diff --git a/Sources/OpenKey/engine/Engine.cpp b/Sources/OpenKey/engine/Engine.cpp index de90833b..c58782c1 100644 --- a/Sources/OpenKey/engine/Engine.cpp +++ b/Sources/OpenKey/engine/Engine.cpp @@ -1556,3 +1556,173 @@ void vKeyHandleEvent(const vKeyEvent& event, //cout<<"backspace "<<(int)hBPC< _seedCharMap; //Unicode character -> how it is made up (code table 0) + +static bool seedToneAllowed(const Uint16& baseKey, const Uint32& tone) { + switch (baseKey) { + case KEY_A: + case KEY_O: + return true; //â ă, ô ơ + case KEY_E: + return tone != TONEW_MASK; //ê + case KEY_U: + return tone != TONE_MASK; //ư + case KEY_D: + return tone == TONE_MASK; //đ + default: + return tone == 0; //i, y + } +} + +static void initSeedCharMap() { + static const Uint16 baseKeys[] = {KEY_A, KEY_E, KEY_I, KEY_O, KEY_U, KEY_Y, KEY_D}; + static const Uint32 tones[] = {0, TONE_MASK, TONEW_MASK}; + static const Uint32 marks[] = {0, MARK1_MASK, MARK2_MASK, MARK3_MASK, MARK4_MASK, MARK5_MASK}; + _seedCharMap.clear(); + for (Uint16 baseKey : baseKeys) { + for (Uint32 tone : tones) { + for (Uint32 mark : marks) { + for (int caps = 0; caps < 2; caps++) { + if (!seedToneAllowed(baseKey, tone) || (tone == 0 && mark == 0)) + continue; //plain letters come from _characterMap + if (baseKey == KEY_D && mark != 0) + continue; + //ask the forward function what this combination looks like on the screen + Uint32 code = getCharacterCode(baseKey | tone | mark | (caps ? CAPS_MASK : 0)); + if (!(code & CHAR_CODE_MASK)) + continue; //no such character + Uint32 shown = code & ~CHAR_CODE_MASK; + if (shown != 0) + _seedCharMap[shown] = {baseKey, tone, mark, caps != 0}; + } + } + } + } +} + +static bool seedIsLetter(const Uint32& ch) { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || _seedCharMap.find(ch) != _seedCharMap.end(); +} + +//same reset as a mouse click, plus what a click does not touch +static void seedReset() { + startNewSession(); + _spaceCount = 0; + _specialChar.clear(); + _typingStates.clear(); + _upperCaseStatus = 0; + _willTempOffEngine = false; + vCheckSpelling = _useSpellCheckingBefore; + hMacroKey.clear(); +} + +static bool seedFail() { + seedReset(); + return false; +} + +bool vSeedTypingWord(const Uint32* chars, const int& count) { + if (vCodeTable != 0) + return false; //the text on the screen is only plain Unicode for table 0 + if (_seedCharMap.empty()) + initSeedCharMap(); + + seedReset(); + if (chars == NULL || count <= 0) + return false; + + //the word is the run of letters right before the caret + int wordStart = count; + while (wordStart > 0 && seedIsLetter(chars[wordStart - 1])) + wordStart--; + int wordLength = count - wordStart; + if (wordLength == 0 || wordLength > SEED_MAX_WORD) + return false; + + //keys that would have typed it (key code, CAPS_MASK when upper case) + vector seedKeys; + Uint32 markKey = 0; + for (int pos = wordStart; pos < count; pos++) { + Uint16 baseKey; + Uint32 tone = 0, mark = 0; + bool caps; + if (chars[pos] < 0x80) { + map::iterator letter = _characterMap.find(chars[pos]); + if (letter == _characterMap.end()) + return seedFail(); + baseKey = letter->second & CHAR_MASK; + caps = (letter->second & CAPS_MASK) != 0; + } else { + map::iterator made = _seedCharMap.find(chars[pos]); + if (made == _seedCharMap.end()) + return seedFail(); + baseKey = made->second.baseKey; + tone = made->second.tone; + mark = made->second.mark; + caps = made->second.caps; + } + const Uint32 capsFlag = caps ? CAPS_MASK : 0; + seedKeys.push_back(baseKey | capsFlag); + + if (tone == TONE_MASK) { //â ê ô đ + if (vInputType == vVNI) + seedKeys.push_back((baseKey == KEY_D ? KEY_9 : KEY_6) | capsFlag); + else + seedKeys.push_back(baseKey | capsFlag); + } else if (tone == TONEW_MASK) { //ă ơ ư + if (vInputType == vVNI) + seedKeys.push_back((baseKey == KEY_A ? KEY_8 : KEY_7) | capsFlag); + else + seedKeys.push_back(KEY_W | capsFlag); + } + + if (mark != 0) { + if (markKey != 0) + return seedFail(); //two marks in one word + Uint32 markIndex = mark == MARK1_MASK ? 0 : (mark == MARK2_MASK ? 1 : (mark == MARK3_MASK ? 2 : (mark == MARK4_MASK ? 3 : 4))); + markKey = ProcessingChar[vInputType][markIndex]; + } + } + if (markKey != 0) + seedKeys.push_back(markKey); //typed last, like most people do + + for (size_t pos = 0; pos < seedKeys.size(); pos++) { + vKeyHandleEvent(vKeyEvent::Keyboard, vKeyEventState::KeyDown, (Uint16)(seedKeys[pos] & CHAR_MASK), + (seedKeys[pos] & CAPS_MASK) ? 1 : 0, false); + } + + //the engine must now hold exactly what is on the screen + if (_index != wordLength || _longWordHelper.size() > 0) + return seedFail(); + for (int pos = 0; pos < wordLength; pos++) { + const Uint32 data = TypingWord[pos]; + Uint32 shown; + if (data & (MARK_MASK | TONE_MASK | TONEW_MASK)) { + Uint32 code = getCharacterCode(data); + shown = (code & CHAR_CODE_MASK) ? (code & ~CHAR_CODE_MASK) : 0; + } else { + shown = keyCodeToCharacter(data & (CHAR_MASK | CAPS_MASK)); + } + if (shown != chars[wordStart + pos]) + return seedFail(); + } + return true; +} diff --git a/Sources/OpenKey/engine/Engine.h b/Sources/OpenKey/engine/Engine.h index f288f8c5..3ba58600 100644 --- a/Sources/OpenKey/engine/Engine.h +++ b/Sources/OpenKey/engine/Engine.h @@ -216,6 +216,16 @@ void vKeyHandleEvent(const vKeyEvent& event, */ void startNewSession(); +/** + * Fill the engine with the word that is already on the screen right before the caret, so the user can + * keep adding marks to it as if it had just been typed. + * chars: code points (NFC) of the text before the caret; only the last word of it is used. + * Only works for the Unicode code table. The word is replayed through vKeyHandleEvent, so all engine + * states are the ones a typed word would have left. Returns false, and leaves the engine empty, when the + * word can not be mapped back exactly (too long, not a word, typed with other settings...). + */ +bool vSeedTypingWord(const Uint32* chars, const int& count); + /** * do some task in english mode (use for macro) */ diff --git a/Sources/OpenKey/macOS/ModernKey/AXTextContext.h b/Sources/OpenKey/macOS/ModernKey/AXTextContext.h new file mode 100644 index 00000000..c7933fe0 --- /dev/null +++ b/Sources/OpenKey/macOS/ModernKey/AXTextContext.h @@ -0,0 +1,47 @@ +// +// AXTextContext.h +// OpenKey +// +// Read the text around the caret of the focused text field with the Accessibility API. +// + +#ifndef AXTextContext_h +#define AXTextContext_h + +#import + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Number of characters read before the caret: one more than the longest word the engine takes (10), + * so the start of that word can be told from the middle of a longer one. + */ +#define AX_CONTEXT_LENGTH 11 + +typedef enum { + AXContextOK, //before and after are filled + AXContextRetry, //accessibility was just switched on for this app, try again shortly + AXContextNotYet, //the focus is not on what was clicked yet (or is still moving): try again in a moment + AXContextUnavailable, //the app does not answer the Accessibility API; another way of reading may work + AXContextSkipped //must not be read at all (password, terminal, text is selected) or is not a text field +} AXContextStatus; + +/** + * Read the text right before and right after the caret (NFC) of the focused text field of an application. + * It talks to another process and can block for a moment, so never call it from the event tap callback. + * Only call it from one serial queue. + * clickPoint (screen coordinates, may be NULL): where the user clicked. The app moves its focus a moment after + * the click, so right after it the focused field can still be the previous one; reading that would give the + * text of another field. When the focused field does not contain the point, the answer is AXContextNotYet. + * It never reads password fields or terminals, and nothing is written to the log except the bundle id + * and Accessibility error codes (turn it on with: defaults write com.tuyenmai.openkey vSeedFromCaretLog -bool YES). + */ +AXContextStatus AXReadCaretContext(pid_t pid, NSString* bundleId, const CGPoint* clickPoint, NSString** before, NSString** after); + +#ifdef __cplusplus +} +#endif + +#endif /* AXTextContext_h */ diff --git a/Sources/OpenKey/macOS/ModernKey/AXTextContext.m b/Sources/OpenKey/macOS/ModernKey/AXTextContext.m new file mode 100644 index 00000000..58fb20ba --- /dev/null +++ b/Sources/OpenKey/macOS/ModernKey/AXTextContext.m @@ -0,0 +1,241 @@ +// +// AXTextContext.m +// OpenKey +// +// Read the text around the caret of the focused text field with the Accessibility API. +// + +#import +#import +#import "AXTextContext.h" + +#define AX_TIMEOUT 0.15f //seconds; a stuck app must not hold us up +#define AX_MAX_FALLBACK_LENGTH 100000 //do not pull a whole big document just to look at the caret + +//apps whose accessibility text is not "the text before the caret" (terminals) or that we must stay out of +static BOOL isDeniedApp(NSString* bundleId) { + static NSSet* denied; + static dispatch_once_t once; + dispatch_once(&once, ^{ + denied = [NSSet setWithObjects:@"com.apple.Terminal", @"com.googlecode.iterm2", @"dev.warp.Warp-Stable", + @"net.kovidgoyal.kitty", @"com.mitchellh.ghostty", @"org.alacritty", @"io.alacritty", + @"com.github.wez.wezterm", @"co.zeit.hyper", nil]; + }); + return bundleId != nil && ([denied containsObject:bundleId] || [bundleId isEqualToString:[[NSBundle mainBundle] bundleIdentifier]]); +} + +static void logStep(NSString* bundleId, const char* step, AXError error) { + if (![[NSUserDefaults standardUserDefaults] boolForKey:@"vSeedFromCaretLog"]) + return; + os_log(OS_LOG_DEFAULT, "seed %{public}@ %{public}s -> AXError %d", bundleId, step, (int)error); +} + +//Chromium and Electron only build their accessibility tree for a client that asks for it, and Firefox +//answers to AXEnhancedUserInterface. Other apps do not support these attributes, so they are left alone. +//Returns YES when something was switched on and a retry is worth it. +static NSMutableSet* enabledPids; //only used from the serial queue that calls AXReadCaretContext +static BOOL switchOnAccessibility(AXUIElementRef app, pid_t pid, NSString* bundleId) { + if (enabledPids == nil) + enabledPids = [NSMutableSet new]; + NSNumber* pidKey = @(pid); + if ([enabledPids containsObject:pidKey]) + return NO; + [enabledPids addObject:pidKey]; + + BOOL changed = AXUIElementSetAttributeValue(app, CFSTR("AXManualAccessibility"), kCFBooleanTrue) == kAXErrorSuccess; + if ([bundleId hasPrefix:@"org.mozilla."] || [bundleId hasPrefix:@"app.zen-browser."]) { + changed |= AXUIElementSetAttributeValue(app, CFSTR("AXEnhancedUserInterface"), kCFBooleanTrue) == kAXErrorSuccess; + } + logStep(bundleId, changed ? "accessibility switched on" : "accessibility not supported", kAXErrorSuccess); + return changed; +} + +//the focused element of the app; the system-wide one is only a fallback, it often fails with Chromium +static AXUIElementRef copyFocusedElement(AXUIElementRef app, pid_t pid, NSString* bundleId) { + CFTypeRef focused = NULL; + AXError error = AXUIElementCopyAttributeValue(app, kAXFocusedUIElementAttribute, &focused); + if (error == kAXErrorSuccess && focused != NULL && CFGetTypeID(focused) == AXUIElementGetTypeID()) + return (AXUIElementRef)focused; + if (focused != NULL) + CFRelease(focused); + logStep(bundleId, "focused element of the app", error); + + AXUIElementRef systemWide = AXUIElementCreateSystemWide(); + AXUIElementSetMessagingTimeout(systemWide, AX_TIMEOUT); + focused = NULL; + error = AXUIElementCopyAttributeValue(systemWide, kAXFocusedUIElementAttribute, &focused); + CFRelease(systemWide); + if (error == kAXErrorSuccess && focused != NULL && CFGetTypeID(focused) == AXUIElementGetTypeID()) { + pid_t owner = 0; + if (AXUIElementGetPid((AXUIElementRef)focused, &owner) == kAXErrorSuccess && owner == pid) + return (AXUIElementRef)focused; + } + if (focused != NULL) + CFRelease(focused); + logStep(bundleId, "focused element (system wide)", error); + return NULL; +} + +//does the element cover the screen point? YES when the app does not tell where the element is +static BOOL frameContainsPoint(AXUIElementRef element, CGPoint point) { + CFTypeRef positionValue = NULL, sizeValue = NULL; + CGPoint position = CGPointZero; + CGSize size = CGSizeZero; + BOOL known = NO; + if (AXUIElementCopyAttributeValue(element, kAXPositionAttribute, &positionValue) == kAXErrorSuccess && positionValue != NULL && + AXUIElementCopyAttributeValue(element, kAXSizeAttribute, &sizeValue) == kAXErrorSuccess && sizeValue != NULL) { + known = CFGetTypeID(positionValue) == AXValueGetTypeID() && CFGetTypeID(sizeValue) == AXValueGetTypeID() && + AXValueGetValue(positionValue, kAXValueTypeCGPoint, &position) && AXValueGetValue(sizeValue, kAXValueTypeCGSize, &size); + } + if (positionValue != NULL) + CFRelease(positionValue); + if (sizeValue != NULL) + CFRelease(sizeValue); + if (!known || size.width <= 0 || size.height <= 0) + return YES; + return CGRectContainsPoint(CGRectInset(CGRectMake(position.x, position.y, size.width, size.height), -3, -3), point); +} + +static BOOL isSecureField(AXUIElementRef element) { + CFTypeRef subrole = NULL; + BOOL secure = NO; + if (AXUIElementCopyAttributeValue(element, kAXSubroleAttribute, &subrole) == kAXErrorSuccess && subrole != NULL) { + secure = CFGetTypeID(subrole) == CFStringGetTypeID() && CFEqual(subrole, kAXSecureTextFieldSubrole); + CFRelease(subrole); + } + return secure; +} + +static AXError copySelectedRange(AXUIElementRef element, CFRange* range, NSString* bundleId) { + CFTypeRef value = NULL; + AXError error = AXUIElementCopyAttributeValue(element, kAXSelectedTextRangeAttribute, &value); + if (error != kAXErrorSuccess || value == NULL) { + logStep(bundleId, "selected text range", error); + return error != kAXErrorSuccess ? error : kAXErrorNoValue; + } + BOOL ok = CFGetTypeID(value) == AXValueGetTypeID() && AXValueGetValue(value, kAXValueTypeCFRange, range); + CFRelease(value); + return ok ? kAXErrorSuccess : kAXErrorFailure; +} + +//a piece of the text, as the app itself reports it +static NSString* copyStringForRange(AXUIElementRef element, CFRange range, NSString* bundleId) { + if (range.length <= 0) + return @""; + AXValueRef rangeValue = AXValueCreate(kAXValueTypeCFRange, &range); + CFTypeRef result = NULL; + AXError error = AXUIElementCopyParameterizedAttributeValue(element, kAXStringForRangeParameterizedAttribute, rangeValue, &result); + CFRelease(rangeValue); + if (error == kAXErrorSuccess && result != NULL && CFGetTypeID(result) == CFStringGetTypeID()) + return CFBridgingRelease(result); + if (result != NULL) + CFRelease(result); + logStep(bundleId, "string for range", error); + return nil; +} + +//number of characters of the text field, -1 when the app does not tell +static CFIndex characterCount(AXUIElementRef element) { + CFTypeRef count = NULL; + if (AXUIElementCopyAttributeValue(element, kAXNumberOfCharactersAttribute, &count) != kAXErrorSuccess || count == NULL) + return -1; + CFIndex total = -1; + if (CFGetTypeID(count) == CFNumberGetTypeID()) + CFNumberGetValue(count, kCFNumberCFIndexType, &total); + CFRelease(count); + return total; +} + +//fallback for apps without AXStringForRange: cut the piece out of the whole value +static NSString* copyStringFromValue(AXUIElementRef element, CFRange range, NSString* bundleId) { + CFIndex total = characterCount(element); + if (total < 0 || total > AX_MAX_FALLBACK_LENGTH) + return nil; + + CFTypeRef value = NULL; + AXError error = AXUIElementCopyAttributeValue(element, kAXValueAttribute, &value); + if (error != kAXErrorSuccess || value == NULL) { + logStep(bundleId, "value", error); + return nil; + } + if (CFGetTypeID(value) != CFStringGetTypeID()) { + CFRelease(value); + return nil; + } + NSString* text = CFBridgingRelease(value); + NSUInteger start = MIN((NSUInteger)range.location, text.length); + NSUInteger end = MIN((NSUInteger)(range.location + range.length), text.length); + return [text substringWithRange:NSMakeRange(start, end - start)]; +} + +//do not start or end in the middle of a surrogate pair +static NSString* trimSurrogates(NSString* text) { + if (text.length > 0 && CFStringIsSurrogateLowCharacter([text characterAtIndex:0])) + text = [text substringFromIndex:1]; + if (text.length > 0 && CFStringIsSurrogateHighCharacter([text characterAtIndex:text.length - 1])) + text = [text substringToIndex:text.length - 1]; + return text; +} + +AXContextStatus AXReadCaretContext(pid_t pid, NSString* bundleId, const CGPoint* clickPoint, NSString** before, NSString** after) { + *before = nil; + *after = nil; + if (isDeniedApp(bundleId)) + return AXContextSkipped; + + AXUIElementRef app = AXUIElementCreateApplication(pid); + if (app == NULL) + return AXContextUnavailable; + AXUIElementSetMessagingTimeout(app, AX_TIMEOUT); + + AXContextStatus status = AXContextUnavailable; + AXUIElementRef focused = copyFocusedElement(app, pid, bundleId); + if (focused == NULL) { + if (switchOnAccessibility(app, pid, bundleId)) + status = AXContextRetry; + CFRelease(app); + return status; + } + AXUIElementSetMessagingTimeout(focused, AX_TIMEOUT); + + CFRange caret = CFRangeMake(0, 0); + AXError rangeError = kAXErrorSuccess; + if (clickPoint != NULL && !frameContainsPoint(focused, *clickPoint)) { + logStep(bundleId, "focus is not under the click yet", kAXErrorSuccess); + status = AXContextNotYet; + } else if (isSecureField(focused)) { + logStep(bundleId, "secure field, skipped", kAXErrorSuccess); + status = AXContextSkipped; + } else if ((rangeError = copySelectedRange(focused, &caret, bundleId)) != kAXErrorSuccess) { + if (switchOnAccessibility(app, pid, bundleId)) + status = AXContextRetry; + else if (rangeError == kAXErrorNoValue || rangeError == kAXErrorAttributeUnsupported) + status = AXContextSkipped; //the app answers, and this element is not a text field + } else if (caret.length != 0 || caret.location < 0) { + logStep(bundleId, "text is selected, skipped", kAXErrorSuccess); + status = AXContextSkipped; + } else { + CFIndex start = MAX(0, caret.location - AX_CONTEXT_LENGTH); + CFRange beforeRange = CFRangeMake(start, caret.location - start); + NSString* text = beforeRange.length == 0 ? @"" : copyStringForRange(focused, beforeRange, bundleId); + if (text == nil) + text = copyStringFromValue(focused, beforeRange, bundleId); + if (text != nil) { + *before = [trimSurrogates(text) precomposedStringWithCanonicalMapping]; + //the part after the caret is only looked at to know where the caret is, it is best effort. + //AXStringForRange fails for a range that goes past the end, so ask only for what is there + CFIndex total = characterCount(focused); + CFIndex available = total >= 0 ? MAX(0, total - caret.location) : AX_CONTEXT_LENGTH; + CFRange afterRange = CFRangeMake(caret.location, MIN(available, AX_CONTEXT_LENGTH)); + NSString* tail = afterRange.length == 0 ? @"" : copyStringForRange(focused, afterRange, bundleId); + if (tail == nil) + tail = copyStringFromValue(focused, afterRange, bundleId); + *after = tail != nil ? [trimSurrogates(tail) precomposedStringWithCanonicalMapping] : @""; + status = AXContextOK; + } + } + + CFRelease(focused); + CFRelease(app); + return status; +} diff --git a/Sources/OpenKey/macOS/ModernKey/AppDelegate.m b/Sources/OpenKey/macOS/ModernKey/AppDelegate.m index 94f84eac..9eebb5e2 100644 --- a/Sources/OpenKey/macOS/ModernKey/AppDelegate.m +++ b/Sources/OpenKey/macOS/ModernKey/AppDelegate.m @@ -23,6 +23,7 @@ extern void OnInputMethodChanged(void); extern void RequestNewSession(void); extern void OnActiveAppChanged(void); +extern void OnActiveAppForSeed(void); //see document in Engine.h int vLanguage = 1; @@ -54,6 +55,10 @@ int vPerformLayoutCompat = 0; +//continue the word under the caret after a click or a change of window: read the text with Accessibility, +//and when the app does not answer, select it and copy it instead (see SelectionProbe) +int vSeedFromCaret = 0; + //beta feature int vFixChromiumBrowser = 0; //new on version 2.0 @@ -311,6 +316,7 @@ -(void)loadDefaultConfig { vShowIconOnDock = 0;[[NSUserDefaults standardUserDefaults] setInteger:vShowIconOnDock forKey:@"vShowIconOnDock"]; vFixChromiumBrowser = 0;[[NSUserDefaults standardUserDefaults] setInteger:vFixChromiumBrowser forKey:@"vFixChromiumBrowser"]; vPerformLayoutCompat = 0;[[NSUserDefaults standardUserDefaults] setInteger:vPerformLayoutCompat forKey:@"vPerformLayoutCompat"]; + vSeedFromCaret = 0;[[NSUserDefaults standardUserDefaults] setInteger:vSeedFromCaret forKey:@"vSeedFromCaret"]; [[NSUserDefaults standardUserDefaults] setInteger:1 forKey:@"GrayIcon"]; [[NSUserDefaults standardUserDefaults] setInteger:1 forKey:@"RunOnStartup"]; @@ -550,6 +556,9 @@ -(void)activeAppChanged: (NSNotification*)note { if (vUseSmartSwitchKey && [OpenKeyManager isInited]) { OnActiveAppChanged(); } + if (vSeedFromCaret && [OpenKeyManager isInited]) { + OnActiveAppForSeed(); + } } -(void)registerSupportedNotification { diff --git a/Sources/OpenKey/macOS/ModernKey/Base.lproj/Main.storyboard b/Sources/OpenKey/macOS/ModernKey/Base.lproj/Main.storyboard index 1ad0aedf..210c2cbd 100644 --- a/Sources/OpenKey/macOS/ModernKey/Base.lproj/Main.storyboard +++ b/Sources/OpenKey/macOS/ModernKey/Base.lproj/Main.storyboard @@ -1291,7 +1291,7 @@ +