Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(macos): make auto-paste layout-aware for non-QWERTY keyboards (#1612)
Idris Gadi76fcb2830116
diff --git a/.github/workflows/build-and-notarize.yml b/.github/workflows/build-and-notarize.ymlindex f40aa6836..5607e485e 100644--- a/.github/workflows/build-and-notarize.yml+++ b/.github/workflows/build-and-notarize.yml@@ -296,6 +296,9 @@ jobs:- name: Compile macos-fast-paste binaryrun: npm run compile:fast-paste -- --arch ${{ matrix.arch }}+ - name: Test macOS copy and paste keyboard layouts+ run: node --test test/helpers/macosPasteLayout.test.js+- name: Compile macos-media-remote binaryrun: npm run compile:media-remote -- --arch ${{ matrix.arch }}diff --git a/resources/macos-fast-paste.swift b/resources/macos-fast-paste.swiftindex faeba79d2..784f24156 100644--- a/resources/macos-fast-paste.swift+++ b/resources/macos-fast-paste.swift@@ -1,4 +1,5 @@import Cocoa+import Carbon.HIToolboxif !AXIsProcessTrusted() {exit(2)@@ -8,7 +9,62 @@ if !AXIsProcessTrusted() {// can tell a copied selection from a target that changed underneath it. With no// arguments this stays what the paste path expects: ⌘V, no output.let copyMode = CommandLine.arguments.contains("--copy")-let virtualKey: CGKeyCode = copyMode ? 0x08 : 0x09 // kVK_ANSI_C : kVK_ANSI_V+let shortcutCharacter = copyMode ? "c" : "v"+let commandModifierState = UInt32(cmdKey) >> 8++func keyboardLayoutData(from inputSource: TISInputSource?) -> Data? {+ guard let inputSource = inputSource,+ let layoutDataPointer = TISGetInputSourceProperty(inputSource, kTISPropertyUnicodeKeyLayoutData) else {+ return nil+ }++ return Unmanaged<CFData>.fromOpaque(layoutDataPointer).takeUnretainedValue() as Data+}++func lookupVirtualKey(for character: String) -> CGKeyCode? {+ // Non-ASCII layouts can define their own Command shortcuts. Use the last+ // ASCII layout only when the active input method provides no layout data.+ guard let layoutData = keyboardLayoutData(from: TISCopyCurrentKeyboardLayoutInputSource()?.takeRetainedValue())+ ?? keyboardLayoutData(from: TISCopyCurrentASCIICapableKeyboardLayoutInputSource()?.takeRetainedValue()) else {+ return nil+ }++ return layoutData.withUnsafeBytes { rawBuffer in+ guard let baseAddress = rawBuffer.baseAddress else { return nil }+ let keyboardLayout = baseAddress.assumingMemoryBound(to: UCKeyboardLayout.self)+ var deadKeyState: UInt32 = 0+ var actualStringLength = 0+ var unicodeString = [UniChar](repeating: 0, count: 4)++ for keyCode in 0..<128 {+ deadKeyState = 0+ let status = UCKeyTranslate(+ keyboardLayout,+ UInt16(keyCode),+ UInt16(kUCKeyActionDisplay),+ commandModifierState,+ UInt32(LMGetKbdType()),+ OptionBits(kUCKeyTranslateNoDeadKeysBit),+ &deadKeyState,+ unicodeString.count,+ &actualStringLength,+ &unicodeString+ )+ if status == noErr,+ String(utf16CodeUnits: unicodeString, count: actualStringLength) == character {+ return CGKeyCode(keyCode)+ }+ }++ return nil+ }+}++// Do not post the old US-ANSI fallback key code: on a non-QWERTY layout it+// can invoke a different shortcut while still reporting a successful paste.+guard let virtualKey = lookupVirtualKey(for: shortcutCharacter) else {+ exit(3)+}// Resolved before the keystroke is posted: this is the app that will receive it.let target = copyMode ? NSWorkspace.shared.frontmostApplication : nildiff --git a/src/helpers/clipboard.js b/src/helpers/clipboard.jsindex f55730f53..8a365ce7b 100644--- a/src/helpers/clipboard.js+++ b/src/helpers/clipboard.js@@ -1012,6 +1012,15 @@ class ClipboardManager {} else {resolve({ restoreComplete: Promise.resolve() });}+ } else if (useFastPaste && code === 3) {+ this.safeLog("CGEvent paste could not resolve the active keyboard layout", {+ stderr: errorOutput.trim(),+ });+ reject(+ new Error(+ "Paste could not resolve the active keyboard layout. Text is copied to clipboard - please paste manually with Cmd+V."+ )+ );} else if (useFastPaste) {this.safeLog(code === 2diff --git a/test/helpers/clipboardRestore.test.js b/test/helpers/clipboardRestore.test.jsindex 931f496dc..512136ae9 100644--- a/test/helpers/clipboardRestore.test.js+++ b/test/helpers/clipboardRestore.test.js@@ -761,6 +761,34 @@ test("pasteMacOS restores clipboard after the short macOS delay on successful fa});});+test("pasteMacOS leaves text on the clipboard when the keyboard layout cannot be resolved", async () => {+ resetClipboard({ text: "dictated text" });+ const spawnCalls = [];+ const TestClipboardManager = loadClipboardManager({+ spawn: createSpawn(spawnCalls, [3]),+ });+ const manager = new TestClipboardManager();+ manager.fastPastePath = "/tmp/openwhispr-fast-paste";+ manager.fastPasteChecked = true;+ let restoreCalls = 0;+ manager._restoreClipboardAfterDelay = () => {+ restoreCalls++;+ return Promise.resolve();+ };++ await assert.rejects(+ manager.pasteMacOS({ type: "text", data: "previous clipboard" }),+ /could not resolve the active keyboard layout/+ );++ assert.deepEqual(spawnCalls, [{ command: "/tmp/openwhispr-fast-paste", args: [] }]);+ assert.equal(fakeClipboard.text, "dictated text");+ assert.deepEqual(fakeClipboard.writes, []);+ assert.equal(restoreCalls, 0);+ assert.equal(manager.fastPastePath, "/tmp/openwhispr-fast-paste");+ assert.equal(manager.fastPasteChecked, true);+});+test("pasteMacOSWithOsascript fallback uses the short macOS restore delay", async () => {const spawnCalls = [];const TestClipboardManager = loadClipboardManager({diff --git a/test/helpers/macosPasteLayout.test.js b/test/helpers/macosPasteLayout.test.jsnew file mode 100644index 000000000..45ecd4b55--- /dev/null+++ b/test/helpers/macosPasteLayout.test.js@@ -0,0 +1,106 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const fs = require("node:fs");+const os = require("node:os");+const path = require("node:path");+const { spawnSync } = require("node:child_process");++test(+ "macOS shortcuts follow the active layout and fall back only without layout data",+ {+ skip: process.platform !== "darwin",+ },+ (t) => {+ const source = fs.readFileSync(+ path.join(__dirname, "../../resources/macos-fast-paste.swift"),+ "utf8"+ );+ // Compile the real resolver without the permission check or event posting.+ const start = source.indexOf("let commandModifierState");+ const end = source.indexOf("// Do not post", start);+ assert.ok(start >= 0 && end > start);+ const directory = fs.mkdtempSync(path.join(os.tmpdir(), "openwhispr-paste-layout-"));+ t.after(() => fs.rmSync(directory, { recursive: true, force: true }));+ const harness = path.join(directory, "main.swift");+ const binary = path.join(directory, "layout-test");+ fs.writeFileSync(+ harness,+ `+import Cocoa+import Carbon.HIToolbox++var currentSource: TISInputSource?+var asciiSource: TISInputSource?++// Stub source selection so tests never switch the user's input source.+func TISCopyCurrentKeyboardLayoutInputSource() -> Unmanaged<TISInputSource>? {+ currentSource.map { Unmanaged.passRetained($0) }+}+func TISCopyCurrentASCIICapableKeyboardLayoutInputSource() -> Unmanaged<TISInputSource>? {+ asciiSource.map { Unmanaged.passRetained($0) }+}++${source.slice(start, end)}++func source(named identifier: String) -> TISInputSource {+ let filter = [kTISPropertyInputSourceID as String: identifier] as CFDictionary+ let sources = TISCreateInputSourceList(filter, true).takeRetainedValue() as! [TISInputSource]+ guard let source = sources.first else { fatalError("Missing layout: " + identifier) }+ return source+}++let dvorak = source(named: "com.apple.keylayout.Dvorak")+let scenarios: [(String, String?, CGKeyCode, CGKeyCode)] = [+ ("Russian after Dvorak", "com.apple.keylayout.Russian", 8, 9),+ ("Dvorak", "com.apple.keylayout.Dvorak", 34, 47),+ ("Dvorak QWERTY Command", "com.apple.keylayout.DVORAK-QWERTYCMD", 8, 9),+ ("US", "com.apple.keylayout.US", 8, 9),+ ("Colemak", "com.apple.keylayout.Colemak", 8, 9),+ ("French", "com.apple.keylayout.French", 8, 9),+ ("German", "com.apple.keylayout.German", 8, 9),+ ("missing current source", nil, 34, 47)+]+for (name, identifier, copyKey, pasteKey) in scenarios {+ currentSource = identifier.map { source(named: $0) }+ asciiSource = dvorak+ let actualCopy = lookupVirtualKey(for: "c")+ let actualPaste = lookupVirtualKey(for: "v")+ guard actualCopy == copyKey && actualPaste == pasteKey else {+ fputs("Wrong shortcuts for " + name + ": " + String(describing: actualCopy) + ", " + String(describing: actualPaste) + "\\n", stderr)+ exit(1)+ }+}+let inputModes = TISCreateInputSourceList(+ [kTISPropertyInputSourceType as String: kTISTypeKeyboardInputMode] as CFDictionary,+ true+).takeRetainedValue() as! [TISInputSource]+guard let inputMode = inputModes.first(where: {+ TISGetInputSourceProperty($0, kTISPropertyUnicodeKeyLayoutData) == nil+}) else { fatalError("Missing input mode without layout data") }+currentSource = inputMode+asciiSource = dvorak+guard lookupVirtualKey(for: "c") == 34 && lookupVirtualKey(for: "v") == 47 else {+ fputs("Input method did not fall back to Dvorak\\n", stderr)+ exit(1)+}+currentSource = nil+asciiSource = nil+guard lookupVirtualKey(for: "v") == nil else { exit(1) }+// A present layout without the shortcut must fail safely, not use another layout.+currentSource = source(named: "com.apple.keylayout.Turkmen")+asciiSource = dvorak+guard lookupVirtualKey(for: "c") == nil && lookupVirtualKey(for: "v") == nil else { exit(1) }+print("layout scenarios passed")+`+ );+ const compiled = spawnSync(+ "xcrun",+ ["swiftc", harness, "-module-cache-path", path.join(directory, "cache"), "-o", binary],+ { encoding: "utf8", timeout: 120000 }+ );+ assert.equal(compiled.status, 0, compiled.stderr || compiled.error?.message);+ const result = spawnSync(binary, [], { encoding: "utf8", timeout: 10000 });+ assert.equal(result.status, 0, result.stderr || result.error?.message);+ assert.match(result.stdout, /layout scenarios passed/);+ }+);