Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(macos): make auto-paste layout-aware for non-QWERTY keyboards (#1612)

Idris Gadi76fcb2830116
diff --git a/.github/workflows/build-and-notarize.yml b/.github/workflows/build-and-notarize.yml
index f40aa6836..5607e485e 100644
--- a/.github/workflows/build-and-notarize.yml
+++ b/.github/workflows/build-and-notarize.yml
@@ -296,6 +296,9 @@ jobs:
- name: Compile macos-fast-paste binary
run: npm run compile:fast-paste -- --arch ${{ matrix.arch }}
+ - name: Test macOS copy and paste keyboard layouts
+ run: node --test test/helpers/macosPasteLayout.test.js
+
- name: Compile macos-media-remote binary
run: npm run compile:media-remote -- --arch ${{ matrix.arch }}
diff --git a/resources/macos-fast-paste.swift b/resources/macos-fast-paste.swift
index faeba79d2..784f24156 100644
--- a/resources/macos-fast-paste.swift
+++ b/resources/macos-fast-paste.swift
@@ -1,4 +1,5 @@
import Cocoa
+import Carbon.HIToolbox
if !AXIsProcessTrusted() {
exit(2)
@@ -8,7 +9,62 @@ if !AXIsProcessTrusted() {
// can tell a copied selection from a target that changed underneath it. With no
// arguments this stays what the paste path expects: ⌘V, no output.
let copyMode = CommandLine.arguments.contains("--copy")
-let virtualKey: CGKeyCode = copyMode ? 0x08 : 0x09 // kVK_ANSI_C : kVK_ANSI_V
+let shortcutCharacter = copyMode ? "c" : "v"
+let commandModifierState = UInt32(cmdKey) >> 8
+
+func keyboardLayoutData(from inputSource: TISInputSource?) -> Data? {
+ guard let inputSource = inputSource,
+ let layoutDataPointer = TISGetInputSourceProperty(inputSource, kTISPropertyUnicodeKeyLayoutData) else {
+ return nil
+ }
+
+ return Unmanaged<CFData>.fromOpaque(layoutDataPointer).takeUnretainedValue() as Data
+}
+
+func lookupVirtualKey(for character: String) -> CGKeyCode? {
+ // Non-ASCII layouts can define their own Command shortcuts. Use the last
+ // ASCII layout only when the active input method provides no layout data.
+ guard let layoutData = keyboardLayoutData(from: TISCopyCurrentKeyboardLayoutInputSource()?.takeRetainedValue())
+ ?? keyboardLayoutData(from: TISCopyCurrentASCIICapableKeyboardLayoutInputSource()?.takeRetainedValue()) else {
+ return nil
+ }
+
+ return layoutData.withUnsafeBytes { rawBuffer in
+ guard let baseAddress = rawBuffer.baseAddress else { return nil }
+ let keyboardLayout = baseAddress.assumingMemoryBound(to: UCKeyboardLayout.self)
+ var deadKeyState: UInt32 = 0
+ var actualStringLength = 0
+ var unicodeString = [UniChar](repeating: 0, count: 4)
+
+ for keyCode in 0..<128 {
+ deadKeyState = 0
+ let status = UCKeyTranslate(
+ keyboardLayout,
+ UInt16(keyCode),
+ UInt16(kUCKeyActionDisplay),
+ commandModifierState,
+ UInt32(LMGetKbdType()),
+ OptionBits(kUCKeyTranslateNoDeadKeysBit),
+ &deadKeyState,
+ unicodeString.count,
+ &actualStringLength,
+ &unicodeString
+ )
+ if status == noErr,
+ String(utf16CodeUnits: unicodeString, count: actualStringLength) == character {
+ return CGKeyCode(keyCode)
+ }
+ }
+
+ return nil
+ }
+}
+
+// Do not post the old US-ANSI fallback key code: on a non-QWERTY layout it
+// can invoke a different shortcut while still reporting a successful paste.
+guard let virtualKey = lookupVirtualKey(for: shortcutCharacter) else {
+ exit(3)
+}
// Resolved before the keystroke is posted: this is the app that will receive it.
let target = copyMode ? NSWorkspace.shared.frontmostApplication : nil
diff --git a/src/helpers/clipboard.js b/src/helpers/clipboard.js
index f55730f53..8a365ce7b 100644
--- a/src/helpers/clipboard.js
+++ b/src/helpers/clipboard.js
@@ -1012,6 +1012,15 @@ class ClipboardManager {
} else {
resolve({ restoreComplete: Promise.resolve() });
}
+ } else if (useFastPaste && code === 3) {
+ this.safeLog("CGEvent paste could not resolve the active keyboard layout", {
+ stderr: errorOutput.trim(),
+ });
+ reject(
+ new Error(
+ "Paste could not resolve the active keyboard layout. Text is copied to clipboard - please paste manually with Cmd+V."
+ )
+ );
} else if (useFastPaste) {
this.safeLog(
code === 2
diff --git a/test/helpers/clipboardRestore.test.js b/test/helpers/clipboardRestore.test.js
index 931f496dc..512136ae9 100644
--- a/test/helpers/clipboardRestore.test.js
+++ b/test/helpers/clipboardRestore.test.js
@@ -761,6 +761,34 @@ test("pasteMacOS restores clipboard after the short macOS delay on successful fa
});
});
+test("pasteMacOS leaves text on the clipboard when the keyboard layout cannot be resolved", async () => {
+ resetClipboard({ text: "dictated text" });
+ const spawnCalls = [];
+ const TestClipboardManager = loadClipboardManager({
+ spawn: createSpawn(spawnCalls, [3]),
+ });
+ const manager = new TestClipboardManager();
+ manager.fastPastePath = "/tmp/openwhispr-fast-paste";
+ manager.fastPasteChecked = true;
+ let restoreCalls = 0;
+ manager._restoreClipboardAfterDelay = () => {
+ restoreCalls++;
+ return Promise.resolve();
+ };
+
+ await assert.rejects(
+ manager.pasteMacOS({ type: "text", data: "previous clipboard" }),
+ /could not resolve the active keyboard layout/
+ );
+
+ assert.deepEqual(spawnCalls, [{ command: "/tmp/openwhispr-fast-paste", args: [] }]);
+ assert.equal(fakeClipboard.text, "dictated text");
+ assert.deepEqual(fakeClipboard.writes, []);
+ assert.equal(restoreCalls, 0);
+ assert.equal(manager.fastPastePath, "/tmp/openwhispr-fast-paste");
+ assert.equal(manager.fastPasteChecked, true);
+});
+
test("pasteMacOSWithOsascript fallback uses the short macOS restore delay", async () => {
const spawnCalls = [];
const TestClipboardManager = loadClipboardManager({
diff --git a/test/helpers/macosPasteLayout.test.js b/test/helpers/macosPasteLayout.test.js
new file mode 100644
index 000000000..45ecd4b55
--- /dev/null
+++ b/test/helpers/macosPasteLayout.test.js
@@ -0,0 +1,106 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+const { spawnSync } = require("node:child_process");
+
+test(
+ "macOS shortcuts follow the active layout and fall back only without layout data",
+ {
+ skip: process.platform !== "darwin",
+ },
+ (t) => {
+ const source = fs.readFileSync(
+ path.join(__dirname, "../../resources/macos-fast-paste.swift"),
+ "utf8"
+ );
+ // Compile the real resolver without the permission check or event posting.
+ const start = source.indexOf("let commandModifierState");
+ const end = source.indexOf("// Do not post", start);
+ assert.ok(start >= 0 && end > start);
+ const directory = fs.mkdtempSync(path.join(os.tmpdir(), "openwhispr-paste-layout-"));
+ t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
+ const harness = path.join(directory, "main.swift");
+ const binary = path.join(directory, "layout-test");
+ fs.writeFileSync(
+ harness,
+ `
+import Cocoa
+import Carbon.HIToolbox
+
+var currentSource: TISInputSource?
+var asciiSource: TISInputSource?
+
+// Stub source selection so tests never switch the user's input source.
+func TISCopyCurrentKeyboardLayoutInputSource() -> Unmanaged<TISInputSource>? {
+ currentSource.map { Unmanaged.passRetained($0) }
+}
+func TISCopyCurrentASCIICapableKeyboardLayoutInputSource() -> Unmanaged<TISInputSource>? {
+ asciiSource.map { Unmanaged.passRetained($0) }
+}
+
+${source.slice(start, end)}
+
+func source(named identifier: String) -> TISInputSource {
+ let filter = [kTISPropertyInputSourceID as String: identifier] as CFDictionary
+ let sources = TISCreateInputSourceList(filter, true).takeRetainedValue() as! [TISInputSource]
+ guard let source = sources.first else { fatalError("Missing layout: " + identifier) }
+ return source
+}
+
+let dvorak = source(named: "com.apple.keylayout.Dvorak")
+let scenarios: [(String, String?, CGKeyCode, CGKeyCode)] = [
+ ("Russian after Dvorak", "com.apple.keylayout.Russian", 8, 9),
+ ("Dvorak", "com.apple.keylayout.Dvorak", 34, 47),
+ ("Dvorak QWERTY Command", "com.apple.keylayout.DVORAK-QWERTYCMD", 8, 9),
+ ("US", "com.apple.keylayout.US", 8, 9),
+ ("Colemak", "com.apple.keylayout.Colemak", 8, 9),
+ ("French", "com.apple.keylayout.French", 8, 9),
+ ("German", "com.apple.keylayout.German", 8, 9),
+ ("missing current source", nil, 34, 47)
+]
+for (name, identifier, copyKey, pasteKey) in scenarios {
+ currentSource = identifier.map { source(named: $0) }
+ asciiSource = dvorak
+ let actualCopy = lookupVirtualKey(for: "c")
+ let actualPaste = lookupVirtualKey(for: "v")
+ guard actualCopy == copyKey && actualPaste == pasteKey else {
+ fputs("Wrong shortcuts for " + name + ": " + String(describing: actualCopy) + ", " + String(describing: actualPaste) + "\\n", stderr)
+ exit(1)
+ }
+}
+let inputModes = TISCreateInputSourceList(
+ [kTISPropertyInputSourceType as String: kTISTypeKeyboardInputMode] as CFDictionary,
+ true
+).takeRetainedValue() as! [TISInputSource]
+guard let inputMode = inputModes.first(where: {
+ TISGetInputSourceProperty($0, kTISPropertyUnicodeKeyLayoutData) == nil
+}) else { fatalError("Missing input mode without layout data") }
+currentSource = inputMode
+asciiSource = dvorak
+guard lookupVirtualKey(for: "c") == 34 && lookupVirtualKey(for: "v") == 47 else {
+ fputs("Input method did not fall back to Dvorak\\n", stderr)
+ exit(1)
+}
+currentSource = nil
+asciiSource = nil
+guard lookupVirtualKey(for: "v") == nil else { exit(1) }
+// A present layout without the shortcut must fail safely, not use another layout.
+currentSource = source(named: "com.apple.keylayout.Turkmen")
+asciiSource = dvorak
+guard lookupVirtualKey(for: "c") == nil && lookupVirtualKey(for: "v") == nil else { exit(1) }
+print("layout scenarios passed")
+`
+ );
+ const compiled = spawnSync(
+ "xcrun",
+ ["swiftc", harness, "-module-cache-path", path.join(directory, "cache"), "-o", binary],
+ { encoding: "utf8", timeout: 120000 }
+ );
+ assert.equal(compiled.status, 0, compiled.stderr || compiled.error?.message);
+ const result = spawnSync(binary, [], { encoding: "utf8", timeout: 10000 });
+ assert.equal(result.status, 0, result.stderr || result.error?.message);
+ assert.match(result.stdout, /layout scenarios passed/);
+ }
+);