Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Paste into a Bot's browser on a layout that does not write Latin letters (#596)

kevine9e0e728560d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 47f7f71f..6e5f737a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,14 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
## Unreleased
+### Pasting into a Bot's browser works on a layout that does not write Latin letters
+
+While somebody drives a Bot's browser, Ctrl+V or Cmd+V is left to the local page so its paste event
+can send the clipboard text across. The shortcut was recognised by the character the key writes,
+and on a Russian or Greek layout the V key writes "м" or "ω", so the keystroke went to the Bot's
+browser instead and nothing was pasted. The V is now read the way the app's own shortcuts read it
+since Shift+N was fixed for the same layouts: from the physical key when the layout writes a
+character outside ASCII there.
### Reading a large file from Google Drive no longer downloads all of it
`read_file_content` shows a Bot at most the first 20,000 characters of a file, but it downloaded the
diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx
index 56d4c729..ce85392c 100644
--- a/app/src/components/computer/live-screen.tsx
+++ b/app/src/components/computer/live-screen.tsx
@@ -1,4 +1,5 @@
import { useCallback, useEffect, useRef, useState } from "react";
+import { keyOf } from "@/lib/hotkeys/hotkeys";
import { socketUrl } from "@/lib/socket-url";
import { currentPageVisible } from "./preview-visibility";
import { pageCoordinates } from "./take-the-wheel";
@@ -33,9 +34,14 @@ function modifierBits(event: {
);
}
-/** Let the local browser create a paste event, whose clipboard text is forwarded separately. */
+/**
+ * Let the local browser create a paste event, whose clipboard text is forwarded separately.
+ *
+ * The V is read the way a shortcut is (`keyOf`), so a layout that writes another script still has
+ * one: Ctrl and the V key report "м" on Russian and "ω" on Greek.
+ */
function isPasteShortcut(event: KeyboardEvent): boolean {
- return (event.ctrlKey || event.metaKey) && event.key.toLowerCase() === "v";
+ return (event.ctrlKey || event.metaKey) && keyOf(event) === "v";
}
type Props = {
diff --git a/app/src/lib/hotkeys/hotkeys.ts b/app/src/lib/hotkeys/hotkeys.ts
index 7c2c3b7a..831f6ab8 100644
--- a/app/src/lib/hotkeys/hotkeys.ts
+++ b/app/src/lib/hotkeys/hotkeys.ts
@@ -61,7 +61,7 @@ const isMac =
* Latin N) on Greek, so a shortcut read from `key` alone never fired there. When `key` is a single
* character outside ASCII, the physical key in `code` is the only N there is.
*/
-function keyOf(event: KeyboardEvent): string {
+export function keyOf(event: KeyboardEvent): string {
const written = event.key.toLowerCase();
if (written.length !== 1 || written.charCodeAt(0) < 0x80) return written;
const physical = /^(?:Key|Digit)([A-Z0-9])$/.exec(event.code);
diff --git a/app/tests/live-screen-keyboard.test.tsx b/app/tests/live-screen-keyboard.test.tsx
index d7cd677d..0a6e4edf 100644
--- a/app/tests/live-screen-keyboard.test.tsx
+++ b/app/tests/live-screen-keyboard.test.tsx
@@ -75,6 +75,81 @@ for (const [name, modifier] of [
});
}
+/*
+ * A layout that writes another script has no key that writes a V. Ctrl and the V key report "м" on
+ * Russian and "ω" on Greek, with `code` still `KeyV`, so reading `key` alone sent the shortcut to the
+ * remote browser and stopped the local paste event that carries the clipboard text.
+ */
+for (const [layout, written] of [
+ ["Russian", "м"],
+ ["Greek", "ω"],
+] as const) {
+ test(`Ctrl+V on a ${layout} layout stays in the local page too`, async () => {
+ const socket = await liveSocket();
+ const shortcut = new KeyboardEvent("keydown", {
+ key: written,
+ code: "KeyV",
+ keyCode: 86,
+ ctrlKey: true,
+ bubbles: true,
+ cancelable: true,
+ });
+
+ window.dispatchEvent(shortcut);
+ window.dispatchEvent(
+ new KeyboardEvent("keyup", {
+ key: written,
+ code: "KeyV",
+ keyCode: 86,
+ bubbles: true,
+ cancelable: true,
+ }),
+ );
+
+ expect(socket.sent).toEqual([]);
+ expect(shortcut.defaultPrevented).toBe(false);
+ });
+}
+
+test("Ctrl on a key that writes V on Dvorak is still paste, and Ctrl on the QWERTY V key is not", async () => {
+ const socket = await liveSocket();
+ // On Dvorak the key that writes V is the one QWERTY calls Period.
+ const paste = new KeyboardEvent("keydown", {
+ key: "v",
+ code: "Period",
+ keyCode: 86,
+ ctrlKey: true,
+ bubbles: true,
+ cancelable: true,
+ });
+ // And the key QWERTY calls V writes K.
+ const other = new KeyboardEvent("keydown", {
+ key: "k",
+ code: "KeyV",
+ keyCode: 75,
+ ctrlKey: true,
+ bubbles: true,
+ cancelable: true,
+ });
+
+ window.dispatchEvent(paste);
+ window.dispatchEvent(other);
+
+ expect(paste.defaultPrevented).toBe(false);
+ expect(other.defaultPrevented).toBe(true);
+ expect(socket.sent).toEqual([
+ {
+ type: "key",
+ event: "down",
+ key: "k",
+ code: "KeyV",
+ text: "k",
+ windowsVirtualKeyCode: 75,
+ modifiers: 2,
+ },
+ ]);
+});
+
test("a paste keyup stays local when the modifier was released first", async () => {
const socket = await liveSocket();
window.dispatchEvent(