Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Fix punctuation and paste in remote browser control (#422)

maximus79234da3cf91a66
diff --git a/agent-computer/src/screencast.ts b/agent-computer/src/screencast.ts
index 9bc27c11..6b998188 100644
--- a/agent-computer/src/screencast.ts
+++ b/agent-computer/src/screencast.ts
@@ -39,6 +39,8 @@ export type InputMessage =
key: string;
code: string;
text?: string;
+ /** Legacy browser keyCode, which CDP uses as its Windows virtual key code. */
+ windowsVirtualKeyCode?: number;
modifiers?: number;
}
| { type: "text"; text: string };
@@ -78,6 +80,17 @@ const VIRTUAL_KEY_CODES: Record<string, number> = {
ArrowRight: 39,
ArrowDown: 40,
Delete: 46,
+ ";": 186,
+ "=": 187,
+ ",": 188,
+ "-": 189,
+ ".": 190,
+ "/": 191,
+ "`": 192,
+ "[": 219,
+ "\\": 220,
+ "]": 221,
+ "'": 222,
};
function virtualKeyCode(key: string): number {
@@ -184,7 +197,13 @@ export async function startScreencast(
}
if (message.type === "key") {
- const code = virtualKeyCode(message.key);
+ const offeredCode = message.windowsVirtualKeyCode;
+ const code =
+ Number.isInteger(offeredCode) &&
+ (offeredCode ?? 0) > 0 &&
+ (offeredCode ?? 0) <= 255
+ ? (offeredCode as number)
+ : virtualKeyCode(message.key);
await client.send("Input.dispatchKeyEvent", {
// `keyDown` only when there is text to insert; otherwise `rawKeyDown`, which is what Chrome
// expects for keys that do not produce a character. Sending keyDown with no text makes
diff --git a/agent-computer/tests/live-screen.test.ts b/agent-computer/tests/live-screen.test.ts
index ee306ce7..c3bc40c7 100644
--- a/agent-computer/tests/live-screen.test.ts
+++ b/agent-computer/tests/live-screen.test.ts
@@ -75,6 +75,13 @@ const TYPING_PAGE =
"<body>start</body><script>addEventListener('keydown',e=>{document.body.textContent+=e.key})</script>",
);
+/** A focused field whose value becomes page text, so character insertion is observable via /read. */
+const TEXT_FIELD_PAGE =
+ "data:text/html," +
+ encodeURIComponent(
+ "<input autofocus><output>empty</output><script>const input=document.querySelector('input');const output=document.querySelector('output');input.addEventListener('input',()=>output.textContent=input.value||'empty')</script>",
+ );
+
let root = "";
let closing: Array<() => void> = [];
@@ -205,6 +212,7 @@ afterAll(async () => {
"reset-viewer",
"wont-launch",
"still-starting",
+ "punctuation",
]) {
await api("/computers/stop", botId, { method: "POST" }).catch(
() => undefined,
@@ -316,6 +324,75 @@ describe.skipIf(!asked)("a superseded socket closing later", () => {
}, 30_000);
});
+describe.skipIf(!asked)("printable punctuation from the live screen", () => {
+ test("inserts a period using the browser key code sent by the surface", async () => {
+ const botId = "punctuation";
+ await api("/navigate", botId, {
+ method: "POST",
+ body: JSON.stringify({ url: TEXT_FIELD_PAGE }),
+ });
+ const viewer = watch(botId);
+ await viewer.casting;
+ await api("/control/take", botId, { method: "POST" });
+
+ viewer.socket.send(
+ JSON.stringify({
+ type: "key",
+ event: "down",
+ key: ".",
+ code: "Period",
+ text: ".",
+ windowsVirtualKeyCode: 190,
+ modifiers: 0,
+ }),
+ );
+ viewer.socket.send(
+ JSON.stringify({
+ type: "key",
+ event: "up",
+ key: ".",
+ code: "Period",
+ windowsVirtualKeyCode: 190,
+ modifiers: 0,
+ }),
+ );
+ // Older OpenBot surfaces did not send the browser keyCode. Keep their punctuation usable while
+ // a deployment rolls the frontend and computer images independently.
+ viewer.socket.send(
+ JSON.stringify({
+ type: "key",
+ event: "down",
+ key: ".",
+ code: "Period",
+ text: ".",
+ modifiers: 0,
+ }),
+ );
+ viewer.socket.send(
+ JSON.stringify({
+ type: "key",
+ event: "up",
+ key: ".",
+ code: "Period",
+ modifiers: 0,
+ }),
+ );
+
+ let landed = "";
+ await until(
+ () => landed.includes(".."),
+ 5_000,
+ "periods from current and legacy surfaces to be inserted into the focused field",
+ async () => {
+ const read = await api("/read", botId);
+ landed = ((await read.json()) as { text: string }).text;
+ },
+ );
+
+ expect(viewer.errors).toEqual([]);
+ }, 30_000);
+});
+
describe.skipIf(!asked)(
"the wheel, with the ownership check in front of it",
() => {
diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx
index 37023112..356e0f2b 100644
--- a/app/src/components/computer/live-screen.tsx
+++ b/app/src/components/computer/live-screen.tsx
@@ -31,6 +31,11 @@ function modifierBits(event: {
);
}
+/** Let the local browser create a paste event, whose clipboard text is forwarded separately. */
+function isPasteShortcut(event: KeyboardEvent): boolean {
+ return (event.ctrlKey || event.metaKey) && event.key.toLowerCase() === "v";
+}
+
type Props = {
/**
* Computer identity is part of the stream URL so input and frames stay scoped to the active Bot.
@@ -45,6 +50,8 @@ type Props = {
export function LiveScreen({ computerId, driving, onProblem }: Props) {
const canvasRef = useRef<HTMLCanvasElement | null>(null);
const socketRef = useRef<WebSocket | null>(null);
+ /** Keydowns handled locally whose matching keyup must not leak to the remote browser. */
+ const localKeyUps = useRef(new Set<string>());
/** The size of the frames Chrome is sending, which is what input coordinates are relative to. */
const frameSize = useRef<{ width: number; height: number } | null>(null);
const [connected, setConnected] = useState(false);
@@ -196,12 +203,17 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {
if (!driving) return;
const onKeyDown = (event: KeyboardEvent) => {
if (event.key === "Escape") return; // Escape still closes the view.
+ if (isPasteShortcut(event)) {
+ localKeyUps.current.add(event.code);
+ return;
+ }
event.preventDefault();
send({
type: "key",
event: "down",
key: event.key,
code: event.code,
+ windowsVirtualKeyCode: event.keyCode,
// Only a printable character carries text. Sending text for Backspace makes Chrome insert a
// character instead of deleting one.
...(event.key.length === 1 ? { text: event.key } : {}),
@@ -210,12 +222,16 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {
};
const onKeyUp = (event: KeyboardEvent) => {
if (event.key === "Escape") return;
+ if (localKeyUps.current.delete(event.code) || isPasteShortcut(event)) {
+ return;
+ }
event.preventDefault();
send({
type: "key",
event: "up",
key: event.key,
code: event.code,
+ windowsVirtualKeyCode: event.keyCode,
modifiers: modifierBits(event),
});
};
@@ -234,6 +250,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {
window.removeEventListener("keydown", onKeyDown);
window.removeEventListener("keyup", onKeyUp);
window.removeEventListener("paste", onPaste);
+ localKeyUps.current.clear();
};
}, [driving, send]);
diff --git a/app/tests/live-screen-keyboard.test.tsx b/app/tests/live-screen-keyboard.test.tsx
new file mode 100644
index 00000000..d7cd677d
--- /dev/null
+++ b/app/tests/live-screen-keyboard.test.tsx
@@ -0,0 +1,164 @@
+import { afterAll, afterEach, beforeAll, expect, test } from "bun:test";
+import { GlobalRegistrator } from "@happy-dom/global-registrator";
+import { cleanup, render, waitFor } from "@testing-library/react";
+import { LiveScreen } from "@/components/computer/live-screen";
+
+class SocketDouble {
+ static readonly OPEN = 1;
+ static readonly CLOSED = 3;
+ static latest: SocketDouble | undefined;
+
+ readyState = SocketDouble.OPEN;
+ onopen: (() => void) | null = null;
+ onmessage: (() => void) | null = null;
+ onerror: (() => void) | null = null;
+ onclose: (() => void) | null = null;
+ readonly sent: Record<string, unknown>[] = [];
+
+ constructor(_url: string) {
+ SocketDouble.latest = this;
+ queueMicrotask(() => this.onopen?.());
+ }
+
+ send(payload: string) {
+ this.sent.push(JSON.parse(payload) as Record<string, unknown>);
+ }
+
+ close() {
+ this.readyState = SocketDouble.CLOSED;
+ this.onclose?.();
+ }
+}
+
+let originalWebSocket: typeof WebSocket;
+
+beforeAll(() => {
+ GlobalRegistrator.register();
+ originalWebSocket = globalThis.WebSocket;
+ globalThis.WebSocket = SocketDouble as unknown as typeof WebSocket;
+});
+
+afterEach(() => {
+ cleanup();
+ SocketDouble.latest = undefined;
+});
+
+afterAll(() => {
+ globalThis.WebSocket = originalWebSocket;
+ GlobalRegistrator.unregister();
+});
+
+async function liveSocket(): Promise<SocketDouble> {
+ render(<LiveScreen computerId="keyboard-test" driving />);
+ await waitFor(() => expect(SocketDouble.latest).toBeDefined());
+ return SocketDouble.latest as SocketDouble;
+}
+
+for (const [name, modifier] of [
+ ["Ctrl+V", { ctrlKey: true }],
+ ["Cmd+V", { metaKey: true }],
+] as const) {
+ test(`${name} stays in the local page so it can produce a paste event`, async () => {
+ const socket = await liveSocket();
+ const shortcut = new KeyboardEvent("keydown", {
+ key: "v",
+ code: "KeyV",
+ ...modifier,
+ bubbles: true,
+ cancelable: true,
+ });
+
+ window.dispatchEvent(shortcut);
+
+ expect(shortcut.defaultPrevented).toBe(false);
+ expect(socket.sent).toEqual([]);
+ });
+}
+
+test("a paste keyup stays local when the modifier was released first", async () => {
+ const socket = await liveSocket();
+ window.dispatchEvent(
+ new KeyboardEvent("keydown", {
+ key: "v",
+ code: "KeyV",
+ ctrlKey: true,
+ bubbles: true,
+ cancelable: true,
+ }),
+ );
+ window.dispatchEvent(
+ new KeyboardEvent("keyup", {
+ key: "v",
+ code: "KeyV",
+ bubbles: true,
+ cancelable: true,
+ }),
+ );
+
+ expect(socket.sent).toEqual([]);
+});
+
+test("a period carries the browser's virtual key code to the remote screen", async () => {
+ const socket = await liveSocket();
+ window.dispatchEvent(
+ new KeyboardEvent("keydown", {
+ key: ".",
+ code: "Period",
+ keyCode: 190,
+ bubbles: true,
+ cancelable: true,
+ }),
+ );
+
+ expect(socket.sent).toEqual([
+ {
+ type: "key",
+ event: "down",
+ key: ".",
+ code: "Period",
+ text: ".",
+ windowsVirtualKeyCode: 190,
+ modifiers: 0,
+ },
+ ]);
+});
+
+test("Ctrl+A remains a remote keyboard shortcut", async () => {
+ const socket = await liveSocket();
+ const shortcut = new KeyboardEvent("keydown", {
+ key: "a",
+ code: "KeyA",
+ keyCode: 65,
+ ctrlKey: true,
+ bubbles: true,
+ cancelable: true,
+ });
+
+ window.dispatchEvent(shortcut);
+
+ expect(shortcut.defaultPrevented).toBe(true);
+ expect(socket.sent).toEqual([
+ {
+ type: "key",
+ event: "down",
+ key: "a",
+ code: "KeyA",
+ text: "a",
+ windowsVirtualKeyCode: 65,
+ modifiers: 2,
+ },
+ ]);
+});
+
+test("the paste event sends clipboard text without forwarding it through a key event", async () => {
+ const socket = await liveSocket();
+ const paste = new Event("paste", { bubbles: true, cancelable: true });
+ Object.defineProperty(paste, "clipboardData", {
+ value: { getData: () => "." },
+ });
+
+ window.dispatchEvent(paste);
+
+ expect(paste.defaultPrevented).toBe(true);
+ expect(socket.sent).toEqual([{ type: "text", text: "." }]);
+});