Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Fix punctuation and paste in remote browser control (#422)
maximus79234da3cf91a66
diff --git a/agent-computer/src/screencast.ts b/agent-computer/src/screencast.tsindex 9bc27c11..6b998188 100644--- a/agent-computer/src/screencast.ts+++ b/agent-computer/src/screencast.ts@@ -39,6 +39,8 @@ export type InputMessage =key: string;code: string;text?: string;+ /** Legacy browser keyCode, which CDP uses as its Windows virtual key code. */+ windowsVirtualKeyCode?: number;modifiers?: number;}| { type: "text"; text: string };@@ -78,6 +80,17 @@ const VIRTUAL_KEY_CODES: Record<string, number> = {ArrowRight: 39,ArrowDown: 40,Delete: 46,+ ";": 186,+ "=": 187,+ ",": 188,+ "-": 189,+ ".": 190,+ "/": 191,+ "`": 192,+ "[": 219,+ "\\": 220,+ "]": 221,+ "'": 222,};function virtualKeyCode(key: string): number {@@ -184,7 +197,13 @@ export async function startScreencast(}if (message.type === "key") {- const code = virtualKeyCode(message.key);+ const offeredCode = message.windowsVirtualKeyCode;+ const code =+ Number.isInteger(offeredCode) &&+ (offeredCode ?? 0) > 0 &&+ (offeredCode ?? 0) <= 255+ ? (offeredCode as number)+ : virtualKeyCode(message.key);await client.send("Input.dispatchKeyEvent", {// `keyDown` only when there is text to insert; otherwise `rawKeyDown`, which is what Chrome// expects for keys that do not produce a character. Sending keyDown with no text makesdiff --git a/agent-computer/tests/live-screen.test.ts b/agent-computer/tests/live-screen.test.tsindex ee306ce7..c3bc40c7 100644--- a/agent-computer/tests/live-screen.test.ts+++ b/agent-computer/tests/live-screen.test.ts@@ -75,6 +75,13 @@ const TYPING_PAGE ="<body>start</body><script>addEventListener('keydown',e=>{document.body.textContent+=e.key})</script>",);+/** A focused field whose value becomes page text, so character insertion is observable via /read. */+const TEXT_FIELD_PAGE =+ "data:text/html," ++ encodeURIComponent(+ "<input autofocus><output>empty</output><script>const input=document.querySelector('input');const output=document.querySelector('output');input.addEventListener('input',()=>output.textContent=input.value||'empty')</script>",+ );+let root = "";let closing: Array<() => void> = [];@@ -205,6 +212,7 @@ afterAll(async () => {"reset-viewer","wont-launch","still-starting",+ "punctuation",]) {await api("/computers/stop", botId, { method: "POST" }).catch(() => undefined,@@ -316,6 +324,75 @@ describe.skipIf(!asked)("a superseded socket closing later", () => {}, 30_000);});+describe.skipIf(!asked)("printable punctuation from the live screen", () => {+ test("inserts a period using the browser key code sent by the surface", async () => {+ const botId = "punctuation";+ await api("/navigate", botId, {+ method: "POST",+ body: JSON.stringify({ url: TEXT_FIELD_PAGE }),+ });+ const viewer = watch(botId);+ await viewer.casting;+ await api("/control/take", botId, { method: "POST" });++ viewer.socket.send(+ JSON.stringify({+ type: "key",+ event: "down",+ key: ".",+ code: "Period",+ text: ".",+ windowsVirtualKeyCode: 190,+ modifiers: 0,+ }),+ );+ viewer.socket.send(+ JSON.stringify({+ type: "key",+ event: "up",+ key: ".",+ code: "Period",+ windowsVirtualKeyCode: 190,+ modifiers: 0,+ }),+ );+ // Older OpenBot surfaces did not send the browser keyCode. Keep their punctuation usable while+ // a deployment rolls the frontend and computer images independently.+ viewer.socket.send(+ JSON.stringify({+ type: "key",+ event: "down",+ key: ".",+ code: "Period",+ text: ".",+ modifiers: 0,+ }),+ );+ viewer.socket.send(+ JSON.stringify({+ type: "key",+ event: "up",+ key: ".",+ code: "Period",+ modifiers: 0,+ }),+ );++ let landed = "";+ await until(+ () => landed.includes(".."),+ 5_000,+ "periods from current and legacy surfaces to be inserted into the focused field",+ async () => {+ const read = await api("/read", botId);+ landed = ((await read.json()) as { text: string }).text;+ },+ );++ expect(viewer.errors).toEqual([]);+ }, 30_000);+});+describe.skipIf(!asked)("the wheel, with the ownership check in front of it",() => {diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsxindex 37023112..356e0f2b 100644--- a/app/src/components/computer/live-screen.tsx+++ b/app/src/components/computer/live-screen.tsx@@ -31,6 +31,11 @@ function modifierBits(event: {);}+/** Let the local browser create a paste event, whose clipboard text is forwarded separately. */+function isPasteShortcut(event: KeyboardEvent): boolean {+ return (event.ctrlKey || event.metaKey) && event.key.toLowerCase() === "v";+}+type Props = {/*** Computer identity is part of the stream URL so input and frames stay scoped to the active Bot.@@ -45,6 +50,8 @@ type Props = {export function LiveScreen({ computerId, driving, onProblem }: Props) {const canvasRef = useRef<HTMLCanvasElement | null>(null);const socketRef = useRef<WebSocket | null>(null);+ /** Keydowns handled locally whose matching keyup must not leak to the remote browser. */+ const localKeyUps = useRef(new Set<string>());/** The size of the frames Chrome is sending, which is what input coordinates are relative to. */const frameSize = useRef<{ width: number; height: number } | null>(null);const [connected, setConnected] = useState(false);@@ -196,12 +203,17 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {if (!driving) return;const onKeyDown = (event: KeyboardEvent) => {if (event.key === "Escape") return; // Escape still closes the view.+ if (isPasteShortcut(event)) {+ localKeyUps.current.add(event.code);+ return;+ }event.preventDefault();send({type: "key",event: "down",key: event.key,code: event.code,+ windowsVirtualKeyCode: event.keyCode,// Only a printable character carries text. Sending text for Backspace makes Chrome insert a// character instead of deleting one....(event.key.length === 1 ? { text: event.key } : {}),@@ -210,12 +222,16 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {};const onKeyUp = (event: KeyboardEvent) => {if (event.key === "Escape") return;+ if (localKeyUps.current.delete(event.code) || isPasteShortcut(event)) {+ return;+ }event.preventDefault();send({type: "key",event: "up",key: event.key,code: event.code,+ windowsVirtualKeyCode: event.keyCode,modifiers: modifierBits(event),});};@@ -234,6 +250,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {window.removeEventListener("keydown", onKeyDown);window.removeEventListener("keyup", onKeyUp);window.removeEventListener("paste", onPaste);+ localKeyUps.current.clear();};}, [driving, send]);diff --git a/app/tests/live-screen-keyboard.test.tsx b/app/tests/live-screen-keyboard.test.tsxnew file mode 100644index 00000000..d7cd677d--- /dev/null+++ b/app/tests/live-screen-keyboard.test.tsx@@ -0,0 +1,164 @@+import { afterAll, afterEach, beforeAll, expect, test } from "bun:test";+import { GlobalRegistrator } from "@happy-dom/global-registrator";+import { cleanup, render, waitFor } from "@testing-library/react";+import { LiveScreen } from "@/components/computer/live-screen";++class SocketDouble {+ static readonly OPEN = 1;+ static readonly CLOSED = 3;+ static latest: SocketDouble | undefined;++ readyState = SocketDouble.OPEN;+ onopen: (() => void) | null = null;+ onmessage: (() => void) | null = null;+ onerror: (() => void) | null = null;+ onclose: (() => void) | null = null;+ readonly sent: Record<string, unknown>[] = [];++ constructor(_url: string) {+ SocketDouble.latest = this;+ queueMicrotask(() => this.onopen?.());+ }++ send(payload: string) {+ this.sent.push(JSON.parse(payload) as Record<string, unknown>);+ }++ close() {+ this.readyState = SocketDouble.CLOSED;+ this.onclose?.();+ }+}++let originalWebSocket: typeof WebSocket;++beforeAll(() => {+ GlobalRegistrator.register();+ originalWebSocket = globalThis.WebSocket;+ globalThis.WebSocket = SocketDouble as unknown as typeof WebSocket;+});++afterEach(() => {+ cleanup();+ SocketDouble.latest = undefined;+});++afterAll(() => {+ globalThis.WebSocket = originalWebSocket;+ GlobalRegistrator.unregister();+});++async function liveSocket(): Promise<SocketDouble> {+ render(<LiveScreen computerId="keyboard-test" driving />);+ await waitFor(() => expect(SocketDouble.latest).toBeDefined());+ return SocketDouble.latest as SocketDouble;+}++for (const [name, modifier] of [+ ["Ctrl+V", { ctrlKey: true }],+ ["Cmd+V", { metaKey: true }],+] as const) {+ test(`${name} stays in the local page so it can produce a paste event`, async () => {+ const socket = await liveSocket();+ const shortcut = new KeyboardEvent("keydown", {+ key: "v",+ code: "KeyV",+ ...modifier,+ bubbles: true,+ cancelable: true,+ });++ window.dispatchEvent(shortcut);++ expect(shortcut.defaultPrevented).toBe(false);+ expect(socket.sent).toEqual([]);+ });+}++test("a paste keyup stays local when the modifier was released first", async () => {+ const socket = await liveSocket();+ window.dispatchEvent(+ new KeyboardEvent("keydown", {+ key: "v",+ code: "KeyV",+ ctrlKey: true,+ bubbles: true,+ cancelable: true,+ }),+ );+ window.dispatchEvent(+ new KeyboardEvent("keyup", {+ key: "v",+ code: "KeyV",+ bubbles: true,+ cancelable: true,+ }),+ );++ expect(socket.sent).toEqual([]);+});++test("a period carries the browser's virtual key code to the remote screen", async () => {+ const socket = await liveSocket();+ window.dispatchEvent(+ new KeyboardEvent("keydown", {+ key: ".",+ code: "Period",+ keyCode: 190,+ bubbles: true,+ cancelable: true,+ }),+ );++ expect(socket.sent).toEqual([+ {+ type: "key",+ event: "down",+ key: ".",+ code: "Period",+ text: ".",+ windowsVirtualKeyCode: 190,+ modifiers: 0,+ },+ ]);+});++test("Ctrl+A remains a remote keyboard shortcut", async () => {+ const socket = await liveSocket();+ const shortcut = new KeyboardEvent("keydown", {+ key: "a",+ code: "KeyA",+ keyCode: 65,+ ctrlKey: true,+ bubbles: true,+ cancelable: true,+ });++ window.dispatchEvent(shortcut);++ expect(shortcut.defaultPrevented).toBe(true);+ expect(socket.sent).toEqual([+ {+ type: "key",+ event: "down",+ key: "a",+ code: "KeyA",+ text: "a",+ windowsVirtualKeyCode: 65,+ modifiers: 2,+ },+ ]);+});++test("the paste event sends clipboard text without forwarding it through a key event", async () => {+ const socket = await liveSocket();+ const paste = new Event("paste", { bubbles: true, cancelable: true });+ Object.defineProperty(paste, "clipboardData", {+ value: { getData: () => "." },+ });++ window.dispatchEvent(paste);++ expect(paste.defaultPrevented).toBe(true);+ expect(socket.sent).toEqual([{ type: "text", text: "." }]);+});