Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Keep a keystroke typed into a Bot's browser out of the app's own shortcuts (#602)
kevina187355659c9
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6fcac1fe..c8507f3a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.## Unreleased+### Typing into a Bot's browser no longer triggers the app's own shortcuts++While somebody drives a Bot's browser, every keystroke is sent to it. The app's shortcuts listen for+keystrokes too, and they heard each one first, so typing a capital N into the Bot's browser, as in+"New York", started a new chat and took the person away from the Bot mid-word, and Ctrl+B there+also showed or hid the sidebar. A keystroke sent to the Bot's browser now reaches only the Bot's+browser. Escape still closes the view, and the paste shortcut still pastes.### The Python LangGraph Bot on Anthropic answers after a skill was pickedA skill somebody picks reaches the Bot as a system message just ahead of their message, and itdiff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsxindex ce85392c..70ad2214 100644--- a/app/src/components/computer/live-screen.tsx+++ b/app/src/components/computer/live-screen.tsx@@ -279,6 +279,13 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {** Listen on window because canvas cannot hold focus. `preventDefault` keeps Tab and typing directed* at the remote page while takeover is active.+ *+ * The keydown in the capture phase, and stopped as well as prevented, because a keystroke sent to+ * the Bot's browser is not also this page's. The app's own shortcuts listen on this window too,+ * and they were bound first, when the signed-in app mounted, so they saw every keystroke before+ * this did: a capital N typed into the remote page started a new chat, and Ctrl+B there toggled+ * the sidebar here. Escape and the paste shortcut are not stopped, because both are meant for this+ * page.*/useEffect(() => {if (!driving) return;@@ -289,6 +296,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {return;}event.preventDefault();+ event.stopPropagation();send({type: "key",event: "down",@@ -324,11 +332,11 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {send({ type: "text", text });};- window.addEventListener("keydown", onKeyDown);+ window.addEventListener("keydown", onKeyDown, true);window.addEventListener("keyup", onKeyUp);window.addEventListener("paste", onPaste);return () => {- window.removeEventListener("keydown", onKeyDown);+ window.removeEventListener("keydown", onKeyDown, true);window.removeEventListener("keyup", onKeyUp);window.removeEventListener("paste", onPaste);localKeyUps.current.clear();diff --git a/app/tests/live-screen-app-shortcuts.test.tsx b/app/tests/live-screen-app-shortcuts.test.tsxnew file mode 100644index 00000000..8a563ee5--- /dev/null+++ b/app/tests/live-screen-app-shortcuts.test.tsx@@ -0,0 +1,174 @@+import { afterAll, afterEach, beforeAll, expect, test } from "bun:test";+import { GlobalRegistrator } from "@happy-dom/global-registrator";+import {+ createMemoryHistory,+ createRootRoute,+ createRoute,+ createRouter,+ Outlet,+ RouterProvider,+} from "@tanstack/react-router";+import { act, cleanup, render, waitFor } from "@testing-library/react";+import { LiveScreen } from "@/components/computer/live-screen";+import { SidebarProvider, useSidebar } from "@/components/ui/sidebar";+import { AppHotkeys } from "@/lib/hotkeys/app-hotkeys";++/**+ * The app's own shortcuts while a person drives a Bot's browser.+ *+ * Every keystroke then belongs to the Bot's browser: LiveScreen forwards it and prevents its default.+ * The app's shortcuts listen on the same window, and they were bound first, when the signed-in app+ * mounted, so they saw each keystroke before LiveScreen did and acted on it as well. Typing a+ * capital N into a page, "New York" in a search box, started a new chat and took the person away+ * from the Bot mid-word; Ctrl+B, bold in a document, showed or hid the sidebar here as well.+ */++class SocketDouble {+ static readonly OPEN = 1;+ static readonly CLOSED = 3;+ static latest: SocketDouble | undefined;++ readyState = SocketDouble.OPEN;+ onopen: (() => void) | null = null;+ onmessage: (() => void) | null = null;+ onerror: (() => void) | null = null;+ onclose: (() => void) | null = null;+ readonly sent: Record<string, unknown>[] = [];++ constructor(_url: string) {+ SocketDouble.latest = this;+ queueMicrotask(() => this.onopen?.());+ }++ send(payload: string) {+ this.sent.push(JSON.parse(payload) as Record<string, unknown>);+ }++ close() {+ this.readyState = SocketDouble.CLOSED;+ this.onclose?.();+ }+}++let originalWebSocket: typeof WebSocket;++beforeAll(() => {+ GlobalRegistrator.register();+ originalWebSocket = globalThis.WebSocket;+ globalThis.WebSocket = SocketDouble as unknown as typeof WebSocket;+});++afterEach(() => {+ cleanup();+ SocketDouble.latest = undefined;+});++afterAll(() => {+ globalThis.WebSocket = originalWebSocket;+ GlobalRegistrator.unregister();+});++function SidebarState() {+ return <output>{useSidebar().open ? "open" : "closed"}</output>;+}++/** The signed-in shell's shortcuts and sidebar, around a conversation showing a Bot's screen. */+async function renderDriving(driving: boolean) {+ const rootRoute = createRootRoute({+ component: () => (+ <SidebarProvider>+ <AppHotkeys />+ <SidebarState />+ <Outlet />+ </SidebarProvider>+ ),+ });+ const conversation = createRoute({+ getParentRoute: () => rootRoute,+ path: "/",+ component: () => (+ <LiveScreen computerId="shortcut-test" driving={driving} />+ ),+ });+ const newChat = createRoute({+ getParentRoute: () => rootRoute,+ path: "/channel/new",+ component: () => <p>New chat</p>,+ });+ const router = createRouter({+ routeTree: rootRoute.addChildren([conversation, newChat]),+ history: createMemoryHistory({ initialEntries: ["/"] }),+ });+ const view = render(<RouterProvider router={router} />);+ await waitFor(() =>+ expect(view.container.querySelector("canvas")).not.toBeNull(),+ );+ if (driving) await waitFor(() => expect(SocketDouble.latest).toBeDefined());+ return {+ router,+ sidebar: () => view.container.querySelector("output")?.textContent,+ /** A keystroke where the browser sends one while the page has no focused field: at the body. */+ press(init: KeyboardEventInit) {+ act(() => {+ document.body.dispatchEvent(+ new KeyboardEvent("keydown", {+ ...init,+ bubbles: true,+ cancelable: true,+ }),+ );+ });+ },+ };+}++test("a capital N typed into the Bot's browser goes to it, and does not start a new chat", async () => {+ const screen = await renderDriving(true);++ screen.press({ key: "N", code: "KeyN", keyCode: 78, shiftKey: true });+ await new Promise((resolve) => setTimeout(resolve, 50));++ expect(screen.router.state.location.pathname).toBe("/");+ expect(SocketDouble.latest?.sent).toEqual([+ {+ type: "key",+ event: "down",+ key: "N",+ code: "KeyN",+ text: "N",+ windowsVirtualKeyCode: 78,+ modifiers: 8,+ },+ ]);+});++test("Ctrl+B typed into the Bot's browser goes to it, and leaves the sidebar alone", async () => {+ const screen = await renderDriving(true);++ screen.press({ key: "b", code: "KeyB", keyCode: 66, ctrlKey: true });++ expect(screen.sidebar()).toBe("open");+ expect(SocketDouble.latest?.sent).toEqual([+ {+ type: "key",+ event: "down",+ key: "b",+ code: "KeyB",+ text: "b",+ windowsVirtualKeyCode: 66,+ modifiers: 2,+ },+ ]);+});++test("the same shortcuts still work while nobody is driving", async () => {+ const screen = await renderDriving(false);++ screen.press({ key: "b", code: "KeyB", keyCode: 66, ctrlKey: true });+ expect(screen.sidebar()).toBe("closed");++ screen.press({ key: "N", code: "KeyN", keyCode: 78, shiftKey: true });+ await waitFor(() =>+ expect(screen.router.state.location.pathname).toBe("/channel/new"),+ );+});