Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Keep a keystroke typed into a Bot's browser out of the app's own shortcuts (#602)

kevina187355659c9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6fcac1fe..c8507f3a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
## Unreleased
+### Typing into a Bot's browser no longer triggers the app's own shortcuts
+
+While somebody drives a Bot's browser, every keystroke is sent to it. The app's shortcuts listen for
+keystrokes too, and they heard each one first, so typing a capital N into the Bot's browser, as in
+"New York", started a new chat and took the person away from the Bot mid-word, and Ctrl+B there
+also showed or hid the sidebar. A keystroke sent to the Bot's browser now reaches only the Bot's
+browser. Escape still closes the view, and the paste shortcut still pastes.
### The Python LangGraph Bot on Anthropic answers after a skill was picked
A skill somebody picks reaches the Bot as a system message just ahead of their message, and it
diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx
index ce85392c..70ad2214 100644
--- a/app/src/components/computer/live-screen.tsx
+++ b/app/src/components/computer/live-screen.tsx
@@ -279,6 +279,13 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {
*
* Listen on window because canvas cannot hold focus. `preventDefault` keeps Tab and typing directed
* at the remote page while takeover is active.
+ *
+ * The keydown in the capture phase, and stopped as well as prevented, because a keystroke sent to
+ * the Bot's browser is not also this page's. The app's own shortcuts listen on this window too,
+ * and they were bound first, when the signed-in app mounted, so they saw every keystroke before
+ * this did: a capital N typed into the remote page started a new chat, and Ctrl+B there toggled
+ * the sidebar here. Escape and the paste shortcut are not stopped, because both are meant for this
+ * page.
*/
useEffect(() => {
if (!driving) return;
@@ -289,6 +296,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {
return;
}
event.preventDefault();
+ event.stopPropagation();
send({
type: "key",
event: "down",
@@ -324,11 +332,11 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) {
send({ type: "text", text });
};
- window.addEventListener("keydown", onKeyDown);
+ window.addEventListener("keydown", onKeyDown, true);
window.addEventListener("keyup", onKeyUp);
window.addEventListener("paste", onPaste);
return () => {
- window.removeEventListener("keydown", onKeyDown);
+ window.removeEventListener("keydown", onKeyDown, true);
window.removeEventListener("keyup", onKeyUp);
window.removeEventListener("paste", onPaste);
localKeyUps.current.clear();
diff --git a/app/tests/live-screen-app-shortcuts.test.tsx b/app/tests/live-screen-app-shortcuts.test.tsx
new file mode 100644
index 00000000..8a563ee5
--- /dev/null
+++ b/app/tests/live-screen-app-shortcuts.test.tsx
@@ -0,0 +1,174 @@
+import { afterAll, afterEach, beforeAll, expect, test } from "bun:test";
+import { GlobalRegistrator } from "@happy-dom/global-registrator";
+import {
+ createMemoryHistory,
+ createRootRoute,
+ createRoute,
+ createRouter,
+ Outlet,
+ RouterProvider,
+} from "@tanstack/react-router";
+import { act, cleanup, render, waitFor } from "@testing-library/react";
+import { LiveScreen } from "@/components/computer/live-screen";
+import { SidebarProvider, useSidebar } from "@/components/ui/sidebar";
+import { AppHotkeys } from "@/lib/hotkeys/app-hotkeys";
+
+/**
+ * The app's own shortcuts while a person drives a Bot's browser.
+ *
+ * Every keystroke then belongs to the Bot's browser: LiveScreen forwards it and prevents its default.
+ * The app's shortcuts listen on the same window, and they were bound first, when the signed-in app
+ * mounted, so they saw each keystroke before LiveScreen did and acted on it as well. Typing a
+ * capital N into a page, "New York" in a search box, started a new chat and took the person away
+ * from the Bot mid-word; Ctrl+B, bold in a document, showed or hid the sidebar here as well.
+ */
+
+class SocketDouble {
+ static readonly OPEN = 1;
+ static readonly CLOSED = 3;
+ static latest: SocketDouble | undefined;
+
+ readyState = SocketDouble.OPEN;
+ onopen: (() => void) | null = null;
+ onmessage: (() => void) | null = null;
+ onerror: (() => void) | null = null;
+ onclose: (() => void) | null = null;
+ readonly sent: Record<string, unknown>[] = [];
+
+ constructor(_url: string) {
+ SocketDouble.latest = this;
+ queueMicrotask(() => this.onopen?.());
+ }
+
+ send(payload: string) {
+ this.sent.push(JSON.parse(payload) as Record<string, unknown>);
+ }
+
+ close() {
+ this.readyState = SocketDouble.CLOSED;
+ this.onclose?.();
+ }
+}
+
+let originalWebSocket: typeof WebSocket;
+
+beforeAll(() => {
+ GlobalRegistrator.register();
+ originalWebSocket = globalThis.WebSocket;
+ globalThis.WebSocket = SocketDouble as unknown as typeof WebSocket;
+});
+
+afterEach(() => {
+ cleanup();
+ SocketDouble.latest = undefined;
+});
+
+afterAll(() => {
+ globalThis.WebSocket = originalWebSocket;
+ GlobalRegistrator.unregister();
+});
+
+function SidebarState() {
+ return <output>{useSidebar().open ? "open" : "closed"}</output>;
+}
+
+/** The signed-in shell's shortcuts and sidebar, around a conversation showing a Bot's screen. */
+async function renderDriving(driving: boolean) {
+ const rootRoute = createRootRoute({
+ component: () => (
+ <SidebarProvider>
+ <AppHotkeys />
+ <SidebarState />
+ <Outlet />
+ </SidebarProvider>
+ ),
+ });
+ const conversation = createRoute({
+ getParentRoute: () => rootRoute,
+ path: "/",
+ component: () => (
+ <LiveScreen computerId="shortcut-test" driving={driving} />
+ ),
+ });
+ const newChat = createRoute({
+ getParentRoute: () => rootRoute,
+ path: "/channel/new",
+ component: () => <p>New chat</p>,
+ });
+ const router = createRouter({
+ routeTree: rootRoute.addChildren([conversation, newChat]),
+ history: createMemoryHistory({ initialEntries: ["/"] }),
+ });
+ const view = render(<RouterProvider router={router} />);
+ await waitFor(() =>
+ expect(view.container.querySelector("canvas")).not.toBeNull(),
+ );
+ if (driving) await waitFor(() => expect(SocketDouble.latest).toBeDefined());
+ return {
+ router,
+ sidebar: () => view.container.querySelector("output")?.textContent,
+ /** A keystroke where the browser sends one while the page has no focused field: at the body. */
+ press(init: KeyboardEventInit) {
+ act(() => {
+ document.body.dispatchEvent(
+ new KeyboardEvent("keydown", {
+ ...init,
+ bubbles: true,
+ cancelable: true,
+ }),
+ );
+ });
+ },
+ };
+}
+
+test("a capital N typed into the Bot's browser goes to it, and does not start a new chat", async () => {
+ const screen = await renderDriving(true);
+
+ screen.press({ key: "N", code: "KeyN", keyCode: 78, shiftKey: true });
+ await new Promise((resolve) => setTimeout(resolve, 50));
+
+ expect(screen.router.state.location.pathname).toBe("/");
+ expect(SocketDouble.latest?.sent).toEqual([
+ {
+ type: "key",
+ event: "down",
+ key: "N",
+ code: "KeyN",
+ text: "N",
+ windowsVirtualKeyCode: 78,
+ modifiers: 8,
+ },
+ ]);
+});
+
+test("Ctrl+B typed into the Bot's browser goes to it, and leaves the sidebar alone", async () => {
+ const screen = await renderDriving(true);
+
+ screen.press({ key: "b", code: "KeyB", keyCode: 66, ctrlKey: true });
+
+ expect(screen.sidebar()).toBe("open");
+ expect(SocketDouble.latest?.sent).toEqual([
+ {
+ type: "key",
+ event: "down",
+ key: "b",
+ code: "KeyB",
+ text: "b",
+ windowsVirtualKeyCode: 66,
+ modifiers: 2,
+ },
+ ]);
+});
+
+test("the same shortcuts still work while nobody is driving", async () => {
+ const screen = await renderDriving(false);
+
+ screen.press({ key: "b", code: "KeyB", keyCode: 66, ctrlKey: true });
+ expect(screen.sidebar()).toBe("closed");
+
+ screen.press({ key: "N", code: "KeyN", keyCode: 78, shiftKey: true });
+ await waitFor(() =>
+ expect(screen.router.state.location.pathname).toBe("/channel/new"),
+ );
+});