Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Make Enter act while a person is driving the browser (#477)

kevin1c2c87b61437
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cde4540e..9236e397 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,19 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
## Unreleased
+### Pressing Enter works while a person is driving a Bot's browser
+
+Taking the wheel of a Bot's browser is mostly for the sign-in it cannot do itself, and Enter is how a
+sign-in ends. Every keystroke reached the page, and Enter reached it as a key press that produces no
+character — which Chrome delivers to the page's own listeners and then does nothing further with. So
+the form did not submit, a new line in a text box did not start, and a button somebody had tabbed to
+was not pressed, while anything on the page listening for the key saw it arrive. There was nothing on
+screen to explain it: the keystroke was not refused, it simply had no effect, and the way out was to
+click the submit button instead. Enter now carries the carriage return a keyboard sends, which is
+what makes Chrome carry out what the key means. Measured against Chromium 151: every other editing
+key — Backspace, Delete, Tab, Home, End and the arrows — already did what it meant and is unchanged,
+and a single-line field still holds exactly what was typed into it.
+
### A half-ticked box is no longer described to a Bot as ticked
The snapshot a Bot reads before it acts on a page says whether each box is ticked, and Playwright
diff --git a/agent-computer/src/screencast.ts b/agent-computer/src/screencast.ts
index 6b998188..13207a06 100644
--- a/agent-computer/src/screencast.ts
+++ b/agent-computer/src/screencast.ts
@@ -93,6 +93,29 @@ const VIRTUAL_KEY_CODES: Record<string, number> = {
"'": 222,
};
+/**
+ * The character a key stands for, where Chrome will not act on the key without one.
+ *
+ * `Input.dispatchKeyEvent` has two kinds of key-down. `rawKeyDown` is a key that produces no
+ * character: Chrome delivers it to the page and stops there, so a listener fires and nothing else
+ * happens. `keyDown` carries text, and that is what makes Chrome perform the key's own default
+ * action.
+ *
+ * Sending text only for a printable character therefore left Enter as a key a page could hear and
+ * not act on. Measured against Chromium 151 through this file: with a person holding the wheel,
+ * Enter did not submit the form they had just filled in, did not start a new line in a textarea and
+ * did not press the button they had tabbed to, while the page's own `keydown` listener saw every
+ * one of them. Which is the sign-in at the end of almost every takeover.
+ *
+ * One key rather than a list, measured the same way: Backspace, Delete, Tab, Home, End and the four
+ * arrows all do what they mean as a `rawKeyDown`, because their default action is not the insertion
+ * of a character.
+ *
+ * A carriage return, which is what a keyboard's Enter carries. Chrome turns it into whatever the
+ * field it lands in needs, so a single-line input is left holding no extra character.
+ */
+const TEXT_FOR_KEY: Record<string, string | undefined> = { Enter: "\r" };
+
function virtualKeyCode(key: string): number {
if (VIRTUAL_KEY_CODES[key] !== undefined) return VIRTUAL_KEY_CODES[key];
// A single printable character carries its own code point, upper-cased, which is what Chrome expects
@@ -204,19 +227,26 @@ export async function startScreencast(
(offeredCode ?? 0) <= 255
? (offeredCode as number)
: virtualKeyCode(message.key);
+ /*
+ * The text this key carries: the surface supplies it for a printable character, and
+ * `TEXT_FOR_KEY` fills in the keys whose default action Chrome will not perform without one.
+ *
+ * Only on the way down, because that is where a default action happens. A key going up
+ * carries whatever it was given and nothing more.
+ */
+ const text =
+ message.event === "up"
+ ? message.text
+ : (message.text ?? TEXT_FOR_KEY[message.key]);
await client.send("Input.dispatchKeyEvent", {
// `keyDown` only when there is text to insert; otherwise `rawKeyDown`, which is what Chrome
// expects for keys that do not produce a character. Sending keyDown with no text makes
// editing keys arrive as nothing.
type:
- message.event === "up"
- ? "keyUp"
- : message.text
- ? "keyDown"
- : "rawKeyDown",
+ message.event === "up" ? "keyUp" : text ? "keyDown" : "rawKeyDown",
key: message.key,
code: message.code,
- ...(message.text ? { text: message.text } : {}),
+ ...(text ? { text } : {}),
windowsVirtualKeyCode: code,
nativeVirtualKeyCode: code,
modifiers: message.modifiers ?? 0,
diff --git a/agent-computer/tests/screencast.test.ts b/agent-computer/tests/screencast.test.ts
new file mode 100644
index 00000000..0527a1e6
--- /dev/null
+++ b/agent-computer/tests/screencast.test.ts
@@ -0,0 +1,109 @@
+import { describe, expect, test } from "bun:test";
+import type { Page } from "playwright";
+import { type InputMessage, startScreencast } from "../src/screencast";
+
+/**
+ * What a person's keystroke turns into on the wire.
+ *
+ * `screencast.ts` imports Playwright for its types only, and a type import is gone at run time, so
+ * this reaches the translation without a browser. What it cannot reach is what Chrome then does with
+ * it, which is the half that was wrong: `rawKeyDown` and `keyDown` are both delivered to the page,
+ * and only one of them makes Chrome carry out what the key means. That difference was measured
+ * against Chromium 151 rather than guessed, and this file pins the parameters that measurement
+ * settled on.
+ */
+
+/** A CDP session that records what was asked of it, and answers everything. */
+function recording() {
+ const sent: { method: string; params: Record<string, unknown> }[] = [];
+ const client = {
+ on() {},
+ async send(method: string, params: Record<string, unknown> = {}) {
+ sent.push({ method, params });
+ },
+ async detach() {},
+ };
+ const page = {
+ context: () => ({ newCDPSession: async () => client }),
+ } as unknown as Page;
+ return { page, sent };
+}
+
+/** The parameters of the one key event a message produced. */
+async function dispatched(message: InputMessage) {
+ const { page, sent } = recording();
+ const cast = await startScreencast(page, () => {});
+ sent.length = 0;
+ await cast.send(message);
+ const keyEvents = sent.filter(
+ (call) => call.method === "Input.dispatchKeyEvent",
+ );
+ expect(keyEvents).toHaveLength(1);
+ return keyEvents[0]?.params as Record<string, unknown>;
+}
+
+/** A keystroke exactly as the live screen sends one: text only for a printable character. */
+function typed(
+ event: "down" | "up",
+ key: string,
+ code: string,
+ keyCode: number,
+): InputMessage {
+ return {
+ type: "key",
+ event,
+ key,
+ code,
+ windowsVirtualKeyCode: keyCode,
+ ...(key.length === 1 ? { text: key } : {}),
+ modifiers: 0,
+ };
+}
+
+describe("a key a person presses while they hold the wheel", () => {
+ test("Enter is a press the page acts on, not one it merely hears", async () => {
+ // Measured through this file against Chromium 151: as a `rawKeyDown` the page's own `keydown`
+ // listener saw Enter and nothing else happened -- the sign-in form did not submit, a textarea
+ // did not take a new line, and a focused button was not pressed.
+ const params = await dispatched(typed("down", "Enter", "Enter", 13));
+ expect(params.type).toBe("keyDown");
+ expect(params.text).toBe("\r");
+ });
+
+ test("a printable character keeps the text the person typed", async () => {
+ const params = await dispatched(typed("down", "a", "KeyA", 65));
+ expect(params.type).toBe("keyDown");
+ expect(params.text).toBe("a");
+ });
+
+ test("a key that produces no character is still a rawKeyDown", async () => {
+ // Backspace, Delete, Tab, Home, End and the arrows were all measured to do what they mean
+ // without text, so filling one in for them would be inventing an insertion.
+ for (const [key, code, keyCode] of [
+ ["Backspace", "Backspace", 8],
+ ["Delete", "Delete", 46],
+ ["Tab", "Tab", 9],
+ ["ArrowLeft", "ArrowLeft", 37],
+ ["Home", "Home", 36],
+ ] as const) {
+ const params = await dispatched(typed("down", key, code, keyCode));
+ expect(params.type).toBe("rawKeyDown");
+ expect(params).not.toHaveProperty("text");
+ }
+ });
+
+ test("Enter going up is a key up carrying nothing it was not given", async () => {
+ // The fill-in belongs to the way down, where a default action happens. A key up that carried a
+ // carriage return would be a second one.
+ const params = await dispatched(typed("up", "Enter", "Enter", 13));
+ expect(params.type).toBe("keyUp");
+ expect(params).not.toHaveProperty("text");
+ });
+
+ test("the virtual key code the surface offered is the one that is sent", async () => {
+ // A form field ignores a bare Enter or Backspace without this, which is why it is sent at all.
+ const params = await dispatched(typed("down", "Enter", "Enter", 13));
+ expect(params.windowsVirtualKeyCode).toBe(13);
+ expect(params.nativeVirtualKeyCode).toBe(13);
+ });
+});