Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Validate scroll deltas and live-screen input on the Bot computer (#488)

Felix-Ayushe197209fdd0c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e809d6ae..1f29a867 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,12 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
## Unreleased
+### The Bot computer refuses a malformed scroll or live input before the browser sees it
+
+A non-finite wheel delta travelled into Playwright and came back as a 502 that read as a broken
+computer, and any JSON object on the live-screen socket fell through to `Input.insertText` or
+forwarded wrong-typed coordinates to CDP. Scroll deltas must be finite numbers now, and live
+input must match its mouse, wheel, key, or text shape; anything else is a 400 naming the field.
### A non-string plugin grant or tool call is refused before it reaches the store
`POST /api/plugins/grants` and `POST /api/plugins/call` checked presence, not shape, so a JSON
diff --git a/agent-computer/src/index.ts b/agent-computer/src/index.ts
index 2468c3fd..16868122 100644
--- a/agent-computer/src/index.ts
+++ b/agent-computer/src/index.ts
@@ -19,7 +19,12 @@ import {
} from "./control";
import { identity } from "./identity";
import { createProfiles, numberFromEnv, VIEWPORT } from "./profiles";
-import { parseExecTimeout, parseNavigateUrl } from "./request-validation";
+import {
+ parseExecTimeout,
+ parseInputMessage,
+ parseNavigateUrl,
+ parseScrollDelta,
+} from "./request-validation";
import { type InputMessage, startScreencast } from "./screencast";
import { createShell } from "./shell";
import { createViewerSlot, type ViewerSlot } from "./viewer";
@@ -498,7 +503,13 @@ serve<StreamData>({
}
let message: InputMessage;
try {
- message = JSON.parse(String(raw)) as InputMessage;
+ const parsed: unknown = JSON.parse(String(raw));
+ const validated = parseInputMessage(parsed);
+ if (!validated.ok) {
+ ws.send(JSON.stringify({ type: "error", error: validated.error }));
+ return;
+ }
+ message = validated.message;
} catch {
return;
}
@@ -744,6 +755,12 @@ serve<StreamData>({
string,
unknown
> | null;
+ if (url.pathname === "/human/scroll") {
+ const parsed = parseScrollDelta(body?.deltaY);
+ if (!parsed.ok) {
+ return json({ error: parsed.error }, 400);
+ }
+ }
try {
const target = await currentPage(botId);
return json(await performHumanInput(target, url.pathname, body ?? {}));
@@ -1008,6 +1025,12 @@ serve<StreamData>({
if (!body) {
return json({ error: "An action needs a JSON body." }, 400);
}
+ if (url.pathname === "/scroll") {
+ const parsed = parseScrollDelta(body.deltaY);
+ if (!parsed.ok) {
+ return json({ error: parsed.error }, 400);
+ }
+ }
const startedAt = Date.now();
try {
@@ -1136,7 +1159,11 @@ async function performHumanInput(
return { action: "human_key", key: body.key, url: target.url() };
}
- const deltaY = typeof body.deltaY === "number" ? body.deltaY : 400;
+ const parsed = parseScrollDelta(body.deltaY);
+ if (!parsed.ok) {
+ throw new Error(parsed.error);
+ }
+ const deltaY = parsed.deltaY ?? 400;
await target.mouse.wheel(0, deltaY);
return { action: "human_scroll", deltaY, url: target.url() };
}
@@ -1214,7 +1241,11 @@ async function performAction(
// Scroll. A plain wheel event on the page, which is what moves a long form, rather than scrolling a
// specific element into view: the Bot asked to see further down, not to hunt for one control.
- const deltaY = typeof body.deltaY === "number" ? body.deltaY : 600;
+ const parsed = parseScrollDelta(body.deltaY);
+ if (!parsed.ok) {
+ throw new Error(parsed.error);
+ }
+ const deltaY = parsed.deltaY ?? 600;
await target.mouse.wheel(0, deltaY);
return { action: "scroll", deltaY, url: target.url() };
}
diff --git a/agent-computer/src/request-validation.ts b/agent-computer/src/request-validation.ts
index 02379a96..99dcccc0 100644
--- a/agent-computer/src/request-validation.ts
+++ b/agent-computer/src/request-validation.ts
@@ -5,6 +5,7 @@
* is checked here is whether the request is well-formed at all, so a typo'd caller gets a 400
* naming the field rather than a 502 that reads as a broken computer.
*/
+import type { InputMessage } from "./screencast";
/** Long enough for an install, short enough that a hung command is not a hung Bot. */
export const EXEC_DEFAULT_TIMEOUT_MS = 120_000;
@@ -44,6 +45,32 @@ export type ExecTimeoutParseResult =
| { ok: true; timeoutMs: number | undefined }
| { ok: false; error: string };
+export type ScrollDeltaParseResult =
+ | { ok: true; deltaY: number | undefined }
+ | { ok: false; error: string };
+
+const SCROLL_DELTA_ERROR = "deltaY must be a finite number of pixels.";
+
+/**
+ * A wheel delta, checked before Playwright is asked.
+ *
+ * `typeof` alone accepts `Infinity` (valid JSON: `1e999` parses to it), which then travels into
+ * `mouse.wheel` and comes back as a 502 that reads as a broken computer. Anything that is not a
+ * finite number answers 400 here, matching the server gateway's validation of the same field.
+ * Absent stays absent: the caller gets the endpoint's own default distance.
+ */
+export function parseScrollDelta(input: unknown): ScrollDeltaParseResult {
+ if (input === undefined) return { ok: true, deltaY: undefined };
+ if (typeof input !== "number" || !Number.isFinite(input)) {
+ return { ok: false, error: SCROLL_DELTA_ERROR };
+ }
+ return { ok: true, deltaY: input };
+}
+
+export type InputMessageParseResult =
+ | { ok: true; message: InputMessage }
+ | { ok: false; error: string };
+
const EXEC_TIMEOUT_ERROR =
"timeoutMs must be a whole number of milliseconds between 1000 and 600000.";
@@ -67,3 +94,76 @@ export function parseExecTimeout(input: unknown): ExecTimeoutParseResult {
}
return { ok: true, timeoutMs: input };
}
+
+function isFiniteNumber(value: unknown): value is number {
+ return typeof value === "number" && Number.isFinite(value);
+}
+
+/**
+ * A live-screen input message, checked before it reaches CDP.
+ *
+ * Anything JSON-decodable used to pass the cast and fall through to `Input.insertText` with an
+ * undefined text, or forward wrong-typed coordinates into `Input.dispatchMouseEvent`. An unknown
+ * type and every malformed field is refused here so a buggy surface gets an error naming the
+ * message, not a CDP failure that reads as a broken computer.
+ */
+export function parseInputMessage(input: unknown): InputMessageParseResult {
+ if (typeof input !== "object" || input === null) {
+ return { ok: false, error: "That input message is not an object." };
+ }
+ const body = input as Record<string, unknown>;
+ if (body.type === "mouse") {
+ if (
+ body.event !== "pressed" &&
+ body.event !== "released" &&
+ body.event !== "moved"
+ ) {
+ return { ok: false, error: "A mouse message needs an event." };
+ }
+ if (!isFiniteNumber(body.x) || !isFiniteNumber(body.y)) {
+ return { ok: false, error: "A mouse message needs an x and a y." };
+ }
+ if (
+ body.button !== undefined &&
+ body.button !== "left" &&
+ body.button !== "right" &&
+ body.button !== "middle"
+ ) {
+ return { ok: false, error: "A mouse button must be named." };
+ }
+ return { ok: true, message: body as unknown as InputMessage };
+ }
+ if (body.type === "wheel") {
+ if (
+ !isFiniteNumber(body.x) ||
+ !isFiniteNumber(body.y) ||
+ !isFiniteNumber(body.deltaX) ||
+ !isFiniteNumber(body.deltaY)
+ ) {
+ return {
+ ok: false,
+ error: "A wheel message needs a position and a delta.",
+ };
+ }
+ return { ok: true, message: body as unknown as InputMessage };
+ }
+ if (body.type === "key") {
+ if (body.event !== "down" && body.event !== "up") {
+ return { ok: false, error: "A key message needs an event." };
+ }
+ if (typeof body.key !== "string" || !body.key) {
+ return { ok: false, error: "A key message needs a key name." };
+ }
+ if (typeof body.code !== "string" || !body.code) {
+ return { ok: false, error: "A key message needs a code." };
+ }
+ return { ok: true, message: body as unknown as InputMessage };
+ }
+ if (body.type === "text") {
+ if (typeof body.text !== "string" || !body.text) {
+ return { ok: false, error: "A text message needs text." };
+ }
+ return { ok: true, message: body as unknown as InputMessage };
+ }
+ return { ok: false, error: "That input message has an unknown type." };
+}
diff --git a/agent-computer/tests/request-validation.test.ts b/agent-computer/tests/request-validation.test.ts
index 839556b0..269854e7 100644
--- a/agent-computer/tests/request-validation.test.ts
+++ b/agent-computer/tests/request-validation.test.ts
@@ -1,5 +1,10 @@
import { describe, expect, test } from "bun:test";
-import { parseExecTimeout, parseNavigateUrl } from "../src/request-validation";
+import {
+ parseExecTimeout,
+ parseInputMessage,
+ parseNavigateUrl,
+ parseScrollDelta,
+} from "../src/request-validation";
/**
* The shape of a call, checked before it reaches the browser or the shell.
@@ -72,3 +77,73 @@ describe("a shell timeout", () => {
});
});
});
+
+describe("a wheel delta", () => {
+ test("absent stays absent: the endpoint default applies", () => {
+ expect(parseScrollDelta(undefined)).toEqual({
+ ok: true,
+ deltaY: undefined,
+ });
+ });
+
+ test("a finite delta passes through", () => {
+ expect(parseScrollDelta(400)).toEqual({ ok: true, deltaY: 400 });
+ expect(parseScrollDelta(-300)).toEqual({ ok: true, deltaY: -300 });
+ expect(parseScrollDelta(0)).toEqual({ ok: true, deltaY: 0 });
+ });
+
+ test.each([
+ ["NaN", Number.NaN],
+ ["Infinity", Number.POSITIVE_INFINITY],
+ ["negative Infinity", Number.NEGATIVE_INFINITY],
+ ["a string", "400"],
+ ["null", null],
+ ["a boolean", true],
+ ["an object", { deltaY: 400 }],
+ ])("rejects %s with a 400 message", (_name, input) => {
+ expect(parseScrollDelta(input)).toEqual({
+ ok: false,
+ error: "deltaY must be a finite number of pixels.",
+ });
+ });
+});
+
+describe("a live-screen input message", () => {
+ test("a well-formed mouse message passes through", () => {
+ expect(
+ parseInputMessage({ type: "mouse", event: "moved", x: 10, y: 20 }),
+ ).toEqual({
+ ok: true,
+ message: { type: "mouse", event: "moved", x: 10, y: 20 },
+ });
+ });
+
+ test.each([
+ ["unknown type", { type: "bogus" }],
+ ["missing type", {}],
+ ["null", null],
+ ["a string", "mouse"],
+ ["mouse without event", { type: "mouse", x: 1, y: 2 }],
+ [
+ "mouse with string coords",
+ { type: "mouse", event: "moved", x: "1", y: 2 },
+ ],
+ [
+ "mouse with infinite x",
+ {
+ type: "mouse",
+ event: "moved",
+ // `1e999` is valid JSON and parses to `Infinity`, which is what the socket carries.
+ x: JSON.parse("1e999"),
+ y: 2,
+ },
+ ],
+ ["wheel without delta", { type: "wheel", x: 1, y: 2, deltaX: 0 }],
+ ["key without code", { type: "key", event: "down", key: "a" }],
+ ["text without text", { type: "text" }],
+ ])("rejects %s with a 400 message", (_name, input) => {
+ const parsed = parseInputMessage(input);
+ expect(parsed.ok).toBe(false);
+ expect((parsed as { error: string }).error).toBeString();
+ });
+});