Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Validate scroll deltas and live-screen input on the Bot computer (#488)
Felix-Ayushe197209fdd0c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e809d6ae..1f29a867 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,12 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.## Unreleased+### The Bot computer refuses a malformed scroll or live input before the browser sees it++A non-finite wheel delta travelled into Playwright and came back as a 502 that read as a broken+computer, and any JSON object on the live-screen socket fell through to `Input.insertText` or+forwarded wrong-typed coordinates to CDP. Scroll deltas must be finite numbers now, and live+input must match its mouse, wheel, key, or text shape; anything else is a 400 naming the field.### A non-string plugin grant or tool call is refused before it reaches the store`POST /api/plugins/grants` and `POST /api/plugins/call` checked presence, not shape, so a JSONdiff --git a/agent-computer/src/index.ts b/agent-computer/src/index.tsindex 2468c3fd..16868122 100644--- a/agent-computer/src/index.ts+++ b/agent-computer/src/index.ts@@ -19,7 +19,12 @@ import {} from "./control";import { identity } from "./identity";import { createProfiles, numberFromEnv, VIEWPORT } from "./profiles";-import { parseExecTimeout, parseNavigateUrl } from "./request-validation";+import {+ parseExecTimeout,+ parseInputMessage,+ parseNavigateUrl,+ parseScrollDelta,+} from "./request-validation";import { type InputMessage, startScreencast } from "./screencast";import { createShell } from "./shell";import { createViewerSlot, type ViewerSlot } from "./viewer";@@ -498,7 +503,13 @@ serve<StreamData>({}let message: InputMessage;try {- message = JSON.parse(String(raw)) as InputMessage;+ const parsed: unknown = JSON.parse(String(raw));+ const validated = parseInputMessage(parsed);+ if (!validated.ok) {+ ws.send(JSON.stringify({ type: "error", error: validated.error }));+ return;+ }+ message = validated.message;} catch {return;}@@ -744,6 +755,12 @@ serve<StreamData>({string,unknown> | null;+ if (url.pathname === "/human/scroll") {+ const parsed = parseScrollDelta(body?.deltaY);+ if (!parsed.ok) {+ return json({ error: parsed.error }, 400);+ }+ }try {const target = await currentPage(botId);return json(await performHumanInput(target, url.pathname, body ?? {}));@@ -1008,6 +1025,12 @@ serve<StreamData>({if (!body) {return json({ error: "An action needs a JSON body." }, 400);}+ if (url.pathname === "/scroll") {+ const parsed = parseScrollDelta(body.deltaY);+ if (!parsed.ok) {+ return json({ error: parsed.error }, 400);+ }+ }const startedAt = Date.now();try {@@ -1136,7 +1159,11 @@ async function performHumanInput(return { action: "human_key", key: body.key, url: target.url() };}- const deltaY = typeof body.deltaY === "number" ? body.deltaY : 400;+ const parsed = parseScrollDelta(body.deltaY);+ if (!parsed.ok) {+ throw new Error(parsed.error);+ }+ const deltaY = parsed.deltaY ?? 400;await target.mouse.wheel(0, deltaY);return { action: "human_scroll", deltaY, url: target.url() };}@@ -1214,7 +1241,11 @@ async function performAction(// Scroll. A plain wheel event on the page, which is what moves a long form, rather than scrolling a// specific element into view: the Bot asked to see further down, not to hunt for one control.- const deltaY = typeof body.deltaY === "number" ? body.deltaY : 600;+ const parsed = parseScrollDelta(body.deltaY);+ if (!parsed.ok) {+ throw new Error(parsed.error);+ }+ const deltaY = parsed.deltaY ?? 600;await target.mouse.wheel(0, deltaY);return { action: "scroll", deltaY, url: target.url() };}diff --git a/agent-computer/src/request-validation.ts b/agent-computer/src/request-validation.tsindex 02379a96..99dcccc0 100644--- a/agent-computer/src/request-validation.ts+++ b/agent-computer/src/request-validation.ts@@ -5,6 +5,7 @@* is checked here is whether the request is well-formed at all, so a typo'd caller gets a 400* naming the field rather than a 502 that reads as a broken computer.*/+import type { InputMessage } from "./screencast";/** Long enough for an install, short enough that a hung command is not a hung Bot. */export const EXEC_DEFAULT_TIMEOUT_MS = 120_000;@@ -44,6 +45,32 @@ export type ExecTimeoutParseResult =| { ok: true; timeoutMs: number | undefined }| { ok: false; error: string };+export type ScrollDeltaParseResult =+ | { ok: true; deltaY: number | undefined }+ | { ok: false; error: string };++const SCROLL_DELTA_ERROR = "deltaY must be a finite number of pixels.";++/**+ * A wheel delta, checked before Playwright is asked.+ *+ * `typeof` alone accepts `Infinity` (valid JSON: `1e999` parses to it), which then travels into+ * `mouse.wheel` and comes back as a 502 that reads as a broken computer. Anything that is not a+ * finite number answers 400 here, matching the server gateway's validation of the same field.+ * Absent stays absent: the caller gets the endpoint's own default distance.+ */+export function parseScrollDelta(input: unknown): ScrollDeltaParseResult {+ if (input === undefined) return { ok: true, deltaY: undefined };+ if (typeof input !== "number" || !Number.isFinite(input)) {+ return { ok: false, error: SCROLL_DELTA_ERROR };+ }+ return { ok: true, deltaY: input };+}++export type InputMessageParseResult =+ | { ok: true; message: InputMessage }+ | { ok: false; error: string };+const EXEC_TIMEOUT_ERROR ="timeoutMs must be a whole number of milliseconds between 1000 and 600000.";@@ -67,3 +94,76 @@ export function parseExecTimeout(input: unknown): ExecTimeoutParseResult {}return { ok: true, timeoutMs: input };}++function isFiniteNumber(value: unknown): value is number {+ return typeof value === "number" && Number.isFinite(value);+}++/**+ * A live-screen input message, checked before it reaches CDP.+ *+ * Anything JSON-decodable used to pass the cast and fall through to `Input.insertText` with an+ * undefined text, or forward wrong-typed coordinates into `Input.dispatchMouseEvent`. An unknown+ * type and every malformed field is refused here so a buggy surface gets an error naming the+ * message, not a CDP failure that reads as a broken computer.+ */+export function parseInputMessage(input: unknown): InputMessageParseResult {+ if (typeof input !== "object" || input === null) {+ return { ok: false, error: "That input message is not an object." };+ }+ const body = input as Record<string, unknown>;+ if (body.type === "mouse") {+ if (+ body.event !== "pressed" &&+ body.event !== "released" &&+ body.event !== "moved"+ ) {+ return { ok: false, error: "A mouse message needs an event." };+ }+ if (!isFiniteNumber(body.x) || !isFiniteNumber(body.y)) {+ return { ok: false, error: "A mouse message needs an x and a y." };+ }+ if (+ body.button !== undefined &&+ body.button !== "left" &&+ body.button !== "right" &&+ body.button !== "middle"+ ) {+ return { ok: false, error: "A mouse button must be named." };+ }+ return { ok: true, message: body as unknown as InputMessage };+ }+ if (body.type === "wheel") {+ if (+ !isFiniteNumber(body.x) ||+ !isFiniteNumber(body.y) ||+ !isFiniteNumber(body.deltaX) ||+ !isFiniteNumber(body.deltaY)+ ) {+ return {+ ok: false,+ error: "A wheel message needs a position and a delta.",+ };+ }+ return { ok: true, message: body as unknown as InputMessage };+ }+ if (body.type === "key") {+ if (body.event !== "down" && body.event !== "up") {+ return { ok: false, error: "A key message needs an event." };+ }+ if (typeof body.key !== "string" || !body.key) {+ return { ok: false, error: "A key message needs a key name." };+ }+ if (typeof body.code !== "string" || !body.code) {+ return { ok: false, error: "A key message needs a code." };+ }+ return { ok: true, message: body as unknown as InputMessage };+ }+ if (body.type === "text") {+ if (typeof body.text !== "string" || !body.text) {+ return { ok: false, error: "A text message needs text." };+ }+ return { ok: true, message: body as unknown as InputMessage };+ }+ return { ok: false, error: "That input message has an unknown type." };+}diff --git a/agent-computer/tests/request-validation.test.ts b/agent-computer/tests/request-validation.test.tsindex 839556b0..269854e7 100644--- a/agent-computer/tests/request-validation.test.ts+++ b/agent-computer/tests/request-validation.test.ts@@ -1,5 +1,10 @@import { describe, expect, test } from "bun:test";-import { parseExecTimeout, parseNavigateUrl } from "../src/request-validation";+import {+ parseExecTimeout,+ parseInputMessage,+ parseNavigateUrl,+ parseScrollDelta,+} from "../src/request-validation";/*** The shape of a call, checked before it reaches the browser or the shell.@@ -72,3 +77,73 @@ describe("a shell timeout", () => {});});});++describe("a wheel delta", () => {+ test("absent stays absent: the endpoint default applies", () => {+ expect(parseScrollDelta(undefined)).toEqual({+ ok: true,+ deltaY: undefined,+ });+ });++ test("a finite delta passes through", () => {+ expect(parseScrollDelta(400)).toEqual({ ok: true, deltaY: 400 });+ expect(parseScrollDelta(-300)).toEqual({ ok: true, deltaY: -300 });+ expect(parseScrollDelta(0)).toEqual({ ok: true, deltaY: 0 });+ });++ test.each([+ ["NaN", Number.NaN],+ ["Infinity", Number.POSITIVE_INFINITY],+ ["negative Infinity", Number.NEGATIVE_INFINITY],+ ["a string", "400"],+ ["null", null],+ ["a boolean", true],+ ["an object", { deltaY: 400 }],+ ])("rejects %s with a 400 message", (_name, input) => {+ expect(parseScrollDelta(input)).toEqual({+ ok: false,+ error: "deltaY must be a finite number of pixels.",+ });+ });+});++describe("a live-screen input message", () => {+ test("a well-formed mouse message passes through", () => {+ expect(+ parseInputMessage({ type: "mouse", event: "moved", x: 10, y: 20 }),+ ).toEqual({+ ok: true,+ message: { type: "mouse", event: "moved", x: 10, y: 20 },+ });+ });++ test.each([+ ["unknown type", { type: "bogus" }],+ ["missing type", {}],+ ["null", null],+ ["a string", "mouse"],+ ["mouse without event", { type: "mouse", x: 1, y: 2 }],+ [+ "mouse with string coords",+ { type: "mouse", event: "moved", x: "1", y: 2 },+ ],+ [+ "mouse with infinite x",+ {+ type: "mouse",+ event: "moved",+ // `1e999` is valid JSON and parses to `Infinity`, which is what the socket carries.+ x: JSON.parse("1e999"),+ y: 2,+ },+ ],+ ["wheel without delta", { type: "wheel", x: 1, y: 2, deltaX: 0 }],+ ["key without code", { type: "key", event: "down", key: "a" }],+ ["text without text", { type: "text" }],+ ])("rejects %s with a 400 message", (_name, input) => {+ const parsed = parseInputMessage(input);+ expect(parsed.ok).toBe(false);+ expect((parsed as { error: string }).error).toBeString();+ });+});