Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

Shape human click, type and key payloads on POST /api/computers/:botId/human/:kind with 400 (#517)

Felix-Ayushe7bbe7755ae2
diff --git a/server/src/computer/routes.ts b/server/src/computer/routes.ts
index 2f0a3fd9..a7ad2a4c 100644
--- a/server/src/computer/routes.ts
+++ b/server/src/computer/routes.ts
@@ -485,6 +485,41 @@ export function createComputerRoutes(
string,
unknown
> | null;
+ /*
+ * Shaped per gesture, like the Bot's acting routes shape theirs. Only scroll was checked:
+ * a click with `{"x": "ten"}`, a type with `{"text": 123}` or a key with `{}` travelled
+ * to the computer untouched, and the failure surfaced as whatever the computer returned
+ * for garbage — mapped here to a 500, or a 200 no-op. The shapes below are the ones the
+ * gateway's `HumanInput` type already promises the computer: coordinates in viewport
+ * pixels, text to enter, a key name. Scroll keeps its existing check, which allows an
+ * absent delta the computer reads as its own default distance.
+ */
+ if (kind === "click") {
+ if (
+ typeof body?.x !== "number" ||
+ !Number.isFinite(body.x) ||
+ typeof body?.y !== "number" ||
+ !Number.isFinite(body.y)
+ ) {
+ return context.json(
+ { error: "A click needs numeric x and y coordinates." },
+ 400,
+ );
+ }
+ }
+ if (kind === "type") {
+ if (typeof body?.text !== "string") {
+ return context.json({ error: "The text to enter is required." }, 400);
+ }
+ }
+ if (kind === "key") {
+ if (typeof body?.key !== "string" || !body.key) {
+ return context.json(
+ { error: "A key name is required, such as Enter or Tab." },
+ 400,
+ );
+ }
+ }
if (kind === "scroll" && !usableDeltaY(body?.deltaY)) {
return context.json(badDeltaY, 400);
}
diff --git a/server/tests/computer-human-input.test.ts b/server/tests/computer-human-input.test.ts
new file mode 100644
index 00000000..c8cbdb87
--- /dev/null
+++ b/server/tests/computer-human-input.test.ts
@@ -0,0 +1,142 @@
+import { describe, expect, test } from "bun:test";
+import type { ComputerGateway } from "../src/computer/gateway";
+import type { PolicyStore } from "../src/computer/policy-store";
+import { createComputerRoutes } from "../src/computer/routes";
+import type { AppVariables } from "../src/auth/guards";
+import type { MiddlewareHandler } from "hono";
+
+/**
+ * A person's own mouse and keyboard, shaped before it travels.
+ *
+ * Only scroll was checked on this route. A click with `{"x": "ten"}`, a type with
+ * `{"text": 123}` or a key with `{}` travelled to the computer untouched, and the failure
+ * surfaced as whatever the computer returned for garbage — a 500 here, or a 200 no-op.
+ * The shapes below are the ones the gateway's `HumanInput` type already promises: `click`
+ * carries viewport-pixel coordinates, `type` carries text, `key` carries a key name.
+ */
+
+const member = {
+ id: "u1",
+ email: "[email protected]",
+ role: "user",
+} as const;
+
+function asActor(
+ actor: typeof member,
+): MiddlewareHandler<{ Variables: AppVariables }> {
+ return async (context, next) => {
+ context.set("actor", { ...actor });
+ await next();
+ };
+}
+
+function recordingGateway() {
+ const calls: Array<{ botId: string; input: Record<string, unknown> }> = [];
+ const gateway = {
+ humanInput: async (botId: string, input: Record<string, unknown>) => {
+ calls.push({ botId, input });
+ return { ok: true };
+ },
+ } as unknown as ComputerGateway;
+ return {
+ calls,
+ app: createComputerRoutes(
+ gateway,
+ {} as PolicyStore,
+ asActor(member),
+ async () => true,
+ ),
+ };
+}
+
+async function send(body: unknown, kind: string) {
+ const { app, calls } = recordingGateway();
+ const response = await app.request(
+ `http://openbot.test/bot-1/human/${kind}`,
+ {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify(body),
+ },
+ );
+ return { response, calls };
+}
+
+describe("human input payloads", () => {
+ test("a click with coordinates still reaches the computer", async () => {
+ const { response, calls } = await send({ x: 10, y: 20 }, "click");
+
+ expect(response.status).toBe(200);
+ expect(calls).toHaveLength(1);
+ expect(calls[0]?.input).toMatchObject({ kind: "click", x: 10, y: 20 });
+ });
+
+ test("a type with text still reaches the computer", async () => {
+ const { response, calls } = await send({ text: "hello" }, "type");
+
+ expect(response.status).toBe(200);
+ expect(calls).toHaveLength(1);
+ });
+
+ test("a key with a name still reaches the computer", async () => {
+ const { response, calls } = await send({ key: "Enter" }, "key");
+
+ expect(response.status).toBe(200);
+ expect(calls).toHaveLength(1);
+ });
+
+ test.each([
+ ["a string x", { x: "ten", y: 20 }],
+ ["a missing x", { y: 20 }],
+ ["a missing y", { x: 10 }],
+ ["NaN x", { x: Number.NaN, y: 20 }],
+ ["Infinity y", { x: 10, y: Number.POSITIVE_INFINITY }],
+ ["null", null],
+ ["an empty body", {}],
+ ])("refuses a click with %s and never forwards it", async (_name, body) => {
+ const { response, calls } = await send(body, "click");
+
+ expect(response.status).toBe(400);
+ await expect(response.json()).resolves.toEqual({
+ error: "A click needs numeric x and y coordinates.",
+ });
+ expect(calls).toHaveLength(0);
+ });
+
+ test.each([
+ ["a number", { text: 123 }],
+ ["null", { text: null }],
+ ["an object", { text: {} }],
+ ["a missing text", {}],
+ ])("refuses a type with %s and never forwards it", async (_name, body) => {
+ const { response, calls } = await send(body, "type");
+
+ expect(response.status).toBe(400);
+ await expect(response.json()).resolves.toEqual({
+ error: "The text to enter is required.",
+ });
+ expect(calls).toHaveLength(0);
+ });
+
+ test.each([
+ ["a number", { key: 123 }],
+ ["an empty name", { key: "" }],
+ ["a missing key", {}],
+ ["null", null],
+ ])("refuses a key with %s and never forwards it", async (_name, body) => {
+ const { response, calls } = await send(body, "key");
+
+ expect(response.status).toBe(400);
+ await expect(response.json()).resolves.toEqual({
+ error: "A key name is required, such as Enter or Tab.",
+ });
+ expect(calls).toHaveLength(0);
+ });
+
+ test("a scroll without a delta still travels, as before", async () => {
+ const { response, calls } = await send({}, "scroll");
+
+ expect(response.status).toBe(200);
+ expect(calls).toHaveLength(1);
+ });
+});
diff --git a/server/tests/computer-routes.test.ts b/server/tests/computer-routes.test.ts
index 36f82b4e..cae6ff40 100644
--- a/server/tests/computer-routes.test.ts
+++ b/server/tests/computer-routes.test.ts
@@ -198,7 +198,7 @@ describe("human input", () => {
test("a body naming its own kind cannot reach the shell either", async () => {
const { calls } = await send(
- { kind: "../exec", command: "cat /workspace/notes" },
+ { kind: "../exec", text: "cat /workspace/notes" },
"type",
);
diff --git a/server/tests/human-input-end-to-end.test.ts b/server/tests/human-input-end-to-end.test.ts
index 3e3bdc00..c11e96a0 100644
--- a/server/tests/human-input-end-to-end.test.ts
+++ b/server/tests/human-input-end-to-end.test.ts
@@ -147,7 +147,12 @@ describe("a person's input, end to end", () => {
test("the four gestures still reach their own endpoints", async () => {
for (const kind of ["click", "type", "key", "scroll"]) {
- const { received } = await drive(kind, { x: 1, y: 1, text: "hello" });
+ const { received } = await drive(kind, {
+ x: 1,
+ y: 1,
+ text: "hello",
+ key: "Enter",
+ });
expect(received.map((request) => request.path)).toEqual([
`/human/${kind}`,