Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Tell a Bot a person has the wheel, not that its refs are stale (#164)
beardthelion71fe47a145ff
diff --git a/app/src/lib/copilot/computer-tools.tsx b/app/src/lib/copilot/computer-tools.tsxindex c4d93726..c4087b49 100644--- a/app/src/lib/copilot/computer-tools.tsx+++ b/app/src/lib/copilot/computer-tools.tsx@@ -42,7 +42,14 @@ async function waitForPerson(return "gave up";}-async function callComputer(+/**+ * Exported for the test that covers what a Bot is told when a call is refused.+ *+ * The distinctions this draws from a status and a body decide the model's next step, and they are+ * drawn nowhere else, so they are worth pinning without standing up the tool registrations and the+ * runtime around them.+ */+export async function callComputer(botId: string,path: string,/*diff --git a/app/tests/computer-tool-refusals.test.ts b/app/tests/computer-tool-refusals.test.tsnew file mode 100644index 00000000..8404f7fb--- /dev/null+++ b/app/tests/computer-tool-refusals.test.ts@@ -0,0 +1,68 @@+import { afterEach, describe, expect, test } from "bun:test";+import { callComputer } from "../src/lib/copilot/computer-tools";++/**+ * What a Bot is told when its action did not happen.+ *+ * Every refusal reaches the model as this object, and the fields decide what it does next: `staleRefs`+ * is the one its own tool description turns into "the page changed, call this again with the new+ * refs", so labelling a takeover with it sends the Bot back round the same action against the person+ * who just took the browser. The server carries `humanHasControl` for exactly that reason; this is+ * the end of that wire, and the only place the two conditions are told apart for the model.+ */++const realFetch = globalThis.fetch;++afterEach(() => {+ globalThis.fetch = realFetch;+});++function serverAnswering(status: number, body: unknown) {+ globalThis.fetch = (async () =>+ new Response(body === undefined ? null : JSON.stringify(body), {+ status,+ headers: { "content-type": "application/json" },+ })) as unknown as typeof fetch;+}++describe("a computer call the server refused", () => {+ test("a person holding the wheel is reported as that, not as stale refs", async () => {+ serverAnswering(409, {+ error: "A person has taken control of this computer.",+ humanHasControl: true,+ });++ const outcome = await callComputer("bot-1", "/click", { method: "POST" });++ expect(outcome.ok).toBe(false);+ expect(outcome.humanHasControl).toBe(true);+ // The instruction that must not be attached: it would send the Bot round again.+ expect(outcome.staleRefs).toBeUndefined();+ expect(outcome.reason).toBe("A person has taken control of this computer.");+ });++ test("a stale snapshot is still reported as stale refs", async () => {+ // The half that must not move. Losing this would park a Bot with genuinely stale refs waiting for+ // a person who is not coming.+ serverAnswering(409, { error: "Snapshot 3 is not the current one." });++ const outcome = await callComputer("bot-1", "/click", { method: "POST" });++ expect(outcome.ok).toBe(false);+ expect(outcome.staleRefs).toBe(true);+ expect(outcome.humanHasControl).toBeUndefined();+ });++ test("a policy refusal is neither", async () => {+ serverAnswering(403, {+ error: "That is not allowed here.",+ rule: 'url.host == "example.com"',+ });++ const outcome = await callComputer("bot-1", "/click", { method: "POST" });++ expect(outcome.refused).toBe(true);+ expect(outcome.staleRefs).toBeUndefined();+ expect(outcome.humanHasControl).toBeUndefined();+ });+});diff --git a/server/src/computer/client.ts b/server/src/computer/client.tsindex d084cfe7..1b1a12a3 100644--- a/server/src/computer/client.ts+++ b/server/src/computer/client.ts@@ -49,6 +49,21 @@ export class StaleSnapshotError extends Error {}}+/**+ * A person has the wheel, so the Bot's action was not carried out.+ *+ * Its own condition rather than a stale snapshot, though both arrive as 409, because what the caller+ * should do next is the opposite in each case. Stale refs mean take a fresh snapshot and go again;+ * this means stop and leave the browser to the person holding it. Told apart by the flag the computer+ * puts on the body, which is the only thing in the response that distinguishes them.+ */+export class HumanHasControlError extends Error {+ constructor(reason: string) {+ super(reason);+ this.name = "HumanHasControlError";+ }+}+/*** Transport options used inside the computer gateway.*@@ -206,6 +221,10 @@ function throwMappedError(const detail =typeof body?.error === "string" ? body.error : `HTTP ${status}`;if (status === 409) {+ // The computer says which kind of 409 this is. Absent, it is the ordinary one.+ if (body?.humanHasControl === true) {+ throw new HumanHasControlError(detail);+ }throw new StaleSnapshotError(detail);}if (status === 403) {diff --git a/server/src/computer/gateway.ts b/server/src/computer/gateway.tsindex 7d5fdac6..672842b3 100644--- a/server/src/computer/gateway.ts+++ b/server/src/computer/gateway.ts@@ -24,6 +24,7 @@ import { checkComputerAddress } from "./target";export {ComputerUnavailableError,ElementNotFoundError,+ HumanHasControlError,NavigationRefusedError,StaleSnapshotError,WorkspaceRefusedError,diff --git a/server/src/computer/routes.ts b/server/src/computer/routes.tsindex b2be2c51..e8a88be6 100644--- a/server/src/computer/routes.ts+++ b/server/src/computer/routes.ts@@ -10,6 +10,7 @@ import {ComputerUnavailableError,ElementNotFoundError,NavigationRefusedError,+ HumanHasControlError,StaleSnapshotError,WorkspaceRefusedError,WorkspaceRequestError,@@ -86,7 +87,7 @@ export function createComputerRoutes(try {return context.json(await gateway.screenshot(context.req.param("botId")));} catch (error) {- return context.json({ error: describe(error) }, statusFor(error));+ return context.json(errorBody(error), statusFor(error));}});@@ -94,7 +95,7 @@ export function createComputerRoutes(try {return context.json(await gateway.read(context.req.param("botId")));} catch (error) {- return context.json({ error: describe(error) }, statusFor(error));+ return context.json(errorBody(error), statusFor(error));}});@@ -129,7 +130,7 @@ export function createComputerRoutes(if (error instanceof NavigationRefusedError) {return context.json({ error: error.message }, 403);}- return context.json({ error: describe(error) }, statusFor(error));+ return context.json(errorBody(error), statusFor(error));}});@@ -137,7 +138,7 @@ export function createComputerRoutes(try {return context.json(await gateway.snapshot(context.req.param("botId")));} catch (error) {- return context.json({ error: describe(error) }, statusFor(error));+ return context.json(errorBody(error), statusFor(error));}});@@ -209,7 +210,7 @@ export function createComputerRoutes(try {return context.json(await gateway.control(context.req.param("botId")));} catch (error) {- return context.json({ error: describe(error) }, statusFor(error));+ return context.json(errorBody(error), statusFor(error));}});@@ -248,7 +249,7 @@ export function createComputerRoutes(try {return context.json(await gateway.computers());} catch (error) {- return context.json({ error: describe(error) }, statusFor(error));+ return context.json(errorBody(error), statusFor(error));}});@@ -262,7 +263,7 @@ export function createComputerRoutes(try {return context.json(await gateway.computers());} catch (error) {- return context.json({ error: describe(error) }, statusFor(error));+ return context.json(errorBody(error), statusFor(error));}});@@ -355,7 +356,7 @@ export function createComputerRoutes(} as Parameters<typeof gateway.humanInput>[1]),);} catch (error) {- return context.json({ error: describe(error) }, statusFor(error));+ return context.json(errorBody(error), statusFor(error));}});@@ -548,7 +549,7 @@ async function act(if (error instanceof WorkspaceRequestError) {return context.json({ error: error.message }, 400);}- return context.json({ error: describe(error) }, statusFor(error));+ return context.json(errorBody(error), statusFor(error));}}@@ -588,8 +589,27 @@ function describe(error: unknown): string {* not running (an operator fixes it), the refs are stale (the model fixes it by snapshotting again),* and everything else. Navigation established this; the acting routes follow it.*/+/**+ * What a failed computer call looks like to the caller.+ *+ * One place, because the flag below decides what a Bot does next and a route that renders its own+ * error would silently not carry it. The computer marks a takeover on both the acting path and+ * navigate, and those two are answered by different code here, so the version of this that lived in+ * the acting helper alone fixed one of them.+ */+function errorBody(error: unknown): Record<string, unknown> {+ return {+ error: describe(error),+ // Not "the refs are stale, take another snapshot", which is what the surface says without it.+ ...(error instanceof HumanHasControlError ? { humanHasControl: true } : {}),+ };+}+function statusFor(error: unknown): 409 | 500 | 503 {if (error instanceof StaleSnapshotError) return 409;+ // Same status as a stale snapshot and for the same reason: nothing is broken, the caller has to do+ // something else first. What differs is what that something is, which the body carries.+ if (error instanceof HumanHasControlError) return 409;// The same answer as a stale snapshot, because it is the same instruction: the refs are wrong, take// another snapshot. Not 503, which says the computer is unavailable and sends an operator hunting a// container that is running perfectly.diff --git a/server/tests/computer-client.test.ts b/server/tests/computer-client.test.tsindex 66dd9d2e..f7c0f0eb 100644--- a/server/tests/computer-client.test.ts+++ b/server/tests/computer-client.test.ts@@ -2,7 +2,9 @@ import { describe, expect, test } from "bun:test";import {createComputerTransport,ElementNotFoundError,+ HumanHasControlError,NavigationRefusedError,+ StaleSnapshotError,} from "../src/computer/client";function clientWith(@@ -191,6 +193,88 @@ describe("acting on an element that is not there", () => {});/**+ * A refusal because a person took the wheel, told apart from stale refs.+ *+ * Both arrive as 409, and the computer says which is which by putting `humanHasControl` on the body+ * (agent-computer/src/index.ts). Mapping every 409 to the same condition throws that away, and the+ * surface then tells the model its refs are stale and to take a fresh snapshot: advice that sends a+ * Bot round the loop again against a person who has deliberately taken the browser. The status is+ * the same either way; what differs is what the caller should do next, which is the only thing the+ * distinction is for.+ */+describe("a 409 from the computer", () => {+ const refusing = (body: unknown) =>+ clientWith(+ () =>+ new Response(JSON.stringify(body), {+ status: 409,+ headers: { "content-type": "application/json" },+ }),+ );++ test("is a person holding the wheel when the computer says so", async () => {+ expect(+ refusing({+ error: "A person has taken control of this computer.",+ humanHasControl: true,+ }).click({ ref: "e1", snapshotId: 1 }),+ ).rejects.toBeInstanceOf(HumanHasControlError);+ });++ test("is a stale snapshot when it does not", async () => {+ // The permissive half, and the one that must not change: an ordinary stale-ref 409 carries no+ // flag and keeps its own condition, so the refs-are-stale instruction still reaches the model.+ expect(+ refusing({ error: "Snapshot 3 is not the current one." }).click({+ ref: "e1",+ snapshotId: 1,+ }),+ ).rejects.toBeInstanceOf(StaleSnapshotError);+ });++ test("is a stale snapshot for every shape that is not the flag", async () => {+ // The flag decides an instruction a Bot acts on, so only the computer's own `true` counts. A+ // string, a false, or a 409 with nothing on it at all stays the ordinary condition rather than+ // parking a Bot to wait for a person who is not there.+ for (const body of [+ { error: "no flag" },+ { error: "explicitly not a takeover", humanHasControl: false },+ { error: "a string is not a boolean", humanHasControl: "true" },+ {},+ ]) {+ expect(+ refusing(body).click({ ref: "e1", snapshotId: 1 }),+ ).rejects.toBeInstanceOf(StaleSnapshotError);+ }+ });++ test("a 409 with no body at all is still a refusal, not a crash", async () => {+ const bodyless = clientWith(() => new Response(null, { status: 409 }));+ expect(bodyless.click({ ref: "e1", snapshotId: 1 })).rejects.toBeInstanceOf(+ StaleSnapshotError,+ );+ });++ test("carries the reason the computer gave, either way", async () => {+ for (const body of [+ {+ error: "A person has taken control of this computer.",+ humanHasControl: true,+ },+ { error: "Snapshot 3 is not the current one." },+ ]) {+ try {+ await refusing(body).click({ ref: "e1", snapshotId: 1 });+ throw new Error("should have refused");+ } catch (error) {+ expect((error as Error).message).toBe(body.error);+ }+ }+ });+});++/**+ * Stop has to travel./*** Stop has to travel.** Pressing Stop aborts the surface's request. That abort is only useful if it reaches the browser: adiff --git a/server/tests/control-refusal-end-to-end.test.ts b/server/tests/control-refusal-end-to-end.test.tsnew file mode 100644index 00000000..0ec79f11--- /dev/null+++ b/server/tests/control-refusal-end-to-end.test.ts@@ -0,0 +1,174 @@+import { afterEach, describe, expect, test } from "bun:test";+import type { MiddlewareHandler } from "hono";+import type { AuditEventInput, AuditStore } from "../src/audit";+import type { AppVariables, AuthenticatedActor } from "../src/auth/guards";+import { createComputerGateway } from "../src/computer/gateway";+import type { PolicyStore } from "../src/computer/policy-store";+import type { ComputerProvider } from "../src/computer/provider";+import { createComputerRoutes } from "../src/computer/routes";++/**+ * What a Bot is told when a person has taken the wheel, over a real socket.+ *+ * The computer answers a refused action with `409 { error, humanHasControl: true }`, and the surface+ * reads that flag to decide what the model hears next: with it, a person has control; without it,+ * `staleRefs`, which renders as "your refs are stale, the page changed, call this again with the new+ * ones". So a flag lost in the middle is not a cosmetic loss. It sends a Bot back round the same+ * action against somebody who deliberately took the browser, which is the one moment it should stop.+ *+ * Driven over a socket rather than a stubbed fetch because the property is that a field survives+ * being serialised, thrown as an error, and re-serialised by the route. A stub proves the shape this+ * process built; only a listener proves what the far side actually sent and what the caller finally+ * reads.+ */++const servers: Array<{ stop: (force?: boolean) => void }> = [];++afterEach(() => {+ for (const server of servers.splice(0)) server.stop(true);+});++/** The computer, refusing exactly the way `agent-computer/src/index.ts` does. */+function computerAnswering(body: unknown, status: number) {+ const server = Bun.serve({+ port: 0,+ fetch: async () => Response.json(body, { status }),+ });+ servers.push(server);+ return `http://127.0.0.1:${server.port}`;+}++function routesFor(baseUrl: string) {+ const rows: AuditEventInput[] = [];+ const provider: ComputerProvider = {+ name: "test",+ isolation: "per-bot",+ locate: async () => baseUrl,+ status: async (botId) => ({ botId, state: "ready" }),+ stop: async () => ({ wasRunning: true }),+ reset: async () => ({ cleared: true }),+ list: async () => [],+ };+ const gateway = createComputerGateway({+ provider,+ auditStore: {+ insert: async (event: AuditEventInput) => {+ rows.push(event);+ },+ } as unknown as AuditStore,+ policy: () => ({ mode: "enforce", deny: [], allow: ["true"] }),+ token: "computer-token-for-this-test",+ });+ const actor: AuthenticatedActor = {+ id: "user-1",+ email: "[email protected]",+ role: "user",+ };+ const asActor: MiddlewareHandler<{ Variables: AppVariables }> = async (+ context,+ next,+ ) => {+ context.set("actor", actor);+ await next();+ };+ return {+ rows,+ app: createComputerRoutes(+ gateway,+ {} as PolicyStore,+ asActor,+ async () => true,+ ),+ };+}++async function navigate(baseUrl: string) {+ const { app } = routesFor(baseUrl);+ const response = await app.request("http://openbot.test/bot-1/navigate", {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({ url: "https://example.com/" }),+ });+ return {+ status: response.status,+ body: (await response.json()) as Record<string, unknown>,+ };+}++async function click(baseUrl: string) {+ const { app, rows } = routesFor(baseUrl);+ const response = await app.request("http://openbot.test/bot-1/click", {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({ ref: "e1", snapshotId: 1 }),+ });+ return {+ status: response.status,+ body: (await response.json()) as Record<string, unknown>,+ rows,+ };+}++describe("a refusal because a person has the wheel", () => {+ test("reaches the caller as one, not as stale refs", async () => {+ const { status, body } = await click(+ computerAnswering(+ {+ error: "A person has taken control of this computer.",+ humanHasControl: true,+ },+ 409,+ ),+ );++ expect(status).toBe(409);+ expect(body.humanHasControl).toBe(true);+ expect(body.error).toBe("A person has taken control of this computer.");+ });++ test("a stale snapshot is still a stale snapshot", async () => {+ // The half that must not move. Without this, marking every 409 as a takeover would pass the test+ // above while telling a Bot with genuinely stale refs to go and wait for a person who is not there.+ const { status, body } = await click(+ computerAnswering({ error: "Snapshot 3 is not the current one." }, 409),+ );++ expect(status).toBe(409);+ expect(body.humanHasControl).toBeUndefined();+ expect(body.error).toBe("Snapshot 3 is not the current one.");+ });++ test("reaches the caller from navigate too, which is the other place the computer refuses", async () => {+ // The computer marks the takeover on its action path and on navigate (agent-computer/src/index.ts+ // 938 and 721). Navigate does not go through the shared acting helper, so carrying the flag in+ // that helper alone would fix one of the two and leave the other telling a Bot to re-snapshot.+ const { status, body } = await navigate(+ computerAnswering(+ {+ error: "A person has taken control of this computer.",+ humanHasControl: true,+ },+ 409,+ ),+ );++ expect(status).toBe(409);+ expect(body.humanHasControl).toBe(true);+ });++ test("the refusal is still audited as a refusal", async () => {+ // The gateway writes a row when a forwarded action fails. Telling the caller something new must+ // not quietly cost the trail the record that the action did not happen.+ const { rows } = await click(+ computerAnswering(+ {+ error: "A person has taken control of this computer.",+ humanHasControl: true,+ },+ 409,+ ),+ );++ expect(rows.length).toBeGreaterThan(0);+ });+});