Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Let a Bot choose from a dropdown, and carry on after a browser step fails (#833)
David McKay46642d6c91da
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 8c488b2f..301973e2 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -139,7 +139,7 @@ jobs:- run: >-bun test tests/sign-in.test.ts tests/session-cookies.test.ts tests/follows-popup.test.tstests/demonstration-live.test.ts tests/webrtc-proxy.test.ts tests/web-bot-auth-browser.test.ts- tests/handoff-page-restore.test.ts+ tests/handoff-page-restore.test.ts tests/enter-text.test.tsworking-directory: agent-computerenv:OPENBOT_COMPUTER_BROWSER: '1'diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 2cd51b8e..3f9bc54a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -35,6 +35,23 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.request for an administrator to switch on a work app, grant it to a Bot, or connect a Shared app'steam account is kept, with who answered it and when. It starts empty and nothing is backfilled.+### A Bot can choose from a dropdown++A Bot filling in a form in its browser could not set a dropdown (a native `<select>`, such as a CRM's+Industry or Stage). Typing into one waited until it timed out and was reported as a field that had+gone from the page, so the Bot took a fresh snapshot and tried the same thing again. Typing into a+dropdown now chooses the option it names, by the option's label or its value, and the page sees the+same change a person's choice makes. Text that names no option is refused with a message saying so.+A page snapshot now lists each dropdown's options and the one it shows, so the Bot can see what it+may choose and what is already chosen.++### A Bot carries on after a browser step fails++When a click, typed entry or key press the Bot made in its browser failed, the Bot's turn ended on+the spot: the step showed "Didn't work" and the Bot said nothing more. The failure now reaches the+Bot like any other result, so it can take a fresh snapshot, try another way, or tell you what+stopped it. A refused step reads as a refusal the Bot can explain.+### A Bot can send, draft and reply in GmailOpenBot left out every Composio action that takes a file, because a Bot has no way to supply one.diff --git a/agent-computer/src/aria-snapshot.ts b/agent-computer/src/aria-snapshot.tsindex 88db6ad7..037ba140 100644--- a/agent-computer/src/aria-snapshot.ts+++ b/agent-computer/src/aria-snapshot.ts@@ -73,8 +73,16 @@ export type SnapshotElement = {value?: string;disabled?: boolean;checked?: boolean;+ /**+ * A native dropdown's choices, by what each says. Its options carry no refs, so without this a Bot+ * saw `combobox "Industry"` and nothing it could choose; it chooses by sending one of these as text.+ */+ options?: string[];};+/** The most choices one dropdown reports, so a list of every country cannot crowd out the form. */+const OPTION_LIMIT = 50;+/** The descriptor half of an entry: everything before the colon. */type Descriptor = {role: string;@@ -199,6 +207,17 @@ function toElement(if (descriptor.flags.has("disabled")) element.disabled = true;+ // A native `<select>`: its value in the YAML is the list of its options, not text. The option marked+ // `[selected]` is what the field shows, which is the value a person reads off it.+ if (descriptor.role === "combobox" || descriptor.role === "listbox") {+ const list = optionsOf(value);+ if (list) {+ element.options = list.options;+ if (list.selected !== undefined && element.value === undefined)+ element.value = list.selected;+ }+ }+// Playwright emits `[checked]` only when something is checked, so absence is ambiguous on its own: a// Bot cannot tell an unticked box from a control that does not tick. Reported as false for the roles// that can be checked, and left off entirely for the ones that cannot.@@ -218,6 +237,35 @@ function toElement(return element;}+/**+ * The options listed under a dropdown, and which one is selected.+ *+ * Only options with no ref of their own: one with a ref is a custom listbox's option, already an+ * element the Bot clicks, and listing it twice would offer two ways to do one thing.+ */+function optionsOf(+ value: unknown,+): { options: string[]; selected?: string } | null {+ if (!Array.isArray(value)) return null;+ const options: string[] = [];+ let selected: string | undefined;+ for (const child of value) {+ const key =+ typeof child === "string"+ ? child+ : child && typeof child === "object"+ ? Object.keys(child)[0]+ : undefined;+ const descriptor = key ? parseDescriptor(key) : null;+ if (descriptor?.role !== "option" || descriptor.flags.has("ref")) continue;+ const name = cutAtCodeUnits(descriptor.name, 200);+ if (descriptor.flags.has("selected") && selected === undefined)+ selected = name;+ if (options.length < OPTION_LIMIT) options.push(name);+ }+ return options.length ? { options, selected } : null;+}+/** Turn an aria snapshot into the flat element list the tool contract publishes. */export function parseAriaSnapshot(yaml: string): {elements: SnapshotElement[];diff --git a/agent-computer/src/enter-text.ts b/agent-computer/src/enter-text.tsnew file mode 100644index 00000000..5db55ffa--- /dev/null+++ b/agent-computer/src/enter-text.ts@@ -0,0 +1,62 @@+import type { Locator } from "playwright";++/** What every Playwright call here is given: the action timeout, and Stop when there is one. */+type Acting = { timeout: number; signal?: AbortSignal };++/**+ * The text a Bot sent cannot go in this field: a list with no option by that name.+ *+ * The caller's to fix, like a stale ref, so the route answers 400 rather than 502. The message names+ * neither the text (a typed value is never echoed) nor the options (they are page content, and page+ * content reaches the model only inside the untrusted envelope a snapshot puts it in).+ */+export class FieldValueError extends Error {+ constructor(message: string) {+ super(message);+ this.name = "FieldValueError";+ }+}++/**+ * Put `text` in a field the way a person would.+ *+ * A text box is filled: `fill` clears it first, which is what "put this value in this box" means.+ * A native `<select>` takes no typing at all, and `fill` waits on one until it times out, which the+ * server then reported as a ref that had gone stale. A person picks the option by what it says, so+ * this does too: the option whose label is the text, ignoring case and surrounding spaces, and+ * failing that the option whose value is. Picking it fires the same input and change events a+ * person's choice does, so the page reacts as it would to them.+ */+export async function enterText(+ field: Locator,+ text: string,+ acting: Acting,+): Promise<void> {+ const options = await field.evaluate(+ (element) =>+ element instanceof HTMLSelectElement+ ? Array.from(element.options, (option) => ({+ label: option.label,+ value: option.value,+ disabled: option.disabled,+ }))+ : null,+ undefined,+ { timeout: acting.timeout },+ );+ if (!options) {+ await field.fill(text, acting);+ return;+ }+ const wanted = text.trim().toLowerCase();+ const choosable = options.filter((option) => !option.disabled);+ const chosen =+ choosable.find((option) => option.label.trim().toLowerCase() === wanted) ??+ choosable.find((option) => option.value.trim().toLowerCase() === wanted);+ if (!chosen) {+ throw new FieldValueError(+ "That field is a list, and none of its options matches that text. Take a snapshot to see its options and send one of them exactly.",+ );+ }+ await field.selectOption({ value: chosen.value }, acting);+}diff --git a/agent-computer/src/index.ts b/agent-computer/src/index.tsindex e05165ac..08486402 100644--- a/agent-computer/src/index.ts+++ b/agent-computer/src/index.ts@@ -27,6 +27,7 @@ import {TAKE_CONTROL_FIRST,} from "./control";import { describeHumanGesture } from "./demonstration";+import { enterText, FieldValueError } from "./enter-text";import { handleEgressPolicyRequest, startEgressFilter } from "./egress";import { identity } from "./identity";import {@@ -1423,6 +1424,10 @@ serve<StreamData>({if (error instanceof StaleSnapshotError) {return json({ error: error.message, stale: true }, 409);}+ // The same reasoning: the computer is fine, and the Bot fixes it by sending another value.+ if (error instanceof FieldValueError) {+ return json({ error: error.message }, 400);+ }return json({ error: describe(error, "The action failed.") }, 502);}}@@ -1585,8 +1590,8 @@ async function performAction(const field = await resolveRef(session, target, ref, expected);// `fill` rather than keystrokes: it clears the field first, which is what "put this value in// this box" means. Typing into a field a previous attempt half-filled otherwise appends, and the- // form ends up with "AlicAlice" in it.- await field.fill(body.text, acting);+ // form ends up with "AlicAlice" in it. A dropdown is chosen from instead (see enterText).+ await enterText(field, body.text, acting);if (body.submit === true) {await field.press("Enter", acting);}diff --git a/agent-computer/tests/aria-snapshot.test.ts b/agent-computer/tests/aria-snapshot.test.tsindex 4052b47e..79d93d50 100644--- a/agent-computer/tests/aria-snapshot.test.ts+++ b/agent-computer/tests/aria-snapshot.test.ts@@ -325,3 +325,64 @@ describe("parseDescriptor", () => {expect(parseDescriptor(" ")).toBeNull();});});++/**+ * Captured from `ariaSnapshot({ mode: "ai" })` in this package's Playwright: two native `<select>`s+ * and a custom listbox. A native select's options carry no refs, so they used to be dropped and a Bot+ * saw `combobox "Industry"` with nothing it could choose and no way to tell what it was set to.+ */+const DROPDOWNS = `- generic [active] [ref=e1]:+ - generic [ref=e2]:+ - text: Industry+ - combobox "Industry" [ref=e3]:+ - option "Select an industry" [selected]+ - option "Logistics"+ - option "Retail"+ - generic [ref=e4]:+ - text: Stage+ - combobox "Stage" [ref=e5]:+ - option "Prospect" [selected]+ - option "Qualified"+ - listbox "Owner" [ref=e6]:+ - option "Alex" [ref=e7]`;++describe("parseAriaSnapshot, dropdowns", () => {+ test("a native select reports its options and the one it shows", () => {+ const { elements } = parseAriaSnapshot(DROPDOWNS);+ expect(elements.find((element) => element.ref === "e3")).toEqual({+ ref: "e3",+ role: "combobox",+ name: "Industry",+ options: ["Select an industry", "Logistics", "Retail"],+ value: "Select an industry",+ });+ expect(elements.find((element) => element.ref === "e5")?.value).toBe(+ "Prospect",+ );+ });++ test("a custom listbox's options stay elements of their own, listed once", () => {+ const { elements } = parseAriaSnapshot(DROPDOWNS);+ expect(elements.find((element) => element.ref === "e6")).toEqual({+ ref: "e6",+ role: "listbox",+ name: "Owner",+ });+ expect(elements.find((element) => element.ref === "e7")).toEqual({+ ref: "e7",+ role: "option",+ name: "Alex",+ });+ });++ test("a very long list is cut to fifty choices", () => {+ const many = Array.from(+ { length: 120 },+ (_, index) => ` - option "Country ${index}"`,+ ).join("\n");+ const { elements } = parseAriaSnapshot(+ `- combobox "Country" [ref=e9]:\n${many}`,+ );+ expect(elements[0]?.options).toHaveLength(50);+ });+});diff --git a/agent-computer/tests/enter-text.test.ts b/agent-computer/tests/enter-text.test.tsnew file mode 100644index 00000000..ac9510cf--- /dev/null+++ b/agent-computer/tests/enter-text.test.ts@@ -0,0 +1,136 @@+import { afterAll, beforeAll, describe, expect, test } from "bun:test";+import type { Browser, Page } from "playwright";+import { parseAriaSnapshot } from "../src/aria-snapshot";+import { enterText, FieldValueError } from "../src/enter-text";++/**+ * Typing into a form the way a Bot does: an AI snapshot, then the field by its `aria-ref`.+ *+ * A CRM's New account form has text boxes and dropdowns side by side. Typing "Logistics" into the+ * Industry dropdown used `fill`, which waits on a `<select>` until it times out; the server reported+ * that as a ref gone stale, the Bot's turn ended, and no account was ever saved.+ *+ * OPENBOT_COMPUTER_BROWSER=1 bun test tests/enter-text.test.ts+ */+const asked = process.env.OPENBOT_COMPUTER_BROWSER === "1";+const ACTING = { timeout: 2_000 };++const FORM = `<!doctype html><title>New account</title>+<form>+ <label>Account name <input name="name" value="Draft"></label>+ <label>Industry+ <select name="industry">+ <option value="">Select an industry</option>+ <option value="fin">Financial services</option>+ <option value="logistics">Logistics</option>+ <option value="retired" disabled>Retired</option>+ </select>+ </label>+</form>+<output id="changes">0</output>+<script>+ const changes = document.getElementById("changes");+ document.querySelector("select").addEventListener("change", () => {+ changes.textContent = String(Number(changes.textContent) + 1);+ });+</script>`;++let browser: Browser;++beforeAll(async () => {+ if (!asked) return;+ const { chromium } = await import("playwright");+ browser = await chromium.launch();+});++afterAll(async () => {+ await browser?.close();+});++/** The form, and the refs its AI snapshot gives each field, as the computer reads them. */+async function openForm(): Promise<{+ tab: Page;+ refOf: (name: string) => string;+ industry: { options?: string[]; value?: string } | undefined;+}> {+ const tab = await browser.newPage();+ await tab.setContent(FORM);+ const snapshot = await tab.ariaSnapshot({ mode: "ai" });+ const { elements } = parseAriaSnapshot(snapshot);+ return {+ tab,+ industry: elements.find((element) => element.name === "Industry"),+ refOf: (name) => {+ const found = elements.find((element) => element.name === name);+ if (!found) throw new Error(`No ${name} in the snapshot.`);+ return found.ref;+ },+ };+}++describe.skipIf(!asked)("entering text in a form", () => {+ test("a dropdown is chosen from by the option's label, and the page sees a change", async () => {+ const { tab, refOf } = await openForm();+ await enterText(+ tab.locator(`aria-ref=${refOf("Industry")}`),+ " logistics ",+ ACTING,+ );+ expect(await tab.locator("select").inputValue()).toBe("logistics");+ expect(await tab.locator("#changes").textContent()).toBe("1");+ await tab.close();+ });++ test("a dropdown falls back to the option's value", async () => {+ const { tab, refOf } = await openForm();+ await enterText(+ tab.locator(`aria-ref=${refOf("Industry")}`),+ "fin",+ ACTING,+ );+ expect(await tab.locator("select").inputValue()).toBe("fin");+ await tab.close();+ });++ test("text that is not an option, or names a disabled one, is refused and changes nothing", async () => {+ const { tab, refOf } = await openForm();+ const industry = tab.locator(`aria-ref=${refOf("Industry")}`);+ for (const text of ["Aerospace", "Retired"]) {+ const refused = await enterText(industry, text, ACTING).catch(+ (error: unknown) => error,+ );+ expect(refused).toBeInstanceOf(FieldValueError);+ // Never the text it was sent, never the page's own option labels.+ expect((refused as Error).message).not.toContain(text);+ expect((refused as Error).message).not.toContain("Logistics");+ }+ expect(await tab.locator("select").inputValue()).toBe("");+ expect(await tab.locator("#changes").textContent()).toBe("0");+ await tab.close();+ });++ test("the snapshot the refusal points to lists the dropdown's options and its choice", async () => {+ const { tab, industry } = await openForm();+ expect(industry?.options).toEqual([+ "Select an industry",+ "Financial services",+ "Logistics",+ "Retired",+ ]);+ expect(industry?.value).toBe("Select an industry");+ await tab.close();+ });++ test("a text box is still filled, replacing what was there", async () => {+ const { tab, refOf } = await openForm();+ await enterText(+ tab.locator(`aria-ref=${refOf("Account name")}`),+ "Navia Freight",+ ACTING,+ );+ expect(await tab.locator("input[name=name]").inputValue()).toBe(+ "Navia Freight",+ );+ await tab.close();+ });+});diff --git a/app/src/lib/computers/call.ts b/app/src/lib/computers/call.tsindex 44ce4f1b..38357e56 100644--- a/app/src/lib/computers/call.ts+++ b/app/src/lib/computers/call.ts@@ -76,26 +76,39 @@ export async function callComputer(*/if (response.status === 403 && run !== null)withdrawComputerActivity(botId, run);- return {- ok: false,- reason: (body?.error as string) ?? "That did not work.",- // Preserve refusal/stale-ref/control distinctions for the model's next step.- ...(response.status === 403- ? { refused: true, rule: body?.rule ?? null }- : {}),- ...(response.status === 409- ? body?.humanHasControl === true- ? {- humanHasControl: true,- requestId: body?.requestId,- handoff: body?.handoff,- }- : body?.snapshotRequired === true- ? { staleRefs: true, snapshotRequired: true }- : { staleRefs: true }- : {}),- };+ return failedComputerCall(response.status, body);}return { ok: true, ...(body ?? {}) };}++/**+ * What a Bot is told when a computer call came back refused or failed.+ *+ * A result, never a thrown error. The SDK starts no follow-up run after a frontend tool throws, so a+ * throw ends the Bot's turn on the spot with nothing said; a result reaches the model, which can take+ * a fresh snapshot, pick another field, or tell the person what stopped it. Shared with the approval+ * gate's wrapper, which reaches the same computer by another route and must answer the same way.+ */+export function failedComputerCall(+ status: number,+ body: Record<string, unknown> | null,+): ToolOutcome {+ return {+ ok: false,+ reason: typeof body?.error === "string" ? body.error : "That did not work.",+ // Preserve refusal/stale-ref/control distinctions for the model's next step.+ ...(status === 403 ? { refused: true, rule: body?.rule ?? null } : {}),+ ...(status === 409+ ? body?.humanHasControl === true+ ? {+ humanHasControl: true,+ requestId: body?.requestId,+ handoff: body?.handoff,+ }+ : body?.snapshotRequired === true+ ? { staleRefs: true, snapshotRequired: true }+ : { staleRefs: true }+ : {}),+ };+}diff --git a/app/src/lib/copilot/approval-tools.tsx b/app/src/lib/copilot/approval-tools.tsxindex c6eeb111..ca7014b8 100644--- a/app/src/lib/copilot/approval-tools.tsx+++ b/app/src/lib/copilot/approval-tools.tsx@@ -7,6 +7,7 @@ import { useQueryClient } from "@tanstack/react-query";import type { ReactNode } from "react";import { InlineApproval } from "@/components/approvals/inline-approval";import { tryClient } from "@/lib/client";+import { failedComputerCall } from "@/lib/computers/call";import { useActiveBotHolder } from "./active-bot";import { frontendApprovalSnapshot } from "./approval-context";@@ -69,11 +70,22 @@ export function useFrontendTool<T extends Record<string, unknown>>(// run it when nothing asks, and says why when something does.return tool.handler?.(args, context);}- const response = await tryClient(- `/api/approvals/computer/${encodeURIComponent(bot.current)}`,- { method: "POST", body: snapshot, signal: context.signal },- );- const payload: unknown = await response.json();+ let response: Response;+ try {+ response = await tryClient(+ `/api/approvals/computer/${encodeURIComponent(bot.current)}`,+ { method: "POST", body: snapshot, signal: context.signal },+ );+ } catch (error) {+ // The same two answers callComputer gives when the request itself does not complete.+ if (error instanceof DOMException && error.name === "AbortError")+ return { ok: false, reason: "Stopped.", stopped: true };+ return {+ ok: false,+ reason: "The assistant's computer could not be reached.",+ };+ }+ const payload: unknown = await response.json().catch(() => null);if (response.status === 202) {if (!context.agent)throw new Error("The interrupted conversation is unavailable.");@@ -84,17 +96,19 @@ export function useFrontendTool<T extends Record<string, unknown>>(void queryClient.invalidateQueries({ queryKey: ["approvals"] });return { waiting: true };}+ // Answered, not thrown: a thrown handler ends the Bot's turn with nothing said (see+ // failedComputerCall), where a failed result lets it snapshot again or try another way.if (!response.ok)- throw new Error(- payload &&- typeof payload === "object" &&- "error" in payload &&- typeof payload.error === "string"- ? payload.error- : "The computer action could not be approved.",+ return failedComputerCall(+ response.status,+ payload && typeof payload === "object"+ ? (payload as Record<string, unknown>)+ : null,);if (!payload || typeof payload !== "object" || !("result" in payload))- throw new Error("The computer action returned no result.");+ return failedComputerCall(response.status, {+ error: "The computer action returned no result.",+ });return payload.result;}: undefined,diff --git a/app/tests/approval-tools.test.tsx b/app/tests/approval-tools.test.tsxindex 8945763f..77c881ef 100644--- a/app/tests/approval-tools.test.tsx+++ b/app/tests/approval-tools.test.tsx@@ -155,3 +155,121 @@ test("a change goes to the server's gate even when nothing about approvals has lglobal.fetch = originalFetch;}});++/**+ * Runs the kept computer_click handler against a gate that answers with `answer`.+ *+ * The SDK starts no follow-up run after a frontend tool throws (core's executeSpecificTool returns+ * false on a handler error), so a handler that throws ends the Bot's turn on the spot with nothing+ * said. A form field the computer could not fill did exactly that: the Bot opened a CRM form, failed+ * on one dropdown, and the conversation simply stopped.+ */+async function clickAgainstGate(+ answer: () => Promise<Response>,+): Promise<unknown> {+ const originalFetch = global.fetch;+ global.fetch = Object.assign(async () => answer(), {+ preconnect: originalFetch.preconnect,+ });+ try {+ render(+ <QueryClientProvider client={new QueryClient()}>+ <Tools />+ </QueryClientProvider>,+ );+ const tool = registered.find((entry) => entry.name === "computer_click");+ const toolCall = {+ id: "click-call",+ type: "function" as const,+ function: {+ name: "computer_click",+ arguments: '{"ref":"e7","snapshotId":3}',+ },+ };+ const subscribers: {+ onRunInitialized?: (input: { input: unknown }) => void;+ }[] = [];+ const agent = {+ threadId: "thread",+ state: {},+ messages: [+ { id: "u1", role: "user", content: "Submit the form" },+ { id: "a1", role: "assistant", content: "", toolCalls: [toolCall] },+ ],+ subscribe: (subscriber: (typeof subscribers)[number]) => {+ subscribers.push(subscriber);+ return { unsubscribe: () => undefined };+ },+ };+ const stop = observeApprovalAgent(agent as never);+ for (const subscriber of subscribers)+ subscriber.onRunInitialized?.({+ input: {+ runId: "run-1",+ threadId: "thread",+ messages: agent.messages,+ tools: [],+ context: [],+ state: {},+ forwardedProps: {},+ },+ });+ try {+ return await tool?.handler?.({ ref: "e7", snapshotId: 3 }, {+ agent,+ toolCall,+ } as never);+ } finally {+ stop();+ }+ } finally {+ global.fetch = originalFetch;+ }+}++test("a change the computer could not make comes back to the Bot as a result, not a throw", async () => {+ const result = await clickAgainstGate(async () =>+ Response.json(+ {+ error:+ "Element e7 is not on the page any more. Take a fresh snapshot and use the refs from it.",+ },+ { status: 409 },+ ),+ );+ expect(result).toEqual({+ ok: false,+ reason:+ "Element e7 is not on the page any more. Take a fresh snapshot and use the refs from it.",+ staleRefs: true,+ });+});++test("a refused change comes back as a refusal the Bot can explain", async () => {+ const result = await clickAgainstGate(async () =>+ Response.json(+ { error: "Your team does not allow this.", rule: "team-rule-1" },+ { status: 403 },+ ),+ );+ expect(result).toEqual({+ ok: false,+ reason: "Your team does not allow this.",+ refused: true,+ rule: "team-rule-1",+ });+});++test("a gate that cannot be reached, or answers with no result, is still a result", async () => {+ expect(+ await clickAgainstGate(async () => {+ throw new TypeError("fetch failed");+ }),+ ).toEqual({+ ok: false,+ reason: "The assistant's computer could not be reached.",+ });+ expect(+ await clickAgainstGate(async () => Response.json({}, { status: 200 })),+ ).toEqual({ ok: false, reason: "The computer action returned no result." });+});diff --git a/server/src/computer/schema.ts b/server/src/computer/schema.tsindex 4836eaed..62da7af7 100644--- a/server/src/computer/schema.ts+++ b/server/src/computer/schema.ts@@ -133,6 +133,8 @@ export type SnapshotElement = {type?: string;disabled?: boolean;checked?: boolean;+ /** A native dropdown's choices, by label. A Bot chooses one by typing it into the dropdown. */+ options?: string[];};export type SnapshotResult = {