Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { randomUUID } from "node:crypto";
import {
type AbstractAgent,
type BaseEvent,
EventType,
HttpAgent,
type RunAgentInput,
} from "@ag-ui/client";
import { NOT_SHOWN } from "@openbot/shared/component-markers";
import { createTurnRunner, type RunnerLike } from "@openbot/toolkit/run-turn";
import { inArray } from "drizzle-orm";
import { Observable } from "rxjs";
import { GALLERY } from "../../app/src/components/gallery/charts";
import { serverTurnAdmission } from "../src/agents/turn-admission";
import { type AuditEventInput, type AuditStore, auditSink } from "../src/audit";
import {
createHeadlessComponentTools,
headlessComponentCatalog,
} from "../src/components/headless";
import { createComponentStore } from "../src/components/store";
import { botSeat } from "../src/computer/computer-key";
import { createComputerGateway } from "../src/computer/gateway";
import { createHeadlessComputerTools } from "../src/computer/headless-tools";
import { createDatabase } from "../src/db/client";
import { agents, componentExclusions, components } from "../src/db/schema";
import { TEST_POOL, testDatabaseUrl } from "./support/database";
/**
* An unattended routine turn that browses a page and answers with a chart, against the real
* component governance tables. The Intelligence runner and the remote Bot are local seams; this is
* not a live model, Kubernetes or browser validation.
*/
const database = createDatabase(testDatabaseUrl(), TEST_POOL);
const store = createComponentStore(database);
const suite = randomUUID().slice(0, 8);
const allowedBot = `agent_headless_chart_${suite}`;
const deniedBot = `agent_headless_withheld_${suite}`;
let addedComponents: string[] = [];
const chartArgs = {
title: "Stories on the front page by source",
basis: "read",
caption: "Read from the page the routine opened.",
points: [
{ label: "github.com", value: 4 },
{ label: "nytimes.com", value: 2 },
],
};
beforeAll(async () => {
for (const id of [allowedBot, deniedBot])
await database
.insert(agents)
.values({ id, name: id, type: "remote_ag_ui", configuration: {} })
.onConflictDoNothing();
// What the browser announces on first load, so the rows are the build's own declarations.
({ added: addedComponents } = await store.syncCatalogue(
GALLERY.map(({ name, title, kind, description }) => ({
name,
title,
kind,
description,
})),
));
const decision = await store.decide("showBarChart", allowedBot);
if (!decision.allowed)
throw new Error(
`The test database has showBarChart unpublished: ${decision.reason}`,
);
await store.revoke("showBarChart", deniedBot, "[email protected]");
});
afterAll(async () => {
await database
.delete(componentExclusions)
.where(inArray(componentExclusions.agentId, [allowedBot, deniedBot]));
if (addedComponents.length > 0)
await database
.delete(components)
.where(inArray(components.name, addedComponents));
await database
.delete(agents)
.where(inArray(agents.id, [allowedBot, deniedBot]));
});
function lifecycle(input: RunAgentInput, body: BaseEvent[]): BaseEvent[] {
return [
{
type: EventType.RUN_STARTED,
threadId: input.threadId,
runId: input.runId,
} as BaseEvent,
...body,
{
type: EventType.RUN_FINISHED,
threadId: input.threadId,
runId: input.runId,
} as BaseEvent,
];
}
function call(id: string, name: string, args: unknown): BaseEvent[] {
return [
{
type: EventType.TOOL_CALL_START,
toolCallId: id,
toolCallName: name,
parentMessageId: `message-${id}`,
},
{
type: EventType.TOOL_CALL_ARGS,
toolCallId: id,
delta: JSON.stringify(args),
},
{ type: EventType.TOOL_CALL_END, toolCallId: id },
] as BaseEvent[];
}
function reply(text: string): BaseEvent[] {
return [
{
type: EventType.TEXT_MESSAGE_START,
messageId: "reply",
role: "assistant",
},
{ type: EventType.TEXT_MESSAGE_CONTENT, messageId: "reply", delta: text },
{ type: EventType.TEXT_MESSAGE_END, messageId: "reply" },
] as BaseEvent[];
}
/** A remote AG-UI Bot that opens a page, charts it, then answers. */
function remoteBot() {
const requests: RunAgentInput[] = [];
const agent = new HttpAgent({
agentId: "bot",
url: "https://agent.example/ag-ui",
fetch: async (_url, init) => {
const input = JSON.parse(String(init?.body)) as RunAgentInput;
requests.push(input);
const body =
requests.length === 1
? call("navigate-call", "computer_navigate", {
url: "https://news.example/",
})
: requests.length === 2
? call("chart-call", "showBarChart", chartArgs)
: reply("I charted the front page for you.");
return new Response(
lifecycle(input, body)
.map((event) => `data: ${JSON.stringify(event)}\n\n`)
.join(""),
{ headers: { "content-type": "text/event-stream" } },
);
},
});
return { agent, requests };
}
function harness(agent: AbstractAgent, botId: string) {
const persisted: BaseEvent[] = [];
const audit: AuditEventInput[] = [];
const auditStore = {
insert: async (event: AuditEventInput) => {
audit.push(event);
},
};
const visited: string[] = [];
const gateway = createComputerGateway({
provider: {
name: "test",
isolation: "per-person",
locate: async () => "http://computer:4100",
status: async (seat) => ({ botId: seat.botId, state: "ready" }),
list: async () => [],
stop: async () => ({ wasRunning: true }),
reset: async () => ({ cleared: true }),
},
auditStore,
policy: () => ({ mode: "enforce", allow: ["true"], deny: [] }),
fetchImpl: async (url, init) => {
const path = new URL(String(url)).pathname;
if (path !== "/navigate")
throw new Error(`Unexpected computer request ${path}`);
visited.push(JSON.parse(String(init?.body)).url);
return Response.json({
url: "https://news.example/",
title: "News",
text: "github.com 4 stories, nytimes.com 2 stories",
truncated: false,
elapsedMs: 1,
});
},
});
const runner: RunnerLike = {
run: ({ agent: selected, input }) =>
new Observable((subscriber) => {
selected
.runAgent(input, {
onEvent: ({ event }) => {
persisted.push(event);
subscriber.next(event);
},
})
.then(
() => subscriber.complete(),
(error) => subscriber.error(error),
);
}),
stop: async () => true,
};
const run = createTurnRunner({
admission: serverTurnAdmission,
intelligence: {
getOrCreateThread: async () => undefined,
getThreadMessages: async () => ({ messages: [] }),
ɵacquireThreadLock: async () => undefined,
ɵrenewThreadLock: async () => undefined,
ɵcleanupThreadLock: async () => undefined,
},
runner,
buildAgentFor: async () => agent,
toolsForTurn: async ({ ownerUserId, initiator }) =>
createHeadlessComputerTools({
gateway,
seat: botSeat(botId),
actor: { id: ownerUserId, userId: ownerUserId, initiator },
}),
components: headlessComponentCatalog({
store,
audit: auditSink(auditStore as unknown as AuditStore),
}),
});
return {
persisted,
audit,
visited,
run: () =>
run({
ownerUserId: "owner",
routineId: "routine-front-page",
agentId: botId,
threadId: `thread-${botId}`,
instruction: "Chart today's front page by source",
}),
};
}
function resultFor(persisted: BaseEvent[], toolCallId: string) {
return persisted.find(
(event) =>
event.type === EventType.TOOL_CALL_RESULT &&
(event as { toolCallId?: string }).toolCallId === toolCallId,
) as { content: string } | undefined;
}
describe("an unattended routine turn answering with a governed chart", () => {
test("browses, records the chart as its component in the persisted transcript, and replies", async () => {
const remote = remoteBot();
const turn = harness(remote.agent, allowedBot);
// The chart comes back with the reply, so a chat surface (Teams, Slack) can draw it natively.
await expect(turn.run()).resolves.toEqual({
replyText: "I charted the front page for you.",
components: [{ name: "showBarChart", args: chartArgs }],
});
expect(turn.visited).toEqual(["https://news.example/"]);
// Offered with the deployment's published wording, as the browser would offer it.
const offered = remote.requests[0]?.tools.find(
(tool) => tool.name === "showBarChart",
);
expect(offered?.description).toBe(
GALLERY.find((entry) => entry.name === "showBarChart")?.description,
);
// The persisted events are what the transcript renderer reads: the call by component name with
// its arguments, and a result that is not a refusal.
const start = turn.persisted.find(
(event) =>
event.type === EventType.TOOL_CALL_START &&
(event as { toolCallId?: string }).toolCallId === "chart-call",
) as { toolCallName: string } | undefined;
expect(start?.toolCallName).toBe("showBarChart");
const args = turn.persisted
.filter(
(event) =>
event.type === EventType.TOOL_CALL_ARGS &&
(event as { toolCallId?: string }).toolCallId === "chart-call",
)
.map((event) => (event as { delta: string }).delta)
.join("");
expect(JSON.parse(args)).toEqual(chartArgs);
const result = resultFor(turn.persisted, "chart-call");
expect(result?.content).toBe(
"The bar chart is saved in this conversation and is drawn when the person opens it.",
);
expect(resultFor(turn.persisted, "navigate-call")?.content).toContain(
'"title":"News"',
);
// The Bot saw the result and continued the same AG-UI run.
expect(remote.requests[2]?.messages).toContainEqual(
expect.objectContaining({ role: "tool", toolCallId: "chart-call" }),
);
expect(
turn.persisted.filter((event) => event.type === EventType.RUN_STARTED),
).toHaveLength(1);
expect(
turn.persisted.filter((event) => event.type === EventType.RUN_FINISHED),
).toHaveLength(1);
expect(
turn.audit.filter((event) => event.eventType.startsWith("component.")),
).toEqual([]);
});
test("a Bot withheld from the chart is not offered it, and a call anyway is recorded as not shown", async () => {
const remote = remoteBot();
const turn = harness(remote.agent, deniedBot);
await expect(turn.run()).resolves.toEqual({
replyText: "I charted the front page for you.",
});
expect(
remote.requests[0]?.tools.some((tool) => tool.name === "showBarChart"),
).toBeFalse();
const result = resultFor(turn.persisted, "chart-call");
expect(result?.content.startsWith(`${NOT_SHOWN}Bar chart. `)).toBeTrue();
expect(result?.content).toContain("withheld from this Bot");
expect(turn.audit).toContainEqual(
expect.objectContaining({
eventType: "component.refused",
targetType: "component",
targetId: "showBarChart",
actorUserId: "owner",
initiator: { kind: "routine", id: "routine-front-page" },
payload: expect.objectContaining({ bot: deniedBot, headless: true }),
}),
);
});
test("a grant revoked after the turn started is refused at call time", async () => {
const tools = await createHeadlessComponentTools({
store,
botId: allowedBot,
ownerUserId: "owner",
initiator: { kind: "routine", id: "routine-front-page" },
});
const bar = tools.find((tool) => tool.definition.name === "showBarChart");
expect(bar?.hidden).toBeUndefined();
await store.revoke("showBarChart", allowedBot, "[email protected]");
try {
const result = String(
await bar?.execute(chartArgs, {
toolCallId: "late",
signal: new AbortController().signal,
}),
);
expect(result.startsWith(NOT_SHOWN)).toBeTrue();
} finally {
await store.grant("showBarChart", allowedBot);
}
});
});