Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { randomUUID } from "node:crypto";
import {
type ExchangeResponse,
LEARNING_SPACE_STATES,
RECORDER_ACTIVITY_STALE_MS,
RECORDER_SIGNED_OUT,
type RecorderConnectionsResponse,
} from "@openbot/shared/recorder-api";
import { RECORDER_EXTENSION_ID } from "@openbot/shared/recorder-extension";
import { eq, inArray } from "drizzle-orm";
import { createApp } from "../src/app";
import type { AuditEventInput } from "../src/audit";
import { loadConfig } from "../src/config";
import { createDatabase } from "../src/db/client";
import { users } from "../src/db/schema/core";
import { recorderConnections } from "../src/db/schema/recorder";
import { createRecorderExtension } from "../src/recorder/wiring";
import { TEST_POOL, testDatabaseUrl } from "./support/database";
import { SESSION_LIFETIME_MS, testEnvironment } from "./support/environment";
const database = createDatabase(testDatabaseUrl(), TEST_POOL);
const prefix = `recorder-routes-${randomUUID()}`;
const owner = `${prefix}-owner`;
const busy = `${prefix}-busy`;
const ORIGIN = `chrome-extension://${RECORDER_EXTENSION_ID}`;
const INSTALL = "c".repeat(32);
const OTHER_EXTENSION = `chrome-extension://${"a".repeat(32)}`;
let transcribed = 0;
const transcriber = Bun.serve({
hostname: "127.0.0.1",
port: 0,
fetch: () => {
transcribed += 1;
return Response.json({ text: "hello from the fake" });
},
});
type Harness = ReturnType<typeof build>;
function build(
extra: Record<string, string | undefined> = {},
options: {
kickThrows?: boolean;
/** The trail refuses every row written with a connection (connections.ts writes them in its transaction). */
auditFails?: boolean;
} = {},
) {
const config = loadConfig(
testEnvironment({
TRANSCRIPTION_PROVIDER: "openai-compatible",
TRANSCRIPTION_BASE_URL: `http://127.0.0.1:${transcriber.port}`,
TRANSCRIPTION_MODEL: "fake",
...extra,
}),
);
const audits: AuditEventInput[] = [];
const auditStore = {
insert: async (event: AuditEventInput) => {
audits.push(event);
},
};
const roleRepository = { rolesForUser: async () => ["user" as const] };
/** The person's own Learning space, faked: what the Recorder is told, and whose browsers connected. */
const kicked: string[] = [];
const recorder = createRecorderExtension({
config,
database,
organizationAuth: undefined,
roleRepository,
peopleStore: { isRevoked: async () => false },
personalSpaces: {
containerFor: (userId) => `space-of-${userId}`,
state: async (userId) => ({
state: "assigned",
containerId: `space-of-${userId}`,
}),
kick: (userId) => {
kicked.push(userId);
if (options.kickThrows) throw new Error("credential store down");
},
},
sessionLifetimeMs: SESSION_LIFETIME_MS,
auditStore,
...(options.auditFails
? {
auditFor: () => ({
insert: async () => {
throw new Error("the audit trail is unavailable");
},
}),
}
: {}),
});
/*
* createApp is positional. Placed by index rather than after a spread of `undefined`s, whose
* length TypeScript cannot see, so each dependency is type-checked against the parameter it
* lands on: the audit store the Recorder's routes write to is `auditStore`.
*/
const args = new Array(createApp.length) as unknown as Parameters<
typeof createApp
>;
args[0] = config;
args[1] = {
handler: () => new Response(null, { status: 204 }),
api: {
getSession: async ({ headers }: { headers: Headers }) => {
const cookie = headers.get("cookie") ?? "";
const id = cookie.includes("test-session=owner")
? owner
: cookie.includes("test-session=busy")
? busy
: null;
return id
? { user: { id, email: `${id}@example.test`, name: "The Owner" } }
: null;
},
},
} as never;
args[2] = roleRepository;
args[12] = auditStore;
args[34] = { recorder };
const app = createApp(...args);
return { app, audits, config, kicked };
}
const call = (
{ app }: Harness,
path: string,
init: RequestInit & { json?: unknown } = {},
) => {
const { json, headers, ...rest } = init;
return app.request(`http://openbot.test${path}`, {
...rest,
headers: {
...(json === undefined ? {} : { "content-type": "application/json" }),
...(headers as Record<string, string>),
},
...(json === undefined ? {} : { body: JSON.stringify(json) }),
});
};
const COOKIE = { cookie: "test-session=owner" };
async function connect(
harness: Harness,
installId = INSTALL,
): Promise<ExchangeResponse> {
const minted = await call(harness, "/api/recorder/connect-codes", {
method: "POST",
headers: COOKIE,
json: { via: "web" },
});
const { code } = (await minted.json()) as { code: string };
const exchanged = await call(harness, "/api/recorder/connections", {
method: "POST",
headers: { origin: ORIGIN },
json: { code, label: "Chrome on macOS", installId },
});
expect(exchanged.status).toBe(201);
return exchanged.json();
}
const bearer = (token: string) => ({ authorization: `Bearer ${token}` });
beforeAll(async () => {
await database.insert(users).values([
{ id: owner, email: `${owner}@example.test`, name: "The Owner" },
{ id: busy, email: `${busy}@example.test`, name: "The Owner" },
]);
});
afterAll(async () => {
await database.delete(users).where(inArray(users.id, [owner, busy]));
await database.$client.close();
await transcriber.stop(true);
});
describe("connecting", () => {
test("a connect code needs JSON and a signed-in person", async () => {
const h = build();
const form = await call(h, "/api/recorder/connect-codes", {
method: "POST",
headers: { ...COOKIE, "content-type": "text/plain" },
body: '{"via":"web"}',
});
expect(form.status).toBe(415);
const anonymous = await call(h, "/api/recorder/connect-codes", {
method: "POST",
json: { via: "web" },
});
expect(anonymous.status).toBe(401);
const minted = await call(h, "/api/recorder/connect-codes", {
method: "POST",
headers: COOKIE,
json: { via: "web" },
});
expect(minted.status).toBe(201);
expect(((await minted.json()) as { code: string }).code).toMatch(/^obc_/);
});
test("a person who already holds ten unused codes is told to wait", async () => {
const h = build();
const mint = () =>
call(h, "/api/recorder/connect-codes", {
method: "POST",
headers: { cookie: "test-session=busy" },
json: { via: "web" },
});
for (let index = 0; index < 10; index += 1) {
expect((await mint()).status).toBe(201);
}
const refused = await mint();
expect(refused.status).toBe(429);
expect(await refused.json()).toEqual({
error:
"Too many connection codes were asked for at once. Wait a few minutes and try again.",
});
// A refusal like any other of the Recorder's: on the trail, as the person's.
expect(
h.audits.filter((a) => a.eventType === "recorder.connect_refused"),
).toEqual([
{
eventType: "recorder.connect_refused",
targetType: "recorder_connection",
actorUserId: busy,
payload: { reason: "too_many_codes", via: "web" },
},
]);
});
test("a code that is unknown, expired or already used is refused on the trail", async () => {
const h = build();
const unknown = await call(h, "/api/recorder/connections", {
method: "POST",
headers: { origin: ORIGIN },
json: { code: `obc_${"A".repeat(43)}`, label: "x", installId: INSTALL },
});
expect(unknown.status).toBe(400);
const refusals = h.audits.filter(
(a) => a.eventType === "recorder.connect_refused",
);
expect(refusals).toEqual([
{
eventType: "recorder.connect_refused",
targetType: "recorder_connection",
initiator: { kind: "deployment" },
payload: {
reason: "code",
extensionId: RECORDER_EXTENSION_ID,
installId: INSTALL,
},
},
]);
// Never the code itself.
expect(JSON.stringify(refusals)).not.toContain("AAAA");
});
test("a connection whose row cannot be written is not made, and the person is told to try again", async () => {
const h = build({}, { auditFails: true });
const minted = await call(h, "/api/recorder/connect-codes", {
method: "POST",
headers: COOKIE,
json: { via: "web" },
});
const { code } = (await minted.json()) as { code: string };
const failed = await call(h, "/api/recorder/connections", {
method: "POST",
headers: { origin: ORIGIN },
json: { code, label: "Chrome on macOS", installId: "9".repeat(32) },
});
expect(failed.status).toBe(500);
expect(await failed.json()).toEqual({
error: "This browser could not be connected. Try again.",
});
const listed = (await (
await call(h, "/api/recorder/connections", { headers: COOKIE })
).json()) as { connections: { installId: string }[] };
expect(listed.connections.map((c) => c.installId)).not.toContain(
"9".repeat(32),
);
// The code was not spent: with the trail back, the same code connects.
const done = await call(build(), "/api/recorder/connections", {
method: "POST",
headers: { origin: ORIGIN },
json: { code, label: "Chrome on macOS", installId: "9".repeat(32) },
});
expect(done.status).toBe(201);
});
test("a body over 2 KiB is refused", async () => {
const h = build();
const big = await call(h, "/api/recorder/connect-codes", {
method: "POST",
headers: COOKIE,
json: { via: "web", padding: "x".repeat(3000) },
});
expect(big.status).toBe(413);
});
test("only the Recorder's own origin may finish connecting", async () => {
const h = build();
const minted = await call(h, "/api/recorder/connect-codes", {
method: "POST",
headers: COOKIE,
json: { via: "web" },
});
const { code } = (await minted.json()) as { code: string };
const body = { code, label: "x", installId: INSTALL };
for (const origin of ["https://evil.example.test", OTHER_EXTENSION]) {
const refused = await call(h, "/api/recorder/connections", {
method: "POST",
headers: { origin },
json: body,
});
expect(refused.status).toBe(403);
}
expect(
h.audits.filter((a) => a.eventType === "recorder.connect_refused"),
).toHaveLength(2);
const missing = await call(h, "/api/recorder/connections", {
method: "POST",
headers: { origin: ORIGIN },
json: { code, label: "x" },
});
expect(missing.status).toBe(400);
const done = await call(h, "/api/recorder/connections", {
method: "POST",
headers: { origin: ORIGIN },
json: body,
});
expect(done.status).toBe(201);
const made = (await done.json()) as ExchangeResponse;
expect(made.connection.installId).toBe(INSTALL);
expect(made.user.id).toBe(owner);
expect(
h.audits.filter((a) => a.eventType === "recorder.connected"),
).toHaveLength(1);
const again = await call(h, "/api/recorder/connections", {
method: "POST",
headers: { origin: ORIGIN },
json: body,
});
expect(again.status).toBe(400);
});
test("connecting still succeeds when starting the person's Learning space fails", async () => {
const h = build({}, { kickThrows: true });
const made = await connect(h, "7".repeat(32));
expect(made.token).toMatch(/^obr_/);
expect(h.kicked).toEqual([owner]);
});
test("the same copy connecting again replaces its older connection", async () => {
const h = build();
const first = await connect(h, "d".repeat(32));
const second = await connect(h, "d".repeat(32));
const replaced = h.audits.filter((a) => a.eventType === "recorder.revoked");
expect(replaced).toHaveLength(1);
expect(replaced[0]?.payload).toEqual({
connectionId: first.connection.id,
via: "web",
reason: "replaced",
});
expect(
(await call(h, "/api/recorder/me", { headers: bearer(first.token) }))
.status,
).toBe(401);
expect(
(await call(h, "/api/recorder/me", { headers: bearer(second.token) }))
.status,
).toBe(200);
});
test("extra extension ids are listed after the fixed one and accepted", async () => {
const extra = "b".repeat(32);
const plain = build();
const refused = await call(plain, "/api/recorder/connections", {
method: "POST",
headers: { origin: `chrome-extension://${extra}` },
json: { code: "obc_x", label: "x", installId: INSTALL },
});
expect(refused.status).toBe(403);
const h = build({ OPENBOT_RECORDER_EXTRA_EXTENSION_IDS: extra });
const listed = await call(h, "/api/recorder/connections", {
headers: COOKIE,
});
expect(
((await listed.json()) as { extensionIds: string[] }).extensionIds,
).toEqual([RECORDER_EXTENSION_ID, extra]);
const minted = await call(h, "/api/recorder/connect-codes", {
method: "POST",
headers: COOKIE,
json: { via: "desktop" },
});
const { code } = (await minted.json()) as { code: string };
const accepted = await call(h, "/api/recorder/connections", {
method: "POST",
headers: { origin: `chrome-extension://${extra}/` },
json: { code, label: "Fork", installId: "f".repeat(32) },
});
expect(accepted.status).toBe(201);
});
});
describe("using the bearer", () => {
test("a disconnect whose row cannot be written does not happen, and the person is told to try again", async () => {
const made = await connect(build(), "6".repeat(32));
const broken = build({}, { auditFails: true });
const fromSettings = await call(
broken,
`/api/recorder/connections/${made.connection.id}`,
{ method: "DELETE", headers: COOKIE },
);
expect(fromSettings.status).toBe(500);
expect(await fromSettings.json()).toEqual({
error: "That browser could not be disconnected. Try again.",
});
const fromTheRecorder = await call(
broken,
"/api/recorder/connections/current",
{ method: "DELETE", headers: bearer(made.token) },
);
expect(fromTheRecorder.status).toBe(500);
expect(await fromTheRecorder.json()).toEqual({
error: "This browser could not be disconnected. Try again.",
});
// Still connected: neither disconnect happened without its row.
expect(
(await call(build(), "/api/recorder/me", { headers: bearer(made.token) }))
.status,
).toBe(200);
});
test("a connection unused for as long as a session lasts is signed out, and one used within it stands", async () => {
const h = build();
const made = await connect(h, "5".repeat(32));
const me = () =>
call(h, "/api/recorder/me", { headers: bearer(made.token) });
const lastUsedAgo = async (ms: number) => {
const at = new Date(Date.now() - ms);
await database
.update(recorderConnections)
.set({ createdAt: at, lastUsedAt: at })
.where(eq(recorderConnections.id, made.connection.id));
};
await lastUsedAgo(SESSION_LIFETIME_MS - 60_000);
expect((await me()).status).toBe(200);
await lastUsedAgo(SESSION_LIFETIME_MS + 60_000);
const expired = await me();
expect(expired.status).toBe(401);
expect(await expired.json()).toMatchObject({
code: RECORDER_SIGNED_OUT,
reason: "revoked",
});
const listed = (await (
await call(h, "/api/recorder/connections", { headers: COOKIE })
).json()) as { connections: { id: string }[] };
expect(listed.connections.map((c) => c.id)).not.toContain(
made.connection.id,
);
});
test("me, the person's own Learning space and the connections list", async () => {
const h = build();
const made = await connect(h, "1".repeat(32));
// Connecting the browser starts making the person's space, so it is there before the first recording.
expect(h.kicked).toEqual([owner]);
const me = await call(h, "/api/recorder/me", {
headers: bearer(made.token),
});
expect(me.status).toBe(200);
expect(await me.json()).toEqual({
user: { id: owner, email: `${owner}@example.test`, name: "The Owner" },
connectionId: made.connection.id,
});
const learning = await call(h, "/api/recorder/learning", {
headers: bearer(made.token),
});
expect(learning.status).toBe(200);
const answer = (await learning.json()) as {
learning: { state: string; containerId: string | null };
};
expect(LEARNING_SPACE_STATES as readonly string[]).toContain(
answer.learning.state,
);
expect(answer.learning).toEqual({
state: "assigned",
containerId: `space-of-${owner}`,
});
// There is no Bot list any more: a recording is the person's, not a Bot's.
expect(
(await call(h, "/api/recorder/bots", { headers: bearer(made.token) }))
.status,
).toBe(404);
const listed = (await (
await call(h, "/api/recorder/connections", { headers: COOKIE })
).json()) as {
connections: { id: string }[];
extensionIds: string[];
};
expect(listed.connections.map((c) => c.id)).toContain(made.connection.id);
expect(listed.extensionIds).toEqual([RECORDER_EXTENSION_ID]);
});
test("the connections list says which browser is recording now, from its Recorder's own recent report", async () => {
// End-to-end run: a browser recording right now read "last used 49 seconds ago", since a recording streams to the
// gateway and reaches OpenBot only with its activity reports.
const h = build();
const made = await connect(h, "5".repeat(32));
const recordingNow = async () => {
const listed = (await (
await call(h, "/api/recorder/connections", { headers: COOKIE })
).json()) as RecorderConnectionsResponse;
return listed.connections.find(
(connection) => connection.id === made.connection.id,
)?.recording;
};
const report = async (phase: "recording" | "idle") =>
expect(
(
await call(h, "/api/recorder/activity", {
method: "POST",
headers: bearer(made.token),
json: { phase, recordingId: "rec-1", actions: 3 },
})
).status,
).toBe(204);
expect(await recordingNow()).toBe(false);
await report("recording");
expect(await recordingNow()).toBe(true);
// A report from a browser that went quiet mid-recording stops standing after its window.
await database
.update(recorderConnections)
.set({
activityAt: new Date(Date.now() - RECORDER_ACTIVITY_STALE_MS - 1_000),
})
.where(eq(recorderConnections.id, made.connection.id));
expect(await recordingNow()).toBe(false);
await report("recording");
expect(await recordingNow()).toBe(true);
await report("idle");
expect(await recordingNow()).toBe(false);
});
test("disconnecting signs the browser out", async () => {
const h = build();
const made = await connect(h, "3".repeat(32));
const gone = await call(h, "/api/recorder/connections/current", {
method: "DELETE",
headers: bearer(made.token),
});
expect(gone.status).toBe(204);
expect(
h.audits.find((a) => a.eventType === "recorder.revoked")?.payload,
).toMatchObject({ connectionId: made.connection.id });
const after = await call(h, "/api/recorder/me", {
headers: bearer(made.token),
});
expect(after.status).toBe(401);
expect(((await after.json()) as { code: string }).code).toBe(
RECORDER_SIGNED_OUT,
);
expect(
h.audits.filter((a) => a.eventType === "recorder.bearer_refused"),
).toHaveLength(1);
});
test("a person removes one of their connections from the list", async () => {
const h = build();
const made = await connect(h, "4".repeat(32));
const removed = await call(
h,
`/api/recorder/connections/${made.connection.id}`,
{ method: "DELETE", headers: COOKIE },
);
expect(removed.status).toBe(204);
const missing = await call(h, "/api/recorder/connections/nope", {
method: "DELETE",
headers: COOKIE,
});
expect(missing.status).toBe(404);
expect(
(await call(h, "/api/recorder/me", { headers: bearer(made.token) }))
.status,
).toBe(401);
});
test("the bearer cannot transcribe speech, even with the dictation header", async () => {
const h = build();
const made = await connect(h, "5".repeat(32));
const form = new FormData();
form.set(
"file",
new File([new Uint8Array(8)], "recording.webm", { type: "audio/webm" }),
);
const answer = await h.app.request(
"http://openbot.test/api/audio/transcriptions",
{
method: "POST",
headers: { ...bearer(made.token), "x-openbot-dictation": "1" },
body: form,
},
);
expect(answer.status).toBe(401);
expect(await answer.json()).toEqual({ error: "Authentication required." });
expect(transcribed).toBe(0);
});
test("OpenBot takes no recording from the Recorder, not even with its bearer", async () => {
const h = build();
const made = await connect(h, "6".repeat(32));
for (const [method, path] of [
["POST", "/api/recorder/demonstrations"],
["GET", "/api/recorder/demonstrations/open?botId=mine"],
["POST", "/api/recorder/demonstrations/x/actions"],
["POST", "/api/recorder/demonstrations/x/media"],
["PUT", "/api/recorder/demonstrations/x/media/m/chunks/0"],
]) {
const answer = await call(h, path, {
method,
headers: bearer(made.token),
...(method === "GET" ? {} : { json: {} }),
});
expect([method, path, answer.status]).toEqual([method, path, 404]);
}
});
});