Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { afterAll, beforeEach, describe, expect, test } from "bun:test";
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import {
createHandoffRunner,
type HandoffWork,
} from "../src/agents/handoff-runner";
import type { AuditStore } from "../src/audit";
import { createDatabase } from "../src/db/client";
import { workItems } from "../src/db/schema";
import { createWorkQueue } from "../src/work/queue";
import { TEST_POOL, testDatabaseUrl } from "./support/database";
/**
* Two replicas and one batch of hops, against a real PostgreSQL.
*
* A lease is a promise the database keeps about time passing, and every stub of this queue answers
* whatever it was told to. The whole suite was green while the tail of every batch was delivered
* twice, because a fake cannot let a lease quietly run out.
*/
const database = createDatabase(testDatabaseUrl(), TEST_POOL);
const queue = createWorkQueue(database);
const kind = "bot.message";
const silent: AuditStore = { insert: async () => {} };
afterAll(async () => {
await database.delete(workItems).where(eq(workItems.kind, kind));
await database.$client.close();
});
beforeEach(async () => {
await database.delete(workItems).where(eq(workItems.kind, kind));
});
function hop(run: string, n: number): HandoffWork {
return {
fromBotId: "assistant",
toBotId: `bot-${n}`,
actorId: "user-1",
threadId: "thread-1",
runId: run,
depth: 1,
task: `task ${n}`,
};
}
describe("a batch of hops and a lease that can run out", () => {
/*
* A claim leases the whole batch from one moment, and the batch is delivered one at a time. A
* heartbeat that only covers the hop in flight leaves the rest on a lease that expires while the
* first one runs: another replica claims them, and this one delivers them anyway. Two model calls,
* two answers in somebody's conversation, and both replicas reporting success.
*/
test("the tail of a batch is not delivered twice while its head is running", async () => {
const run = randomUUID();
for (const n of [1, 2, 3]) {
await queue.offer({
kind,
key: `${run}:${n}`,
payload: hop(run, n) as unknown as Record<string, unknown>,
});
}
const ran: string[] = [];
const held = Promise.withResolvers<void>();
const shared = {
queue,
sign: () => "signed",
auditStore: silent,
// Small enough to drive in milliseconds; the property is one duration outrunning another.
leaseMs: 400,
renewEveryMs: 100,
limit: 3,
};
const slow = createHandoffRunner({
...shared,
owner: "replica-a",
delivery: {
deliver: async ({ work }) => {
ran.push(`a:${work.toBotId}`);
if (work.toBotId === "bot-1") await held.promise;
return { answer: null };
},
},
});
const quick = createHandoffRunner({
...shared,
owner: "replica-b",
delivery: {
deliver: async ({ work }) => {
ran.push(`b:${work.toBotId}`);
return { answer: null };
},
},
});
const sweepA = slow.sweep();
// Long enough that an unrenewed lease taken at the same moment would have lapsed twice over.
await new Promise((resolve) => setTimeout(resolve, 900));
const reportB = await quick.sweep();
held.resolve();
const reportA = await sweepA;
expect(reportA.delivered).toEqual(["bot-1", "bot-2", "bot-3"]);
// Nothing was left for the other replica to take, so nothing ran twice.
expect(reportB.delivered).toEqual([]);
expect(ran).toEqual(["a:bot-1", "a:bot-2", "a:bot-3"]);
});
/*
* And when a lease really has gone, the model call is the thing not to spend. Finding out after
* delivering is finding out too late.
*/
test("a hop whose lease went elsewhere is not run again by its old owner", async () => {
const run = randomUUID();
for (const n of [1, 2]) {
await queue.offer({
kind,
key: `${run}:${n}`,
payload: hop(run, n) as unknown as Record<string, unknown>,
});
}
const ran: string[] = [];
const held = Promise.withResolvers<void>();
const slow = createHandoffRunner({
queue,
owner: "replica-a",
sign: () => "signed",
auditStore: silent,
leaseMs: 400,
// Never refreshed, which is what a paused process looks like from the database's side.
renewEveryMs: 60_000,
limit: 2,
delivery: {
deliver: async ({ work }) => {
ran.push(work.toBotId);
if (work.toBotId === "bot-1") await held.promise;
return { answer: null };
},
},
});
const sweep = slow.sweep();
await new Promise((resolve) => setTimeout(resolve, 700));
// Somebody else takes the lapsed hop while the first is still running.
const taken = await queue.claim({
kind,
owner: "replica-b",
leaseMs: 10_000,
limit: 5,
});
held.resolve();
const report = await sweep;
expect(taken.map((item) => item.key)).toContain(`${run}:2`);
// Delivered once, by whoever holds it now, and not a second time by its old owner.
expect(ran).toEqual(["bot-1"]);
expect(report.skipped.map((entry) => entry.reason)).toContain(
"the lease went elsewhere",
);
});
});