Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { afterAll, afterEach, describe, expect, test } from "bun:test";
import { randomUUID } from "node:crypto";
import { FIRING_FRAME, frameFiring } from "@openbot/shared/routine-firing";
import { eq } from "drizzle-orm";
import { createAgentProfileStore } from "../src/agents/profile-store";
import type { AgentActor } from "../src/agents/profile-types";
import { createChannelStore } from "../src/channels/routes";
import {
CHANNEL_SUMMARY_KIND,
type ChannelTitler,
offerChannelsAwaitingSummary,
summariseClaimedChannels,
type ThreadTranscript,
} from "../src/channels/summary";
import { createThreadIdentity } from "../src/channels/thread-identity";
import { createDatabase } from "../src/db/client";
import {
agentProfiles,
agents,
channelMemberships,
channels,
intelligenceChannelMappings,
users,
workItems,
} from "../src/db/schema";
import { createWorkQueue } from "../src/work/queue";
import { TEST_POOL, testDatabaseUrl } from "./support/database";
const databaseUrl = testDatabaseUrl();
const database = createDatabase(databaseUrl, TEST_POOL);
const profileStore = createAgentProfileStore(
database,
new URL("https://managed.example.test/ag-ui"),
);
const store = createChannelStore(
database,
profileStore,
createThreadIdentity("test-deployment"),
);
const queue = createWorkQueue(database);
const testPrefix = `channel-summary-${randomUUID()}`;
const createdUserIds: string[] = [];
const createdAgentIds: string[] = [];
const createdChannelIds: string[] = [];
afterEach(async () => {
for (const channelId of createdChannelIds.splice(0)) {
await database.delete(workItems).where(eq(workItems.key, channelId));
await database
.delete(intelligenceChannelMappings)
.where(eq(intelligenceChannelMappings.channelId, channelId));
await database.delete(channels).where(eq(channels.id, channelId));
}
for (const agentId of createdAgentIds.splice(0)) {
await database
.delete(agentProfiles)
.where(eq(agentProfiles.agentId, agentId));
await database.delete(agents).where(eq(agents.id, agentId));
}
for (const userId of createdUserIds.splice(0)) {
await database.delete(users).where(eq(users.id, userId));
}
});
afterAll(async () => {
await database.$client.close();
});
async function createUser(): Promise<AgentActor> {
const id = `${testPrefix}-user-${randomUUID()}`;
await database.insert(users).values({
id,
email: `${id}@example.test`,
name: "Channel Summary Test User",
});
createdUserIds.push(id);
return { id, role: "user" };
}
async function createAgent(owner: AgentActor) {
const profile = await profileStore.create(owner, {
name: "Expense Manager",
title: "Finance Operations",
roleDescription: "Review receipts.",
visibility: "private",
});
createdAgentIds.push(profile.id);
return profile.id;
}
/** A channel somebody has actually said something in, which is the only kind worth naming. */
async function createUsedChannel(owner: AgentActor) {
const agentId = await createAgent(owner);
const channel = await store.create(owner, [agentId]);
createdChannelIds.push(channel.id);
await store.recordActivity(owner, channel.id, {
text: "Which of these receipts count as travel?",
agentId: null,
at: new Date(),
});
return channel;
}
/** A transcript with one question and one answer in it, which is what a title is made from. */
function transcriptOf(question: string, answer?: string): ThreadTranscript {
return {
getThreadMessages: async () => ({
messages: [
{ role: "user", content: question },
...(answer ? [{ role: "assistant", content: answer }] : []),
],
}),
};
}
function titler(answer: string | null): ChannelTitler {
return async () => answer;
}
async function summaryOf(channelId: string) {
const [row] = await database
.select({ summary: channels.summary, summaryAt: channels.summaryAt })
.from(channels)
.where(eq(channels.id, channelId));
return row;
}
/**
* Queue one conversation, by name.
*
* Deliberately not `offerChannelsAwaitingSummary`, which offers every unnamed conversation in the
* database. A claim takes whatever is queued, so a test that offers the whole table and then claims
* will happily write a fabricated title onto somebody's real conversation when the suite is pointed
* at a database that has any — a development one, for instance. Tests that exercise the offer query
* itself assert on what it returns and never claim.
*/
async function offer(channelId: string) {
await queue.offer({ kind: CHANNEL_SUMMARY_KIND, key: channelId });
}
const options = (overrides: {
transcript?: ThreadTranscript;
title?: ChannelTitler;
owner?: string;
}) => ({
database,
queue,
transcript: overrides.transcript ?? transcriptOf("Which receipts?"),
title: overrides.title ?? titler("Travel receipt rules"),
owner: overrides.owner ?? `test-${randomUUID()}`,
});
describe("offering conversations to be named", () => {
test("offers one that has been used and has no name yet", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
const { offered } = await offerChannelsAwaitingSummary({
database,
queue,
limit: 200,
});
expect(offered).toContain(channel.id);
});
test("does not offer a channel nobody has said anything in", async () => {
const owner = await createUser();
const agentId = await createAgent(owner);
const channel = await store.create(owner, [agentId]);
createdChannelIds.push(channel.id);
const { offered } = await offerChannelsAwaitingSummary({
database,
queue,
limit: 200,
});
// There is nothing for a model to read, and asking it to name an empty thread spends money to
// produce a guess.
expect(offered).not.toContain(channel.id);
});
test("offering the same conversation twice leaves one piece of work", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offerChannelsAwaitingSummary({ database, queue, limit: 200 });
await offerChannelsAwaitingSummary({ database, queue, limit: 200 });
const rows = await database
.select({ key: workItems.key })
.from(workItems)
.where(eq(workItems.key, channel.id));
expect(rows).toHaveLength(1);
});
/*
* A pass offers at most `limit` conversations, twenty by default, and a conversation it could not
* name keeps no summary, so it stays in the set the pass reads. Offering it again while its settled
* work row stands does nothing, but it still used one of those places. Twenty such conversations
* ahead of a new one left nothing for it, pass after pass.
*/
test("does not spend a place on a conversation whose settled work still stands", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
// Settled without a name, the way a model that answers with nothing leaves it.
await summariseClaimedChannels(options({ title: titler(null) }));
expect((await summaryOf(channel.id))?.summary).toBeNull();
const { offered } = await offerChannelsAwaitingSummary({
database,
queue,
limit: 200,
});
expect(offered).not.toContain(channel.id);
});
test("offers it again once its settled work has been forgotten", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
await summariseClaimedChannels(options({ title: titler(null) }));
// What `forgetSettledSummaries` does once the finished row is an hour old.
await database.delete(workItems).where(eq(workItems.key, channel.id));
const { offered } = await offerChannelsAwaitingSummary({
database,
queue,
limit: 200,
});
expect(offered).toContain(channel.id);
});
});
/**
* What the first pass on a deployment with a history looks like.
*
* Every conversation ever used and never named is eligible at once, so the only thing standing
* between a deploy and a model request per historical conversation is the claim limit. These pin
* that limit as behaviour rather than leaving it a constant somebody can raise without noticing what
* it is holding back.
*/
describe("a backlog of unnamed conversations", () => {
test("names at most one batch per pass, not one per conversation", async () => {
const owner = await createUser();
const agentId = await createAgent(owner);
const made: string[] = [];
for (let index = 0; index < 6; index += 1) {
const channel = await store.create(owner, [agentId]);
createdChannelIds.push(channel.id);
await store.recordActivity(owner, channel.id, {
text: `Question ${index}`,
agentId: null,
at: new Date(),
});
made.push(channel.id);
}
for (const id of made) await offer(id);
let calls = 0;
const counting = options({
title: async () => {
calls += 1;
return "A title";
},
});
const first = await summariseClaimedChannels({ ...counting, limit: 2 });
expect(first.written).toHaveLength(2);
expect(calls).toBe(2);
const second = await summariseClaimedChannels({ ...counting, limit: 2 });
expect(second.written).toHaveLength(2);
// The next batch, not the first one again.
expect(second.written.some((id) => first.written.includes(id))).toBe(false);
expect(calls).toBe(4);
});
});
describe("naming a claimed conversation", () => {
test("writes the title and stamps when it was written", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
const report = await summariseClaimedChannels(
options({
transcript: transcriptOf(
"Which of these receipts count as travel?",
"The three flights and the hotel do.",
),
title: titler("Travel receipt rules"),
}),
);
expect(report.written).toContain(channel.id);
const row = await summaryOf(channel.id);
expect(row?.summary).toBe("Travel receipt rules");
expect(row?.summaryAt).toBeInstanceOf(Date);
});
test("titles a routine-opened channel from the instruction, not the firing frame", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
let excerptSeen = "";
await summariseClaimedChannels(
options({
// A channel a routine opened has its first message framed, not typed by a person: see
// `shared/routine-firing.ts`. The title has to come from the instruction inside the frame.
transcript: transcriptOf(frameFiring("Post the standup summary.")),
title: async (excerpt) => {
excerptSeen = excerpt;
return "Standup summary";
},
}),
);
expect(excerptSeen).toContain("Post the standup summary.");
expect(excerptSeen).not.toContain(FIRING_FRAME[0]);
expect((await summaryOf(channel.id))?.summary).toBe("Standup summary");
});
test("two replicas racing for the same conversation name it once", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
/*
* Overlapping, not sequential. Two sweeps run one after the other pass just as happily with the
* row-level locking removed, so they prove the conditional update and nothing about the claim.
* These two are in flight together on separate pooled connections, which is what `for update
* skip locked` is there for: the second finds nothing to take rather than waiting behind the
* first and then doing the work twice.
*/
const [left, right] = await Promise.all([
summariseClaimedChannels(options({ title: titler("Left title") })),
summariseClaimedChannels(options({ title: titler("Right title") })),
]);
expect([...left.written, ...right.written]).toEqual([channel.id]);
expect(left.considered + right.considered).toBe(1);
const written = (await summaryOf(channel.id))?.summary;
expect(["Left title", "Right title"]).toContain(written);
});
test("never overwrites a name the conversation already has", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await database
.update(channels)
.set({ summary: "Named already", summaryAt: new Date() })
.where(eq(channels.id, channel.id));
// Offered before it was named, which is exactly the race two replicas produce.
await queue.offer({ kind: CHANNEL_SUMMARY_KIND, key: channel.id });
await summariseClaimedChannels(
options({ title: titler("A second opinion") }),
);
expect((await summaryOf(channel.id))?.summary).toBe("Named already");
});
test("leaves the conversation unnamed when the model has no answer", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
const report = await summariseClaimedChannels(
options({ title: titler(null) }),
);
// A deployment with no model key reaches this on every pass. Nothing is written, nothing is
// broken, and the roster keeps drawing the Bot's name.
expect(report.written).not.toContain(channel.id);
expect((await summaryOf(channel.id))?.summary).toBeNull();
});
test("comes back later when the conversation is not readable yet", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
// What a conversation looks like in the seconds between somebody sending a message and
// Intelligence holding it. Finishing the work here is how an ordinary conversation ends up
// never named at all.
const report = await summariseClaimedChannels(
options({
transcript: { getThreadMessages: async () => ({ messages: [] }) },
}),
);
expect(report.skipped).toContainEqual({
channelId: channel.id,
reason: "not readable yet",
});
const [item] = await database
.select({ finishedAt: workItems.finishedAt, runAt: workItems.runAt })
.from(workItems)
.where(eq(workItems.key, channel.id));
expect(item?.finishedAt).toBeNull();
expect(item?.runAt.getTime()).toBeGreaterThan(Date.now());
});
test("a conversation that no longer exists is dropped, not retried", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
// The channel goes while its naming is still queued. A queue key is not a foreign key, so the
// work outlives the thing it was about — which happens for real on a delete, and happens on
// every integration test that seeds a channel and tears it down.
await database.delete(channels).where(eq(channels.id, channel.id));
await summariseClaimedChannels(options({}));
const [item] = await database
.select({
finishedAt: workItems.finishedAt,
attempts: workItems.attempts,
})
.from(workItems)
.where(eq(workItems.key, channel.id));
// Finished rather than released: nothing about a deleted conversation is different in fifteen
// seconds, and retrying it to the attempt cap burns a model call's worth of work each time.
expect(item?.finishedAt).not.toBeNull();
expect(item?.attempts).toBe(1);
});
test("a soft-deleted conversation is dropped the same way", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
await database
.update(channels)
.set({ deletedAt: new Date() })
.where(eq(channels.id, channel.id));
await summariseClaimedChannels(options({}));
const [item] = await database
.select({ finishedAt: workItems.finishedAt })
.from(workItems)
.where(eq(workItems.key, channel.id));
expect(item?.finishedAt).not.toBeNull();
expect((await summaryOf(channel.id))?.summary).toBeNull();
});
test("stays inside the NOTIFY payload limit on a crowded channel", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
// The title is short and fixed; the term that grows is the member list the payload carries, so
// that is the term the test grows. `pg_notify` refuses a payload over 8000 bytes outright, and
// it refuses it inside the transaction that writes the title.
const crowd = [];
for (let index = 0; index < 40; index += 1) {
const member = await createUser();
crowd.push(member.id);
await database
.insert(channelMemberships)
.values({ channelId: channel.id, userId: member.id });
}
await offer(channel.id);
const report = await summariseClaimedChannels(
options({ title: titler("Travel receipt rules") }),
);
expect(report.written).toContain(channel.id);
// Recorded rather than merely survived: this is the number that says how much room is left, and
// it is what a later change adding a field to the event has to be measured against.
const payload = JSON.stringify({
channelId: channel.id,
memberIds: [owner.id, ...crowd],
lastMessage: null,
lastMessageAt: null,
lastMessageAgentId: null,
summary: "Travel receipt rules",
});
expect(payload.length).toBeLessThan(8000);
});
test("a model that fails leaves the work to be tried again", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
await summariseClaimedChannels(
options({
title: async () => {
throw new Error("the model is unreachable");
},
}),
);
expect((await summaryOf(channel.id))?.summary).toBeNull();
const [item] = await database
.select({ attempts: workItems.attempts, error: workItems.lastError })
.from(workItems)
.where(eq(workItems.key, channel.id));
expect(item?.attempts).toBe(1);
expect(item?.error).toContain("the model is unreachable");
});
test("flattens and shortens whatever the model answers with", async () => {
const owner = await createUser();
const channel = await createUsedChannel(owner);
await offer(channel.id);
await summariseClaimedChannels(
options({ title: titler(' "Travel\nreceipt rules" ') }),
);
// The quotes a model wraps a title in, the newline, and the run of spaces all go: a roster row
// is one line of plain text.
expect((await summaryOf(channel.id))?.summary).toBe("Travel receipt rules");
});
});