Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { afterAll, afterEach, describe, expect, test } from "bun:test";
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { createAgentProfileStore } from "../src/agents/profile-store";
import type { AgentActor } from "../src/agents/profile-types";
import { createChannelStore } from "../src/channels/routes";
import { createThreadIdentity } from "../src/channels/thread-identity";
import { createDatabase } from "../src/db/client";
import {
agentProfiles,
agents,
channels,
intelligenceChannelMappings,
routineRuns,
routines,
users,
} from "../src/db/schema";
import {
createRoutineStore,
MAX_ENABLED_ROUTINES,
MAX_INSTRUCTION_CODE_POINTS,
MAX_RUN_ERROR,
RoutineNotFoundError,
RoutineRefusedError,
} from "../src/routines/store";
import { TEST_POOL, testDatabaseUrl } from "./support/database";
const databaseUrl = testDatabaseUrl();
const database = createDatabase(databaseUrl, TEST_POOL);
const profileStore = createAgentProfileStore(
database,
new URL("https://managed.example.test/ag-ui"),
);
const channelStore = createChannelStore(
database,
profileStore,
createThreadIdentity("test-deployment"),
);
const store = createRoutineStore(database);
const testPrefix = `routines-store-${randomUUID()}`;
const createdUserIds: string[] = [];
const createdAgentIds: string[] = [];
const createdChannelIds: string[] = [];
/** Every day at 09:00 UTC: comfortably above the floor and stable to describe. */
const DAILY = "0 9 * * *";
afterEach(async () => {
// Routines cascade from both the owner and the agent, but they are cleared first so a failure
// part-way through cleanup leaves nothing pointing at rows this file is about to delete.
for (const userId of createdUserIds) {
await database.delete(routines).where(eq(routines.ownerUserId, userId));
}
for (const channelId of createdChannelIds.splice(0)) {
await database
.delete(intelligenceChannelMappings)
.where(eq(intelligenceChannelMappings.channelId, channelId));
await database.delete(channels).where(eq(channels.id, channelId));
}
for (const agentId of createdAgentIds.splice(0)) {
await database
.delete(agentProfiles)
.where(eq(agentProfiles.agentId, agentId));
await database.delete(agents).where(eq(agents.id, agentId));
}
for (const userId of createdUserIds.splice(0)) {
await database.delete(users).where(eq(users.id, userId));
}
});
afterAll(async () => {
await database.$client.close();
});
async function createUser(): Promise<AgentActor> {
const id = `${testPrefix}-user-${randomUUID()}`;
await database.insert(users).values({
id,
email: `${id}@example.test`,
name: "Routine Store Test User",
});
createdUserIds.push(id);
return { id, role: "user" };
}
async function createAgent(owner: AgentActor, name = "Expense Manager") {
const profile = await profileStore.create(owner, {
name,
title: "Finance Operations",
roleDescription: "Review receipts.",
visibility: "private",
});
createdAgentIds.push(profile.id);
return profile.id;
}
async function createChannel(owner: AgentActor, agentIds: string[]) {
const channel = await channelStore.create(owner, agentIds);
createdChannelIds.push(channel.id);
return channel;
}
/** A person, a Bot and the one channel they share: the ordinary starting point. */
async function setUp() {
const owner = await createUser();
const agentId = await createAgent(owner);
const channel = await createChannel(owner, [agentId]);
return { owner, agentId, channel };
}
/**
* A routine is one person's standing instruction, so every method takes the owner and every
* statement is guarded by it. A routine belonging to somebody else has to be indistinguishable from
* one that was never created — the `setPinned` rule — because anything else tells a stranger which
* ids exist.
*/
describe("owner-guarding", () => {
test("another person cannot update, remove or switch off a routine", async () => {
const { owner, agentId, channel } = await setUp();
const stranger = await createUser();
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Summarise yesterday's receipts.",
cron: DAILY,
});
await expect(
store.update(stranger.id, routine.id, { instruction: "Do my bidding." }),
).rejects.toBeInstanceOf(RoutineNotFoundError);
await expect(store.remove(stranger.id, routine.id)).rejects.toBeInstanceOf(
RoutineNotFoundError,
);
await expect(
store.setEnabled(stranger.id, routine.id, false),
).rejects.toBeInstanceOf(RoutineNotFoundError);
const [mine] = await store.listFor(owner.id);
expect(mine?.instruction).toBe("Summarise yesterday's receipts.");
expect(mine?.enabled).toBe(true);
});
test("a stranger's list does not contain the routine", async () => {
const { owner, agentId, channel } = await setUp();
const stranger = await createUser();
await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Summarise yesterday's receipts.",
cron: DAILY,
});
expect(await store.listFor(stranger.id)).toEqual([]);
});
});
describe("what the schedule has to be", () => {
test("refuses a cron under the floor, in the schedule's own words", async () => {
const { owner, agentId, channel } = await setUp();
// Every minute. The sentence is the schedule module's; the store only passes it along, because
// a model recovers from prose and this is the prose that says what to try instead.
await expect(
store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Check my inbox.",
cron: "* * * * *",
}),
).rejects.toThrow(/15 minutes/);
await expect(
store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Check my inbox.",
cron: "* * * * *",
}),
).rejects.toBeInstanceOf(RoutineRefusedError);
});
test("refuses an instruction over the cap and accepts one at it", async () => {
const { owner, agentId, channel } = await setUp();
const atTheCap = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "x".repeat(MAX_INSTRUCTION_CODE_POINTS),
cron: DAILY,
});
expect(Array.from(atTheCap.instruction)).toHaveLength(
MAX_INSTRUCTION_CODE_POINTS,
);
await expect(
store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "x".repeat(MAX_INSTRUCTION_CODE_POINTS + 1),
cron: DAILY,
}),
).rejects.toBeInstanceOf(RoutineRefusedError);
});
test("refuses a blank instruction with the sentence a model is meant to read", async () => {
const { owner, agentId, channel } = await setUp();
const failure = await store
.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: " ",
cron: DAILY,
})
.then(
() => null,
(error: unknown) => error,
);
expect(failure).toBeInstanceOf(RoutineRefusedError);
expect((failure as Error).message).toBe(
"A routine needs an instruction to carry out.",
);
});
});
/**
* The cap is a constant with a reason: every enabled routine is a headless turn somebody's Bot will
* take without being watched, and a model that can be talked into creating them one at a time can be
* talked into creating a hundred. It counts what is switched ON, so switching one off makes room.
*/
describe("how many a person may have switched on", () => {
test("refuses the one past the cap, and accepts it once another is off", async () => {
const { owner, agentId, channel } = await setUp();
const created: string[] = [];
for (let index = 0; index < MAX_ENABLED_ROUTINES; index += 1) {
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: `Routine ${index}`,
cron: DAILY,
});
created.push(routine.id);
}
await expect(
store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "One too many.",
cron: DAILY,
}),
).rejects.toBeInstanceOf(RoutineRefusedError);
await store.setEnabled(owner.id, created[0] as string, false);
const room = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Now there is room.",
cron: DAILY,
});
expect(room.enabled).toBe(true);
// And switching the disabled one back on is refused for the same reason.
await expect(
store.setEnabled(owner.id, created[0] as string, true),
).rejects.toBeInstanceOf(RoutineRefusedError);
});
/**
* The cap has to hold under concurrency, not just in sequence. A bare count-then-insert lets two
* creates racing at 19 both count 19 and both insert — the person holds 21 — which is why the
* store serializes the count and the write per owner under an advisory lock. This drives the race
* for real: two creates in flight at once on separate pooled connections.
*/
test("two creates racing at the cap admit exactly one", async () => {
const { owner, agentId, channel } = await setUp();
for (let index = 0; index < MAX_ENABLED_ROUTINES - 1; index += 1) {
await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: `Routine ${index}`,
cron: DAILY,
});
}
const outcomes = await Promise.allSettled([
store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Racer one.",
cron: DAILY,
}),
store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Racer two.",
cron: DAILY,
}),
]);
const refused = outcomes.filter(
(outcome) => outcome.status === "rejected",
) as PromiseRejectedResult[];
expect(outcomes.filter((o) => o.status === "fulfilled")).toHaveLength(1);
expect(refused).toHaveLength(1);
// The loser gets the friendly sentence, not a constraint violation: the same refusal a
// sequential twenty-first create gets.
expect(refused[0]?.reason).toBeInstanceOf(RoutineRefusedError);
const enabled = (await store.listFor(owner.id)).filter(
(summary) => summary.enabled,
);
expect(enabled).toHaveLength(MAX_ENABLED_ROUTINES);
});
});
/**
* The conversation's channel is not reachable at the tool layer, so the store resolves it. All four
* branches matter: the wrong channel is a place a Bot could post something private, and the two
* ambiguous branches are refusals a model has to be able to recover from in language.
*/
describe("resolving the channel to post into", () => {
test("accepts a channel the owner and the agent share", async () => {
const { owner, agentId, channel } = await setUp();
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Post the summary here.",
cron: DAILY,
});
expect(routine.channelId).toBe(channel.id);
});
test("refuses a channel the owner is not in", async () => {
const { owner, agentId } = await setUp();
const stranger = await createUser();
const strangerAgentId = await createAgent(stranger, "Their Bot");
const theirChannel = await createChannel(stranger, [strangerAgentId]);
await expect(
store.create({
ownerUserId: owner.id,
agentId,
channelId: theirChannel.id,
instruction: "Post into somebody else's channel.",
cron: DAILY,
}),
).rejects.toThrow(/both in/);
});
test("refuses a channel this agent is not in", async () => {
const { owner, channel } = await setUp();
const otherAgentId = await createAgent(owner, "Unrelated Bot");
await expect(
store.create({
ownerUserId: owner.id,
agentId: otherAgentId,
channelId: channel.id,
instruction: "Post where I do not live.",
cron: DAILY,
}),
).rejects.toBeInstanceOf(RoutineRefusedError);
});
test("uses the only shared channel when none was named", async () => {
const { owner, agentId, channel } = await setUp();
const routine = await store.create({
ownerUserId: owner.id,
agentId,
instruction: "Wherever we talk.",
cron: DAILY,
});
expect(routine.channelId).toBe(channel.id);
});
test("refuses with no channel at all, and says to start one", async () => {
const owner = await createUser();
const agentId = await createAgent(owner);
await expect(
store.create({
ownerUserId: owner.id,
agentId,
instruction: "Nowhere to put this.",
cron: DAILY,
}),
).rejects.toThrow(/Start one/);
});
test("refuses ambiguity by naming the channels, so a model can ask", async () => {
const { owner, agentId, channel } = await setUp();
const secondAgentId = await createAgent(owner, "Second Bot");
const other = await createChannel(owner, [agentId, secondAgentId]);
const failure = await store
.create({
ownerUserId: owner.id,
agentId,
instruction: "Which one?",
cron: DAILY,
})
.then(
() => null,
(error: unknown) => error,
);
expect(failure).toBeInstanceOf(RoutineRefusedError);
const message = (failure as Error).message;
expect(message).toContain(channel.name);
expect(message).toContain(other.name);
expect(message).toContain("Say which one.");
});
test("names at most five channels before it gives up and says 'and others'", async () => {
const owner = await createUser();
const agentId = await createAgent(owner);
const sharedChannels = [];
for (let index = 0; index < 6; index += 1) {
sharedChannels.push(await createChannel(owner, [agentId]));
}
const failure = await store
.create({
ownerUserId: owner.id,
agentId,
instruction: "Which one?",
cron: DAILY,
})
.then(
() => null,
(error: unknown) => error,
);
expect(failure).toBeInstanceOf(RoutineRefusedError);
const message = (failure as Error).message;
expect(message).toContain(", and others");
// All six channels share one agent, so they share one name; the cap names five of them, not
// six, so that one name appears exactly five times rather than six.
const name = sharedChannels[0]?.name as string;
const occurrences = message.split(name).length - 1;
expect(occurrences).toBe(5);
});
test("names a colliding channel by id when its name cannot tell it apart from another", async () => {
// The finding this guards: a real account had six channels all named "General Assistant" with
// the same Bot, and the refusal read "General Assistant, General Assistant, … and others" —
// circular, because the person cannot answer it and the model cannot map an answer back to a
// channelId. Two same-named channels plus one distinctly-named one is the smallest case that
// proves the fix: the colliding pair gets ids, the distinct one stays bare.
const owner = await createUser();
const agentId = await createAgent(owner);
const twinOne = await createChannel(owner, [agentId]);
const twinTwo = await createChannel(owner, [agentId]);
// channelStore.create names channels itself, so force the collision directly rather than
// trusting default naming to produce two identical names.
await database
.update(channels)
.set({ name: "General Assistant" })
.where(eq(channels.id, twinOne.id));
await database
.update(channels)
.set({ name: "General Assistant" })
.where(eq(channels.id, twinTwo.id));
const distinct = await createChannel(owner, [agentId]);
await database
.update(channels)
.set({ name: "Marketing Bot" })
.where(eq(channels.id, distinct.id));
const failure = await store
.create({
ownerUserId: owner.id,
agentId,
instruction: "Which one?",
cron: DAILY,
})
.then(
() => null,
(error: unknown) => error,
);
expect(failure).toBeInstanceOf(RoutineRefusedError);
const message = (failure as Error).message;
// Both same-named channels are named by their full id, because that is the one thing the model
// can pass back as `channelId` when the names alone cannot disambiguate.
expect(message).toContain(twinOne.id);
expect(message).toContain(twinTwo.id);
// The distinctly-named channel is not: its name alone already disambiguates it, and appending
// an id it does not need would make the common, non-colliding case's sentence longer for
// nothing.
expect(message).toContain("Marketing Bot");
expect(message).not.toContain(`Marketing Bot (${distinct.id})`);
});
});
describe("reading a person's routines", () => {
test("says the schedule in words and never in cron", async () => {
const { owner, agentId, channel } = await setUp();
await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Summarise the day.",
cron: DAILY,
timezone: "Europe/Madrid",
});
const [summary] = await store.listFor(owner.id);
expect(summary?.schedule).toBe("Every day at 09:00");
expect(summary?.schedule).not.toContain("*");
expect(summary?.timezone).toBe("Europe/Madrid");
expect(summary?.channelName).toBe(channel.name);
expect(summary?.channelDeleted).toBe(false);
expect(summary?.nextRunAt).toBeInstanceOf(Date);
// A routine that has never fired has no run row, so the join has nothing to report.
expect(summary?.lastRun).toBeNull();
});
test("says a step schedule in words too, never in cron", async () => {
const { owner, agentId, channel } = await setUp();
await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Check in often.",
cron: "*/20 * * * *",
});
const [summary] = await store.listFor(owner.id);
expect(summary?.schedule).not.toContain("*");
});
test("puts the newest first", async () => {
const { owner, agentId, channel } = await setUp();
const first = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "First.",
cron: DAILY,
});
const second = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Second.",
cron: DAILY,
});
expect((await store.listFor(owner.id)).map((row) => row.id)).toEqual([
second.id,
first.id,
]);
});
test("keeps a routine whose channel was deleted and reports it as gone", async () => {
// The whole reason `routines.channel_id` is not a foreign key: a cascade here would delete the
// person's standing instruction silently, and they would find out by it never running again.
const { owner, agentId, channel } = await setUp();
await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Post into a channel that is about to go.",
cron: DAILY,
});
await channelStore.softDelete(owner, channel.id);
const [summary] = await store.listFor(owner.id);
expect(summary?.channelId).toBe(channel.id);
expect(summary?.channelDeleted).toBe(true);
});
});
describe("changing a routine", () => {
test("recomputes the next run when the cron changes", async () => {
const { owner, agentId, channel } = await setUp();
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Summarise the day.",
cron: DAILY,
});
const updated = await store.update(owner.id, routine.id, {
cron: "30 21 * * *",
});
expect(updated.cron).toBe("30 21 * * *");
expect(updated.nextRunAt.getTime()).not.toBe(routine.nextRunAt.getTime());
});
test("recomputes the next run when only the timezone changes, and refuses an unknown zone", async () => {
const { owner, agentId, channel } = await setUp();
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Summarise the day.",
cron: DAILY,
});
const updated = await store.update(owner.id, routine.id, {
timezone: "Asia/Tokyo",
});
expect(updated.timezone).toBe("Asia/Tokyo");
expect(updated.nextRunAt.getTime()).not.toBe(routine.nextRunAt.getTime());
await expect(
store.update(owner.id, routine.id, { timezone: "Mars/Olympus" }),
).rejects.toBeInstanceOf(RoutineRefusedError);
// The refusal must not have partially applied: the row still carries the timezone from the
// update that succeeded, not the rejected "Mars/Olympus".
const [summary] = await store.listFor(owner.id);
expect(summary?.timezone).toBe("Asia/Tokyo");
});
test("leaves the next run alone when only the instruction changes", async () => {
const { owner, agentId, channel } = await setUp();
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Summarise the day.",
cron: DAILY,
});
const updated = await store.update(owner.id, routine.id, {
instruction: "Summarise the week.",
});
expect(updated.instruction).toBe("Summarise the week.");
expect(updated.nextRunAt.getTime()).toBe(routine.nextRunAt.getTime());
});
test("re-validates only what was supplied", async () => {
const { owner, agentId, channel } = await setUp();
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Summarise the day.",
cron: DAILY,
});
await expect(
store.update(owner.id, routine.id, { cron: "* * * * *" }),
).rejects.toBeInstanceOf(RoutineRefusedError);
await expect(
store.update(owner.id, routine.id, {
instruction: "x".repeat(MAX_INSTRUCTION_CODE_POINTS + 1),
}),
).rejects.toBeInstanceOf(RoutineRefusedError);
// Refused writes changed nothing.
const [summary] = await store.listFor(owner.id);
expect(summary?.instruction).toBe("Summarise the day.");
expect(summary?.schedule).toBe("Every day at 09:00");
});
test("re-resolves the channel when a new one is named", async () => {
const { owner, agentId, channel } = await setUp();
const secondAgentId = await createAgent(owner, "Second Bot");
const other = await createChannel(owner, [agentId, secondAgentId]);
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Summarise the day.",
cron: DAILY,
});
const moved = await store.update(owner.id, routine.id, {
channelId: other.id,
});
expect(moved.channelId).toBe(other.id);
const stranger = await createUser();
const strangerAgentId = await createAgent(stranger, "Their Bot");
const theirChannel = await createChannel(stranger, [strangerAgentId]);
await expect(
store.update(owner.id, routine.id, { channelId: theirChannel.id }),
).rejects.toBeInstanceOf(RoutineRefusedError);
});
test("switching off and on again recomputes the next run", async () => {
const { owner, agentId, channel } = await setUp();
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Summarise the day.",
cron: DAILY,
});
// A routine switched off for a month has a next run a month in the past. Enabling it must not
// hand the sweep a firing that was due in June. Forcing the stamp into the past makes that
// stale state real rather than assumed: without this, `nextRunAt > now` would already have
// been true before the disable, and deleting the recompute branch would leave this green.
await store.setEnabled(owner.id, routine.id, false);
const stale = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
await database
.update(routines)
.set({ nextRunAt: stale })
.where(eq(routines.id, routine.id));
const enabled = await store.update(owner.id, routine.id, { enabled: true });
expect(enabled.enabled).toBe(true);
expect(enabled.nextRunAt.getTime()).toBeGreaterThan(Date.now());
// The mirror image: an instruction-only change on an already-enabled routine must not recompute,
// even when the stored stamp is one that recomputing would obviously move.
await database
.update(routines)
.set({ nextRunAt: stale })
.where(eq(routines.id, routine.id));
const untouched = await store.update(owner.id, routine.id, {
instruction: "Summarise the week instead.",
});
expect(untouched.nextRunAt.getTime()).toBe(stale.getTime());
});
test("removing one is a hard delete, and only once", async () => {
const { owner, agentId, channel } = await setUp();
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Summarise the day.",
cron: DAILY,
});
await store.remove(owner.id, routine.id);
expect(await store.listFor(owner.id)).toEqual([]);
await expect(store.remove(owner.id, routine.id)).rejects.toBeInstanceOf(
RoutineNotFoundError,
);
});
});
/*
* ---------------------------------------------------------------------------------------------
* The sweep's half. Nothing below is asked a question by a person, and everything below is about
* two things happening at once: several replicas reading the same due row in the same second.
*
* NO CLAIM OR LEASE SEMANTICS ARE TESTED HERE. Firing mechanics live in the shared `work_items`
* queue, and `server/tests/work-queue.integration.test.ts` owns `for update skip locked`, leases
* on the database's clock, and the attempt count. What this file tests is the routines table's own
* compare-and-set on `next_run_at`, which is a different guarantee: the clock moves once.
*/
/**
* The create path always computes a future stamp, so a due-in-the-past row has to be written
* directly — the same direct write the staleness test above uses. The stamps are deliberately
* ancient: `dueRoutines` is not owner-scoped, so a test that asserts on ordering has to be sure
* its own rows sort ahead of anything else in the database.
*/
async function makeDueAt(routineId: string, nextRunAt: Date): Promise<Date> {
await database
.update(routines)
.set({ nextRunAt })
.where(eq(routines.id, routineId));
const [row] = await database
.select({ nextRunAt: routines.nextRunAt })
.from(routines)
.where(eq(routines.id, routineId))
.limit(1);
// Read back rather than trusting the Date we wrote: the comparison the CAS makes is Postgres's.
return row?.nextRunAt as Date;
}
async function readRoutine(routineId: string) {
const [row] = await database
.select()
.from(routines)
.where(eq(routines.id, routineId))
.limit(1);
return row;
}
async function makeRoutine(instruction = "Summarise the day.") {
const { owner, agentId, channel } = await setUp();
const routine = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction,
cron: DAILY,
});
return { owner, agentId, channel, routine };
}
describe("moving a routine's clock", () => {
test("two sweeps racing on the same stamp advance it exactly once", async () => {
const { routine } = await makeRoutine();
const from = await makeDueAt(routine.id, new Date("2001-03-04T09:00:00Z"));
const outcomes = await Promise.all([
store.advanceNextRun(routine.id, from),
store.advanceNextRun(routine.id, from),
]);
expect(outcomes.filter((won) => won)).toHaveLength(1);
expect(outcomes.filter((won) => !won)).toHaveLength(1);
// The loser's re-read sees the advanced value: whichever call lost, the clock has moved on and
// nothing is left holding the stamp it was asked to move from.
const after = await readRoutine(routine.id);
expect(after?.nextRunAt.getTime()).toBeGreaterThan(from.getTime());
});
test("a stale `from` returns false and moves nothing", async () => {
const { routine } = await makeRoutine();
const from = await makeDueAt(routine.id, new Date("2001-03-04T09:00:00Z"));
const moved = await store.advanceNextRun(
routine.id,
new Date("2001-03-03T09:00:00Z"),
);
expect(moved).toBe(false);
const after = await readRoutine(routine.id);
expect(after?.nextRunAt.getTime()).toBe(from.getTime());
expect(after?.lastRunAt).toBeNull();
});
test("an explicit computeFrom drains a stale clock current in one move, CAS intact", async () => {
const { routine } = await makeRoutine();
const from = await makeDueAt(routine.id, new Date("2001-01-01T09:00:00Z"));
// The sweep's stale-backlog call: compare against the month-old stamp it read, but land on the
// occurrence after the moment the sweep is standing in — not one day along a 31-day drain.
const moved = await store.advanceNextRun(
routine.id,
from,
new Date("2001-02-01T10:00:00Z"),
);
expect(moved).toBe(true);
const after = await readRoutine(routine.id);
expect(after?.nextRunAt.toISOString()).toBe("2001-02-02T09:00:00.000Z");
// The bookmark still records the stamp the CAS compared against.
expect(after?.lastRunAt?.getTime()).toBe(from.getTime());
// And the CAS still guards: a second call holding the drained stamp moves nothing.
expect(
await store.advanceNextRun(
routine.id,
from,
new Date("2001-03-01T10:00:00Z"),
),
).toBe(false);
});
test("advancing stamps last_run_at with the `from` it was given", async () => {
const { routine } = await makeRoutine();
const from = await makeDueAt(routine.id, new Date("2001-03-04T09:00:00Z"));
expect(await store.advanceNextRun(routine.id, from)).toBe(true);
const after = await readRoutine(routine.id);
// The moment the routine was due, not the moment the sweep happened to look at it.
expect(after?.lastRunAt?.getTime()).toBe(from.getTime());
expect(after?.nextRunAt.getTime()).toBeGreaterThan(from.getTime());
});
});
describe("reading which routines are due", () => {
test("returns the due ones oldest first and respects the limit", async () => {
const { owner, agentId, channel } = await setUp();
const oldest = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Oldest.",
cron: DAILY,
});
const middle = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Middle.",
cron: DAILY,
});
const newest = await store.create({
ownerUserId: owner.id,
agentId,
channelId: channel.id,
instruction: "Newest.",
cron: DAILY,
});
await makeDueAt(oldest.id, new Date("2001-01-01T09:00:00Z"));
await makeDueAt(middle.id, new Date("2001-01-02T09:00:00Z"));
await makeDueAt(newest.id, new Date("2001-01-03T09:00:00Z"));
const all = await store.dueRoutines(10);
expect(all.map((row) => row.id).slice(0, 3)).toEqual([
oldest.id,
middle.id,
newest.id,
]);
expect(all[0]?.nextRunAt).toBeInstanceOf(Date);
const limited = await store.dueRoutines(2);
expect(limited.map((row) => row.id)).toEqual([oldest.id, middle.id]);
});
test("excludes a routine that is switched off", async () => {
const { owner, routine } = await makeRoutine();
await store.setEnabled(owner.id, routine.id, false);
await makeDueAt(routine.id, new Date("2001-01-01T09:00:00Z"));
const due = await store.dueRoutines(50);
expect(due.map((row) => row.id)).not.toContain(routine.id);
});
test("excludes a routine whose next run is still ahead", async () => {
const { routine } = await makeRoutine();
// The create path already put this in the future; the point is that nothing rounds it down.
const due = await store.dueRoutines(50);
expect(due.map((row) => row.id)).not.toContain(routine.id);
await makeDueAt(routine.id, new Date(Date.now() + 60 * 60 * 1000));
const stillAhead = await store.dueRoutines(50);
expect(stillAhead.map((row) => row.id)).not.toContain(routine.id);
});
});
describe("opening and closing a run", () => {
test("a finished run stamps finished_at and leaves the error null", async () => {
const { routine } = await makeRoutine();
const { runId } = await store.insertRun(routine.id);
const [inFlight] = await database
.select()
.from(routineRuns)
.where(eq(routineRuns.id, runId));
// Null status is the in-flight state, which is why the column is nullable.
expect(inFlight?.status).toBeNull();
expect(inFlight?.finishedAt).toBeNull();
expect(inFlight?.startedAt).toBeInstanceOf(Date);
await store.finishRun(runId, "succeeded");
const [finished] = await database
.select()
.from(routineRuns)
.where(eq(routineRuns.id, runId));
expect(finished?.status).toBe("succeeded");
expect(finished?.finishedAt).toBeInstanceOf(Date);
expect(finished?.error).toBeNull();
});
test("caps a long error rather than storing whatever a failure produced", async () => {
const { routine } = await makeRoutine();
const { runId } = await store.insertRun(routine.id);
await store.finishRun(runId, "failed", "x".repeat(600));
const [finished] = await database
.select()
.from(routineRuns)
.where(eq(routineRuns.id, runId));
expect(finished?.status).toBe("failed");
expect(finished?.error).toHaveLength(MAX_RUN_ERROR);
expect(MAX_RUN_ERROR).toBe(400);
});
test("finishing a run twice cannot relabel what the first finish recorded", async () => {
const { routine } = await makeRoutine();
const { runId } = await store.insertRun(routine.id);
await store.finishRun(runId, "succeeded");
const [firstFinish] = await database
.select()
.from(routineRuns)
.where(eq(routineRuns.id, runId));
// The natural next-task implementation: finish "succeeded", then something downstream throws,
// and the catch calls finishRun("failed", ...). Without a finish-once guard this would
// relabel a run that had already succeeded.
await store.finishRun(runId, "failed", "late");
const [secondFinish] = await database
.select()
.from(routineRuns)
.where(eq(routineRuns.id, runId));
expect(secondFinish?.status).toBe("succeeded");
expect(secondFinish?.error).toBeNull();
expect(secondFinish?.finishedAt?.getTime()).toBe(
firstFinish?.finishedAt?.getTime(),
);
});
test("the page's last run reads the newest of several", async () => {
const { owner, routine } = await makeRoutine();
const first = await store.insertRun(routine.id);
await store.finishRun(first.runId, "failed", "the first one");
const second = await store.insertRun(routine.id);
await store.finishRun(second.runId, "succeeded");
const [summary] = await store.listFor(owner.id);
expect(summary?.lastRun?.status).toBe("succeeded");
expect(summary?.lastRun?.finishedAt).toBeInstanceOf(Date);
});
});
/**
* A server that dies mid-turn strands its run row open forever: the work item was finished on the
* 202, so no retry ever comes back for that row, and nothing else writes it — the routines page
* reads "running now" for a run no process is running. `reapAbandonedRuns` is the sweep's mop for
* exactly that, and its age bound is what keeps it off live work: a young open row may be a turn
* some server is still running, and closing it would turn the real `finishRun` into a silent no-op.
*/
describe("reaping the runs the server never finished", () => {
/** Age one run row past the reaper's cutoff by hand; the reaper compares on the database's clock. */
async function ageRun(runId: string, byMs: number) {
await database
.update(routineRuns)
.set({ startedAt: new Date(Date.now() - byMs) })
.where(eq(routineRuns.id, runId));
}
test("closes only the rows past the cutoff, as skipped, and leaves live work alone", async () => {
const { routine } = await makeRoutine();
const abandoned = await store.insertRun(routine.id);
const inFlight = await store.insertRun(routine.id);
const finished = await store.insertRun(routine.id);
await store.finishRun(finished.runId, "succeeded");
await ageRun(abandoned.runId, 11 * 60_000);
const reaped = await store.reapAbandonedRuns(
10 * 60_000,
"the server did not finish this run; it may have restarted mid-turn",
);
// At least the row this test aged: the reaper is deliberately not routine-scoped, so a stray
// abandoned row from elsewhere in the database may be swept up in the same call.
expect(reaped).toBeGreaterThanOrEqual(1);
const rows = await database
.select()
.from(routineRuns)
.where(eq(routineRuns.routineId, routine.id));
const byId = new Map(rows.map((row) => [row.id, row]));
const abandonedRow = byId.get(abandoned.runId);
expect(abandonedRow?.status).toBe("skipped");
expect(abandonedRow?.finishedAt).toBeInstanceOf(Date);
expect(abandonedRow?.error).toContain("did not finish this run");
// The fresh open row is a turn some server may still be running: closing it would make the real
// finishRun a silent no-op, which is the exact overreach the age bound exists to prevent.
expect(byId.get(inFlight.runId)?.status).toBeNull();
// And a run with an outcome already has its truth; the reaper closes abandonment, not history.
expect(byId.get(finished.runId)?.status).toBe("succeeded");
});
test("caps the reason the same way finishRun does", async () => {
const { routine } = await makeRoutine();
const { runId } = await store.insertRun(routine.id);
await ageRun(runId, 11 * 60_000);
await store.reapAbandonedRuns(10 * 60_000, "x".repeat(600));
const [row] = await database
.select()
.from(routineRuns)
.where(eq(routineRuns.id, runId));
expect(row?.error).toHaveLength(MAX_RUN_ERROR);
});
});
/**
* The sweep's off switch for a routine whose own schedule refuses to advance — a cron written before
* a validation existed, or hand-edited under it. Left enabled, such a routine throws out of
* `advanceNextRun` on every pass forever: its clock never moves and it burns a due slot each time.
*/
describe("switching off a routine the sweep cannot schedule", () => {
test("disables it and leaves a skipped run carrying the reason where the page reads it", async () => {
const { owner, routine } = await makeRoutine();
await store.markUnschedulable(
routine.id,
"Routines may run at most every 15 minutes.",
);
const [summary] = await store.listFor(owner.id);
expect(summary?.enabled).toBe(false);
// The run row is the announcement: the sweep has no channel to say this in, so the page's
// last-run column is where the owner learns why their routine stopped.
expect(summary?.lastRun?.status).toBe("skipped");
const [row] = await database
.select()
.from(routineRuns)
.where(eq(routineRuns.routineId, routine.id));
expect(row?.error).toBe("Routines may run at most every 15 minutes.");
expect(row?.finishedAt).toBeInstanceOf(Date);
});
test("a skipped announcement neither counts as a failure nor breaks a streak", async () => {
const { routine } = await makeRoutine();
const { runId } = await store.insertRun(routine.id);
await store.finishRun(runId, "failed", "it threw");
await store.markUnschedulable(routine.id, "unschedulable");
// The fatigue rule reads through the announcement to the real failure tail.
expect(await store.consecutiveFailures(routine.id)).toBe(1);
});
test("switching a routine back on starts its failure count again", async () => {
const { owner, routine } = await makeRoutine();
for (let failure = 0; failure < 10; failure += 1) {
const { runId } = await store.insertRun(routine.id);
await store.finishRun(runId, "failed", "the integration is down");
}
expect(await store.consecutiveFailures(routine.id)).toBe(10);
// What the fatigue rule does at ten, and what the owner does once they have fixed the cause.
await store.setEnabled(owner.id, routine.id, false);
await store.setEnabled(owner.id, routine.id, true);
expect(await store.consecutiveFailures(routine.id)).toBe(0);
// The next failure is the first of a new streak, so it is announced as one rather than
// switching the routine straight back off as the eleventh in a row.
const { runId } = await store.insertRun(routine.id);
await store.finishRun(runId, "failed", "still down");
expect(await store.consecutiveFailures(routine.id)).toBe(1);
});
test("a routine deleted in the meantime is left alone", async () => {
const { owner, routine } = await makeRoutine();
await store.remove(owner.id, routine.id);
// Gone is gone: no throw, and no orphaned run row for a routine nobody can see.
await store.markUnschedulable(routine.id, "unschedulable");
expect(
await database
.select()
.from(routineRuns)
.where(eq(routineRuns.routineId, routine.id)),
).toEqual([]);
});
});
describe("the runner's read of one firing", () => {
test("joins the run to its routine, owner included", async () => {
const { owner, agentId, channel, routine } = await makeRoutine(
"Post the standup summary.",
);
const { runId } = await store.insertRun(routine.id);
// The owner comes back because the runner acts AS the owner: the channel it posts into and the
// thread it continues are that person's, and nothing else in the run row says who that is.
expect(await store.runContext(runId)).toEqual({
routineId: routine.id,
ownerUserId: owner.id,
agentId,
channelId: channel.id,
// Read at dispatch so a routine paused after it was queued still does not run.
enabled: true,
instruction: "Post the standup summary.",
});
});
test("a run id nothing wrote reads as no firing", async () => {
expect(await store.runContext("routine_run_missing")).toBeNull();
});
test("a deleted routine takes its queued firing with it", async () => {
const { owner, routine } = await makeRoutine();
const { runId } = await store.insertRun(routine.id);
await store.remove(owner.id, routine.id);
// Nobody's problem: the delete cascades, so there is no row left to finish and the runner has
// nothing to say to anybody.
expect(await store.runContext(runId)).toBeNull();
});
});
/**
* The consumer's re-read: "gone or disabled since the offer -> finish without dispatching" needs a
* by-id read that is not owner-scoped, because the id it is given comes from a work item's own
* payload, not from a person calling in with their own identity. `runContext` cannot answer this —
* it needs a run row to exist and does not report `enabled` — so this is a second, narrower read.
*/
describe("the sweep's by-id read of a routine before firing", () => {
test("an existing enabled routine reads back enabled", async () => {
const { routine } = await makeRoutine();
expect(await store.routineForFiring(routine.id)).toEqual({
id: routine.id,
enabled: true,
});
});
test("a disabled routine reads back disabled, not missing", async () => {
const { owner, routine } = await makeRoutine();
await store.setEnabled(owner.id, routine.id, false);
expect(await store.routineForFiring(routine.id)).toEqual({
id: routine.id,
enabled: false,
});
});
test("a deleted routine reads as no routine at all", async () => {
const { owner, routine } = await makeRoutine();
await store.remove(owner.id, routine.id);
expect(await store.routineForFiring(routine.id)).toBeNull();
});
});
/**
* The fatigue rule counts the failures at the tail, and this file pins what a `skipped` run does to
* that tail: A SKIP IS NOT A FAILURE AND DOES NOT BREAK THE STREAK. A skip means the channel was
* gone, not that the turn failed, so it is not counted; and it does not reset the count either,
* because a routine whose channel flaps must not be able to escape the fatigue rule by skipping
* between its failures.
*/
describe("counting the failures at the tail", () => {
async function finish(
routineId: string,
status: "succeeded" | "failed" | "skipped",
) {
const { runId } = await store.insertRun(routineId);
await store.finishRun(
runId,
status,
status === "failed" ? "it threw" : undefined,
);
}
test("counts up, resets on a success, and is not broken by a skip", async () => {
const { routine } = await makeRoutine();
expect(await store.consecutiveFailures(routine.id)).toBe(0);
await finish(routine.id, "succeeded");
expect(await store.consecutiveFailures(routine.id)).toBe(0);
await finish(routine.id, "failed");
expect(await store.consecutiveFailures(routine.id)).toBe(1);
await finish(routine.id, "failed");
await finish(routine.id, "failed");
expect(await store.consecutiveFailures(routine.id)).toBe(3);
await finish(routine.id, "succeeded");
expect(await store.consecutiveFailures(routine.id)).toBe(0);
});
test("a skip between failures neither counts nor resets", async () => {
const { routine } = await makeRoutine();
await finish(routine.id, "failed");
await finish(routine.id, "skipped");
await finish(routine.id, "failed");
expect(await store.consecutiveFailures(routine.id)).toBe(2);
});
test("an in-flight run is not an outcome", async () => {
const { routine } = await makeRoutine();
await finish(routine.id, "failed");
await store.insertRun(routine.id);
expect(await store.consecutiveFailures(routine.id)).toBe(1);
});
/**
* A routine with a flapping channel skips far more often than it fails. If the bounded window
* is filled with every finished run — skips included — the skips eat slots a failure needed,
* and a routine that has genuinely failed a dozen times in a row can look like it has failed
* only six or seven. That falsifies the fatigue rule for exactly the scenario the skip rule was
* written for: it would never reach the disable threshold. The window has to be filled with the
* rows the fatigue rule can act on, not with every finished row.
*/
test("skips do not dilute the bounded window the fatigue rule reads", async () => {
const { routine } = await makeRoutine();
// Twelve failures, each trailed by two skips, and nothing after them: a flapping channel's
// shape. Post-fix, all twelve failures are read (well under the 20-row limit) because skips
// never occupy a slot. Pre-fix, the 20-row window is filled with the interleaved skips too,
// so it only reaches back a handful of failures — short of the >=10 disable threshold.
for (let i = 0; i < 12; i++) {
await finish(routine.id, "failed");
await finish(routine.id, "skipped");
await finish(routine.id, "skipped");
}
expect(await store.consecutiveFailures(routine.id)).toBeGreaterThanOrEqual(
10,
);
});
});