A personal agent with a browser, terminal, files, and work that keeps going built with CopilotKit and AG-UI.
TypeScript
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { after, before, test } from "node:test";
import { createApp } from "../apps/server/src/app.ts";
import { createStore, type Store } from "../apps/server/src/db.ts";
import { LostLeaseError } from "../apps/server/src/engine/worker.ts";
import type { AgentNotification, AgentTask, Monitor } from "../packages/domain/src/agent.ts";
let db: Store, server: Awaited<ReturnType<typeof createApp>>, directory: string, token: string;
const owner = "local-user";
const headers = () => ({ Authorization: `Bearer ${token}`, "Content-Type": "application/json" });
const request = (path: string, body?: unknown) =>
server.app.request(`/api/agent${path}`, {
headers: headers(),
...(body === undefined ? {} : { method: "POST", body: JSON.stringify(body) }),
});
async function read<T>(path: string, body?: unknown, status = 200): Promise<T> {
const response = await request(path, body);
assert.equal(response.status, status, await response.clone().text());
return response.json();
}
type CompareAndSwap = Store["compareAndSwap"];
// Replaces the next tasks CAS that matches `when` with a lost write (null), then restores.
function loseNextTaskWrite(when: (patch: Record<string, unknown>) => boolean) {
const original: CompareAndSwap = db.compareAndSwap.bind(db);
let lost = 0;
db.compareAndSwap = (async (o, kind, id, expected, patch) => {
if (!lost && kind === "tasks" && when(patch)) {
lost++;
return null;
}
return original(o, kind, id, expected, patch);
}) as CompareAndSwap;
return {
get lost() {
return lost;
},
restore() {
db.compareAndSwap = original;
},
};
}
async function createMonitor(title: string) {
return read<Monitor>(
"/monitors",
{ title, url: "sample://availability", condition: "change", intervalMinutes: 1 },
201,
);
}
// Makes the sample page throw until the returned restore() is called.
function failPage() {
const originalGet = db.get.bind(db);
db.get = (async (o: string, kind: string, id: string) => {
if (kind === "sample-pages") throw new Error("Page unavailable");
return originalGet(o, kind, id);
}) as Store["get"];
return () => {
db.get = originalGet;
};
}
// Failures back off into the future; make the scheduled retry due now.
const makeDue = (taskId: string) =>
db.compareAndSwap(
owner,
"tasks",
taskId,
{ status: "scheduled" },
{ nextRunAt: new Date(0).toISOString() },
);
const maintain = () => (server.agent as unknown as { maintain(): Promise<void> }).maintain();
before(async () => {
directory = await mkdtemp(join(tmpdir(), "openmuse-monitor-recovery-"));
db = await createStore({ dataDir: join(directory, "db") });
server = await createApp(db, {
mode: "sample",
port: 8787,
host: "127.0.0.1",
publicUrl: "http://localhost:8787",
dataDir: directory,
agentBackend: "model",
intelligenceApiKey: "test-project-key-never-sent",
googleRedirectUri: "http://localhost:8787/api/google/callback",
allowedOrigins: ["http://localhost:8081"],
});
const session = await server.app.request("/api/session", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: "{}",
});
assert.equal(session.status, 200);
token = (await session.json()).token;
});
after(async () => {
await server?.agent?.stop();
await db?.close();
if (directory) await rm(directory, { recursive: true, force: true });
});
test("a page change stays alertable when the task outcome is lost after the baseline moves", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Recovery availability");
const notifications = async () =>
(await read<AgentNotification[]>("/notifications")).filter(
(item) => item.taskId === monitor.taskId,
);
await server.agent.worker.tick();
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");
assert.equal((await notifications()).length, 0);
await read("/sample-page", { text: "One table at 7 pm" });
await read(`/monitors/${monitor.id}/control`, { action: "check" });
// The worker's final checkpoint for this run loses its lease after observe() committed
// the new monitor baseline, as if the process stopped or another worker took over.
const fault = loseNextTaskWrite(
(patch) => patch.status === "scheduled" && patch.leaseId === null,
);
try {
await server.agent.worker.tick();
} finally {
fault.restore();
}
assert.equal(fault.lost, 1);
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");
// Recovery: the requeued run executes, then a later scheduled check runs as well.
await server.agent.worker.tick();
await read(`/monitors/${monitor.id}/control`, { action: "check" });
await server.agent.worker.tick();
const found = await notifications();
assert.equal(found.length, 1, "the No tables -> One table change should notify exactly once");
assert.match(found[0].body, /One table at 7 pm/);
});
for (const scenario of [
{
condition: "change",
value: "",
pages: ["Sold out", "Available now", "Sold out", "Available now"],
expectedAlerts: 3,
},
{
condition: "contains",
value: "available now",
pages: ["Sold out", "Available now", "Sold out", "Available now"],
expectedAlerts: 2,
},
{
condition: "price_below",
value: "50",
pages: ["Price: $80", "Price: $40", "Price: $80", "Price: $40"],
expectedAlerts: 2,
},
]) {
test(`${scenario.condition} alerts on recurring changes while deduplicating outcome replay`, async () => {
const monitor = await read<Monitor>(
"/monitors",
{
title: `Recurring ${scenario.condition}`,
url: "sample://availability",
condition: scenario.condition,
value: scenario.value,
intervalMinutes: 1,
},
201,
);
const alerts = async () =>
(await read<AgentNotification[]>("/notifications")).filter(
(item) => item.taskId === monitor.taskId,
);
for (const text of scenario.pages) {
await read("/sample-page", { text });
await read(`/monitors/${monitor.id}/control`, { action: "check" });
await server.agent.worker.tick();
// A restart can publish the same saved notice again; that is not a new alert.
await maintain();
await maintain();
}
assert.equal((await alerts()).length, scenario.expectedAlerts);
for (let i = 0; i < 2; i++) {
await read(`/monitors/${monitor.id}/control`, { action: "check" });
await server.agent.worker.tick();
}
assert.equal((await alerts()).length, scenario.expectedAlerts, "unchanged pages stay quiet");
await read(`/monitors/${monitor.id}/control`, { action: "stop" });
});
}
test("resuming a monitor fails without activating it when the task transition does not commit", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Resume availability");
await server.agent.worker.tick();
assert.equal(
(await read<Monitor>(`/monitors/${monitor.id}/control`, { action: "pause" })).status,
"paused",
);
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "paused");
// Another writer changes the task between the read and the CAS in controlMonitor().
const fault = loseNextTaskWrite((patch) => patch.status === "queued");
let response: Response;
try {
response = await request(`/monitors/${monitor.id}/control`, { action: "resume" });
} finally {
fault.restore();
}
assert.equal(fault.lost, 1);
assert.equal(response.status, 409, await response.clone().text());
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "paused");
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "paused");
});
test("the paused alert is delivered when the final failure outcome is lost", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Failing availability");
await server.agent.worker.tick();
const restorePage = failPage();
try {
for (let failure = 1; failure < 5; failure++) {
await makeDue(monitor.taskId);
await server.agent.worker.tick();
const task = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
assert.equal(task?.status, "scheduled");
assert.equal(task?.state.failures, failure);
}
await makeDue(monitor.taskId);
// The fifth failure's paused outcome is lost after the run already recorded the failure.
const fault = loseNextTaskWrite((patch) => patch.status === "paused" && patch.leaseId === null);
try {
await server.agent.worker.tick();
} finally {
fault.restore();
}
assert.equal(fault.lost, 1);
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");
await server.agent.worker.tick();
} finally {
restorePage();
}
const task = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
assert.equal(task?.status, "paused");
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "paused");
const pausedId = createHash("sha256")
.update(`watch-error:${monitor.taskId}:1:paused`)
.digest("hex");
const paused = (await read<AgentNotification[]>("/notifications")).filter(
(item) => item.id === pausedId,
);
assert.equal(paused.length, 1, "the watch paused alert should be delivered exactly once");
const resumed = await read<Monitor>(`/monitors/${monitor.id}/control`, { action: "resume" });
assert.equal(resumed.status, "active");
assert.equal(resumed.error, null);
const requeued = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
assert.equal(requeued?.status, "queued");
assert.equal(requeued?.state.failures, 0);
await server.agent.worker.tick();
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");
});
test("maintenance finishes a resume that stopped after the monitor was activated", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Interrupted resume");
await server.agent.worker.tick();
await read(`/monitors/${monitor.id}/control`, { action: "pause" });
const paused = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
assert.equal(paused?.status, "paused");
// The durable state left when a resume stops after activating the monitor, before queueing.
await db.compareAndSwap(
owner,
"tasks",
monitor.taskId,
{ status: "paused" },
{ state: { ...paused?.state, resumingMonitor: true } },
);
const current = await db.get<Monitor>(owner, "monitors", monitor.id);
assert.ok(current);
await db.put(owner, "monitors", { ...current, status: "active" });
await maintain();
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");
await server.agent.worker.tick();
const task = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
assert.equal(task?.status, "scheduled");
assert.equal(task?.state.resumingMonitor, false);
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");
});
test("checking a watch retries once when its run finishes during the request", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Check during run");
await server.agent.worker.tick();
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");
const fault = loseNextTaskWrite((patch) => patch.status === "queued");
let response: Response;
try {
response = await request(`/monitors/${monitor.id}/control`, { action: "check" });
} finally {
fault.restore();
}
assert.equal(fault.lost, 1);
assert.equal(response.status, 200, await response.clone().text());
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");
});
test("a resume during outcome publication is not paused again by the failure reconcile", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Resume during publication");
await server.agent.worker.tick();
// A failure pause has committed on the task, but the monitor has not been paused yet.
const task = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
assert.ok(task);
await db.compareAndSwap(
owner,
"tasks",
task.id,
{ status: "scheduled" },
{
status: "paused",
error: "Page unavailable",
state: {
...task.state,
failures: 5,
notice: { title: "Watch needs attention", body: "Page unavailable", key: "race-paused" },
},
},
);
await db.compareAndSwap(
owner,
"monitors",
monitor.id,
{ status: "active" },
{ error: "Page unavailable" },
);
// The user resumes while publishOutcome() is sending the paused notice.
const originalInsert = db.insertIfAbsent.bind(db);
let resuming: Promise<Response> | undefined;
db.insertIfAbsent = (async (o: string, kind: string, value: { id: string }) => {
if (!resuming && kind === "notifications") {
resuming = Promise.resolve(request(`/monitors/${monitor.id}/control`, { action: "resume" }));
await resuming;
}
return originalInsert(o, kind, value);
}) as Store["insertIfAbsent"];
try {
await (
server.agent as unknown as { publishOutcome(o: string, t: AgentTask): Promise<void> }
).publishOutcome(owner, task);
} finally {
db.insertIfAbsent = originalInsert;
}
const resumed = await resuming;
assert.equal(resumed?.status, 200, await resumed?.clone().text());
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");
});
test("a resume finished by maintenance during the request still succeeds", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Resume with maintenance");
await server.agent.worker.tick();
await read(`/monitors/${monitor.id}/control`, { action: "pause" });
// Maintenance runs right after the request activates the monitor, and queues the task first.
const originalPut = db.put.bind(db);
const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);
let fired = false;
const afterActivation = async (kind: string, status: unknown) => {
if (!fired && kind === "monitors" && status === "active") {
fired = true;
await maintain();
}
};
db.put = (async (o: string, kind: string, value: { id: string; status?: string }) => {
const saved = await originalPut(o, kind, value);
await afterActivation(kind, value.status);
return saved;
}) as Store["put"];
db.compareAndSwap = (async (o, kind, id, expected, patch) => {
const saved = await originalCas(o, kind, id, expected, patch);
if (saved) await afterActivation(kind, patch.status);
return saved;
}) as CompareAndSwap;
let response: Response;
try {
response = await request(`/monitors/${monitor.id}/control`, { action: "resume" });
} finally {
db.put = originalPut;
db.compareAndSwap = originalCas;
}
assert.ok(fired);
assert.equal(response.status, 200, await response.clone().text());
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");
});
test("two overlapping resume requests both leave the watch active", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Double resume");
await server.agent.worker.tick();
await read(`/monitors/${monitor.id}/control`, { action: "pause" });
// A second resume runs to completion right after the first one marks the task.
const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);
let second: Promise<Response> | undefined;
db.compareAndSwap = (async (o, kind, id, expected, patch) => {
const saved = await originalCas(o, kind, id, expected, patch);
const state = patch.state as { resumingMonitor?: boolean } | undefined;
if (!second && saved && kind === "tasks" && state?.resumingMonitor === true) {
second = Promise.resolve(request(`/monitors/${monitor.id}/control`, { action: "resume" }));
await second;
}
return saved;
}) as CompareAndSwap;
let first: Response;
try {
first = await request(`/monitors/${monitor.id}/control`, { action: "resume" });
} finally {
db.compareAndSwap = originalCas;
}
const secondResponse = await second;
assert.equal(secondResponse?.status, 200, await secondResponse?.clone().text());
assert.equal(first.status, 200, await first.clone().text());
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");
});
test("a leftover resume marker does not keep a failing watch running", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Leftover marker");
await server.agent.worker.tick();
// The task is paused directly, then a resume stops right after marking it.
await read(`/tasks/${monitor.taskId}/control`, { action: "pause" });
const paused = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
await db.compareAndSwap(
owner,
"tasks",
monitor.taskId,
{ status: "paused" },
{ state: { ...paused?.state, resumingMonitor: true } },
);
await read(`/tasks/${monitor.taskId}/control`, { action: "resume" });
const restorePage = failPage();
try {
await server.agent.worker.tick();
for (let failure = 2; failure <= 5; failure++) {
await makeDue(monitor.taskId);
await server.agent.worker.tick();
}
} finally {
restorePage();
}
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "paused");
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "paused");
await maintain();
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "paused");
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "paused");
});
test("a leftover resume marker does not undo a later task pause", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Leftover marker pause");
await server.agent.worker.tick();
await read(`/tasks/${monitor.taskId}/control`, { action: "pause" });
const paused = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
await db.compareAndSwap(
owner,
"tasks",
monitor.taskId,
{ status: "paused" },
{ state: { ...paused?.state, resumingMonitor: true } },
);
await read(`/tasks/${monitor.taskId}/control`, { action: "resume" });
await server.agent.worker.tick();
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");
// A later task pause stops after pausing the task, before it reaches the monitor.
await db.compareAndSwap(
owner,
"tasks",
monitor.taskId,
{ status: "scheduled" },
{ status: "paused", leaseId: null, leaseUntil: null },
);
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");
await maintain();
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "paused");
});
test("a stop during a resume keeps the watch stopped", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Stop during resume");
await server.agent.worker.tick();
await read(`/monitors/${monitor.id}/control`, { action: "pause" });
// The user stops the watch right after the resume marks the task.
const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);
let stopping: Promise<Response> | undefined;
db.compareAndSwap = (async (o, kind, id, expected, patch) => {
const saved = await originalCas(o, kind, id, expected, patch);
const state = patch.state as { resumingMonitor?: boolean } | undefined;
if (!stopping && saved && kind === "tasks" && state?.resumingMonitor === true) {
stopping = Promise.resolve(request(`/monitors/${monitor.id}/control`, { action: "stop" }));
await stopping;
}
return saved;
}) as CompareAndSwap;
let response: Response;
try {
response = await request(`/monitors/${monitor.id}/control`, { action: "resume" });
} finally {
db.compareAndSwap = originalCas;
}
assert.equal((await stopping)?.status, 200);
assert.equal(response.status, 409, await response.clone().text());
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "stopped");
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "cancelled");
});
test("a resume from the task screen is not paused again by a stale failure reconcile", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Task screen resume");
await server.agent.worker.tick();
const restorePage = failPage();
try {
for (let failure = 1; failure <= 5; failure++) {
await makeDue(monitor.taskId);
await server.agent.worker.tick();
}
} finally {
restorePage();
}
const stale = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
assert.equal(stale?.status, "paused");
assert.ok(stale?.error);
await read(`/tasks/${monitor.taskId}/control`, { action: "resume" });
// A failure reconcile that read the paused task before the resume finishes afterwards.
const originalGet = db.get.bind(db);
let served = false;
db.get = (async (o: string, kind: string, id: string) => {
if (!served && kind === "tasks" && id === monitor.taskId) {
served = true;
return stale;
}
return originalGet(o, kind, id);
}) as Store["get"];
try {
await (
server.agent as unknown as { publishOutcome(o: string, t: AgentTask): Promise<void> }
).publishOutcome(owner, stale);
} finally {
db.get = originalGet;
}
assert.ok(served);
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");
await server.agent.worker.tick();
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");
});
test("a stop during a check keeps the watch stopped", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Stop during check");
await server.agent.worker.tick();
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");
// The user stops the watch right before the check reactivates the monitor.
const originalPut = db.put.bind(db);
const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);
let stopping: Response | undefined;
const beforeActivation = async (kind: string, status: unknown) => {
if (!stopping && kind === "monitors" && status === "active") {
db.put = originalPut;
db.compareAndSwap = originalCas;
stopping = await request(`/monitors/${monitor.id}/control`, { action: "stop" });
}
};
db.put = (async (o: string, kind: string, value: { id: string; status?: string }) => {
await beforeActivation(kind, value.status);
return originalPut(o, kind, value);
}) as Store["put"];
db.compareAndSwap = (async (o, kind, id, expected, patch) => {
await beforeActivation(kind, patch.status);
return originalCas(o, kind, id, expected, patch);
}) as CompareAndSwap;
let response: Response;
try {
response = await request(`/monitors/${monitor.id}/control`, { action: "check" });
} finally {
db.put = originalPut;
db.compareAndSwap = originalCas;
}
assert.equal(stopping?.status, 200, await stopping?.clone().text());
assert.equal(response.status, 409, await response.clone().text());
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "stopped");
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "cancelled");
});
test("a check does not revive a task cancelled from the task screen", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Cancel during check");
await server.agent.worker.tick();
// The check arrives after the task is cancelled, before the cancel reaches the monitor.
const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);
let checking: Response | undefined;
db.compareAndSwap = (async (o, kind, id, expected, patch) => {
if (!checking && kind === "monitors" && patch.status === "stopped") {
db.compareAndSwap = originalCas;
checking = await request(`/monitors/${monitor.id}/control`, { action: "check" });
}
return originalCas(o, kind, id, expected, patch);
}) as CompareAndSwap;
try {
await read(`/tasks/${monitor.taskId}/control`, { action: "cancel" });
} finally {
db.compareAndSwap = originalCas;
}
assert.equal(checking?.status, 409, await checking?.clone().text());
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "stopped");
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "cancelled");
});
test("a check keeps the baseline from a run that finishes during the request", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Baseline during check");
await server.agent.worker.tick();
const before = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
assert.equal(before?.status, "scheduled");
await read("/sample-page", { text: "One table at 7 pm" });
// A whole scheduled run completes between the check's read and its requeue.
const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);
let ran: AgentTask | null = null;
db.compareAndSwap = (async (o, kind, id, expected, patch) => {
if (!ran && kind === "tasks" && patch.status === "queued" && expected.status === "scheduled") {
db.compareAndSwap = originalCas;
await makeDue(monitor.taskId);
await server.agent.worker.tick();
ran = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
}
return originalCas(o, kind, id, expected, patch);
}) as CompareAndSwap;
try {
await read(`/monitors/${monitor.id}/control`, { action: "check" });
} finally {
db.compareAndSwap = originalCas;
}
const finished = ran as AgentTask | null;
const afterRun = finished?.state.lastHash;
assert.equal(finished?.status, "scheduled");
assert.ok(afterRun);
assert.notEqual(afterRun, before?.state.lastHash);
const task = await db.get<AgentTask>(owner, "tasks", monitor.taskId);
assert.equal(task?.status, "queued");
assert.equal(task?.state.lastHash, afterRun);
});
test("resuming from Activity cannot let a worker claim the task before its monitor is active", async () => {
const monitor = await createMonitor("Activity resume with worker");
await server.agent.worker.tick();
await read(`/tasks/${monitor.taskId}/control`, { action: "pause" });
const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);
let ran = false;
db.compareAndSwap = (async (o, kind, id, expected, patch) => {
if (!ran && kind === "monitors" && id === monitor.id && patch.status === "active") {
ran = true;
// A separate worker can run between the request's two record updates.
await server.agent.worker.tick();
}
return originalCas(o, kind, id, expected, patch);
}) as CompareAndSwap;
try {
await read(`/tasks/${monitor.taskId}/control`, { action: "resume" });
} finally {
db.compareAndSwap = originalCas;
}
assert.ok(ran);
await maintain();
await server.agent.worker.tick();
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");
});
test("maintenance finishes an Activity resume interrupted after activating its monitor", async () => {
const monitor = await createMonitor("Interrupted Activity resume");
await server.agent.worker.tick();
await read(`/tasks/${monitor.taskId}/control`, { action: "pause" });
const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);
let interrupted = false;
db.compareAndSwap = (async (o, kind, id, expected, patch) => {
const saved = await originalCas(o, kind, id, expected, patch);
if (
!interrupted &&
saved &&
kind === "monitors" &&
id === monitor.id &&
patch.status === "active"
) {
interrupted = true;
throw new Error("Simulated interruption after activating the monitor");
}
return saved;
}) as CompareAndSwap;
try {
const response = await request(`/tasks/${monitor.taskId}/control`, { action: "resume" });
assert.equal(response.status, 502);
} finally {
db.compareAndSwap = originalCas;
}
assert.ok(interrupted);
await maintain();
await server.agent.worker.tick();
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");
});
test("a stop during an Activity resume cannot be undone by that resume", async () => {
const monitor = await createMonitor("Stop during Activity resume");
await server.agent.worker.tick();
await read(`/tasks/${monitor.taskId}/control`, { action: "pause" });
const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);
let stopping: Response | undefined;
db.compareAndSwap = (async (o, kind, id, expected, patch) => {
if (!stopping && kind === "monitors" && id === monitor.id && patch.status === "active") {
db.compareAndSwap = originalCas;
stopping = await request(`/monitors/${monitor.id}/control`, { action: "stop" });
}
return originalCas(o, kind, id, expected, patch);
}) as CompareAndSwap;
let response: Response;
try {
response = await request(`/tasks/${monitor.taskId}/control`, { action: "resume" });
} finally {
db.compareAndSwap = originalCas;
}
assert.equal(stopping?.status, 200);
assert.equal(response.status, 409);
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "stopped");
assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "cancelled");
});
test("a second observe from a stale snapshot loses its monitor commit instead of clobbering it", async () => {
await read("/sample-page", { text: "No tables available" });
const monitor = await createMonitor("Stale observe");
await server.agent.worker.tick();
const base = await db.get<Monitor>(owner, "monitors", monitor.id);
assert.equal(base?.checks, 1);
// Two workers read the same monitor snapshot (lease overlap), then both run
// observe() to completion. Freeze the monitor read at the pre-run snapshot.
const originalGet = db.get.bind(db);
db.get = (async (o: string, kind: string, id: string) => {
if (kind === "monitors" && id === monitor.id) return base;
return originalGet(o, kind, id);
}) as Store["get"];
const task = await originalGet<AgentTask>(owner, "tasks", monitor.taskId);
assert.ok(task);
const context = {
signal: new AbortController().signal,
guard: async () => {},
checkpoint: async (_patch: Partial<AgentTask>) => task,
event: async () => {},
};
const observe = (
server.agent as unknown as {
observe(owner: string, task: AgentTask, ctx: typeof context): Promise<Partial<AgentTask>>;
}
).observe.bind(server.agent);
try {
await observe(owner, task, context);
await assert.rejects(observe(owner, task, context), LostLeaseError);
} finally {
db.get = originalGet;
}
// The loser's commit was rejected: exactly one increment landed.
assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.checks, 2);
});
test("pausing a watch preserves a concurrent observe() commit", async () => {
const monitor = await createMonitor("Pause during observe");
// Simulate an observe() committing between the pause request's read and its
// write: the pause must move only the control fields, never clobber the commit.
const stale = (await db.get<Monitor>(owner, "monitors", monitor.id)) as Monitor;
await db.compareAndSwap(
owner,
"monitors",
monitor.id,
{},
{
checks: 41,
lastCheckedAt: new Date().toISOString(),
lastHash: "deadbeef",
lastValue: "committed value",
},
);
const originalGet: Store["get"] = db.get.bind(db);
let served = false;
db.get = (async (o: string, kind: string, key: string) => {
if (!served && kind === "monitors" && key === monitor.id) {
served = true;
return stale;
}
return originalGet(o, kind, key);
}) as Store["get"];
try {
await read<Monitor>(`/monitors/${monitor.id}/control`, { action: "pause" });
} finally {
db.get = originalGet;
}
const after = (await db.get<Monitor>(owner, "monitors", monitor.id)) as Monitor;
assert.equal(after.status, "paused");
assert.equal(after.checks, 41);
assert.equal(after.lastHash, "deadbeef");
assert.equal(after.lastValue, "committed value");
});