A personal agent with a browser, terminal, files, and work that keeps going built with CopilotKit and AG-UI.
TypeScript
fix: keep monitor state from committing ahead of its task outcome (#30) (#41)
Nathan 🔶 Tarbert6593991d8a3a
diff --git a/apps/server/src/engine/service.ts b/apps/server/src/engine/service.tsindex ce5f0f7..8b49d25 100644--- a/apps/server/src/engine/service.ts+++ b/apps/server/src/engine/service.ts@@ -271,7 +271,7 @@ export class AgentService {? "Paused. Resume when you're ready.": "",...(task.kind === "monitor" && action === "resume"- ? { state: { ...task.state, failures: 0, notice: null } }+ ? { state: { ...task.state, failures: 0, notice: null, resumingMonitor: false } }: {}),},);@@ -286,6 +286,8 @@ export class AgentService {{status: action === "cancel" ? "stopped" : action === "pause" ? "paused" : "active",nextCheckAt: date(),+ // Clearing the error fences out a failure reconcile that read the task before this.+ ...(action === "resume" || action === "retry" ? { error: null } : {}),},);if (action === "cancel" && task.actionId) {@@ -395,10 +397,26 @@ export class AgentService {return monitor;}private async activateMonitor(owner: string, monitor: Monitor) {- if (monitor.status !== "active") return;+ if (monitor.status !== "active") return null;const task = await this.getTask(owner, monitor.taskId);- if (task.status !== "paused" || !task.state.initializingMonitor) return;- await this.db.compareAndSwap(+ if (task.status !== "paused") return null;+ if (task.state.resumingMonitor)+ return this.db.compareAndSwap(+ owner,+ "tasks",+ task.id,+ { status: "paused", state: { resumingMonitor: true } },+ {+ status: "queued",+ nextRunAt: date(),+ leaseId: null,+ leaseUntil: null,+ error: null,+ state: { ...task.state, resumingMonitor: false, failures: 0, notice: null },+ },+ );+ if (!task.state.initializingMonitor) return null;+ return this.db.compareAndSwap(owner,"tasks",task.id,@@ -414,18 +432,79 @@ export class AgentService {if (!monitor) throw new AppError("Monitor not found", 404);if (monitor.status === "stopped" && action !== "stop")throw new AppError("Create a new watch to restart this stopped monitor", 409);- const status = action === "pause" ? "paused" : action === "stop" ? "stopped" : "active";- const saved = await this.db.put(owner, "monitors", { ...monitor, status, nextCheckAt: date() });- const task = await this.getTask(owner, monitor.taskId);- if (action === "pause" || action === "stop")+ if (action === "pause" || action === "stop") {+ const status = action === "pause" ? "paused" : "stopped";+ const saved = await this.db.put(owner, "monitors", {+ ...monitor,+ status,+ nextCheckAt: date(),+ });+ const task = await this.getTask(owner, monitor.taskId);await this.control(owner, task.id, action === "pause" ? "pause" : "cancel");- else {+ return saved;+ }+ let monitorStatus = monitor.status;+ for (let attempt = 0; attempt < 2; attempt++) {+ const task = await this.getTask(owner, monitor.taskId);+ if (task.status === "cancelled") break;+ if (task.status === "paused") {+ // Mark the paused task before activating the monitor so no worker can claim it in between.+ const marked = await this.db.compareAndSwap(+ owner,+ "tasks",+ task.id,+ { status: "paused", leaseId: task.leaseId ?? null },+ { state: { ...task.state, resumingMonitor: true } },+ );+ if (!marked) break;+ const activated = await this.db.compareAndSwap<Monitor>(+ owner,+ "monitors",+ id,+ { status: monitor.status },+ { status: "active", nextCheckAt: date(), error: null },+ );+ const saved = activated ?? (await this.db.get<Monitor>(owner, "monitors", id));+ if (saved?.status === "active" && (await this.activateMonitor(owner, saved))) return saved;+ const unmarked = await this.db.compareAndSwap(+ owner,+ "tasks",+ task.id,+ { status: "paused", state: { resumingMonitor: true } },+ { state: { ...task.state, resumingMonitor: false } },+ );+ // Another request or maintenance may have finished this resume first.+ if (!unmarked && saved?.status === "active") {+ const latest = await this.getTask(owner, task.id);+ if (["queued", "running", "scheduled"].includes(latest.status)) return saved;+ }+ if (activated)+ await this.db.compareAndSwap(+ owner,+ "monitors",+ id,+ { status: "active" },+ { status: monitor.status, error: monitor.error ?? null },+ );+ break;+ }+ // Only activate the monitor we read, so a concurrent stop is never undone.+ const saved = await this.db.compareAndSwap<Monitor>(+ owner,+ "monitors",+ id,+ { status: monitorStatus },+ { status: "active", nextCheckAt: date() },+ );+ if (!saved) break;+ monitorStatus = "active";this.worker.abort(task.id);- await this.db.compareAndSwap(+ const queued = await this.db.compareAndSwap(owner,"tasks",task.id,- { status: task.status, leaseId: task.leaseId ?? null },+ // updatedAt fences out a whole run finishing in between, which would move the baseline.+ { status: task.status, leaseId: task.leaseId ?? null, updatedAt: task.updatedAt },{status: "queued",nextRunAt: date(),@@ -435,8 +514,9 @@ export class AgentService {state: { ...task.state, failures: 0, notice: null },},);+ if (queued) return saved;}- return saved;+ throw new AppError("The watch changed while updating. Try again.", 409);}async refreshIdeas(owner: string) {const w = await this.workspace.snapshot(owner);@@ -683,11 +763,7 @@ export class AgentService {"monitors",String(task.input.monitorId),{ status: "active" },- {- error: detail,- nextCheckAt,- ...(failures >= 5 ? { status: "paused" } : {}),- },+ { error: detail, nextCheckAt },);await context.event("error",@@ -701,6 +777,7 @@ export class AgentService {state: {...task.state,failures,+ resumingMonitor: false,notice: {title: "Watch needs attention",body: detail,@@ -804,6 +881,15 @@ export class AgentService {.safeParse(task.state.notice);if ((task.status === "scheduled" || (task.status === "paused" && task.error)) && notice.success)await this.notify(owner, notice.data.title, notice.data.body, task.id, notice.data.key);+ // A watch pauses after repeated failures only once that task outcome has committed.+ if (task.kind === "monitor" && task.status === "paused" && task.error)+ await this.db.compareAndSwap(+ owner,+ "monitors",+ String(task.input.monitorId),+ { status: "active", error: task.error },+ { status: "paused" },+ );}private async document(owner: string,@@ -925,7 +1011,8 @@ export class AgentService {}const text = observation.text.replace(/\s+/g, " ").trim();const currentHash = hash(text);- const previousHash = monitor.lastHash;+ const previousHash =+ typeof task.state.lastHash === "string" ? task.state.lastHash : monitor.lastHash;const matched =monitor.condition === "change"? Boolean(previousHash && previousHash !== currentHash)@@ -970,6 +1057,8 @@ export class AgentService {state: {...task.state,sessionId: observation.sessionId,+ lastHash: currentHash,+ resumingMonitor: false,matched,failures: 0,notice: shouldNotifydiff --git a/tests/monitor-recovery.test.ts b/tests/monitor-recovery.test.tsnew file mode 100644index 0000000..023cc6f--- /dev/null+++ b/tests/monitor-recovery.test.ts@@ -0,0 +1,613 @@+import assert from "node:assert/strict";+import { createHash } from "node:crypto";+import { mkdtemp, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { after, before, test } from "node:test";+import { createApp } from "../apps/server/src/app.ts";+import { createStore, type Store } from "../apps/server/src/db.ts";+import type { AgentNotification, AgentTask, Monitor } from "../packages/domain/src/agent.ts";++let db: Store, server: Awaited<ReturnType<typeof createApp>>, directory: string, token: string;+const owner = "local-user";+const headers = () => ({ Authorization: `Bearer ${token}`, "Content-Type": "application/json" });+const request = (path: string, body?: unknown) =>+ server.app.request(`/api/agent${path}`, {+ headers: headers(),+ ...(body === undefined ? {} : { method: "POST", body: JSON.stringify(body) }),+ });+async function read<T>(path: string, body?: unknown, status = 200): Promise<T> {+ const response = await request(path, body);+ assert.equal(response.status, status, await response.clone().text());+ return response.json();+}+type CompareAndSwap = Store["compareAndSwap"];+// Replaces the next tasks CAS that matches `when` with a lost write (null), then restores.+function loseNextTaskWrite(when: (patch: Record<string, unknown>) => boolean) {+ const original: CompareAndSwap = db.compareAndSwap.bind(db);+ let lost = 0;+ db.compareAndSwap = (async (o, kind, id, expected, patch) => {+ if (!lost && kind === "tasks" && when(patch)) {+ lost++;+ return null;+ }+ return original(o, kind, id, expected, patch);+ }) as CompareAndSwap;+ return {+ get lost() {+ return lost;+ },+ restore() {+ db.compareAndSwap = original;+ },+ };+}+async function createMonitor(title: string) {+ return read<Monitor>(+ "/monitors",+ { title, url: "sample://availability", condition: "change", intervalMinutes: 1 },+ 201,+ );+}+// Makes the sample page throw until the returned restore() is called.+function failPage() {+ const originalGet = db.get.bind(db);+ db.get = (async (o: string, kind: string, id: string) => {+ if (kind === "sample-pages") throw new Error("Page unavailable");+ return originalGet(o, kind, id);+ }) as Store["get"];+ return () => {+ db.get = originalGet;+ };+}+// Failures back off into the future; make the scheduled retry due now.+const makeDue = (taskId: string) =>+ db.compareAndSwap(+ owner,+ "tasks",+ taskId,+ { status: "scheduled" },+ { nextRunAt: new Date(0).toISOString() },+ );+const maintain = () => (server.agent as unknown as { maintain(): Promise<void> }).maintain();++before(async () => {+ directory = await mkdtemp(join(tmpdir(), "openmuse-monitor-recovery-"));+ db = await createStore({ dataDir: join(directory, "db") });+ server = await createApp(db, {+ mode: "sample",+ port: 8787,+ host: "127.0.0.1",+ publicUrl: "http://localhost:8787",+ dataDir: directory,+ agentBackend: "model",+ intelligenceApiKey: "test-project-key-never-sent",+ googleRedirectUri: "http://localhost:8787/api/google/callback",+ allowedOrigins: ["http://localhost:8081"],+ });+ const session = await server.app.request("/api/session", {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: "{}",+ });+ assert.equal(session.status, 200);+ token = (await session.json()).token;+});+after(async () => {+ await server?.agent?.stop();+ await db?.close();+ if (directory) await rm(directory, { recursive: true, force: true });+});++test("a page change stays alertable when the task outcome is lost after the baseline moves", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Recovery availability");+ const notifications = async () =>+ (await read<AgentNotification[]>("/notifications")).filter(+ (item) => item.taskId === monitor.taskId,+ );+ await server.agent.worker.tick();+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");+ assert.equal((await notifications()).length, 0);++ await read("/sample-page", { text: "One table at 7 pm" });+ await read(`/monitors/${monitor.id}/control`, { action: "check" });+ // The worker's final checkpoint for this run loses its lease after observe() committed+ // the new monitor baseline, as if the process stopped or another worker took over.+ const fault = loseNextTaskWrite(+ (patch) => patch.status === "scheduled" && patch.leaseId === null,+ );+ try {+ await server.agent.worker.tick();+ } finally {+ fault.restore();+ }+ assert.equal(fault.lost, 1);+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");++ // Recovery: the requeued run executes, then a later scheduled check runs as well.+ await server.agent.worker.tick();+ await read(`/monitors/${monitor.id}/control`, { action: "check" });+ await server.agent.worker.tick();++ const found = await notifications();+ assert.equal(found.length, 1, "the No tables -> One table change should notify exactly once");+ assert.match(found[0].body, /One table at 7 pm/);+});++test("resuming a monitor fails without activating it when the task transition does not commit", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Resume availability");+ await server.agent.worker.tick();+ assert.equal(+ (await read<Monitor>(`/monitors/${monitor.id}/control`, { action: "pause" })).status,+ "paused",+ );+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "paused");++ // Another writer changes the task between the read and the CAS in controlMonitor().+ const fault = loseNextTaskWrite((patch) => patch.status === "queued");+ let response: Response;+ try {+ response = await request(`/monitors/${monitor.id}/control`, { action: "resume" });+ } finally {+ fault.restore();+ }+ assert.equal(fault.lost, 1);+ assert.equal(response.status, 409, await response.clone().text());+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "paused");+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "paused");+});++test("the paused alert is delivered when the final failure outcome is lost", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Failing availability");+ await server.agent.worker.tick();+ const restorePage = failPage();+ try {+ for (let failure = 1; failure < 5; failure++) {+ await makeDue(monitor.taskId);+ await server.agent.worker.tick();+ const task = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ assert.equal(task?.status, "scheduled");+ assert.equal(task?.state.failures, failure);+ }+ await makeDue(monitor.taskId);+ // The fifth failure's paused outcome is lost after the run already recorded the failure.+ const fault = loseNextTaskWrite((patch) => patch.status === "paused" && patch.leaseId === null);+ try {+ await server.agent.worker.tick();+ } finally {+ fault.restore();+ }+ assert.equal(fault.lost, 1);+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");+ await server.agent.worker.tick();+ } finally {+ restorePage();+ }++ const task = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ assert.equal(task?.status, "paused");+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "paused");+ const pausedId = createHash("sha256")+ .update(`watch-error:${monitor.taskId}:paused`)+ .digest("hex");+ const paused = (await read<AgentNotification[]>("/notifications")).filter(+ (item) => item.id === pausedId,+ );+ assert.equal(paused.length, 1, "the watch paused alert should be delivered exactly once");++ const resumed = await read<Monitor>(`/monitors/${monitor.id}/control`, { action: "resume" });+ assert.equal(resumed.status, "active");+ assert.equal(resumed.error, null);+ const requeued = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ assert.equal(requeued?.status, "queued");+ assert.equal(requeued?.state.failures, 0);+ await server.agent.worker.tick();+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");+});++test("maintenance finishes a resume that stopped after the monitor was activated", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Interrupted resume");+ await server.agent.worker.tick();+ await read(`/monitors/${monitor.id}/control`, { action: "pause" });+ const paused = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ assert.equal(paused?.status, "paused");++ // The durable state left when a resume stops after activating the monitor, before queueing.+ await db.compareAndSwap(+ owner,+ "tasks",+ monitor.taskId,+ { status: "paused" },+ { state: { ...paused?.state, resumingMonitor: true } },+ );+ const current = await db.get<Monitor>(owner, "monitors", monitor.id);+ assert.ok(current);+ await db.put(owner, "monitors", { ...current, status: "active" });++ await maintain();+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");+ await server.agent.worker.tick();+ const task = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ assert.equal(task?.status, "scheduled");+ assert.equal(task?.state.resumingMonitor, false);+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");+});++test("checking a watch retries once when its run finishes during the request", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Check during run");+ await server.agent.worker.tick();+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");++ const fault = loseNextTaskWrite((patch) => patch.status === "queued");+ let response: Response;+ try {+ response = await request(`/monitors/${monitor.id}/control`, { action: "check" });+ } finally {+ fault.restore();+ }+ assert.equal(fault.lost, 1);+ assert.equal(response.status, 200, await response.clone().text());+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");+});++test("a resume during outcome publication is not paused again by the failure reconcile", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Resume during publication");+ await server.agent.worker.tick();+ // A failure pause has committed on the task, but the monitor has not been paused yet.+ const task = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ assert.ok(task);+ await db.compareAndSwap(+ owner,+ "tasks",+ task.id,+ { status: "scheduled" },+ {+ status: "paused",+ error: "Page unavailable",+ state: {+ ...task.state,+ failures: 5,+ notice: { title: "Watch needs attention", body: "Page unavailable", key: "race-paused" },+ },+ },+ );+ await db.compareAndSwap(+ owner,+ "monitors",+ monitor.id,+ { status: "active" },+ { error: "Page unavailable" },+ );++ // The user resumes while publishOutcome() is sending the paused notice.+ const originalInsert = db.insertIfAbsent.bind(db);+ let resuming: Promise<Response> | undefined;+ db.insertIfAbsent = (async (o: string, kind: string, value: { id: string }) => {+ if (!resuming && kind === "notifications") {+ resuming = Promise.resolve(request(`/monitors/${monitor.id}/control`, { action: "resume" }));+ await resuming;+ }+ return originalInsert(o, kind, value);+ }) as Store["insertIfAbsent"];+ try {+ await (+ server.agent as unknown as { publishOutcome(o: string, t: AgentTask): Promise<void> }+ ).publishOutcome(owner, task);+ } finally {+ db.insertIfAbsent = originalInsert;+ }+ const resumed = await resuming;+ assert.equal(resumed?.status, 200, await resumed?.clone().text());+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");+});++test("a resume finished by maintenance during the request still succeeds", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Resume with maintenance");+ await server.agent.worker.tick();+ await read(`/monitors/${monitor.id}/control`, { action: "pause" });++ // Maintenance runs right after the request activates the monitor, and queues the task first.+ const originalPut = db.put.bind(db);+ const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);+ let fired = false;+ const afterActivation = async (kind: string, status: unknown) => {+ if (!fired && kind === "monitors" && status === "active") {+ fired = true;+ await maintain();+ }+ };+ db.put = (async (o: string, kind: string, value: { id: string; status?: string }) => {+ const saved = await originalPut(o, kind, value);+ await afterActivation(kind, value.status);+ return saved;+ }) as Store["put"];+ db.compareAndSwap = (async (o, kind, id, expected, patch) => {+ const saved = await originalCas(o, kind, id, expected, patch);+ if (saved) await afterActivation(kind, patch.status);+ return saved;+ }) as CompareAndSwap;+ let response: Response;+ try {+ response = await request(`/monitors/${monitor.id}/control`, { action: "resume" });+ } finally {+ db.put = originalPut;+ db.compareAndSwap = originalCas;+ }+ assert.ok(fired);+ assert.equal(response.status, 200, await response.clone().text());+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");+});++test("two overlapping resume requests both leave the watch active", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Double resume");+ await server.agent.worker.tick();+ await read(`/monitors/${monitor.id}/control`, { action: "pause" });++ // A second resume runs to completion right after the first one marks the task.+ const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);+ let second: Promise<Response> | undefined;+ db.compareAndSwap = (async (o, kind, id, expected, patch) => {+ const saved = await originalCas(o, kind, id, expected, patch);+ const state = patch.state as { resumingMonitor?: boolean } | undefined;+ if (!second && saved && kind === "tasks" && state?.resumingMonitor === true) {+ second = Promise.resolve(request(`/monitors/${monitor.id}/control`, { action: "resume" }));+ await second;+ }+ return saved;+ }) as CompareAndSwap;+ let first: Response;+ try {+ first = await request(`/monitors/${monitor.id}/control`, { action: "resume" });+ } finally {+ db.compareAndSwap = originalCas;+ }+ const secondResponse = await second;+ assert.equal(secondResponse?.status, 200, await secondResponse?.clone().text());+ assert.equal(first.status, 200, await first.clone().text());+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "queued");+});++test("a leftover resume marker does not keep a failing watch running", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Leftover marker");+ await server.agent.worker.tick();+ // The task is paused directly, then a resume stops right after marking it.+ await read(`/tasks/${monitor.taskId}/control`, { action: "pause" });+ const paused = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ await db.compareAndSwap(+ owner,+ "tasks",+ monitor.taskId,+ { status: "paused" },+ { state: { ...paused?.state, resumingMonitor: true } },+ );+ await read(`/tasks/${monitor.taskId}/control`, { action: "resume" });++ const restorePage = failPage();+ try {+ await server.agent.worker.tick();+ for (let failure = 2; failure <= 5; failure++) {+ await makeDue(monitor.taskId);+ await server.agent.worker.tick();+ }+ } finally {+ restorePage();+ }+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "paused");+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "paused");+ await maintain();+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "paused");+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "paused");+});++test("a leftover resume marker does not undo a later task pause", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Leftover marker pause");+ await server.agent.worker.tick();+ await read(`/tasks/${monitor.taskId}/control`, { action: "pause" });+ const paused = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ await db.compareAndSwap(+ owner,+ "tasks",+ monitor.taskId,+ { status: "paused" },+ { state: { ...paused?.state, resumingMonitor: true } },+ );+ await read(`/tasks/${monitor.taskId}/control`, { action: "resume" });+ await server.agent.worker.tick();+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");++ // A later task pause stops after pausing the task, before it reaches the monitor.+ await db.compareAndSwap(+ owner,+ "tasks",+ monitor.taskId,+ { status: "scheduled" },+ { status: "paused", leaseId: null, leaseUntil: null },+ );+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");+ await maintain();+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "paused");+});++test("a stop during a resume keeps the watch stopped", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Stop during resume");+ await server.agent.worker.tick();+ await read(`/monitors/${monitor.id}/control`, { action: "pause" });++ // The user stops the watch right after the resume marks the task.+ const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);+ let stopping: Promise<Response> | undefined;+ db.compareAndSwap = (async (o, kind, id, expected, patch) => {+ const saved = await originalCas(o, kind, id, expected, patch);+ const state = patch.state as { resumingMonitor?: boolean } | undefined;+ if (!stopping && saved && kind === "tasks" && state?.resumingMonitor === true) {+ stopping = Promise.resolve(request(`/monitors/${monitor.id}/control`, { action: "stop" }));+ await stopping;+ }+ return saved;+ }) as CompareAndSwap;+ let response: Response;+ try {+ response = await request(`/monitors/${monitor.id}/control`, { action: "resume" });+ } finally {+ db.compareAndSwap = originalCas;+ }+ assert.equal((await stopping)?.status, 200);+ assert.equal(response.status, 409, await response.clone().text());+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "stopped");+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "cancelled");+});++test("a resume from the task screen is not paused again by a stale failure reconcile", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Task screen resume");+ await server.agent.worker.tick();+ const restorePage = failPage();+ try {+ for (let failure = 1; failure <= 5; failure++) {+ await makeDue(monitor.taskId);+ await server.agent.worker.tick();+ }+ } finally {+ restorePage();+ }+ const stale = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ assert.equal(stale?.status, "paused");+ assert.ok(stale?.error);+ await read(`/tasks/${monitor.taskId}/control`, { action: "resume" });++ // A failure reconcile that read the paused task before the resume finishes afterwards.+ const originalGet = db.get.bind(db);+ let served = false;+ db.get = (async (o: string, kind: string, id: string) => {+ if (!served && kind === "tasks" && id === monitor.taskId) {+ served = true;+ return stale;+ }+ return originalGet(o, kind, id);+ }) as Store["get"];+ try {+ await (+ server.agent as unknown as { publishOutcome(o: string, t: AgentTask): Promise<void> }+ ).publishOutcome(owner, stale);+ } finally {+ db.get = originalGet;+ }+ assert.ok(served);+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");+ await server.agent.worker.tick();+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "active");+});++test("a stop during a check keeps the watch stopped", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Stop during check");+ await server.agent.worker.tick();+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "scheduled");++ // The user stops the watch right before the check reactivates the monitor.+ const originalPut = db.put.bind(db);+ const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);+ let stopping: Response | undefined;+ const beforeActivation = async (kind: string, status: unknown) => {+ if (!stopping && kind === "monitors" && status === "active") {+ db.put = originalPut;+ db.compareAndSwap = originalCas;+ stopping = await request(`/monitors/${monitor.id}/control`, { action: "stop" });+ }+ };+ db.put = (async (o: string, kind: string, value: { id: string; status?: string }) => {+ await beforeActivation(kind, value.status);+ return originalPut(o, kind, value);+ }) as Store["put"];+ db.compareAndSwap = (async (o, kind, id, expected, patch) => {+ await beforeActivation(kind, patch.status);+ return originalCas(o, kind, id, expected, patch);+ }) as CompareAndSwap;+ let response: Response;+ try {+ response = await request(`/monitors/${monitor.id}/control`, { action: "check" });+ } finally {+ db.put = originalPut;+ db.compareAndSwap = originalCas;+ }+ assert.equal(stopping?.status, 200, await stopping?.clone().text());+ assert.equal(response.status, 409, await response.clone().text());+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "stopped");+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "cancelled");+});++test("a check does not revive a task cancelled from the task screen", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Cancel during check");+ await server.agent.worker.tick();++ // The check arrives after the task is cancelled, before the cancel reaches the monitor.+ const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);+ let checking: Response | undefined;+ db.compareAndSwap = (async (o, kind, id, expected, patch) => {+ if (!checking && kind === "monitors" && patch.status === "stopped") {+ db.compareAndSwap = originalCas;+ checking = await request(`/monitors/${monitor.id}/control`, { action: "check" });+ }+ return originalCas(o, kind, id, expected, patch);+ }) as CompareAndSwap;+ try {+ await read(`/tasks/${monitor.taskId}/control`, { action: "cancel" });+ } finally {+ db.compareAndSwap = originalCas;+ }+ assert.equal(checking?.status, 409, await checking?.clone().text());+ assert.equal((await db.get<Monitor>(owner, "monitors", monitor.id))?.status, "stopped");+ assert.equal((await db.get<AgentTask>(owner, "tasks", monitor.taskId))?.status, "cancelled");+});++test("a check keeps the baseline from a run that finishes during the request", async () => {+ await read("/sample-page", { text: "No tables available" });+ const monitor = await createMonitor("Baseline during check");+ await server.agent.worker.tick();+ const before = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ assert.equal(before?.status, "scheduled");+ await read("/sample-page", { text: "One table at 7 pm" });++ // A whole scheduled run completes between the check's read and its requeue.+ const originalCas: CompareAndSwap = db.compareAndSwap.bind(db);+ let ran: AgentTask | null = null;+ db.compareAndSwap = (async (o, kind, id, expected, patch) => {+ if (!ran && kind === "tasks" && patch.status === "queued" && expected.status === "scheduled") {+ db.compareAndSwap = originalCas;+ await makeDue(monitor.taskId);+ await server.agent.worker.tick();+ ran = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ }+ return originalCas(o, kind, id, expected, patch);+ }) as CompareAndSwap;+ try {+ await read(`/monitors/${monitor.id}/control`, { action: "check" });+ } finally {+ db.compareAndSwap = originalCas;+ }+ const finished = ran as AgentTask | null;+ const afterRun = finished?.state.lastHash;+ assert.equal(finished?.status, "scheduled");+ assert.ok(afterRun);+ assert.notEqual(afterRun, before?.state.lastHash);+ const task = await db.get<AgentTask>(owner, "tasks", monitor.taskId);+ assert.equal(task?.status, "queued");+ assert.equal(task?.state.lastHash, afterRun);+});