Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): prevent duplicate monitor finalization (#4542)

Gergő Móricz6616c0f2efd0
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 674d3dfc4..6f4445045 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -218,6 +218,16 @@ jobs:
env:
npm_config_ignore_scripts: 'true'
+ - name: Run monitor finalization regression tests
+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
+ run: >-
+ pnpm exec vitest run
+ src/services/monitoring/runner.finalization.test.ts
+ src/services/monitoring/store.finalization.test.ts
+ src/services/monitoring/scheduler.test.ts
+ --reporter=default
+ working-directory: apps/api
+
- name: Install test site dependencies
run: pnpm install --frozen-lockfile
working-directory: apps/test-site
diff --git a/apps/api/src/services/monitoring/runner.finalization.test.ts b/apps/api/src/services/monitoring/runner.finalization.test.ts
new file mode 100644
index 000000000..25bd42853
--- /dev/null
+++ b/apps/api/src/services/monitoring/runner.finalization.test.ts
@@ -0,0 +1,748 @@
+import type { MonitorCheckRow, MonitorRow } from "./types";
+
+vi.mock("../../config", () => ({ config: { USE_DB_AUTHENTICATION: true } }));
+vi.mock("../../lib/logger", () => {
+ const logger = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ child: vi.fn(),
+ };
+ logger.child.mockReturnValue(logger);
+ return { logger };
+});
+vi.mock("../logging/log_job", () => ({}));
+vi.mock("../../lib/gcs-monitoring", () => ({}));
+vi.mock("../worker/scrape-worker", () => ({}));
+vi.mock("../worker/nuq-router", () => ({}));
+vi.mock("./diff", () => ({}));
+vi.mock("../../lib/crawl-redis", () => ({}));
+vi.mock("../queue-jobs", () => ({}));
+vi.mock("../../controllers/v2/types", () => ({}));
+vi.mock("../webhook", () => ({}));
+vi.mock("./results", () => ({}));
+vi.mock("../notification/monitoring_email", () => ({}));
+vi.mock("../notification/monitoring_slack", () => ({}));
+vi.mock("./types", () => ({}));
+vi.mock("./interest", () => ({
+ trackMonitorCheckStartedInterest: async () => {},
+}));
+vi.mock("./search/run", () => ({}));
+vi.mock("./search/judge", () => ({}));
+vi.mock("./search/dedupe", () => ({}));
+vi.mock("./search/persist", () => ({}));
+vi.mock("../../scraper/WebScraper/utils/blocklist", () => ({}));
+vi.mock("../../controllers/auth", () => ({}));
+vi.mock("./store", () => ({
+ getMonitorCheckForUpdate: vi.fn(),
+ getMonitorForUpdate: vi.fn(),
+ listRunningMonitorChecks: vi.fn(),
+ updateMonitorCheck: vi.fn(),
+ updateMonitorCheckIfRunning: vi.fn(),
+ updateMonitorCheckIfStatus: vi.fn(),
+ markMonitorRunning: vi.fn(),
+ countMonitorCheckPages: vi.fn(),
+ calculateMonitorCheckActualCredits: vi.fn(),
+ updateMonitorScheduleAfterRun: vi.fn(),
+}));
+vi.mock("../autumn/autumn.service", () => ({
+ autumnService: { finalizeCreditsLock: vi.fn(), lockCredits: vi.fn() },
+}));
+vi.mock("../queue-service", () => ({ getBillingQueue: vi.fn() }));
+vi.mock("../redis", () => ({
+ redisEvictConnection: { set: vi.fn(), del: vi.fn(), eval: vi.fn() },
+}));
+
+import {
+ processMonitorCheckJob,
+ reconcileRunningMonitorChecks,
+} from "./runner";
+import * as store from "./store";
+import { autumnService } from "../autumn/autumn.service";
+import { getBillingQueue } from "../queue-service";
+import { redisEvictConnection } from "../redis";
+
+function deferred() {
+ let resolve!: () => void;
+ const promise = new Promise<void>(r => {
+ resolve = r;
+ });
+ return { promise, resolve };
+}
+
+describe("monitor check finalization ownership", () => {
+ let current: MonitorCheckRow;
+ let snapshot: MonitorCheckRow;
+ let monitor: MonitorRow;
+ const locks = new Map<string, string>();
+ const bill = vi.fn();
+ const lockKey = "monitor-check-finalize:check-1";
+
+ beforeEach(() => {
+ vi.resetAllMocks();
+ locks.clear();
+ current = {
+ id: "check-1",
+ monitor_id: "monitor-1",
+ team_id: "team-1",
+ status: "running",
+ billing_status: "reserved",
+ autumn_lock_id: "monitor_check-1",
+ reserved_credits: 1,
+ started_at: new Date().toISOString(),
+ target_results: [
+ { targetId: "target-1", type: "scrape", expectedJobs: ["scrape-1"] },
+ ],
+ } as MonitorCheckRow;
+ snapshot = structuredClone(current);
+ monitor = {
+ id: current.monitor_id,
+ team_id: current.team_id,
+ current_check_id: current.id,
+ targets: [
+ { id: "target-1", type: "scrape", urls: ["https://example.com"] },
+ ],
+ } as MonitorRow;
+ vi.mocked(store.listRunningMonitorChecks).mockImplementation(async () => [
+ structuredClone(snapshot),
+ ]);
+ vi.mocked(store.getMonitorCheckForUpdate).mockImplementation(async () =>
+ structuredClone(current),
+ );
+ vi.mocked(store.getMonitorForUpdate).mockImplementation(
+ async () => monitor,
+ );
+ vi.mocked(store.updateMonitorCheck).mockImplementation(
+ async (_id, patch) => {
+ current = { ...current, ...patch };
+ return structuredClone(current);
+ },
+ );
+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
+ async (_id, patch) => {
+ if (current.status !== "running") return null;
+ current = { ...current, ...patch };
+ return structuredClone(current);
+ },
+ );
+ vi.mocked(store.updateMonitorCheckIfStatus).mockImplementation(
+ async (_id, expectedStatus, patch) => {
+ if (current.status !== expectedStatus) return null;
+ current = { ...current, ...patch };
+ return structuredClone(current);
+ },
+ );
+ vi.mocked(store.countMonitorCheckPages).mockImplementation(
+ async ({ status }) => (!status || status === "same" ? 1 : 0),
+ );
+ vi.mocked(store.calculateMonitorCheckActualCredits).mockResolvedValue(1);
+ vi.mocked(autumnService.finalizeCreditsLock).mockResolvedValue(true);
+ vi.mocked(getBillingQueue).mockReturnValue({
+ add: bill,
+ } as unknown as ReturnType<typeof getBillingQueue>);
+ vi.mocked(redisEvictConnection.set).mockImplementation((async (
+ key: string,
+ token: string,
+ ) => {
+ // Notifications are tested independently; this suite exercises settlement and scheduling.
+ if (key.startsWith("monitor-check-notify:")) return null;
+ if (locks.has(key)) return null;
+ locks.set(key, token);
+ return "OK";
+ }) as any);
+ vi.mocked(redisEvictConnection.del).mockImplementation((async (
+ key: string,
+ ) => Number(locks.delete(key))) as any);
+ vi.mocked(redisEvictConnection.eval).mockImplementation((async (
+ _script: string,
+ _keys: number,
+ key: string,
+ token: string,
+ ) => {
+ if (locks.get(key) !== token) return 0;
+ return Number(locks.delete(key));
+ }) as any);
+ });
+
+ it("settles and schedules a completed check once when another batch retained its running snapshot", async () => {
+ await reconcileRunningMonitorChecks();
+ await reconcileRunningMonitorChecks();
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "confirmed",
+ });
+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledTimes(1);
+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledWith(
+ expect.objectContaining({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ overrideValue: 1,
+ }),
+ );
+ expect(bill).toHaveBeenCalledTimes(1);
+ expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);
+ });
+
+ it("uses the refreshed target state instead of completing an obsolete snapshot", async () => {
+ current.target_results = [
+ { type: "search", targetId: "target-1", searchCompleted: false },
+ ];
+ await reconcileRunningMonitorChecks();
+ expect(current.status).toBe("running");
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+ });
+
+ it.each(["completed", "partial", "failed"] as const)(
+ "leaves an already %s check untouched",
+ async status => {
+ current.status = status;
+ current.billing_status = "confirmed";
+ await reconcileRunningMonitorChecks();
+ expect(store.updateMonitorCheck).not.toHaveBeenCalled();
+ expect(store.updateMonitorCheckIfRunning).not.toHaveBeenCalled();
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+ },
+ );
+
+ it("skips a deleted check and releases its Redis lock", async () => {
+ vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValue(null);
+ await reconcileRunningMonitorChecks();
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+ expect(locks.has(lockKey)).toBe(false);
+ });
+
+ it("allows only one settlement when the Redis lease expires while a worker is still preparing completion", async () => {
+ const entered = deferred();
+ const resume = deferred();
+ vi.mocked(store.calculateMonitorCheckActualCredits).mockImplementationOnce(
+ async () => {
+ entered.resolve();
+ await resume.promise;
+ return 1;
+ },
+ );
+ const first = reconcileRunningMonitorChecks();
+ await entered.promise;
+ locks.delete(lockKey); // The second worker can acquire an expired lease.
+ await reconcileRunningMonitorChecks();
+ resume.resolve();
+ await first;
+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledTimes(1);
+ expect(bill).toHaveBeenCalledTimes(1);
+ expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);
+ expect(current.billing_status).toBe("confirmed");
+ });
+
+ it("does not delete a replacement owner's Redis lock", async () => {
+ vi.mocked(store.calculateMonitorCheckActualCredits).mockImplementation(
+ async () => {
+ locks.set(lockKey, "replacement-owner");
+ return 1;
+ },
+ );
+ await reconcileRunningMonitorChecks();
+ expect(locks.get(lockKey)).toBe("replacement-owner");
+ });
+
+ it.each(["stale", "orphan"])(
+ "does not release a %s check whose terminal claim was won by another worker",
+ async kind => {
+ if (kind === "stale") {
+ current.started_at = new Date(
+ Date.now() - 2 * 60 * 60 * 1000,
+ ).toISOString();
+ snapshot = structuredClone(current);
+ } else {
+ vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);
+ }
+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
+ async () => {
+ current = {
+ ...current,
+ status: "completed",
+ billing_status: "confirmed",
+ };
+ return null;
+ },
+ );
+ await reconcileRunningMonitorChecks();
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+ expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "confirmed",
+ });
+ },
+ );
+
+ it("records a failed settlement without enqueueing billing", async () => {
+ vi.mocked(autumnService.finalizeCreditsLock).mockResolvedValue(false);
+ await reconcileRunningMonitorChecks();
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "failed",
+ });
+ expect(bill).not.toHaveBeenCalled();
+ });
+
+ it.each(["stale", "orphan"])(
+ "releases a %s hold after winning the failure transition",
+ async kind => {
+ if (kind === "stale") {
+ current.started_at = new Date(
+ Date.now() - 2 * 60 * 60 * 1000,
+ ).toISOString();
+ } else {
+ vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);
+ }
+ await reconcileRunningMonitorChecks();
+ expect(current).toMatchObject({
+ status: "failed",
+ billing_status: "released",
+ });
+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ action: "release",
+ lockId: "monitor_check-1",
+ }),
+ );
+ expect(
+ vi.mocked(store.updateMonitorCheckIfRunning).mock
+ .invocationCallOrder[0],
+ ).toBeLessThan(
+ vi.mocked(autumnService.finalizeCreditsLock).mock
+ .invocationCallOrder[0],
+ );
+ },
+ );
+
+ it.each([
+ { kind: "stale", throws: false },
+ { kind: "stale", throws: true },
+ { kind: "orphan", throws: false },
+ { kind: "orphan", throws: true },
+ { kind: "handler", throws: false },
+ { kind: "handler", throws: true },
+ ])(
+ "records failed settlement for a $kind release that throws: $throws",
+ async ({ kind, throws }) => {
+ if (throws) {
+ vi.mocked(autumnService.finalizeCreditsLock).mockRejectedValue(
+ new Error("Release unavailable"),
+ );
+ } else {
+ vi.mocked(autumnService.finalizeCreditsLock).mockResolvedValue(false);
+ }
+ if (kind === "handler") {
+ monitor.targets = [];
+ const failure = new Error("Target write failed");
+ vi.mocked(autumnService.lockCredits).mockResolvedValue({
+ status: "locked",
+ lockId: "monitor_check-1",
+ });
+ const updateIfRunning = vi
+ .mocked(store.updateMonitorCheckIfRunning)
+ .getMockImplementation()!;
+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
+ async (id, patch) => {
+ if (patch.target_results) throw failure;
+ return updateIfRunning(id, patch);
+ },
+ );
+ await expect(
+ processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ }),
+ ).rejects.toThrow(failure);
+ } else {
+ if (kind === "stale") {
+ current.started_at = new Date(
+ Date.now() - 2 * 60 * 60 * 1000,
+ ).toISOString();
+ } else {
+ vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);
+ }
+ await reconcileRunningMonitorChecks();
+ }
+ expect(current).toMatchObject({
+ status: "failed",
+ billing_status: "failed",
+ });
+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ lockId: "monitor_check-1",
+ action: "release",
+ }),
+ );
+ expect(bill).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each(["completed", "stale", "orphan"])(
+ "settles the hold returned by the terminal claim for a %s check",
+ async kind => {
+ current.autumn_lock_id = null;
+ current.billing_status = "not_applicable";
+ if (kind === "stale") {
+ current.started_at = new Date(
+ Date.now() - 2 * 60 * 60 * 1000,
+ ).toISOString();
+ } else if (kind === "orphan") {
+ vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);
+ }
+ const updateIfRunning = vi
+ .mocked(store.updateMonitorCheckIfRunning)
+ .getMockImplementation()!;
+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
+ async (id, patch) => {
+ if (patch.status) {
+ // Reservation persisted after the primary read, just before the claim.
+ current = {
+ ...current,
+ autumn_lock_id: "late-lock",
+ reserved_credits: 2,
+ partner_run_token: "late-token",
+ billing_status: "reserved",
+ };
+ }
+ return updateIfRunning(id, patch);
+ },
+ );
+ await reconcileRunningMonitorChecks();
+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ lockId: "late-lock",
+ action: kind === "completed" ? "confirm" : "release",
+ ...(kind === "completed"
+ ? { externalRequestId: "late-token", heldValue: 2 }
+ : {}),
+ }),
+ );
+ expect(current.billing_status).toBe(
+ kind === "completed" ? "confirmed" : "released",
+ );
+ },
+ );
+
+ it("does not reopen a check that became terminal after the job handler read it", async () => {
+ const queued = { ...current, status: "queued" as const };
+ vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValue(queued);
+ current.status = "completed";
+ current.billing_status = "confirmed";
+ await processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ });
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "confirmed",
+ });
+ expect(store.markMonitorRunning).not.toHaveBeenCalled();
+ expect(autumnService.lockCredits).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ { status: "locked" as const, lockId: "monitor_check-1" },
+ { status: "skipped" as const },
+ { status: "denied" as const },
+ ])(
+ "preserves completion when a slow reservation returns $status",
+ async result => {
+ monitor.targets = [];
+ vi.mocked(autumnService.lockCredits).mockImplementation(async () => {
+ current = {
+ ...current,
+ status: "completed",
+ billing_status: "confirmed",
+ };
+ return result;
+ });
+ await processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ });
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "confirmed",
+ });
+ expect(store.updateMonitorCheck).not.toHaveBeenCalled();
+ expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+ },
+ );
+ it.each([
+ {
+ name: "an unowned hold",
+ persistedLock: null,
+ deleted: false,
+ release: true,
+ },
+ {
+ name: "a different hold",
+ persistedLock: "other-lock",
+ deleted: false,
+ release: true,
+ },
+ {
+ name: "the finalizer's hold",
+ persistedLock: "monitor_check-1",
+ deleted: false,
+ release: false,
+ },
+ {
+ name: "a deleted check's hold",
+ persistedLock: null,
+ deleted: true,
+ release: true,
+ },
+ ])(
+ "cleans up $name when a reservation arrives after completion",
+ async ({ persistedLock, deleted, release }) => {
+ current.autumn_lock_id = null;
+ current.billing_status = "not_applicable";
+ monitor.targets = [];
+ vi.mocked(autumnService.lockCredits).mockImplementation(async () => {
+ current = {
+ ...current,
+ status: "completed",
+ billing_status: "confirmed",
+ autumn_lock_id: persistedLock,
+ };
+ if (deleted) {
+ vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValueOnce(null);
+ }
+ return {
+ status: "locked",
+ lockId: "monitor_check-1",
+ operationToken: "operation-1",
+ };
+ });
+ await processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ });
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "confirmed",
+ autumn_lock_id: persistedLock,
+ });
+ expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();
+ if (release) {
+ expect(
+ autumnService.finalizeCreditsLock,
+ ).toHaveBeenCalledExactlyOnceWith({
+ lockId: "monitor_check-1",
+ action: "release",
+ properties: {
+ source: "monitorCheck",
+ endpoint: "monitor",
+ jobId: current.id,
+ },
+ teamId: monitor.team_id,
+ });
+ } else {
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+ }
+ },
+ );
+
+ it.each(["queued", "running"] as const)(
+ "does not release a late reservation whose current row is %s",
+ async status => {
+ current.autumn_lock_id = null;
+ vi.mocked(autumnService.lockCredits).mockImplementation(async () => {
+ current.status = "completed";
+ vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValueOnce({
+ ...current,
+ status,
+ });
+ return { status: "locked", lockId: "monitor_check-1" };
+ });
+ await processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ });
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+ },
+ );
+
+ it("does not release a late reservation without an authoritative ownership read", async () => {
+ current.autumn_lock_id = null;
+ const failure = new Error("Primary read unavailable");
+ vi.mocked(autumnService.lockCredits).mockImplementation(async () => {
+ current.status = "completed";
+ vi.mocked(store.getMonitorCheckForUpdate).mockRejectedValue(failure);
+ return { status: "locked", lockId: "monitor_check-1" };
+ });
+ await expect(
+ processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ }),
+ ).rejects.toThrow(failure);
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ { localLock: "monitor_check-1", persistedLock: "other-lock" },
+ { localLock: null, persistedLock: "other-lock" },
+ { localLock: "monitor_check-1", persistedLock: null },
+ { localLock: "monitor_check-1", persistedLock: "monitor_check-1" },
+ ])(
+ "releases owned holds after a handler failure with $localLock and $persistedLock",
+ async ({ localLock, persistedLock }) => {
+ monitor.targets = [];
+ const failure = new Error("Target write failed");
+ vi.mocked(autumnService.lockCredits).mockResolvedValue(
+ localLock
+ ? { status: "locked", lockId: localLock }
+ : { status: "skipped" },
+ );
+ const updateIfRunning = vi
+ .mocked(store.updateMonitorCheckIfRunning)
+ .getMockImplementation()!;
+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
+ async (id, patch) => {
+ if (patch.target_results) {
+ current.autumn_lock_id = persistedLock;
+ throw failure;
+ }
+ return updateIfRunning(id, patch);
+ },
+ );
+ await expect(
+ processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ }),
+ ).rejects.toThrow(failure);
+ const expectedLocks = [
+ ...new Set([localLock, persistedLock].filter(Boolean)),
+ ].sort();
+ expect(
+ vi
+ .mocked(autumnService.finalizeCreditsLock)
+ .mock.calls.map(([call]) => call.lockId)
+ .sort(),
+ ).toEqual(expectedLocks);
+ expect(
+ vi
+ .mocked(autumnService.finalizeCreditsLock)
+ .mock.calls.every(([call]) => call.action === "release"),
+ ).toBe(true);
+ expect(current).toMatchObject({
+ status: "failed",
+ billing_status: "released",
+ });
+ },
+ );
+
+ it.each([false, true])(
+ "releases an unpersisted hold when reservation storage throws and another finalizer wins: %s",
+ async terminal => {
+ current.autumn_lock_id = null;
+ const failure = new Error("Reservation write failed");
+ vi.mocked(autumnService.lockCredits).mockResolvedValue({
+ status: "locked",
+ lockId: "monitor_check-1",
+ });
+ const updateIfRunning = vi
+ .mocked(store.updateMonitorCheckIfRunning)
+ .getMockImplementation()!;
+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
+ async (id, patch) => {
+ if (Object.hasOwn(patch, "autumn_lock_id")) {
+ if (terminal) {
+ current.status = "completed";
+ current.billing_status = "not_applicable";
+ }
+ throw failure;
+ }
+ return updateIfRunning(id, patch);
+ },
+ );
+ await expect(
+ processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ }),
+ ).rejects.toThrow(failure);
+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ lockId: "monitor_check-1",
+ action: "release",
+ }),
+ );
+ expect(current.status).toBe(terminal ? "completed" : "failed");
+ if (terminal)
+ expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each([1, 2])(
+ "preserves finalized results when target write %i arrives late",
+ async lateWrite => {
+ monitor.targets = [];
+ vi.mocked(autumnService.lockCredits).mockResolvedValue({
+ status: "locked",
+ lockId: "monitor_check-1",
+ });
+ const finalizedResults = [
+ {
+ type: "scrape",
+ targetId: "target-1",
+ expectedJobs: ["finished-scrape"],
+ },
+ ];
+ let targetWrites = 0;
+ const finishBeforeWrite = (patch: Partial<MonitorCheckRow>) => {
+ if (patch.target_results && ++targetWrites === lateWrite) {
+ current = {
+ ...current,
+ status: "completed",
+ billing_status: "confirmed",
+ target_results: finalizedResults,
+ };
+ }
+ };
+ const update = vi
+ .mocked(store.updateMonitorCheck)
+ .getMockImplementation()!;
+ const updateIfRunning = vi
+ .mocked(store.updateMonitorCheckIfRunning)
+ .getMockImplementation()!;
+ vi.mocked(store.updateMonitorCheck).mockImplementation(
+ async (id, patch) => {
+ finishBeforeWrite(patch);
+ return update(id, patch);
+ },
+ );
+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
+ async (id, patch) => {
+ finishBeforeWrite(patch);
+ return updateIfRunning(id, patch);
+ },
+ );
+ await processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ });
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "confirmed",
+ target_results: finalizedResults,
+ });
+ expect(targetWrites).toBe(lateWrite);
+ },
+ );
+});
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 3b6c5b94e..e72706d1b 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -37,7 +37,7 @@ import {
bulkUpsertMonitorPages,
calculateMonitorCheckActualCredits,
countRecentConsecutiveSkippedForCredits,
- getMonitorCheck,
+ getMonitorCheckForUpdate,
getMonitorForUpdate,
countMonitorCheckPages,
insertMonitorCheckPages,
@@ -49,6 +49,7 @@ import {
pauseMonitor,
updateMonitorCheck,
updateMonitorCheckIfRunning,
+ updateMonitorCheckIfStatus,
updateMonitorScheduleAfterRun,
upsertMonitorPage,
} from "./store";
@@ -95,13 +96,6 @@ const MONITOR_CHECK_REVOKED_ERROR =
* reached by a 402: the current denial must itself be `job_revoked`.
*/
const MONITOR_GATE_REVOKED_STREAK = 3;
-const TERMINAL_CHECK_STATUSES = new Set([
- "completed",
- "partial",
- "failed",
- "skipped_overlap",
- "skipped_no_credits",
-]);
async function claimMonitorNotification(checkId: string): Promise<boolean> {
const result = await redisEvictConnection.set(
@@ -936,6 +930,38 @@ export function findCompletedSearchTargetRun(
return (match as Record<string, unknown>) ?? null;
}
+async function releaseUnpersistedMonitorHold(
+ monitor: MonitorRow,
+ checkId: string,
+ lockId: string | null,
+): Promise<void> {
+ if (!lockId) return;
+ const latest = await getMonitorCheckForUpdate(
+ monitor.team_id,
+ monitor.id,
+ checkId,
+ );
+ // A duplicate delivery can receive the same lock ID as the finalizer.
+ // Only release an unpersisted hold after no running writer can adopt it.
+ if (
+ !latest ||
+ (latest.status !== "queued" &&
+ latest.status !== "running" &&
+ latest.autumn_lock_id !== lockId)
+ ) {
+ await autumnService.finalizeCreditsLock({
+ lockId,
+ action: "release",
+ properties: {
+ source: "monitorCheck",
+ endpoint: "monitor",
+ jobId: checkId,
+ },
+ teamId: monitor.team_id,
+ });
+ }
+}
+
export async function processMonitorCheckJob(
job: MonitorCheckJobData,
): Promise<void> {
@@ -944,7 +970,7 @@ export async function processMonitorCheckJob(
throw new Error("Monitor not found");
}
- const initialCheck = await getMonitorCheck(
+ const initialCheck = await getMonitorCheckForUpdate(
job.teamId,
job.monitorId,
job.checkId,
@@ -952,20 +978,26 @@ export async function processMonitorCheckJob(
if (!initialCheck) {
throw new Error("Monitor check not found");
}
- if (TERMINAL_CHECK_STATUSES.has(initialCheck.status)) {
+ if (initialCheck.status !== "queued" && initialCheck.status !== "running") {
return;
}
+ const started = await updateMonitorCheckIfStatus(
+ job.checkId,
+ initialCheck.status,
+ {
+ status: "running",
+ started_at: new Date().toISOString(),
+ },
+ );
+ if (!started) return;
+ let check: MonitorCheckRow = started;
+
await markMonitorRunning({
monitorId: monitor.id,
checkId: job.checkId,
});
- let check: MonitorCheckRow = await updateMonitorCheck(job.checkId, {
- status: "running",
- started_at: new Date().toISOString(),
- });
-
trackMonitorCheckStartedInterest({ monitor, check }).catch(error =>
logger.warn("Failed to track monitor target interest", {
error,
@@ -997,7 +1029,7 @@ export async function processMonitorCheckJob(
if (lock.status === "denied") {
const revoked = lock.reason === "job_revoked";
- check = await updateMonitorCheck(check.id, {
+ const skipped = await updateMonitorCheckIfRunning(check.id, {
status: "skipped_no_credits",
finished_at: new Date().toISOString(),
actual_credits: 0,
@@ -1007,6 +1039,9 @@ export async function processMonitorCheckJob(
: MONITOR_CHECK_NO_CREDITS_ERROR,
});
+ if (!skipped) return;
+ check = skipped;
+
// A revoked job never becomes unrevoked; see MONITOR_GATE_REVOKED_STREAK
// for why it is waited out rather than acted on at once.
const paused =
@@ -1045,7 +1080,7 @@ export async function processMonitorCheckJob(
lockId = lock.status === "locked" ? lock.lockId : null;
- check = await updateMonitorCheck(check.id, {
+ const reserved = await updateMonitorCheckIfRunning(check.id, {
autumn_lock_id: lockId,
// A token already on the row wins: the gate was not re-asked, so there
// is no newer one, and null would lose the authorized operation.
@@ -1056,10 +1091,17 @@ export async function processMonitorCheckJob(
billing_status: lockId ? "reserved" : "not_applicable",
});
+ if (!reserved) {
+ await releaseUnpersistedMonitorHold(monitor, check.id, lockId);
+ return;
+ }
+ check = reserved;
+
const targetResults = monitor.targets.map(createMonitorTargetRun);
- await updateMonitorCheck(check.id, {
+ const initialized = await updateMonitorCheckIfRunning(check.id, {
target_results: targetResults,
});
+ if (!initialized) return;
for (const [index, target] of monitor.targets.entries()) {
const targetRun = targetResults[index];
@@ -1097,7 +1139,7 @@ export async function processMonitorCheckJob(
targetRun.searchCompleted = true;
// Persist searchCompleted now so a crash/redelivery short-circuits via
// findCompletedSearchTargetRun instead of re-running and re-billing.
- await withFinalizeTimeout(
+ const persisted = await withFinalizeTimeout(
signal =>
signal.aborted
? Promise.resolve(null)
@@ -1108,10 +1150,11 @@ export async function processMonitorCheckJob(
}),
"monitor search searchCompleted flush",
);
+ if (!persisted) return;
}
}
- await updateMonitorCheck(check.id, {
+ await updateMonitorCheckIfRunning(check.id, {
target_results: targetResults,
});
} catch (error) {
@@ -1123,27 +1166,46 @@ export async function processMonitorCheckJob(
const failed = await updateMonitorCheckIfRunning(check.id, {
status: "failed",
finished_at: new Date().toISOString(),
- billing_status: lockId ? "released" : "failed",
error: error instanceof Error ? error.message : String(error),
});
if (!failed) {
+ await releaseUnpersistedMonitorHold(monitor, check.id, lockId);
throw error;
}
check = failed;
- if (lockId) {
- await autumnService.finalizeCreditsLock({
- lockId,
- action: "release",
- properties: {
- source: "monitorCheck",
- endpoint: "monitor",
- jobId: check.id,
- },
- teamId: monitor.team_id,
- });
+ // The claim owns the persisted hold; a failed reservation write can also
+ // leave this handler with a newly acquired hold that was never stored.
+ let released = true;
+ for (const ownedLockId of new Set([failed.autumn_lock_id, lockId])) {
+ if (!ownedLockId) continue;
+ const settled = await autumnService
+ .finalizeCreditsLock({
+ lockId: ownedLockId,
+ action: "release",
+ properties: {
+ source: "monitorCheck",
+ endpoint: "monitor",
+ jobId: check.id,
+ },
+ teamId: monitor.team_id,
+ })
+ .catch(releaseError => {
+ logger.warn("Failed to release monitor check credit lock", {
+ error: releaseError,
+ monitorId: monitor.id,
+ checkId: check.id,
+ lockId: ownedLockId,
+ });
+ return false;
+ });
+ released = released && settled;
}
+ check = await updateMonitorCheck(failed.id, {
+ billing_status:
+ (failed.autumn_lock_id || lockId) && released ? "released" : "failed",
+ });
if (
await claimMonitorNotification(check.id).catch(error => {
@@ -1307,10 +1369,19 @@ async function failStaleMonitorCheck(params: {
return false;
const error = MONITOR_CHECK_STALE_ERROR;
- if (params.check.autumn_lock_id) {
- await autumnService
+ const claimed = await updateMonitorCheckIfRunning(params.check.id, {
+ status: "failed",
+ finished_at: new Date().toISOString(),
+ actual_credits: 0,
+ error,
+ });
+ if (!claimed) return true;
+
+ let released = true;
+ if (claimed.autumn_lock_id) {
+ released = await autumnService
.finalizeCreditsLock({
- lockId: params.check.autumn_lock_id,
+ lockId: claimed.autumn_lock_id,
action: "release",
properties: {
source: "monitorCheck",
@@ -1324,19 +1395,19 @@ async function failStaleMonitorCheck(params: {
error: releaseError,
monitorId: params.monitor.id,
checkId: params.check.id,
- lockId: params.check.autumn_lock_id,
+ lockId: claimed.autumn_lock_id,
});
+ return false;
});
}
- const finalized = await updateMonitorCheck(params.check.id, {
- status: "failed",
- finished_at: new Date().toISOString(),
- actual_credits: 0,
- billing_status: params.check.autumn_lock_id ? "released" : "not_applicable",
- error,
+ const finalized = await updateMonitorCheck(claimed.id, {
+ billing_status: !claimed.autumn_lock_id
+ ? "not_applicable"
+ : released
+ ? "released"
+ : "failed",
});
-
let withNotifications = finalized;
if (await claimMonitorNotification(params.check.id)) {
const notificationStatus = await sendNotifications({
@@ -1403,21 +1474,46 @@ export async function reconcileRunningMonitorChecks(
limit: number = 50,
): Promise<void> {
const checks = await listRunningMonitorChecks(limit);
- for (const check of checks) {
- const lockKey = `monitor-check-finalize:${check.id}`;
- const lock = await redisEvictConnection.set(lockKey, "1", "EX", 60, "NX");
+ for (const candidate of checks) {
+ const lockKey = `monitor-check-finalize:${candidate.id}`;
+ const lockToken = uuidv7();
+ const lock = await redisEvictConnection.set(
+ lockKey,
+ lockToken,
+ "EX",
+ 60,
+ "NX",
+ );
if (lock !== "OK") continue;
try {
+ // The batch can outlive another finalizer. Read from the primary after
+ // acquiring the lease rather than acting on that old running snapshot.
+ const check = await getMonitorCheckForUpdate(
+ candidate.team_id,
+ candidate.monitor_id,
+ candidate.id,
+ );
+ if (!check || check.status !== "running") continue;
+
const monitor = await getMonitorForUpdate(
check.team_id,
check.monitor_id,
);
if (!monitor) {
- if (check.autumn_lock_id) {
- await autumnService
+ const failed = await updateMonitorCheckIfRunning(check.id, {
+ status: "failed",
+ finished_at: new Date().toISOString(),
+ actual_credits: 0,
+ error: "Monitor no longer exists.",
+ });
+ if (!failed) continue;
+
+ let released = true;
+ if (failed.autumn_lock_id) {
+ released = await autumnService
.finalizeCreditsLock({
- lockId: check.autumn_lock_id,
+ lockId: failed.autumn_lock_id,
action: "release",
properties: {
source: "monitorCheck",
@@ -1433,20 +1529,20 @@ export async function reconcileRunningMonitorChecks(
error,
monitorId: check.monitor_id,
checkId: check.id,
- lockId: check.autumn_lock_id,
+ lockId: failed.autumn_lock_id,
},
);
+ return false;
});
}
- await updateMonitorCheck(check.id, {
- status: "failed",
- finished_at: new Date().toISOString(),
- actual_credits: 0,
- billing_status: check.autumn_lock_id ? "released" : "not_applicable",
- error: "Monitor no longer exists.",
+ await updateMonitorCheck(failed.id, {
+ billing_status: !failed.autumn_lock_id
+ ? "not_applicable"
+ : released
+ ? "released"
+ : "failed",
});
-
logger.warn("Failed orphaned monitor check", {
monitorId: check.monitor_id,
checkId: check.id,
@@ -1456,8 +1552,7 @@ export async function reconcileRunningMonitorChecks(
if (await failStaleMonitorCheck({ monitor, check })) continue;
- // Snapshot from listRunningMonitorChecks; may be stale relative to the
- // inline handler that is still writing this check.
+ // The inline handler may still write target results after our primary read.
let targetResults = Array.isArray(check.target_results)
? ([...check.target_results] as any[])
: [];
@@ -1494,7 +1589,9 @@ export async function reconcileRunningMonitorChecks(
// reaper. The complete-path write below is safe: a search target can only
// be complete once its snapshot already carries searchCompleted=true.
if (recoveredFromEmpty && targetResults.length > 0) {
- await updateMonitorCheck(check.id, { target_results: targetResults });
+ await updateMonitorCheckIfRunning(check.id, {
+ target_results: targetResults,
+ });
}
continue;
}
@@ -1514,14 +1611,12 @@ export async function reconcileRunningMonitorChecks(
targetResults,
});
- let finalized = await updateMonitorCheck(check.id, {
+ // This conditional write is the durable claim. Even if the Redis lease
+ // expires during preparation, only one worker may settle this check.
+ const claimed = await updateMonitorCheckIfRunning(check.id, {
status: errorCount > 0 ? "partial" : "completed",
finished_at: new Date().toISOString(),
actual_credits: actualCredits,
- // Not `confirmed` yet — the settle has not been attempted. Claiming it
- // here and then finding firebill refused leaves a run permanently
- // recorded as billed that nobody billed, with no retry.
- billing_status: check.autumn_lock_id ? "reserved" : "not_applicable",
total_pages: totalPages,
same_count: same,
changed_count: changed,
@@ -1531,13 +1626,16 @@ export async function reconcileRunningMonitorChecks(
target_results: targetResults,
});
+ if (!claimed) continue;
+ let finalized = claimed;
+
let settled = false;
try {
settled = await billMonitorCheck({
monitor,
check: finalized,
actualCredits,
- lockId: check.autumn_lock_id,
+ lockId: claimed.autumn_lock_id,
});
} catch (error) {
logger.warn("Failed to bill monitor check during reconciliation", {
@@ -1550,14 +1648,14 @@ export async function reconcileRunningMonitorChecks(
// A refusal and a throw are the same fact — the settle did not land — and
// both must be recorded as such. `firebillFinalize` answers `false`
// without throwing, so the catch alone never saw them.
- if (check.autumn_lock_id) {
+ if (claimed.autumn_lock_id) {
if (!settled) {
logger.error(
"Monitor check settle did not land; the hold is unsettled and this run is unbilled",
{
monitorId: monitor.id,
checkId: finalized.id,
- lockId: check.autumn_lock_id,
+ lockId: claimed.autumn_lock_id,
actualCredits,
},
);
@@ -1655,10 +1753,19 @@ export async function reconcileRunningMonitorChecks(
} catch (error) {
logger.warn("Failed to reconcile monitor check", {
error,
- checkId: check.id,
+ checkId: candidate.id,
});
} finally {
- await redisEvictConnection.del(lockKey);
+ // An expired lease may already belong to another worker.
+ await redisEvictConnection.eval(
+ `if redis.call("get", KEYS[1]) == ARGV[1] then
+ return redis.call("del", KEYS[1])
+ end
+ return 0`,
+ 1,
+ lockKey,
+ lockToken,
+ );
}
}
}
diff --git a/apps/api/src/services/monitoring/scheduler.test.ts b/apps/api/src/services/monitoring/scheduler.test.ts
index 25e29e7c9..bdded5c43 100644
--- a/apps/api/src/services/monitoring/scheduler.test.ts
+++ b/apps/api/src/services/monitoring/scheduler.test.ts
@@ -7,8 +7,9 @@ import {
claimDueMonitors,
createMonitorCheck,
dispatchScheduledMonitorCheck,
- getMonitorCheck,
+ getMonitorCheckForUpdate,
updateMonitorCheck,
+ updateMonitorCheckIfStatus,
updateMonitorScheduleAfterRun,
} from "./store";
import { autumnService } from "../autumn/autumn.service";
@@ -22,8 +23,9 @@ vi.mock("./store", () => ({
claimDueMonitors: vi.fn(),
createMonitorCheck: vi.fn(),
dispatchScheduledMonitorCheck: vi.fn(),
- getMonitorCheck: vi.fn(),
+ getMonitorCheckForUpdate: vi.fn(),
updateMonitorCheck: vi.fn(),
+ updateMonitorCheckIfStatus: vi.fn(),
updateMonitorScheduleAfterRun: vi.fn(),
}));
@@ -52,8 +54,8 @@ const mockDispatchScheduledMonitorCheck =
dispatchScheduledMonitorCheck as MockedFunction<
typeof dispatchScheduledMonitorCheck
>;
-const mockGetMonitorCheck = getMonitorCheck as MockedFunction<
- typeof getMonitorCheck
+const mockGetMonitorCheckForUpdate = getMonitorCheckForUpdate as MockedFunction<
+ typeof getMonitorCheckForUpdate
>;
const mockIsMonitorCheckStale = isMonitorCheckStale as MockedFunction<
typeof isMonitorCheckStale
@@ -92,11 +94,18 @@ describe("monitoring scheduler", () => {
mockCreateMonitorCheck.mockResolvedValue(check);
mockDispatchScheduledMonitorCheck.mockResolvedValue(true);
mockAddMonitorCheckJob.mockResolvedValue(undefined);
+ mockUpdateMonitorCheck.mockImplementation(
+ async (id, patch) =>
+ ({
+ id,
+ ...patch,
+ }) as any,
+ );
mockAdvanceMonitorAfterSkippedCheck.mockResolvedValue(undefined);
mockUpdateMonitorScheduleAfterRun.mockResolvedValue(undefined);
- mockGetMonitorCheck.mockResolvedValue(null);
+ mockGetMonitorCheckForUpdate.mockResolvedValue(null);
mockIsMonitorCheckStale.mockReturnValue(false);
- mockFinalizeCreditsLock.mockResolvedValue(undefined as any);
+ mockFinalizeCreditsLock.mockResolvedValue(true);
});
it("dispatches and advances a scheduled monitor before enqueueing its job", async () => {
@@ -187,6 +196,36 @@ describe("monitoring scheduler", () => {
});
});
+ it("records an overlap while the current check is still active", async () => {
+ const active = { id: "active-check", status: "running" } as any;
+ const monitorWithCurrentCheck = {
+ ...monitor,
+ current_check_id: active.id,
+ };
+ const skipped = { id: check.id, status: "skipped_overlap" } as any;
+ mockClaimDueMonitors.mockResolvedValue([monitorWithCurrentCheck]);
+ mockGetMonitorCheckForUpdate.mockResolvedValue(active);
+ mockUpdateMonitorCheck.mockResolvedValue(skipped);
+
+ await expect(enqueueDueMonitorChecks()).resolves.toBe(0);
+
+ expect(mockCreateMonitorCheck).toHaveBeenCalledExactlyOnceWith({
+ monitor: monitorWithCurrentCheck,
+ trigger: "scheduled",
+ scheduledFor: monitor.next_run_at,
+ status: "skipped_overlap",
+ });
+ expect(mockAdvanceMonitorAfterSkippedCheck).toHaveBeenCalledExactlyOnceWith(
+ {
+ monitor: monitorWithCurrentCheck,
+ check: skipped,
+ },
+ );
+ expect(updateMonitorCheckIfStatus).not.toHaveBeenCalled();
+ expect(mockDispatchScheduledMonitorCheck).not.toHaveBeenCalled();
+ expect(mockAddMonitorCheckJob).not.toHaveBeenCalled();
+ });
+
it("clears a stale current check before enqueueing a scheduled run", async () => {
const monitorWithCurrentCheck = {
...monitor,
@@ -194,25 +233,33 @@ describe("monitoring scheduler", () => {
} as any;
const staleCheck = { id: "stale-check", status: "running" } as any;
const failedStaleCheck = { ...staleCheck, status: "failed" } as any;
+ const finalizedStaleCheck = {
+ ...failedStaleCheck,
+ billing_status: "not_applicable",
+ };
mockClaimDueMonitors.mockResolvedValue([monitorWithCurrentCheck]);
- mockGetMonitorCheck.mockResolvedValue(staleCheck);
+ mockGetMonitorCheckForUpdate.mockResolvedValue(staleCheck);
mockIsMonitorCheckStale.mockReturnValue(true);
- mockUpdateMonitorCheck.mockResolvedValue(failedStaleCheck);
+ vi.mocked(updateMonitorCheckIfStatus).mockResolvedValue(failedStaleCheck);
+ mockUpdateMonitorCheck.mockResolvedValue(finalizedStaleCheck);
await expect(
enqueueDueMonitorChecks({ workerId: "worker-1" }),
).resolves.toBe(1);
- expect(mockUpdateMonitorCheck).toHaveBeenCalledWith(staleCheck.id, {
- status: "failed",
- finished_at: expect.any(String),
- actual_credits: 0,
- billing_status: "not_applicable",
- error: "Monitor check exceeded the 1 hour running timeout.",
- });
+ expect(updateMonitorCheckIfStatus).toHaveBeenCalledWith(
+ staleCheck.id,
+ "running",
+ {
+ status: "failed",
+ finished_at: expect.any(String),
+ actual_credits: 0,
+ error: "Monitor check exceeded the 1 hour running timeout.",
+ },
+ );
expect(mockUpdateMonitorScheduleAfterRun).toHaveBeenCalledWith({
monitor: monitorWithCurrentCheck,
- check: failedStaleCheck,
+ check: finalizedStaleCheck,
});
expect(mockCreateMonitorCheck).toHaveBeenCalledWith({
monitor: { ...monitorWithCurrentCheck, current_check_id: null },
@@ -228,4 +275,160 @@ describe("monitoring scheduler", () => {
{ search: false },
);
});
+ it.each(["running", "queued"] as const)(
+ "releases a stale %s hold only after claiming failure",
+ async status => {
+ const stale = {
+ id: "stale-check",
+ status,
+ autumn_lock_id: "monitor_stale-check",
+ } as any;
+ mockClaimDueMonitors.mockResolvedValue([
+ { ...monitor, current_check_id: stale.id },
+ ]);
+ mockGetMonitorCheckForUpdate.mockResolvedValue(stale);
+ mockIsMonitorCheckStale.mockReturnValue(true);
+ vi.mocked(updateMonitorCheckIfStatus).mockResolvedValue({
+ ...stale,
+ status: "failed",
+ });
+
+ await enqueueDueMonitorChecks();
+
+ expect(updateMonitorCheckIfStatus).toHaveBeenCalledWith(
+ stale.id,
+ status,
+ expect.objectContaining({ status: "failed" }),
+ );
+ expect(mockFinalizeCreditsLock).toHaveBeenCalledWith(
+ expect.objectContaining({
+ lockId: stale.autumn_lock_id,
+ action: "release",
+ }),
+ );
+ expect(
+ vi.mocked(updateMonitorCheckIfStatus).mock.invocationCallOrder[0],
+ ).toBeLessThan(mockFinalizeCreditsLock.mock.invocationCallOrder[0]);
+ },
+ );
+
+ it("does not release or rewrite a check when another finalizer wins the terminal claim", async () => {
+ const stale = {
+ id: "stale-check",
+ status: "running",
+ autumn_lock_id: "monitor_stale-check",
+ } as any;
+ mockClaimDueMonitors.mockResolvedValue([
+ { ...monitor, current_check_id: stale.id },
+ ]);
+ mockGetMonitorCheckForUpdate.mockResolvedValue(stale);
+ mockIsMonitorCheckStale.mockReturnValue(true);
+ vi.mocked(updateMonitorCheckIfStatus).mockResolvedValue(null);
+
+ await enqueueDueMonitorChecks();
+
+ expect(mockFinalizeCreditsLock).not.toHaveBeenCalled();
+ expect(mockUpdateMonitorScheduleAfterRun).not.toHaveBeenCalled();
+ expect(mockCreateMonitorCheck).not.toHaveBeenCalled();
+ expect(mockUpdateMonitorCheck).not.toHaveBeenCalled();
+ expect(mockAdvanceMonitorAfterSkippedCheck).not.toHaveBeenCalled();
+ expect(mockDispatchScheduledMonitorCheck).not.toHaveBeenCalled();
+ expect(mockAddMonitorCheckJob).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ { previousLock: null, claimedLock: "current-lock" },
+ { previousLock: "previous-lock", claimedLock: null },
+ { previousLock: "previous-lock", claimedLock: "current-lock" },
+ ])(
+ "settles the claimed hold when it changes from $previousLock to $claimedLock",
+ async ({ previousLock, claimedLock }) => {
+ const current = {
+ id: "stale-check",
+ status: "running",
+ autumn_lock_id: previousLock,
+ } as any;
+ const claimed = {
+ ...current,
+ status: "failed",
+ autumn_lock_id: claimedLock,
+ };
+ mockClaimDueMonitors.mockResolvedValue([
+ { ...monitor, current_check_id: current.id },
+ ]);
+ mockGetMonitorCheckForUpdate.mockResolvedValue(current);
+ mockIsMonitorCheckStale.mockReturnValue(true);
+ vi.mocked(updateMonitorCheckIfStatus).mockResolvedValue(claimed);
+ mockUpdateMonitorCheck.mockImplementation(async (_id, patch) => ({
+ ...claimed,
+ ...patch,
+ }));
+
+ await enqueueDueMonitorChecks();
+
+ expect(
+ vi.mocked(updateMonitorCheckIfStatus).mock.calls[0][2],
+ ).not.toHaveProperty("billing_status");
+ if (claimedLock) {
+ expect(mockFinalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ lockId: claimedLock, action: "release" }),
+ );
+ } else {
+ expect(mockFinalizeCreditsLock).not.toHaveBeenCalled();
+ }
+ expect(mockUpdateMonitorCheck).toHaveBeenCalledWith(current.id, {
+ billing_status: claimedLock ? "released" : "not_applicable",
+ });
+ expect(mockUpdateMonitorScheduleAfterRun).toHaveBeenCalledWith({
+ monitor: expect.objectContaining({ id: monitor.id }),
+ check: expect.objectContaining({
+ autumn_lock_id: claimedLock,
+ billing_status: claimedLock ? "released" : "not_applicable",
+ }),
+ });
+ },
+ );
+
+ it.each(["success", "refused", "throw"] as const)(
+ "records a stale hold release outcome of %s before advancing the schedule",
+ async outcome => {
+ const current = {
+ id: "stale-check",
+ status: "running",
+ autumn_lock_id: "monitor_stale-check",
+ } as any;
+ const claimed = { ...current, status: "failed" };
+ mockClaimDueMonitors.mockResolvedValue([
+ { ...monitor, current_check_id: current.id },
+ ]);
+ mockGetMonitorCheckForUpdate.mockResolvedValue(current);
+ mockIsMonitorCheckStale.mockReturnValue(true);
+ vi.mocked(updateMonitorCheckIfStatus).mockResolvedValue(claimed);
+ mockUpdateMonitorCheck.mockImplementation(async (_id, patch) => ({
+ ...claimed,
+ ...patch,
+ }));
+ if (outcome === "throw") {
+ mockFinalizeCreditsLock.mockRejectedValue(
+ new Error("Release unavailable"),
+ );
+ } else {
+ mockFinalizeCreditsLock.mockResolvedValue(outcome === "success");
+ }
+
+ await expect(enqueueDueMonitorChecks()).resolves.toBe(1);
+
+ const billingStatus = outcome === "success" ? "released" : "failed";
+ expect(mockUpdateMonitorCheck).toHaveBeenCalledWith(current.id, {
+ billing_status: billingStatus,
+ });
+ expect(mockUpdateMonitorScheduleAfterRun).toHaveBeenCalledExactlyOnceWith(
+ {
+ monitor: expect.objectContaining({ id: monitor.id }),
+ check: { ...claimed, billing_status: billingStatus },
+ },
+ );
+ expect(mockAddMonitorCheckJob).toHaveBeenCalledTimes(1);
+ },
+ );
});
diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.ts
index 20c3855e7..d9704a543 100644
--- a/apps/api/src/services/monitoring/scheduler.ts
+++ b/apps/api/src/services/monitoring/scheduler.ts
@@ -7,8 +7,9 @@ import {
createMonitorCheck,
deferMonitorClaim,
dispatchScheduledMonitorCheck,
- getMonitorCheck,
+ getMonitorCheckForUpdate,
updateMonitorCheck,
+ updateMonitorCheckIfStatus,
updateMonitorScheduleAfterRun,
} from "./store";
import { autumnService } from "../autumn/autumn.service";
@@ -62,8 +63,9 @@ export async function enqueueDueMonitorChecks(
if (await deferForJitter(monitor)) continue;
if (monitor.current_check_id) {
- const cleared = await clearFinishedOrStaleCurrentCheck(monitor);
- if (cleared) {
+ const resolution = await clearFinishedOrStaleCurrentCheck(monitor);
+ if (resolution === "lost_claim") continue;
+ if (resolution === "cleared") {
monitor = { ...monitor, current_check_id: null };
}
}
@@ -178,29 +180,44 @@ async function deferForJitter(monitor: MonitorRow): Promise<boolean> {
async function clearFinishedOrStaleCurrentCheck(
monitor: MonitorRow,
-): Promise<boolean> {
- if (!monitor.current_check_id) return true;
+): Promise<"cleared" | "active" | "lost_claim"> {
+ if (!monitor.current_check_id) return "cleared";
- const current = await getMonitorCheck(
+ const current = await getMonitorCheckForUpdate(
monitor.team_id,
monitor.id,
monitor.current_check_id,
);
- if (!current) return false;
+ if (!current) return "active";
if (current.status === "running" || current.status === "queued") {
if (!isMonitorCheckStale(current, new Date(), monitor.targets))
- return false;
-
- if (current.autumn_lock_id) {
- await autumnService
+ return "active";
+
+ // Compete with the reconciler before releasing its hold. A failed claim
+ // means another worker owns completion; leave its billing and schedule alone.
+ const failed = await updateMonitorCheckIfStatus(
+ current.id,
+ current.status,
+ {
+ status: "failed",
+ finished_at: new Date().toISOString(),
+ actual_credits: 0,
+ error: MONITOR_CHECK_STALE_ERROR,
+ },
+ );
+ if (!failed) return "lost_claim";
+
+ let released = true;
+ if (failed.autumn_lock_id) {
+ released = await autumnService
.finalizeCreditsLock({
- lockId: current.autumn_lock_id,
+ lockId: failed.autumn_lock_id,
action: "release",
properties: {
source: "monitorCheck",
endpoint: "monitor",
- jobId: current.id,
+ jobId: failed.id,
},
teamId: monitor.team_id,
})
@@ -208,29 +225,30 @@ async function clearFinishedOrStaleCurrentCheck(
logger.warn("Failed to release stale monitor check credit lock", {
error,
monitorId: monitor.id,
- checkId: current.id,
- lockId: current.autumn_lock_id,
+ checkId: failed.id,
+ lockId: failed.autumn_lock_id,
});
+ return false;
});
}
- const failed = await updateMonitorCheck(current.id, {
- status: "failed",
- finished_at: new Date().toISOString(),
- actual_credits: 0,
- billing_status: current.autumn_lock_id ? "released" : "not_applicable",
- error: MONITOR_CHECK_STALE_ERROR,
+ const finalized = await updateMonitorCheck(failed.id, {
+ billing_status: failed.autumn_lock_id
+ ? released
+ ? "released"
+ : "failed"
+ : "not_applicable",
});
await updateMonitorScheduleAfterRun({
monitor,
- check: failed,
+ check: finalized,
});
- return true;
+ return "cleared";
}
await updateMonitorScheduleAfterRun({
monitor,
check: current,
});
- return true;
+ return "cleared";
}
diff --git a/apps/api/src/services/monitoring/store.finalization.test.ts b/apps/api/src/services/monitoring/store.finalization.test.ts
new file mode 100644
index 000000000..e6c941c78
--- /dev/null
+++ b/apps/api/src/services/monitoring/store.finalization.test.ts
@@ -0,0 +1,147 @@
+import { PgDialect } from "drizzle-orm/pg-core";
+
+const {
+ primarySelect,
+ replicaSelect,
+ primaryUpdate,
+ where,
+ set,
+ limit,
+ returning,
+} = vi.hoisted(() => {
+ const where = vi.fn();
+ const set = vi.fn();
+ const limit = vi.fn();
+ const returning = vi.fn();
+ const chain = {
+ from: vi.fn(() => chain),
+ set: set.mockImplementation(() => chain),
+ where: where.mockImplementation(() => chain),
+ limit,
+ returning,
+ };
+ return {
+ primarySelect: vi.fn(() => chain),
+ replicaSelect: vi.fn(() => chain),
+ primaryUpdate: vi.fn(() => chain),
+ where,
+ set,
+ limit,
+ returning,
+ };
+});
+
+vi.mock("../../db/connection", () => ({
+ db: { select: primarySelect, update: primaryUpdate },
+ dbRr: { select: replicaSelect },
+}));
+vi.mock("../../db/rpc", () => ({ monitoringClaimDueMonitors: vi.fn() }));
+
+import {
+ getMonitorCheckForUpdate,
+ updateMonitorCheckIfRunning,
+ updateMonitorCheckIfStatus,
+} from "./store";
+
+const teamId = "11111111-1111-4111-8111-111111111111";
+const monitorId = "22222222-2222-4222-8222-222222222222";
+const checkId = "33333333-3333-4333-8333-333333333333";
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ limit.mockResolvedValue([]);
+ returning.mockResolvedValue([]);
+});
+
+function queryPredicate() {
+ expect(where).toHaveBeenCalledTimes(1);
+ return new PgDialect().sqlToQuery(where.mock.calls[0][0]);
+}
+
+describe("monitor check finalization storage", () => {
+ it("reads the current check from primary scoped to its team and monitor", async () => {
+ const confirmed = {
+ id: checkId,
+ monitor_id: monitorId,
+ team_id: teamId,
+ status: "completed",
+ billing_status: "confirmed",
+ };
+ limit.mockResolvedValue([confirmed]);
+
+ expect(await getMonitorCheckForUpdate(teamId, monitorId, checkId)).toEqual(
+ confirmed,
+ );
+ expect(primarySelect).toHaveBeenCalledTimes(1);
+ expect(replicaSelect).not.toHaveBeenCalled();
+ expect(limit).toHaveBeenCalledWith(1);
+ const predicate = queryPredicate();
+ expect(predicate.sql).toBe(
+ '(("monitor_checks"."id" = $1) and ("monitor_checks"."monitor_id" = $2) and ("monitor_checks"."team_id" = $3))',
+ );
+ expect(predicate.params).toEqual([checkId, monitorId, teamId]);
+ });
+
+ it("returns null when the authoritative check no longer exists", async () => {
+ expect(
+ await getMonitorCheckForUpdate(teamId, monitorId, checkId),
+ ).toBeNull();
+ expect(primarySelect).toHaveBeenCalledTimes(1);
+ expect(replicaSelect).not.toHaveBeenCalled();
+ });
+
+ it("claims finalization with one conditional update while the check is running", async () => {
+ const completed = { id: checkId, status: "completed" };
+ returning.mockResolvedValue([completed]);
+
+ expect(
+ await updateMonitorCheckIfRunning(checkId, {
+ status: "completed",
+ billing_status: "reserved",
+ }),
+ ).toEqual(completed);
+ expect(primarySelect).not.toHaveBeenCalled();
+ expect(replicaSelect).not.toHaveBeenCalled();
+ expect(primaryUpdate).toHaveBeenCalledTimes(1);
+ expect(set).toHaveBeenCalledWith({
+ status: "completed",
+ billing_status: "reserved",
+ updated_at: expect.any(String),
+ });
+ const predicate = queryPredicate();
+ expect(predicate.sql).toBe(
+ '(("monitor_checks"."id" = $1) and ("monitor_checks"."status" = $2))',
+ );
+ expect(predicate.params).toEqual([checkId, "running"]);
+ });
+
+ it("returns null to a finalizer that lost the terminal-state claim", async () => {
+ expect(
+ await updateMonitorCheckIfRunning(checkId, {
+ status: "failed",
+ billing_status: "released",
+ }),
+ ).toBeNull();
+ expect(primaryUpdate).toHaveBeenCalledTimes(1);
+ expect(queryPredicate().params).toEqual([checkId, "running"]);
+ });
+
+ it("conditions a queued check transition on the queued state it observed", async () => {
+ const failed = { id: checkId, status: "failed" };
+ returning.mockResolvedValue([failed]);
+
+ expect(
+ await updateMonitorCheckIfStatus(checkId, "queued", {
+ status: "failed",
+ }),
+ ).toEqual(failed);
+ expect(primarySelect).not.toHaveBeenCalled();
+ expect(replicaSelect).not.toHaveBeenCalled();
+ expect(primaryUpdate).toHaveBeenCalledTimes(1);
+ const predicate = queryPredicate();
+ expect(predicate.sql).toBe(
+ '(("monitor_checks"."id" = $1) and ("monitor_checks"."status" = $2))',
+ );
+ expect(predicate.params).toEqual([checkId, "queued"]);
+ });
+});
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index d3adbed83..9c9f31211 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -777,6 +777,30 @@ export async function getMonitorCheck(
return (data ?? null) as MonitorCheckRow | null;
}
+// Finalizers must observe terminal writes before deciding whether to settle a hold.
+export async function getMonitorCheckForUpdate(
+ teamId: string,
+ monitorId: string,
+ checkId: string,
+): Promise<MonitorCheckRow | null> {
+ const [data] = await run(
+ () =>
+ db
+ .select()
+ .from(schema.monitor_checks)
+ .where(
+ and(
+ eq(schema.monitor_checks.id, checkId),
+ eq(schema.monitor_checks.monitor_id, monitorId),
+ eq(schema.monitor_checks.team_id, teamId),
+ ),
+ )
+ .limit(1),
+ "Failed to get monitor check for update",
+ );
+ return (data ?? null) as MonitorCheckRow | null;
+}
+
export async function listRunningMonitorChecks(
limit: number = 100,
): Promise<MonitorCheckRow[]> {
@@ -902,6 +926,16 @@ export async function updateMonitorCheck(
export async function updateMonitorCheckIfRunning(
checkId: string,
patch: Partial<MonitorCheckRow>,
+): Promise<MonitorCheckRow | null> {
+ return updateMonitorCheckIfStatus(checkId, "running", patch);
+}
+
+// The terminal transition grants ownership of settlement and its follow-up work.
+// A competing worker must not confirm/release the hold when this returns null.
+export async function updateMonitorCheckIfStatus(
+ checkId: string,
+ expectedStatus: "queued" | "running",
+ patch: Partial<MonitorCheckRow>,
): Promise<MonitorCheckRow | null> {
const [data] = await run(
() =>
@@ -914,7 +948,7 @@ export async function updateMonitorCheckIfRunning(
.where(
and(
eq(schema.monitor_checks.id, checkId),
- eq(schema.monitor_checks.status, "running"),
+ eq(schema.monitor_checks.status, expectedStatus),
),
)
.returning(),