Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): prevent duplicate monitor finalization (#4542)
Gergő Móricz6616c0f2efd0
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 674d3dfc4..6f4445045 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -218,6 +218,16 @@ jobs:env:npm_config_ignore_scripts: 'true'+ - name: Run monitor finalization regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/services/monitoring/runner.finalization.test.ts+ src/services/monitoring/store.finalization.test.ts+ src/services/monitoring/scheduler.test.ts+ --reporter=default+ working-directory: apps/api+- name: Install test site dependenciesrun: pnpm install --frozen-lockfileworking-directory: apps/test-sitediff --git a/apps/api/src/services/monitoring/runner.finalization.test.ts b/apps/api/src/services/monitoring/runner.finalization.test.tsnew file mode 100644index 000000000..25bd42853--- /dev/null+++ b/apps/api/src/services/monitoring/runner.finalization.test.ts@@ -0,0 +1,748 @@+import type { MonitorCheckRow, MonitorRow } from "./types";++vi.mock("../../config", () => ({ config: { USE_DB_AUTHENTICATION: true } }));+vi.mock("../../lib/logger", () => {+ const logger = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ child: vi.fn(),+ };+ logger.child.mockReturnValue(logger);+ return { logger };+});+vi.mock("../logging/log_job", () => ({}));+vi.mock("../../lib/gcs-monitoring", () => ({}));+vi.mock("../worker/scrape-worker", () => ({}));+vi.mock("../worker/nuq-router", () => ({}));+vi.mock("./diff", () => ({}));+vi.mock("../../lib/crawl-redis", () => ({}));+vi.mock("../queue-jobs", () => ({}));+vi.mock("../../controllers/v2/types", () => ({}));+vi.mock("../webhook", () => ({}));+vi.mock("./results", () => ({}));+vi.mock("../notification/monitoring_email", () => ({}));+vi.mock("../notification/monitoring_slack", () => ({}));+vi.mock("./types", () => ({}));+vi.mock("./interest", () => ({+ trackMonitorCheckStartedInterest: async () => {},+}));+vi.mock("./search/run", () => ({}));+vi.mock("./search/judge", () => ({}));+vi.mock("./search/dedupe", () => ({}));+vi.mock("./search/persist", () => ({}));+vi.mock("../../scraper/WebScraper/utils/blocklist", () => ({}));+vi.mock("../../controllers/auth", () => ({}));+vi.mock("./store", () => ({+ getMonitorCheckForUpdate: vi.fn(),+ getMonitorForUpdate: vi.fn(),+ listRunningMonitorChecks: vi.fn(),+ updateMonitorCheck: vi.fn(),+ updateMonitorCheckIfRunning: vi.fn(),+ updateMonitorCheckIfStatus: vi.fn(),+ markMonitorRunning: vi.fn(),+ countMonitorCheckPages: vi.fn(),+ calculateMonitorCheckActualCredits: vi.fn(),+ updateMonitorScheduleAfterRun: vi.fn(),+}));+vi.mock("../autumn/autumn.service", () => ({+ autumnService: { finalizeCreditsLock: vi.fn(), lockCredits: vi.fn() },+}));+vi.mock("../queue-service", () => ({ getBillingQueue: vi.fn() }));+vi.mock("../redis", () => ({+ redisEvictConnection: { set: vi.fn(), del: vi.fn(), eval: vi.fn() },+}));++import {+ processMonitorCheckJob,+ reconcileRunningMonitorChecks,+} from "./runner";+import * as store from "./store";+import { autumnService } from "../autumn/autumn.service";+import { getBillingQueue } from "../queue-service";+import { redisEvictConnection } from "../redis";++function deferred() {+ let resolve!: () => void;+ const promise = new Promise<void>(r => {+ resolve = r;+ });+ return { promise, resolve };+}++describe("monitor check finalization ownership", () => {+ let current: MonitorCheckRow;+ let snapshot: MonitorCheckRow;+ let monitor: MonitorRow;+ const locks = new Map<string, string>();+ const bill = vi.fn();+ const lockKey = "monitor-check-finalize:check-1";++ beforeEach(() => {+ vi.resetAllMocks();+ locks.clear();+ current = {+ id: "check-1",+ monitor_id: "monitor-1",+ team_id: "team-1",+ status: "running",+ billing_status: "reserved",+ autumn_lock_id: "monitor_check-1",+ reserved_credits: 1,+ started_at: new Date().toISOString(),+ target_results: [+ { targetId: "target-1", type: "scrape", expectedJobs: ["scrape-1"] },+ ],+ } as MonitorCheckRow;+ snapshot = structuredClone(current);+ monitor = {+ id: current.monitor_id,+ team_id: current.team_id,+ current_check_id: current.id,+ targets: [+ { id: "target-1", type: "scrape", urls: ["https://example.com"] },+ ],+ } as MonitorRow;+ vi.mocked(store.listRunningMonitorChecks).mockImplementation(async () => [+ structuredClone(snapshot),+ ]);+ vi.mocked(store.getMonitorCheckForUpdate).mockImplementation(async () =>+ structuredClone(current),+ );+ vi.mocked(store.getMonitorForUpdate).mockImplementation(+ async () => monitor,+ );+ vi.mocked(store.updateMonitorCheck).mockImplementation(+ async (_id, patch) => {+ current = { ...current, ...patch };+ return structuredClone(current);+ },+ );+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(+ async (_id, patch) => {+ if (current.status !== "running") return null;+ current = { ...current, ...patch };+ return structuredClone(current);+ },+ );+ vi.mocked(store.updateMonitorCheckIfStatus).mockImplementation(+ async (_id, expectedStatus, patch) => {+ if (current.status !== expectedStatus) return null;+ current = { ...current, ...patch };+ return structuredClone(current);+ },+ );+ vi.mocked(store.countMonitorCheckPages).mockImplementation(+ async ({ status }) => (!status || status === "same" ? 1 : 0),+ );+ vi.mocked(store.calculateMonitorCheckActualCredits).mockResolvedValue(1);+ vi.mocked(autumnService.finalizeCreditsLock).mockResolvedValue(true);+ vi.mocked(getBillingQueue).mockReturnValue({+ add: bill,+ } as unknown as ReturnType<typeof getBillingQueue>);+ vi.mocked(redisEvictConnection.set).mockImplementation((async (+ key: string,+ token: string,+ ) => {+ // Notifications are tested independently; this suite exercises settlement and scheduling.+ if (key.startsWith("monitor-check-notify:")) return null;+ if (locks.has(key)) return null;+ locks.set(key, token);+ return "OK";+ }) as any);+ vi.mocked(redisEvictConnection.del).mockImplementation((async (+ key: string,+ ) => Number(locks.delete(key))) as any);+ vi.mocked(redisEvictConnection.eval).mockImplementation((async (+ _script: string,+ _keys: number,+ key: string,+ token: string,+ ) => {+ if (locks.get(key) !== token) return 0;+ return Number(locks.delete(key));+ }) as any);+ });++ it("settles and schedules a completed check once when another batch retained its running snapshot", async () => {+ await reconcileRunningMonitorChecks();+ await reconcileRunningMonitorChecks();+ expect(current).toMatchObject({+ status: "completed",+ billing_status: "confirmed",+ });+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledTimes(1);+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledWith(+ expect.objectContaining({+ lockId: "monitor_check-1",+ action: "confirm",+ overrideValue: 1,+ }),+ );+ expect(bill).toHaveBeenCalledTimes(1);+ expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);+ });++ it("uses the refreshed target state instead of completing an obsolete snapshot", async () => {+ current.target_results = [+ { type: "search", targetId: "target-1", searchCompleted: false },+ ];+ await reconcileRunningMonitorChecks();+ expect(current.status).toBe("running");+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();+ });++ it.each(["completed", "partial", "failed"] as const)(+ "leaves an already %s check untouched",+ async status => {+ current.status = status;+ current.billing_status = "confirmed";+ await reconcileRunningMonitorChecks();+ expect(store.updateMonitorCheck).not.toHaveBeenCalled();+ expect(store.updateMonitorCheckIfRunning).not.toHaveBeenCalled();+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();+ },+ );++ it("skips a deleted check and releases its Redis lock", async () => {+ vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValue(null);+ await reconcileRunningMonitorChecks();+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();+ expect(locks.has(lockKey)).toBe(false);+ });++ it("allows only one settlement when the Redis lease expires while a worker is still preparing completion", async () => {+ const entered = deferred();+ const resume = deferred();+ vi.mocked(store.calculateMonitorCheckActualCredits).mockImplementationOnce(+ async () => {+ entered.resolve();+ await resume.promise;+ return 1;+ },+ );+ const first = reconcileRunningMonitorChecks();+ await entered.promise;+ locks.delete(lockKey); // The second worker can acquire an expired lease.+ await reconcileRunningMonitorChecks();+ resume.resolve();+ await first;+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledTimes(1);+ expect(bill).toHaveBeenCalledTimes(1);+ expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);+ expect(current.billing_status).toBe("confirmed");+ });++ it("does not delete a replacement owner's Redis lock", async () => {+ vi.mocked(store.calculateMonitorCheckActualCredits).mockImplementation(+ async () => {+ locks.set(lockKey, "replacement-owner");+ return 1;+ },+ );+ await reconcileRunningMonitorChecks();+ expect(locks.get(lockKey)).toBe("replacement-owner");+ });++ it.each(["stale", "orphan"])(+ "does not release a %s check whose terminal claim was won by another worker",+ async kind => {+ if (kind === "stale") {+ current.started_at = new Date(+ Date.now() - 2 * 60 * 60 * 1000,+ ).toISOString();+ snapshot = structuredClone(current);+ } else {+ vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);+ }+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(+ async () => {+ current = {+ ...current,+ status: "completed",+ billing_status: "confirmed",+ };+ return null;+ },+ );+ await reconcileRunningMonitorChecks();+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();+ expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();+ expect(current).toMatchObject({+ status: "completed",+ billing_status: "confirmed",+ });+ },+ );++ it("records a failed settlement without enqueueing billing", async () => {+ vi.mocked(autumnService.finalizeCreditsLock).mockResolvedValue(false);+ await reconcileRunningMonitorChecks();+ expect(current).toMatchObject({+ status: "completed",+ billing_status: "failed",+ });+ expect(bill).not.toHaveBeenCalled();+ });++ it.each(["stale", "orphan"])(+ "releases a %s hold after winning the failure transition",+ async kind => {+ if (kind === "stale") {+ current.started_at = new Date(+ Date.now() - 2 * 60 * 60 * 1000,+ ).toISOString();+ } else {+ vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);+ }+ await reconcileRunningMonitorChecks();+ expect(current).toMatchObject({+ status: "failed",+ billing_status: "released",+ });+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({+ action: "release",+ lockId: "monitor_check-1",+ }),+ );+ expect(+ vi.mocked(store.updateMonitorCheckIfRunning).mock+ .invocationCallOrder[0],+ ).toBeLessThan(+ vi.mocked(autumnService.finalizeCreditsLock).mock+ .invocationCallOrder[0],+ );+ },+ );++ it.each([+ { kind: "stale", throws: false },+ { kind: "stale", throws: true },+ { kind: "orphan", throws: false },+ { kind: "orphan", throws: true },+ { kind: "handler", throws: false },+ { kind: "handler", throws: true },+ ])(+ "records failed settlement for a $kind release that throws: $throws",+ async ({ kind, throws }) => {+ if (throws) {+ vi.mocked(autumnService.finalizeCreditsLock).mockRejectedValue(+ new Error("Release unavailable"),+ );+ } else {+ vi.mocked(autumnService.finalizeCreditsLock).mockResolvedValue(false);+ }+ if (kind === "handler") {+ monitor.targets = [];+ const failure = new Error("Target write failed");+ vi.mocked(autumnService.lockCredits).mockResolvedValue({+ status: "locked",+ lockId: "monitor_check-1",+ });+ const updateIfRunning = vi+ .mocked(store.updateMonitorCheckIfRunning)+ .getMockImplementation()!;+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(+ async (id, patch) => {+ if (patch.target_results) throw failure;+ return updateIfRunning(id, patch);+ },+ );+ await expect(+ processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ }),+ ).rejects.toThrow(failure);+ } else {+ if (kind === "stale") {+ current.started_at = new Date(+ Date.now() - 2 * 60 * 60 * 1000,+ ).toISOString();+ } else {+ vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);+ }+ await reconcileRunningMonitorChecks();+ }+ expect(current).toMatchObject({+ status: "failed",+ billing_status: "failed",+ });+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({+ lockId: "monitor_check-1",+ action: "release",+ }),+ );+ expect(bill).not.toHaveBeenCalled();+ },+ );++ it.each(["completed", "stale", "orphan"])(+ "settles the hold returned by the terminal claim for a %s check",+ async kind => {+ current.autumn_lock_id = null;+ current.billing_status = "not_applicable";+ if (kind === "stale") {+ current.started_at = new Date(+ Date.now() - 2 * 60 * 60 * 1000,+ ).toISOString();+ } else if (kind === "orphan") {+ vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);+ }+ const updateIfRunning = vi+ .mocked(store.updateMonitorCheckIfRunning)+ .getMockImplementation()!;+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(+ async (id, patch) => {+ if (patch.status) {+ // Reservation persisted after the primary read, just before the claim.+ current = {+ ...current,+ autumn_lock_id: "late-lock",+ reserved_credits: 2,+ partner_run_token: "late-token",+ billing_status: "reserved",+ };+ }+ return updateIfRunning(id, patch);+ },+ );+ await reconcileRunningMonitorChecks();+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({+ lockId: "late-lock",+ action: kind === "completed" ? "confirm" : "release",+ ...(kind === "completed"+ ? { externalRequestId: "late-token", heldValue: 2 }+ : {}),+ }),+ );+ expect(current.billing_status).toBe(+ kind === "completed" ? "confirmed" : "released",+ );+ },+ );++ it("does not reopen a check that became terminal after the job handler read it", async () => {+ const queued = { ...current, status: "queued" as const };+ vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValue(queued);+ current.status = "completed";+ current.billing_status = "confirmed";+ await processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ });+ expect(current).toMatchObject({+ status: "completed",+ billing_status: "confirmed",+ });+ expect(store.markMonitorRunning).not.toHaveBeenCalled();+ expect(autumnService.lockCredits).not.toHaveBeenCalled();+ });++ it.each([+ { status: "locked" as const, lockId: "monitor_check-1" },+ { status: "skipped" as const },+ { status: "denied" as const },+ ])(+ "preserves completion when a slow reservation returns $status",+ async result => {+ monitor.targets = [];+ vi.mocked(autumnService.lockCredits).mockImplementation(async () => {+ current = {+ ...current,+ status: "completed",+ billing_status: "confirmed",+ };+ return result;+ });+ await processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ });+ expect(current).toMatchObject({+ status: "completed",+ billing_status: "confirmed",+ });+ expect(store.updateMonitorCheck).not.toHaveBeenCalled();+ expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();+ },+ );+ it.each([+ {+ name: "an unowned hold",+ persistedLock: null,+ deleted: false,+ release: true,+ },+ {+ name: "a different hold",+ persistedLock: "other-lock",+ deleted: false,+ release: true,+ },+ {+ name: "the finalizer's hold",+ persistedLock: "monitor_check-1",+ deleted: false,+ release: false,+ },+ {+ name: "a deleted check's hold",+ persistedLock: null,+ deleted: true,+ release: true,+ },+ ])(+ "cleans up $name when a reservation arrives after completion",+ async ({ persistedLock, deleted, release }) => {+ current.autumn_lock_id = null;+ current.billing_status = "not_applicable";+ monitor.targets = [];+ vi.mocked(autumnService.lockCredits).mockImplementation(async () => {+ current = {+ ...current,+ status: "completed",+ billing_status: "confirmed",+ autumn_lock_id: persistedLock,+ };+ if (deleted) {+ vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValueOnce(null);+ }+ return {+ status: "locked",+ lockId: "monitor_check-1",+ operationToken: "operation-1",+ };+ });+ await processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ });+ expect(current).toMatchObject({+ status: "completed",+ billing_status: "confirmed",+ autumn_lock_id: persistedLock,+ });+ expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();+ if (release) {+ expect(+ autumnService.finalizeCreditsLock,+ ).toHaveBeenCalledExactlyOnceWith({+ lockId: "monitor_check-1",+ action: "release",+ properties: {+ source: "monitorCheck",+ endpoint: "monitor",+ jobId: current.id,+ },+ teamId: monitor.team_id,+ });+ } else {+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();+ }+ },+ );++ it.each(["queued", "running"] as const)(+ "does not release a late reservation whose current row is %s",+ async status => {+ current.autumn_lock_id = null;+ vi.mocked(autumnService.lockCredits).mockImplementation(async () => {+ current.status = "completed";+ vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValueOnce({+ ...current,+ status,+ });+ return { status: "locked", lockId: "monitor_check-1" };+ });+ await processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ });+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();+ },+ );++ it("does not release a late reservation without an authoritative ownership read", async () => {+ current.autumn_lock_id = null;+ const failure = new Error("Primary read unavailable");+ vi.mocked(autumnService.lockCredits).mockImplementation(async () => {+ current.status = "completed";+ vi.mocked(store.getMonitorCheckForUpdate).mockRejectedValue(failure);+ return { status: "locked", lockId: "monitor_check-1" };+ });+ await expect(+ processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ }),+ ).rejects.toThrow(failure);+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();+ });++ it.each([+ { localLock: "monitor_check-1", persistedLock: "other-lock" },+ { localLock: null, persistedLock: "other-lock" },+ { localLock: "monitor_check-1", persistedLock: null },+ { localLock: "monitor_check-1", persistedLock: "monitor_check-1" },+ ])(+ "releases owned holds after a handler failure with $localLock and $persistedLock",+ async ({ localLock, persistedLock }) => {+ monitor.targets = [];+ const failure = new Error("Target write failed");+ vi.mocked(autumnService.lockCredits).mockResolvedValue(+ localLock+ ? { status: "locked", lockId: localLock }+ : { status: "skipped" },+ );+ const updateIfRunning = vi+ .mocked(store.updateMonitorCheckIfRunning)+ .getMockImplementation()!;+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(+ async (id, patch) => {+ if (patch.target_results) {+ current.autumn_lock_id = persistedLock;+ throw failure;+ }+ return updateIfRunning(id, patch);+ },+ );+ await expect(+ processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ }),+ ).rejects.toThrow(failure);+ const expectedLocks = [+ ...new Set([localLock, persistedLock].filter(Boolean)),+ ].sort();+ expect(+ vi+ .mocked(autumnService.finalizeCreditsLock)+ .mock.calls.map(([call]) => call.lockId)+ .sort(),+ ).toEqual(expectedLocks);+ expect(+ vi+ .mocked(autumnService.finalizeCreditsLock)+ .mock.calls.every(([call]) => call.action === "release"),+ ).toBe(true);+ expect(current).toMatchObject({+ status: "failed",+ billing_status: "released",+ });+ },+ );++ it.each([false, true])(+ "releases an unpersisted hold when reservation storage throws and another finalizer wins: %s",+ async terminal => {+ current.autumn_lock_id = null;+ const failure = new Error("Reservation write failed");+ vi.mocked(autumnService.lockCredits).mockResolvedValue({+ status: "locked",+ lockId: "monitor_check-1",+ });+ const updateIfRunning = vi+ .mocked(store.updateMonitorCheckIfRunning)+ .getMockImplementation()!;+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(+ async (id, patch) => {+ if (Object.hasOwn(patch, "autumn_lock_id")) {+ if (terminal) {+ current.status = "completed";+ current.billing_status = "not_applicable";+ }+ throw failure;+ }+ return updateIfRunning(id, patch);+ },+ );+ await expect(+ processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ }),+ ).rejects.toThrow(failure);+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({+ lockId: "monitor_check-1",+ action: "release",+ }),+ );+ expect(current.status).toBe(terminal ? "completed" : "failed");+ if (terminal)+ expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();+ },+ );++ it.each([1, 2])(+ "preserves finalized results when target write %i arrives late",+ async lateWrite => {+ monitor.targets = [];+ vi.mocked(autumnService.lockCredits).mockResolvedValue({+ status: "locked",+ lockId: "monitor_check-1",+ });+ const finalizedResults = [+ {+ type: "scrape",+ targetId: "target-1",+ expectedJobs: ["finished-scrape"],+ },+ ];+ let targetWrites = 0;+ const finishBeforeWrite = (patch: Partial<MonitorCheckRow>) => {+ if (patch.target_results && ++targetWrites === lateWrite) {+ current = {+ ...current,+ status: "completed",+ billing_status: "confirmed",+ target_results: finalizedResults,+ };+ }+ };+ const update = vi+ .mocked(store.updateMonitorCheck)+ .getMockImplementation()!;+ const updateIfRunning = vi+ .mocked(store.updateMonitorCheckIfRunning)+ .getMockImplementation()!;+ vi.mocked(store.updateMonitorCheck).mockImplementation(+ async (id, patch) => {+ finishBeforeWrite(patch);+ return update(id, patch);+ },+ );+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(+ async (id, patch) => {+ finishBeforeWrite(patch);+ return updateIfRunning(id, patch);+ },+ );+ await processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ });+ expect(current).toMatchObject({+ status: "completed",+ billing_status: "confirmed",+ target_results: finalizedResults,+ });+ expect(targetWrites).toBe(lateWrite);+ },+ );+});diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 3b6c5b94e..e72706d1b 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -37,7 +37,7 @@ import {bulkUpsertMonitorPages,calculateMonitorCheckActualCredits,countRecentConsecutiveSkippedForCredits,- getMonitorCheck,+ getMonitorCheckForUpdate,getMonitorForUpdate,countMonitorCheckPages,insertMonitorCheckPages,@@ -49,6 +49,7 @@ import {pauseMonitor,updateMonitorCheck,updateMonitorCheckIfRunning,+ updateMonitorCheckIfStatus,updateMonitorScheduleAfterRun,upsertMonitorPage,} from "./store";@@ -95,13 +96,6 @@ const MONITOR_CHECK_REVOKED_ERROR =* reached by a 402: the current denial must itself be `job_revoked`.*/const MONITOR_GATE_REVOKED_STREAK = 3;-const TERMINAL_CHECK_STATUSES = new Set([- "completed",- "partial",- "failed",- "skipped_overlap",- "skipped_no_credits",-]);async function claimMonitorNotification(checkId: string): Promise<boolean> {const result = await redisEvictConnection.set(@@ -936,6 +930,38 @@ export function findCompletedSearchTargetRun(return (match as Record<string, unknown>) ?? null;}+async function releaseUnpersistedMonitorHold(+ monitor: MonitorRow,+ checkId: string,+ lockId: string | null,+): Promise<void> {+ if (!lockId) return;+ const latest = await getMonitorCheckForUpdate(+ monitor.team_id,+ monitor.id,+ checkId,+ );+ // A duplicate delivery can receive the same lock ID as the finalizer.+ // Only release an unpersisted hold after no running writer can adopt it.+ if (+ !latest ||+ (latest.status !== "queued" &&+ latest.status !== "running" &&+ latest.autumn_lock_id !== lockId)+ ) {+ await autumnService.finalizeCreditsLock({+ lockId,+ action: "release",+ properties: {+ source: "monitorCheck",+ endpoint: "monitor",+ jobId: checkId,+ },+ teamId: monitor.team_id,+ });+ }+}+export async function processMonitorCheckJob(job: MonitorCheckJobData,): Promise<void> {@@ -944,7 +970,7 @@ export async function processMonitorCheckJob(throw new Error("Monitor not found");}- const initialCheck = await getMonitorCheck(+ const initialCheck = await getMonitorCheckForUpdate(job.teamId,job.monitorId,job.checkId,@@ -952,20 +978,26 @@ export async function processMonitorCheckJob(if (!initialCheck) {throw new Error("Monitor check not found");}- if (TERMINAL_CHECK_STATUSES.has(initialCheck.status)) {+ if (initialCheck.status !== "queued" && initialCheck.status !== "running") {return;}+ const started = await updateMonitorCheckIfStatus(+ job.checkId,+ initialCheck.status,+ {+ status: "running",+ started_at: new Date().toISOString(),+ },+ );+ if (!started) return;+ let check: MonitorCheckRow = started;+await markMonitorRunning({monitorId: monitor.id,checkId: job.checkId,});- let check: MonitorCheckRow = await updateMonitorCheck(job.checkId, {- status: "running",- started_at: new Date().toISOString(),- });-trackMonitorCheckStartedInterest({ monitor, check }).catch(error =>logger.warn("Failed to track monitor target interest", {error,@@ -997,7 +1029,7 @@ export async function processMonitorCheckJob(if (lock.status === "denied") {const revoked = lock.reason === "job_revoked";- check = await updateMonitorCheck(check.id, {+ const skipped = await updateMonitorCheckIfRunning(check.id, {status: "skipped_no_credits",finished_at: new Date().toISOString(),actual_credits: 0,@@ -1007,6 +1039,9 @@ export async function processMonitorCheckJob(: MONITOR_CHECK_NO_CREDITS_ERROR,});+ if (!skipped) return;+ check = skipped;+// A revoked job never becomes unrevoked; see MONITOR_GATE_REVOKED_STREAK// for why it is waited out rather than acted on at once.const paused =@@ -1045,7 +1080,7 @@ export async function processMonitorCheckJob(lockId = lock.status === "locked" ? lock.lockId : null;- check = await updateMonitorCheck(check.id, {+ const reserved = await updateMonitorCheckIfRunning(check.id, {autumn_lock_id: lockId,// A token already on the row wins: the gate was not re-asked, so there// is no newer one, and null would lose the authorized operation.@@ -1056,10 +1091,17 @@ export async function processMonitorCheckJob(billing_status: lockId ? "reserved" : "not_applicable",});+ if (!reserved) {+ await releaseUnpersistedMonitorHold(monitor, check.id, lockId);+ return;+ }+ check = reserved;+const targetResults = monitor.targets.map(createMonitorTargetRun);- await updateMonitorCheck(check.id, {+ const initialized = await updateMonitorCheckIfRunning(check.id, {target_results: targetResults,});+ if (!initialized) return;for (const [index, target] of monitor.targets.entries()) {const targetRun = targetResults[index];@@ -1097,7 +1139,7 @@ export async function processMonitorCheckJob(targetRun.searchCompleted = true;// Persist searchCompleted now so a crash/redelivery short-circuits via// findCompletedSearchTargetRun instead of re-running and re-billing.- await withFinalizeTimeout(+ const persisted = await withFinalizeTimeout(signal =>signal.aborted? Promise.resolve(null)@@ -1108,10 +1150,11 @@ export async function processMonitorCheckJob(}),"monitor search searchCompleted flush",);+ if (!persisted) return;}}- await updateMonitorCheck(check.id, {+ await updateMonitorCheckIfRunning(check.id, {target_results: targetResults,});} catch (error) {@@ -1123,27 +1166,46 @@ export async function processMonitorCheckJob(const failed = await updateMonitorCheckIfRunning(check.id, {status: "failed",finished_at: new Date().toISOString(),- billing_status: lockId ? "released" : "failed",error: error instanceof Error ? error.message : String(error),});if (!failed) {+ await releaseUnpersistedMonitorHold(monitor, check.id, lockId);throw error;}check = failed;- if (lockId) {- await autumnService.finalizeCreditsLock({- lockId,- action: "release",- properties: {- source: "monitorCheck",- endpoint: "monitor",- jobId: check.id,- },- teamId: monitor.team_id,- });+ // The claim owns the persisted hold; a failed reservation write can also+ // leave this handler with a newly acquired hold that was never stored.+ let released = true;+ for (const ownedLockId of new Set([failed.autumn_lock_id, lockId])) {+ if (!ownedLockId) continue;+ const settled = await autumnService+ .finalizeCreditsLock({+ lockId: ownedLockId,+ action: "release",+ properties: {+ source: "monitorCheck",+ endpoint: "monitor",+ jobId: check.id,+ },+ teamId: monitor.team_id,+ })+ .catch(releaseError => {+ logger.warn("Failed to release monitor check credit lock", {+ error: releaseError,+ monitorId: monitor.id,+ checkId: check.id,+ lockId: ownedLockId,+ });+ return false;+ });+ released = released && settled;}+ check = await updateMonitorCheck(failed.id, {+ billing_status:+ (failed.autumn_lock_id || lockId) && released ? "released" : "failed",+ });if (await claimMonitorNotification(check.id).catch(error => {@@ -1307,10 +1369,19 @@ async function failStaleMonitorCheck(params: {return false;const error = MONITOR_CHECK_STALE_ERROR;- if (params.check.autumn_lock_id) {- await autumnService+ const claimed = await updateMonitorCheckIfRunning(params.check.id, {+ status: "failed",+ finished_at: new Date().toISOString(),+ actual_credits: 0,+ error,+ });+ if (!claimed) return true;++ let released = true;+ if (claimed.autumn_lock_id) {+ released = await autumnService.finalizeCreditsLock({- lockId: params.check.autumn_lock_id,+ lockId: claimed.autumn_lock_id,action: "release",properties: {source: "monitorCheck",@@ -1324,19 +1395,19 @@ async function failStaleMonitorCheck(params: {error: releaseError,monitorId: params.monitor.id,checkId: params.check.id,- lockId: params.check.autumn_lock_id,+ lockId: claimed.autumn_lock_id,});+ return false;});}- const finalized = await updateMonitorCheck(params.check.id, {- status: "failed",- finished_at: new Date().toISOString(),- actual_credits: 0,- billing_status: params.check.autumn_lock_id ? "released" : "not_applicable",- error,+ const finalized = await updateMonitorCheck(claimed.id, {+ billing_status: !claimed.autumn_lock_id+ ? "not_applicable"+ : released+ ? "released"+ : "failed",});-let withNotifications = finalized;if (await claimMonitorNotification(params.check.id)) {const notificationStatus = await sendNotifications({@@ -1403,21 +1474,46 @@ export async function reconcileRunningMonitorChecks(limit: number = 50,): Promise<void> {const checks = await listRunningMonitorChecks(limit);- for (const check of checks) {- const lockKey = `monitor-check-finalize:${check.id}`;- const lock = await redisEvictConnection.set(lockKey, "1", "EX", 60, "NX");+ for (const candidate of checks) {+ const lockKey = `monitor-check-finalize:${candidate.id}`;+ const lockToken = uuidv7();+ const lock = await redisEvictConnection.set(+ lockKey,+ lockToken,+ "EX",+ 60,+ "NX",+ );if (lock !== "OK") continue;try {+ // The batch can outlive another finalizer. Read from the primary after+ // acquiring the lease rather than acting on that old running snapshot.+ const check = await getMonitorCheckForUpdate(+ candidate.team_id,+ candidate.monitor_id,+ candidate.id,+ );+ if (!check || check.status !== "running") continue;+const monitor = await getMonitorForUpdate(check.team_id,check.monitor_id,);if (!monitor) {- if (check.autumn_lock_id) {- await autumnService+ const failed = await updateMonitorCheckIfRunning(check.id, {+ status: "failed",+ finished_at: new Date().toISOString(),+ actual_credits: 0,+ error: "Monitor no longer exists.",+ });+ if (!failed) continue;++ let released = true;+ if (failed.autumn_lock_id) {+ released = await autumnService.finalizeCreditsLock({- lockId: check.autumn_lock_id,+ lockId: failed.autumn_lock_id,action: "release",properties: {source: "monitorCheck",@@ -1433,20 +1529,20 @@ export async function reconcileRunningMonitorChecks(error,monitorId: check.monitor_id,checkId: check.id,- lockId: check.autumn_lock_id,+ lockId: failed.autumn_lock_id,},);+ return false;});}- await updateMonitorCheck(check.id, {- status: "failed",- finished_at: new Date().toISOString(),- actual_credits: 0,- billing_status: check.autumn_lock_id ? "released" : "not_applicable",- error: "Monitor no longer exists.",+ await updateMonitorCheck(failed.id, {+ billing_status: !failed.autumn_lock_id+ ? "not_applicable"+ : released+ ? "released"+ : "failed",});-logger.warn("Failed orphaned monitor check", {monitorId: check.monitor_id,checkId: check.id,@@ -1456,8 +1552,7 @@ export async function reconcileRunningMonitorChecks(if (await failStaleMonitorCheck({ monitor, check })) continue;- // Snapshot from listRunningMonitorChecks; may be stale relative to the- // inline handler that is still writing this check.+ // The inline handler may still write target results after our primary read.let targetResults = Array.isArray(check.target_results)? ([...check.target_results] as any[]): [];@@ -1494,7 +1589,9 @@ export async function reconcileRunningMonitorChecks(// reaper. The complete-path write below is safe: a search target can only// be complete once its snapshot already carries searchCompleted=true.if (recoveredFromEmpty && targetResults.length > 0) {- await updateMonitorCheck(check.id, { target_results: targetResults });+ await updateMonitorCheckIfRunning(check.id, {+ target_results: targetResults,+ });}continue;}@@ -1514,14 +1611,12 @@ export async function reconcileRunningMonitorChecks(targetResults,});- let finalized = await updateMonitorCheck(check.id, {+ // This conditional write is the durable claim. Even if the Redis lease+ // expires during preparation, only one worker may settle this check.+ const claimed = await updateMonitorCheckIfRunning(check.id, {status: errorCount > 0 ? "partial" : "completed",finished_at: new Date().toISOString(),actual_credits: actualCredits,- // Not `confirmed` yet — the settle has not been attempted. Claiming it- // here and then finding firebill refused leaves a run permanently- // recorded as billed that nobody billed, with no retry.- billing_status: check.autumn_lock_id ? "reserved" : "not_applicable",total_pages: totalPages,same_count: same,changed_count: changed,@@ -1531,13 +1626,16 @@ export async function reconcileRunningMonitorChecks(target_results: targetResults,});+ if (!claimed) continue;+ let finalized = claimed;+let settled = false;try {settled = await billMonitorCheck({monitor,check: finalized,actualCredits,- lockId: check.autumn_lock_id,+ lockId: claimed.autumn_lock_id,});} catch (error) {logger.warn("Failed to bill monitor check during reconciliation", {@@ -1550,14 +1648,14 @@ export async function reconcileRunningMonitorChecks(// A refusal and a throw are the same fact — the settle did not land — and// both must be recorded as such. `firebillFinalize` answers `false`// without throwing, so the catch alone never saw them.- if (check.autumn_lock_id) {+ if (claimed.autumn_lock_id) {if (!settled) {logger.error("Monitor check settle did not land; the hold is unsettled and this run is unbilled",{monitorId: monitor.id,checkId: finalized.id,- lockId: check.autumn_lock_id,+ lockId: claimed.autumn_lock_id,actualCredits,},);@@ -1655,10 +1753,19 @@ export async function reconcileRunningMonitorChecks(} catch (error) {logger.warn("Failed to reconcile monitor check", {error,- checkId: check.id,+ checkId: candidate.id,});} finally {- await redisEvictConnection.del(lockKey);+ // An expired lease may already belong to another worker.+ await redisEvictConnection.eval(+ `if redis.call("get", KEYS[1]) == ARGV[1] then+ return redis.call("del", KEYS[1])+ end+ return 0`,+ 1,+ lockKey,+ lockToken,+ );}}}diff --git a/apps/api/src/services/monitoring/scheduler.test.ts b/apps/api/src/services/monitoring/scheduler.test.tsindex 25e29e7c9..bdded5c43 100644--- a/apps/api/src/services/monitoring/scheduler.test.ts+++ b/apps/api/src/services/monitoring/scheduler.test.ts@@ -7,8 +7,9 @@ import {claimDueMonitors,createMonitorCheck,dispatchScheduledMonitorCheck,- getMonitorCheck,+ getMonitorCheckForUpdate,updateMonitorCheck,+ updateMonitorCheckIfStatus,updateMonitorScheduleAfterRun,} from "./store";import { autumnService } from "../autumn/autumn.service";@@ -22,8 +23,9 @@ vi.mock("./store", () => ({claimDueMonitors: vi.fn(),createMonitorCheck: vi.fn(),dispatchScheduledMonitorCheck: vi.fn(),- getMonitorCheck: vi.fn(),+ getMonitorCheckForUpdate: vi.fn(),updateMonitorCheck: vi.fn(),+ updateMonitorCheckIfStatus: vi.fn(),updateMonitorScheduleAfterRun: vi.fn(),}));@@ -52,8 +54,8 @@ const mockDispatchScheduledMonitorCheck =dispatchScheduledMonitorCheck as MockedFunction<typeof dispatchScheduledMonitorCheck>;-const mockGetMonitorCheck = getMonitorCheck as MockedFunction<- typeof getMonitorCheck+const mockGetMonitorCheckForUpdate = getMonitorCheckForUpdate as MockedFunction<+ typeof getMonitorCheckForUpdate>;const mockIsMonitorCheckStale = isMonitorCheckStale as MockedFunction<typeof isMonitorCheckStale@@ -92,11 +94,18 @@ describe("monitoring scheduler", () => {mockCreateMonitorCheck.mockResolvedValue(check);mockDispatchScheduledMonitorCheck.mockResolvedValue(true);mockAddMonitorCheckJob.mockResolvedValue(undefined);+ mockUpdateMonitorCheck.mockImplementation(+ async (id, patch) =>+ ({+ id,+ ...patch,+ }) as any,+ );mockAdvanceMonitorAfterSkippedCheck.mockResolvedValue(undefined);mockUpdateMonitorScheduleAfterRun.mockResolvedValue(undefined);- mockGetMonitorCheck.mockResolvedValue(null);+ mockGetMonitorCheckForUpdate.mockResolvedValue(null);mockIsMonitorCheckStale.mockReturnValue(false);- mockFinalizeCreditsLock.mockResolvedValue(undefined as any);+ mockFinalizeCreditsLock.mockResolvedValue(true);});it("dispatches and advances a scheduled monitor before enqueueing its job", async () => {@@ -187,6 +196,36 @@ describe("monitoring scheduler", () => {});});+ it("records an overlap while the current check is still active", async () => {+ const active = { id: "active-check", status: "running" } as any;+ const monitorWithCurrentCheck = {+ ...monitor,+ current_check_id: active.id,+ };+ const skipped = { id: check.id, status: "skipped_overlap" } as any;+ mockClaimDueMonitors.mockResolvedValue([monitorWithCurrentCheck]);+ mockGetMonitorCheckForUpdate.mockResolvedValue(active);+ mockUpdateMonitorCheck.mockResolvedValue(skipped);++ await expect(enqueueDueMonitorChecks()).resolves.toBe(0);++ expect(mockCreateMonitorCheck).toHaveBeenCalledExactlyOnceWith({+ monitor: monitorWithCurrentCheck,+ trigger: "scheduled",+ scheduledFor: monitor.next_run_at,+ status: "skipped_overlap",+ });+ expect(mockAdvanceMonitorAfterSkippedCheck).toHaveBeenCalledExactlyOnceWith(+ {+ monitor: monitorWithCurrentCheck,+ check: skipped,+ },+ );+ expect(updateMonitorCheckIfStatus).not.toHaveBeenCalled();+ expect(mockDispatchScheduledMonitorCheck).not.toHaveBeenCalled();+ expect(mockAddMonitorCheckJob).not.toHaveBeenCalled();+ });+it("clears a stale current check before enqueueing a scheduled run", async () => {const monitorWithCurrentCheck = {...monitor,@@ -194,25 +233,33 @@ describe("monitoring scheduler", () => {} as any;const staleCheck = { id: "stale-check", status: "running" } as any;const failedStaleCheck = { ...staleCheck, status: "failed" } as any;+ const finalizedStaleCheck = {+ ...failedStaleCheck,+ billing_status: "not_applicable",+ };mockClaimDueMonitors.mockResolvedValue([monitorWithCurrentCheck]);- mockGetMonitorCheck.mockResolvedValue(staleCheck);+ mockGetMonitorCheckForUpdate.mockResolvedValue(staleCheck);mockIsMonitorCheckStale.mockReturnValue(true);- mockUpdateMonitorCheck.mockResolvedValue(failedStaleCheck);+ vi.mocked(updateMonitorCheckIfStatus).mockResolvedValue(failedStaleCheck);+ mockUpdateMonitorCheck.mockResolvedValue(finalizedStaleCheck);await expect(enqueueDueMonitorChecks({ workerId: "worker-1" }),).resolves.toBe(1);- expect(mockUpdateMonitorCheck).toHaveBeenCalledWith(staleCheck.id, {- status: "failed",- finished_at: expect.any(String),- actual_credits: 0,- billing_status: "not_applicable",- error: "Monitor check exceeded the 1 hour running timeout.",- });+ expect(updateMonitorCheckIfStatus).toHaveBeenCalledWith(+ staleCheck.id,+ "running",+ {+ status: "failed",+ finished_at: expect.any(String),+ actual_credits: 0,+ error: "Monitor check exceeded the 1 hour running timeout.",+ },+ );expect(mockUpdateMonitorScheduleAfterRun).toHaveBeenCalledWith({monitor: monitorWithCurrentCheck,- check: failedStaleCheck,+ check: finalizedStaleCheck,});expect(mockCreateMonitorCheck).toHaveBeenCalledWith({monitor: { ...monitorWithCurrentCheck, current_check_id: null },@@ -228,4 +275,160 @@ describe("monitoring scheduler", () => {{ search: false },);});+ it.each(["running", "queued"] as const)(+ "releases a stale %s hold only after claiming failure",+ async status => {+ const stale = {+ id: "stale-check",+ status,+ autumn_lock_id: "monitor_stale-check",+ } as any;+ mockClaimDueMonitors.mockResolvedValue([+ { ...monitor, current_check_id: stale.id },+ ]);+ mockGetMonitorCheckForUpdate.mockResolvedValue(stale);+ mockIsMonitorCheckStale.mockReturnValue(true);+ vi.mocked(updateMonitorCheckIfStatus).mockResolvedValue({+ ...stale,+ status: "failed",+ });++ await enqueueDueMonitorChecks();++ expect(updateMonitorCheckIfStatus).toHaveBeenCalledWith(+ stale.id,+ status,+ expect.objectContaining({ status: "failed" }),+ );+ expect(mockFinalizeCreditsLock).toHaveBeenCalledWith(+ expect.objectContaining({+ lockId: stale.autumn_lock_id,+ action: "release",+ }),+ );+ expect(+ vi.mocked(updateMonitorCheckIfStatus).mock.invocationCallOrder[0],+ ).toBeLessThan(mockFinalizeCreditsLock.mock.invocationCallOrder[0]);+ },+ );++ it("does not release or rewrite a check when another finalizer wins the terminal claim", async () => {+ const stale = {+ id: "stale-check",+ status: "running",+ autumn_lock_id: "monitor_stale-check",+ } as any;+ mockClaimDueMonitors.mockResolvedValue([+ { ...monitor, current_check_id: stale.id },+ ]);+ mockGetMonitorCheckForUpdate.mockResolvedValue(stale);+ mockIsMonitorCheckStale.mockReturnValue(true);+ vi.mocked(updateMonitorCheckIfStatus).mockResolvedValue(null);++ await enqueueDueMonitorChecks();++ expect(mockFinalizeCreditsLock).not.toHaveBeenCalled();+ expect(mockUpdateMonitorScheduleAfterRun).not.toHaveBeenCalled();+ expect(mockCreateMonitorCheck).not.toHaveBeenCalled();+ expect(mockUpdateMonitorCheck).not.toHaveBeenCalled();+ expect(mockAdvanceMonitorAfterSkippedCheck).not.toHaveBeenCalled();+ expect(mockDispatchScheduledMonitorCheck).not.toHaveBeenCalled();+ expect(mockAddMonitorCheckJob).not.toHaveBeenCalled();+ });++ it.each([+ { previousLock: null, claimedLock: "current-lock" },+ { previousLock: "previous-lock", claimedLock: null },+ { previousLock: "previous-lock", claimedLock: "current-lock" },+ ])(+ "settles the claimed hold when it changes from $previousLock to $claimedLock",+ async ({ previousLock, claimedLock }) => {+ const current = {+ id: "stale-check",+ status: "running",+ autumn_lock_id: previousLock,+ } as any;+ const claimed = {+ ...current,+ status: "failed",+ autumn_lock_id: claimedLock,+ };+ mockClaimDueMonitors.mockResolvedValue([+ { ...monitor, current_check_id: current.id },+ ]);+ mockGetMonitorCheckForUpdate.mockResolvedValue(current);+ mockIsMonitorCheckStale.mockReturnValue(true);+ vi.mocked(updateMonitorCheckIfStatus).mockResolvedValue(claimed);+ mockUpdateMonitorCheck.mockImplementation(async (_id, patch) => ({+ ...claimed,+ ...patch,+ }));++ await enqueueDueMonitorChecks();++ expect(+ vi.mocked(updateMonitorCheckIfStatus).mock.calls[0][2],+ ).not.toHaveProperty("billing_status");+ if (claimedLock) {+ expect(mockFinalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({ lockId: claimedLock, action: "release" }),+ );+ } else {+ expect(mockFinalizeCreditsLock).not.toHaveBeenCalled();+ }+ expect(mockUpdateMonitorCheck).toHaveBeenCalledWith(current.id, {+ billing_status: claimedLock ? "released" : "not_applicable",+ });+ expect(mockUpdateMonitorScheduleAfterRun).toHaveBeenCalledWith({+ monitor: expect.objectContaining({ id: monitor.id }),+ check: expect.objectContaining({+ autumn_lock_id: claimedLock,+ billing_status: claimedLock ? "released" : "not_applicable",+ }),+ });+ },+ );++ it.each(["success", "refused", "throw"] as const)(+ "records a stale hold release outcome of %s before advancing the schedule",+ async outcome => {+ const current = {+ id: "stale-check",+ status: "running",+ autumn_lock_id: "monitor_stale-check",+ } as any;+ const claimed = { ...current, status: "failed" };+ mockClaimDueMonitors.mockResolvedValue([+ { ...monitor, current_check_id: current.id },+ ]);+ mockGetMonitorCheckForUpdate.mockResolvedValue(current);+ mockIsMonitorCheckStale.mockReturnValue(true);+ vi.mocked(updateMonitorCheckIfStatus).mockResolvedValue(claimed);+ mockUpdateMonitorCheck.mockImplementation(async (_id, patch) => ({+ ...claimed,+ ...patch,+ }));+ if (outcome === "throw") {+ mockFinalizeCreditsLock.mockRejectedValue(+ new Error("Release unavailable"),+ );+ } else {+ mockFinalizeCreditsLock.mockResolvedValue(outcome === "success");+ }++ await expect(enqueueDueMonitorChecks()).resolves.toBe(1);++ const billingStatus = outcome === "success" ? "released" : "failed";+ expect(mockUpdateMonitorCheck).toHaveBeenCalledWith(current.id, {+ billing_status: billingStatus,+ });+ expect(mockUpdateMonitorScheduleAfterRun).toHaveBeenCalledExactlyOnceWith(+ {+ monitor: expect.objectContaining({ id: monitor.id }),+ check: { ...claimed, billing_status: billingStatus },+ },+ );+ expect(mockAddMonitorCheckJob).toHaveBeenCalledTimes(1);+ },+ );});diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.tsindex 20c3855e7..d9704a543 100644--- a/apps/api/src/services/monitoring/scheduler.ts+++ b/apps/api/src/services/monitoring/scheduler.ts@@ -7,8 +7,9 @@ import {createMonitorCheck,deferMonitorClaim,dispatchScheduledMonitorCheck,- getMonitorCheck,+ getMonitorCheckForUpdate,updateMonitorCheck,+ updateMonitorCheckIfStatus,updateMonitorScheduleAfterRun,} from "./store";import { autumnService } from "../autumn/autumn.service";@@ -62,8 +63,9 @@ export async function enqueueDueMonitorChecks(if (await deferForJitter(monitor)) continue;if (monitor.current_check_id) {- const cleared = await clearFinishedOrStaleCurrentCheck(monitor);- if (cleared) {+ const resolution = await clearFinishedOrStaleCurrentCheck(monitor);+ if (resolution === "lost_claim") continue;+ if (resolution === "cleared") {monitor = { ...monitor, current_check_id: null };}}@@ -178,29 +180,44 @@ async function deferForJitter(monitor: MonitorRow): Promise<boolean> {async function clearFinishedOrStaleCurrentCheck(monitor: MonitorRow,-): Promise<boolean> {- if (!monitor.current_check_id) return true;+): Promise<"cleared" | "active" | "lost_claim"> {+ if (!monitor.current_check_id) return "cleared";- const current = await getMonitorCheck(+ const current = await getMonitorCheckForUpdate(monitor.team_id,monitor.id,monitor.current_check_id,);- if (!current) return false;+ if (!current) return "active";if (current.status === "running" || current.status === "queued") {if (!isMonitorCheckStale(current, new Date(), monitor.targets))- return false;-- if (current.autumn_lock_id) {- await autumnService+ return "active";++ // Compete with the reconciler before releasing its hold. A failed claim+ // means another worker owns completion; leave its billing and schedule alone.+ const failed = await updateMonitorCheckIfStatus(+ current.id,+ current.status,+ {+ status: "failed",+ finished_at: new Date().toISOString(),+ actual_credits: 0,+ error: MONITOR_CHECK_STALE_ERROR,+ },+ );+ if (!failed) return "lost_claim";++ let released = true;+ if (failed.autumn_lock_id) {+ released = await autumnService.finalizeCreditsLock({- lockId: current.autumn_lock_id,+ lockId: failed.autumn_lock_id,action: "release",properties: {source: "monitorCheck",endpoint: "monitor",- jobId: current.id,+ jobId: failed.id,},teamId: monitor.team_id,})@@ -208,29 +225,30 @@ async function clearFinishedOrStaleCurrentCheck(logger.warn("Failed to release stale monitor check credit lock", {error,monitorId: monitor.id,- checkId: current.id,- lockId: current.autumn_lock_id,+ checkId: failed.id,+ lockId: failed.autumn_lock_id,});+ return false;});}- const failed = await updateMonitorCheck(current.id, {- status: "failed",- finished_at: new Date().toISOString(),- actual_credits: 0,- billing_status: current.autumn_lock_id ? "released" : "not_applicable",- error: MONITOR_CHECK_STALE_ERROR,+ const finalized = await updateMonitorCheck(failed.id, {+ billing_status: failed.autumn_lock_id+ ? released+ ? "released"+ : "failed"+ : "not_applicable",});await updateMonitorScheduleAfterRun({monitor,- check: failed,+ check: finalized,});- return true;+ return "cleared";}await updateMonitorScheduleAfterRun({monitor,check: current,});- return true;+ return "cleared";}diff --git a/apps/api/src/services/monitoring/store.finalization.test.ts b/apps/api/src/services/monitoring/store.finalization.test.tsnew file mode 100644index 000000000..e6c941c78--- /dev/null+++ b/apps/api/src/services/monitoring/store.finalization.test.ts@@ -0,0 +1,147 @@+import { PgDialect } from "drizzle-orm/pg-core";++const {+ primarySelect,+ replicaSelect,+ primaryUpdate,+ where,+ set,+ limit,+ returning,+} = vi.hoisted(() => {+ const where = vi.fn();+ const set = vi.fn();+ const limit = vi.fn();+ const returning = vi.fn();+ const chain = {+ from: vi.fn(() => chain),+ set: set.mockImplementation(() => chain),+ where: where.mockImplementation(() => chain),+ limit,+ returning,+ };+ return {+ primarySelect: vi.fn(() => chain),+ replicaSelect: vi.fn(() => chain),+ primaryUpdate: vi.fn(() => chain),+ where,+ set,+ limit,+ returning,+ };+});++vi.mock("../../db/connection", () => ({+ db: { select: primarySelect, update: primaryUpdate },+ dbRr: { select: replicaSelect },+}));+vi.mock("../../db/rpc", () => ({ monitoringClaimDueMonitors: vi.fn() }));++import {+ getMonitorCheckForUpdate,+ updateMonitorCheckIfRunning,+ updateMonitorCheckIfStatus,+} from "./store";++const teamId = "11111111-1111-4111-8111-111111111111";+const monitorId = "22222222-2222-4222-8222-222222222222";+const checkId = "33333333-3333-4333-8333-333333333333";++beforeEach(() => {+ vi.clearAllMocks();+ limit.mockResolvedValue([]);+ returning.mockResolvedValue([]);+});++function queryPredicate() {+ expect(where).toHaveBeenCalledTimes(1);+ return new PgDialect().sqlToQuery(where.mock.calls[0][0]);+}++describe("monitor check finalization storage", () => {+ it("reads the current check from primary scoped to its team and monitor", async () => {+ const confirmed = {+ id: checkId,+ monitor_id: monitorId,+ team_id: teamId,+ status: "completed",+ billing_status: "confirmed",+ };+ limit.mockResolvedValue([confirmed]);++ expect(await getMonitorCheckForUpdate(teamId, monitorId, checkId)).toEqual(+ confirmed,+ );+ expect(primarySelect).toHaveBeenCalledTimes(1);+ expect(replicaSelect).not.toHaveBeenCalled();+ expect(limit).toHaveBeenCalledWith(1);+ const predicate = queryPredicate();+ expect(predicate.sql).toBe(+ '(("monitor_checks"."id" = $1) and ("monitor_checks"."monitor_id" = $2) and ("monitor_checks"."team_id" = $3))',+ );+ expect(predicate.params).toEqual([checkId, monitorId, teamId]);+ });++ it("returns null when the authoritative check no longer exists", async () => {+ expect(+ await getMonitorCheckForUpdate(teamId, monitorId, checkId),+ ).toBeNull();+ expect(primarySelect).toHaveBeenCalledTimes(1);+ expect(replicaSelect).not.toHaveBeenCalled();+ });++ it("claims finalization with one conditional update while the check is running", async () => {+ const completed = { id: checkId, status: "completed" };+ returning.mockResolvedValue([completed]);++ expect(+ await updateMonitorCheckIfRunning(checkId, {+ status: "completed",+ billing_status: "reserved",+ }),+ ).toEqual(completed);+ expect(primarySelect).not.toHaveBeenCalled();+ expect(replicaSelect).not.toHaveBeenCalled();+ expect(primaryUpdate).toHaveBeenCalledTimes(1);+ expect(set).toHaveBeenCalledWith({+ status: "completed",+ billing_status: "reserved",+ updated_at: expect.any(String),+ });+ const predicate = queryPredicate();+ expect(predicate.sql).toBe(+ '(("monitor_checks"."id" = $1) and ("monitor_checks"."status" = $2))',+ );+ expect(predicate.params).toEqual([checkId, "running"]);+ });++ it("returns null to a finalizer that lost the terminal-state claim", async () => {+ expect(+ await updateMonitorCheckIfRunning(checkId, {+ status: "failed",+ billing_status: "released",+ }),+ ).toBeNull();+ expect(primaryUpdate).toHaveBeenCalledTimes(1);+ expect(queryPredicate().params).toEqual([checkId, "running"]);+ });++ it("conditions a queued check transition on the queued state it observed", async () => {+ const failed = { id: checkId, status: "failed" };+ returning.mockResolvedValue([failed]);++ expect(+ await updateMonitorCheckIfStatus(checkId, "queued", {+ status: "failed",+ }),+ ).toEqual(failed);+ expect(primarySelect).not.toHaveBeenCalled();+ expect(replicaSelect).not.toHaveBeenCalled();+ expect(primaryUpdate).toHaveBeenCalledTimes(1);+ const predicate = queryPredicate();+ expect(predicate.sql).toBe(+ '(("monitor_checks"."id" = $1) and ("monitor_checks"."status" = $2))',+ );+ expect(predicate.params).toEqual([checkId, "queued"]);+ });+});diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex d3adbed83..9c9f31211 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -777,6 +777,30 @@ export async function getMonitorCheck(return (data ?? null) as MonitorCheckRow | null;}+// Finalizers must observe terminal writes before deciding whether to settle a hold.+export async function getMonitorCheckForUpdate(+ teamId: string,+ monitorId: string,+ checkId: string,+): Promise<MonitorCheckRow | null> {+ const [data] = await run(+ () =>+ db+ .select()+ .from(schema.monitor_checks)+ .where(+ and(+ eq(schema.monitor_checks.id, checkId),+ eq(schema.monitor_checks.monitor_id, monitorId),+ eq(schema.monitor_checks.team_id, teamId),+ ),+ )+ .limit(1),+ "Failed to get monitor check for update",+ );+ return (data ?? null) as MonitorCheckRow | null;+}+export async function listRunningMonitorChecks(limit: number = 100,): Promise<MonitorCheckRow[]> {@@ -902,6 +926,16 @@ export async function updateMonitorCheck(export async function updateMonitorCheckIfRunning(checkId: string,patch: Partial<MonitorCheckRow>,+): Promise<MonitorCheckRow | null> {+ return updateMonitorCheckIfStatus(checkId, "running", patch);+}++// The terminal transition grants ownership of settlement and its follow-up work.+// A competing worker must not confirm/release the hold when this returns null.+export async function updateMonitorCheckIfStatus(+ checkId: string,+ expectedStatus: "queued" | "running",+ patch: Partial<MonitorCheckRow>,): Promise<MonitorCheckRow | null> {const [data] = await run(() =>@@ -914,7 +948,7 @@ export async function updateMonitorCheckIfRunning(.where(and(eq(schema.monitor_checks.id, checkId),- eq(schema.monitor_checks.status, "running"),+ eq(schema.monitor_checks.status, expectedStatus),),).returning(),