Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Abort orphaned finalize writes on timeout to prevent dedup corruption

Developers Digeste50168877526
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.ts
index ed78c69d7..43fcc0a52 100644
--- a/apps/api/src/services/monitoring/runner.test.ts
+++ b/apps/api/src/services/monitoring/runner.test.ts
@@ -7,6 +7,7 @@ import {
findCompletedSearchTargetRun,
isMonitorCheckStale,
MONITOR_CHECK_STALE_TIMEOUT_MS,
+ withFinalizeTimeout,
} from "./runner";
describe("monitoring runner", () => {
@@ -179,3 +180,37 @@ describe("findCompletedSearchTargetRun (redelivery idempotency)", () => {
expect(findCompletedSearchTargetRun([null, "x", 5], "t1")).toBeNull();
});
});
+
+describe("withFinalizeTimeout", () => {
+ it("resolves with the work result and never aborts on success", async () => {
+ let observed: boolean | undefined;
+ const result = await withFinalizeTimeout(async signal => {
+ observed = signal.aborted;
+ return "ok";
+ }, "fast work");
+ expect(result).toBe("ok");
+ expect(observed).toBe(false);
+ });
+
+ it("aborts the signal so stalled work stops issuing writes after a timeout", async () => {
+ const writes: number[] = [];
+ const promise = withFinalizeTimeout(
+ async signal => {
+ // Simulate a finalize tail that gates each write on the signal: the first
+ // write lands, then the timeout fires and the rest must be skipped.
+ for (let i = 0; i < 5; i++) {
+ if (signal.aborted) return;
+ writes.push(i);
+ await new Promise(resolve => setTimeout(resolve, 20));
+ }
+ },
+ "stalled tail",
+ 10,
+ );
+
+ await expect(promise).rejects.toThrow(/exceeded 10ms/);
+ // Let any orphaned iterations run; they must observe the abort and stop.
+ await new Promise(resolve => setTimeout(resolve, 60));
+ expect(writes).toEqual([0]);
+ });
+});
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 75c168a49..637485b75 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -48,6 +48,7 @@ import {
listRunningMonitorChecks,
markMonitorRunning,
updateMonitorCheck,
+ updateMonitorCheckIfRunning,
updateMonitorScheduleAfterRun,
upsertMonitorPage,
} from "./store";
@@ -1050,20 +1051,24 @@ class MonitorFinalizeTimeoutError extends Error {
}
// Reject (not resolve) on timeout so a stalled write fails fast into the catch path.
-async function withFinalizeTimeout<T>(
- work: () => Promise<T>,
+// The race rejects but can't truly cancel work(); we abort a signal so work() can
+// cooperatively stop issuing further writes, otherwise a stalled write may land
+// AFTER the catch has marked the check terminal — corrupting cross-run dedup state.
+export async function withFinalizeTimeout<T>(
+ work: (signal: AbortSignal) => Promise<T>,
what: string,
ms: number = MONITOR_FINALIZE_WRITE_TIMEOUT_MS,
): Promise<T> {
+ const controller = new AbortController();
let timer: ReturnType<typeof setTimeout> | undefined;
const timeout = new Promise<never>((_, reject) => {
- timer = setTimeout(
- () => reject(new MonitorFinalizeTimeoutError(what, ms)),
- ms,
- );
+ timer = setTimeout(() => {
+ controller.abort();
+ reject(new MonitorFinalizeTimeoutError(what, ms));
+ }, ms);
});
try {
- return await Promise.race([work(), timeout]);
+ return await Promise.race([work(controller.signal), timeout]);
} finally {
if (timer) clearTimeout(timer);
}
@@ -1175,8 +1180,19 @@ async function runMonitorSearchTarget(params: {
};
});
- await withFinalizeTimeout(async () => {
+ await withFinalizeTimeout(async signal => {
+ // Per-check rows first: idempotent (delete+insert replace) and not read across runs.
+ // Clear rows from a prior (crashed/redelivered) run so the insert is a replace, not a duplicate.
+ if (signal.aborted) return;
+ await deleteMonitorCheckPages({ checkId: check.id, targetId: target.id });
+ if (signal.aborted) return;
+ await insertMonitorCheckPages(pages);
+
+ // Durable cross-run dedup baseline last, so a timeout before this point leaves
+ // monitor_pages untouched and the next run re-alerts. Pass the signal so an
+ // orphaned write that outran the timeout can't baseline a now-failed check.
for (let i = 0; i < pages.length; i++) {
+ if (signal.aborted) return;
const page = pages[i];
await upsertMonitorPage({
monitorId: monitor.id,
@@ -1188,12 +1204,9 @@ async function runMonitorSearchTarget(params: {
scrapeId: null,
status: page.status,
metadata: page.metadata as Record<string, unknown>,
+ abortSignal: signal,
});
}
-
- // Clear rows from a prior (crashed/redelivered) run so the insert is a replace, not a duplicate.
- await deleteMonitorCheckPages({ checkId: check.id, targetId: target.id });
- await insertMonitorCheckPages(pages);
}, "monitor search page-write tail");
for (const page of pages) {
@@ -1362,10 +1375,14 @@ export async function processMonitorCheckJob(
// Persist searchCompleted now so a crash/redelivery short-circuits via
// findCompletedSearchTargetRun instead of re-running and re-billing.
await withFinalizeTimeout(
- () =>
- updateMonitorCheck(check.id, {
- target_results: targetResults,
- }),
+ signal =>
+ signal.aborted
+ ? Promise.resolve(null)
+ : // Atomic guard: if this write outran the timeout and the catch
+ // already failed the check, no-op instead of stamping searchCompleted.
+ updateMonitorCheckIfRunning(check.id, {
+ target_results: targetResults,
+ }),
"monitor search searchCompleted flush",
);
}
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index bbeefcf64..d13e9b0f3 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -840,6 +840,34 @@ export async function updateMonitorCheck(
return data as MonitorCheckRow;
}
+// Atomic variant of updateMonitorCheck that only writes while the check is still
+// running. A late finalize write that lost the race to the catch path (which marks
+// the check failed) becomes a no-op instead of stamping results/searchCompleted onto
+// an already-terminal check. Returns the row if it applied, else null.
+export async function updateMonitorCheckIfRunning(
+ checkId: string,
+ patch: Partial<MonitorCheckRow>,
+): Promise<MonitorCheckRow | null> {
+ const [data] = await run(
+ () =>
+ db
+ .update(schema.monitor_checks)
+ .set({
+ ...patch,
+ updated_at: new Date().toISOString(),
+ })
+ .where(
+ and(
+ eq(schema.monitor_checks.id, checkId),
+ eq(schema.monitor_checks.status, "running"),
+ ),
+ )
+ .returning(),
+ "Failed to update monitor check",
+ );
+ return (data as MonitorCheckRow) ?? null;
+}
+
export async function insertMonitorCheckPages(
pages: MonitorCheckPageInsert[],
): Promise<void> {
@@ -1003,6 +1031,9 @@ export async function upsertMonitorPage(params: {
scrapeId: string | null;
status: "same" | "new" | "changed" | "removed" | "error";
metadata?: unknown;
+ // When the caller's finalize times out it aborts this signal; we then skip the
+ // write so an orphaned baseline can't poison the next run's dedup state.
+ abortSignal?: AbortSignal;
}): Promise<void> {
const now = new Date().toISOString();
@@ -1012,6 +1043,8 @@ export async function upsertMonitorPage(params: {
url: params.url,
});
+ if (params.abortSignal?.aborted) return;
+
if (!existing) {
await run(
() =>