Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Shorten search check timeout & handle redeliveries

Developers Digest52ca397a86e7
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.ts
index 892bd6195..586ff26b2 100644
--- a/apps/api/src/services/monitoring/runner.test.ts
+++ b/apps/api/src/services/monitoring/runner.test.ts
@@ -66,5 +66,31 @@ describe("monitoring runner", () => {
),
).toBe(true);
});
+
+ it("uses the shorter search timeout when the check has a search target", () => {
+ // 11 minutes old: past the 10-min search cutoff but well under 1 hour.
+ const elevenMinAgo = new Date(
+ now.getTime() - 11 * 60 * 1000,
+ ).toISOString();
+ const base = {
+ started_at: elevenMinAgo,
+ updated_at: now.toISOString(),
+ created_at: now.toISOString(),
+ };
+ // Scrape check: not stale yet (1-hour timeout).
+ expect(
+ isMonitorCheckStale(
+ { ...base, target_results: [{ type: "scrape", targetId: "t1" }] },
+ now,
+ ),
+ ).toBe(false);
+ // Search check: stale (10-minute timeout).
+ expect(
+ isMonitorCheckStale(
+ { ...base, target_results: [{ type: "search", targetId: "t1" }] },
+ now,
+ ),
+ ).toBe(true);
+ });
});
});
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index a1bf99320..b67883e79 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -42,6 +42,7 @@ import {
countMonitorCheckPages,
hashMonitorUrl,
insertMonitorCheckPages,
+ deleteMonitorCheckPages,
listActiveMonitorPages,
listMonitorCheckPages,
listRunningMonitorChecks,
@@ -63,6 +64,7 @@ import {
MONITOR_CHECK_STALE_ERROR,
isMonitorCheckStale,
MONITOR_CHECK_STALE_TIMEOUT_MS,
+ monitorCheckStaleTimeoutMs,
} from "./stale";
import { trackMonitorCheckStartedInterest } from "./interest";
import { runSearchTarget, type ScrapeSearchResult } from "./search/run";
@@ -1164,6 +1166,9 @@ async function runMonitorSearchTarget(params: {
});
}
+ // Clear any rows from a prior (crashed/redelivered) run of this same check so
+ // the insert is effectively a replace — a redelivery can't duplicate pages.
+ await deleteMonitorCheckPages({ checkId: check.id, targetId: target.id });
await insertMonitorCheckPages(pages);
for (const page of pages) {
@@ -1192,6 +1197,24 @@ async function runMonitorSearchTarget(params: {
};
}
+// Find a persisted search target run that already completed, so a redelivered
+// check can restore its figures instead of re-running (and re-billing) it.
+function findCompletedSearchTargetRun(
+ targetResults: unknown,
+ targetId: string,
+): Record<string, unknown> | null {
+ if (!Array.isArray(targetResults)) return null;
+ const match = targetResults.find(
+ tr =>
+ tr != null &&
+ typeof tr === "object" &&
+ (tr as { type?: unknown }).type === "search" &&
+ (tr as { targetId?: unknown }).targetId === targetId &&
+ (tr as { searchCompleted?: unknown }).searchCompleted === true,
+ );
+ return (match as Record<string, unknown>) ?? null;
+}
+
export async function processMonitorCheckJob(
job: MonitorCheckJobData,
): Promise<void> {
@@ -1284,6 +1307,19 @@ export async function processMonitorCheckJob(
} else if (target.type === "crawl" && targetRun.type === "crawl") {
await enqueueMonitorCrawlTarget({ monitor, check, target, targetRun });
} else if (target.type === "search" && targetRun.type === "search") {
+ // If this check was already run to completion for this target (a
+ // redelivery after the inline work finished but before the ack),
+ // restore the persisted figures instead of re-running — re-running
+ // would re-bill the search and re-scrape every page.
+ const priorRun = findCompletedSearchTargetRun(
+ initialCheck.target_results,
+ target.id,
+ );
+ if (priorRun) {
+ Object.assign(targetRun, priorRun);
+ targetRun.searchCompleted = true;
+ continue;
+ }
// Search runs synchronously; fold its outcome into target_results.
const searchResult = await runMonitorSearchTarget({
monitor,
@@ -1573,7 +1609,7 @@ async function failStaleMonitorCheck(params: {
monitorId: params.monitor.id,
checkId: params.check.id,
startedAt: params.check.started_at,
- timeoutMs: MONITOR_CHECK_STALE_TIMEOUT_MS,
+ timeoutMs: monitorCheckStaleTimeoutMs(params.check),
});
return true;
diff --git a/apps/api/src/services/monitoring/stale.ts b/apps/api/src/services/monitoring/stale.ts
index 669cf22dd..73d217eb5 100644
--- a/apps/api/src/services/monitoring/stale.ts
+++ b/apps/api/src/services/monitoring/stale.ts
@@ -1,15 +1,41 @@
import type { MonitorCheckRow } from "./types";
export const MONITOR_CHECK_STALE_TIMEOUT_MS = 60 * 60 * 1000;
+// Search checks run their whole search+scrape+judge inline and realistically
+// finish in seconds-to-low-minutes. A stranded one (crash between the inline
+// work and the ack) should self-heal quickly rather than look dead for an hour.
+const MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS = 10 * 60 * 1000;
export const MONITOR_CHECK_STALE_ERROR =
- "Monitor check exceeded the 1 hour running timeout.";
+ "Monitor check exceeded the running timeout.";
+
+// A check is a search check if any of its target runs is of type "search".
+function isSearchCheck(check: { target_results?: unknown }): boolean {
+ const targetResults = check.target_results;
+ if (!Array.isArray(targetResults)) return false;
+ return targetResults.some(
+ tr =>
+ tr != null &&
+ typeof tr === "object" &&
+ (tr as { type?: unknown }).type === "search",
+ );
+}
+
+export function monitorCheckStaleTimeoutMs(check: {
+ target_results?: unknown;
+}): number {
+ return isSearchCheck(check)
+ ? MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS
+ : MONITOR_CHECK_STALE_TIMEOUT_MS;
+}
export function isMonitorCheckStale(
- check: Pick<MonitorCheckRow, "started_at" | "updated_at" | "created_at">,
+ check: Pick<MonitorCheckRow, "started_at" | "updated_at" | "created_at"> & {
+ target_results?: unknown;
+ },
now: Date = new Date(),
): boolean {
const startedAt = check.started_at ?? check.updated_at ?? check.created_at;
const startedAtMs = Date.parse(startedAt);
if (!Number.isFinite(startedAtMs)) return false;
- return now.getTime() - startedAtMs >= MONITOR_CHECK_STALE_TIMEOUT_MS;
+ return now.getTime() - startedAtMs >= monitorCheckStaleTimeoutMs(check);
}
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index dd39502d1..8cf931314 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -858,6 +858,28 @@ export async function insertMonitorCheckPages(
);
}
+// Delete all check-page rows for a (check, target). Used to make the inline
+// search write idempotent: a redelivered search check clears its prior rows
+// before re-inserting, so a crash-and-redeliver can't leave duplicate pages.
+// Partition-safe (no unique constraint needed on the partitioned table).
+export async function deleteMonitorCheckPages(params: {
+ checkId: string;
+ targetId: string;
+}): Promise<void> {
+ await run(
+ () =>
+ db
+ .delete(schema.monitor_check_pages)
+ .where(
+ and(
+ eq(schema.monitor_check_pages.check_id, params.checkId),
+ eq(schema.monitor_check_pages.target_id, params.targetId),
+ ),
+ ),
+ "Failed to delete monitor check pages",
+ );
+}
+
export async function listMonitorCheckPages(params: {
teamId: string;
monitorId: string;