Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Shorten search check timeout & handle redeliveries
Developers Digest52ca397a86e7
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.tsindex 892bd6195..586ff26b2 100644--- a/apps/api/src/services/monitoring/runner.test.ts+++ b/apps/api/src/services/monitoring/runner.test.ts@@ -66,5 +66,31 @@ describe("monitoring runner", () => {),).toBe(true);});++ it("uses the shorter search timeout when the check has a search target", () => {+ // 11 minutes old: past the 10-min search cutoff but well under 1 hour.+ const elevenMinAgo = new Date(+ now.getTime() - 11 * 60 * 1000,+ ).toISOString();+ const base = {+ started_at: elevenMinAgo,+ updated_at: now.toISOString(),+ created_at: now.toISOString(),+ };+ // Scrape check: not stale yet (1-hour timeout).+ expect(+ isMonitorCheckStale(+ { ...base, target_results: [{ type: "scrape", targetId: "t1" }] },+ now,+ ),+ ).toBe(false);+ // Search check: stale (10-minute timeout).+ expect(+ isMonitorCheckStale(+ { ...base, target_results: [{ type: "search", targetId: "t1" }] },+ now,+ ),+ ).toBe(true);+ });});});diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex a1bf99320..b67883e79 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -42,6 +42,7 @@ import {countMonitorCheckPages,hashMonitorUrl,insertMonitorCheckPages,+ deleteMonitorCheckPages,listActiveMonitorPages,listMonitorCheckPages,listRunningMonitorChecks,@@ -63,6 +64,7 @@ import {MONITOR_CHECK_STALE_ERROR,isMonitorCheckStale,MONITOR_CHECK_STALE_TIMEOUT_MS,+ monitorCheckStaleTimeoutMs,} from "./stale";import { trackMonitorCheckStartedInterest } from "./interest";import { runSearchTarget, type ScrapeSearchResult } from "./search/run";@@ -1164,6 +1166,9 @@ async function runMonitorSearchTarget(params: {});}+ // Clear any rows from a prior (crashed/redelivered) run of this same check so+ // the insert is effectively a replace — a redelivery can't duplicate pages.+ await deleteMonitorCheckPages({ checkId: check.id, targetId: target.id });await insertMonitorCheckPages(pages);for (const page of pages) {@@ -1192,6 +1197,24 @@ async function runMonitorSearchTarget(params: {};}+// Find a persisted search target run that already completed, so a redelivered+// check can restore its figures instead of re-running (and re-billing) it.+function findCompletedSearchTargetRun(+ targetResults: unknown,+ targetId: string,+): Record<string, unknown> | null {+ if (!Array.isArray(targetResults)) return null;+ const match = targetResults.find(+ tr =>+ tr != null &&+ typeof tr === "object" &&+ (tr as { type?: unknown }).type === "search" &&+ (tr as { targetId?: unknown }).targetId === targetId &&+ (tr as { searchCompleted?: unknown }).searchCompleted === true,+ );+ return (match as Record<string, unknown>) ?? null;+}+export async function processMonitorCheckJob(job: MonitorCheckJobData,): Promise<void> {@@ -1284,6 +1307,19 @@ export async function processMonitorCheckJob(} else if (target.type === "crawl" && targetRun.type === "crawl") {await enqueueMonitorCrawlTarget({ monitor, check, target, targetRun });} else if (target.type === "search" && targetRun.type === "search") {+ // If this check was already run to completion for this target (a+ // redelivery after the inline work finished but before the ack),+ // restore the persisted figures instead of re-running — re-running+ // would re-bill the search and re-scrape every page.+ const priorRun = findCompletedSearchTargetRun(+ initialCheck.target_results,+ target.id,+ );+ if (priorRun) {+ Object.assign(targetRun, priorRun);+ targetRun.searchCompleted = true;+ continue;+ }// Search runs synchronously; fold its outcome into target_results.const searchResult = await runMonitorSearchTarget({monitor,@@ -1573,7 +1609,7 @@ async function failStaleMonitorCheck(params: {monitorId: params.monitor.id,checkId: params.check.id,startedAt: params.check.started_at,- timeoutMs: MONITOR_CHECK_STALE_TIMEOUT_MS,+ timeoutMs: monitorCheckStaleTimeoutMs(params.check),});return true;diff --git a/apps/api/src/services/monitoring/stale.ts b/apps/api/src/services/monitoring/stale.tsindex 669cf22dd..73d217eb5 100644--- a/apps/api/src/services/monitoring/stale.ts+++ b/apps/api/src/services/monitoring/stale.ts@@ -1,15 +1,41 @@import type { MonitorCheckRow } from "./types";export const MONITOR_CHECK_STALE_TIMEOUT_MS = 60 * 60 * 1000;+// Search checks run their whole search+scrape+judge inline and realistically+// finish in seconds-to-low-minutes. A stranded one (crash between the inline+// work and the ack) should self-heal quickly rather than look dead for an hour.+const MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS = 10 * 60 * 1000;export const MONITOR_CHECK_STALE_ERROR =- "Monitor check exceeded the 1 hour running timeout.";+ "Monitor check exceeded the running timeout.";++// A check is a search check if any of its target runs is of type "search".+function isSearchCheck(check: { target_results?: unknown }): boolean {+ const targetResults = check.target_results;+ if (!Array.isArray(targetResults)) return false;+ return targetResults.some(+ tr =>+ tr != null &&+ typeof tr === "object" &&+ (tr as { type?: unknown }).type === "search",+ );+}++export function monitorCheckStaleTimeoutMs(check: {+ target_results?: unknown;+}): number {+ return isSearchCheck(check)+ ? MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS+ : MONITOR_CHECK_STALE_TIMEOUT_MS;+}export function isMonitorCheckStale(- check: Pick<MonitorCheckRow, "started_at" | "updated_at" | "created_at">,+ check: Pick<MonitorCheckRow, "started_at" | "updated_at" | "created_at"> & {+ target_results?: unknown;+ },now: Date = new Date(),): boolean {const startedAt = check.started_at ?? check.updated_at ?? check.created_at;const startedAtMs = Date.parse(startedAt);if (!Number.isFinite(startedAtMs)) return false;- return now.getTime() - startedAtMs >= MONITOR_CHECK_STALE_TIMEOUT_MS;+ return now.getTime() - startedAtMs >= monitorCheckStaleTimeoutMs(check);}diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex dd39502d1..8cf931314 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -858,6 +858,28 @@ export async function insertMonitorCheckPages();}+// Delete all check-page rows for a (check, target). Used to make the inline+// search write idempotent: a redelivered search check clears its prior rows+// before re-inserting, so a crash-and-redeliver can't leave duplicate pages.+// Partition-safe (no unique constraint needed on the partitioned table).+export async function deleteMonitorCheckPages(params: {+ checkId: string;+ targetId: string;+}): Promise<void> {+ await run(+ () =>+ db+ .delete(schema.monitor_check_pages)+ .where(+ and(+ eq(schema.monitor_check_pages.check_id, params.checkId),+ eq(schema.monitor_check_pages.target_id, params.targetId),+ ),+ ),+ "Failed to delete monitor check pages",+ );+}+export async function listMonitorCheckPages(params: {teamId: string;monitorId: string;