Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Harden scrape/crawl monitor result recording
Developers Digest6fe4977cf8ff
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.tsindex 45c4921e7..0b4b2bb78 100644--- a/apps/api/src/services/monitoring/results.ts+++ b/apps/api/src/services/monitoring/results.ts@@ -2,9 +2,11 @@ import { NuQJob } from "../worker/nuq";import { ScrapeJobData } from "../../types";import { logger as _logger } from "../../lib/logger";import { createWebhookSender, WebhookEvent } from "../webhook";+import { redisEvictConnection } from "../redis";import { computeAndPersistPageDiff } from "./diff-orchestrator";import { derivePageIsMeaningful } from "./page-events";import {+ deleteMonitorCheckPages,getMonitorForUpdate,getMonitorPage,hashMonitorUrl,@@ -14,6 +16,82 @@ import {const logger = _logger.child({ module: "monitoring-results" });+// Per-(check, url) webhook claim. checkIds are unique per run, so this only needs+// to outlive a job redelivery; we match runner.ts's notify-claim horizon.+const MONITOR_PAGE_WEBHOOK_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;++// Mirror runner.ts's claimMonitorNotification: a redelivered scrape job must not+// re-send the MONITOR_PAGE webhook. Returns true only for the first claimant; a+// redis hiccup degrades to "don't send" rather than throwing out of the caller+// (the page row is already persisted and stays pollable).+async function claimMonitorPageWebhook(+ checkId: string,+ url: string,+): Promise<boolean> {+ try {+ const result = await redisEvictConnection.set(+ `monitor-page-notify:${checkId}:${hashMonitorUrl(url).toString("hex")}`,+ "1",+ "EX",+ MONITOR_PAGE_WEBHOOK_CLAIM_TTL_SECONDS,+ "NX",+ );+ return result === "OK";+ } catch (error) {+ logger.warn("Failed to claim monitor page webhook", {+ error,+ checkId,+ url,+ });+ return false;+ }+}++// Idempotently record an error-status check page and (once) notify. Shared by the+// scrape-failure path and the success path's diff/persist guard so a transient+// failure still completes the reconciler's fan-in tally.+async function persistMonitorCheckError(params: {+ monitoring: NonNullable<ScrapeJobData["monitoring"]>;+ teamId: string;+ url: string;+ scrapeId: string;+ error: string;+ statusCode?: number | null;+ metadata?: Record<string, unknown>;+}): Promise<void> {+ await deleteMonitorCheckPages({+ checkId: params.monitoring.checkId,+ targetId: params.monitoring.targetId,+ url: params.url,+ });+ await insertMonitorCheckPages([+ {+ check_id: params.monitoring.checkId,+ monitor_id: params.monitoring.monitorId,+ team_id: params.teamId,+ target_id: params.monitoring.targetId,+ url: params.url,+ status: "error",+ current_scrape_id: params.scrapeId,+ error: params.error,+ status_code: params.statusCode ?? null,+ metadata: params.metadata ?? null,+ },+ ]);++ if (await claimMonitorPageWebhook(params.monitoring.checkId, params.url)) {+ await sendMonitorPageWebhook({+ teamId: params.teamId,+ monitorId: params.monitoring.monitorId,+ checkId: params.monitoring.checkId,+ url: params.url,+ status: "error",+ currentScrapeId: params.scrapeId,+ error: params.error,+ });+ }+}+function getDocumentUrl(doc: any, fallback: string): string {return doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? doc?.url ?? fallback;}@@ -130,6 +208,52 @@ export async function recordMonitorScrapeSuccess(const targetCtFormat = Array.isArray(targetFormats)? (targetFormats as any[]).find((f: any) => f?.type === "changeTracking"): undefined;+ let diff: Awaited<ReturnType<typeof computeAndPersistPageDiff>>;+ try {+ diff = await computeAndPersistPageDiff({+ teamId: job.data.team_id,+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ url,+ scrapeId: job.id,+ doc,+ previous: previous+ ? {+ last_scrape_id: previous.last_scrape_id,+ is_removed: previous.is_removed,+ }+ : null,+ formats: targetFormats,+ goal:+ monitorForRun?.judge_enabled && monitorForRun?.goal+ ? monitorForRun.goal+ : null,+ extractionPrompt: targetCtFormat?.prompt ?? null,+ });+ } catch (error) {+ // A transient diff/persist failure (e.g. GCS) must not strand the check:+ // still record an error page so the reconciler's fan-in tally completes+ // instead of hanging until the stale reaper.+ logger.warn("Failed to compute monitor page diff; recording error page", {+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ targetId: monitoring.targetId,+ scrapeId: job.id,+ url,+ error,+ });+ await persistMonitorCheckError({+ monitoring,+ teamId: job.data.team_id,+ url,+ scrapeId: job.id,+ error: error instanceof Error ? error.message : String(error),+ statusCode: getDocumentStatusCode(doc),+ metadata: { creditsUsed: doc?.metadata?.creditsUsed ?? null },+ });+ return;+ }+const {status,diffGcsKey,@@ -139,47 +263,17 @@ export async function recordMonitorScrapeSuccess(diffText,diffJson,error,- } = await computeAndPersistPageDiff({- teamId: job.data.team_id,- monitorId: monitoring.monitorId,- checkId: monitoring.checkId,- url,- scrapeId: job.id,- doc,- previous: previous- ? {- last_scrape_id: previous.last_scrape_id,- is_removed: previous.is_removed,- }- : null,- formats: targetFormats,- goal:- monitorForRun?.judge_enabled && monitorForRun?.goal- ? monitorForRun.goal- : null,- extractionPrompt: targetCtFormat?.prompt ?? null,- });+ } = diff;- await upsertMonitorPage({- monitorId: monitoring.monitorId,- teamId: job.data.team_id,+ // Tally first (the reconciler's fan-in gate), durable baseline last: a crash+ // between the two completes the check rather than poisoning the cross-run dedup+ // baseline against an unrecorded page. The delete makes redelivery a replace,+ // not a duplicate.+ await deleteMonitorCheckPages({+ checkId: monitoring.checkId,targetId: monitoring.targetId,url,- source: monitoring.source,- checkId: monitoring.checkId,- scrapeId: job.id,- status,- metadata: {- title: doc?.metadata?.title ?? null,- statusCode: getDocumentStatusCode(doc),- contentType: doc?.metadata?.contentType ?? null,- numPages: doc?.metadata?.numPages ?? null,- proxyUsed: doc?.metadata?.proxyUsed ?? null,- postprocessorsUsed: doc?.metadata?.postprocessorsUsed ?? null,- creditsUsed: doc?.metadata?.creditsUsed ?? null,- },});-await insertMonitorCheckPages([{check_id: monitoring.checkId,@@ -208,30 +302,52 @@ export async function recordMonitorScrapeSuccess(},]);- logger.info("Recorded monitor scrape result", {+ await upsertMonitorPage({monitorId: monitoring.monitorId,- checkId: monitoring.checkId,+ teamId: job.data.team_id,targetId: monitoring.targetId,- scrapeId: job.id,url,+ source: monitoring.source,+ checkId: monitoring.checkId,+ scrapeId: job.id,status,- previousScrapeId: previous?.last_scrape_id ?? null,- diffGcsKey,- judgmentMeaningful: judgment?.meaningful,+ metadata: {+ title: doc?.metadata?.title ?? null,+ statusCode: getDocumentStatusCode(doc),+ contentType: doc?.metadata?.contentType ?? null,+ numPages: doc?.metadata?.numPages ?? null,+ proxyUsed: doc?.metadata?.proxyUsed ?? null,+ postprocessorsUsed: doc?.metadata?.postprocessorsUsed ?? null,+ creditsUsed: doc?.metadata?.creditsUsed ?? null,+ },});- await sendMonitorPageWebhook({- teamId: job.data.team_id,+ logger.info("Recorded monitor scrape result", {monitorId: monitoring.monitorId,checkId: monitoring.checkId,+ targetId: monitoring.targetId,+ scrapeId: job.id,url,status,previousScrapeId: previous?.last_scrape_id ?? null,- currentScrapeId: job.id,- judgment: judgment ?? null,- diffText: diffText ?? null,- diffJson: diffJson ?? null,+ diffGcsKey,+ judgmentMeaningful: judgment?.meaningful,});++ if (await claimMonitorPageWebhook(monitoring.checkId, url)) {+ await sendMonitorPageWebhook({+ teamId: job.data.team_id,+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ url,+ status,+ previousScrapeId: previous?.last_scrape_id ?? null,+ currentScrapeId: job.id,+ judgment: judgment ?? null,+ diffText: diffText ?? null,+ diffJson: diffJson ?? null,+ });+ }}export async function recordMonitorScrapeFailure(@@ -242,39 +358,21 @@ export async function recordMonitorScrapeFailure(const monitoring = job.data.monitoring;if (!monitoring || job.data.mode !== "single_urls") return;- await insertMonitorCheckPages([- {- check_id: monitoring.checkId,- monitor_id: monitoring.monitorId,- team_id: job.data.team_id,- target_id: monitoring.targetId,- url: job.data.url,- url_hash: hashMonitorUrl(job.data.url),- status: "error",- current_scrape_id: job.id,- error: error instanceof Error ? error.message : String(error),- metadata: {- creditsUsed: creditsUsed ?? null,- },- },- ]);-- logger.info("Recorded monitor scrape failure", {- monitorId: monitoring.monitorId,- checkId: monitoring.checkId,- targetId: monitoring.targetId,- scrapeId: job.id,+ await persistMonitorCheckError({+ monitoring,+ teamId: job.data.team_id,url: job.data.url,+ scrapeId: job.id,error: error instanceof Error ? error.message : String(error),+ metadata: { creditsUsed: creditsUsed ?? null },});- await sendMonitorPageWebhook({- teamId: job.data.team_id,+ logger.info("Recorded monitor scrape failure", {monitorId: monitoring.monitorId,checkId: monitoring.checkId,+ targetId: monitoring.targetId,+ scrapeId: job.id,url: job.data.url,- status: "error",- currentScrapeId: job.id,error: error instanceof Error ? error.message : String(error),});}diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 3939b8cb7..e83ef5097 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -886,23 +886,27 @@ export async function insertMonitorCheckPages();}-// Makes the inline search write idempotent: a redelivered check clears its prior-// (check, target) rows before re-inserting, so crash-and-redeliver can't duplicate-// pages. Partition-safe (no unique constraint needed).+// Makes an inline write idempotent: a redelivered check clears its prior rows+// before re-inserting, so crash-and-redeliver can't duplicate pages. Pass `url`+// to scope the clear to a single page so the per-URL scrape path replaces only+// its own row without clobbering sibling pages of the same target. Partition-safe+// (no unique constraint needed).export async function deleteMonitorCheckPages(params: {checkId: string;targetId: string;+ url?: string;}): Promise<void> {+ const conditions = [+ eq(schema.monitor_check_pages.check_id, params.checkId),+ eq(schema.monitor_check_pages.target_id, params.targetId),+ ];+ if (params.url !== undefined) {+ conditions.push(+ eq(schema.monitor_check_pages.url_hash, hashMonitorUrl(params.url)),+ );+ }await run(- () =>- db- .delete(schema.monitor_check_pages)- .where(- and(- eq(schema.monitor_check_pages.check_id, params.checkId),- eq(schema.monitor_check_pages.target_id, params.targetId),- ),- ),+ () => db.delete(schema.monitor_check_pages).where(and(...conditions)),"Failed to delete monitor check pages",);}