Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Harden scrape/crawl monitor result recording

Developers Digest6fe4977cf8ff
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.ts
index 45c4921e7..0b4b2bb78 100644
--- a/apps/api/src/services/monitoring/results.ts
+++ b/apps/api/src/services/monitoring/results.ts
@@ -2,9 +2,11 @@ import { NuQJob } from "../worker/nuq";
import { ScrapeJobData } from "../../types";
import { logger as _logger } from "../../lib/logger";
import { createWebhookSender, WebhookEvent } from "../webhook";
+import { redisEvictConnection } from "../redis";
import { computeAndPersistPageDiff } from "./diff-orchestrator";
import { derivePageIsMeaningful } from "./page-events";
import {
+ deleteMonitorCheckPages,
getMonitorForUpdate,
getMonitorPage,
hashMonitorUrl,
@@ -14,6 +16,82 @@ import {
const logger = _logger.child({ module: "monitoring-results" });
+// Per-(check, url) webhook claim. checkIds are unique per run, so this only needs
+// to outlive a job redelivery; we match runner.ts's notify-claim horizon.
+const MONITOR_PAGE_WEBHOOK_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;
+
+// Mirror runner.ts's claimMonitorNotification: a redelivered scrape job must not
+// re-send the MONITOR_PAGE webhook. Returns true only for the first claimant; a
+// redis hiccup degrades to "don't send" rather than throwing out of the caller
+// (the page row is already persisted and stays pollable).
+async function claimMonitorPageWebhook(
+ checkId: string,
+ url: string,
+): Promise<boolean> {
+ try {
+ const result = await redisEvictConnection.set(
+ `monitor-page-notify:${checkId}:${hashMonitorUrl(url).toString("hex")}`,
+ "1",
+ "EX",
+ MONITOR_PAGE_WEBHOOK_CLAIM_TTL_SECONDS,
+ "NX",
+ );
+ return result === "OK";
+ } catch (error) {
+ logger.warn("Failed to claim monitor page webhook", {
+ error,
+ checkId,
+ url,
+ });
+ return false;
+ }
+}
+
+// Idempotently record an error-status check page and (once) notify. Shared by the
+// scrape-failure path and the success path's diff/persist guard so a transient
+// failure still completes the reconciler's fan-in tally.
+async function persistMonitorCheckError(params: {
+ monitoring: NonNullable<ScrapeJobData["monitoring"]>;
+ teamId: string;
+ url: string;
+ scrapeId: string;
+ error: string;
+ statusCode?: number | null;
+ metadata?: Record<string, unknown>;
+}): Promise<void> {
+ await deleteMonitorCheckPages({
+ checkId: params.monitoring.checkId,
+ targetId: params.monitoring.targetId,
+ url: params.url,
+ });
+ await insertMonitorCheckPages([
+ {
+ check_id: params.monitoring.checkId,
+ monitor_id: params.monitoring.monitorId,
+ team_id: params.teamId,
+ target_id: params.monitoring.targetId,
+ url: params.url,
+ status: "error",
+ current_scrape_id: params.scrapeId,
+ error: params.error,
+ status_code: params.statusCode ?? null,
+ metadata: params.metadata ?? null,
+ },
+ ]);
+
+ if (await claimMonitorPageWebhook(params.monitoring.checkId, params.url)) {
+ await sendMonitorPageWebhook({
+ teamId: params.teamId,
+ monitorId: params.monitoring.monitorId,
+ checkId: params.monitoring.checkId,
+ url: params.url,
+ status: "error",
+ currentScrapeId: params.scrapeId,
+ error: params.error,
+ });
+ }
+}
+
function getDocumentUrl(doc: any, fallback: string): string {
return doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? doc?.url ?? fallback;
}
@@ -130,6 +208,52 @@ export async function recordMonitorScrapeSuccess(
const targetCtFormat = Array.isArray(targetFormats)
? (targetFormats as any[]).find((f: any) => f?.type === "changeTracking")
: undefined;
+ let diff: Awaited<ReturnType<typeof computeAndPersistPageDiff>>;
+ try {
+ diff = await computeAndPersistPageDiff({
+ teamId: job.data.team_id,
+ monitorId: monitoring.monitorId,
+ checkId: monitoring.checkId,
+ url,
+ scrapeId: job.id,
+ doc,
+ previous: previous
+ ? {
+ last_scrape_id: previous.last_scrape_id,
+ is_removed: previous.is_removed,
+ }
+ : null,
+ formats: targetFormats,
+ goal:
+ monitorForRun?.judge_enabled && monitorForRun?.goal
+ ? monitorForRun.goal
+ : null,
+ extractionPrompt: targetCtFormat?.prompt ?? null,
+ });
+ } catch (error) {
+ // A transient diff/persist failure (e.g. GCS) must not strand the check:
+ // still record an error page so the reconciler's fan-in tally completes
+ // instead of hanging until the stale reaper.
+ logger.warn("Failed to compute monitor page diff; recording error page", {
+ monitorId: monitoring.monitorId,
+ checkId: monitoring.checkId,
+ targetId: monitoring.targetId,
+ scrapeId: job.id,
+ url,
+ error,
+ });
+ await persistMonitorCheckError({
+ monitoring,
+ teamId: job.data.team_id,
+ url,
+ scrapeId: job.id,
+ error: error instanceof Error ? error.message : String(error),
+ statusCode: getDocumentStatusCode(doc),
+ metadata: { creditsUsed: doc?.metadata?.creditsUsed ?? null },
+ });
+ return;
+ }
+
const {
status,
diffGcsKey,
@@ -139,47 +263,17 @@ export async function recordMonitorScrapeSuccess(
diffText,
diffJson,
error,
- } = await computeAndPersistPageDiff({
- teamId: job.data.team_id,
- monitorId: monitoring.monitorId,
- checkId: monitoring.checkId,
- url,
- scrapeId: job.id,
- doc,
- previous: previous
- ? {
- last_scrape_id: previous.last_scrape_id,
- is_removed: previous.is_removed,
- }
- : null,
- formats: targetFormats,
- goal:
- monitorForRun?.judge_enabled && monitorForRun?.goal
- ? monitorForRun.goal
- : null,
- extractionPrompt: targetCtFormat?.prompt ?? null,
- });
+ } = diff;
- await upsertMonitorPage({
- monitorId: monitoring.monitorId,
- teamId: job.data.team_id,
+ // Tally first (the reconciler's fan-in gate), durable baseline last: a crash
+ // between the two completes the check rather than poisoning the cross-run dedup
+ // baseline against an unrecorded page. The delete makes redelivery a replace,
+ // not a duplicate.
+ await deleteMonitorCheckPages({
+ checkId: monitoring.checkId,
targetId: monitoring.targetId,
url,
- source: monitoring.source,
- checkId: monitoring.checkId,
- scrapeId: job.id,
- status,
- metadata: {
- title: doc?.metadata?.title ?? null,
- statusCode: getDocumentStatusCode(doc),
- contentType: doc?.metadata?.contentType ?? null,
- numPages: doc?.metadata?.numPages ?? null,
- proxyUsed: doc?.metadata?.proxyUsed ?? null,
- postprocessorsUsed: doc?.metadata?.postprocessorsUsed ?? null,
- creditsUsed: doc?.metadata?.creditsUsed ?? null,
- },
});
-
await insertMonitorCheckPages([
{
check_id: monitoring.checkId,
@@ -208,30 +302,52 @@ export async function recordMonitorScrapeSuccess(
},
]);
- logger.info("Recorded monitor scrape result", {
+ await upsertMonitorPage({
monitorId: monitoring.monitorId,
- checkId: monitoring.checkId,
+ teamId: job.data.team_id,
targetId: monitoring.targetId,
- scrapeId: job.id,
url,
+ source: monitoring.source,
+ checkId: monitoring.checkId,
+ scrapeId: job.id,
status,
- previousScrapeId: previous?.last_scrape_id ?? null,
- diffGcsKey,
- judgmentMeaningful: judgment?.meaningful,
+ metadata: {
+ title: doc?.metadata?.title ?? null,
+ statusCode: getDocumentStatusCode(doc),
+ contentType: doc?.metadata?.contentType ?? null,
+ numPages: doc?.metadata?.numPages ?? null,
+ proxyUsed: doc?.metadata?.proxyUsed ?? null,
+ postprocessorsUsed: doc?.metadata?.postprocessorsUsed ?? null,
+ creditsUsed: doc?.metadata?.creditsUsed ?? null,
+ },
});
- await sendMonitorPageWebhook({
- teamId: job.data.team_id,
+ logger.info("Recorded monitor scrape result", {
monitorId: monitoring.monitorId,
checkId: monitoring.checkId,
+ targetId: monitoring.targetId,
+ scrapeId: job.id,
url,
status,
previousScrapeId: previous?.last_scrape_id ?? null,
- currentScrapeId: job.id,
- judgment: judgment ?? null,
- diffText: diffText ?? null,
- diffJson: diffJson ?? null,
+ diffGcsKey,
+ judgmentMeaningful: judgment?.meaningful,
});
+
+ if (await claimMonitorPageWebhook(monitoring.checkId, url)) {
+ await sendMonitorPageWebhook({
+ teamId: job.data.team_id,
+ monitorId: monitoring.monitorId,
+ checkId: monitoring.checkId,
+ url,
+ status,
+ previousScrapeId: previous?.last_scrape_id ?? null,
+ currentScrapeId: job.id,
+ judgment: judgment ?? null,
+ diffText: diffText ?? null,
+ diffJson: diffJson ?? null,
+ });
+ }
}
export async function recordMonitorScrapeFailure(
@@ -242,39 +358,21 @@ export async function recordMonitorScrapeFailure(
const monitoring = job.data.monitoring;
if (!monitoring || job.data.mode !== "single_urls") return;
- await insertMonitorCheckPages([
- {
- check_id: monitoring.checkId,
- monitor_id: monitoring.monitorId,
- team_id: job.data.team_id,
- target_id: monitoring.targetId,
- url: job.data.url,
- url_hash: hashMonitorUrl(job.data.url),
- status: "error",
- current_scrape_id: job.id,
- error: error instanceof Error ? error.message : String(error),
- metadata: {
- creditsUsed: creditsUsed ?? null,
- },
- },
- ]);
-
- logger.info("Recorded monitor scrape failure", {
- monitorId: monitoring.monitorId,
- checkId: monitoring.checkId,
- targetId: monitoring.targetId,
- scrapeId: job.id,
+ await persistMonitorCheckError({
+ monitoring,
+ teamId: job.data.team_id,
url: job.data.url,
+ scrapeId: job.id,
error: error instanceof Error ? error.message : String(error),
+ metadata: { creditsUsed: creditsUsed ?? null },
});
- await sendMonitorPageWebhook({
- teamId: job.data.team_id,
+ logger.info("Recorded monitor scrape failure", {
monitorId: monitoring.monitorId,
checkId: monitoring.checkId,
+ targetId: monitoring.targetId,
+ scrapeId: job.id,
url: job.data.url,
- status: "error",
- currentScrapeId: job.id,
error: error instanceof Error ? error.message : String(error),
});
}
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 3939b8cb7..e83ef5097 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -886,23 +886,27 @@ export async function insertMonitorCheckPages(
);
}
-// Makes the inline search write idempotent: a redelivered check clears its prior
-// (check, target) rows before re-inserting, so crash-and-redeliver can't duplicate
-// pages. Partition-safe (no unique constraint needed).
+// Makes an inline write idempotent: a redelivered check clears its prior rows
+// before re-inserting, so crash-and-redeliver can't duplicate pages. Pass `url`
+// to scope the clear to a single page so the per-URL scrape path replaces only
+// its own row without clobbering sibling pages of the same target. Partition-safe
+// (no unique constraint needed).
export async function deleteMonitorCheckPages(params: {
checkId: string;
targetId: string;
+ url?: string;
}): Promise<void> {
+ const conditions = [
+ eq(schema.monitor_check_pages.check_id, params.checkId),
+ eq(schema.monitor_check_pages.target_id, params.targetId),
+ ];
+ if (params.url !== undefined) {
+ conditions.push(
+ eq(schema.monitor_check_pages.url_hash, hashMonitorUrl(params.url)),
+ );
+ }
await run(
- () =>
- db
- .delete(schema.monitor_check_pages)
- .where(
- and(
- eq(schema.monitor_check_pages.check_id, params.checkId),
- eq(schema.monitor_check_pages.target_id, params.targetId),
- ),
- ),
+ () => db.delete(schema.monitor_check_pages).where(and(...conditions)),
"Failed to delete monitor check pages",
);
}