Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
re-judge live results on a cadence even when SERP snippet is unchanged
Developers Digest41880ec21dc5
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 2eff5d857..8310769c0 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -1000,6 +1000,7 @@ async function runMonitorSearchTarget(params: {alertMode: target.alertMode,includeDomains: target.includeDomains,excludeDomains: target.excludeDomains,+ recheckAfter: target.recheckAfter,maxResults: target.maxResults,},goalVersion,diff --git a/apps/api/src/services/monitoring/search/judge.ts b/apps/api/src/services/monitoring/search/judge.tsindex fabc9d7b3..e41b8a635 100644--- a/apps/api/src/services/monitoring/search/judge.ts+++ b/apps/api/src/services/monitoring/search/judge.ts@@ -94,6 +94,10 @@ const WINDOW_MS: Record<string, number> = {"7d": 7 * 24 * 60 * 60_000,};+export function windowToMs(window: string): number {+ return WINDOW_MS[window] ?? WINDOW_MS["24h"];+}+// Freshness from a real publish date vs the search window. Returns null when no usable date// exists, so the caller can fall back to the LLM's freshness guess. Future-dated β fresh.export function freshnessFromDate(diff --git a/apps/api/src/services/monitoring/search/persist.ts b/apps/api/src/services/monitoring/search/persist.tsindex 831b1b8c9..c99f388c0 100644--- a/apps/api/src/services/monitoring/search/persist.ts+++ b/apps/api/src/services/monitoring/search/persist.ts@@ -16,7 +16,12 @@ export function searchPageWasScraped(status: string): boolean {return status === "alert" || status === "watching" || status === "ignored";}-type PriorPage = { url: string; metadata: unknown | null };+type PriorPage = {+ url: string;+ metadata: unknown | null;+ updated_at?: string;+ last_status?: string;+};// Rebuild dedup memory + event index from prior pages. Fingerprints load regardless of// goalVersion (the runner gates freshness); events only carry the current goalVersion.@@ -35,6 +40,8 @@ export function reconstructKnownState(knownPages.set(canonicalizeUrl(page.url), {fingerprint: meta.fingerprint,goalVersion: meta.goalVersion,+ lastCheckedAt: page.updated_at,+ lastStatus: page.last_status,});}if (diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 9ff39fda6..942f73e75 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -76,7 +76,7 @@ const baseMonitor = {function run(over: {- target?: Partial<typeof baseTarget>;+ target?: Partial<typeof baseTarget> & { recheckAfter?: string };knownPages?: Map<string, KnownPage>;knownEvents?: EventResolution[] extends never? never@@ -281,6 +281,75 @@ describe("runSearchTarget orchestration", () => {});});+describe("re-judge cadence", () => {+ const url = "https://sec.gov/openai";+ const fp = stableSerpFingerprint({+ url,+ title: "OpenAI S-1",+ snippet: "filing",+ });+ const serp = [{ url, title: "OpenAI S-1", description: "filing" }];+ const old = new Date(Date.now() - 48 * 3600_000).toISOString();+ const recent = new Date(Date.now() - 1000).toISOString();++ it("re-judges a live page whose last check is older than recheckAfter (SERP unchanged)", async () => {+ setSearchResults(serp);+ setVerdictsByUrl({ [url]: verdict() });+ const knownPages = new Map<string, KnownPage>([+ [+ canonicalizeUrl(url),+ {+ fingerprint: fp,+ goalVersion: "gv1",+ lastCheckedAt: old,+ lastStatus: "watching",+ },+ ],+ ]);+ const out = await run({ target: { recheckAfter: "24h" }, knownPages });+ expect(scrapeURLMock).toHaveBeenCalledTimes(1);+ expect(out.sources[0].status).toBe("alert");+ });++ it("does NOT re-judge when the last check is within recheckAfter", async () => {+ setSearchResults(serp);+ setVerdictsByUrl({ [url]: verdict() });+ const knownPages = new Map<string, KnownPage>([+ [+ canonicalizeUrl(url),+ {+ fingerprint: fp,+ goalVersion: "gv1",+ lastCheckedAt: recent,+ lastStatus: "watching",+ },+ ],+ ]);+ const out = await run({ target: { recheckAfter: "24h" }, knownPages });+ expect(scrapeURLMock).not.toHaveBeenCalled();+ expect(out.sources[0].status).toBe("already_seen");+ });++ it("does NOT re-judge a non-live (ignored) page even when stale", async () => {+ setSearchResults(serp);+ setVerdictsByUrl({ [url]: verdict() });+ const knownPages = new Map<string, KnownPage>([+ [+ canonicalizeUrl(url),+ {+ fingerprint: fp,+ goalVersion: "gv1",+ lastCheckedAt: old,+ lastStatus: "ignored",+ },+ ],+ ]);+ const out = await run({ target: { recheckAfter: "24h" }, knownPages });+ expect(scrapeURLMock).not.toHaveBeenCalled();+ expect(out.sources[0].status).toBe("already_seen");+ });+});+describe("date-based freshness", () => {it("a real stale publish date vetoes an LLM fresh+alert verdict", async () => {setSearchResults([diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 99dd5d629..aba06cbad 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -13,6 +13,7 @@ import {parseVerdict,verdictJsonSchema,verdictToDecision,+ windowToMs,} from "./judge";import { resolveEvent, summarizeRun, type KnownEvent } from "./llm";@@ -40,12 +41,17 @@ type SearchTargetInput = {alertMode: "first_match" | "every_new_result";includeDomains?: string[];excludeDomains?: string[];+ // Re-judge a still-live result (last status alert/watching) when its last check is older than+ // this window, even if the SERP snippet is unchanged β catches content updates SERP text misses.+ recheckAfter?: string;maxResults: number;};export type KnownPage = {fingerprint: string;goalVersion: string;+ lastCheckedAt?: string;+ lastStatus?: string;};type SearchSource = {@@ -147,8 +153,21 @@ export async function runSearchTarget(params: {const known = knownPages.get(canonical);const knownCurrent =known && known.goalVersion === goalVersion ? known : undefined;+ // Re-judge a still-live result on a cadence even if its SERP snippet is unchanged.+ const recheckMs = target.recheckAfter ? windowToMs(target.recheckAfter) : 0;+ const isLive =+ knownCurrent?.lastStatus === "alert" ||+ knownCurrent?.lastStatus === "watching";+ const dueForRecheck = Boolean(+ recheckMs &&+ isLive &&+ knownCurrent?.lastCheckedAt &&+ Date.now() - Date.parse(knownCurrent.lastCheckedAt) > recheckMs,+ );const isNewOrChanged =- !knownCurrent || knownCurrent.fingerprint !== fingerprint;+ !knownCurrent ||+ knownCurrent.fingerprint !== fingerprint ||+ dueForRecheck;// Already-seen + unchanged under the current goal β reuse, no scrape/judge/LLM.if (!isNewOrChanged) {diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsindex bfb913c8d..9c2c3ce1c 100644--- a/apps/api/src/services/monitoring/types.ts+++ b/apps/api/src/services/monitoring/types.ts@@ -63,6 +63,7 @@ const searchTargetSchema = z.default("first_match"),includeDomains: z.array(monitorDomainSchema).max(50).optional(),excludeDomains: z.array(monitorDomainSchema).max(50).optional(),+ recheckAfter: z.enum(["1h", "6h", "24h", "7d"]).optional(),maxResults: z.number().int().min(1).max(50).optional().default(10),scrapeOptions: scrapeOptionsSchema,})