Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
freshness from real publish date with llm fallback
Developers Digestcb0e68d7ce46
diff --git a/apps/api/src/services/monitoring/search/judge.test.ts b/apps/api/src/services/monitoring/search/judge.test.tsindex c5bdd9eb9..0d71d7c9a 100644--- a/apps/api/src/services/monitoring/search/judge.test.ts+++ b/apps/api/src/services/monitoring/search/judge.test.ts@@ -1,6 +1,30 @@-import { parseVerdict, verdictToDecision, type SearchVerdict } from "./judge";+import {+ freshnessFromDate,+ parseVerdict,+ verdictToDecision,+ type SearchVerdict,+} from "./judge";import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";+describe("freshnessFromDate", () => {+ const now = Date.parse("2026-06-09T12:00:00Z");+ it("returns null with no usable date (caller falls back to LLM)", () => {+ expect(freshnessFromDate(null, "24h", now)).toBeNull();+ expect(freshnessFromDate("not-a-date", "24h", now)).toBeNull();+ });+ it("fresh when within the window, stale when older", () => {+ expect(freshnessFromDate("2026-06-09T06:00:00Z", "24h", now)).toBe("fresh");+ expect(freshnessFromDate("2026-06-07T06:00:00Z", "24h", now)).toBe("stale");+ });+ it("respects the window size", () => {+ expect(freshnessFromDate("2026-06-09T10:00:00Z", "1h", now)).toBe("stale");+ expect(freshnessFromDate("2026-06-09T11:30:00Z", "1h", now)).toBe("fresh");+ });+ it("future-dated counts as fresh", () => {+ expect(freshnessFromDate("2026-06-10T00:00:00Z", "24h", now)).toBe("fresh");+ });+});+const v = (over: Partial<SearchVerdict>): SearchVerdict => ({relevant: true,alertAction: "alert",diff --git a/apps/api/src/services/monitoring/search/judge.ts b/apps/api/src/services/monitoring/search/judge.tsindex 030b2fee5..fabc9d7b3 100644--- a/apps/api/src/services/monitoring/search/judge.ts+++ b/apps/api/src/services/monitoring/search/judge.ts@@ -85,6 +85,29 @@ export function parseVerdict(json: unknown): SearchVerdict | null {};}+const WINDOW_MS: Record<string, number> = {+ "5m": 5 * 60_000,+ "15m": 15 * 60_000,+ "1h": 60 * 60_000,+ "6h": 6 * 60 * 60_000,+ "24h": 24 * 60 * 60_000,+ "7d": 7 * 24 * 60 * 60_000,+};++// Freshness from a real publish date vs the search window. Returns null when no usable date+// exists, so the caller can fall back to the LLM's freshness guess. Future-dated → fresh.+export function freshnessFromDate(+ dateIso: string | null | undefined,+ searchWindow: string,+ nowMs: number,+): "fresh" | "stale" | null {+ if (!dateIso) return null;+ const t = Date.parse(dateIso);+ if (Number.isNaN(t)) return null;+ const window = WINDOW_MS[searchWindow] ?? WINDOW_MS["24h"];+ return nowMs - t <= window ? "fresh" : "stale";+}+// "alert" only when relevant, fresh, trusted-enough, and the judge asked to alert.export function verdictToDecision(v: SearchVerdict,diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex baa5b5b38..9ff39fda6 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -281,6 +281,50 @@ describe("runSearchTarget orchestration", () => {});});+describe("date-based freshness", () => {+ it("a real stale publish date vetoes an LLM fresh+alert verdict", async () => {+ setSearchResults([+ {+ url: "https://news.com/openai",+ title: "OpenAI files S-1",+ description: "x",+ },+ ]);+ // LLM says fresh+alert, but the page's real publishedTime is ~2 years old.+ scrapeURLMock.mockResolvedValue({+ success: true,+ document: {+ json: verdict({ freshness: "fresh", alertAction: "alert" }),+ metadata: { publishedTime: "2024-01-01T00:00:00Z" },+ },+ });+ const out = await run();+ expect(out.matches).toBe(0);+ expect(out.sources[0].status).toBe("watching");+ expect(out.sources[0].freshness).toBe("stale");+ });++ it("a real fresh date keeps an alert and is recorded as date-sourced", async () => {+ setSearchResults([+ {+ url: "https://news.com/openai",+ title: "OpenAI files S-1",+ description: "x",+ },+ ]);+ scrapeURLMock.mockResolvedValue({+ success: true,+ document: {+ json: verdict({ freshness: "fresh", alertAction: "alert" }),+ metadata: { publishedTime: new Date().toISOString() },+ },+ });+ const out = await run();+ expect(out.matches).toBe(1);+ expect(out.pageUpserts[0].metadata.freshnessSource).toBe("date");+ });+});+describe("domain scoping", () => {beforeEach(() => setSearchResults([]));diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 6b59719d0..99dd5d629 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -9,6 +9,7 @@ import { hashMonitorUrl } from "../store";import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";import {buildJudgePrompt,+ freshnessFromDate,parseVerdict,verdictJsonSchema,verdictToDecision,@@ -202,15 +203,31 @@ export async function runSearchTarget(params: {continue;}- const decision = verdictToDecision(verdict);+ // Prefer a real publish date over the LLM's freshness guess (freshness is an alert veto).+ const realDate =+ res.document.metadata?.publishedTime ??+ res.document.metadata?.modifiedTime ??+ null;+ const dateFreshness = freshnessFromDate(+ realDate,+ target.searchWindow,+ Date.now(),+ );+ const effectiveVerdict = dateFreshness+ ? { ...verdict, freshness: dateFreshness }+ : verdict;++ const decision = verdictToDecision(effectiveVerdict);const baseMeta = {fingerprint,goalVersion,- alertAction: verdict.alertAction,- freshness: verdict.freshness,- sourceQuality: verdict.sourceQuality,- concept: verdict.concept,- rationale: verdict.rationale,+ alertAction: effectiveVerdict.alertAction,+ freshness: effectiveVerdict.freshness,+ freshnessSource: dateFreshness ? "date" : "llm",+ publishedAt: realDate,+ sourceQuality: effectiveVerdict.sourceQuality,+ concept: effectiveVerdict.concept,+ rationale: effectiveVerdict.rationale,};if (decision === "ignore") {