Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
preserve event metadata and mirror search status on monitor page upserts
Developers Digest1f6eecc49a53
diff --git a/apps/api/src/services/monitoring/search/persist.test.ts b/apps/api/src/services/monitoring/search/persist.test.tsindex 58ba4283f..a2e17823f 100644--- a/apps/api/src/services/monitoring/search/persist.test.ts+++ b/apps/api/src/services/monitoring/search/persist.test.ts@@ -27,9 +27,68 @@ describe("reconstructKnownState", () => {expect(knownPages.get(canonicalizeUrl("https://example.com/a"))).toEqual({fingerprint: "fp1",goalVersion: "gv1",+ metadata: { fingerprint: "fp1", goalVersion: "gv1" },});});+ it("yields the search-internal lastStatus from metadata.searchStatus", () => {+ // last_status on monitor_pages stores the page-status mapping (alert→new),+ // so the search-internal status must come from metadata.searchStatus.+ const { knownPages } = reconstructKnownState(+ [+ {+ url: "https://a.com/1",+ metadata: {+ fingerprint: "f1",+ goalVersion: "gv1",+ searchStatus: "alert",+ },+ updated_at: "2026-06-10T00:00:00Z",+ last_status: "new",+ },+ ],+ "gv1",+ );+ const known = knownPages.get(canonicalizeUrl("https://a.com/1"))!;+ expect(known.lastStatus).toBe("alert");+ expect(known.lastCheckedAt).toBe("2026-06-10T00:00:00Z");+ });++ it("falls back to last_status for legacy rows without metadata.searchStatus", () => {+ const { knownPages } = reconstructKnownState(+ [+ {+ url: "https://a.com/1",+ metadata: { fingerprint: "f1", goalVersion: "gv1" },+ last_status: "same",+ },+ ],+ "gv1",+ );+ expect(knownPages.get(canonicalizeUrl("https://a.com/1"))!.lastStatus).toBe(+ "same",+ );+ });++ it("carries the full stored metadata through to KnownPage (reuse upserts need it)", () => {+ const meta = {+ fingerprint: "f1",+ goalVersion: "gv1",+ searchStatus: "alert",+ eventKey: "evt-1",+ eventLabel: "openai ipo",+ eventSatisfiedAt: "2026-06-01T00:00:00Z",+ eventAlertCount: 2,+ };+ const { knownPages } = reconstructKnownState(+ [{ url: "https://a.com/1", metadata: meta }],+ "gv1",+ );+ expect(+ knownPages.get(canonicalizeUrl("https://a.com/1"))!.metadata,+ ).toEqual(meta);+ });+it("only includes events from the current goalVersion (goal change starts clean)", () => {const { knownEvents } = reconstructKnownState([diff --git a/apps/api/src/services/monitoring/search/persist.ts b/apps/api/src/services/monitoring/search/persist.tsindex f598af203..66979bf43 100644--- a/apps/api/src/services/monitoring/search/persist.ts+++ b/apps/api/src/services/monitoring/search/persist.ts@@ -33,7 +33,14 @@ export function reconstructKnownState(fingerprint: meta.fingerprint,goalVersion: meta.goalVersion,lastCheckedAt: page.updated_at,- lastStatus: page.last_status,+ // monitor_pages.last_status holds the page-status mapping (new/same/+ // error); the search-internal status (alert/watching/...) is mirrored+ // into metadata.searchStatus. Fall back to last_status for legacy rows.+ lastStatus:+ typeof meta.searchStatus === "string"+ ? meta.searchStatus+ : page.last_status,+ metadata: meta,});}if (meta.goalVersion === goalVersion && typeof meta.eventKey === "string") {diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.tsindex 3706cd3f8..6b1b20409 100644--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts@@ -164,6 +164,8 @@ describe("standard depth (snippet judging, no scrapes)", () => {"alert","watching",]);+ // snippet-only judgments never bill a scrape+ expect(result.pageUpserts.every(u => u.scraped === false)).toBe(true);});it("skips results the snippet judge failed to cover", async () => {diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 77a584ce6..33c9dccb3 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -164,6 +164,89 @@ describe("runSearchTarget orchestration", () => {expect(out.matches).toBe(0);});+ it("reuse upsert preserves prior event metadata (eventKey survives unchanged pages)", async () => {+ setSearchResults([+ {+ url: "https://sec.gov/openai",+ title: "OpenAI S-1",+ description: "filing",+ },+ ]);+ const canonical = canonicalizeUrl("https://sec.gov/openai");+ const fingerprint = stableSerpFingerprint({+ url: "https://sec.gov/openai",+ title: "OpenAI S-1",+ snippet: "filing",+ });+ const knownPages = new Map<string, KnownPage>([+ [+ canonical,+ {+ fingerprint,+ goalVersion: "gv1",+ lastStatus: "alert",+ metadata: {+ fingerprint,+ goalVersion: "gv1",+ searchStatus: "alert",+ eventKey: "evt-1",+ eventLabel: "OpenAI IPO",+ eventSatisfiedAt: "2026-06-01T00:00:00Z",+ eventAlertCount: 2,+ },+ },+ ],+ ]);++ const out = await run({ knownPages });++ expect(scrapeURLMock).not.toHaveBeenCalled();+ expect(out.pageUpserts[0].status).toBe("already_seen");+ expect(out.pageUpserts[0].metadata).toMatchObject({+ eventKey: "evt-1",+ eventLabel: "OpenAI IPO",+ eventSatisfiedAt: "2026-06-01T00:00:00Z",+ eventAlertCount: 2,+ fingerprint,+ goalVersion: "gv1",+ searchStatus: "already_seen",+ });+ });++ it("mirrors the search-internal status into metadata and marks deep judgments scraped", async () => {+ setSearchResults([+ {+ url: "https://sec.gov/openai",+ title: "OpenAI S-1",+ description: "filing",+ },+ {+ url: "https://old.com/openai",+ title: "OpenAI old news",+ description: "2019",+ },+ {+ url: "https://spam.com/x",+ title: "irrelevant",+ description: "noise",+ },+ ]);+ setVerdictsByUrl({+ "https://sec.gov/openai": verdict(),+ "https://old.com/openai": verdict({ alertAction: "watch" }),+ "https://spam.com/x": verdict({ relevant: false }),+ });++ const out = await run();++ const byStatus = (status: string) =>+ out.pageUpserts.find(u => u.status === status)!;+ for (const status of ["alert", "watching", "ignored"]) {+ expect(byStatus(status).metadata.searchStatus).toBe(status);+ expect(byStatus(status).scraped).toBe(true);+ }+ });+it("re-evaluates a known page when the goalVersion changed (stale memory)", async () => {setSearchResults([{@@ -226,6 +309,10 @@ describe("runSearchTarget orchestration", () => {expect(statuses).toContain("alert");expect(statuses).toContain("already_seen");expect(out.matches).toBe(1);+ // the suppressed result was still deep-scraped and judged+ const seen = out.pageUpserts.find(u => u.status === "already_seen")!;+ expect(seen.scraped).toBe(true);+ expect(seen.metadata.searchStatus).toBe("already_seen");});it("mints a stable opaque key for new events (not a label slug)", async () => {diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 452b738fb..a208c454f 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -85,6 +85,9 @@ export type KnownPage = {goalVersion: string;lastCheckedAt?: string;lastStatus?: string;+ // Full stored monitor_pages metadata, carried through so unchanged-page+ // upserts don't wipe event/verdict fields persisted by earlier runs.+ metadata?: Record<string, unknown>;};type SearchSource = {@@ -349,7 +352,15 @@ export async function runSearchTarget(params: {url: canonical,urlHash: hashMonitorUrl(canonical),status: reusedStatus,- metadata: { fingerprint, goalVersion },+ // Upserts replace the stored metadata wholesale, so spread the prior+ // metadata to preserve event stamps (eventKey/eventLabel/...) that+ // event reconstruction depends on.+ metadata: {+ ...(knownCurrent?.metadata ?? {}),+ fingerprint,+ goalVersion,+ searchStatus: reusedStatus,+ },});continue;}@@ -485,7 +496,8 @@ export async function runSearchTarget(params: {url: canonical,urlHash: hashMonitorUrl(canonical),status: "ignored",- metadata: baseMeta,+ scraped: depth === "deep",+ metadata: { ...baseMeta, searchStatus: "ignored" },});continue;}@@ -500,7 +512,8 @@ export async function runSearchTarget(params: {url: canonical,urlHash: hashMonitorUrl(canonical),status: "watching",- metadata: baseMeta,+ scraped: depth === "deep",+ metadata: { ...baseMeta, searchStatus: "watching" },});continue;}@@ -569,7 +582,8 @@ export async function runSearchTarget(params: {url: canonical,urlHash: hashMonitorUrl(canonical),status: "already_seen",- metadata: eventMeta,+ scraped: depth === "deep",+ metadata: { ...eventMeta, searchStatus: "already_seen" },});continue;}@@ -605,6 +619,7 @@ export async function runSearchTarget(params: {scraped: depth === "deep",metadata: {...eventMeta,+ searchStatus: "alert",eventSatisfiedAt,eventAlertCount,eventLastAlertAt: nowIso,