Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

preserve event metadata and mirror search status on monitor page upserts

Developers Digest1f6eecc49a53
diff --git a/apps/api/src/services/monitoring/search/persist.test.ts b/apps/api/src/services/monitoring/search/persist.test.ts
index 58ba4283f..a2e17823f 100644
--- a/apps/api/src/services/monitoring/search/persist.test.ts
+++ b/apps/api/src/services/monitoring/search/persist.test.ts
@@ -27,9 +27,68 @@ describe("reconstructKnownState", () => {
expect(knownPages.get(canonicalizeUrl("https://example.com/a"))).toEqual({
fingerprint: "fp1",
goalVersion: "gv1",
+ metadata: { fingerprint: "fp1", goalVersion: "gv1" },
});
});
+ it("yields the search-internal lastStatus from metadata.searchStatus", () => {
+ // last_status on monitor_pages stores the page-status mapping (alert→new),
+ // so the search-internal status must come from metadata.searchStatus.
+ const { knownPages } = reconstructKnownState(
+ [
+ {
+ url: "https://a.com/1",
+ metadata: {
+ fingerprint: "f1",
+ goalVersion: "gv1",
+ searchStatus: "alert",
+ },
+ updated_at: "2026-06-10T00:00:00Z",
+ last_status: "new",
+ },
+ ],
+ "gv1",
+ );
+ const known = knownPages.get(canonicalizeUrl("https://a.com/1"))!;
+ expect(known.lastStatus).toBe("alert");
+ expect(known.lastCheckedAt).toBe("2026-06-10T00:00:00Z");
+ });
+
+ it("falls back to last_status for legacy rows without metadata.searchStatus", () => {
+ const { knownPages } = reconstructKnownState(
+ [
+ {
+ url: "https://a.com/1",
+ metadata: { fingerprint: "f1", goalVersion: "gv1" },
+ last_status: "same",
+ },
+ ],
+ "gv1",
+ );
+ expect(knownPages.get(canonicalizeUrl("https://a.com/1"))!.lastStatus).toBe(
+ "same",
+ );
+ });
+
+ it("carries the full stored metadata through to KnownPage (reuse upserts need it)", () => {
+ const meta = {
+ fingerprint: "f1",
+ goalVersion: "gv1",
+ searchStatus: "alert",
+ eventKey: "evt-1",
+ eventLabel: "openai ipo",
+ eventSatisfiedAt: "2026-06-01T00:00:00Z",
+ eventAlertCount: 2,
+ };
+ const { knownPages } = reconstructKnownState(
+ [{ url: "https://a.com/1", metadata: meta }],
+ "gv1",
+ );
+ expect(
+ knownPages.get(canonicalizeUrl("https://a.com/1"))!.metadata,
+ ).toEqual(meta);
+ });
+
it("only includes events from the current goalVersion (goal change starts clean)", () => {
const { knownEvents } = reconstructKnownState(
[
diff --git a/apps/api/src/services/monitoring/search/persist.ts b/apps/api/src/services/monitoring/search/persist.ts
index f598af203..66979bf43 100644
--- a/apps/api/src/services/monitoring/search/persist.ts
+++ b/apps/api/src/services/monitoring/search/persist.ts
@@ -33,7 +33,14 @@ export function reconstructKnownState(
fingerprint: meta.fingerprint,
goalVersion: meta.goalVersion,
lastCheckedAt: page.updated_at,
- lastStatus: page.last_status,
+ // monitor_pages.last_status holds the page-status mapping (new/same/
+ // error); the search-internal status (alert/watching/...) is mirrored
+ // into metadata.searchStatus. Fall back to last_status for legacy rows.
+ lastStatus:
+ typeof meta.searchStatus === "string"
+ ? meta.searchStatus
+ : page.last_status,
+ metadata: meta,
});
}
if (meta.goalVersion === goalVersion && typeof meta.eventKey === "string") {
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
index 3706cd3f8..6b1b20409 100644
--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -164,6 +164,8 @@ describe("standard depth (snippet judging, no scrapes)", () => {
"alert",
"watching",
]);
+ // snippet-only judgments never bill a scrape
+ expect(result.pageUpserts.every(u => u.scraped === false)).toBe(true);
});
it("skips results the snippet judge failed to cover", async () => {
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.ts
index 77a584ce6..33c9dccb3 100644
--- a/apps/api/src/services/monitoring/search/run.test.ts
+++ b/apps/api/src/services/monitoring/search/run.test.ts
@@ -164,6 +164,89 @@ describe("runSearchTarget orchestration", () => {
expect(out.matches).toBe(0);
});
+ it("reuse upsert preserves prior event metadata (eventKey survives unchanged pages)", async () => {
+ setSearchResults([
+ {
+ url: "https://sec.gov/openai",
+ title: "OpenAI S-1",
+ description: "filing",
+ },
+ ]);
+ const canonical = canonicalizeUrl("https://sec.gov/openai");
+ const fingerprint = stableSerpFingerprint({
+ url: "https://sec.gov/openai",
+ title: "OpenAI S-1",
+ snippet: "filing",
+ });
+ const knownPages = new Map<string, KnownPage>([
+ [
+ canonical,
+ {
+ fingerprint,
+ goalVersion: "gv1",
+ lastStatus: "alert",
+ metadata: {
+ fingerprint,
+ goalVersion: "gv1",
+ searchStatus: "alert",
+ eventKey: "evt-1",
+ eventLabel: "OpenAI IPO",
+ eventSatisfiedAt: "2026-06-01T00:00:00Z",
+ eventAlertCount: 2,
+ },
+ },
+ ],
+ ]);
+
+ const out = await run({ knownPages });
+
+ expect(scrapeURLMock).not.toHaveBeenCalled();
+ expect(out.pageUpserts[0].status).toBe("already_seen");
+ expect(out.pageUpserts[0].metadata).toMatchObject({
+ eventKey: "evt-1",
+ eventLabel: "OpenAI IPO",
+ eventSatisfiedAt: "2026-06-01T00:00:00Z",
+ eventAlertCount: 2,
+ fingerprint,
+ goalVersion: "gv1",
+ searchStatus: "already_seen",
+ });
+ });
+
+ it("mirrors the search-internal status into metadata and marks deep judgments scraped", async () => {
+ setSearchResults([
+ {
+ url: "https://sec.gov/openai",
+ title: "OpenAI S-1",
+ description: "filing",
+ },
+ {
+ url: "https://old.com/openai",
+ title: "OpenAI old news",
+ description: "2019",
+ },
+ {
+ url: "https://spam.com/x",
+ title: "irrelevant",
+ description: "noise",
+ },
+ ]);
+ setVerdictsByUrl({
+ "https://sec.gov/openai": verdict(),
+ "https://old.com/openai": verdict({ alertAction: "watch" }),
+ "https://spam.com/x": verdict({ relevant: false }),
+ });
+
+ const out = await run();
+
+ const byStatus = (status: string) =>
+ out.pageUpserts.find(u => u.status === status)!;
+ for (const status of ["alert", "watching", "ignored"]) {
+ expect(byStatus(status).metadata.searchStatus).toBe(status);
+ expect(byStatus(status).scraped).toBe(true);
+ }
+ });
+
it("re-evaluates a known page when the goalVersion changed (stale memory)", async () => {
setSearchResults([
{
@@ -226,6 +309,10 @@ describe("runSearchTarget orchestration", () => {
expect(statuses).toContain("alert");
expect(statuses).toContain("already_seen");
expect(out.matches).toBe(1);
+ // the suppressed result was still deep-scraped and judged
+ const seen = out.pageUpserts.find(u => u.status === "already_seen")!;
+ expect(seen.scraped).toBe(true);
+ expect(seen.metadata.searchStatus).toBe("already_seen");
});
it("mints a stable opaque key for new events (not a label slug)", async () => {
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index 452b738fb..a208c454f 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -85,6 +85,9 @@ export type KnownPage = {
goalVersion: string;
lastCheckedAt?: string;
lastStatus?: string;
+ // Full stored monitor_pages metadata, carried through so unchanged-page
+ // upserts don't wipe event/verdict fields persisted by earlier runs.
+ metadata?: Record<string, unknown>;
};
type SearchSource = {
@@ -349,7 +352,15 @@ export async function runSearchTarget(params: {
url: canonical,
urlHash: hashMonitorUrl(canonical),
status: reusedStatus,
- metadata: { fingerprint, goalVersion },
+ // Upserts replace the stored metadata wholesale, so spread the prior
+ // metadata to preserve event stamps (eventKey/eventLabel/...) that
+ // event reconstruction depends on.
+ metadata: {
+ ...(knownCurrent?.metadata ?? {}),
+ fingerprint,
+ goalVersion,
+ searchStatus: reusedStatus,
+ },
});
continue;
}
@@ -485,7 +496,8 @@ export async function runSearchTarget(params: {
url: canonical,
urlHash: hashMonitorUrl(canonical),
status: "ignored",
- metadata: baseMeta,
+ scraped: depth === "deep",
+ metadata: { ...baseMeta, searchStatus: "ignored" },
});
continue;
}
@@ -500,7 +512,8 @@ export async function runSearchTarget(params: {
url: canonical,
urlHash: hashMonitorUrl(canonical),
status: "watching",
- metadata: baseMeta,
+ scraped: depth === "deep",
+ metadata: { ...baseMeta, searchStatus: "watching" },
});
continue;
}
@@ -569,7 +582,8 @@ export async function runSearchTarget(params: {
url: canonical,
urlHash: hashMonitorUrl(canonical),
status: "already_seen",
- metadata: eventMeta,
+ scraped: depth === "deep",
+ metadata: { ...eventMeta, searchStatus: "already_seen" },
});
continue;
}
@@ -605,6 +619,7 @@ export async function runSearchTarget(params: {
scraped: depth === "deep",
metadata: {
...eventMeta,
+ searchStatus: "alert",
eventSatisfiedAt,
eventAlertCount,
eventLastAlertAt: nowIso,