Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
carry prior page status through unchanged-page reuse so already_seen means previously alerted
Developers Digesta0cbeee15289
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 1167d5ba8..3aef68fd7 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -154,12 +154,15 @@ describe("runSearchTarget orchestration", () => {snippet: "filing",});const knownPages = new Map<string, KnownPage>([- [canonical, { fingerprint, goalVersion: "gv1" }],+ [canonical, { fingerprint, goalVersion: "gv1", lastStatus: "alert" }],]);const out = await run({ knownPages });+ // Reuse carries the prior outcome forward: this page alerted before, so the+ // repeat is "already_seen". A page with no recorded prior alert must not be.expect(out.sources[0].status).toBe("already_seen");+ expect(out.pageUpserts[0].status).toBe("already_seen");expect(scrapeURLMock).not.toHaveBeenCalled();expect(out.matches).toBe(0);});@@ -446,7 +449,9 @@ describe("re-judge cadence", () => {]);const out = await run({ target: { recheckAfter: "24h" }, knownPages });expect(scrapeURLMock).not.toHaveBeenCalled();- expect(out.sources[0].status).toBe("already_seen");+ // A page that only ever watched repeats as watching β "already_seen" is+ // reserved for pages that actually alerted on a prior run.+ expect(out.sources[0].status).toBe("watching");});it("does NOT re-judge a non-live (ignored) page even when stale", async () => {@@ -465,7 +470,7 @@ describe("re-judge cadence", () => {]);const out = await run({ target: { recheckAfter: "24h" }, knownPages });expect(scrapeURLMock).not.toHaveBeenCalled();- expect(out.sources[0].status).toBe("already_seen");+ expect(out.sources[0].status).toBe("ignored");});});diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex c7c888c6f..52e65bc38 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -188,13 +188,25 @@ export async function runSearchTarget(params: {knownCurrent.fingerprint !== fingerprint ||dueForRecheck;- // Already-seen + unchanged under the current goal β reuse, no scrape/judge/LLM.+ // Known + unchanged under the current goal β reuse, no scrape/judge/LLM. The reused+ // status carries the page's prior outcome forward: "already_seen" must mean "this+ // alerted before" β a page that only ever watched/ignored repeats as such, otherwise+ // the UI reports a prior alert that never happened (and flipping it to already_seen+ // would also drop it out of the isLive recheck cadence above).if (!isNewOrChanged) {- sources.push({ url: c.url, title: c.title, status: "already_seen" });+ const reusedStatus: SearchSource["status"] =+ knownCurrent?.lastStatus === "alert" ||+ knownCurrent?.lastStatus === "already_seen"+ ? "already_seen"+ : knownCurrent?.lastStatus === "ignored" ||+ knownCurrent?.lastStatus === "skipped"+ ? knownCurrent.lastStatus+ : "watching";+ sources.push({ url: c.url, title: c.title, status: reusedStatus });pageUpserts.push({url: canonical,urlHash: hashMonitorUrl(canonical),- status: "already_seen",+ status: reusedStatus,metadata: { fingerprint, goalVersion },});continue;