Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
trim search tests to crucial behavior coverage
Developers Digest6dd17cceeabc
diff --git a/apps/api/src/services/monitoring/search/judge.test.ts b/apps/api/src/services/monitoring/search/judge.test.tsindex a89db7ca6..9c8a8f3d0 100644--- a/apps/api/src/services/monitoring/search/judge.test.ts+++ b/apps/api/src/services/monitoring/search/judge.test.ts@@ -23,12 +23,6 @@ describe("verdictToDecision", () => {it("ignores when not relevant", () => {expect(verdictToDecision(v({ relevant: false }))).toBe("ignore");});- it("ignores when judge says ignore", () => {- expect(verdictToDecision(v({ alertAction: "ignore" }))).toBe("ignore");- });- it("watches when judge says watch", () => {- expect(verdictToDecision(v({ alertAction: "watch" }))).toBe("watch");- });});describe("parseVerdict", () => {@@ -47,10 +41,6 @@ describe("parseVerdict", () => {}),).toMatchObject({ relevant: true, alertAction: "alert", concept: "c" });});- it("defaults unknown enum values safely", () => {- const out = parseVerdict({ relevant: true, alertAction: "bogus" });- expect(out?.alertAction).toBe("watch");- });});describe("dedupe", () => {@@ -72,19 +62,6 @@ describe("dedupe", () => {});expect(a).toBe(b);});- it("different snippet → different fingerprint", () => {- const a = stableSerpFingerprint({- url: "https://a.com",- title: "T",- snippet: "S1",- });- const b = stableSerpFingerprint({- url: "https://a.com",- title: "T",- snippet: "S2",- });- expect(a).not.toBe(b);- });});describe("verdict defenses", () => {@@ -107,20 +84,6 @@ describe("verdict defenses", () => {).toBe("no");});- it("detects insufficient-evidence contradictions as unclear", () => {- expect(- contradictionFromRationale(- "There is not enough evidence to confirm a launch.",- ),- ).toBe("unclear");- });-- it("returns empty for a rationale that supports the verdict", () => {- expect(- contradictionFromRationale("Anthropic's own blog announces the release."),- ).toBe("");- });-it("applyVerdictDefenses flips a self-contradicting alert to ignore", () => {const corrected = applyVerdictDefenses(v({ rationale: "The page does not mention the subject." }),diff --git a/apps/api/src/services/monitoring/search/persist.test.ts b/apps/api/src/services/monitoring/search/persist.test.tsindex a2baa27bb..6fe1df3b2 100644--- a/apps/api/src/services/monitoring/search/persist.test.ts+++ b/apps/api/src/services/monitoring/search/persist.test.ts@@ -42,21 +42,6 @@ describe("reconstructKnownState", () => {});});- it("loads fingerprints from any goalVersion (the runner gate decides freshness)", () => {- const { knownPages } = reconstructKnownState(- [- {- url: "https://a.com",- metadata: { fingerprint: "old", goalVersion: "gvOLD" },- },- ],- "gvNEW",- );- expect(knownPages.get(canonicalizeUrl("https://a.com"))?.goalVersion).toBe(- "gvOLD",- );- });-it("only includes events from the current goalVersion (goal change starts clean)", () => {const { knownEvents } = reconstructKnownState([@@ -84,28 +69,6 @@ describe("reconstructKnownState", () => {expect(knownEvents).toEqual([{ key: "e1", label: "Event One" }]);});- it("dedups repeated event keys and falls back label→key", () => {- const { knownEvents } = reconstructKnownState(- [- {- url: "https://a.com",- metadata: { fingerprint: "f1", goalVersion: "gv1", eventKey: "e1" },- },- {- url: "https://b.com",- metadata: {- fingerprint: "f2",- goalVersion: "gv1",- eventKey: "e1",- eventLabel: "Later",- },- },- ],- "gv1",- );- expect(knownEvents).toEqual([{ key: "e1", label: "e1" }]);- });-it("orders events most-recently-seen first (resolver window holds active stories)", () => {const page = (url: string, eventKey: string, updated_at: string) => ({url,@@ -128,18 +91,6 @@ describe("reconstructKnownState", () => {);expect(knownEvents.map(e => e.key)).toEqual(["old-story", "active-story"]);});-- it("ignores pages with no usable metadata", () => {- const { knownPages, knownEvents } = reconstructKnownState(- [- { url: "https://a.com", metadata: null },- { url: "https://b.com", metadata: {} },- ],- "gv1",- );- expect(knownPages.size).toBe(0);- expect(knownEvents).toEqual([]);- });});describe("event state aggregation (jsonb stamps on alerting pages)", () => {diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 1ee9e0ff9..0474201f1 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -458,59 +458,11 @@ describe("re-judge cadence", () => {// reserved for pages that actually alerted on a prior run.expect(out.sources[0].status).toBe("watching");});-- it("does NOT re-judge a non-live (ignored) page even when stale", async () => {- setSearchResults(serp);- setVerdictsByUrl({ [url]: verdict() });- const knownPages = new Map<string, KnownPage>([- [- canonicalizeUrl(url),- {- fingerprint: fp,- goalVersion: "gv1",- lastCheckedAt: old,- lastStatus: "ignored",- },- ],- ]);- const out = await run({ target: { recheckAfter: "24h" }, knownPages });- expect(scrapeURLMock).not.toHaveBeenCalled();- expect(out.sources[0].status).toBe("ignored");- });});describe("domain scoping", () => {beforeEach(() => setSearchResults([]));- it("includeDomains → site: OR filter appended to the query", async () => {- await runSearchTarget({- monitor: baseMonitor,- target: { ...baseTarget, includeDomains: ["sec.gov", "reuters.com"] },- goalVersion: "gv1",- knownPages: new Map(),- knownEvents: [],- zeroDataRetention: false,- logger,- });- const sentQuery = searchMock.mock.calls[0][0].query as string;- expect(sentQuery).toContain("openai ipo");- expect(sentQuery).toContain("(site:sec.gov OR site:reuters.com)");- });-- it("excludeDomains → -site: filter appended to the query", async () => {- await runSearchTarget({- monitor: baseMonitor,- target: { ...baseTarget, excludeDomains: ["pinterest.com"] },- goalVersion: "gv1",- knownPages: new Map(),- knownEvents: [],- zeroDataRetention: false,- logger,- });- const sentQuery = searchMock.mock.calls[0][0].query as string;- expect(sentQuery).toContain("-site:pinterest.com");- });-it("includeDomains and excludeDomains combine; exclude wins in the query", async () => {await run({target: {@@ -551,17 +503,4 @@ describe("domain scoping", () => {expect(out.sources.map(s => s.url)).toEqual(["https://news.com/openai"]);expect(scrapeURLMock).toHaveBeenCalledTimes(1);});-- it("a host merely containing an excluded domain is not dropped", async () => {- setSearchResults([- { url: "https://notpinterest.com/a", title: "t", description: "x" },- ]);- setVerdictsByUrl({ "https://notpinterest.com/a": verdict() });- const out = await run({- target: { excludeDomains: ["pinterest.com"] } as Partial<- typeof baseTarget- >,- });- expect(out.resultCount).toBe(1);- });});diff --git a/apps/api/src/services/monitoring/search/verify.test.ts b/apps/api/src/services/monitoring/search/verify.test.tsindex bbf0f65ea..7adaba9d2 100644--- a/apps/api/src/services/monitoring/search/verify.test.ts+++ b/apps/api/src/services/monitoring/search/verify.test.ts@@ -1,8 +1,6 @@import {compileGoalCriteria,- containsAlias,mergeCompiledCriteria,- tokenizeContent,type GoalCriteria,} from "./criteria";import { classifyPageShape, verifyAlertCandidate } from "./verify";@@ -30,33 +28,6 @@ const evidence = (});describe("criteria", () => {- it("tokenizes content words and drops instruction words", () => {- expect(- tokenizeContent("Alert me when Anthropic releases a new Claude model"),- ).toEqual(["anthropic", "releases", "claude", "model"]);- });-- it("containsAlias matches whole words only", () => {- expect(containsAlias("Firecrawl launches today", "Firecrawl")).toBe(true);- expect(containsAlias("Firecrawler launches today", "Firecrawl")).toBe(- false,- );- });-- it("deterministic compile derives subject aliases + mustConcern from the goal", () => {- const criteria = compileGoalCriteria({- goal: "Alert me when Firecrawl launches a new product",- subject: "Firecrawl",- goalVersion: "v1",- });- expect(criteria.generatedBy).toBe("deterministic");- expect(criteria.subjectAliases).toEqual(["Firecrawl"]);- expect(criteria.mustConcern).toContain("launches");- expect(criteria.mustConcern).not.toContain("firecrawl");- expect(criteria.thirdPartyOnly).toBe(false);- expect(criteria.excludedSubjects).toEqual([]);- });-it("merge drops an excluded subject that is also an alias (alias wins)", () => {const deterministic = compileGoalCriteria({goal: "Alert me when Firecrawl launches a new product",@@ -176,14 +147,6 @@ describe("verifyAlertCandidate", () => {});describe("classifyPageShape", () => {- it("declared article metadata wins", () => {- expect(- classifyPageShape(- evidence({ metadata: { ogType: "article" }, pageText: "" }),- ).kind,- ).toBe("article");- });-it("link walls without a publish date classify as listing", () => {const links = Array.from({ length: 40 },@@ -193,15 +156,4 @@ describe("classifyPageShape", () => {"listing",);});-- it("prose with few links classifies as article", () => {- const prose = `${"word ".repeat(500)} [one](https://example.com/a) [two](https://example.com/b)`;- expect(classifyPageShape(evidence({ pageText: prose })).kind).toBe(- "article",- );- });-- it("snippet-only evidence is unknown (never a veto)", () => {- expect(classifyPageShape(evidence({ pageText: "" })).kind).toBe("unknown");- });});