Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
verifier checks page body for subject, keeps 2-char tokens, strips ports from owned hosts
Developers Digest85fa903e0dff
diff --git a/apps/api/src/services/monitoring/search/criteria.ts b/apps/api/src/services/monitoring/search/criteria.tsindex 77834d147..a9be0f233 100644--- a/apps/api/src/services/monitoring/search/criteria.ts+++ b/apps/api/src/services/monitoring/search/criteria.ts@@ -16,10 +16,13 @@ export type GoalCriteria = {};export function tokenizeContent(text: string | null | undefined): string[] {- return String(text ?? "")- .toLowerCase()- .split(/[^a-z0-9]+/)- .filter(token => token.length > 2);+ return (+ String(text ?? "")+ .toLowerCase()+ .split(/[^a-z0-9]+/)+ // Keep 2-char tokens: acronyms like AI, EU, or Go are real content terms.+ .filter(token => token.length >= 2)+ );}function normalizeForContainment(text: string | null | undefined): string {@@ -110,7 +113,8 @@ export function mergeCompiledCriteria(.toLowerCase().replace(/^https?:\/\//, "").replace(/^www\./, "")- .split("/")[0],+ .split("/")[0]+ .split(":")[0],).filter(Boolean),),diff --git a/apps/api/src/services/monitoring/search/verify.test.ts b/apps/api/src/services/monitoring/search/verify.test.tsindex d7e652096..1554a04f4 100644--- a/apps/api/src/services/monitoring/search/verify.test.ts+++ b/apps/api/src/services/monitoring/search/verify.test.ts@@ -1,6 +1,7 @@import {compileGoalCriteria,mergeCompiledCriteria,+ tokenizeContent,type GoalCriteria,} from "./criteria";import { verifyAlertCandidate } from "./verify";@@ -45,6 +46,38 @@ describe("criteria", () => {expect(merged.ownedHosts).toEqual(["firecrawl.dev"]);expect(merged.subjectAliases).toContain("FIRE-1");});++ it("strips ports when normalizing ownedHosts", () => {+ const deterministic = compileGoalCriteria({+ goal: "Alert me when Firecrawl launches a new product",+ subject: "Firecrawl",+ goalVersion: "v1",+ });+ const merged = mergeCompiledCriteria(deterministic, {+ subjectAliases: ["Firecrawl"],+ mustConcern: [],+ excludedSubjects: [],+ ownedHosts: ["https://firecrawl.dev:3000/blog", "localhost:8080"],+ thirdPartyOnly: true,+ });+ expect(merged.ownedHosts).toEqual(["firecrawl.dev", "localhost"]);+ });++ it("tokenizeContent keeps 2-char acronyms (AI, EU, Go) and drops 1-char noise", () => {+ expect(+ tokenizeContent("Go 1.0 released in the EU under the AI Act"),+ ).toEqual([+ "go",+ "released",+ "in",+ "the",+ "eu",+ "under",+ "the",+ "ai",+ "act",+ ]);+ });});describe("verifyAlertCandidate", () => {@@ -65,11 +98,25 @@ describe("verifyAlertCandidate", () => {evidence: evidence({titleText: "A company shipped something",claimText: "Some launch happened.",+ pageText: "Some company shipped something today. Details follow.",}),});expect(result.failures.map(f => f.check)).toContain("subject_missing");});+ it("passes subject containment when the subject appears only in the page body", () => {+ const result = verifyAlertCandidate({+ criteria: baseCriteria(),+ concept: "Firecrawl product launch",+ evidence: evidence({+ titleText: "A scraping startup shipped something",+ claimText: "A new product launched today.",+ pageText: "The company behind Firecrawl announced a new product.",+ }),+ });+ expect(result.failures.map(f => f.check)).not.toContain("subject_missing");+ });+it("skips subject containment for snippet-only results (no page content)", () => {const result = verifyAlertCandidate({criteria: baseCriteria(),diff --git a/apps/api/src/services/monitoring/search/verify.ts b/apps/api/src/services/monitoring/search/verify.tsindex d465016ba..39b0ccdbb 100644--- a/apps/api/src/services/monitoring/search/verify.ts+++ b/apps/api/src/services/monitoring/search/verify.ts@@ -23,7 +23,13 @@ export function verifyAlertCandidate(params: {const { criteria, evidence } = params;const failures: VerificationFailure[] = [];- const pageBackedText = [evidence.titleText, evidence.claimText]+ const pageBackedText = [+ evidence.titleText,+ evidence.claimText,+ // The subject often appears only in the body (e.g. a headline that uses a+ // product name); cap the slice so containment stays cheap on huge pages.+ String(evidence.pageText ?? "").slice(0, 20000),+ ].filter(Boolean).join(" ");const conceptText = String(params.concept ?? "");