Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add alert verification and goal criteria
Developers Digesta5ed23902768
diff --git a/apps/api/src/services/monitoring/search/criteria.ts b/apps/api/src/services/monitoring/search/criteria.tsnew file mode 100644index 000000000..eac3d63a8--- /dev/null+++ b/apps/api/src/services/monitoring/search/criteria.ts@@ -0,0 +1,123 @@+import { z } from "zod";++export type GoalCriteria = {+ goalVersion: string;+ generatedBy: "deterministic" | "llm";+ subjectAliases: string[];+ mustConcern: string[];+ excludedSubjects: string[];+ ownedHosts: string[];+ thirdPartyOnly: boolean;+};++export function tokenizeContent(text: string | null | undefined): string[] {+ return String(text ?? "")+ .toLowerCase()+ .split(/[^a-z0-9]+/)+ .filter(token => token.length >= 2);+}++function normalizeForContainment(text: string | null | undefined): string {+ return ` ${String(text ?? "")+ .toLowerCase()+ .replace(/[^a-z0-9]+/g, " ")+ .trim()} `;+}++export function containsAlias(+ text: string | null | undefined,+ alias: string,+): boolean {+ const haystack = normalizeForContainment(text);+ const needle = normalizeForContainment(alias).trim();+ return needle.length > 0 && haystack.includes(` ${needle} `);+}++export function compileGoalCriteria(params: {+ goal: string;+ subject: string;+ goalVersion: string;+}): GoalCriteria {+ const subject = params.subject.trim();+ return {+ goalVersion: params.goalVersion,+ generatedBy: "deterministic",+ subjectAliases: subject ? [subject] : [],+ mustConcern: [],+ excludedSubjects: [],+ ownedHosts: [],+ thirdPartyOnly: false,+ };+}++const criteriaSchema = z.object({+ subjectAliases: z+ .array(z.string())+ .describe(+ "Every name the monitored subject goes by, INCLUDING its well-known products and brand names. Empty if the goal has no specific subject.",+ ),+ mustConcern: z+ .array(z.string())+ .describe(+ "Single lowercase content words that name the goal's SPECIFIC topic/scope β the subject matter a matching story must actually be about, not just the broad field. Derived from the goal's topic, not its phrasing. For a goal like 'AI safety or alignment research', use the precise topic words (safety, alignment, interpretability, evaluation) rather than broad-field words (ai, research) that adjacent off-topic stories would also share. Matching is EXACT string comparison, so include inflection variants explicitly (e.g. release, releases, released, launch, launches, launched).",+ ),+ excludedSubjects: z+ .array(z.string())+ .describe(+ "Entities whose OWN news must not trigger this monitor (e.g. competitors merely name-dropping the subject). Empty when the goal doesn't imply any.",+ ),+ ownedHosts: z+ .array(z.string())+ .describe(+ "Hostnames operated by the subject itself (official site, docs, blog).",+ ),+ thirdPartyOnly: z+ .boolean()+ .describe(+ "true ONLY when the goal asks for third-party coverage of the subject (mentions, press, discussion, reviews) β the subject's own pages then don't satisfy it. false when the subject's own announcements DO satisfy the goal (releases, launches, filings, pricing changes, changelogs).",+ ),+});++export function mergeCompiledCriteria(+ deterministic: GoalCriteria,+ llm: z.infer<typeof criteriaSchema>,+): GoalCriteria {+ const merged: GoalCriteria = {+ ...deterministic,+ generatedBy: "llm",+ subjectAliases: [+ ...new Set([+ ...deterministic.subjectAliases,+ ...llm.subjectAliases.map(s => s.trim()).filter(Boolean),+ ]),+ ],+ mustConcern: [...new Set(llm.mustConcern.flatMap(tokenizeContent))],+ excludedSubjects: [+ ...new Set(llm.excludedSubjects.map(s => s.trim()).filter(Boolean)),+ ],+ ownedHosts: [+ ...new Set(+ llm.ownedHosts+ .map(+ h =>+ String(h ?? "")+ .trim()+ .toLowerCase()+ .replace(/^https?:\/\//, "")+ .replace(/^www\./, "")+ .split("/")[0]+ .split(":")[0],+ )+ .filter(Boolean),+ ),+ ],+ thirdPartyOnly: llm.thirdPartyOnly === true,+ };+ merged.excludedSubjects = merged.excludedSubjects.filter(+ excluded =>+ !merged.subjectAliases.some(+ alias => alias.toLowerCase() === excluded.toLowerCase(),+ ),+ );+ return merged;+}diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 503aa6bab..d3e3a8ff2 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -12,6 +12,12 @@ import {type SearchVerdict,} from "./judge";import { judgeSnippets, type KnownEvent, type SnippetVerdict } from "./llm";+import { compileGoalCriteria, type GoalCriteria } from "./criteria";+import {+ verifyAlertCandidate,+ type VerificationResult,+ type VerifyEvidence,+} from "./verify";import { hasLlmProvider } from "./tuning";import {searchCreditsForResultCount,@@ -312,6 +318,13 @@ export async function runSearchTarget(params: {const judgeGoal: string = goal ?? "";+ // Deterministic criteria for the (non-LLM) verifier β no LLM enrich/router.+ const criteria: GoalCriteria = compileGoalCriteria({+ goal: judgeGoal,+ subject,+ goalVersion,+ });+const selected = candidates.slice(0, target.maxResults);const recheckMs = target.recheckAfter ? windowToMs(target.recheckAfter) : 0;@@ -532,6 +545,7 @@ export async function runSearchTarget(params: {}let verdict: SearchVerdict | null = null;+ let pageText = "";let realDate: string | null = null;if (depth === "standard") {verdict = snippetVerdicts.get(canonical) ?? null;@@ -594,15 +608,37 @@ export async function runSearchTarget(params: {pushUnevaluatedPage("verdict unparseable");continue;}+ pageText = doc.markdown ?? "";realDate =doc.metadata?.publishedTime ?? doc.metadata?.modifiedTime ?? null;}resultsJudged += 1;- // The per-page JSON verdict is the gate β alert/watch/ignore comes straight- // from the judge that read the scraped page.- const decision = verdictToDecision(verdict);+ let decision = verdictToDecision(verdict);++ let verification: VerificationResult | null = null;+ if (decision === "notify") {+ const evidence: VerifyEvidence = {+ url: c.url,+ titleText: c.title,+ claimText: verdict.rationale,+ pageText,+ };+ const result = verifyAlertCandidate({+ criteria,+ concept: verdict.concept,+ evidence,+ });+ verification = result;+ if (!result.pass) {+ decision = "watch";+ logger.info("search monitor alert downgraded by verifier", {+ url: c.url,+ failures: result.failures,+ });+ }+ }const baseMeta = {fingerprint,@@ -613,6 +649,7 @@ export async function runSearchTarget(params: {rationale: verdict.rationale,matchedQueries: c.matchedQueries,judgedThisRun: true,+ ...(verification && !verification.pass ? { verification } : {}),};if (decision === "ignore") {diff --git a/apps/api/src/services/monitoring/search/verify.test.ts b/apps/api/src/services/monitoring/search/verify.test.tsnew file mode 100644index 000000000..220ae2d68--- /dev/null+++ b/apps/api/src/services/monitoring/search/verify.test.ts@@ -0,0 +1,204 @@+import {+ compileGoalCriteria,+ mergeCompiledCriteria,+ tokenizeContent,+ type GoalCriteria,+} from "./criteria";+import { verifyAlertCandidate } from "./verify";++const baseCriteria = (over: Partial<GoalCriteria> = {}): GoalCriteria => ({+ goalVersion: "v1",+ generatedBy: "llm",+ subjectAliases: ["Firecrawl"],+ mustConcern: ["launch", "launches", "launched", "product"],+ excludedSubjects: ["Parallel"],+ ownedHosts: ["firecrawl.dev"],+ thirdPartyOnly: false,+ ...over,+});++const evidence = (+ over: Partial<Parameters<typeof verifyAlertCandidate>[0]["evidence"]> = {},+) => ({+ url: "https://news.example.com/story",+ titleText: "Firecrawl launches a new product",+ claimText: "Firecrawl announced a new product today.",+ pageText: "Firecrawl announced a new product today. Details follow in prose.",+ ...over,+});++describe("criteria", () => {+ it("merge drops an excluded subject that is also an alias (alias wins)", () => {+ const deterministic = compileGoalCriteria({+ goal: "Alert me when Firecrawl launches a new product",+ subject: "Firecrawl",+ goalVersion: "v1",+ });+ const merged = mergeCompiledCriteria(deterministic, {+ subjectAliases: ["Firecrawl", "FIRE-1"],+ mustConcern: ["launch"],+ excludedSubjects: ["firecrawl", "Parallel"],+ ownedHosts: ["https://www.firecrawl.dev/blog"],+ thirdPartyOnly: false,+ });+ expect(merged.generatedBy).toBe("llm");+ expect(merged.excludedSubjects).toEqual(["Parallel"]);+ expect(merged.ownedHosts).toEqual(["firecrawl.dev"]);+ expect(merged.subjectAliases).toContain("FIRE-1");+ });++ it("strips ports when normalizing ownedHosts", () => {+ const deterministic = compileGoalCriteria({+ goal: "Alert me when Firecrawl launches a new product",+ subject: "Firecrawl",+ goalVersion: "v1",+ });+ const merged = mergeCompiledCriteria(deterministic, {+ subjectAliases: ["Firecrawl"],+ mustConcern: [],+ excludedSubjects: [],+ ownedHosts: ["https://firecrawl.dev:3000/blog", "localhost:8080"],+ thirdPartyOnly: true,+ });+ expect(merged.ownedHosts).toEqual(["firecrawl.dev", "localhost"]);+ });++ it("tokenizeContent keeps 2-char acronyms (AI, EU, Go) and drops 1-char noise", () => {+ expect(+ tokenizeContent("Go 1.0 released in the EU under the AI Act"),+ ).toEqual([+ "go",+ "released",+ "in",+ "the",+ "eu",+ "under",+ "the",+ "ai",+ "act",+ ]);+ });+});++describe("verifyAlertCandidate", () => {+ it("passes a clean alert", () => {+ const result = verifyAlertCandidate({+ criteria: baseCriteria(),+ concept: "Firecrawl product launch",+ evidence: evidence(),+ });+ expect(result.pass).toBe(true);+ expect(result.failures).toEqual([]);+ });++ it("fails subject containment when the subject never appears in page-backed evidence", () => {+ const result = verifyAlertCandidate({+ criteria: baseCriteria(),+ concept: "Firecrawl product launch",+ evidence: evidence({+ titleText: "A company shipped something",+ claimText: "Some launch happened.",+ pageText: "Some company shipped something today. Details follow.",+ }),+ });+ expect(result.failures.map(f => f.check)).toContain("subject_missing");+ });++ it("passes subject containment when the subject appears only in the page body", () => {+ const result = verifyAlertCandidate({+ criteria: baseCriteria(),+ concept: "Firecrawl product launch",+ evidence: evidence({+ titleText: "A scraping startup shipped something",+ claimText: "A new product launched today.",+ pageText: "The company behind Firecrawl announced a new product.",+ }),+ });+ expect(result.failures.map(f => f.check)).not.toContain("subject_missing");+ });++ it("skips subject containment for snippet-only results (no page content)", () => {+ const result = verifyAlertCandidate({+ criteria: baseCriteria(),+ concept: "Firecrawl product launch",+ evidence: evidence({+ titleText: "A company shipped something",+ claimText: "Some launch happened.",+ pageText: "",+ }),+ });+ expect(result.failures.map(f => f.check)).not.toContain("subject_missing");+ });++ it("fails excluded-subject dominance when a competitor owns the story", () => {+ const result = verifyAlertCandidate({+ criteria: baseCriteria(),+ concept: "Parallel launches Monitor API",+ evidence: evidence({+ titleText: "Parallel launches Monitor API",+ claimText:+ "Parallel launched its Monitor API; Firecrawl is named as a competitor.",+ }),+ });+ expect(result.failures.map(f => f.check)).toContain("excluded_subject");+ });++ it("fails concept relevance when the concept shares no goal terms (llm criteria only)", () => {+ const offGoal = verifyAlertCandidate({+ criteria: baseCriteria(),+ concept: "quarterly earnings beat",+ evidence: evidence(),+ });+ expect(offGoal.failures.map(f => f.check)).toContain("concept_off_goal");++ const deterministic = verifyAlertCandidate({+ criteria: baseCriteria({ generatedBy: "deterministic" }),+ concept: "quarterly earnings beat",+ evidence: evidence(),+ });+ expect(deterministic.failures.map(f => f.check)).not.toContain(+ "concept_off_goal",+ );+ });++ it("downgrades a subject-only concept that misses the topic (funding vs launch)", () => {+ // "Firecrawl funding" names the subject but not the goal's topic (launch);+ // a subject-alias token must NOT satisfy the must-concern gate.+ const result = verifyAlertCandidate({+ criteria: baseCriteria(),+ concept: "Firecrawl Series B funding",+ evidence: evidence(),+ });+ expect(result.failures.map(f => f.check)).toContain("concept_off_goal");+ });++ it("fails owned-surface only when the goal wants third-party coverage", () => {+ const owned = verifyAlertCandidate({+ criteria: baseCriteria({ thirdPartyOnly: true }),+ concept: "Firecrawl product launch",+ evidence: evidence({ url: "https://www.firecrawl.dev/blog/launch" }),+ });+ expect(owned.failures.map(f => f.check)).toContain("owned_surface");++ const eventGoal = verifyAlertCandidate({+ criteria: baseCriteria({ thirdPartyOnly: false }),+ concept: "Firecrawl product launch",+ evidence: evidence({ url: "https://www.firecrawl.dev/blog/launch" }),+ });+ expect(eventGoal.failures.map(f => f.check)).not.toContain("owned_surface");+ });++ it("empty criteria disables every check (fail-open for unknown goal shapes)", () => {+ const result = verifyAlertCandidate({+ criteria: baseCriteria({+ subjectAliases: [],+ mustConcern: [],+ excludedSubjects: [],+ ownedHosts: [],+ }),+ concept: "anything at all",+ evidence: evidence({ titleText: "unrelated", claimText: "unrelated" }),+ });+ expect(result.pass).toBe(true);+ });+});diff --git a/apps/api/src/services/monitoring/search/verify.ts b/apps/api/src/services/monitoring/search/verify.tsnew file mode 100644index 000000000..82edae5ae--- /dev/null+++ b/apps/api/src/services/monitoring/search/verify.ts@@ -0,0 +1,124 @@+import { containsAlias, tokenizeContent, type GoalCriteria } from "./criteria";++export type VerifyEvidence = {+ url: string;+ titleText: string;+ claimText: string;+ pageText: string;+};++type VerificationFailure = { check: string; detail: string };++export type VerificationResult = {+ pass: boolean;+ failures: VerificationFailure[];+ criteriaSource: GoalCriteria["generatedBy"];+};++export function verifyAlertCandidate(params: {+ criteria: GoalCriteria;+ evidence: VerifyEvidence;+ concept: string;+}): VerificationResult {+ const { criteria, evidence } = params;+ const failures: VerificationFailure[] = [];++ const pageBackedText = [+ evidence.titleText,+ evidence.claimText,+ String(evidence.pageText ?? "").slice(0, 20000),+ ]+ .filter(Boolean)+ .join(" ");+ const conceptText = String(params.concept ?? "");+ const hasPageContent = Boolean(String(evidence.pageText ?? "").trim());++ if (+ criteria.subjectAliases.length > 0 &&+ hasPageContent &&+ pageBackedText.trim()+ ) {+ const present = criteria.subjectAliases.some(+ alias =>+ containsAlias(pageBackedText, alias) ||+ tokenizeContent(alias).some(token =>+ containsAlias(pageBackedText, token),+ ),+ );+ if (!present) {+ failures.push({+ check: "subject_missing",+ detail: `none of [${criteria.subjectAliases.join(", ")}] appear in the page evidence`,+ });+ }+ }++ if (criteria.excludedSubjects.length > 0 && conceptText) {+ const conceptNamesAlias = criteria.subjectAliases.some(alias =>+ containsAlias(conceptText, alias),+ );+ const excludedInConcept = criteria.excludedSubjects.find(+ excluded =>+ containsAlias(conceptText, excluded) ||+ containsAlias(evidence.titleText, excluded),+ );+ if (excludedInConcept && !conceptNamesAlias) {+ failures.push({+ check: "excluded_subject",+ detail: `the story is about "${excludedInConcept}", which this monitor excludes`,+ });+ }+ }++ if (+ criteria.generatedBy === "llm" &&+ criteria.mustConcern.length > 0 &&+ conceptText+ ) {+ const conceptTokens = new Set(tokenizeContent(conceptText));+ const topicPresent = criteria.mustConcern.some(token =>+ conceptTokens.has(token),+ );+ if (conceptTokens.size > 0 && !topicPresent) {+ failures.push({+ check: "concept_off_goal",+ detail: `alert concept "${conceptText.slice(0, 80)}" shares no goal terms with the monitor`,+ });+ }+ }++ if (criteria.thirdPartyOnly === true && criteria.ownedHosts.length > 0) {+ const host = hostOf(evidence.url);+ const owned =+ host &&+ criteria.ownedHosts.find(+ ownedHost => host === ownedHost || host.endsWith(`.${ownedHost}`),+ );+ if (owned) {+ failures.push({+ check: "owned_surface",+ detail: `${host} is operated by the monitored subject`,+ });+ }+ }++ // `subject_missing` is a fragile literal alias-in-page check, so keep it as a+ // diagnostic but don't let it alone suppress an alert β only the stronger checks block.+ const blockingFailures = failures.filter(f => f.check !== "subject_missing");++ return {+ pass: blockingFailures.length === 0,+ failures,+ criteriaSource: criteria.generatedBy,+ };+}++function hostOf(url: string): string {+ try {+ return new URL(String(url ?? "")).hostname+ .toLowerCase()+ .replace(/^www\./, "");+ } catch {+ return "";+ }+}diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex 9f0208ed8..eecb9e102 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -231,9 +231,9 @@ describe("FLAT deterministic search-monitor billing", () => {expect(estimateMonitorCreditsPerRun([searchTarget()], false)).toBe(2);});- it("judge ON: search call + flat 5 per judged result (all results, all queries)", () => {- // search = ceil(10/10)*2*1 = 2; judge = 5 * (10 results * 1 query) = 50.- expect(estimateMonitorCreditsPerRun([searchTarget()], true)).toBe(52);+ it("judge ON: search call + flat 1 per judged result (capped at maxResults)", () => {+ // search = ceil(10/10)*2*1 = 2; judge = 1 * min(maxResults, 10*1) = 10.+ expect(estimateMonitorCreditsPerRun([searchTarget()], true)).toBe(12);});it("legacy depth:raw target is never judged even when judging is on", () => {@@ -247,9 +247,9 @@ describe("FLAT deterministic search-monitor billing", () => {maxResults: 20,queries: ["a", "b", "c"],});- // search = ceil((20*3)/10)*2 = 12; judge = 5 * 20 (total cap, NOT *queries)- // = 100 β 112. The runner judges at most maxResults candidates total.- expect(estimateMonitorCreditsPerRun([t], true)).toBe(112);+ // search = ceil((20*3)/10)*2 = 12; judge = 1 * 20 (maxResults cap, NOT *queries)+ // = 20 β 32. The runner judges at most maxResults candidates total.+ expect(estimateMonitorCreditsPerRun([t], true)).toBe(32);});});