Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Add alert verification and goal criteria

Developers Digesta5ed23902768
diff --git a/apps/api/src/services/monitoring/search/criteria.ts b/apps/api/src/services/monitoring/search/criteria.ts
new file mode 100644
index 000000000..eac3d63a8
--- /dev/null
+++ b/apps/api/src/services/monitoring/search/criteria.ts
@@ -0,0 +1,123 @@
+import { z } from "zod";
+
+export type GoalCriteria = {
+ goalVersion: string;
+ generatedBy: "deterministic" | "llm";
+ subjectAliases: string[];
+ mustConcern: string[];
+ excludedSubjects: string[];
+ ownedHosts: string[];
+ thirdPartyOnly: boolean;
+};
+
+export function tokenizeContent(text: string | null | undefined): string[] {
+ return String(text ?? "")
+ .toLowerCase()
+ .split(/[^a-z0-9]+/)
+ .filter(token => token.length >= 2);
+}
+
+function normalizeForContainment(text: string | null | undefined): string {
+ return ` ${String(text ?? "")
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, " ")
+ .trim()} `;
+}
+
+export function containsAlias(
+ text: string | null | undefined,
+ alias: string,
+): boolean {
+ const haystack = normalizeForContainment(text);
+ const needle = normalizeForContainment(alias).trim();
+ return needle.length > 0 && haystack.includes(` ${needle} `);
+}
+
+export function compileGoalCriteria(params: {
+ goal: string;
+ subject: string;
+ goalVersion: string;
+}): GoalCriteria {
+ const subject = params.subject.trim();
+ return {
+ goalVersion: params.goalVersion,
+ generatedBy: "deterministic",
+ subjectAliases: subject ? [subject] : [],
+ mustConcern: [],
+ excludedSubjects: [],
+ ownedHosts: [],
+ thirdPartyOnly: false,
+ };
+}
+
+const criteriaSchema = z.object({
+ subjectAliases: z
+ .array(z.string())
+ .describe(
+ "Every name the monitored subject goes by, INCLUDING its well-known products and brand names. Empty if the goal has no specific subject.",
+ ),
+ mustConcern: z
+ .array(z.string())
+ .describe(
+ "Single lowercase content words that name the goal's SPECIFIC topic/scope β€” the subject matter a matching story must actually be about, not just the broad field. Derived from the goal's topic, not its phrasing. For a goal like 'AI safety or alignment research', use the precise topic words (safety, alignment, interpretability, evaluation) rather than broad-field words (ai, research) that adjacent off-topic stories would also share. Matching is EXACT string comparison, so include inflection variants explicitly (e.g. release, releases, released, launch, launches, launched).",
+ ),
+ excludedSubjects: z
+ .array(z.string())
+ .describe(
+ "Entities whose OWN news must not trigger this monitor (e.g. competitors merely name-dropping the subject). Empty when the goal doesn't imply any.",
+ ),
+ ownedHosts: z
+ .array(z.string())
+ .describe(
+ "Hostnames operated by the subject itself (official site, docs, blog).",
+ ),
+ thirdPartyOnly: z
+ .boolean()
+ .describe(
+ "true ONLY when the goal asks for third-party coverage of the subject (mentions, press, discussion, reviews) β€” the subject's own pages then don't satisfy it. false when the subject's own announcements DO satisfy the goal (releases, launches, filings, pricing changes, changelogs).",
+ ),
+});
+
+export function mergeCompiledCriteria(
+ deterministic: GoalCriteria,
+ llm: z.infer<typeof criteriaSchema>,
+): GoalCriteria {
+ const merged: GoalCriteria = {
+ ...deterministic,
+ generatedBy: "llm",
+ subjectAliases: [
+ ...new Set([
+ ...deterministic.subjectAliases,
+ ...llm.subjectAliases.map(s => s.trim()).filter(Boolean),
+ ]),
+ ],
+ mustConcern: [...new Set(llm.mustConcern.flatMap(tokenizeContent))],
+ excludedSubjects: [
+ ...new Set(llm.excludedSubjects.map(s => s.trim()).filter(Boolean)),
+ ],
+ ownedHosts: [
+ ...new Set(
+ llm.ownedHosts
+ .map(
+ h =>
+ String(h ?? "")
+ .trim()
+ .toLowerCase()
+ .replace(/^https?:\/\//, "")
+ .replace(/^www\./, "")
+ .split("/")[0]
+ .split(":")[0],
+ )
+ .filter(Boolean),
+ ),
+ ],
+ thirdPartyOnly: llm.thirdPartyOnly === true,
+ };
+ merged.excludedSubjects = merged.excludedSubjects.filter(
+ excluded =>
+ !merged.subjectAliases.some(
+ alias => alias.toLowerCase() === excluded.toLowerCase(),
+ ),
+ );
+ return merged;
+}
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index 503aa6bab..d3e3a8ff2 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -12,6 +12,12 @@ import {
type SearchVerdict,
} from "./judge";
import { judgeSnippets, type KnownEvent, type SnippetVerdict } from "./llm";
+import { compileGoalCriteria, type GoalCriteria } from "./criteria";
+import {
+ verifyAlertCandidate,
+ type VerificationResult,
+ type VerifyEvidence,
+} from "./verify";
import { hasLlmProvider } from "./tuning";
import {
searchCreditsForResultCount,
@@ -312,6 +318,13 @@ export async function runSearchTarget(params: {
const judgeGoal: string = goal ?? "";
+ // Deterministic criteria for the (non-LLM) verifier β€” no LLM enrich/router.
+ const criteria: GoalCriteria = compileGoalCriteria({
+ goal: judgeGoal,
+ subject,
+ goalVersion,
+ });
+
const selected = candidates.slice(0, target.maxResults);
const recheckMs = target.recheckAfter ? windowToMs(target.recheckAfter) : 0;
@@ -532,6 +545,7 @@ export async function runSearchTarget(params: {
}
let verdict: SearchVerdict | null = null;
+ let pageText = "";
let realDate: string | null = null;
if (depth === "standard") {
verdict = snippetVerdicts.get(canonical) ?? null;
@@ -594,15 +608,37 @@ export async function runSearchTarget(params: {
pushUnevaluatedPage("verdict unparseable");
continue;
}
+ pageText = doc.markdown ?? "";
realDate =
doc.metadata?.publishedTime ?? doc.metadata?.modifiedTime ?? null;
}
resultsJudged += 1;
- // The per-page JSON verdict is the gate β€” alert/watch/ignore comes straight
- // from the judge that read the scraped page.
- const decision = verdictToDecision(verdict);
+ let decision = verdictToDecision(verdict);
+
+ let verification: VerificationResult | null = null;
+ if (decision === "notify") {
+ const evidence: VerifyEvidence = {
+ url: c.url,
+ titleText: c.title,
+ claimText: verdict.rationale,
+ pageText,
+ };
+ const result = verifyAlertCandidate({
+ criteria,
+ concept: verdict.concept,
+ evidence,
+ });
+ verification = result;
+ if (!result.pass) {
+ decision = "watch";
+ logger.info("search monitor alert downgraded by verifier", {
+ url: c.url,
+ failures: result.failures,
+ });
+ }
+ }
const baseMeta = {
fingerprint,
@@ -613,6 +649,7 @@ export async function runSearchTarget(params: {
rationale: verdict.rationale,
matchedQueries: c.matchedQueries,
judgedThisRun: true,
+ ...(verification && !verification.pass ? { verification } : {}),
};
if (decision === "ignore") {
diff --git a/apps/api/src/services/monitoring/search/verify.test.ts b/apps/api/src/services/monitoring/search/verify.test.ts
new file mode 100644
index 000000000..220ae2d68
--- /dev/null
+++ b/apps/api/src/services/monitoring/search/verify.test.ts
@@ -0,0 +1,204 @@
+import {
+ compileGoalCriteria,
+ mergeCompiledCriteria,
+ tokenizeContent,
+ type GoalCriteria,
+} from "./criteria";
+import { verifyAlertCandidate } from "./verify";
+
+const baseCriteria = (over: Partial<GoalCriteria> = {}): GoalCriteria => ({
+ goalVersion: "v1",
+ generatedBy: "llm",
+ subjectAliases: ["Firecrawl"],
+ mustConcern: ["launch", "launches", "launched", "product"],
+ excludedSubjects: ["Parallel"],
+ ownedHosts: ["firecrawl.dev"],
+ thirdPartyOnly: false,
+ ...over,
+});
+
+const evidence = (
+ over: Partial<Parameters<typeof verifyAlertCandidate>[0]["evidence"]> = {},
+) => ({
+ url: "https://news.example.com/story",
+ titleText: "Firecrawl launches a new product",
+ claimText: "Firecrawl announced a new product today.",
+ pageText: "Firecrawl announced a new product today. Details follow in prose.",
+ ...over,
+});
+
+describe("criteria", () => {
+ it("merge drops an excluded subject that is also an alias (alias wins)", () => {
+ const deterministic = compileGoalCriteria({
+ goal: "Alert me when Firecrawl launches a new product",
+ subject: "Firecrawl",
+ goalVersion: "v1",
+ });
+ const merged = mergeCompiledCriteria(deterministic, {
+ subjectAliases: ["Firecrawl", "FIRE-1"],
+ mustConcern: ["launch"],
+ excludedSubjects: ["firecrawl", "Parallel"],
+ ownedHosts: ["https://www.firecrawl.dev/blog"],
+ thirdPartyOnly: false,
+ });
+ expect(merged.generatedBy).toBe("llm");
+ expect(merged.excludedSubjects).toEqual(["Parallel"]);
+ expect(merged.ownedHosts).toEqual(["firecrawl.dev"]);
+ expect(merged.subjectAliases).toContain("FIRE-1");
+ });
+
+ it("strips ports when normalizing ownedHosts", () => {
+ const deterministic = compileGoalCriteria({
+ goal: "Alert me when Firecrawl launches a new product",
+ subject: "Firecrawl",
+ goalVersion: "v1",
+ });
+ const merged = mergeCompiledCriteria(deterministic, {
+ subjectAliases: ["Firecrawl"],
+ mustConcern: [],
+ excludedSubjects: [],
+ ownedHosts: ["https://firecrawl.dev:3000/blog", "localhost:8080"],
+ thirdPartyOnly: true,
+ });
+ expect(merged.ownedHosts).toEqual(["firecrawl.dev", "localhost"]);
+ });
+
+ it("tokenizeContent keeps 2-char acronyms (AI, EU, Go) and drops 1-char noise", () => {
+ expect(
+ tokenizeContent("Go 1.0 released in the EU under the AI Act"),
+ ).toEqual([
+ "go",
+ "released",
+ "in",
+ "the",
+ "eu",
+ "under",
+ "the",
+ "ai",
+ "act",
+ ]);
+ });
+});
+
+describe("verifyAlertCandidate", () => {
+ it("passes a clean alert", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "Firecrawl product launch",
+ evidence: evidence(),
+ });
+ expect(result.pass).toBe(true);
+ expect(result.failures).toEqual([]);
+ });
+
+ it("fails subject containment when the subject never appears in page-backed evidence", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "Firecrawl product launch",
+ evidence: evidence({
+ titleText: "A company shipped something",
+ claimText: "Some launch happened.",
+ pageText: "Some company shipped something today. Details follow.",
+ }),
+ });
+ expect(result.failures.map(f => f.check)).toContain("subject_missing");
+ });
+
+ it("passes subject containment when the subject appears only in the page body", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "Firecrawl product launch",
+ evidence: evidence({
+ titleText: "A scraping startup shipped something",
+ claimText: "A new product launched today.",
+ pageText: "The company behind Firecrawl announced a new product.",
+ }),
+ });
+ expect(result.failures.map(f => f.check)).not.toContain("subject_missing");
+ });
+
+ it("skips subject containment for snippet-only results (no page content)", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "Firecrawl product launch",
+ evidence: evidence({
+ titleText: "A company shipped something",
+ claimText: "Some launch happened.",
+ pageText: "",
+ }),
+ });
+ expect(result.failures.map(f => f.check)).not.toContain("subject_missing");
+ });
+
+ it("fails excluded-subject dominance when a competitor owns the story", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "Parallel launches Monitor API",
+ evidence: evidence({
+ titleText: "Parallel launches Monitor API",
+ claimText:
+ "Parallel launched its Monitor API; Firecrawl is named as a competitor.",
+ }),
+ });
+ expect(result.failures.map(f => f.check)).toContain("excluded_subject");
+ });
+
+ it("fails concept relevance when the concept shares no goal terms (llm criteria only)", () => {
+ const offGoal = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "quarterly earnings beat",
+ evidence: evidence(),
+ });
+ expect(offGoal.failures.map(f => f.check)).toContain("concept_off_goal");
+
+ const deterministic = verifyAlertCandidate({
+ criteria: baseCriteria({ generatedBy: "deterministic" }),
+ concept: "quarterly earnings beat",
+ evidence: evidence(),
+ });
+ expect(deterministic.failures.map(f => f.check)).not.toContain(
+ "concept_off_goal",
+ );
+ });
+
+ it("downgrades a subject-only concept that misses the topic (funding vs launch)", () => {
+ // "Firecrawl funding" names the subject but not the goal's topic (launch);
+ // a subject-alias token must NOT satisfy the must-concern gate.
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "Firecrawl Series B funding",
+ evidence: evidence(),
+ });
+ expect(result.failures.map(f => f.check)).toContain("concept_off_goal");
+ });
+
+ it("fails owned-surface only when the goal wants third-party coverage", () => {
+ const owned = verifyAlertCandidate({
+ criteria: baseCriteria({ thirdPartyOnly: true }),
+ concept: "Firecrawl product launch",
+ evidence: evidence({ url: "https://www.firecrawl.dev/blog/launch" }),
+ });
+ expect(owned.failures.map(f => f.check)).toContain("owned_surface");
+
+ const eventGoal = verifyAlertCandidate({
+ criteria: baseCriteria({ thirdPartyOnly: false }),
+ concept: "Firecrawl product launch",
+ evidence: evidence({ url: "https://www.firecrawl.dev/blog/launch" }),
+ });
+ expect(eventGoal.failures.map(f => f.check)).not.toContain("owned_surface");
+ });
+
+ it("empty criteria disables every check (fail-open for unknown goal shapes)", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria({
+ subjectAliases: [],
+ mustConcern: [],
+ excludedSubjects: [],
+ ownedHosts: [],
+ }),
+ concept: "anything at all",
+ evidence: evidence({ titleText: "unrelated", claimText: "unrelated" }),
+ });
+ expect(result.pass).toBe(true);
+ });
+});
diff --git a/apps/api/src/services/monitoring/search/verify.ts b/apps/api/src/services/monitoring/search/verify.ts
new file mode 100644
index 000000000..82edae5ae
--- /dev/null
+++ b/apps/api/src/services/monitoring/search/verify.ts
@@ -0,0 +1,124 @@
+import { containsAlias, tokenizeContent, type GoalCriteria } from "./criteria";
+
+export type VerifyEvidence = {
+ url: string;
+ titleText: string;
+ claimText: string;
+ pageText: string;
+};
+
+type VerificationFailure = { check: string; detail: string };
+
+export type VerificationResult = {
+ pass: boolean;
+ failures: VerificationFailure[];
+ criteriaSource: GoalCriteria["generatedBy"];
+};
+
+export function verifyAlertCandidate(params: {
+ criteria: GoalCriteria;
+ evidence: VerifyEvidence;
+ concept: string;
+}): VerificationResult {
+ const { criteria, evidence } = params;
+ const failures: VerificationFailure[] = [];
+
+ const pageBackedText = [
+ evidence.titleText,
+ evidence.claimText,
+ String(evidence.pageText ?? "").slice(0, 20000),
+ ]
+ .filter(Boolean)
+ .join(" ");
+ const conceptText = String(params.concept ?? "");
+ const hasPageContent = Boolean(String(evidence.pageText ?? "").trim());
+
+ if (
+ criteria.subjectAliases.length > 0 &&
+ hasPageContent &&
+ pageBackedText.trim()
+ ) {
+ const present = criteria.subjectAliases.some(
+ alias =>
+ containsAlias(pageBackedText, alias) ||
+ tokenizeContent(alias).some(token =>
+ containsAlias(pageBackedText, token),
+ ),
+ );
+ if (!present) {
+ failures.push({
+ check: "subject_missing",
+ detail: `none of [${criteria.subjectAliases.join(", ")}] appear in the page evidence`,
+ });
+ }
+ }
+
+ if (criteria.excludedSubjects.length > 0 && conceptText) {
+ const conceptNamesAlias = criteria.subjectAliases.some(alias =>
+ containsAlias(conceptText, alias),
+ );
+ const excludedInConcept = criteria.excludedSubjects.find(
+ excluded =>
+ containsAlias(conceptText, excluded) ||
+ containsAlias(evidence.titleText, excluded),
+ );
+ if (excludedInConcept && !conceptNamesAlias) {
+ failures.push({
+ check: "excluded_subject",
+ detail: `the story is about "${excludedInConcept}", which this monitor excludes`,
+ });
+ }
+ }
+
+ if (
+ criteria.generatedBy === "llm" &&
+ criteria.mustConcern.length > 0 &&
+ conceptText
+ ) {
+ const conceptTokens = new Set(tokenizeContent(conceptText));
+ const topicPresent = criteria.mustConcern.some(token =>
+ conceptTokens.has(token),
+ );
+ if (conceptTokens.size > 0 && !topicPresent) {
+ failures.push({
+ check: "concept_off_goal",
+ detail: `alert concept "${conceptText.slice(0, 80)}" shares no goal terms with the monitor`,
+ });
+ }
+ }
+
+ if (criteria.thirdPartyOnly === true && criteria.ownedHosts.length > 0) {
+ const host = hostOf(evidence.url);
+ const owned =
+ host &&
+ criteria.ownedHosts.find(
+ ownedHost => host === ownedHost || host.endsWith(`.${ownedHost}`),
+ );
+ if (owned) {
+ failures.push({
+ check: "owned_surface",
+ detail: `${host} is operated by the monitored subject`,
+ });
+ }
+ }
+
+ // `subject_missing` is a fragile literal alias-in-page check, so keep it as a
+ // diagnostic but don't let it alone suppress an alert β€” only the stronger checks block.
+ const blockingFailures = failures.filter(f => f.check !== "subject_missing");
+
+ return {
+ pass: blockingFailures.length === 0,
+ failures,
+ criteriaSource: criteria.generatedBy,
+ };
+}
+
+function hostOf(url: string): string {
+ try {
+ return new URL(String(url ?? "")).hostname
+ .toLowerCase()
+ .replace(/^www\./, "");
+ } catch {
+ return "";
+ }
+}
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index 9f0208ed8..eecb9e102 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -231,9 +231,9 @@ describe("FLAT deterministic search-monitor billing", () => {
expect(estimateMonitorCreditsPerRun([searchTarget()], false)).toBe(2);
});
- it("judge ON: search call + flat 5 per judged result (all results, all queries)", () => {
- // search = ceil(10/10)*2*1 = 2; judge = 5 * (10 results * 1 query) = 50.
- expect(estimateMonitorCreditsPerRun([searchTarget()], true)).toBe(52);
+ it("judge ON: search call + flat 1 per judged result (capped at maxResults)", () => {
+ // search = ceil(10/10)*2*1 = 2; judge = 1 * min(maxResults, 10*1) = 10.
+ expect(estimateMonitorCreditsPerRun([searchTarget()], true)).toBe(12);
});
it("legacy depth:raw target is never judged even when judging is on", () => {
@@ -247,9 +247,9 @@ describe("FLAT deterministic search-monitor billing", () => {
maxResults: 20,
queries: ["a", "b", "c"],
});
- // search = ceil((20*3)/10)*2 = 12; judge = 5 * 20 (total cap, NOT *queries)
- // = 100 β†’ 112. The runner judges at most maxResults candidates total.
- expect(estimateMonitorCreditsPerRun([t], true)).toBe(112);
+ // search = ceil((20*3)/10)*2 = 12; judge = 1 * 20 (maxResults cap, NOT *queries)
+ // = 20 β†’ 32. The runner judges at most maxResults candidates total.
+ expect(estimateMonitorCreditsPerRun([t], true)).toBe(32);
});
});