Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

port search monitor p0s: verifier+criteria+skeptic stack, serp router, standard/deep depth modes, thinking suppression, verdict defenses

Developers Digest7024347a22b7
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 8310769c0..89e6df458 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -1002,6 +1002,7 @@ async function runMonitorSearchTarget(params: {
excludeDomains: target.excludeDomains,
recheckAfter: target.recheckAfter,
maxResults: target.maxResults,
+ depth: target.depth,
},
goalVersion,
knownPages,
diff --git a/apps/api/src/services/monitoring/search/criteria.ts b/apps/api/src/services/monitoring/search/criteria.ts
new file mode 100644
index 000000000..ca3ae56aa
--- /dev/null
+++ b/apps/api/src/services/monitoring/search/criteria.ts
@@ -0,0 +1,272 @@
+import { generateObject } from "ai";
+import { google } from "@ai-sdk/google";
+import { z } from "zod";
+import { googleProviderOptions } from "./tuning";
+
+// Compile a monitor goal into a structured criteria artifact that the alert
+// verifier can evaluate mechanically. ALL goal-specific knowledge lives in this
+// artifact (data); the verifier has no per-goal-type code paths. The artifact is
+// keyed to goalVersion, so editing the goal recompiles it automatically.
+// Ported from the POC's goal-criteria.js.
+
+const CRITERIA_MODEL =
+ process.env.SEARCH_MONITOR_CRITERIA_MODEL ?? "gemini-flash-latest";
+
+export type GoalCriteria = {
+ goalVersion: string;
+ generatedBy: "deterministic" | "llm";
+ // Entity names (and product names, when LLM-compiled) that anchor an alert; empty = not enforced.
+ subjectAliases: string[];
+ // Content tokens an alert's concept/evidence must overlap; empty = not enforced.
+ mustConcern: string[];
+ // Entities that must NOT be the story's subject (LLM only β€” e.g. competitors); empty = not enforced.
+ excludedSubjects: string[];
+ // Hosts whose own pages should not alert on mention feeds (LLM only); empty = not enforced.
+ ownedHosts: string[];
+ thirdPartyOnly: boolean;
+};
+
+// Words that describe the *instruction*, not the *content* β€” never criteria.
+const INSTRUCTION_WORDS = new Set([
+ "about",
+ "alert",
+ "alerts",
+ "all",
+ "any",
+ "anything",
+ "are",
+ "article",
+ "articles",
+ "because",
+ "been",
+ "between",
+ "blog",
+ "blogs",
+ "can",
+ "confirm",
+ "could",
+ "each",
+ "every",
+ "evidence",
+ "for",
+ "fresh",
+ "from",
+ "get",
+ "has",
+ "have",
+ "her",
+ "his",
+ "how",
+ "into",
+ "its",
+ "know",
+ "let",
+ "may",
+ "mention",
+ "mentioned",
+ "mentions",
+ "monitor",
+ "new",
+ "news",
+ "not",
+ "notify",
+ "now",
+ "online",
+ "only",
+ "our",
+ "out",
+ "page",
+ "pages",
+ "post",
+ "posts",
+ "related",
+ "requested",
+ "result",
+ "results",
+ "see",
+ "should",
+ "site",
+ "sites",
+ "some",
+ "tell",
+ "that",
+ "the",
+ "their",
+ "them",
+ "then",
+ "there",
+ "these",
+ "they",
+ "this",
+ "those",
+ "track",
+ "want",
+ "watch",
+ "web",
+ "websites",
+ "what",
+ "when",
+ "whenever",
+ "where",
+ "which",
+ "who",
+ "will",
+ "with",
+ "would",
+ "you",
+ "your",
+]);
+
+export function tokenizeContent(text: string | null | undefined): string[] {
+ return String(text ?? "")
+ .toLowerCase()
+ .split(/[^a-z0-9]+/)
+ .filter(token => token.length > 2 && !INSTRUCTION_WORDS.has(token));
+}
+
+function normalizeForContainment(text: string | null | undefined): string {
+ return ` ${String(text ?? "")
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, " ")
+ .trim()} `;
+}
+
+export function containsAlias(
+ text: string | null | undefined,
+ alias: string,
+): boolean {
+ const haystack = normalizeForContainment(text);
+ const needle = normalizeForContainment(alias).trim();
+ return needle.length > 0 && haystack.includes(` ${needle} `);
+}
+
+// Pure, synchronous, no-network compile. Always available; the LLM compile
+// below only ever ADDS knowledge (product aliases, competitors, owned hosts).
+export function compileGoalCriteria(params: {
+ goal: string;
+ subject: string;
+ goalVersion: string;
+}): GoalCriteria {
+ const subject = params.subject.trim();
+ const subjectAliases = subject ? [subject] : [];
+ const subjectTokens = new Set(tokenizeContent(subject));
+ const mustConcern = [
+ ...new Set(
+ tokenizeContent(params.goal).filter(token => !subjectTokens.has(token)),
+ ),
+ ];
+ return {
+ goalVersion: params.goalVersion,
+ generatedBy: "deterministic",
+ subjectAliases,
+ mustConcern,
+ excludedSubjects: [],
+ ownedHosts: [],
+ // Never assume a goal excludes the subject's own sources without the
+ // compiler explicitly saying so β€” vetoing official sources is recall loss.
+ thirdPartyOnly: false,
+ };
+}
+
+const criteriaSchema = z.object({
+ subjectAliases: z
+ .array(z.string())
+ .describe(
+ "Every name the monitored subject goes by, INCLUDING its well-known products and brand names. Empty if the goal has no specific subject.",
+ ),
+ mustConcern: z
+ .array(z.string())
+ .describe(
+ "Single lowercase content words a matching story must plausibly concern. Derived from the goal's topic, not its phrasing. Matching is EXACT string comparison, so include inflection variants explicitly (e.g. release, releases, released, launch, launches, launched).",
+ ),
+ excludedSubjects: z
+ .array(z.string())
+ .describe(
+ "Entities whose OWN news must not trigger this monitor (e.g. competitors merely name-dropping the subject). Empty when the goal doesn't imply any.",
+ ),
+ ownedHosts: z
+ .array(z.string())
+ .describe(
+ "Hostnames operated by the subject itself (official site, docs, blog).",
+ ),
+ thirdPartyOnly: z
+ .boolean()
+ .describe(
+ "true ONLY when the goal asks for third-party coverage of the subject (mentions, press, discussion, reviews) β€” the subject's own pages then don't satisfy it. false when the subject's own announcements DO satisfy the goal (releases, launches, filings, pricing changes, changelogs).",
+ ),
+});
+
+export function mergeCompiledCriteria(
+ deterministic: GoalCriteria,
+ llm: z.infer<typeof criteriaSchema>,
+): GoalCriteria {
+ const merged: GoalCriteria = {
+ ...deterministic,
+ generatedBy: "llm",
+ subjectAliases: [
+ ...new Set([
+ ...deterministic.subjectAliases,
+ ...llm.subjectAliases.map(s => s.trim()).filter(Boolean),
+ ]),
+ ],
+ mustConcern: [
+ ...new Set([
+ ...deterministic.mustConcern,
+ ...llm.mustConcern.flatMap(tokenizeContent),
+ ]),
+ ],
+ excludedSubjects: [
+ ...new Set(llm.excludedSubjects.map(s => s.trim()).filter(Boolean)),
+ ],
+ ownedHosts: [
+ ...new Set(
+ llm.ownedHosts
+ .map(
+ h =>
+ String(h ?? "")
+ .trim()
+ .toLowerCase()
+ .replace(/^https?:\/\//, "")
+ .replace(/^www\./, "")
+ .split("/")[0],
+ )
+ .filter(Boolean),
+ ),
+ ],
+ thirdPartyOnly: llm.thirdPartyOnly === true,
+ };
+ // An alias that is also an excluded subject would make every alert
+ // contradictory β€” the alias wins (the compiler over-reached).
+ merged.excludedSubjects = merged.excludedSubjects.filter(
+ excluded =>
+ !merged.subjectAliases.some(
+ alias => alias.toLowerCase() === excluded.toLowerCase(),
+ ),
+ );
+ return merged;
+}
+
+// LLM enrichment of the deterministic compile. Throws on model failure β€” the
+// caller keeps the deterministic artifact (fail-safe, never blocks the run).
+export async function compileGoalCriteriaWithLlm(params: {
+ goal: string;
+ subject: string;
+ queries: string[];
+ goalVersion: string;
+}): Promise<GoalCriteria> {
+ const deterministic = compileGoalCriteria(params);
+ const { object } = await generateObject({
+ model: google(CRITERIA_MODEL),
+ schema: criteriaSchema,
+ system:
+ "You compile a web-monitoring goal into machine-checkable criteria. The criteria are evaluated with exact string containment and set membership β€” no interpretation happens later, so be complete (list product names, abbreviations, obvious competitors). Only include entries you are confident about; an empty list disables that check, which is always safe.",
+ prompt: JSON.stringify({
+ goal: params.goal,
+ subject: params.subject,
+ queries: params.queries,
+ }),
+ temperature: 0,
+ ...googleProviderOptions(CRITERIA_MODEL),
+ });
+ return mergeCompiledCriteria(deterministic, object);
+}
diff --git a/apps/api/src/services/monitoring/search/judge.test.ts b/apps/api/src/services/monitoring/search/judge.test.ts
index 0d71d7c9a..54902fa23 100644
--- a/apps/api/src/services/monitoring/search/judge.test.ts
+++ b/apps/api/src/services/monitoring/search/judge.test.ts
@@ -1,6 +1,9 @@
import {
+ applyVerdictDefenses,
+ contradictionFromRationale,
freshnessFromDate,
parseVerdict,
+ stripJudgeMetaClaims,
verdictToDecision,
type SearchVerdict,
} from "./judge";
@@ -117,3 +120,64 @@ describe("dedupe", () => {
expect(a).not.toBe(b);
});
});
+
+describe("verdict defenses", () => {
+ it("notify requires a non-empty concept (event dedup needs a label)", () => {
+ expect(verdictToDecision(v({ concept: "" }))).toBe("watch");
+ expect(verdictToDecision(v({ concept: " " }))).toBe("watch");
+ });
+
+ it("detects evidence-absent contradictions as no", () => {
+ expect(
+ contradictionFromRationale("The page does not mention Firecrawl at all."),
+ ).toBe("no");
+ expect(
+ contradictionFromRationale(
+ "The monitored subject is not present on this page.",
+ ),
+ ).toBe("no");
+ expect(
+ contradictionFromRationale("This is only a related background piece."),
+ ).toBe("no");
+ });
+
+ it("detects insufficient-evidence contradictions as unclear", () => {
+ expect(
+ contradictionFromRationale(
+ "There is not enough evidence to confirm a launch.",
+ ),
+ ).toBe("unclear");
+ });
+
+ it("returns empty for a rationale that supports the verdict", () => {
+ expect(
+ contradictionFromRationale("Anthropic's own blog announces the release."),
+ ).toBe("");
+ });
+
+ it("applyVerdictDefenses flips a self-contradicting alert to ignore", () => {
+ const corrected = applyVerdictDefenses(
+ v({ rationale: "The page does not mention the subject." }),
+ );
+ expect(corrected.relevant).toBe(false);
+ expect(corrected.alertAction).toBe("ignore");
+ });
+
+ it("applyVerdictDefenses caps insufficient-evidence alerts at watch", () => {
+ const corrected = applyVerdictDefenses(
+ v({ rationale: "Not enough evidence to alert on this." }),
+ );
+ expect(corrected.alertAction).toBe("watch");
+ expect(corrected.relevant).toBe(true);
+ });
+
+ it("strips meta-claims and field boilerplate, keeps page facts", () => {
+ const stripped = stripJudgeMetaClaims(
+ "Anthropic released Claude 5 today. This fits within the requested topic lanes. Source quality is first-party. Alert action: alert.",
+ );
+ expect(stripped).toContain("Anthropic released Claude 5 today.");
+ expect(stripped).not.toMatch(/topic lanes/i);
+ expect(stripped).not.toMatch(/Source quality/i);
+ expect(stripped).not.toMatch(/Alert action/i);
+ });
+});
diff --git a/apps/api/src/services/monitoring/search/judge.ts b/apps/api/src/services/monitoring/search/judge.ts
index e41b8a635..8ce55f824 100644
--- a/apps/api/src/services/monitoring/search/judge.ts
+++ b/apps/api/src/services/monitoring/search/judge.ts
@@ -112,7 +112,9 @@ export function freshnessFromDate(
return nowMs - t <= window ? "fresh" : "stale";
}
-// "alert" only when relevant, fresh, trusted-enough, and the judge asked to alert.
+// "alert" only when relevant, fresh, trusted-enough, concept-labeled, and the
+// judge asked to alert. A notify without a concept can't be event-deduped, so
+// it would re-alert forever β€” the POC requires the full field set for notify.
export function verdictToDecision(
v: SearchVerdict,
): "notify" | "watch" | "ignore" {
@@ -121,5 +123,110 @@ export function verdictToDecision(
if (v.freshness !== "fresh") return "watch";
if (v.sourceQuality === "unverified" || v.sourceQuality === "unclear")
return "watch";
+ if (!v.concept.trim()) return "watch";
return "notify";
}
+
+// ── Verdict defenses (ported from the POC's assessment.js) ──────────────────
+// The judge's own prose is evidence for downstream stages (event resolver,
+// verifier, skeptic). Two failure modes need mechanical correction before any
+// of that runs: (1) the verdict's boolean contradicts its rationale, and
+// (2) the rationale is self-referential meta-claims ("aligns with the monitor
+// goal") instead of page facts.
+
+// Does the rationale text negate the verdict? Returns the corrected stance:
+// "no" (evidence absent β†’ not relevant), "unclear" (insufficient evidence β†’
+// never alert), or "" (no contradiction).
+export function contradictionFromRationale(
+ rationale: string,
+): "no" | "unclear" | "" {
+ const value = String(rationale ?? "")
+ .trim()
+ .toLowerCase();
+ if (!value) return "";
+ if (
+ /\b(?:not enough|insufficient|too little)\s+evidence\s+(?:to|for)\s+(?:alert|notify|confirm|determine|judge)\b|\bnot\s+enough\s+to\s+(?:alert|notify|confirm|determine|judge)\b/.test(
+ value,
+ )
+ ) {
+ return "unclear";
+ }
+ if (
+ /\b(?:does not|doesn't|do not|don't|did not|cannot|can't|fails to|failed to)\s+(?:mention|name|state|show|provide|confirm|establish|include|identify|report|document)\b/.test(
+ value,
+ )
+ ) {
+ return "no";
+ }
+ if (
+ /\b(?:mention|subject|topic|brand|company|entity|monitored\s+subject)\b.{0,100}\b(?:is\s+)?(?:not\s+present|absent|missing|not\s+found)\b/.test(
+ value,
+ ) ||
+ /\b(?:not\s+present|absent|missing|not\s+found)\b.{0,80}\b(?:on|from|in)\s+(?:this|the)\s+(?:page|result|article|snippet|content)\b/.test(
+ value,
+ )
+ ) {
+ return "no";
+ }
+ if (
+ /\b(?:not|no)\s+(?:an?\s+|the\s+)?(?:direct|exact|matching|concrete|fresh|completed|substantive|user-visible|goal-satisfying)\s+(?:match|event|announcement|release|launch|approval|customer|contract|partnership|deal|acquisition|lawsuit|filing|outage|incident|route|concert|mention)\b/.test(
+ value,
+ )
+ ) {
+ return "no";
+ }
+ if (
+ /\b(?:only|just|merely)\s+(?:a\s+)?(?:related|tangential|adjacent|generic|background)\b/.test(
+ value,
+ )
+ ) {
+ return "no";
+ }
+ return "";
+}
+
+// Apply contradiction correction to a parsed verdict. A "no" contradiction
+// flips relevant off (β†’ ignore); an "unclear" one caps the action at watch.
+export function applyVerdictDefenses(v: SearchVerdict): SearchVerdict {
+ const contradiction = contradictionFromRationale(v.rationale);
+ if (contradiction === "no" && v.relevant) {
+ return { ...v, relevant: false, alertAction: "ignore" };
+ }
+ if (contradiction === "unclear" && v.alertAction === "alert") {
+ return { ...v, alertAction: "watch" };
+ }
+ return v;
+}
+
+// Strip the judge's self-referential meta-claims and field boilerplate from its
+// rationale so downstream stages see page facts, not the judge grading itself.
+// ("…which aligns with the monitor goal" is not evidence; LLM-side chrome like
+// "Related topics: …" is page furniture, not a story.)
+export function stripJudgeMetaClaims(text: string): string {
+ return String(text ?? "")
+ .replace(
+ /\b(?:related|more|recommended|popular|trending|latest|similar|also)\s+(?:topics|tags|categories|sections|links|stories|articles|posts|coverage)\s*:?\s*[^.?!]*(?:[.?!]|$)/gi,
+ " ",
+ )
+ .replace(
+ /[^.?!]*\b(?:align|aligns|aligned|fit|fits|matched?|matches|satisf(?:y|ies|ied))\s+with\s+(?:the\s+)?monitor\s+goal\b[^.?!]*(?:[.?!]|$)/gi,
+ " ",
+ )
+ .replace(
+ /[^.?!]*\b(?:fall|falls|fit|fits|align|aligns|belong|belongs|sit|sits)\s+(?:within|into|under|with)\s+(?:the\s+)?(?:requested\s+)?(?:scope|criteria|categor(?:y|ies)|topic\s+lanes?|lanes?|contexts?)\b[^.?!]*(?:[.?!]|$)/gi,
+ " ",
+ )
+ .replace(
+ /[^.?!]*\b(?:fit|fits|fitting|meet|meets|meeting|satisf(?:y|ies|ying|ied)|match(?:es|ing|ed)?)\s+(?:the\s+)?(?:criteria|requirements)\s+(?:for|of)\b[^.?!]*(?:[.?!]|$)/gi,
+ " ",
+ )
+ .replace(/\bSource quality\s+is\b[^.?!]*(?:[.?!]|$)/gi, " ")
+ .replace(/\bFreshness\s+is\b[^.?!]*(?:[.?!]|$)/gi, " ")
+ .replace(/\bReusable topic\s+is\b[^.?!]*(?:[.?!]|$)/gi, " ")
+ .replace(
+ /\bAlert action\s*:?\s*(?:alert|watch|ignore)\b[^.?!]*(?:[.?!]|$)/gi,
+ " ",
+ )
+ .replace(/\s{2,}/g, " ")
+ .trim();
+}
diff --git a/apps/api/src/services/monitoring/search/llm.ts b/apps/api/src/services/monitoring/search/llm.ts
index c971b4b0f..25b87211f 100644
--- a/apps/api/src/services/monitoring/search/llm.ts
+++ b/apps/api/src/services/monitoring/search/llm.ts
@@ -1,15 +1,21 @@
import { generateObject } from "ai";
import { google } from "@ai-sdk/google";
import { z } from "zod";
+import { googleProviderOptions } from "./tuning";
+import type { GoalCriteria } from "./criteria";
// Gemini LLM calls for the search monitor (mirrors services/monitoring/judgeChange.ts).
-// Cost: Gemini 3 Flash reasoning tokens dominate output β€” use a low thinking budget and
-// batch the resolver when scaling.
+// Every call runs with thinking suppressed (see tuning.ts) β€” reasoning tokens measured
+// at ~85% of output spend on these structured tasks with no accuracy gain.
const EVENT_MODEL =
process.env.SEARCH_MONITOR_EVENT_MODEL ?? "gemini-3-flash-preview";
const SUMMARY_MODEL =
process.env.SEARCH_MONITOR_SUMMARY_MODEL ?? "gemini-3-flash-preview";
+const SKEPTIC_MODEL =
+ process.env.SEARCH_MONITOR_SKEPTIC_MODEL ?? "gemini-3-flash-preview";
+const ROUTER_MODEL =
+ process.env.SEARCH_MONITOR_ROUTER_MODEL ?? "gemini-3-flash-preview";
export type KnownEvent = { key: string; label: string };
@@ -56,6 +62,7 @@ export async function resolveEvent(params: {
],
}),
temperature: 0,
+ ...googleProviderOptions(EVENT_MODEL),
});
return {
@@ -97,10 +104,200 @@ export async function judgeMaterialDevelopment(params: {
],
}),
temperature: 0,
+ ...googleProviderOptions(EVENT_MODEL),
});
return { material: object.material === true, reason: object.reason ?? "" };
}
+const alertSkepticSchema = z.object({
+ refuted: z.boolean(),
+ failureMode: z.enum([
+ "wrong_subject",
+ "listing_surface",
+ "not_completed",
+ "adjacent_event",
+ "query_echo",
+ "other",
+ "none",
+ ]),
+ reason: z.string(),
+});
+
+export type SkepticVerdict = z.infer<typeof alertSkepticSchema>;
+
+// Adversarial review of an alert candidate β€” runs ONLY on results the judge
+// already marked alert-worthy, so its cost is bounded by the (rare) alert rate.
+// The judge that produced a verdict cannot be the one defending it; this is the
+// independent second opinion. Callers must fail OPEN: a skeptic outage must not
+// silently drop real alerts.
+export async function reviewAlert(params: {
+ goal: string;
+ subject: string;
+ criteria: GoalCriteria | null;
+ result: {
+ title: string;
+ url: string;
+ snippet: string;
+ concept: string;
+ judgeAnswer: string;
+ };
+}): Promise<SkepticVerdict> {
+ const { object } = await generateObject({
+ model: google(SKEPTIC_MODEL),
+ schema: alertSkepticSchema,
+ system:
+ "You are the adversarial reviewer for a web-monitoring alert that is about to be sent to a paying user. Try to REFUTE it. Refute when any of these hold: (1) wrong_subject β€” the story's protagonist is a different entity than the monitored subject; the subject being name-dropped, compared against, or listed among others is NOT coverage of the subject; (2) listing_surface β€” the page is an aggregator, directory, category, calendar, homepage, or feed surface rather than a single story; (3) not_completed β€” the goal asks for a completed event but this is upcoming, rumored, or planned; (4) adjacent_event β€” a related-but-different event type (funding news on a release monitor, a partnership on a lawsuit monitor); (5) query_echo β€” the only evidence is the search query's own wording reflected back in a snippet. UPHOLD the alert (refuted=false, failureMode=none) when the result directly and concretely satisfies the goal β€” do not refute clear matches on technicalities. The subject's OWN official announcement satisfies an event goal (release, launch, filing, pricing change); never refute an official primary source unless the goal explicitly asks for third-party coverage. Return structured JSON only.",
+ prompt: JSON.stringify({
+ monitor: {
+ goal: params.goal,
+ subject: params.subject,
+ ...(params.criteria
+ ? {
+ subjectAliases: params.criteria.subjectAliases,
+ excludedSubjects: params.criteria.excludedSubjects,
+ mustConcern: params.criteria.mustConcern,
+ }
+ : {}),
+ },
+ alertCandidate: {
+ title: params.result.title,
+ url: params.result.url,
+ snippet: params.result.snippet.slice(0, 600),
+ judgeConcept: params.result.concept,
+ judgeAnswer: params.result.judgeAnswer.slice(0, 800),
+ },
+ }),
+ temperature: 0,
+ ...googleProviderOptions(SKEPTIC_MODEL),
+ });
+ return object;
+}
+
+const routerSchema = z.object({
+ decisions: z.array(
+ z.object({
+ id: z.string(),
+ decision: z.enum(["scrape", "skip"]),
+ priority: z.number(),
+ reason: z.string(),
+ }),
+ ),
+});
+
+export type RouteDecision = {
+ id: string;
+ decision: "scrape" | "skip";
+ priority: number;
+ reason: string;
+};
+
+// SERP router: decides which results are worth scrape + judge spend, from the
+// SERP row alone. The single biggest cost lever on a deep run β€” every skipped
+// candidate saves a full extract-tier scrape. Callers must fail OPEN to
+// deterministic top-K routing when this throws.
+export async function routeSearchResults(params: {
+ goal: string;
+ subject: string;
+ searchWindow: string;
+ maxResults: number;
+ candidates: Array<{
+ id: string;
+ query: string;
+ title: string;
+ url: string;
+ snippet: string;
+ }>;
+}): Promise<RouteDecision[]> {
+ const candidates = params.candidates.slice(0, 50);
+ const { object } = await generateObject({
+ model: google(ROUTER_MODEL),
+ schema: routerSchema,
+ system:
+ "You route search results for a monitoring product. Decide which indexed results are worth spending scrape/judge credits on. Use only the SERP title, URL, snippet, query, monitor goal, and search window. Return structured JSON only.",
+ prompt: JSON.stringify({
+ monitor: { goal: params.goal, subject: params.subject },
+ searchWindow: params.searchWindow,
+ maxResults: params.maxResults,
+ instructions: [
+ "Select scrape only when the result could plausibly contain primary evidence or substantive discussion satisfying the monitor goal.",
+ "Skip obvious jobs, directories, tag pages, search pages, marketplaces, profiles, irrelevant foreign-language spam, generic tool lists, or pages where the monitored subject appears only as a skill, integration, or incidental keyword.",
+ "Do not use keyword overlap alone. Explain the user-facing reason.",
+ "Never select more than maxResults.",
+ ],
+ candidates,
+ }),
+ temperature: 0,
+ ...googleProviderOptions(ROUTER_MODEL),
+ });
+ const validIds = new Set(candidates.map(c => c.id));
+ return object.decisions
+ .filter(d => validIds.has(d.id))
+ .map(d => ({
+ id: d.id,
+ decision: d.decision,
+ priority: Math.min(1, Math.max(0, d.priority)),
+ reason: d.reason.slice(0, 500),
+ }));
+}
+
+const snippetVerdictSchema = z.object({
+ verdicts: z.array(
+ z.object({
+ id: z.string(),
+ relevant: z.boolean(),
+ alertAction: z.enum(["alert", "watch", "ignore"]),
+ freshness: z.enum(["fresh", "stale", "unknown"]),
+ sourceQuality: z.enum([
+ "first-party",
+ "authoritative",
+ "unverified",
+ "resale",
+ "unclear",
+ ]),
+ concept: z.string(),
+ rationale: z.string(),
+ }),
+ ),
+});
+
+export type SnippetVerdict = z.infer<
+ typeof snippetVerdictSchema
+>["verdicts"][number];
+
+// Standard-depth judging: one batched call over SERP rows, no page fetches.
+// (The POC's standard tier judges snippets deterministically; a single batched
+// flash call is the backend-appropriate equivalent β€” same verdict contract as
+// the in-scrape judge, ~free, and strictly better than keyword heuristics.)
+export async function judgeSnippets(params: {
+ goal: string;
+ subject: string;
+ searchWindow: string;
+ candidates: Array<{
+ id: string;
+ query: string;
+ title: string;
+ url: string;
+ snippet: string;
+ }>;
+}): Promise<SnippetVerdict[]> {
+ if (params.candidates.length === 0) return [];
+ const { object } = await generateObject({
+ model: google(ROUTER_MODEL),
+ schema: snippetVerdictSchema,
+ system:
+ "You judge search results for a monitoring product using ONLY each result's SERP title, URL, and snippet β€” you cannot see the page. Be conservative: a snippet is thin evidence, so alertAction alert requires the snippet itself to concretely state a completed event satisfying the goal from a credible source. Query wording echoed in a snippet is not evidence. Competitors, look-alikes, listings, stale or unconfirmed results are watch/ignore. concept: a short reusable label naming the real-world event. Return one verdict per candidate id. Structured JSON only.",
+ prompt: JSON.stringify({
+ monitor: { goal: params.goal, subject: params.subject },
+ searchWindow: params.searchWindow,
+ candidates: params.candidates.slice(0, 50),
+ }),
+ temperature: 0,
+ ...googleProviderOptions(ROUTER_MODEL),
+ });
+ const validIds = new Set(params.candidates.map(c => c.id));
+ return object.verdicts.filter(v => validIds.has(v.id));
+}
+
const runSummarySchema = z.object({
label: z.enum(["meaningful", "already_satisfied", "not_meaningful"]),
summary: z.string(),
@@ -125,6 +322,7 @@ export async function summarizeRun(params: {
],
}),
temperature: 0,
+ ...googleProviderOptions(SUMMARY_MODEL),
});
return { label: object.label, summary: object.summary };
diff --git a/apps/api/src/services/monitoring/search/persist.test.ts b/apps/api/src/services/monitoring/search/persist.test.ts
index 0f1d2e855..f89ced6ff 100644
--- a/apps/api/src/services/monitoring/search/persist.test.ts
+++ b/apps/api/src/services/monitoring/search/persist.test.ts
@@ -109,7 +109,12 @@ describe("reconstructKnownState", () => {
it("orders events most-recently-seen first (resolver window holds active stories)", () => {
const page = (url: string, eventKey: string, updated_at: string) => ({
url,
- metadata: { fingerprint: "f", goalVersion: "gv1", eventKey, eventLabel: eventKey },
+ metadata: {
+ fingerprint: "f",
+ goalVersion: "gv1",
+ eventKey,
+ eventLabel: eventKey,
+ },
updated_at,
});
const { knownEvents } = reconstructKnownState(
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
new file mode 100644
index 000000000..b61db2cf0
--- /dev/null
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -0,0 +1,297 @@
+import type { Logger } from "winston";
+import type { SearchVerdict } from "./judge";
+
+// --- Mock the external boundaries; the orchestration logic in run.ts runs for real. ---
+const searchMock = jest.fn();
+const scrapeURLMock = jest.fn();
+const resolveEventMock = jest.fn();
+const summarizeRunMock = jest.fn();
+const materialDevMock = jest.fn();
+const reviewAlertMock = jest.fn();
+const routeMock = jest.fn();
+const snippetsMock = jest.fn();
+const criteriaLlmMock = jest.fn();
+
+jest.mock("uuid", () => ({ v7: () => "00000000-0000-7000-8000-000000000000" }));
+jest.mock("../../../search", () => ({
+ search: (...a: unknown[]) => searchMock(...a),
+}));
+jest.mock("../../../scraper/scrapeURL", () => ({
+ scrapeURL: (...a: unknown[]) => scrapeURLMock(...a),
+}));
+jest.mock("./llm", () => ({
+ resolveEvent: (...a: unknown[]) => resolveEventMock(...a),
+ summarizeRun: (...a: unknown[]) => summarizeRunMock(...a),
+ judgeMaterialDevelopment: (...a: unknown[]) => materialDevMock(...a),
+ reviewAlert: (...a: unknown[]) => reviewAlertMock(...a),
+ routeSearchResults: (...a: unknown[]) => routeMock(...a),
+ judgeSnippets: (...a: unknown[]) => snippetsMock(...a),
+}));
+// LLM stages on: these tests exercise the router / skeptic / verifier wiring.
+jest.mock("./tuning", () => ({
+ hasGeminiKey: () => true,
+ googleProviderOptions: () => ({}),
+}));
+jest.mock("./criteria", () => ({
+ ...jest.requireActual("./criteria"),
+ compileGoalCriteriaWithLlm: (...a: unknown[]) => criteriaLlmMock(...a),
+}));
+
+import { runSearchTarget } from "./run";
+
+const logger = {
+ warn: jest.fn(),
+ info: jest.fn(),
+ error: jest.fn(),
+} as unknown as Logger;
+
+const verdict = (over: Partial<SearchVerdict> = {}): SearchVerdict => ({
+ relevant: true,
+ alertAction: "alert",
+ freshness: "fresh",
+ sourceQuality: "authoritative",
+ concept: "Firecrawl product launch",
+ rationale: "Firecrawl announced a new product today.",
+ ...over,
+});
+
+const llmCriteria = {
+ goalVersion: "v1",
+ generatedBy: "llm" as const,
+ subjectAliases: ["Firecrawl"],
+ mustConcern: ["launch", "launches", "launched", "product"],
+ excludedSubjects: ["Parallel"],
+ ownedHosts: ["firecrawl.dev"],
+ thirdPartyOnly: false,
+};
+
+function runParams(targetOver: Record<string, unknown> = {}) {
+ return {
+ monitor: {
+ id: "mon_1",
+ teamId: "team_1",
+ goal: "Alert me when Firecrawl launches a new product",
+ subject: "Firecrawl",
+ },
+ target: {
+ id: "tgt_1",
+ queries: ["Firecrawl launch"],
+ searchWindow: "24h",
+ alertMode: "first_match" as const,
+ maxResults: 10,
+ ...targetOver,
+ },
+ goalVersion: "v1",
+ knownPages: new Map(),
+ knownEvents: [],
+ zeroDataRetention: false,
+ logger,
+ };
+}
+
+beforeEach(() => {
+ jest.clearAllMocks();
+ criteriaLlmMock.mockResolvedValue(llmCriteria);
+ resolveEventMock.mockResolvedValue({
+ matchedKey: null,
+ isNew: true,
+ label: "Firecrawl product launch",
+ reason: "",
+ });
+ summarizeRunMock.mockResolvedValue({ label: "meaningful", summary: "ok" });
+ reviewAlertMock.mockResolvedValue({
+ refuted: false,
+ failureMode: "none",
+ reason: "",
+ });
+});
+
+const serpRow = (n: number) => ({
+ url: `https://news${n}.example.com/story`,
+ title: `Firecrawl launches product ${n}`,
+ description: `Firecrawl announced product ${n} today.`,
+});
+
+describe("router gating (deep)", () => {
+ it("scrapes only candidates the router selects", async () => {
+ searchMock.mockResolvedValue([serpRow(1), serpRow(2), serpRow(3)]);
+ routeMock.mockResolvedValue([
+ { id: "result_2", decision: "scrape", priority: 1, reason: "promising" },
+ ]);
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict(),
+ markdown: "Firecrawl announced product 2 today in prose.",
+ metadata: {},
+ },
+ });
+
+ const result = await runSearchTarget(runParams());
+ expect(scrapeURLMock).toHaveBeenCalledTimes(1);
+ expect(scrapeURLMock.mock.calls[0][1]).toBe(serpRow(2).url);
+ expect(result.matches).toBe(1);
+ });
+
+ it("fails open to top-K when the router throws", async () => {
+ searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
+ routeMock.mockRejectedValue(new Error("router down"));
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict({ alertAction: "watch" }),
+ markdown: "",
+ metadata: {},
+ },
+ });
+
+ const result = await runSearchTarget(runParams());
+ expect(scrapeURLMock).toHaveBeenCalledTimes(2);
+ expect(result.matches).toBe(0);
+ });
+});
+
+describe("standard depth (snippet judging, no scrapes)", () => {
+ it("judges from snippets without any page fetch", async () => {
+ searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
+ snippetsMock.mockResolvedValue([
+ { id: "result_1", ...verdict() },
+ { id: "result_2", ...verdict({ alertAction: "watch" }) },
+ ]);
+
+ const result = await runSearchTarget(runParams({ depth: "standard" }));
+ expect(scrapeURLMock).not.toHaveBeenCalled();
+ expect(routeMock).not.toHaveBeenCalled();
+ expect(result.pagesChecked).toBe(2);
+ expect(result.matches).toBe(1);
+ expect(result.sources.map(s => s.status).sort()).toEqual([
+ "alert",
+ "watching",
+ ]);
+ });
+
+ it("skips results the snippet judge failed to cover", async () => {
+ searchMock.mockResolvedValue([serpRow(1)]);
+ snippetsMock.mockRejectedValue(new Error("batch failed"));
+
+ const result = await runSearchTarget(runParams({ depth: "standard" }));
+ expect(result.skipped).toBe(1);
+ expect(result.matches).toBe(0);
+ });
+});
+
+describe("alert boundary", () => {
+ beforeEach(() => {
+ searchMock.mockResolvedValue([serpRow(1)]);
+ routeMock.mockResolvedValue([
+ { id: "result_1", decision: "scrape", priority: 1, reason: "" },
+ ]);
+ });
+
+ it("verifier downgrades a competitor-dominated story to watching", async () => {
+ searchMock.mockResolvedValue([
+ {
+ url: "https://news.example.com/parallel",
+ title: "Parallel launches Monitor API",
+ description: "Parallel launched a product; Firecrawl is a competitor.",
+ },
+ ]);
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict({ concept: "Parallel Monitor API launch" }),
+ markdown:
+ "Parallel launched its Monitor API today. Firecrawl is named as a competitor.",
+ metadata: {},
+ },
+ });
+
+ const result = await runSearchTarget(runParams());
+ expect(result.matches).toBe(0);
+ expect(result.sources[0].status).toBe("watching");
+ expect(reviewAlertMock).not.toHaveBeenCalled(); // refuted before the skeptic spends tokens
+ });
+
+ it("skeptic refutation downgrades to watching", async () => {
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict(),
+ markdown: "Firecrawl announced a new product today in prose.",
+ metadata: {},
+ },
+ });
+ reviewAlertMock.mockResolvedValue({
+ refuted: true,
+ failureMode: "adjacent_event",
+ reason: "funding, not a launch",
+ });
+
+ const result = await runSearchTarget(runParams());
+ expect(result.matches).toBe(0);
+ expect(result.sources[0].status).toBe("watching");
+ });
+
+ it("skeptic outage fails open (alert proceeds)", async () => {
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict(),
+ markdown: "Firecrawl announced a new product today in prose.",
+ metadata: {},
+ },
+ });
+ reviewAlertMock.mockRejectedValue(new Error("skeptic down"));
+
+ const result = await runSearchTarget(runParams());
+ expect(result.matches).toBe(1);
+ });
+
+ it("resolver outage fails open to a new event (alert proceeds)", async () => {
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict(),
+ markdown: "Firecrawl announced a new product today in prose.",
+ metadata: {},
+ },
+ });
+ resolveEventMock.mockRejectedValue(new Error("resolver down"));
+
+ const result = await runSearchTarget(runParams());
+ expect(result.matches).toBe(1);
+ expect(result.sources[0].status).toBe("alert");
+ });
+
+ it("criteria LLM failure keeps the deterministic compile (run completes)", async () => {
+ criteriaLlmMock.mockRejectedValue(new Error("criteria down"));
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict(),
+ markdown: "Firecrawl announced a new product today in prose.",
+ metadata: {},
+ },
+ });
+
+ const result = await runSearchTarget(runParams());
+ expect(result.matches).toBe(1);
+ });
+
+ it("self-contradicting verdict is corrected before alerting", async () => {
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict({
+ rationale: "The page does not mention Firecrawl anywhere.",
+ }),
+ markdown: "Unrelated prose about something else entirely.",
+ metadata: {},
+ },
+ });
+
+ const result = await runSearchTarget(runParams());
+ expect(result.matches).toBe(0);
+ expect(result.sources[0].status).toBe("ignored");
+ });
+});
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.ts
index 3aef68fd7..ff5f7a6d3 100644
--- a/apps/api/src/services/monitoring/search/run.test.ts
+++ b/apps/api/src/services/monitoring/search/run.test.ts
@@ -22,6 +22,13 @@ jest.mock("./llm", () => ({
summarizeRun: (...a: unknown[]) => summarizeRunMock(...a),
judgeMaterialDevelopment: (...a: unknown[]) => materialDevMock(...a),
}));
+// These tests exercise the core pipeline with the optional LLM stages (router,
+// skeptic, criteria enrichment, snippet judge) gated off β€” the no-Gemini-key
+// configuration. run-defenses.test.ts covers the stages-on wiring.
+jest.mock("./tuning", () => ({
+ hasGeminiKey: () => false,
+ googleProviderOptions: () => ({}),
+}));
import { runSearchTarget, type KnownPage } from "./run";
@@ -564,13 +571,27 @@ describe("domain scoping", () => {
it("excluded hosts returned by the provider are dropped before judging", async () => {
setSearchResults([
- { url: "https://www.pinterest.com/pin/1", title: "pin", description: "x" },
- { url: "https://boards.pinterest.com/pin/2", title: "pin", description: "x" },
- { url: "https://news.com/openai", title: "OpenAI files S-1", description: "x" },
+ {
+ url: "https://www.pinterest.com/pin/1",
+ title: "pin",
+ description: "x",
+ },
+ {
+ url: "https://boards.pinterest.com/pin/2",
+ title: "pin",
+ description: "x",
+ },
+ {
+ url: "https://news.com/openai",
+ title: "OpenAI files S-1",
+ description: "x",
+ },
]);
setVerdictsByUrl({ "https://news.com/openai": verdict() });
const out = await run({
- target: { excludeDomains: ["pinterest.com"] } as Partial<typeof baseTarget>,
+ target: { excludeDomains: ["pinterest.com"] } as Partial<
+ typeof baseTarget
+ >,
});
expect(out.resultCount).toBe(1);
expect(out.sources.map(s => s.url)).toEqual(["https://news.com/openai"]);
@@ -583,7 +604,9 @@ describe("domain scoping", () => {
]);
setVerdictsByUrl({ "https://notpinterest.com/a": verdict() });
const out = await run({
- target: { excludeDomains: ["pinterest.com"] } as Partial<typeof baseTarget>,
+ target: { excludeDomains: ["pinterest.com"] } as Partial<
+ typeof baseTarget
+ >,
});
expect(out.resultCount).toBe(1);
});
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index 52e65bc38..659cd7b32 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -8,19 +8,39 @@ import { CostTracking } from "../../../lib/cost-tracking";
import { hashMonitorUrl } from "../store";
import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";
import {
+ applyVerdictDefenses,
buildJudgePrompt,
freshnessFromDate,
parseVerdict,
+ stripJudgeMetaClaims,
verdictJsonSchema,
verdictToDecision,
windowToMs,
+ type SearchVerdict,
} from "./judge";
import {
judgeMaterialDevelopment,
+ judgeSnippets,
resolveEvent,
+ reviewAlert,
+ routeSearchResults,
summarizeRun,
type KnownEvent,
+ type RouteDecision,
+ type SkepticVerdict,
+ type SnippetVerdict,
} from "./llm";
+import {
+ compileGoalCriteria,
+ compileGoalCriteriaWithLlm,
+ type GoalCriteria,
+} from "./criteria";
+import {
+ verifyAlertCandidate,
+ type VerificationResult,
+ type VerifyEvidence,
+} from "./verify";
+import { hasGeminiKey } from "./tuning";
// Unified schedule/window step β†’ Firecrawl tbs window.
function windowToTbs(window: string): string {
@@ -32,7 +52,7 @@ function windowToTbs(window: string): string {
// Exclude wins over include (Exa/Parallel semantics). The -site: operators in
// the scoped query are advisory β€” not every search provider honors them β€” so
// the blocklist is also enforced here on the returned results.
-export function isExcludedDomain(
+function isExcludedDomain(
url: string,
excludeDomains: string[] | undefined,
): boolean {
@@ -44,7 +64,10 @@ export function isExcludedDomain(
return false;
}
return excludeDomains.some(domain => {
- const normalized = domain.trim().toLowerCase().replace(/^www\./, "");
+ const normalized = domain
+ .trim()
+ .toLowerCase()
+ .replace(/^www\./, "");
return (
normalized.length > 0 &&
(host === normalized || host.endsWith(`.${normalized}`))
@@ -63,6 +86,10 @@ type SearchTargetInput = {
// this window, even if the SERP snippet is unchanged β€” catches content updates SERP text misses.
recheckAfter?: string;
maxResults: number;
+ // "deep" (default) scrapes + judges routed pages; "standard" judges from SERP
+ // snippets in one batched call β€” no page fetches, the cheap tier. Standard
+ // requires a Gemini key; without one it falls back to deep behavior.
+ depth?: "standard" | "deep";
};
export type KnownPage = {
@@ -132,8 +159,12 @@ export async function runSearchTarget(params: {
// 1. Search each query; flatten + dedup by canonical URL within this run.
const seenThisRun = new Set<string>();
- const candidates: Array<{ url: string; title: string; description: string }> =
- [];
+ const candidates: Array<{
+ url: string;
+ title: string;
+ description: string;
+ query: string;
+ }> = [];
for (const query of target.queries) {
// Scope to domains the same way the v2 search API does (site:/-site: operators).
const { query: scopedQuery } = buildSearchQuery(query, undefined, {
@@ -156,13 +187,118 @@ export async function runSearchTarget(params: {
url: r.url,
title: r.title,
description: r.description,
+ query,
});
}
}
resultCount = candidates.length;
+ const llmStagesAvailable = hasGeminiKey();
+ const depth: "standard" | "deep" =
+ target.depth === "standard" && llmStagesAvailable ? "standard" : "deep";
+
+ // Criteria artifact for the verifier + skeptic. Deterministic compile is
+ // free and always available; the LLM enrichment (aliases, competitors, owned
+ // hosts) fails safe back to deterministic. Compiled per run β€” cheap on a
+ // thinking-suppressed flash-class model; persistence keyed to goalVersion is
+ // a follow-up optimization, not a correctness need.
+ let criteria: GoalCriteria = compileGoalCriteria({
+ goal,
+ subject,
+ goalVersion,
+ });
+ if (llmStagesAvailable) {
+ try {
+ criteria = await compileGoalCriteriaWithLlm({
+ goal,
+ subject,
+ queries: target.queries,
+ goalVersion,
+ });
+ } catch (error) {
+ logger.warn(
+ "search monitor criteria compile failed, keeping deterministic",
+ {
+ error,
+ },
+ );
+ }
+ }
+
+ // Route which candidates are worth scrape/judge spend (deep mode). The LLM
+ // router fails open to deterministic top-K β€” the pre-port behavior.
+ let selected = candidates.slice(0, target.maxResults);
+ if (depth === "deep" && llmStagesAvailable && candidates.length > 0) {
+ try {
+ const decisions = await routeSearchResults({
+ goal,
+ subject,
+ searchWindow: target.searchWindow,
+ maxResults: target.maxResults,
+ candidates: candidates.map((c, i) => ({
+ id: `result_${i + 1}`,
+ query: c.query,
+ title: c.title,
+ url: c.url,
+ snippet: c.description,
+ })),
+ });
+ const byId = new Map<string, RouteDecision>(
+ decisions.map(d => [d.id, d]),
+ );
+ const routed = candidates
+ .map((c, i) => ({ c, routing: byId.get(`result_${i + 1}`) }))
+ .filter(r => r.routing?.decision === "scrape")
+ .sort((a, b) => (b.routing?.priority ?? 0) - (a.routing?.priority ?? 0))
+ .map(r => r.c);
+ selected = routed.slice(0, target.maxResults);
+ } catch (error) {
+ logger.warn("search monitor router failed open to top-K", { error });
+ }
+ }
+
+ // Standard depth: one batched snippet-judge call for the whole selection β€”
+ // no page fetches. Verdicts keyed by canonical URL for the loop below.
+ const snippetVerdicts = new Map<string, SearchVerdict>();
+ if (depth === "standard" && selected.length > 0) {
+ try {
+ const verdicts = await judgeSnippets({
+ goal,
+ subject,
+ searchWindow: target.searchWindow,
+ candidates: selected.map((c, i) => ({
+ id: `result_${i + 1}`,
+ query: c.query,
+ title: c.title,
+ url: c.url,
+ snippet: c.description,
+ })),
+ });
+ const byId = new Map<string, SnippetVerdict>(
+ verdicts.map(v => [v.id, v]),
+ );
+ selected.forEach((c, i) => {
+ const v = byId.get(`result_${i + 1}`);
+ if (v) {
+ snippetVerdicts.set(canonicalizeUrl(c.url), {
+ relevant: v.relevant,
+ alertAction: v.alertAction,
+ freshness: v.freshness,
+ sourceQuality: v.sourceQuality,
+ concept: v.concept,
+ rationale: v.rationale,
+ });
+ }
+ });
+ } catch (error) {
+ logger.warn("search monitor snippet judge failed; results skipped", {
+ error,
+ });
+ }
+ }
+
// 2. Per candidate: dedup β†’ (judge if new/changed) β†’ decide β†’ event-resolve.
- for (const c of candidates.slice(0, target.maxResults)) {
+ for (const c of selected) {
const canonical = canonicalizeUrl(c.url);
const fingerprint = stableSerpFingerprint({
url: c.url,
@@ -212,52 +348,77 @@ export async function runSearchTarget(params: {
continue;
}
- // Judge in-scrape via the json format; verdict returns on document.json.
- let res;
- try {
- res = await scrapeURL(
- "search-monitor;" + uuidv7(),
- c.url,
- scrapeOptions.parse({
- formats: [
- {
- type: "json",
- schema: verdictJsonSchema,
- prompt: buildJudgePrompt(goal, subject, target.searchWindow),
- },
- ],
- timeout: 20000,
- }),
- { teamId, zeroDataRetention: params.zeroDataRetention },
- new CostTracking(),
- );
- } catch (error) {
- logger.warn("search monitor scrape threw", { url: c.url, error });
- skipped += 1;
- sources.push({ url: c.url, title: c.title, status: "skipped" });
- continue;
- }
+ // Judge: deep scrapes the page (verdict returns on document.json, markdown
+ // kept for the verifier); standard reads the batched snippet verdict.
+ let verdict: SearchVerdict | null = null;
+ let pageText = "";
+ let pageMetadata: Record<string, unknown> | null = null;
+ let realDate: string | null = null;
+ if (depth === "standard") {
+ verdict = snippetVerdicts.get(canonical) ?? null;
+ if (!verdict) {
+ skipped += 1;
+ sources.push({ url: c.url, title: c.title, status: "skipped" });
+ continue;
+ }
+ pagesChecked += 1;
+ } else {
+ let res;
+ try {
+ res = await scrapeURL(
+ "search-monitor;" + uuidv7(),
+ c.url,
+ scrapeOptions.parse({
+ formats: [
+ { type: "markdown" },
+ {
+ type: "json",
+ schema: verdictJsonSchema,
+ prompt: buildJudgePrompt(goal, subject, target.searchWindow),
+ },
+ ],
+ timeout: 20000,
+ }),
+ { teamId, zeroDataRetention: params.zeroDataRetention },
+ new CostTracking(),
+ );
+ } catch (error) {
+ logger.warn("search monitor scrape threw", { url: c.url, error });
+ skipped += 1;
+ sources.push({ url: c.url, title: c.title, status: "skipped" });
+ continue;
+ }
- if (!res.success) {
- logger.warn("search monitor scrape failed", { url: c.url });
- skipped += 1;
- sources.push({ url: c.url, title: c.title, status: "skipped" });
- continue;
- }
- pagesChecked += 1;
+ if (!res.success) {
+ logger.warn("search monitor scrape failed", { url: c.url });
+ skipped += 1;
+ sources.push({ url: c.url, title: c.title, status: "skipped" });
+ continue;
+ }
+ pagesChecked += 1;
- const verdict = parseVerdict(res.document.json);
- if (!verdict) {
- skipped += 1;
- sources.push({ url: c.url, title: c.title, status: "skipped" });
- continue;
+ verdict = parseVerdict(res.document.json);
+ if (!verdict) {
+ skipped += 1;
+ sources.push({ url: c.url, title: c.title, status: "skipped" });
+ continue;
+ }
+ pageText = res.document.markdown ?? "";
+ pageMetadata = (res.document.metadata ?? null) as Record<
+ string,
+ unknown
+ > | null;
+ realDate =
+ res.document.metadata?.publishedTime ??
+ res.document.metadata?.modifiedTime ??
+ null;
}
+ // Mechanical defense: a verdict whose rationale negates its own booleans is
+ // corrected before anything downstream consumes it.
+ verdict = applyVerdictDefenses(verdict);
+
// Prefer a real publish date over the LLM's freshness guess (freshness is an alert veto).
- const realDate =
- res.document.metadata?.publishedTime ??
- res.document.metadata?.modifiedTime ??
- null;
const dateFreshness = freshnessFromDate(
realDate,
target.searchWindow,
@@ -266,8 +427,67 @@ export async function runSearchTarget(params: {
const effectiveVerdict = dateFreshness
? { ...verdict, freshness: dateFreshness }
: verdict;
+ // Downstream stages (verifier, skeptic, resolver) must see page facts, not
+ // the judge grading itself.
+ const strippedRationale = stripJudgeMetaClaims(effectiveVerdict.rationale);
+
+ let decision = verdictToDecision(effectiveVerdict);
+
+ // Alert boundary, stage 1 β€” deterministic verification against the compiled
+ // criteria. Downgrade-only (notify β†’ watch), fail-open on unknown shapes.
+ let verification: VerificationResult | null = null;
+ if (decision === "notify") {
+ const evidence: VerifyEvidence = {
+ url: c.url,
+ titleText: c.title,
+ claimText: strippedRationale,
+ pageText,
+ metadata: pageMetadata,
+ };
+ const result = verifyAlertCandidate({
+ criteria,
+ concept: effectiveVerdict.concept,
+ evidence,
+ });
+ verification = result;
+ if (!result.pass) {
+ decision = "watch";
+ logger.info("search monitor alert downgraded by verifier", {
+ url: c.url,
+ failures: result.failures,
+ });
+ }
+ }
- const decision = verdictToDecision(effectiveVerdict);
+ // Alert boundary, stage 2 β€” adversarial skeptic. Runs only on surviving
+ // notify candidates (cost bounded by the alert rate); fails OPEN so a
+ // skeptic outage can't silently drop real alerts.
+ if (decision === "notify" && llmStagesAvailable) {
+ try {
+ const skeptic: SkepticVerdict = await reviewAlert({
+ goal,
+ subject,
+ criteria,
+ result: {
+ title: c.title,
+ url: c.url,
+ snippet: c.description,
+ concept: effectiveVerdict.concept,
+ judgeAnswer: strippedRationale,
+ },
+ });
+ if (skeptic.refuted) {
+ decision = "watch";
+ logger.info("search monitor alert refuted by skeptic", {
+ url: c.url,
+ failureMode: skeptic.failureMode,
+ reason: skeptic.reason,
+ });
+ }
+ } catch (error) {
+ logger.warn("search monitor skeptic failed open", { error });
+ }
+ }
const baseMeta = {
fingerprint,
goalVersion,
@@ -278,6 +498,7 @@ export async function runSearchTarget(params: {
sourceQuality: effectiveVerdict.sourceQuality,
concept: effectiveVerdict.concept,
rationale: effectiveVerdict.rationale,
+ ...(verification && !verification.pass ? { verification } : {}),
};
if (decision === "ignore") {
@@ -312,17 +533,31 @@ export async function runSearchTarget(params: {
}
// notify β†’ resolve the real-world event against the known events (small list, handed straight
- // to the LLM). Matches reuse the existing stable key; new events mint one.
- const resolution = await resolveEvent({
- goal,
- subject,
- result: {
- title: c.title,
- url: c.url,
- evidence: effectiveVerdict.rationale,
- },
- candidates: events,
- });
+ // to the LLM). Matches reuse the existing stable key; new events mint one. Resolver failure
+ // fails open to a new event: a possible duplicate alert beats a silently dropped one.
+ let resolution;
+ try {
+ resolution = await resolveEvent({
+ goal,
+ subject,
+ result: {
+ title: c.title,
+ url: c.url,
+ evidence: strippedRationale,
+ },
+ candidates: events,
+ });
+ } catch (error) {
+ logger.warn("search monitor event resolver failed open to new event", {
+ error,
+ });
+ resolution = {
+ matchedKey: null,
+ isNew: true,
+ label: effectiveVerdict.concept,
+ reason: "resolver_failed",
+ };
+ }
const matched =
resolution.matchedKey &&
events.find(e => e.key === resolution.matchedKey);
@@ -340,13 +575,20 @@ export async function runSearchTarget(params: {
if (target.alertMode === "first_match") {
alreadySatisfied = true;
} else if (target.alertMode === "material_dev") {
- const dev = await judgeMaterialDevelopment({
- goal,
- subject,
- eventLabel,
- result: { title: c.title, evidence: effectiveVerdict.rationale },
- });
- alreadySatisfied = !dev.material;
+ // Fails CLOSED: an unjudgeable "material development" stays suppressed β€”
+ // re-alerting an already-alerted event on an LLM outage is the worse error.
+ try {
+ const dev = await judgeMaterialDevelopment({
+ goal,
+ subject,
+ eventLabel,
+ result: { title: c.title, evidence: strippedRationale },
+ });
+ alreadySatisfied = !dev.material;
+ } catch (error) {
+ logger.warn("search monitor material judge failed closed", { error });
+ alreadySatisfied = true;
+ }
}
}
const eventMeta = { ...baseMeta, eventKey, eventLabel };
@@ -397,18 +639,26 @@ export async function runSearchTarget(params: {
let summary = "";
if (matches > 0) {
- const out = await summarizeRun({
- goal,
- subject,
- evidence: sources
- .filter(s => s.status === "alert")
- .map(s => ({
- title: s.title,
- url: s.url,
- rationale: s.rationale ?? "",
- })),
- });
- summary = out.summary;
+ // Summarizer failure must not fail the run β€” fall back to a counting summary.
+ try {
+ const out = await summarizeRun({
+ goal,
+ subject,
+ evidence: sources
+ .filter(s => s.status === "alert")
+ .map(s => ({
+ title: s.title,
+ url: s.url,
+ rationale: s.rationale ?? "",
+ })),
+ });
+ summary = out.summary;
+ } catch (error) {
+ logger.warn("search monitor summarizer failed; using fallback summary", {
+ error,
+ });
+ summary = `${matches} new result${matches === 1 ? "" : "s"} matched the monitor goal.`;
+ }
}
return {
diff --git a/apps/api/src/services/monitoring/search/tuning.ts b/apps/api/src/services/monitoring/search/tuning.ts
new file mode 100644
index 000000000..9acaabfec
--- /dev/null
+++ b/apps/api/src/services/monitoring/search/tuning.ts
@@ -0,0 +1,51 @@
+// Thinking suppression for search-monitor orchestration calls. Reasoning tokens
+// measured at ~85% of LLM output spend on these structured tasks, so every call
+// defaults to the lowest thinking the model supports. Gemini 3 models take
+// thinkingLevel ("minimal" is the floor β€” thinking cannot be fully disabled);
+// Gemini 2.5-era models (flash-latest, flash-lite) take thinkingBudget, where 0
+// disables thinking entirely.
+//
+// SEARCH_MONITOR_THINKING: "off" (default) | "minimal" | "low" | "medium" |
+// "high" | "default" ("default" restores the provider's full-thinking behavior).
+const GEMINI_3_LEVELS = ["minimal", "low", "medium", "high"] as const;
+
+type GoogleThinkingOptions = {
+ providerOptions?: {
+ google: {
+ thinkingConfig:
+ | { thinkingLevel: (typeof GEMINI_3_LEVELS)[number] }
+ | { thinkingBudget: number };
+ };
+ };
+};
+
+export function googleProviderOptions(model: string): GoogleThinkingOptions {
+ const setting = String(
+ process.env.SEARCH_MONITOR_THINKING ?? "off",
+ ).toLowerCase();
+ if (setting === "default") {
+ return {};
+ }
+ if (/^gemini-3/.test(model)) {
+ const thinkingLevel = (GEMINI_3_LEVELS as readonly string[]).includes(
+ setting,
+ )
+ ? (setting as (typeof GEMINI_3_LEVELS)[number])
+ : "minimal";
+ return {
+ providerOptions: { google: { thinkingConfig: { thinkingLevel } } },
+ };
+ }
+ return {
+ providerOptions: { google: { thinkingConfig: { thinkingBudget: 0 } } },
+ };
+}
+
+// The LLM stages (router, skeptic, criteria enrichment) are optional: without a
+// Gemini key they are skipped and deterministic fallbacks run instead, exactly
+// like the POC's *FromEnv factories returning null.
+export function hasGeminiKey(): boolean {
+ return Boolean(
+ process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? process.env.GEMINI_API_KEY,
+ );
+}
diff --git a/apps/api/src/services/monitoring/search/verify.test.ts b/apps/api/src/services/monitoring/search/verify.test.ts
new file mode 100644
index 000000000..bbf0f65ea
--- /dev/null
+++ b/apps/api/src/services/monitoring/search/verify.test.ts
@@ -0,0 +1,207 @@
+import {
+ compileGoalCriteria,
+ containsAlias,
+ mergeCompiledCriteria,
+ tokenizeContent,
+ type GoalCriteria,
+} from "./criteria";
+import { classifyPageShape, verifyAlertCandidate } from "./verify";
+
+const baseCriteria = (over: Partial<GoalCriteria> = {}): GoalCriteria => ({
+ goalVersion: "v1",
+ generatedBy: "llm",
+ subjectAliases: ["Firecrawl"],
+ mustConcern: ["launch", "launches", "launched", "product"],
+ excludedSubjects: ["Parallel"],
+ ownedHosts: ["firecrawl.dev"],
+ thirdPartyOnly: false,
+ ...over,
+});
+
+const evidence = (
+ over: Partial<Parameters<typeof verifyAlertCandidate>[0]["evidence"]> = {},
+) => ({
+ url: "https://news.example.com/story",
+ titleText: "Firecrawl launches a new product",
+ claimText: "Firecrawl announced a new product today.",
+ pageText: "Firecrawl announced a new product today. Details follow in prose.",
+ metadata: null,
+ ...over,
+});
+
+describe("criteria", () => {
+ it("tokenizes content words and drops instruction words", () => {
+ expect(
+ tokenizeContent("Alert me when Anthropic releases a new Claude model"),
+ ).toEqual(["anthropic", "releases", "claude", "model"]);
+ });
+
+ it("containsAlias matches whole words only", () => {
+ expect(containsAlias("Firecrawl launches today", "Firecrawl")).toBe(true);
+ expect(containsAlias("Firecrawler launches today", "Firecrawl")).toBe(
+ false,
+ );
+ });
+
+ it("deterministic compile derives subject aliases + mustConcern from the goal", () => {
+ const criteria = compileGoalCriteria({
+ goal: "Alert me when Firecrawl launches a new product",
+ subject: "Firecrawl",
+ goalVersion: "v1",
+ });
+ expect(criteria.generatedBy).toBe("deterministic");
+ expect(criteria.subjectAliases).toEqual(["Firecrawl"]);
+ expect(criteria.mustConcern).toContain("launches");
+ expect(criteria.mustConcern).not.toContain("firecrawl");
+ expect(criteria.thirdPartyOnly).toBe(false);
+ expect(criteria.excludedSubjects).toEqual([]);
+ });
+
+ it("merge drops an excluded subject that is also an alias (alias wins)", () => {
+ const deterministic = compileGoalCriteria({
+ goal: "Alert me when Firecrawl launches a new product",
+ subject: "Firecrawl",
+ goalVersion: "v1",
+ });
+ const merged = mergeCompiledCriteria(deterministic, {
+ subjectAliases: ["Firecrawl", "FIRE-1"],
+ mustConcern: ["launch"],
+ excludedSubjects: ["firecrawl", "Parallel"],
+ ownedHosts: ["https://www.firecrawl.dev/blog"],
+ thirdPartyOnly: false,
+ });
+ expect(merged.generatedBy).toBe("llm");
+ expect(merged.excludedSubjects).toEqual(["Parallel"]);
+ expect(merged.ownedHosts).toEqual(["firecrawl.dev"]);
+ expect(merged.subjectAliases).toContain("FIRE-1");
+ });
+});
+
+describe("verifyAlertCandidate", () => {
+ it("passes a clean alert", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "Firecrawl product launch",
+ evidence: evidence(),
+ });
+ expect(result.pass).toBe(true);
+ expect(result.failures).toEqual([]);
+ });
+
+ it("fails subject containment when the subject never appears in page-backed evidence", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "Firecrawl product launch",
+ evidence: evidence({
+ titleText: "A company shipped something",
+ claimText: "Some launch happened.",
+ }),
+ });
+ expect(result.failures.map(f => f.check)).toContain("subject_missing");
+ });
+
+ it("skips subject containment for snippet-only results (no page content)", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "Firecrawl product launch",
+ evidence: evidence({
+ titleText: "A company shipped something",
+ claimText: "Some launch happened.",
+ pageText: "",
+ }),
+ });
+ expect(result.failures.map(f => f.check)).not.toContain("subject_missing");
+ });
+
+ it("fails excluded-subject dominance when a competitor owns the story", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "Parallel launches Monitor API",
+ evidence: evidence({
+ titleText: "Parallel launches Monitor API",
+ claimText:
+ "Parallel launched its Monitor API; Firecrawl is named as a competitor.",
+ }),
+ });
+ expect(result.failures.map(f => f.check)).toContain("excluded_subject");
+ });
+
+ it("fails concept relevance when the concept shares no goal terms (llm criteria only)", () => {
+ const offGoal = verifyAlertCandidate({
+ criteria: baseCriteria(),
+ concept: "quarterly earnings beat",
+ evidence: evidence(),
+ });
+ expect(offGoal.failures.map(f => f.check)).toContain("concept_off_goal");
+
+ const deterministic = verifyAlertCandidate({
+ criteria: baseCriteria({ generatedBy: "deterministic" }),
+ concept: "quarterly earnings beat",
+ evidence: evidence(),
+ });
+ expect(deterministic.failures.map(f => f.check)).not.toContain(
+ "concept_off_goal",
+ );
+ });
+
+ it("fails owned-surface only when the goal wants third-party coverage", () => {
+ const owned = verifyAlertCandidate({
+ criteria: baseCriteria({ thirdPartyOnly: true }),
+ concept: "Firecrawl product launch",
+ evidence: evidence({ url: "https://www.firecrawl.dev/blog/launch" }),
+ });
+ expect(owned.failures.map(f => f.check)).toContain("owned_surface");
+
+ const eventGoal = verifyAlertCandidate({
+ criteria: baseCriteria({ thirdPartyOnly: false }),
+ concept: "Firecrawl product launch",
+ evidence: evidence({ url: "https://www.firecrawl.dev/blog/launch" }),
+ });
+ expect(eventGoal.failures.map(f => f.check)).not.toContain("owned_surface");
+ });
+
+ it("empty criteria disables every check (fail-open for unknown goal shapes)", () => {
+ const result = verifyAlertCandidate({
+ criteria: baseCriteria({
+ subjectAliases: [],
+ mustConcern: [],
+ excludedSubjects: [],
+ ownedHosts: [],
+ }),
+ concept: "anything at all",
+ evidence: evidence({ titleText: "unrelated", claimText: "unrelated" }),
+ });
+ expect(result.pass).toBe(true);
+ });
+});
+
+describe("classifyPageShape", () => {
+ it("declared article metadata wins", () => {
+ expect(
+ classifyPageShape(
+ evidence({ metadata: { ogType: "article" }, pageText: "" }),
+ ).kind,
+ ).toBe("article");
+ });
+
+ it("link walls without a publish date classify as listing", () => {
+ const links = Array.from(
+ { length: 40 },
+ (_, i) => `[item ${i}](https://example.com/${i})`,
+ ).join(" ");
+ expect(classifyPageShape(evidence({ pageText: links })).kind).toBe(
+ "listing",
+ );
+ });
+
+ it("prose with few links classifies as article", () => {
+ const prose = `${"word ".repeat(500)} [one](https://example.com/a) [two](https://example.com/b)`;
+ expect(classifyPageShape(evidence({ pageText: prose })).kind).toBe(
+ "article",
+ );
+ });
+
+ it("snippet-only evidence is unknown (never a veto)", () => {
+ expect(classifyPageShape(evidence({ pageText: "" })).kind).toBe("unknown");
+ });
+});
diff --git a/apps/api/src/services/monitoring/search/verify.ts b/apps/api/src/services/monitoring/search/verify.ts
new file mode 100644
index 000000000..85d890a05
--- /dev/null
+++ b/apps/api/src/services/monitoring/search/verify.ts
@@ -0,0 +1,282 @@
+import { containsAlias, tokenizeContent, type GoalCriteria } from "./criteria";
+
+// Deterministic verification of a judge "alert" verdict. Every check is a
+// universal mechanic (string containment, set membership, token overlap, page
+// shape measurement) β€” goal-specific knowledge comes exclusively from the
+// compiled criteria, and a check whose criteria field is empty is a no-op, so
+// unknown goal shapes degrade to trusting the judge rather than breaking.
+//
+// Verification gates (alert β†’ watch); it never escalates and never ignores.
+// Ported from the POC's alert-verifier.js.
+
+const LISTING_PAGE_TYPES = new Set([
+ "collectionpage",
+ "itemlist",
+ "searchresultspage",
+ "profilepage",
+]);
+
+const ARTICLE_PAGE_TYPES = new Set([
+ "article",
+ "newsarticle",
+ "blogposting",
+ "report",
+ "pressrelease",
+ "techarticle",
+ "scholarlyarticle",
+]);
+
+export type VerifyEvidence = {
+ url: string;
+ titleText: string;
+ // The judge's claim about the page (rationale), already meta-claim-stripped.
+ claimText: string;
+ // Scraped page markdown when available; empty for snippet-only results.
+ pageText: string;
+ metadata?: Record<string, unknown> | null;
+};
+
+type VerificationFailure = { check: string; detail: string };
+
+export type VerificationResult = {
+ pass: boolean;
+ failures: VerificationFailure[];
+ criteriaSource: GoalCriteria["generatedBy"];
+};
+
+export function verifyAlertCandidate(params: {
+ criteria: GoalCriteria;
+ evidence: VerifyEvidence;
+ concept: string;
+}): VerificationResult {
+ const { criteria, evidence } = params;
+ const failures: VerificationFailure[] = [];
+
+ const pageBackedText = [evidence.titleText, evidence.claimText]
+ .filter(Boolean)
+ .join(" ");
+ const conceptText = String(params.concept ?? "");
+ const hasPageContent = Boolean(String(evidence.pageText ?? "").trim());
+
+ // 1. Subject containment β€” the monitored entity must actually appear in the
+ // visible evidence, not just in the query we sent. Only enforceable when
+ // we hold the page content the judge read: on snippet-only results the
+ // judge saw more than we did, so absence here proves nothing.
+ if (
+ criteria.subjectAliases.length > 0 &&
+ hasPageContent &&
+ pageBackedText.trim()
+ ) {
+ // Whole-alias containment, or any single alias token: an inferred subject
+ // can be a multi-word phrase ("Anthropic models") whose exact phrasing
+ // never appears even though the entity plainly does.
+ const present = criteria.subjectAliases.some(
+ alias =>
+ containsAlias(pageBackedText, alias) ||
+ tokenizeContent(alias).some(token =>
+ containsAlias(pageBackedText, token),
+ ),
+ );
+ if (!present) {
+ failures.push({
+ check: "subject_missing",
+ detail: `none of [${criteria.subjectAliases.join(", ")}] appear in the page evidence`,
+ });
+ }
+ }
+
+ // 2. Excluded-subject dominance β€” the story is ABOUT an entity the goal
+ // excludes (e.g. a competitor) and the alert concept doesn't name the
+ // subject. Name-drops of the subject inside someone else's story.
+ if (criteria.excludedSubjects.length > 0 && conceptText) {
+ const conceptNamesAlias = criteria.subjectAliases.some(alias =>
+ containsAlias(conceptText, alias),
+ );
+ const excludedInConcept = criteria.excludedSubjects.find(
+ excluded =>
+ containsAlias(conceptText, excluded) ||
+ containsAlias(evidence.titleText, excluded),
+ );
+ if (excludedInConcept && !conceptNamesAlias) {
+ failures.push({
+ check: "excluded_subject",
+ detail: `the story is about "${excludedInConcept}", which this monitor excludes`,
+ });
+ }
+ }
+
+ // 3. Concept relevance β€” the alert's event/topic label must share at least
+ // one content token with what the goal is about. Exact token comparison
+ // only: inflection variants ("lawsuit"/"lawsuits") are the compiler's job
+ // to enumerate into mustConcern as data, never code's job to derive.
+ // Enforced only with LLM-compiled criteria: the deterministic compile's
+ // token list is known-incomplete, and an incomplete allowlist must not veto.
+ if (
+ criteria.generatedBy === "llm" &&
+ criteria.mustConcern.length > 0 &&
+ conceptText
+ ) {
+ const conceptTokens = new Set(tokenizeContent(conceptText));
+ const aliasTokens = criteria.subjectAliases.flatMap(tokenizeContent);
+ const relevant = [...criteria.mustConcern, ...aliasTokens].some(token =>
+ conceptTokens.has(token),
+ );
+ if (conceptTokens.size > 0 && !relevant) {
+ failures.push({
+ check: "concept_off_goal",
+ detail: `alert concept "${conceptText.slice(0, 80)}" shares no content terms with the goal`,
+ });
+ }
+ }
+
+ // 4. Page shape β€” listing/aggregator/hub pages are surfaces, not stories.
+ // Structured metadata wins when present; otherwise measure link density on
+ // the scraped markdown (counting, not pattern matching). Skipped entirely
+ // for snippet-only results.
+ const shape = classifyPageShape(evidence);
+ if (shape.kind === "listing") {
+ failures.push({ check: "listing_surface", detail: shape.detail });
+ }
+
+ // 5. Owned surface β€” only when the goal explicitly wants third-party
+ // coverage. For event goals (releases, filings) the subject's own site is
+ // the most authoritative source there is, never a disqualified one.
+ if (criteria.thirdPartyOnly === true && criteria.ownedHosts.length > 0) {
+ const host = hostOf(evidence.url);
+ const owned =
+ host &&
+ criteria.ownedHosts.find(
+ ownedHost => host === ownedHost || host.endsWith(`.${ownedHost}`),
+ );
+ if (owned) {
+ failures.push({
+ check: "owned_surface",
+ detail: `${host} is operated by the monitored subject`,
+ });
+ }
+ }
+
+ return {
+ pass: failures.length === 0,
+ failures,
+ criteriaSource: criteria.generatedBy,
+ };
+}
+
+export function classifyPageShape(evidence: VerifyEvidence): {
+ kind: "article" | "listing" | "unknown";
+ detail: string;
+} {
+ const declared = declaredPageTypes(evidence.metadata);
+ if (declared.some(type => ARTICLE_PAGE_TYPES.has(type))) {
+ return { kind: "article", detail: `declared ${declared.join("/")}` };
+ }
+ if (declared.some(type => LISTING_PAGE_TYPES.has(type))) {
+ return {
+ kind: "listing",
+ detail: `page declares itself ${declared.join("/")}`,
+ };
+ }
+
+ const pageText = String(evidence.pageText ?? "");
+ if (!pageText.trim()) {
+ return { kind: "unknown", detail: "no page content to measure" };
+ }
+
+ const words = pageText.split(/\s+/).filter(Boolean).length;
+ const links = countLinks(pageText);
+ const publishedAt = publishedDate(evidence.metadata);
+ const density = words > 0 ? links / words : 0;
+
+ // A real article is prose; an aggregator/directory is a wall of links.
+ // Calibrated on live pages: link-heavy ARTICLES (citation-dense blogs, sites
+ // with nav/related-post chrome in the scrape) sit around 1 link per 30-50
+ // words, while true link walls run 1 per 6-12 words. Only the latter is a
+ // listing, and a declared publish date always vouches for the page.
+ if (links >= 30 && density > 0.08 && !publishedAt) {
+ return {
+ kind: "listing",
+ detail: `${links} links across ${words} words with no publish date`,
+ };
+ }
+ return { kind: "article", detail: `${links} links across ${words} words` };
+}
+
+function declaredPageTypes(
+ metadata: Record<string, unknown> | null | undefined,
+): string[] {
+ const meta = metadata && typeof metadata === "object" ? metadata : {};
+ const candidates = [
+ meta.ogType,
+ meta["og:type"],
+ ...jsonLdTypes(meta.jsonLd ?? meta["json-ld"] ?? meta.structuredData),
+ ];
+ return candidates
+ .filter((value): value is string => typeof value === "string")
+ .map(value => value.toLowerCase().replace(/[^a-z]/g, ""))
+ .filter(Boolean);
+}
+
+function jsonLdTypes(jsonLd: unknown): unknown[] {
+ const blocks = Array.isArray(jsonLd) ? jsonLd : jsonLd ? [jsonLd] : [];
+ return blocks.flatMap(block => {
+ if (!block || typeof block !== "object") return [];
+ const type = (block as Record<string, unknown>)["@type"];
+ return Array.isArray(type) ? type : type ? [type] : [];
+ });
+}
+
+function publishedDate(
+ metadata: Record<string, unknown> | null | undefined,
+): string | null {
+ const meta = (
+ metadata && typeof metadata === "object" ? metadata : {}
+ ) as Record<string, unknown>;
+ const value =
+ meta.publishedTime ||
+ meta["article:published_time"] ||
+ meta.publishedDate ||
+ meta.datePublished ||
+ meta.date;
+ if (!value || typeof value !== "string") return null;
+ const parsed = Date.parse(value);
+ return Number.isNaN(parsed) ? null : new Date(parsed).toISOString();
+}
+
+function countLinks(markdown: string): number {
+ let count = 0;
+ let index = 0;
+ for (;;) {
+ index = markdown.indexOf("](", index);
+ if (index === -1) break;
+ count += 1;
+ index += 2;
+ }
+ // Bare URLs outside markdown link syntax still count toward link mass.
+ index = 0;
+ for (;;) {
+ index = markdown.indexOf("http://", index);
+ if (index === -1) break;
+ count += 1;
+ index += 7;
+ }
+ index = 0;
+ for (;;) {
+ index = markdown.indexOf("https://", index);
+ if (index === -1) break;
+ count += 1;
+ index += 8;
+ }
+ // Markdown links were double-counted (the "](https://" form) β€” correct that.
+ return Math.max(0, Math.round(count / 2));
+}
+
+function hostOf(url: string): string {
+ try {
+ return new URL(String(url ?? "")).hostname
+ .toLowerCase()
+ .replace(/^www\./, "");
+ } catch {
+ return "";
+ }
+}
diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.ts
index f4f50cf0b..a9421f5f7 100644
--- a/apps/api/src/services/monitoring/types.ts
+++ b/apps/api/src/services/monitoring/types.ts
@@ -48,25 +48,27 @@ const monitorDomainSchema = z
"Domain must be a valid hostname without protocol or path",
);
-const searchTargetSchema = z
- .strictObject({
- id: z.string().uuid().optional(),
- type: z.literal("search"),
- queries: z.array(z.string().min(1).max(256)).min(1).max(10),
- searchWindow: z
- .enum(["5m", "15m", "1h", "6h", "24h", "7d"])
- .optional()
- .default("24h"),
- alertMode: z
- .enum(["first_match", "every_new_result", "material_dev"])
- .optional()
- .default("first_match"),
- includeDomains: z.array(monitorDomainSchema).max(50).optional(),
- excludeDomains: z.array(monitorDomainSchema).max(50).optional(),
- recheckAfter: z.enum(["1h", "6h", "24h", "7d"]).optional(),
- maxResults: z.number().int().min(1).max(50).optional().default(10),
- scrapeOptions: scrapeOptionsSchema,
- });
+const searchTargetSchema = z.strictObject({
+ id: z.string().uuid().optional(),
+ type: z.literal("search"),
+ queries: z.array(z.string().min(1).max(256)).min(1).max(10),
+ searchWindow: z
+ .enum(["5m", "15m", "1h", "6h", "24h", "7d"])
+ .optional()
+ .default("24h"),
+ alertMode: z
+ .enum(["first_match", "every_new_result", "material_dev"])
+ .optional()
+ .default("first_match"),
+ // "deep" scrapes + judges routed pages (extract tier); "standard" judges
+ // from SERP snippets only β€” no page fetches, the cheap tier.
+ depth: z.enum(["standard", "deep"]).optional().default("deep"),
+ includeDomains: z.array(monitorDomainSchema).max(50).optional(),
+ excludeDomains: z.array(monitorDomainSchema).max(50).optional(),
+ recheckAfter: z.enum(["1h", "6h", "24h", "7d"]).optional(),
+ maxResults: z.number().int().min(1).max(50).optional().default(10),
+ scrapeOptions: scrapeOptionsSchema,
+});
const monitorTargetSchema = z.union([
scrapeTargetSchema,