Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
strip comments from search monitor code
Developers Digest08e98c4f3216
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.tsindex 81fc85d0d..45c4921e7 100644--- a/apps/api/src/services/monitoring/results.ts+++ b/apps/api/src/services/monitoring/results.ts@@ -36,8 +36,6 @@ interface PageJudgment {}>;}-// Exported for the search-monitor runner: search targets run inline (not via-// the scrape worker), so they fire their per-page webhooks from runner.ts.export async function sendMonitorPageWebhook(params: {teamId: string;monitorId: string;diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 138f0720d..b9efbd2a1 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -1043,10 +1043,6 @@ async function runMonitorSearchTarget(params: {}await insertMonitorCheckPages(pages);- // Per-page webhooks (monitor.page), change-monitor parity: search targets run- // inline rather than through the scrape worker, so the worker's webhook path- // never sees them. Fire for outcomes a subscriber would act on β alerts- // ("new") and failures ("error") β not for unchanged/watching pages.for (const page of pages) {if (page.status !== "new" && page.status !== "error") continue;await sendMonitorPageWebhook({diff --git a/apps/api/src/services/monitoring/search/criteria.ts b/apps/api/src/services/monitoring/search/criteria.tsindex ca3ae56aa..e442fb709 100644--- a/apps/api/src/services/monitoring/search/criteria.ts+++ b/apps/api/src/services/monitoring/search/criteria.ts@@ -3,30 +3,19 @@ import { google } from "@ai-sdk/google";import { z } from "zod";import { googleProviderOptions } from "./tuning";-// Compile a monitor goal into a structured criteria artifact that the alert-// verifier can evaluate mechanically. ALL goal-specific knowledge lives in this-// artifact (data); the verifier has no per-goal-type code paths. The artifact is-// keyed to goalVersion, so editing the goal recompiles it automatically.-// Ported from the POC's goal-criteria.js.-const CRITERIA_MODEL =process.env.SEARCH_MONITOR_CRITERIA_MODEL ?? "gemini-flash-latest";export type GoalCriteria = {goalVersion: string;generatedBy: "deterministic" | "llm";- // Entity names (and product names, when LLM-compiled) that anchor an alert; empty = not enforced.subjectAliases: string[];- // Content tokens an alert's concept/evidence must overlap; empty = not enforced.mustConcern: string[];- // Entities that must NOT be the story's subject (LLM only β e.g. competitors); empty = not enforced.excludedSubjects: string[];- // Hosts whose own pages should not alert on mention feeds (LLM only); empty = not enforced.ownedHosts: string[];thirdPartyOnly: boolean;};-// Words that describe the *instruction*, not the *content* β never criteria.const INSTRUCTION_WORDS = new Set(["about","alert",@@ -140,8 +129,6 @@ export function containsAlias(return needle.length > 0 && haystack.includes(` ${needle} `);}-// Pure, synchronous, no-network compile. Always available; the LLM compile-// below only ever ADDS knowledge (product aliases, competitors, owned hosts).export function compileGoalCriteria(params: {goal: string;subject: string;@@ -162,8 +149,6 @@ export function compileGoalCriteria(params: {mustConcern,excludedSubjects: [],ownedHosts: [],- // Never assume a goal excludes the subject's own sources without the- // compiler explicitly saying so β vetoing official sources is recall loss.thirdPartyOnly: false,};}@@ -235,8 +220,6 @@ export function mergeCompiledCriteria(],thirdPartyOnly: llm.thirdPartyOnly === true,};- // An alias that is also an excluded subject would make every alert- // contradictory β the alias wins (the compiler over-reached).merged.excludedSubjects = merged.excludedSubjects.filter(excluded =>!merged.subjectAliases.some(@@ -246,8 +229,6 @@ export function mergeCompiledCriteria(return merged;}-// LLM enrichment of the deterministic compile. Throws on model failure β the-// caller keeps the deterministic artifact (fail-safe, never blocks the run).export async function compileGoalCriteriaWithLlm(params: {goal: string;subject: string;diff --git a/apps/api/src/services/monitoring/search/dedupe.ts b/apps/api/src/services/monitoring/search/dedupe.tsindex 58ed86c56..bdd78bb20 100644Binary files a/apps/api/src/services/monitoring/search/dedupe.ts and b/apps/api/src/services/monitoring/search/dedupe.ts differdiff --git a/apps/api/src/services/monitoring/search/judge.ts b/apps/api/src/services/monitoring/search/judge.tsindex 416e6506f..93177f2c6 100644--- a/apps/api/src/services/monitoring/search/judge.ts+++ b/apps/api/src/services/monitoring/search/judge.ts@@ -1,10 +1,3 @@-// Page judge runs INSIDE the Firecrawl scrape (the `json` format) β the verdict comes back-// on document.json. No separate Gemini call: Gemini is only for event-resolution + summary.--// Deliberately minimal: recency and source credibility are folded into the-// judge's single alertAction decision (see buildJudgePrompt) instead of being-// separate gateable fields β fewer fields means fewer self-contradictions and-// a simpler contract. The skeptic remains the independent safety net.export type SearchVerdict = {relevant: boolean;alertAction: "alert" | "watch" | "ignore";@@ -12,7 +5,6 @@ export type SearchVerdict = {rationale: string;};-// JSON schema passed to the scrape `json` format (Firecrawl runs the extraction internally).export const verdictJsonSchema = {type: "object",additionalProperties: false,@@ -42,7 +34,6 @@ export function buildJudgePrompt(.join("\n");}-// Validate/coerce the scrape's document.json into a verdict.export function parseVerdict(json: unknown): SearchVerdict | null {if (!json || typeof json !== "object") return null;const v = json as Record<string, unknown>;@@ -70,10 +61,6 @@ export function windowToMs(window: string): number {return WINDOW_MS[window] ?? WINDOW_MS["24h"];}-// "alert" only when relevant, concept-labeled, and the judge asked to alert.-// Recency and credibility live inside the judge's alertAction (see the prompt);-// a notify without a concept can't be event-deduped, so it would re-alert-// forever β concept stays a mechanical requirement.export function verdictToDecision(v: SearchVerdict,): "notify" | "watch" | "ignore" {@@ -83,16 +70,6 @@ export function verdictToDecision(return "notify";}-// ββ Verdict defenses (ported from the POC's assessment.js) ββββββββββββββββββ-// The judge's own prose is evidence for downstream stages (event resolver,-// verifier, skeptic). Two failure modes need mechanical correction before any-// of that runs: (1) the verdict's boolean contradicts its rationale, and-// (2) the rationale is self-referential meta-claims ("aligns with the monitor-// goal") instead of page facts.--// Does the rationale text negate the verdict? Returns the corrected stance:-// "no" (evidence absent β not relevant), "unclear" (insufficient evidence β-// never alert), or "" (no contradiction).export function contradictionFromRationale(rationale: string,): "no" | "unclear" | "" {@@ -141,8 +118,6 @@ export function contradictionFromRationale(return "";}-// Apply contradiction correction to a parsed verdict. A "no" contradiction-// flips relevant off (β ignore); an "unclear" one caps the action at watch.export function applyVerdictDefenses(v: SearchVerdict): SearchVerdict {const contradiction = contradictionFromRationale(v.rationale);if (contradiction === "no" && v.relevant) {@@ -154,10 +129,6 @@ export function applyVerdictDefenses(v: SearchVerdict): SearchVerdict {return v;}-// Strip the judge's self-referential meta-claims and field boilerplate from its-// rationale so downstream stages see page facts, not the judge grading itself.-// ("β¦which aligns with the monitor goal" is not evidence; LLM-side chrome like-// "Related topics: β¦" is page furniture, not a story.)export function stripJudgeMetaClaims(text: string): string {return String(text ?? "").replace(diff --git a/apps/api/src/services/monitoring/search/llm.ts b/apps/api/src/services/monitoring/search/llm.tsindex 21f111ac9..e599789ed 100644--- a/apps/api/src/services/monitoring/search/llm.ts+++ b/apps/api/src/services/monitoring/search/llm.ts@@ -4,10 +4,6 @@ import { z } from "zod";import { googleProviderOptions } from "./tuning";import type { GoalCriteria } from "./criteria";-// Gemini LLM calls for the search monitor (mirrors services/monitoring/judgeChange.ts).-// Every call runs with thinking suppressed (see tuning.ts) β reasoning tokens measured-// at ~85% of output spend on these structured tasks with no accuracy gain.-const EVENT_MODEL =process.env.SEARCH_MONITOR_EVENT_MODEL ?? "gemini-3-flash-preview";const SUMMARY_MODEL =@@ -17,9 +13,6 @@ const SKEPTIC_MODEL =const ROUTER_MODEL =process.env.SEARCH_MONITOR_ROUTER_MODEL ?? "gemini-3-flash-preview";-// Event state rides page-row metadata (no dedicated table): satisfiedAt is the-// first alert time, alertCount the number of alerts β reconstructed per run by-// persist.ts from the stamps run.ts writes on alerting pages.export type KnownEvent = {key: string;label: string;@@ -86,8 +79,6 @@ const materialDevSchema = z.object({reason: z.string(),});-// For alertMode "material_dev": does this new result add materially-new information to an-// already-alerted event (e.g. IPO filed β priced β traded), vs just another retelling?export async function judgeMaterialDevelopment(params: {goal: string;subject: string;@@ -133,11 +124,6 @@ const alertSkepticSchema = z.object({export type SkepticVerdict = z.infer<typeof alertSkepticSchema>;-// Adversarial review of an alert candidate β runs ONLY on results the judge-// already marked alert-worthy, so its cost is bounded by the (rare) alert rate.-// The judge that produced a verdict cannot be the one defending it; this is the-// independent second opinion. Callers must fail OPEN: a skeptic outage must not-// silently drop real alerts.export async function reviewAlert(params: {goal: string;subject: string;@@ -199,10 +185,6 @@ export type RouteDecision = {reason: string;};-// SERP router: decides which results are worth scrape + judge spend, from the-// SERP row alone. The single biggest cost lever on a deep run β every skipped-// candidate saves a full extract-tier scrape. Callers must fail OPEN to-// deterministic top-K routing when this throws.export async function routeSearchResults(params: {goal: string;subject: string;@@ -264,10 +246,6 @@ export type SnippetVerdict = z.infer<typeof snippetVerdictSchema>["verdicts"][number];-// Standard-depth judging: one batched call over SERP rows, no page fetches.-// (The POC's standard tier judges snippets deterministically; a single batched-// flash call is the backend-appropriate equivalent β same verdict contract as-// the in-scrape judge, ~free, and strictly better than keyword heuristics.)export async function judgeSnippets(params: {goal: string;subject: string;diff --git a/apps/api/src/services/monitoring/search/persist.test.ts b/apps/api/src/services/monitoring/search/persist.test.tsindex 6fe1df3b2..42cbe815f 100644--- a/apps/api/src/services/monitoring/search/persist.test.ts+++ b/apps/api/src/services/monitoring/search/persist.test.ts@@ -84,7 +84,6 @@ describe("reconstructKnownState", () => {[page("https://a.com", "old-story", "2026-01-01T00:00:00Z"),page("https://b.com", "active-story", "2026-01-02T00:00:00Z"),- // a newer article re-touches the old story β it should outrank active-storypage("https://c.com", "old-story", "2026-06-01T00:00:00Z"),],"gv1",diff --git a/apps/api/src/services/monitoring/search/persist.ts b/apps/api/src/services/monitoring/search/persist.tsindex b7411f3b9..f90ab63ba 100644--- a/apps/api/src/services/monitoring/search/persist.ts+++ b/apps/api/src/services/monitoring/search/persist.ts@@ -2,7 +2,6 @@ import { canonicalizeUrl } from "./dedupe";import type { KnownPage } from "./run";import type { KnownEvent } from "./llm";-// Search verdict status β the page-status enum the reconciler tallies (new/same/error).export function searchStatusToPageStatus(status: string,): "same" | "new" | "changed" | "removed" | "error" {@@ -11,7 +10,6 @@ export function searchStatusToPageStatus(return "same";}-// True for statuses that came from a successful scrape+judge (used for credit attribution).export function searchPageWasScraped(status: string): boolean {return status === "alert" || status === "watching" || status === "ignored";}@@ -23,8 +21,6 @@ type PriorPage = {last_status?: string;};-// Rebuild dedup memory + event index from prior pages. Fingerprints load regardless of-// goalVersion (the runner gates freshness); events only carry the current goalVersion.export function reconstructKnownState(priorPages: PriorPage[],goalVersion: string,@@ -47,10 +43,6 @@ export function reconstructKnownState(if (meta.goalVersion === goalVersion && typeof meta.eventKey === "string") {const existing = eventsByKey.get(meta.eventKey);const seenAt = page.updated_at ?? "";- // Event state is stamped per alerting page; aggregate across pages:- // satisfiedAt = earliest stamp (first alert), alertCount = highest stamp- // (each alert writes prior+1, so max is the true count even when several- // pages of the same event carry older stamps).const satisfiedAt =typeof meta.eventSatisfiedAt === "string"? meta.eventSatisfiedAt@@ -90,9 +82,6 @@ export function reconstructKnownState(}}}- // Most-recently-seen first: the event resolver truncates this list to ~20- // candidates, and active stories must stay inside that window or every new- // article about them mints a duplicate event (and a duplicate alert).const knownEvents = [...eventsByKey.values()].sort((a, b) => (a.lastSeenAt < b.lastSeenAt ? 1 : -1)).map(({ lastSeenAt: _ignored, ...event }) => event);diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.tsindex ab907f3f6..f6be80b92 100644--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts@@ -1,7 +1,6 @@import type { Logger } from "winston";import type { SearchVerdict } from "./judge";-// --- Mock the external boundaries; the orchestration logic in run.ts runs for real. ---const searchMock = jest.fn();const scrapeURLMock = jest.fn();const resolveEventMock = jest.fn();@@ -27,7 +26,6 @@ jest.mock("./llm", () => ({routeSearchResults: (...a: unknown[]) => routeMock(...a),judgeSnippets: (...a: unknown[]) => snippetsMock(...a),}));-// LLM stages on: these tests exercise the router / skeptic / verifier wiring.jest.mock("./tuning", () => ({hasGeminiKey: () => true,googleProviderOptions: () => ({}),@@ -207,7 +205,7 @@ describe("alert boundary", () => {const result = await runSearchTarget(runParams());expect(result.matches).toBe(0);expect(result.sources[0].status).toBe("watching");- expect(reviewAlertMock).not.toHaveBeenCalled(); // refuted before the skeptic spends tokens+ expect(reviewAlertMock).not.toHaveBeenCalled();});it("skeptic refutation downgrades to watching", async () => {diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 0474201f1..77a584ce6 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -3,7 +3,6 @@ import type { SearchVerdict } from "./judge";import type { EventResolution } from "./llm";import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";-// --- Mock the external boundaries; the orchestration logic in run.ts runs for real. ---const searchMock = jest.fn();const scrapeURLMock = jest.fn();const resolveEventMock = jest.fn();@@ -22,9 +21,6 @@ jest.mock("./llm", () => ({summarizeRun: (...a: unknown[]) => summarizeRunMock(...a),judgeMaterialDevelopment: (...a: unknown[]) => materialDevMock(...a),}));-// These tests exercise the core pipeline with the optional LLM stages (router,-// skeptic, criteria enrichment, snippet judge) gated off β the no-Gemini-key-// configuration. run-defenses.test.ts covers the stages-on wiring.jest.mock("./tuning", () => ({hasGeminiKey: () => false,googleProviderOptions: () => ({}),@@ -57,7 +53,6 @@ function setSearchResults(searchMock.mockResolvedValue(results);}-// scrapeURL returns a verdict chosen by URL.function setVerdictsByUrl(map: Record<string, SearchVerdict>) {scrapeURLMock.mockImplementation((_id: string, url: string) => {const v = map[url];@@ -151,7 +146,6 @@ describe("runSearchTarget orchestration", () => {description: "filing",},]);- // Precompute the fingerprint the same way run.ts does, via the real dedupe module.const canonical = canonicalizeUrl("https://sec.gov/openai");const fingerprint = stableSerpFingerprint({url: "https://sec.gov/openai",@@ -164,8 +158,6 @@ describe("runSearchTarget orchestration", () => {const out = await run({ knownPages });- // Reuse carries the prior outcome forward: this page alerted before, so the- // repeat is "already_seen". A page with no recorded prior alert must not be.expect(out.sources[0].status).toBe("already_seen");expect(out.pageUpserts[0].status).toBe("already_seen");expect(scrapeURLMock).not.toHaveBeenCalled();@@ -187,7 +179,6 @@ describe("runSearchTarget orchestration", () => {title: "OpenAI S-1",snippet: "filing",});- // Known under the OLD goal version β must be treated as new under gv2.const knownPages = new Map<string, KnownPage>([[canonical, { fingerprint, goalVersion: "gv1" }],]);@@ -215,7 +206,6 @@ describe("runSearchTarget orchestration", () => {"https://sec.gov/openai": verdict(),"https://nytimes.com/openai-ipo": verdict({ concept: "openai-ipo" }),});- // First result creates the event; second resolves to the same key.resolveEventMock.mockResolvedValueOnce({matchedKey: null,@@ -254,7 +244,6 @@ describe("runSearchTarget orchestration", () => {reason: "",});const out = await run();- // Key is the uuid (mocked constant here), not slugify("OpenAI's IPO Filing!!!").expect(out.sources[0].eventKey).toBe("00000000-0000-7000-8000-000000000000",);@@ -282,7 +271,6 @@ describe("runSearchTarget orchestration", () => {knownEvents: [{ key: "evt-stable-123", label: "OpenAI IPO" }],});expect(out.sources[0].eventKey).toBe("evt-stable-123");- // Reuses the stored label, ignoring the drifted one from this run.expect(out.pageUpserts[0].metadata.eventLabel).toBe("OpenAI IPO");});@@ -341,9 +329,6 @@ describe("runSearchTarget orchestration", () => {});describe("scrape payload validity (real validator, no mocks)", () => {- // Proves the exact scrape options run.ts sends survive the real v2 validator and yield a- // json format carrying our verdict schema + prompt β the one integration not covered by- // matching the scrape/crawl path.it("scrapeOptions.parse accepts the verdict json format", () => {const { scrapeOptions } = require("../../../controllers/v2/types");const { verdictJsonSchema, buildJudgePrompt } = require("./judge");@@ -454,8 +439,6 @@ describe("re-judge cadence", () => {]);const out = await run({ target: { recheckAfter: "24h" }, knownPages });expect(scrapeURLMock).not.toHaveBeenCalled();- // A page that only ever watched repeats as watching β "already_seen" is- // reserved for pages that actually alerted on a prior run.expect(out.sources[0].status).toBe("watching");});});diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 8145d367a..e51c05812 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -41,16 +41,12 @@ import {} from "./verify";import { hasGeminiKey } from "./tuning";-// Unified schedule/window step β Firecrawl tbs window.function windowToTbs(window: string): string {if (window === "5m" || window === "15m" || window === "1h") return "qdr:h";if (window === "6h" || window === "24h") return "qdr:d";- return "qdr:w"; // 7d+ return "qdr:w";}-// Exclude wins over include (Exa/Parallel semantics). The -site: operators in-// the scoped query are advisory β not every search provider honors them β so-// the blocklist is also enforced here on the returned results.function isExcludedDomain(url: string,excludeDomains: string[] | undefined,@@ -81,13 +77,8 @@ type SearchTargetInput = {alertMode: "first_match" | "every_new_result" | "material_dev";includeDomains?: string[];excludeDomains?: string[];- // Re-judge a still-live result (last status alert/watching) when its last check is older than- // this window, even if the SERP snippet is unchanged β catches content updates SERP text misses.recheckAfter?: string;maxResults: number;- // "deep" (default) scrapes + judges routed pages; "standard" judges from SERP- // snippets in one batched call β no page fetches, the cheap tier. Standard- // requires a Gemini key; without one it falls back to deep behavior.depth?: "standard" | "deep";};@@ -115,7 +106,7 @@ type SearchTargetRunResult = {pagesChecked: number;skipped: number;meaningful: number;- matches: number; // new alerts this run+ matches: number;summary: string;sources: SearchSource[];pageUpserts: Array<{@@ -123,8 +114,6 @@ type SearchTargetRunResult = {urlHash: Buffer;status: string;metadata: Record<string, unknown>;- // Present on alert pages: feeds the check page's judgment jsonb so email +- // per-page webhooks carry the judge's reasoning (change-monitor parity).judgment?: {meaningful: boolean;confidence: "high" | "medium" | "low";@@ -154,7 +143,7 @@ export async function runSearchTarget(params: {const tbs = windowToTbs(target.searchWindow);const events: KnownEvent[] = [...params.knownEvents];- const satisfied = new Set(events.map(e => e.key)); // first_match suppression set+ const satisfied = new Set(events.map(e => e.key));const sources: SearchSource[] = [];const pageUpserts: SearchTargetRunResult["pageUpserts"] = [];let resultCount = 0;@@ -162,7 +151,6 @@ export async function runSearchTarget(params: {let skipped = 0;let matches = 0;- // 1. Search each query; flatten + dedup by canonical URL within this run.const seenThisRun = new Set<string>();const candidates: Array<{url: string;@@ -171,7 +159,6 @@ export async function runSearchTarget(params: {query: string;}> = [];for (const query of target.queries) {- // Scope to domains the same way the v2 search API does (site:/-site: operators).const { query: scopedQuery } = buildSearchQuery(query, undefined, {includeDomains: target.includeDomains,excludeDomains: target.excludeDomains,@@ -202,11 +189,6 @@ export async function runSearchTarget(params: {const depth: "standard" | "deep" =target.depth === "standard" && llmStagesAvailable ? "standard" : "deep";- // Criteria artifact for the verifier + skeptic. Deterministic compile is- // free and always available; the LLM enrichment (aliases, competitors, owned- // hosts) fails safe back to deterministic. Compiled per run β cheap on a- // thinking-suppressed flash-class model; persistence keyed to goalVersion is- // a follow-up optimization, not a correctness need.let criteria: GoalCriteria = compileGoalCriteria({goal,subject,@@ -230,8 +212,6 @@ export async function runSearchTarget(params: {}}- // Route which candidates are worth scrape/judge spend (deep mode). The LLM- // router fails open to deterministic top-K β the pre-port behavior.let selected = candidates.slice(0, target.maxResults);if (depth === "deep" && llmStagesAvailable && candidates.length > 0) {try {@@ -262,8 +242,6 @@ export async function runSearchTarget(params: {}}- // Standard depth: one batched snippet-judge call for the whole selection β- // no page fetches. Verdicts keyed by canonical URL for the loop below.const snippetVerdicts = new Map<string, SearchVerdict>();if (depth === "standard" && selected.length > 0) {try {@@ -300,7 +278,6 @@ export async function runSearchTarget(params: {}}- // 2. Per candidate: dedup β (judge if new/changed) β decide β event-resolve.for (const c of selected) {const canonical = canonicalizeUrl(c.url);const fingerprint = stableSerpFingerprint({@@ -311,7 +288,6 @@ export async function runSearchTarget(params: {const known = knownPages.get(canonical);const knownCurrent =known && known.goalVersion === goalVersion ? known : undefined;- // Re-judge a still-live result on a cadence even if its SERP snippet is unchanged.const recheckMs = target.recheckAfter ? windowToMs(target.recheckAfter) : 0;const isLive =knownCurrent?.lastStatus === "alert" ||@@ -327,11 +303,6 @@ export async function runSearchTarget(params: {knownCurrent.fingerprint !== fingerprint ||dueForRecheck;- // Known + unchanged under the current goal β reuse, no scrape/judge/LLM. The reused- // status carries the page's prior outcome forward: "already_seen" must mean "this- // alerted before" β a page that only ever watched/ignored repeats as such, otherwise- // the UI reports a prior alert that never happened (and flipping it to already_seen- // would also drop it out of the isLive recheck cadence above).if (!isNewOrChanged) {const reusedStatus: SearchSource["status"] =knownCurrent?.lastStatus === "alert" ||@@ -351,8 +322,6 @@ export async function runSearchTarget(params: {continue;}- // Judge: deep scrapes the page (verdict returns on document.json, markdown- // kept for the verifier); standard reads the batched snippet verdict.let verdict: SearchVerdict | null = null;let pageText = "";let pageMetadata: Record<string, unknown> | null = null;@@ -417,18 +386,12 @@ export async function runSearchTarget(params: {null;}- // Mechanical defense: a verdict whose rationale negates its own booleans is- // corrected before anything downstream consumes it.verdict = applyVerdictDefenses(verdict);- // Downstream stages (verifier, skeptic, resolver) must see page facts, not- // the judge grading itself.const strippedRationale = stripJudgeMetaClaims(verdict.rationale);let decision = verdictToDecision(verdict);- // Alert boundary, stage 1 β deterministic verification against the compiled- // criteria. Downgrade-only (notify β watch), fail-open on unknown shapes.let verification: VerificationResult | null = null;if (decision === "notify") {const evidence: VerifyEvidence = {@@ -453,9 +416,6 @@ export async function runSearchTarget(params: {}}- // Alert boundary, stage 2 β adversarial skeptic. Runs only on surviving- // notify candidates (cost bounded by the alert rate); fails OPEN so a- // skeptic outage can't silently drop real alerts.if (decision === "notify" && llmStagesAvailable) {try {const skeptic: SkepticVerdict = await reviewAlert({@@ -486,8 +446,6 @@ export async function runSearchTarget(params: {fingerprint,goalVersion,alertAction: verdict.alertAction,- // Informational only β recency/credibility gating lives in the judge's- // alertAction, not in mechanical field gates.publishedAt: realDate,concept: verdict.concept,rationale: verdict.rationale,@@ -525,9 +483,6 @@ export async function runSearchTarget(params: {continue;}- // notify β resolve the real-world event against the known events (small list, handed straight- // to the LLM). Matches reuse the existing stable key; new events mint one. Resolver failure- // fails open to a new event: a possible duplicate alert beats a silently dropped one.let resolution;try {resolution = await resolveEvent({@@ -554,22 +509,16 @@ export async function runSearchTarget(params: {const matched =resolution.matchedKey &&events.find(e => e.key === resolution.matchedKey);- // New event β mint a stable, content-independent key so a future relabel can't drift it.const eventKey = matched ? matched.key : uuidv7();const eventLabel = matched? matched.label: resolution.label || verdict.concept;- // Suppression by mode. first_match: alert once per event. material_dev: alert once, then- // re-alert only when a later result adds materially-new info to the known event.- // every_new_result: never event-suppressed (alerts each new URL).let alreadySatisfied = false;if (satisfied.has(eventKey)) {if (target.alertMode === "first_match") {alreadySatisfied = true;} else if (target.alertMode === "material_dev") {- // Fails CLOSED: an unjudgeable "material development" stays suppressed β- // re-alerting an already-alerted event on an LLM outage is the worse error.try {const dev = await judgeMaterialDevelopment({goal,@@ -603,9 +552,6 @@ export async function runSearchTarget(params: {continue;}- // New, meaningful, not-yet-satisfied β alert. Event state (satisfiedAt =- // first alert, alertCount) is stamped onto the alerting page's metadata β- // that's its durable home; persist.ts re-aggregates it next run.matches += 1;satisfied.add(eventKey);const nowIso = new Date().toISOString();@@ -616,8 +562,6 @@ export async function runSearchTarget(params: {existingEvent.satisfiedAt = eventSatisfiedAt;existingEvent.alertCount = eventAlertCount;} else {- // Newest first: the resolver only sees the first ~20 candidates, and an- // event minted earlier in this same run must be visible to later results.events.unshift({key: eventKey,label: eventLabel,@@ -657,7 +601,6 @@ export async function runSearchTarget(params: {let summary = "";if (matches > 0) {- // Summarizer failure must not fail the run β fall back to a counting summary.try {const out = await summarizeRun({goal,diff --git a/apps/api/src/services/monitoring/search/tuning.ts b/apps/api/src/services/monitoring/search/tuning.tsindex 9acaabfec..3ce249575 100644--- a/apps/api/src/services/monitoring/search/tuning.ts+++ b/apps/api/src/services/monitoring/search/tuning.ts@@ -1,12 +1,3 @@-// Thinking suppression for search-monitor orchestration calls. Reasoning tokens-// measured at ~85% of LLM output spend on these structured tasks, so every call-// defaults to the lowest thinking the model supports. Gemini 3 models take-// thinkingLevel ("minimal" is the floor β thinking cannot be fully disabled);-// Gemini 2.5-era models (flash-latest, flash-lite) take thinkingBudget, where 0-// disables thinking entirely.-//-// SEARCH_MONITOR_THINKING: "off" (default) | "minimal" | "low" | "medium" |-// "high" | "default" ("default" restores the provider's full-thinking behavior).const GEMINI_3_LEVELS = ["minimal", "low", "medium", "high"] as const;type GoogleThinkingOptions = {@@ -41,9 +32,6 @@ export function googleProviderOptions(model: string): GoogleThinkingOptions {};}-// The LLM stages (router, skeptic, criteria enrichment) are optional: without a-// Gemini key they are skipped and deterministic fallbacks run instead, exactly-// like the POC's *FromEnv factories returning null.export function hasGeminiKey(): boolean {return Boolean(process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? process.env.GEMINI_API_KEY,diff --git a/apps/api/src/services/monitoring/search/verify.ts b/apps/api/src/services/monitoring/search/verify.tsindex 85d890a05..d678f3bad 100644--- a/apps/api/src/services/monitoring/search/verify.ts+++ b/apps/api/src/services/monitoring/search/verify.ts@@ -1,14 +1,5 @@import { containsAlias, tokenizeContent, type GoalCriteria } from "./criteria";-// Deterministic verification of a judge "alert" verdict. Every check is a-// universal mechanic (string containment, set membership, token overlap, page-// shape measurement) β goal-specific knowledge comes exclusively from the-// compiled criteria, and a check whose criteria field is empty is a no-op, so-// unknown goal shapes degrade to trusting the judge rather than breaking.-//-// Verification gates (alert β watch); it never escalates and never ignores.-// Ported from the POC's alert-verifier.js.-const LISTING_PAGE_TYPES = new Set(["collectionpage","itemlist",@@ -29,9 +20,7 @@ const ARTICLE_PAGE_TYPES = new Set([export type VerifyEvidence = {url: string;titleText: string;- // The judge's claim about the page (rationale), already meta-claim-stripped.claimText: string;- // Scraped page markdown when available; empty for snippet-only results.pageText: string;metadata?: Record<string, unknown> | null;};@@ -58,18 +47,11 @@ export function verifyAlertCandidate(params: {const conceptText = String(params.concept ?? "");const hasPageContent = Boolean(String(evidence.pageText ?? "").trim());- // 1. Subject containment β the monitored entity must actually appear in the- // visible evidence, not just in the query we sent. Only enforceable when- // we hold the page content the judge read: on snippet-only results the- // judge saw more than we did, so absence here proves nothing.if (criteria.subjectAliases.length > 0 &&hasPageContent &&pageBackedText.trim()) {- // Whole-alias containment, or any single alias token: an inferred subject- // can be a multi-word phrase ("Anthropic models") whose exact phrasing- // never appears even though the entity plainly does.const present = criteria.subjectAliases.some(alias =>containsAlias(pageBackedText, alias) ||@@ -85,9 +67,6 @@ export function verifyAlertCandidate(params: {}}- // 2. Excluded-subject dominance β the story is ABOUT an entity the goal- // excludes (e.g. a competitor) and the alert concept doesn't name the- // subject. Name-drops of the subject inside someone else's story.if (criteria.excludedSubjects.length > 0 && conceptText) {const conceptNamesAlias = criteria.subjectAliases.some(alias =>containsAlias(conceptText, alias),@@ -105,12 +84,6 @@ export function verifyAlertCandidate(params: {}}- // 3. Concept relevance β the alert's event/topic label must share at least- // one content token with what the goal is about. Exact token comparison- // only: inflection variants ("lawsuit"/"lawsuits") are the compiler's job- // to enumerate into mustConcern as data, never code's job to derive.- // Enforced only with LLM-compiled criteria: the deterministic compile's- // token list is known-incomplete, and an incomplete allowlist must not veto.if (criteria.generatedBy === "llm" &&criteria.mustConcern.length > 0 &&@@ -129,18 +102,11 @@ export function verifyAlertCandidate(params: {}}- // 4. Page shape β listing/aggregator/hub pages are surfaces, not stories.- // Structured metadata wins when present; otherwise measure link density on- // the scraped markdown (counting, not pattern matching). Skipped entirely- // for snippet-only results.const shape = classifyPageShape(evidence);if (shape.kind === "listing") {failures.push({ check: "listing_surface", detail: shape.detail });}- // 5. Owned surface β only when the goal explicitly wants third-party- // coverage. For event goals (releases, filings) the subject's own site is- // the most authoritative source there is, never a disqualified one.if (criteria.thirdPartyOnly === true && criteria.ownedHosts.length > 0) {const host = hostOf(evidence.url);const owned =@@ -188,11 +154,6 @@ export function classifyPageShape(evidence: VerifyEvidence): {const publishedAt = publishedDate(evidence.metadata);const density = words > 0 ? links / words : 0;- // A real article is prose; an aggregator/directory is a wall of links.- // Calibrated on live pages: link-heavy ARTICLES (citation-dense blogs, sites- // with nav/related-post chrome in the scrape) sit around 1 link per 30-50- // words, while true link walls run 1 per 6-12 words. Only the latter is a- // listing, and a declared publish date always vouches for the page.if (links >= 30 && density > 0.08 && !publishedAt) {return {kind: "listing",@@ -252,7 +213,6 @@ function countLinks(markdown: string): number {count += 1;index += 2;}- // Bare URLs outside markdown link syntax still count toward link mass.index = 0;for (;;) {index = markdown.indexOf("http://", index);@@ -267,7 +227,6 @@ function countLinks(markdown: string): number {count += 1;index += 8;}- // Markdown links were double-counted (the "](https://" form) β correct that.return Math.max(0, Math.round(count / 2));}