Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Improve monitoring queue, search-run, and billing
Developers Digestd632e75c5f72
diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.tsindex e0fc5b234..bd50efc67 100644--- a/apps/api/src/services/monitoring/queue.ts+++ b/apps/api/src/services/monitoring/queue.ts@@ -28,6 +28,12 @@ let connection: amqp.ChannelModel | null = null;// other.let publishChannel: amqp.Channel | null = null;let consumeChannel: amqp.Channel | null = null;+// In-flight creation promises. Channel creation suspends on async asserts before+// the channel is assigned, so two concurrent callers (e.g. the two consume+// registrations started together at worker boot) would each create a channel and+// orphan one. Memoizing the promise serializes concurrent callers onto one.+let publishChannelPromise: Promise<amqp.Channel> | null = null;+let consumeChannelPromise: Promise<amqp.Channel> | null = null;// Registry of every consumer registered this process, so we can re-attach them// after a connection/channel drop. Without this the worker — which never@@ -92,7 +98,9 @@ function handleConnectionDrop(reason: string, error?: unknown): void {}connection = null;publishChannel = null;+ publishChannelPromise = null;consumeChannel = null;+ consumeChannelPromise = null;subscribedQueues.clear();if (registeredConsumers.length > 0) {scheduleReconnect(RECONNECT_BASE_DELAY_MS);@@ -120,9 +128,18 @@ function scheduleReconnect(delayMs: number): void {error,nextDelayMs: nextDelay,});+ // Best-effort close the half-set-up channel, then reset ALL cached state —+ // including subscribedQueues. If a mid-loop subscribe() failed on an+ // otherwise-healthy channel (so its "close" never fires), leaving+ // subscribedQueues populated would make the next reconnect skip that queue+ // (subscribe() early-returns on subscribedQueues.has) and deafen it forever.+ await consumeChannel?.close().catch(() => {});connection = null;publishChannel = null;+ publishChannelPromise = null;consumeChannel = null;+ consumeChannelPromise = null;+ subscribedQueues.clear();scheduleReconnect(nextDelay);}}, delayMs);@@ -157,9 +174,13 @@ async function createChannel(label: string): Promise<amqp.Channel> {// PRECONDITION_FAILED from assertQueue). Without these handlers the cached// channel would stay non-null and every send/consume would throw forever.ch.on("close", () => {- if (label === "publish" && publishChannel === ch) publishChannel = null;+ if (label === "publish" && publishChannel === ch) {+ publishChannel = null;+ publishChannelPromise = null;+ }if (label === "consume" && consumeChannel === ch) {consumeChannel = null;+ consumeChannelPromise = null;handleConnectionDrop("consume channel closed");}});@@ -171,14 +192,34 @@ async function createChannel(label: string): Promise<amqp.Channel> {async function getPublishChannel(): Promise<amqp.Channel> {if (publishChannel) return publishChannel;- publishChannel = await createChannel("publish");- return publishChannel;+ if (!publishChannelPromise) {+ publishChannelPromise = createChannel("publish")+ .then(ch => {+ publishChannel = ch;+ return ch;+ })+ .catch(err => {+ publishChannelPromise = null;+ throw err;+ });+ }+ return publishChannelPromise;}async function getConsumeChannel(): Promise<amqp.Channel> {if (consumeChannel) return consumeChannel;- consumeChannel = await createChannel("consume");- return consumeChannel;+ if (!consumeChannelPromise) {+ consumeChannelPromise = createChannel("consume")+ .then(ch => {+ consumeChannel = ch;+ return ch;+ })+ .catch(err => {+ consumeChannelPromise = null;+ throw err;+ });+ }+ return consumeChannelPromise;}export async function addMonitorCheckJob(diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.tsindex 586ff26b2..b7db2a6ee 100644--- a/apps/api/src/services/monitoring/runner.test.ts+++ b/apps/api/src/services/monitoring/runner.test.ts@@ -91,6 +91,20 @@ describe("monitoring runner", () => {now,),).toBe(true);+ // MIXED search+crawl: keeps the 1-hour timeout (the crawl can legitimately+ // run for many minutes) — must NOT be reaped at 11 minutes.+ expect(+ isMonitorCheckStale(+ {+ ...base,+ target_results: [+ { type: "search", targetId: "t1" },+ { type: "crawl", targetId: "t2" },+ ],+ },+ now,+ ),+ ).toBe(false);});});});diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex b67883e79..1c627e893 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -332,11 +332,11 @@ async function runSingleScrape(params: {};}-// Deep-mode search-monitor page scrape. Routes through the same internal-// scrape path as every other monitor scrape (processJobInternal, skipNuq,-// bypassBilling) instead of calling scrapeURL directly, so it respects the-// team's concurrency queue and is never billed per-page (search monitors bill-// flat at the check level).+// Deep-mode search-monitor page scrape. Runs inline via processJobInternal+// (skipNuq:true, bypassBilling) — it is NOT routed through NuQ, so it bypasses+// per-team/global scrape concurrency; the caller bounds the fan-out via+// SEARCH_SCRAPE_CONCURRENCY (search/run.ts). Never billed per-page (search+// monitors bill flat at the check level).async function scrapeSearchMonitorPage(params: {teamId: string;checkId: string;@@ -1337,6 +1337,14 @@ export async function processMonitorCheckJob(// Set LAST, after credits are stamped, so the reconciler never finalizes// with credits still at 0.targetRun.searchCompleted = true;+ // Persist immediately (not just at the end of the loop): the search work+ // — external search calls + per-page scrapes — already happened. If we+ // crash before the end-of-loop flush, a redelivery would re-run and+ // re-scrape everything. Flushing searchCompleted now lets+ // findCompletedSearchTargetRun short-circuit the redelivery.+ await updateMonitorCheck(check.id, {+ target_results: targetResults,+ });}}diff --git a/apps/api/src/services/monitoring/search/billing.ts b/apps/api/src/services/monitoring/search/billing.tsindex a8dfbb933..cca5cf7e0 100644--- a/apps/api/src/services/monitoring/search/billing.ts+++ b/apps/api/src/services/monitoring/search/billing.ts@@ -1,6 +1,9 @@const SEARCH_CREDITS_PER_TEN_RESULTS = 2;const SEARCH_CREDITS_PER_TEN_RESULTS_ZDR = 10;-const SEARCH_JUDGE_CREDITS_PER_RESULT = 1;+// The single source of truth for the per-judged-result judge credit rate.+// Tests MUST import this rather than re-declaring a literal (which previously+// drifted to 5 and silently misrepresented billing).+export const SEARCH_JUDGE_CREDITS_PER_RESULT = 1;export function searchCreditsForResultCount(rawResultCount: number,diff --git a/apps/api/src/services/monitoring/search/cost.test.ts b/apps/api/src/services/monitoring/search/cost.test.tsindex 3077faf5a..88ab71569 100644--- a/apps/api/src/services/monitoring/search/cost.test.ts+++ b/apps/api/src/services/monitoring/search/cost.test.ts@@ -1,7 +1,9 @@import { CostTracking } from "../../../lib/cost-tracking";import { recordLlmCall } from "./cost";--const JUDGE_CREDITS_PER_RESULT = 5;+import {+ SEARCH_JUDGE_CREDITS_PER_RESULT,+ judgeCreditsForJudgedCount,+} from "./billing";describe("search-monitor LLM cost recording (observability only)", () => {it("records token usage into the shared CostTracking with no dollar cost", () => {@@ -15,7 +17,10 @@ describe("search-monitor LLM cost recording (observability only)", () => {const json = ct.toJSON();// Tokens recorded for observability; cost stays 0 (flat per-result billing).expect(json.totalCost).toBe(0);- expect(json.calls[0].tokens).toEqual({ input: 1_000_000, output: 1_000_000 });+ expect(json.calls[0].tokens).toEqual({+ input: 1_000_000,+ output: 1_000_000,+ });});it("normalizes promptTokens/completionTokens (older SDK usage shape)", () => {@@ -26,28 +31,33 @@ describe("search-monitor LLM cost recording (observability only)", () => {usage: { promptTokens: 1_000_000, completionTokens: 0 },stage: "summarizeRun",});- expect(ct.toJSON().calls[0].tokens).toEqual({ input: 1_000_000, output: 0 });+ expect(ct.toJSON().calls[0].tokens).toEqual({+ input: 1_000_000,+ output: 0,+ });});});describe("search-monitor FLAT judge billing rate", () => {- it("is a deterministic 5 credits per judged result", () => {- expect(JUDGE_CREDITS_PER_RESULT).toBe(5);+ it("is a deterministic 1 credit per judged result", () => {+ expect(SEARCH_JUDGE_CREDITS_PER_RESULT).toBe(1);});- it("computes judge credits as 5 * resultsJudged", () => {- const judgeCreditsFor = (resultsJudged: number) =>- resultsJudged * JUDGE_CREDITS_PER_RESULT;- expect(judgeCreditsFor(0)).toBe(0); // raw / judge-off- expect(judgeCreditsFor(3)).toBe(15); // deep, 3 results judged- expect(judgeCreditsFor(10)).toBe(50);+ it("computes judge credits as 1 * resultsJudged", () => {+ expect(judgeCreditsForJudgedCount(0)).toBe(0); // raw / judge-off+ expect(judgeCreditsForJudgedCount(3)).toBe(3); // deep, 3 results judged+ expect(judgeCreditsForJudgedCount(10)).toBe(10);});});// Billed count == pages with judgedThisRun=true (not every page, not every page// with a concept — reused/skipped pages keep a stale concept but aren't judged).describe("canonical judged-result count (billing == persisted signal)", () => {- type Page = { searchStatus: string; judgedThisRun?: boolean; concept?: string };+ type Page = {+ searchStatus: string;+ judgedThisRun?: boolean;+ concept?: string;+ };const billedJudgedCount = (pages: Page[]) =>pages.filter(p => p.judgedThisRun === true).length;@@ -55,13 +65,17 @@ describe("canonical judged-result count (billing == persisted signal)", () => {// 1 judged this run; 2 reused (stale concept, not judged); 1 skipped.const pages: Page[] = [{ searchStatus: "alert", judgedThisRun: true, concept: "fresh-verdict" },- { searchStatus: "already_seen", judgedThisRun: false, concept: "stale-1" },+ {+ searchStatus: "already_seen",+ judgedThisRun: false,+ concept: "stale-1",+ },{ searchStatus: "watching", judgedThisRun: false, concept: "stale-2" },{ searchStatus: "skipped" }, // no verdict, no judgedThisRun];expect(pages.filter(p => p.concept).length).toBe(3); // the misleading countexpect(billedJudgedCount(pages)).toBe(1); // the canonical, billed count- expect(billedJudgedCount(pages) * JUDGE_CREDITS_PER_RESULT).toBe(5);+ expect(judgeCreditsForJudgedCount(billedJudgedCount(pages))).toBe(1);});it("counts judged results across every billed outcome (alert/watch/ignore/already_seen)", () => {@@ -72,6 +86,6 @@ describe("canonical judged-result count (billing == persisted signal)", () => {{ searchStatus: "already_seen", judgedThisRun: true },{ searchStatus: "skipped" }, // verdict failed -> not billed];- expect(billedJudgedCount(pages)).toBe(4); // -> 5*4 = 20 judge credits+ expect(billedJudgedCount(pages)).toBe(4); // -> 1*4 = 4 judge credits});});diff --git a/apps/api/src/services/monitoring/search/criteria.ts b/apps/api/src/services/monitoring/search/criteria.tsdeleted file mode 100644index eac3d63a8..000000000--- a/apps/api/src/services/monitoring/search/criteria.ts+++ /dev/null@@ -1,123 +0,0 @@-import { z } from "zod";--export type GoalCriteria = {- goalVersion: string;- generatedBy: "deterministic" | "llm";- subjectAliases: string[];- mustConcern: string[];- excludedSubjects: string[];- ownedHosts: string[];- thirdPartyOnly: boolean;-};--export function tokenizeContent(text: string | null | undefined): string[] {- return String(text ?? "")- .toLowerCase()- .split(/[^a-z0-9]+/)- .filter(token => token.length >= 2);-}--function normalizeForContainment(text: string | null | undefined): string {- return ` ${String(text ?? "")- .toLowerCase()- .replace(/[^a-z0-9]+/g, " ")- .trim()} `;-}--export function containsAlias(- text: string | null | undefined,- alias: string,-): boolean {- const haystack = normalizeForContainment(text);- const needle = normalizeForContainment(alias).trim();- return needle.length > 0 && haystack.includes(` ${needle} `);-}--export function compileGoalCriteria(params: {- goal: string;- subject: string;- goalVersion: string;-}): GoalCriteria {- const subject = params.subject.trim();- return {- goalVersion: params.goalVersion,- generatedBy: "deterministic",- subjectAliases: subject ? [subject] : [],- mustConcern: [],- excludedSubjects: [],- ownedHosts: [],- thirdPartyOnly: false,- };-}--const criteriaSchema = z.object({- subjectAliases: z- .array(z.string())- .describe(- "Every name the monitored subject goes by, INCLUDING its well-known products and brand names. Empty if the goal has no specific subject.",- ),- mustConcern: z- .array(z.string())- .describe(- "Single lowercase content words that name the goal's SPECIFIC topic/scope — the subject matter a matching story must actually be about, not just the broad field. Derived from the goal's topic, not its phrasing. For a goal like 'AI safety or alignment research', use the precise topic words (safety, alignment, interpretability, evaluation) rather than broad-field words (ai, research) that adjacent off-topic stories would also share. Matching is EXACT string comparison, so include inflection variants explicitly (e.g. release, releases, released, launch, launches, launched).",- ),- excludedSubjects: z- .array(z.string())- .describe(- "Entities whose OWN news must not trigger this monitor (e.g. competitors merely name-dropping the subject). Empty when the goal doesn't imply any.",- ),- ownedHosts: z- .array(z.string())- .describe(- "Hostnames operated by the subject itself (official site, docs, blog).",- ),- thirdPartyOnly: z- .boolean()- .describe(- "true ONLY when the goal asks for third-party coverage of the subject (mentions, press, discussion, reviews) — the subject's own pages then don't satisfy it. false when the subject's own announcements DO satisfy the goal (releases, launches, filings, pricing changes, changelogs).",- ),-});--export function mergeCompiledCriteria(- deterministic: GoalCriteria,- llm: z.infer<typeof criteriaSchema>,-): GoalCriteria {- const merged: GoalCriteria = {- ...deterministic,- generatedBy: "llm",- subjectAliases: [- ...new Set([- ...deterministic.subjectAliases,- ...llm.subjectAliases.map(s => s.trim()).filter(Boolean),- ]),- ],- mustConcern: [...new Set(llm.mustConcern.flatMap(tokenizeContent))],- excludedSubjects: [- ...new Set(llm.excludedSubjects.map(s => s.trim()).filter(Boolean)),- ],- ownedHosts: [- ...new Set(- llm.ownedHosts- .map(- h =>- String(h ?? "")- .trim()- .toLowerCase()- .replace(/^https?:\/\//, "")- .replace(/^www\./, "")- .split("/")[0]- .split(":")[0],- )- .filter(Boolean),- ),- ],- thirdPartyOnly: llm.thirdPartyOnly === true,- };- merged.excludedSubjects = merged.excludedSubjects.filter(- excluded =>- !merged.subjectAliases.some(- alias => alias.toLowerCase() === excluded.toLowerCase(),- ),- );- return merged;-}diff --git a/apps/api/src/services/monitoring/search/llm.ts b/apps/api/src/services/monitoring/search/llm.tsdeleted file mode 100644index 5dd78a88e..000000000--- a/apps/api/src/services/monitoring/search/llm.ts+++ /dev/null@@ -1,74 +0,0 @@-import { generateObject } from "ai";-import { z } from "zod";-import {- googleModel,- googleProviderOptions,- type LlmUsageLabels,-} from "./tuning";-import { recordLlmCall } from "./cost";-import type { CostTracking } from "../../../lib/cost-tracking";--const ROUTER_MODEL = "gemini-3-flash-preview";--export type KnownEvent = {- key: string;- label: string;- satisfiedAt?: string;- alertCount?: number;-};--const snippetVerdictSchema = z.object({- verdicts: z.array(- z.object({- id: z.string(),- relevant: z.boolean(),- alertAction: z.enum(["alert", "watch", "ignore"]),- concept: z.string(),- rationale: z.string(),- }),- ),-});--export type SnippetVerdict = z.infer<- typeof snippetVerdictSchema->["verdicts"][number];--export async function judgeSnippets(params: {- goal: string;- subject: string;- searchWindow: string;- candidates: Array<{- id: string;- query: string;- title: string;- url: string;- snippet: string;- }>;- costTracking?: CostTracking;- labels?: LlmUsageLabels;-}): Promise<SnippetVerdict[]> {- if (params.candidates.length === 0) return [];- const { object, usage } = await generateObject({- model: googleModel(ROUTER_MODEL),- schema: snippetVerdictSchema,- system:- "You judge search results for a monitoring product using ONLY each result's SERP title, URL, and snippet — you cannot see the page. Be conservative: a snippet is thin evidence, so alertAction alert requires the snippet itself to concretely state a completed, recent event satisfying the goal, from a source credible for that claim (the subject itself or established reporting). The goal names a SPECIFIC topic/scope: the result's core subject must actually match that specific subject matter, not merely be related, adjacent, or in the same broad field — a credible source or the right organization on an adjacent topic is NOT enough. Treat any 'Ignore/except/not/exclude' clause in the goal as a hard exclusion. Query wording echoed in a snippet is not evidence. Rumors, content farms, competitors, look-alikes, listings, and old or unconfirmed results are watch/ignore. concept: a short reusable label naming the real-world event; describe what the result is actually about, do not reword it to match the goal. rationale: concrete snippet facts only, never references to the monitor or goal. Return one verdict per candidate id. Structured JSON only.",- prompt: JSON.stringify({- monitor: { goal: params.goal, subject: params.subject },- searchWindow: params.searchWindow,- candidates: params.candidates.slice(0, 50),- }),- temperature: 0,- ...googleProviderOptions("judgeSnippets", params.labels),- });- if (params.costTracking) {- recordLlmCall({- costTracking: params.costTracking,- model: ROUTER_MODEL,- usage,- stage: "judgeSnippets",- });- }- const validIds = new Set(params.candidates.map(c => c.id));- return object.verdicts.filter(v => validIds.has(v.id));-}diff --git a/apps/api/src/services/monitoring/search/persist.ts b/apps/api/src/services/monitoring/search/persist.tsindex 6d9680742..74390bb75 100644--- a/apps/api/src/services/monitoring/search/persist.ts+++ b/apps/api/src/services/monitoring/search/persist.ts@@ -1,6 +1,5 @@import { canonicalizeUrl } from "./dedupe";-import type { KnownPage } from "./run";-import type { KnownEvent } from "./llm";+import type { KnownPage, KnownEvent } from "./run";export function searchStatusToPageStatus(status: string,diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.tsindex 65547194f..850fdf497 100644--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts@@ -4,12 +4,11 @@ import { canonicalizeUrl } from "./dedupe";// vi.mock is hoisted above declarations, so the mocks its factories reference// are created in vi.hoisted() (also hoisted) to avoid any TDZ surprises.-// The lean pipeline has only one LLM stage left in deep mode (the per-page JSON-// verdict, produced by the injected scrapePage) plus judgeSnippets for standard.-const { searchMock, scrapeURLMock, snippetsMock } = vi.hoisted(() => ({+// The lean pipeline has a single LLM stage: the deep-mode per-page JSON verdict,+// produced by the injected scrapePage.+const { searchMock, scrapeURLMock } = vi.hoisted(() => ({searchMock: vi.fn(),scrapeURLMock: vi.fn(),- snippetsMock: vi.fn(),}));vi.mock("uuid", () => ({ v7: () => "00000000-0000-7000-8000-000000000000" }));@@ -21,9 +20,6 @@ vi.mock("../../../search/v2", () => ({return Array.isArray(r) ? { web: r } : r;},}));-vi.mock("./llm", () => ({- judgeSnippets: (...a: unknown[]) => snippetsMock(...a),-}));vi.mock("./tuning", () => ({hasLlmProvider: () => true,googleProviderOptions: () => ({}),@@ -122,36 +118,6 @@ describe("deep mode (scrape every selected candidate)", () => {});});-describe("standard depth (snippet judging, no scrapes)", () => {- it("judges from snippets without any page fetch", async () => {- searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);- snippetsMock.mockResolvedValue([- { id: "result_1", ...verdict() },- { id: "result_2", ...verdict({ alertAction: "watch" }) },- ]);-- const result = await runSearchTarget(runParams({ depth: "standard" }));- expect(scrapeURLMock).not.toHaveBeenCalled();- expect(result.pagesChecked).toBe(2);- expect(result.matches).toBe(1);- expect(result.sources.map(s => s.status).sort()).toEqual([- "alert",- "watching",- ]);- // snippet-only judgments never bill a scrape- expect(result.pageUpserts.every(u => u.scraped === false)).toBe(true);- });-- it("skips results the snippet judge failed to cover", async () => {- searchMock.mockResolvedValue([serpRow(1)]);- snippetsMock.mockRejectedValue(new Error("batch failed"));-- const result = await runSearchTarget(runParams({ depth: "standard" }));- expect(result.skipped).toBe(1);- expect(result.matches).toBe(0);- });-});-describe("alert boundary (the per-page JSON verdict is the gate)", () => {beforeEach(() => {searchMock.mockResolvedValue([serpRow(1)]);@@ -265,8 +231,7 @@ describe("judgeEnabled gates the LLM judge", () => {runParams({ depth: "deep" }, { judgeEnabled: false }),);- // No judge stage ran: raw mode neither snippet-judges nor scrapes.- expect(snippetsMock).not.toHaveBeenCalled();+ // No judge stage ran: raw mode does not scrape.expect(scrapeURLMock).not.toHaveBeenCalled();// No judge credits billed (raw → nothing judged); search still ran and was billed.@@ -410,16 +375,51 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",expect(scrapeURLMock).not.toHaveBeenCalled();});- it("degraded=true when the standard snippet judge fails for every candidate", async () => {- searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);- snippetsMock.mockRejectedValue(new Error("snippet judge outage"));+ it("flags partialScrapeLoss (soft signal, NOT degraded) when most scrapes fail but some judge", async () => {+ searchMock.mockResolvedValue([+ serpRow(1),+ serpRow(2),+ serpRow(3),+ serpRow(4),+ ]);+ // Only result 1 scrapes + judges; results 2-4 all fail to scrape.+ scrapeURLMock.mockImplementation(({ url }: { url: string }) =>+ url === serpRow(1).url+ ? Promise.resolve({+ success: true,+ document: {+ json: verdict({ alertAction: "ignore" }),+ markdown: "prose",+ metadata: {},+ },+ })+ : Promise.resolve({ success: false }),+ );- const result = await runSearchTarget(runParams({ depth: "standard" }));+ const result = await runSearchTarget(runParams({ depth: "deep" }));- expect(result.resultsJudged).toBe(0);- expect(result.matches).toBe(0);- expect(result.judgeDegraded).toBe(true);- expect(result.degradedReason).toMatch(/judged|incomplete/i);+ expect(result.resultsJudged).toBe(1);+ expect(result.scrapeFailures).toBe(3);+ // 3/(1+3) = 0.75 >= 0.5 -> soft partial-loss signal, but the run still+ // produced a verdict so it is NOT a hard degrade.+ expect(result.partialScrapeLoss).toBe(true);+ expect(result.judgeDegraded).toBe(false);+ });++ it("does NOT flag partialScrapeLoss when every scrape succeeds", async () => {+ scrapeURLMock.mockResolvedValue({+ success: true,+ document: {+ json: verdict({ alertAction: "ignore" }),+ markdown: "prose",+ metadata: {},+ },+ });++ const result = await runSearchTarget(runParams({ depth: "deep" }));++ expect(result.scrapeFailures).toBe(0);+ expect(result.partialScrapeLoss).toBe(false);});});diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 904bb269c..91839be11 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -24,8 +24,7 @@ vi.mock("../../../search/v2", () => ({},}));// Deep mode (these tests) scrapes + parses a per-page JSON verdict and dedups by-// canonical URL — no LLM chain. judgeSnippets (standard mode) is never reached.-vi.mock("./llm", () => ({ judgeSnippets: vi.fn() }));+// canonical URL — the only LLM stage is the injected per-page scrape verdict.vi.mock("./tuning", () => ({hasLlmProvider: () => false,googleProviderOptions: () => ({}),diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex ad526bbe4..3b37f21b9 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -1,7 +1,6 @@import type { Logger } from "winston";import { search } from "../../../search/v2";import { buildSearchQuery } from "../../../lib/search-query-builder";-import { CostTracking } from "../../../lib/cost-tracking";import { hashMonitorUrl } from "../store";import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";import {@@ -11,13 +10,6 @@ import {windowToMs,type SearchVerdict,} from "./judge";-import { judgeSnippets, type KnownEvent, type SnippetVerdict } from "./llm";-import { compileGoalCriteria, type GoalCriteria } from "./criteria";-import {- verifyAlertCandidate,- type VerificationResult,- type VerifyEvidence,-} from "./verify";import { hasLlmProvider } from "./tuning";import {searchCreditsForResultCount,@@ -78,6 +70,31 @@ const SEARCH_RETRY_BACKOFF_MS = [400, 1200, 3000, 6000] as const;// 10-minute search stale timeout so a run always returns before the reaper.const SEARCH_RUN_BUDGET_MS = 4 * 60 * 1000;+// Deep-mode page scrapes run INLINE in the worker process (skipNuq), so they are+// NOT bounded by NuQ's per-team/global concurrency. Cap the pre-scrape fan-out+// ourselves so a single maxResults=50 check can't launch 50 simultaneous+// fetch+LLM-extraction scrapes and starve the worker that owns the search consumer.+const SEARCH_SCRAPE_CONCURRENCY = 6;++// Run fn over items with at most `limit` in flight at once. Never rejects per the+// caller's needs — fn is expected to swallow its own errors (it writes outcomes+// into a shared cache).+async function mapWithConcurrency<T>(+ items: T[],+ limit: number,+ fn: (item: T) => Promise<void>,+): Promise<void> {+ let next = 0;+ const worker = async (): Promise<void> => {+ while (next < items.length) {+ const i = next++;+ await fn(items[i]);+ }+ };+ const pool = Array.from({ length: Math.min(limit, items.length) }, worker);+ await Promise.all(pool);+}+// Race a promise against a wall-clock deadline. Resolves false if the work// finishes first, true if the deadline hits. Never rejects; the underlying work// keeps running (bounded by its own per-scrape timeout) and populates its cache.@@ -170,6 +187,13 @@ export type KnownPage = {metadata?: Record<string, unknown>;};+export type KnownEvent = {+ key: string;+ label: string;+ satisfiedAt?: string;+ alertCount?: number;+};+// Deep-mode page scrape. Injected by the caller so it can route through the// shared monitor scrape path (concurrency queue + bypassBilling), keeping this// module free of worker/queue dependencies. Resolves to null on scrape failure.@@ -205,6 +229,11 @@ type SearchTargetRunResult = {summary: string;judgeDegraded: boolean;degradedReason: string | null;+ // Per-page scrape failures in deep mode. partialScrapeLoss is a SOFT signal+ // (distinct from judgeDegraded): some results judged, but a majority of the+ // attempted scrapes failed — the run is usable but likely missed developments.+ scrapeFailures: number;+ partialScrapeLoss: boolean;sources: SearchSource[];searchCredits: number;judgeCredits: number;@@ -250,13 +279,6 @@ export async function runSearchTarget(params: {throw new Error("search monitor target requires a non-empty monitor goal");}const subject = params.monitor.subject ?? "";- const teamId = params.monitor.teamId;- // Vertex billing labels attached to every LLM call for usage tracing.- const labels = {- teamId,- monitorId: params.monitor.id,- monitorCheckId: params.monitorCheckId,- };const tbs = windowToTbs(target.searchWindow);const events: KnownEvent[] = [...params.knownEvents];@@ -267,11 +289,9 @@ export async function runSearchTarget(params: {let pagesChecked = 0;let skipped = 0;let matches = 0;- const costTracking = new CostTracking();let searchResultsBilled = 0;let resultsJudged = 0;let judgeScrapeFailures = 0;- let snippetJudgeFailed = false;const seenThisRun = new Map<string, number>();const candidates: Array<{@@ -335,96 +355,22 @@ export async function runSearchTarget(params: {}const llmStagesAvailable = hasLlmProvider();- const depth: "raw" | "standard" | "deep" = !judgeEnabled- ? "raw"- : target.depth === "raw"- ? "raw"- : target.depth === "standard" && llmStagesAvailable- ? "standard"- : "deep";+ // Judge depth is raw (no LLM) or deep (per-page scrape + verdict). Standard+ // (snippet-only judging) was removed: it was unreachable via the API (depth is+ // stripped) and used a divergent LLM provider; deep is the single judged path.+ const depth: "raw" | "deep" =+ !judgeEnabled || target.depth === "raw" ? "raw" : "deep";const judgeGoal: string = goal ?? "";- // Deterministic criteria for the (non-LLM) verifier — no LLM enrich/router.- const criteria: GoalCriteria = compileGoalCriteria({- goal: judgeGoal,- subject,- goalVersion,- });-const selected = candidates.slice(0, target.maxResults);const recheckMs = target.recheckAfter ? windowToMs(target.recheckAfter) : 0;- const pageNeedsJudgment = (c: {- url: string;- title: string;- description: string;- }) => {- const canonical = canonicalizeUrl(c.url);- const known = knownPages.get(canonical);- const knownCurrent =- known && known.goalVersion === goalVersion ? known : undefined;- if (!knownCurrent) return true;- const fingerprint = stableSerpFingerprint({- url: c.url,- title: c.title,- snippet: c.description,- });- if (knownCurrent.fingerprint !== fingerprint) return true;- const isLive =- knownCurrent.lastStatus === "alert" ||- knownCurrent.lastStatus === "watching";- return Boolean(- recheckMs &&- isLive &&- knownCurrent.lastCheckedAt &&- Date.now() - Date.parse(knownCurrent.lastCheckedAt) > recheckMs,- );- };-- const snippetVerdicts = new Map<string, SearchVerdict>();- const snippetCandidates = selected.filter(pageNeedsJudgment);- if (depth === "standard" && snippetCandidates.length > 0) {- try {- const verdicts = await judgeSnippets({- goal: judgeGoal,- subject,- searchWindow: target.searchWindow,- costTracking,- labels,- candidates: snippetCandidates.map((c, i) => ({- id: `result_${i + 1}`,- query: c.query,- title: c.title,- url: c.url,- snippet: c.description,- })),- });- const byId = new Map<string, SnippetVerdict>(- verdicts.map(v => [v.id, v]),- );- snippetCandidates.forEach((c, i) => {- const v = byId.get(`result_${i + 1}`);- if (v) {- snippetVerdicts.set(canonicalizeUrl(c.url), {- relevant: v.relevant,- alertAction: v.alertAction,- concept: v.concept,- rationale: v.rationale,- });- }- });- } catch (error) {- snippetJudgeFailed = true;- logger.warn("search monitor snippet judge failed; results skipped", {- error,- });- }- }- // Pre-scrape deep-mode pages concurrently and cache them, so the loop below isn't- // a serial scrape→judge→scrape chain. Only the I/O is parallel; all dedup/event- // state stays in the single-threaded loop. (NuQ bounds the actual scrape fan-out.)+ // Pre-scrape deep-mode pages (bounded concurrency) and cache them, so the loop+ // below isn't a serial scrape→judge→scrape chain. Only the I/O is parallel; all+ // dedup/event state stays in the single-threaded loop. These scrapes run inline+ // (skipNuq), so the fan-out is bounded by SEARCH_SCRAPE_CONCURRENCY, not NuQ.const deepDocs = new Map<string,{ doc: ScrapeSearchResult | null; error?: unknown }@@ -439,23 +385,24 @@ export async function runSearchTarget(params: {subject,target.searchWindow,);- const preScrape = Promise.all(- selected- .filter(- c =>- evaluateSerpCandidate(c, knownPages, goalVersion, recheckMs)- .isNewOrChanged,- )- .map(async c => {- const canonical = canonicalizeUrl(c.url);- try {- deepDocs.set(canonical, {- doc: await params.scrapePage({ url: c.url, judgePrompt }),- });- } catch (error) {- deepDocs.set(canonical, { doc: null, error });- }- }),+ const toScrape = selected.filter(+ c =>+ evaluateSerpCandidate(c, knownPages, goalVersion, recheckMs)+ .isNewOrChanged,+ );+ const preScrape = mapWithConcurrency(+ toScrape,+ SEARCH_SCRAPE_CONCURRENCY,+ async c => {+ const canonical = canonicalizeUrl(c.url);+ try {+ deepDocs.set(canonical, {+ doc: await params.scrapePage({ url: c.url, judgePrompt }),+ });+ } catch (error) {+ deepDocs.set(canonical, { doc: null, error });+ }+ },);const remainingBudget = SEARCH_RUN_BUDGET_MS - (Date.now() - runStart);budgetExceeded = await raceDeadline(preScrape, remainingBudget);@@ -589,22 +536,9 @@ export async function runSearchTarget(params: {}let verdict: SearchVerdict | null = null;- let pageText = "";let realDate: string | null = null;- if (depth === "standard") {- verdict = snippetVerdicts.get(canonical) ?? null;- if (!verdict) {- if (snippetJudgeFailed) {- pushUnevaluatedPage("snippet judge failed");- } else {- skipped += 1;- sources.push({ url: c.url, title: c.title, status: "skipped" });- }- continue;- }- pagesChecked += 1;- } else {- // Read the doc from the pre-scrape cache; scrape inline only as a fallback.+ {+ // Deep mode: read the doc from the pre-scrape cache; scrape inline only as a fallback.let doc: ScrapeSearchResult | null;const cached = deepDocs.get(canonical);if (cached?.error !== undefined) {@@ -660,37 +594,13 @@ export async function runSearchTarget(params: {pushUnevaluatedPage("verdict unparseable");continue;}- pageText = doc.markdown ?? "";realDate =doc.metadata?.publishedTime ?? doc.metadata?.modifiedTime ?? null;}resultsJudged += 1;- let decision = verdictToDecision(verdict);-- let verification: VerificationResult | null = null;- if (decision === "notify") {- const evidence: VerifyEvidence = {- url: c.url,- titleText: c.title,- claimText: verdict.rationale,- pageText,- };- const result = verifyAlertCandidate({- criteria,- concept: verdict.concept,- evidence,- });- verification = result;- if (!result.pass) {- decision = "watch";- logger.info("search monitor alert downgraded by verifier", {- url: c.url,- failures: result.failures,- });- }- }+ const decision = verdictToDecision(verdict);const baseMeta = {fingerprint,@@ -701,7 +611,6 @@ export async function runSearchTarget(params: {rationale: verdict.rationale,matchedQueries: c.matchedQueries,judgedThisRun: true,- ...(verification && !verification.pass ? { verification } : {}),};if (decision === "ignore") {@@ -819,14 +728,11 @@ export async function runSearchTarget(params: {? `${matches} match${matches === 1 ? "" : "es"} across ${resultCount} result${resultCount === 1 ? "" : "s"}.`: "";- const deepJudgeFailed =+ const judgeDegraded =depth === "deep" &&candidates.length > 0 &&resultsJudged === 0 &&judgeScrapeFailures > 0;- const standardJudgeFailed =- depth === "standard" && resultsJudged === 0 && snippetJudgeFailed;- const judgeDegraded = deepJudgeFailed || standardJudgeFailed;if (judgeDegraded) {logger.warn("search monitor run is degraded: results were found but none could be judged; results may be incomplete (NOT a clean no-results)",@@ -836,7 +742,6 @@ export async function runSearchTarget(params: {depth,candidates: candidates.length,scrapeFailures: judgeScrapeFailures,- snippetJudgeFailed,resultsJudged,},);@@ -850,6 +755,25 @@ export async function runSearchTarget(params: {? "results were found but none could be judged; results may be incomplete": null;+ // Soft signal (NOT judgeDegraded): the run produced verdicts, but a majority of+ // the attempted deep scrapes failed, so it likely missed real developments.+ // Surfaced for observability; does not fail the check.+ const partialScrapeLoss =+ resultsJudged > 0 &&+ judgeScrapeFailures > 0 &&+ judgeScrapeFailures / (resultsJudged + judgeScrapeFailures) >= 0.5;+ if (partialScrapeLoss) {+ logger.warn(+ "search monitor run had heavy partial scrape loss: judged some results but most scrapes failed; coverage may be incomplete",+ {+ monitorId: params.monitor.id,+ targetId: target.id,+ resultsJudged,+ scrapeFailures: judgeScrapeFailures,+ },+ );+ }+const isZDR = params.zeroDataRetention;const searchCredits = searchCreditsForResultCount(searchResultsBilled, isZDR);const judgeCredits = judgeCreditsForJudgedCount(resultsJudged);@@ -865,6 +789,8 @@ export async function runSearchTarget(params: {summary,judgeDegraded,degradedReason,+ scrapeFailures: judgeScrapeFailures,+ partialScrapeLoss,sources,searchCredits,judgeCredits,diff --git a/apps/api/src/services/monitoring/search/verify.test.ts b/apps/api/src/services/monitoring/search/verify.test.tsdeleted file mode 100644index 220ae2d68..000000000--- a/apps/api/src/services/monitoring/search/verify.test.ts+++ /dev/null@@ -1,204 +0,0 @@-import {- compileGoalCriteria,- mergeCompiledCriteria,- tokenizeContent,- type GoalCriteria,-} from "./criteria";-import { verifyAlertCandidate } from "./verify";--const baseCriteria = (over: Partial<GoalCriteria> = {}): GoalCriteria => ({- goalVersion: "v1",- generatedBy: "llm",- subjectAliases: ["Firecrawl"],- mustConcern: ["launch", "launches", "launched", "product"],- excludedSubjects: ["Parallel"],- ownedHosts: ["firecrawl.dev"],- thirdPartyOnly: false,- ...over,-});--const evidence = (- over: Partial<Parameters<typeof verifyAlertCandidate>[0]["evidence"]> = {},-) => ({- url: "https://news.example.com/story",- titleText: "Firecrawl launches a new product",- claimText: "Firecrawl announced a new product today.",- pageText: "Firecrawl announced a new product today. Details follow in prose.",- ...over,-});--describe("criteria", () => {- it("merge drops an excluded subject that is also an alias (alias wins)", () => {- const deterministic = compileGoalCriteria({- goal: "Alert me when Firecrawl launches a new product",- subject: "Firecrawl",- goalVersion: "v1",- });- const merged = mergeCompiledCriteria(deterministic, {- subjectAliases: ["Firecrawl", "FIRE-1"],- mustConcern: ["launch"],- excludedSubjects: ["firecrawl", "Parallel"],- ownedHosts: ["https://www.firecrawl.dev/blog"],- thirdPartyOnly: false,- });- expect(merged.generatedBy).toBe("llm");- expect(merged.excludedSubjects).toEqual(["Parallel"]);- expect(merged.ownedHosts).toEqual(["firecrawl.dev"]);- expect(merged.subjectAliases).toContain("FIRE-1");- });-- it("strips ports when normalizing ownedHosts", () => {- const deterministic = compileGoalCriteria({- goal: "Alert me when Firecrawl launches a new product",- subject: "Firecrawl",- goalVersion: "v1",- });- const merged = mergeCompiledCriteria(deterministic, {- subjectAliases: ["Firecrawl"],- mustConcern: [],- excludedSubjects: [],- ownedHosts: ["https://firecrawl.dev:3000/blog", "localhost:8080"],- thirdPartyOnly: true,- });- expect(merged.ownedHosts).toEqual(["firecrawl.dev", "localhost"]);- });-- it("tokenizeContent keeps 2-char acronyms (AI, EU, Go) and drops 1-char noise", () => {- expect(- tokenizeContent("Go 1.0 released in the EU under the AI Act"),- ).toEqual([- "go",- "released",- "in",- "the",- "eu",- "under",- "the",- "ai",- "act",- ]);- });-});--describe("verifyAlertCandidate", () => {- it("passes a clean alert", () => {- const result = verifyAlertCandidate({- criteria: baseCriteria(),- concept: "Firecrawl product launch",- evidence: evidence(),- });- expect(result.pass).toBe(true);- expect(result.failures).toEqual([]);- });-- it("fails subject containment when the subject never appears in page-backed evidence", () => {- const result = verifyAlertCandidate({- criteria: baseCriteria(),- concept: "Firecrawl product launch",- evidence: evidence({- titleText: "A company shipped something",- claimText: "Some launch happened.",- pageText: "Some company shipped something today. Details follow.",- }),- });- expect(result.failures.map(f => f.check)).toContain("subject_missing");- });-- it("passes subject containment when the subject appears only in the page body", () => {- const result = verifyAlertCandidate({- criteria: baseCriteria(),- concept: "Firecrawl product launch",- evidence: evidence({- titleText: "A scraping startup shipped something",- claimText: "A new product launched today.",- pageText: "The company behind Firecrawl announced a new product.",- }),- });- expect(result.failures.map(f => f.check)).not.toContain("subject_missing");- });-- it("skips subject containment for snippet-only results (no page content)", () => {- const result = verifyAlertCandidate({- criteria: baseCriteria(),- concept: "Firecrawl product launch",- evidence: evidence({- titleText: "A company shipped something",- claimText: "Some launch happened.",- pageText: "",- }),- });- expect(result.failures.map(f => f.check)).not.toContain("subject_missing");- });-- it("fails excluded-subject dominance when a competitor owns the story", () => {- const result = verifyAlertCandidate({- criteria: baseCriteria(),- concept: "Parallel launches Monitor API",- evidence: evidence({- titleText: "Parallel launches Monitor API",- claimText:- "Parallel launched its Monitor API; Firecrawl is named as a competitor.",- }),- });- expect(result.failures.map(f => f.check)).toContain("excluded_subject");- });-- it("fails concept relevance when the concept shares no goal terms (llm criteria only)", () => {- const offGoal = verifyAlertCandidate({- criteria: baseCriteria(),- concept: "quarterly earnings beat",- evidence: evidence(),- });- expect(offGoal.failures.map(f => f.check)).toContain("concept_off_goal");-- const deterministic = verifyAlertCandidate({- criteria: baseCriteria({ generatedBy: "deterministic" }),- concept: "quarterly earnings beat",- evidence: evidence(),- });- expect(deterministic.failures.map(f => f.check)).not.toContain(- "concept_off_goal",- );- });-- it("downgrades a subject-only concept that misses the topic (funding vs launch)", () => {- // "Firecrawl funding" names the subject but not the goal's topic (launch);- // a subject-alias token must NOT satisfy the must-concern gate.- const result = verifyAlertCandidate({- criteria: baseCriteria(),- concept: "Firecrawl Series B funding",- evidence: evidence(),- });- expect(result.failures.map(f => f.check)).toContain("concept_off_goal");- });-- it("fails owned-surface only when the goal wants third-party coverage", () => {- const owned = verifyAlertCandidate({- criteria: baseCriteria({ thirdPartyOnly: true }),- concept: "Firecrawl product launch",- evidence: evidence({ url: "https://www.firecrawl.dev/blog/launch" }),- });- expect(owned.failures.map(f => f.check)).toContain("owned_surface");-- const eventGoal = verifyAlertCandidate({- criteria: baseCriteria({ thirdPartyOnly: false }),- concept: "Firecrawl product launch",- evidence: evidence({ url: "https://www.firecrawl.dev/blog/launch" }),- });- expect(eventGoal.failures.map(f => f.check)).not.toContain("owned_surface");- });-- it("empty criteria disables every check (fail-open for unknown goal shapes)", () => {- const result = verifyAlertCandidate({- criteria: baseCriteria({- subjectAliases: [],- mustConcern: [],- excludedSubjects: [],- ownedHosts: [],- }),- concept: "anything at all",- evidence: evidence({ titleText: "unrelated", claimText: "unrelated" }),- });- expect(result.pass).toBe(true);- });-});diff --git a/apps/api/src/services/monitoring/search/verify.ts b/apps/api/src/services/monitoring/search/verify.tsdeleted file mode 100644index 82edae5ae..000000000--- a/apps/api/src/services/monitoring/search/verify.ts+++ /dev/null@@ -1,124 +0,0 @@-import { containsAlias, tokenizeContent, type GoalCriteria } from "./criteria";--export type VerifyEvidence = {- url: string;- titleText: string;- claimText: string;- pageText: string;-};--type VerificationFailure = { check: string; detail: string };--export type VerificationResult = {- pass: boolean;- failures: VerificationFailure[];- criteriaSource: GoalCriteria["generatedBy"];-};--export function verifyAlertCandidate(params: {- criteria: GoalCriteria;- evidence: VerifyEvidence;- concept: string;-}): VerificationResult {- const { criteria, evidence } = params;- const failures: VerificationFailure[] = [];-- const pageBackedText = [- evidence.titleText,- evidence.claimText,- String(evidence.pageText ?? "").slice(0, 20000),- ]- .filter(Boolean)- .join(" ");- const conceptText = String(params.concept ?? "");- const hasPageContent = Boolean(String(evidence.pageText ?? "").trim());-- if (- criteria.subjectAliases.length > 0 &&- hasPageContent &&- pageBackedText.trim()- ) {- const present = criteria.subjectAliases.some(- alias =>- containsAlias(pageBackedText, alias) ||- tokenizeContent(alias).some(token =>- containsAlias(pageBackedText, token),- ),- );- if (!present) {- failures.push({- check: "subject_missing",- detail: `none of [${criteria.subjectAliases.join(", ")}] appear in the page evidence`,- });- }- }-- if (criteria.excludedSubjects.length > 0 && conceptText) {- const conceptNamesAlias = criteria.subjectAliases.some(alias =>- containsAlias(conceptText, alias),- );- const excludedInConcept = criteria.excludedSubjects.find(- excluded =>- containsAlias(conceptText, excluded) ||- containsAlias(evidence.titleText, excluded),- );- if (excludedInConcept && !conceptNamesAlias) {- failures.push({- check: "excluded_subject",- detail: `the story is about "${excludedInConcept}", which this monitor excludes`,- });- }- }-- if (- criteria.generatedBy === "llm" &&- criteria.mustConcern.length > 0 &&- conceptText- ) {- const conceptTokens = new Set(tokenizeContent(conceptText));- const topicPresent = criteria.mustConcern.some(token =>- conceptTokens.has(token),- );- if (conceptTokens.size > 0 && !topicPresent) {- failures.push({- check: "concept_off_goal",- detail: `alert concept "${conceptText.slice(0, 80)}" shares no goal terms with the monitor`,- });- }- }-- if (criteria.thirdPartyOnly === true && criteria.ownedHosts.length > 0) {- const host = hostOf(evidence.url);- const owned =- host &&- criteria.ownedHosts.find(- ownedHost => host === ownedHost || host.endsWith(`.${ownedHost}`),- );- if (owned) {- failures.push({- check: "owned_surface",- detail: `${host} is operated by the monitored subject`,- });- }- }-- // `subject_missing` is a fragile literal alias-in-page check, so keep it as a- // diagnostic but don't let it alone suppress an alert — only the stronger checks block.- const blockingFailures = failures.filter(f => f.check !== "subject_missing");-- return {- pass: blockingFailures.length === 0,- failures,- criteriaSource: criteria.generatedBy,- };-}--function hostOf(url: string): string {- try {- return new URL(String(url ?? "")).hostname- .toLowerCase()- .replace(/^www\./, "");- } catch {- return "";- }-}diff --git a/apps/api/src/services/monitoring/stale.ts b/apps/api/src/services/monitoring/stale.tsindex 73d217eb5..97c86775c 100644--- a/apps/api/src/services/monitoring/stale.ts+++ b/apps/api/src/services/monitoring/stale.ts@@ -8,11 +8,14 @@ const MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS = 10 * 60 * 1000;export const MONITOR_CHECK_STALE_ERROR ="Monitor check exceeded the running timeout.";-// A check is a search check if any of its target runs is of type "search".-function isSearchCheck(check: { target_results?: unknown }): boolean {+// The shorter search timeout only applies when EVERY target is a search target.+// A mixed monitor (e.g. search + crawl) must keep the long timeout — its scrape+// fan-out / crawl legitimately runs for many minutes and would otherwise be+// falsely reaped at the 10-minute search cutoff.+function isSearchOnlyCheck(check: { target_results?: unknown }): boolean {const targetResults = check.target_results;- if (!Array.isArray(targetResults)) return false;- return targetResults.some(+ if (!Array.isArray(targetResults) || targetResults.length === 0) return false;+ return targetResults.every(tr =>tr != null &&typeof tr === "object" &&@@ -23,7 +26,7 @@ function isSearchCheck(check: { target_results?: unknown }): boolean {export function monitorCheckStaleTimeoutMs(check: {target_results?: unknown;}): number {- return isSearchCheck(check)+ return isSearchOnlyCheck(check)? MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS: MONITOR_CHECK_STALE_TIMEOUT_MS;}diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex eecb9e102..2c3469148 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -7,6 +7,7 @@ import {estimateMonitorCreditsPerRun,flatSearchTargetCredits,} from "./store";+import { judgeCreditsForJudgedCount } from "./search/billing";import type { MonitorTarget } from "./types";describe("monitoring store credit helpers", () => {@@ -255,17 +256,17 @@ describe("FLAT deterministic search-monitor billing", () => {describe("flatSearchTargetCredits (the deterministic actual)", () => {it("sums searchCredits + judgeCredits across search target_results", () => {- // The headline trace: search 2 + judge 5*3 = 17.+ // The headline trace: search 2 + judge 1*3 = 5.const targetResults = [{targetId: "search-1",type: "search",searchCredits: 2,- judgeCredits: 15,+ judgeCredits: 3,resultsJudged: 3,},];- expect(flatSearchTargetCredits(targetResults)).toBe(17);+ expect(flatSearchTargetCredits(targetResults)).toBe(5);});it("raw / judge-off: just the 2 search credits, 0 judge", () => {@@ -301,7 +302,7 @@ describe("FLAT deterministic search-monitor billing", () => {});});- // Flat billing contract: actual = 2*ceil(totalResults/10) + 5*resultsJudged.+ // Flat billing contract: actual = 2*ceil(totalResults/10) + 1*resultsJudged.describe("flat search billing contract (eval H regression guard)", () => {const searchCreditsFor = (totalResults: number) =>2 * Math.ceil(totalResults / 10);@@ -311,19 +312,19 @@ describe("FLAT deterministic search-monitor billing", () => {searchCompleted: true,resultCount: totalResults,searchCredits: searchCreditsFor(totalResults),- judgeCredits: resultsJudged * 5,+ judgeCredits: judgeCreditsForJudgedCount(resultsJudged),resultsJudged,});- it("is the exact deterministic flat sum (the headline 17-credit trace)", () => {- // results=6 -> search 2; judged=3 -> 15; total 17.- expect(flatSearchTargetCredits([stampedSearchRun(6, 3)])).toBe(17);+ it("is the exact deterministic flat sum (the headline 5-credit trace)", () => {+ // results=6 -> search 2; judged=3 -> 3; total 5.+ expect(flatSearchTargetCredits([stampedSearchRun(6, 3)])).toBe(5);});it("never bills 0 on a check that returned results and judged them", () => {- // results=6, judged=4 -> 2 + 20 = 22 (the judged#2 case that landed on 0).+ // results=6, judged=4 -> 2 + 4 = 6 (the judged#2 case that landed on 0).const credits = flatSearchTargetCredits([stampedSearchRun(6, 4)]);- expect(credits).toBe(22);+ expect(credits).toBe(6);expect(credits).toBeGreaterThan(0);});@@ -333,7 +334,7 @@ describe("FLAT deterministic search-monitor billing", () => {// searchCompleted=true guard ensures credits are stamped before summing.const retried = stampedSearchRun(6, 4);expect(retried.searchCompleted).toBe(true);- expect(flatSearchTargetCredits([retried])).toBe(22);+ expect(flatSearchTargetCredits([retried])).toBe(6);});it("search-only (judge off) bills just the search portion, never the judge", () => {diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 8cf931314..e391c8558 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -502,8 +502,15 @@ export async function updateMonitor(params: {if (params.input.status !== undefined) patch.status = params.input.status;if (params.input.webhook !== undefined)patch.webhook = params.input.webhook ?? null;- if (params.input.notification !== undefined) {- patch.notification = params.input.notification ?? null;+ // Defense-in-depth against a wiped notification: only write when the caller+ // actually sent config. Treat an empty object (the legacy default-materialized+ // {}) as "leave unchanged" rather than clobbering the stored email settings.+ if (+ params.input.notification !== undefined &&+ params.input.notification !== null &&+ Object.keys(params.input.notification).length > 0+ ) {+ patch.notification = params.input.notification;}if (params.input.retentionDays !== undefined) {patch.retention_days = params.input.retentionDays;diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsindex 8122aa6b6..44c7623ef 100644--- a/apps/api/src/services/monitoring/types.ts+++ b/apps/api/src/services/monitoring/types.ts@@ -129,16 +129,19 @@ const monitorScheduleSchema = ztimezone: schedule.timezone,}));-const monitorNotificationSchema = z- .strictObject({- email: z- .strictObject({- enabled: z.boolean().optional().default(false),- recipients: z.array(z.email()).max(25).optional().default([]),- includeDiffs: z.boolean().optional().default(false),- })- .optional(),- })+const monitorNotificationInner = z.strictObject({+ email: z+ .strictObject({+ enabled: z.boolean().optional().default(false),+ recipients: z.array(z.email()).max(25).optional().default([]),+ includeDiffs: z.boolean().optional().default(false),+ })+ .optional(),+});+// Create defaults a missing notification to {}. The UPDATE schema deliberately+// does NOT (see updateMonitorSchema) — a default there would materialize {} on+// any PATCH that omits notification and wipe the stored email config.+const monitorNotificationSchema = monitorNotificationInner.optional().default({});@@ -223,6 +226,10 @@ export const updateMonitorSchema = createMonitorBaseSchema.partial().extend({status: z.enum(["active", "paused"]).optional(),+ // Override the create-path notification (which defaults to {}): on update a+ // default would materialize {} for a PATCH that omits notification and wipe+ // the stored email config. No default => omitted means "leave unchanged".+ notification: monitorNotificationInner.optional(),}).superRefine(rejectGoalClearedWithSearchTargets).refine(x => Object.keys(x).length > 0, "Update body cannot be empty");