Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Improve monitoring queue, search-run, and billing

Developers Digestd632e75c5f72
diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.ts
index e0fc5b234..bd50efc67 100644
--- a/apps/api/src/services/monitoring/queue.ts
+++ b/apps/api/src/services/monitoring/queue.ts
@@ -28,6 +28,12 @@ let connection: amqp.ChannelModel | null = null;
// other.
let publishChannel: amqp.Channel | null = null;
let consumeChannel: amqp.Channel | null = null;
+// In-flight creation promises. Channel creation suspends on async asserts before
+// the channel is assigned, so two concurrent callers (e.g. the two consume
+// registrations started together at worker boot) would each create a channel and
+// orphan one. Memoizing the promise serializes concurrent callers onto one.
+let publishChannelPromise: Promise<amqp.Channel> | null = null;
+let consumeChannelPromise: Promise<amqp.Channel> | null = null;
// Registry of every consumer registered this process, so we can re-attach them
// after a connection/channel drop. Without this the worker — which never
@@ -92,7 +98,9 @@ function handleConnectionDrop(reason: string, error?: unknown): void {
}
connection = null;
publishChannel = null;
+ publishChannelPromise = null;
consumeChannel = null;
+ consumeChannelPromise = null;
subscribedQueues.clear();
if (registeredConsumers.length > 0) {
scheduleReconnect(RECONNECT_BASE_DELAY_MS);
@@ -120,9 +128,18 @@ function scheduleReconnect(delayMs: number): void {
error,
nextDelayMs: nextDelay,
});
+ // Best-effort close the half-set-up channel, then reset ALL cached state —
+ // including subscribedQueues. If a mid-loop subscribe() failed on an
+ // otherwise-healthy channel (so its "close" never fires), leaving
+ // subscribedQueues populated would make the next reconnect skip that queue
+ // (subscribe() early-returns on subscribedQueues.has) and deafen it forever.
+ await consumeChannel?.close().catch(() => {});
connection = null;
publishChannel = null;
+ publishChannelPromise = null;
consumeChannel = null;
+ consumeChannelPromise = null;
+ subscribedQueues.clear();
scheduleReconnect(nextDelay);
}
}, delayMs);
@@ -157,9 +174,13 @@ async function createChannel(label: string): Promise<amqp.Channel> {
// PRECONDITION_FAILED from assertQueue). Without these handlers the cached
// channel would stay non-null and every send/consume would throw forever.
ch.on("close", () => {
- if (label === "publish" && publishChannel === ch) publishChannel = null;
+ if (label === "publish" && publishChannel === ch) {
+ publishChannel = null;
+ publishChannelPromise = null;
+ }
if (label === "consume" && consumeChannel === ch) {
consumeChannel = null;
+ consumeChannelPromise = null;
handleConnectionDrop("consume channel closed");
}
});
@@ -171,14 +192,34 @@ async function createChannel(label: string): Promise<amqp.Channel> {
async function getPublishChannel(): Promise<amqp.Channel> {
if (publishChannel) return publishChannel;
- publishChannel = await createChannel("publish");
- return publishChannel;
+ if (!publishChannelPromise) {
+ publishChannelPromise = createChannel("publish")
+ .then(ch => {
+ publishChannel = ch;
+ return ch;
+ })
+ .catch(err => {
+ publishChannelPromise = null;
+ throw err;
+ });
+ }
+ return publishChannelPromise;
}
async function getConsumeChannel(): Promise<amqp.Channel> {
if (consumeChannel) return consumeChannel;
- consumeChannel = await createChannel("consume");
- return consumeChannel;
+ if (!consumeChannelPromise) {
+ consumeChannelPromise = createChannel("consume")
+ .then(ch => {
+ consumeChannel = ch;
+ return ch;
+ })
+ .catch(err => {
+ consumeChannelPromise = null;
+ throw err;
+ });
+ }
+ return consumeChannelPromise;
}
export async function addMonitorCheckJob(
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.ts
index 586ff26b2..b7db2a6ee 100644
--- a/apps/api/src/services/monitoring/runner.test.ts
+++ b/apps/api/src/services/monitoring/runner.test.ts
@@ -91,6 +91,20 @@ describe("monitoring runner", () => {
now,
),
).toBe(true);
+ // MIXED search+crawl: keeps the 1-hour timeout (the crawl can legitimately
+ // run for many minutes) — must NOT be reaped at 11 minutes.
+ expect(
+ isMonitorCheckStale(
+ {
+ ...base,
+ target_results: [
+ { type: "search", targetId: "t1" },
+ { type: "crawl", targetId: "t2" },
+ ],
+ },
+ now,
+ ),
+ ).toBe(false);
});
});
});
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index b67883e79..1c627e893 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -332,11 +332,11 @@ async function runSingleScrape(params: {
};
}
-// Deep-mode search-monitor page scrape. Routes through the same internal
-// scrape path as every other monitor scrape (processJobInternal, skipNuq,
-// bypassBilling) instead of calling scrapeURL directly, so it respects the
-// team's concurrency queue and is never billed per-page (search monitors bill
-// flat at the check level).
+// Deep-mode search-monitor page scrape. Runs inline via processJobInternal
+// (skipNuq:true, bypassBilling) — it is NOT routed through NuQ, so it bypasses
+// per-team/global scrape concurrency; the caller bounds the fan-out via
+// SEARCH_SCRAPE_CONCURRENCY (search/run.ts). Never billed per-page (search
+// monitors bill flat at the check level).
async function scrapeSearchMonitorPage(params: {
teamId: string;
checkId: string;
@@ -1337,6 +1337,14 @@ export async function processMonitorCheckJob(
// Set LAST, after credits are stamped, so the reconciler never finalizes
// with credits still at 0.
targetRun.searchCompleted = true;
+ // Persist immediately (not just at the end of the loop): the search work
+ // — external search calls + per-page scrapes — already happened. If we
+ // crash before the end-of-loop flush, a redelivery would re-run and
+ // re-scrape everything. Flushing searchCompleted now lets
+ // findCompletedSearchTargetRun short-circuit the redelivery.
+ await updateMonitorCheck(check.id, {
+ target_results: targetResults,
+ });
}
}
diff --git a/apps/api/src/services/monitoring/search/billing.ts b/apps/api/src/services/monitoring/search/billing.ts
index a8dfbb933..cca5cf7e0 100644
--- a/apps/api/src/services/monitoring/search/billing.ts
+++ b/apps/api/src/services/monitoring/search/billing.ts
@@ -1,6 +1,9 @@
const SEARCH_CREDITS_PER_TEN_RESULTS = 2;
const SEARCH_CREDITS_PER_TEN_RESULTS_ZDR = 10;
-const SEARCH_JUDGE_CREDITS_PER_RESULT = 1;
+// The single source of truth for the per-judged-result judge credit rate.
+// Tests MUST import this rather than re-declaring a literal (which previously
+// drifted to 5 and silently misrepresented billing).
+export const SEARCH_JUDGE_CREDITS_PER_RESULT = 1;
export function searchCreditsForResultCount(
rawResultCount: number,
diff --git a/apps/api/src/services/monitoring/search/cost.test.ts b/apps/api/src/services/monitoring/search/cost.test.ts
index 3077faf5a..88ab71569 100644
--- a/apps/api/src/services/monitoring/search/cost.test.ts
+++ b/apps/api/src/services/monitoring/search/cost.test.ts
@@ -1,7 +1,9 @@
import { CostTracking } from "../../../lib/cost-tracking";
import { recordLlmCall } from "./cost";
-
-const JUDGE_CREDITS_PER_RESULT = 5;
+import {
+ SEARCH_JUDGE_CREDITS_PER_RESULT,
+ judgeCreditsForJudgedCount,
+} from "./billing";
describe("search-monitor LLM cost recording (observability only)", () => {
it("records token usage into the shared CostTracking with no dollar cost", () => {
@@ -15,7 +17,10 @@ describe("search-monitor LLM cost recording (observability only)", () => {
const json = ct.toJSON();
// Tokens recorded for observability; cost stays 0 (flat per-result billing).
expect(json.totalCost).toBe(0);
- expect(json.calls[0].tokens).toEqual({ input: 1_000_000, output: 1_000_000 });
+ expect(json.calls[0].tokens).toEqual({
+ input: 1_000_000,
+ output: 1_000_000,
+ });
});
it("normalizes promptTokens/completionTokens (older SDK usage shape)", () => {
@@ -26,28 +31,33 @@ describe("search-monitor LLM cost recording (observability only)", () => {
usage: { promptTokens: 1_000_000, completionTokens: 0 },
stage: "summarizeRun",
});
- expect(ct.toJSON().calls[0].tokens).toEqual({ input: 1_000_000, output: 0 });
+ expect(ct.toJSON().calls[0].tokens).toEqual({
+ input: 1_000_000,
+ output: 0,
+ });
});
});
describe("search-monitor FLAT judge billing rate", () => {
- it("is a deterministic 5 credits per judged result", () => {
- expect(JUDGE_CREDITS_PER_RESULT).toBe(5);
+ it("is a deterministic 1 credit per judged result", () => {
+ expect(SEARCH_JUDGE_CREDITS_PER_RESULT).toBe(1);
});
- it("computes judge credits as 5 * resultsJudged", () => {
- const judgeCreditsFor = (resultsJudged: number) =>
- resultsJudged * JUDGE_CREDITS_PER_RESULT;
- expect(judgeCreditsFor(0)).toBe(0); // raw / judge-off
- expect(judgeCreditsFor(3)).toBe(15); // deep, 3 results judged
- expect(judgeCreditsFor(10)).toBe(50);
+ it("computes judge credits as 1 * resultsJudged", () => {
+ expect(judgeCreditsForJudgedCount(0)).toBe(0); // raw / judge-off
+ expect(judgeCreditsForJudgedCount(3)).toBe(3); // deep, 3 results judged
+ expect(judgeCreditsForJudgedCount(10)).toBe(10);
});
});
// Billed count == pages with judgedThisRun=true (not every page, not every page
// with a concept — reused/skipped pages keep a stale concept but aren't judged).
describe("canonical judged-result count (billing == persisted signal)", () => {
- type Page = { searchStatus: string; judgedThisRun?: boolean; concept?: string };
+ type Page = {
+ searchStatus: string;
+ judgedThisRun?: boolean;
+ concept?: string;
+ };
const billedJudgedCount = (pages: Page[]) =>
pages.filter(p => p.judgedThisRun === true).length;
@@ -55,13 +65,17 @@ describe("canonical judged-result count (billing == persisted signal)", () => {
// 1 judged this run; 2 reused (stale concept, not judged); 1 skipped.
const pages: Page[] = [
{ searchStatus: "alert", judgedThisRun: true, concept: "fresh-verdict" },
- { searchStatus: "already_seen", judgedThisRun: false, concept: "stale-1" },
+ {
+ searchStatus: "already_seen",
+ judgedThisRun: false,
+ concept: "stale-1",
+ },
{ searchStatus: "watching", judgedThisRun: false, concept: "stale-2" },
{ searchStatus: "skipped" }, // no verdict, no judgedThisRun
];
expect(pages.filter(p => p.concept).length).toBe(3); // the misleading count
expect(billedJudgedCount(pages)).toBe(1); // the canonical, billed count
- expect(billedJudgedCount(pages) * JUDGE_CREDITS_PER_RESULT).toBe(5);
+ expect(judgeCreditsForJudgedCount(billedJudgedCount(pages))).toBe(1);
});
it("counts judged results across every billed outcome (alert/watch/ignore/already_seen)", () => {
@@ -72,6 +86,6 @@ describe("canonical judged-result count (billing == persisted signal)", () => {
{ searchStatus: "already_seen", judgedThisRun: true },
{ searchStatus: "skipped" }, // verdict failed -> not billed
];
- expect(billedJudgedCount(pages)).toBe(4); // -> 5*4 = 20 judge credits
+ expect(billedJudgedCount(pages)).toBe(4); // -> 1*4 = 4 judge credits
});
});
diff --git a/apps/api/src/services/monitoring/search/criteria.ts b/apps/api/src/services/monitoring/search/criteria.ts
deleted file mode 100644
index eac3d63a8..000000000
--- a/apps/api/src/services/monitoring/search/criteria.ts
+++ /dev/null
@@ -1,123 +0,0 @@
-import { z } from "zod";
-
-export type GoalCriteria = {
- goalVersion: string;
- generatedBy: "deterministic" | "llm";
- subjectAliases: string[];
- mustConcern: string[];
- excludedSubjects: string[];
- ownedHosts: string[];
- thirdPartyOnly: boolean;
-};
-
-export function tokenizeContent(text: string | null | undefined): string[] {
- return String(text ?? "")
- .toLowerCase()
- .split(/[^a-z0-9]+/)
- .filter(token => token.length >= 2);
-}
-
-function normalizeForContainment(text: string | null | undefined): string {
- return ` ${String(text ?? "")
- .toLowerCase()
- .replace(/[^a-z0-9]+/g, " ")
- .trim()} `;
-}
-
-export function containsAlias(
- text: string | null | undefined,
- alias: string,
-): boolean {
- const haystack = normalizeForContainment(text);
- const needle = normalizeForContainment(alias).trim();
- return needle.length > 0 && haystack.includes(` ${needle} `);
-}
-
-export function compileGoalCriteria(params: {
- goal: string;
- subject: string;
- goalVersion: string;
-}): GoalCriteria {
- const subject = params.subject.trim();
- return {
- goalVersion: params.goalVersion,
- generatedBy: "deterministic",
- subjectAliases: subject ? [subject] : [],
- mustConcern: [],
- excludedSubjects: [],
- ownedHosts: [],
- thirdPartyOnly: false,
- };
-}
-
-const criteriaSchema = z.object({
- subjectAliases: z
- .array(z.string())
- .describe(
- "Every name the monitored subject goes by, INCLUDING its well-known products and brand names. Empty if the goal has no specific subject.",
- ),
- mustConcern: z
- .array(z.string())
- .describe(
- "Single lowercase content words that name the goal's SPECIFIC topic/scope — the subject matter a matching story must actually be about, not just the broad field. Derived from the goal's topic, not its phrasing. For a goal like 'AI safety or alignment research', use the precise topic words (safety, alignment, interpretability, evaluation) rather than broad-field words (ai, research) that adjacent off-topic stories would also share. Matching is EXACT string comparison, so include inflection variants explicitly (e.g. release, releases, released, launch, launches, launched).",
- ),
- excludedSubjects: z
- .array(z.string())
- .describe(
- "Entities whose OWN news must not trigger this monitor (e.g. competitors merely name-dropping the subject). Empty when the goal doesn't imply any.",
- ),
- ownedHosts: z
- .array(z.string())
- .describe(
- "Hostnames operated by the subject itself (official site, docs, blog).",
- ),
- thirdPartyOnly: z
- .boolean()
- .describe(
- "true ONLY when the goal asks for third-party coverage of the subject (mentions, press, discussion, reviews) — the subject's own pages then don't satisfy it. false when the subject's own announcements DO satisfy the goal (releases, launches, filings, pricing changes, changelogs).",
- ),
-});
-
-export function mergeCompiledCriteria(
- deterministic: GoalCriteria,
- llm: z.infer<typeof criteriaSchema>,
-): GoalCriteria {
- const merged: GoalCriteria = {
- ...deterministic,
- generatedBy: "llm",
- subjectAliases: [
- ...new Set([
- ...deterministic.subjectAliases,
- ...llm.subjectAliases.map(s => s.trim()).filter(Boolean),
- ]),
- ],
- mustConcern: [...new Set(llm.mustConcern.flatMap(tokenizeContent))],
- excludedSubjects: [
- ...new Set(llm.excludedSubjects.map(s => s.trim()).filter(Boolean)),
- ],
- ownedHosts: [
- ...new Set(
- llm.ownedHosts
- .map(
- h =>
- String(h ?? "")
- .trim()
- .toLowerCase()
- .replace(/^https?:\/\//, "")
- .replace(/^www\./, "")
- .split("/")[0]
- .split(":")[0],
- )
- .filter(Boolean),
- ),
- ],
- thirdPartyOnly: llm.thirdPartyOnly === true,
- };
- merged.excludedSubjects = merged.excludedSubjects.filter(
- excluded =>
- !merged.subjectAliases.some(
- alias => alias.toLowerCase() === excluded.toLowerCase(),
- ),
- );
- return merged;
-}
diff --git a/apps/api/src/services/monitoring/search/llm.ts b/apps/api/src/services/monitoring/search/llm.ts
deleted file mode 100644
index 5dd78a88e..000000000
--- a/apps/api/src/services/monitoring/search/llm.ts
+++ /dev/null
@@ -1,74 +0,0 @@
-import { generateObject } from "ai";
-import { z } from "zod";
-import {
- googleModel,
- googleProviderOptions,
- type LlmUsageLabels,
-} from "./tuning";
-import { recordLlmCall } from "./cost";
-import type { CostTracking } from "../../../lib/cost-tracking";
-
-const ROUTER_MODEL = "gemini-3-flash-preview";
-
-export type KnownEvent = {
- key: string;
- label: string;
- satisfiedAt?: string;
- alertCount?: number;
-};
-
-const snippetVerdictSchema = z.object({
- verdicts: z.array(
- z.object({
- id: z.string(),
- relevant: z.boolean(),
- alertAction: z.enum(["alert", "watch", "ignore"]),
- concept: z.string(),
- rationale: z.string(),
- }),
- ),
-});
-
-export type SnippetVerdict = z.infer<
- typeof snippetVerdictSchema
->["verdicts"][number];
-
-export async function judgeSnippets(params: {
- goal: string;
- subject: string;
- searchWindow: string;
- candidates: Array<{
- id: string;
- query: string;
- title: string;
- url: string;
- snippet: string;
- }>;
- costTracking?: CostTracking;
- labels?: LlmUsageLabels;
-}): Promise<SnippetVerdict[]> {
- if (params.candidates.length === 0) return [];
- const { object, usage } = await generateObject({
- model: googleModel(ROUTER_MODEL),
- schema: snippetVerdictSchema,
- system:
- "You judge search results for a monitoring product using ONLY each result's SERP title, URL, and snippet — you cannot see the page. Be conservative: a snippet is thin evidence, so alertAction alert requires the snippet itself to concretely state a completed, recent event satisfying the goal, from a source credible for that claim (the subject itself or established reporting). The goal names a SPECIFIC topic/scope: the result's core subject must actually match that specific subject matter, not merely be related, adjacent, or in the same broad field — a credible source or the right organization on an adjacent topic is NOT enough. Treat any 'Ignore/except/not/exclude' clause in the goal as a hard exclusion. Query wording echoed in a snippet is not evidence. Rumors, content farms, competitors, look-alikes, listings, and old or unconfirmed results are watch/ignore. concept: a short reusable label naming the real-world event; describe what the result is actually about, do not reword it to match the goal. rationale: concrete snippet facts only, never references to the monitor or goal. Return one verdict per candidate id. Structured JSON only.",
- prompt: JSON.stringify({
- monitor: { goal: params.goal, subject: params.subject },
- searchWindow: params.searchWindow,
- candidates: params.candidates.slice(0, 50),
- }),
- temperature: 0,
- ...googleProviderOptions("judgeSnippets", params.labels),
- });
- if (params.costTracking) {
- recordLlmCall({
- costTracking: params.costTracking,
- model: ROUTER_MODEL,
- usage,
- stage: "judgeSnippets",
- });
- }
- const validIds = new Set(params.candidates.map(c => c.id));
- return object.verdicts.filter(v => validIds.has(v.id));
-}
diff --git a/apps/api/src/services/monitoring/search/persist.ts b/apps/api/src/services/monitoring/search/persist.ts
index 6d9680742..74390bb75 100644
--- a/apps/api/src/services/monitoring/search/persist.ts
+++ b/apps/api/src/services/monitoring/search/persist.ts
@@ -1,6 +1,5 @@
import { canonicalizeUrl } from "./dedupe";
-import type { KnownPage } from "./run";
-import type { KnownEvent } from "./llm";
+import type { KnownPage, KnownEvent } from "./run";
export function searchStatusToPageStatus(
status: string,
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
index 65547194f..850fdf497 100644
--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -4,12 +4,11 @@ import { canonicalizeUrl } from "./dedupe";
// vi.mock is hoisted above declarations, so the mocks its factories reference
// are created in vi.hoisted() (also hoisted) to avoid any TDZ surprises.
-// The lean pipeline has only one LLM stage left in deep mode (the per-page JSON
-// verdict, produced by the injected scrapePage) plus judgeSnippets for standard.
-const { searchMock, scrapeURLMock, snippetsMock } = vi.hoisted(() => ({
+// The lean pipeline has a single LLM stage: the deep-mode per-page JSON verdict,
+// produced by the injected scrapePage.
+const { searchMock, scrapeURLMock } = vi.hoisted(() => ({
searchMock: vi.fn(),
scrapeURLMock: vi.fn(),
- snippetsMock: vi.fn(),
}));
vi.mock("uuid", () => ({ v7: () => "00000000-0000-7000-8000-000000000000" }));
@@ -21,9 +20,6 @@ vi.mock("../../../search/v2", () => ({
return Array.isArray(r) ? { web: r } : r;
},
}));
-vi.mock("./llm", () => ({
- judgeSnippets: (...a: unknown[]) => snippetsMock(...a),
-}));
vi.mock("./tuning", () => ({
hasLlmProvider: () => true,
googleProviderOptions: () => ({}),
@@ -122,36 +118,6 @@ describe("deep mode (scrape every selected candidate)", () => {
});
});
-describe("standard depth (snippet judging, no scrapes)", () => {
- it("judges from snippets without any page fetch", async () => {
- searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
- snippetsMock.mockResolvedValue([
- { id: "result_1", ...verdict() },
- { id: "result_2", ...verdict({ alertAction: "watch" }) },
- ]);
-
- const result = await runSearchTarget(runParams({ depth: "standard" }));
- expect(scrapeURLMock).not.toHaveBeenCalled();
- expect(result.pagesChecked).toBe(2);
- expect(result.matches).toBe(1);
- expect(result.sources.map(s => s.status).sort()).toEqual([
- "alert",
- "watching",
- ]);
- // snippet-only judgments never bill a scrape
- expect(result.pageUpserts.every(u => u.scraped === false)).toBe(true);
- });
-
- it("skips results the snippet judge failed to cover", async () => {
- searchMock.mockResolvedValue([serpRow(1)]);
- snippetsMock.mockRejectedValue(new Error("batch failed"));
-
- const result = await runSearchTarget(runParams({ depth: "standard" }));
- expect(result.skipped).toBe(1);
- expect(result.matches).toBe(0);
- });
-});
-
describe("alert boundary (the per-page JSON verdict is the gate)", () => {
beforeEach(() => {
searchMock.mockResolvedValue([serpRow(1)]);
@@ -265,8 +231,7 @@ describe("judgeEnabled gates the LLM judge", () => {
runParams({ depth: "deep" }, { judgeEnabled: false }),
);
- // No judge stage ran: raw mode neither snippet-judges nor scrapes.
- expect(snippetsMock).not.toHaveBeenCalled();
+ // No judge stage ran: raw mode does not scrape.
expect(scrapeURLMock).not.toHaveBeenCalled();
// No judge credits billed (raw → nothing judged); search still ran and was billed.
@@ -410,16 +375,51 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",
expect(scrapeURLMock).not.toHaveBeenCalled();
});
- it("degraded=true when the standard snippet judge fails for every candidate", async () => {
- searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
- snippetsMock.mockRejectedValue(new Error("snippet judge outage"));
+ it("flags partialScrapeLoss (soft signal, NOT degraded) when most scrapes fail but some judge", async () => {
+ searchMock.mockResolvedValue([
+ serpRow(1),
+ serpRow(2),
+ serpRow(3),
+ serpRow(4),
+ ]);
+ // Only result 1 scrapes + judges; results 2-4 all fail to scrape.
+ scrapeURLMock.mockImplementation(({ url }: { url: string }) =>
+ url === serpRow(1).url
+ ? Promise.resolve({
+ success: true,
+ document: {
+ json: verdict({ alertAction: "ignore" }),
+ markdown: "prose",
+ metadata: {},
+ },
+ })
+ : Promise.resolve({ success: false }),
+ );
- const result = await runSearchTarget(runParams({ depth: "standard" }));
+ const result = await runSearchTarget(runParams({ depth: "deep" }));
- expect(result.resultsJudged).toBe(0);
- expect(result.matches).toBe(0);
- expect(result.judgeDegraded).toBe(true);
- expect(result.degradedReason).toMatch(/judged|incomplete/i);
+ expect(result.resultsJudged).toBe(1);
+ expect(result.scrapeFailures).toBe(3);
+ // 3/(1+3) = 0.75 >= 0.5 -> soft partial-loss signal, but the run still
+ // produced a verdict so it is NOT a hard degrade.
+ expect(result.partialScrapeLoss).toBe(true);
+ expect(result.judgeDegraded).toBe(false);
+ });
+
+ it("does NOT flag partialScrapeLoss when every scrape succeeds", async () => {
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict({ alertAction: "ignore" }),
+ markdown: "prose",
+ metadata: {},
+ },
+ });
+
+ const result = await runSearchTarget(runParams({ depth: "deep" }));
+
+ expect(result.scrapeFailures).toBe(0);
+ expect(result.partialScrapeLoss).toBe(false);
});
});
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.ts
index 904bb269c..91839be11 100644
--- a/apps/api/src/services/monitoring/search/run.test.ts
+++ b/apps/api/src/services/monitoring/search/run.test.ts
@@ -24,8 +24,7 @@ vi.mock("../../../search/v2", () => ({
},
}));
// Deep mode (these tests) scrapes + parses a per-page JSON verdict and dedups by
-// canonical URL — no LLM chain. judgeSnippets (standard mode) is never reached.
-vi.mock("./llm", () => ({ judgeSnippets: vi.fn() }));
+// canonical URL — the only LLM stage is the injected per-page scrape verdict.
vi.mock("./tuning", () => ({
hasLlmProvider: () => false,
googleProviderOptions: () => ({}),
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index ad526bbe4..3b37f21b9 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -1,7 +1,6 @@
import type { Logger } from "winston";
import { search } from "../../../search/v2";
import { buildSearchQuery } from "../../../lib/search-query-builder";
-import { CostTracking } from "../../../lib/cost-tracking";
import { hashMonitorUrl } from "../store";
import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";
import {
@@ -11,13 +10,6 @@ import {
windowToMs,
type SearchVerdict,
} from "./judge";
-import { judgeSnippets, type KnownEvent, type SnippetVerdict } from "./llm";
-import { compileGoalCriteria, type GoalCriteria } from "./criteria";
-import {
- verifyAlertCandidate,
- type VerificationResult,
- type VerifyEvidence,
-} from "./verify";
import { hasLlmProvider } from "./tuning";
import {
searchCreditsForResultCount,
@@ -78,6 +70,31 @@ const SEARCH_RETRY_BACKOFF_MS = [400, 1200, 3000, 6000] as const;
// 10-minute search stale timeout so a run always returns before the reaper.
const SEARCH_RUN_BUDGET_MS = 4 * 60 * 1000;
+// Deep-mode page scrapes run INLINE in the worker process (skipNuq), so they are
+// NOT bounded by NuQ's per-team/global concurrency. Cap the pre-scrape fan-out
+// ourselves so a single maxResults=50 check can't launch 50 simultaneous
+// fetch+LLM-extraction scrapes and starve the worker that owns the search consumer.
+const SEARCH_SCRAPE_CONCURRENCY = 6;
+
+// Run fn over items with at most `limit` in flight at once. Never rejects per the
+// caller's needs — fn is expected to swallow its own errors (it writes outcomes
+// into a shared cache).
+async function mapWithConcurrency<T>(
+ items: T[],
+ limit: number,
+ fn: (item: T) => Promise<void>,
+): Promise<void> {
+ let next = 0;
+ const worker = async (): Promise<void> => {
+ while (next < items.length) {
+ const i = next++;
+ await fn(items[i]);
+ }
+ };
+ const pool = Array.from({ length: Math.min(limit, items.length) }, worker);
+ await Promise.all(pool);
+}
+
// Race a promise against a wall-clock deadline. Resolves false if the work
// finishes first, true if the deadline hits. Never rejects; the underlying work
// keeps running (bounded by its own per-scrape timeout) and populates its cache.
@@ -170,6 +187,13 @@ export type KnownPage = {
metadata?: Record<string, unknown>;
};
+export type KnownEvent = {
+ key: string;
+ label: string;
+ satisfiedAt?: string;
+ alertCount?: number;
+};
+
// Deep-mode page scrape. Injected by the caller so it can route through the
// shared monitor scrape path (concurrency queue + bypassBilling), keeping this
// module free of worker/queue dependencies. Resolves to null on scrape failure.
@@ -205,6 +229,11 @@ type SearchTargetRunResult = {
summary: string;
judgeDegraded: boolean;
degradedReason: string | null;
+ // Per-page scrape failures in deep mode. partialScrapeLoss is a SOFT signal
+ // (distinct from judgeDegraded): some results judged, but a majority of the
+ // attempted scrapes failed — the run is usable but likely missed developments.
+ scrapeFailures: number;
+ partialScrapeLoss: boolean;
sources: SearchSource[];
searchCredits: number;
judgeCredits: number;
@@ -250,13 +279,6 @@ export async function runSearchTarget(params: {
throw new Error("search monitor target requires a non-empty monitor goal");
}
const subject = params.monitor.subject ?? "";
- const teamId = params.monitor.teamId;
- // Vertex billing labels attached to every LLM call for usage tracing.
- const labels = {
- teamId,
- monitorId: params.monitor.id,
- monitorCheckId: params.monitorCheckId,
- };
const tbs = windowToTbs(target.searchWindow);
const events: KnownEvent[] = [...params.knownEvents];
@@ -267,11 +289,9 @@ export async function runSearchTarget(params: {
let pagesChecked = 0;
let skipped = 0;
let matches = 0;
- const costTracking = new CostTracking();
let searchResultsBilled = 0;
let resultsJudged = 0;
let judgeScrapeFailures = 0;
- let snippetJudgeFailed = false;
const seenThisRun = new Map<string, number>();
const candidates: Array<{
@@ -335,96 +355,22 @@ export async function runSearchTarget(params: {
}
const llmStagesAvailable = hasLlmProvider();
- const depth: "raw" | "standard" | "deep" = !judgeEnabled
- ? "raw"
- : target.depth === "raw"
- ? "raw"
- : target.depth === "standard" && llmStagesAvailable
- ? "standard"
- : "deep";
+ // Judge depth is raw (no LLM) or deep (per-page scrape + verdict). Standard
+ // (snippet-only judging) was removed: it was unreachable via the API (depth is
+ // stripped) and used a divergent LLM provider; deep is the single judged path.
+ const depth: "raw" | "deep" =
+ !judgeEnabled || target.depth === "raw" ? "raw" : "deep";
const judgeGoal: string = goal ?? "";
- // Deterministic criteria for the (non-LLM) verifier — no LLM enrich/router.
- const criteria: GoalCriteria = compileGoalCriteria({
- goal: judgeGoal,
- subject,
- goalVersion,
- });
-
const selected = candidates.slice(0, target.maxResults);
const recheckMs = target.recheckAfter ? windowToMs(target.recheckAfter) : 0;
- const pageNeedsJudgment = (c: {
- url: string;
- title: string;
- description: string;
- }) => {
- const canonical = canonicalizeUrl(c.url);
- const known = knownPages.get(canonical);
- const knownCurrent =
- known && known.goalVersion === goalVersion ? known : undefined;
- if (!knownCurrent) return true;
- const fingerprint = stableSerpFingerprint({
- url: c.url,
- title: c.title,
- snippet: c.description,
- });
- if (knownCurrent.fingerprint !== fingerprint) return true;
- const isLive =
- knownCurrent.lastStatus === "alert" ||
- knownCurrent.lastStatus === "watching";
- return Boolean(
- recheckMs &&
- isLive &&
- knownCurrent.lastCheckedAt &&
- Date.now() - Date.parse(knownCurrent.lastCheckedAt) > recheckMs,
- );
- };
-
- const snippetVerdicts = new Map<string, SearchVerdict>();
- const snippetCandidates = selected.filter(pageNeedsJudgment);
- if (depth === "standard" && snippetCandidates.length > 0) {
- try {
- const verdicts = await judgeSnippets({
- goal: judgeGoal,
- subject,
- searchWindow: target.searchWindow,
- costTracking,
- labels,
- candidates: snippetCandidates.map((c, i) => ({
- id: `result_${i + 1}`,
- query: c.query,
- title: c.title,
- url: c.url,
- snippet: c.description,
- })),
- });
- const byId = new Map<string, SnippetVerdict>(
- verdicts.map(v => [v.id, v]),
- );
- snippetCandidates.forEach((c, i) => {
- const v = byId.get(`result_${i + 1}`);
- if (v) {
- snippetVerdicts.set(canonicalizeUrl(c.url), {
- relevant: v.relevant,
- alertAction: v.alertAction,
- concept: v.concept,
- rationale: v.rationale,
- });
- }
- });
- } catch (error) {
- snippetJudgeFailed = true;
- logger.warn("search monitor snippet judge failed; results skipped", {
- error,
- });
- }
- }
- // Pre-scrape deep-mode pages concurrently and cache them, so the loop below isn't
- // a serial scrape→judge→scrape chain. Only the I/O is parallel; all dedup/event
- // state stays in the single-threaded loop. (NuQ bounds the actual scrape fan-out.)
+ // Pre-scrape deep-mode pages (bounded concurrency) and cache them, so the loop
+ // below isn't a serial scrape→judge→scrape chain. Only the I/O is parallel; all
+ // dedup/event state stays in the single-threaded loop. These scrapes run inline
+ // (skipNuq), so the fan-out is bounded by SEARCH_SCRAPE_CONCURRENCY, not NuQ.
const deepDocs = new Map<
string,
{ doc: ScrapeSearchResult | null; error?: unknown }
@@ -439,23 +385,24 @@ export async function runSearchTarget(params: {
subject,
target.searchWindow,
);
- const preScrape = Promise.all(
- selected
- .filter(
- c =>
- evaluateSerpCandidate(c, knownPages, goalVersion, recheckMs)
- .isNewOrChanged,
- )
- .map(async c => {
- const canonical = canonicalizeUrl(c.url);
- try {
- deepDocs.set(canonical, {
- doc: await params.scrapePage({ url: c.url, judgePrompt }),
- });
- } catch (error) {
- deepDocs.set(canonical, { doc: null, error });
- }
- }),
+ const toScrape = selected.filter(
+ c =>
+ evaluateSerpCandidate(c, knownPages, goalVersion, recheckMs)
+ .isNewOrChanged,
+ );
+ const preScrape = mapWithConcurrency(
+ toScrape,
+ SEARCH_SCRAPE_CONCURRENCY,
+ async c => {
+ const canonical = canonicalizeUrl(c.url);
+ try {
+ deepDocs.set(canonical, {
+ doc: await params.scrapePage({ url: c.url, judgePrompt }),
+ });
+ } catch (error) {
+ deepDocs.set(canonical, { doc: null, error });
+ }
+ },
);
const remainingBudget = SEARCH_RUN_BUDGET_MS - (Date.now() - runStart);
budgetExceeded = await raceDeadline(preScrape, remainingBudget);
@@ -589,22 +536,9 @@ export async function runSearchTarget(params: {
}
let verdict: SearchVerdict | null = null;
- let pageText = "";
let realDate: string | null = null;
- if (depth === "standard") {
- verdict = snippetVerdicts.get(canonical) ?? null;
- if (!verdict) {
- if (snippetJudgeFailed) {
- pushUnevaluatedPage("snippet judge failed");
- } else {
- skipped += 1;
- sources.push({ url: c.url, title: c.title, status: "skipped" });
- }
- continue;
- }
- pagesChecked += 1;
- } else {
- // Read the doc from the pre-scrape cache; scrape inline only as a fallback.
+ {
+ // Deep mode: read the doc from the pre-scrape cache; scrape inline only as a fallback.
let doc: ScrapeSearchResult | null;
const cached = deepDocs.get(canonical);
if (cached?.error !== undefined) {
@@ -660,37 +594,13 @@ export async function runSearchTarget(params: {
pushUnevaluatedPage("verdict unparseable");
continue;
}
- pageText = doc.markdown ?? "";
realDate =
doc.metadata?.publishedTime ?? doc.metadata?.modifiedTime ?? null;
}
resultsJudged += 1;
- let decision = verdictToDecision(verdict);
-
- let verification: VerificationResult | null = null;
- if (decision === "notify") {
- const evidence: VerifyEvidence = {
- url: c.url,
- titleText: c.title,
- claimText: verdict.rationale,
- pageText,
- };
- const result = verifyAlertCandidate({
- criteria,
- concept: verdict.concept,
- evidence,
- });
- verification = result;
- if (!result.pass) {
- decision = "watch";
- logger.info("search monitor alert downgraded by verifier", {
- url: c.url,
- failures: result.failures,
- });
- }
- }
+ const decision = verdictToDecision(verdict);
const baseMeta = {
fingerprint,
@@ -701,7 +611,6 @@ export async function runSearchTarget(params: {
rationale: verdict.rationale,
matchedQueries: c.matchedQueries,
judgedThisRun: true,
- ...(verification && !verification.pass ? { verification } : {}),
};
if (decision === "ignore") {
@@ -819,14 +728,11 @@ export async function runSearchTarget(params: {
? `${matches} match${matches === 1 ? "" : "es"} across ${resultCount} result${resultCount === 1 ? "" : "s"}.`
: "";
- const deepJudgeFailed =
+ const judgeDegraded =
depth === "deep" &&
candidates.length > 0 &&
resultsJudged === 0 &&
judgeScrapeFailures > 0;
- const standardJudgeFailed =
- depth === "standard" && resultsJudged === 0 && snippetJudgeFailed;
- const judgeDegraded = deepJudgeFailed || standardJudgeFailed;
if (judgeDegraded) {
logger.warn(
"search monitor run is degraded: results were found but none could be judged; results may be incomplete (NOT a clean no-results)",
@@ -836,7 +742,6 @@ export async function runSearchTarget(params: {
depth,
candidates: candidates.length,
scrapeFailures: judgeScrapeFailures,
- snippetJudgeFailed,
resultsJudged,
},
);
@@ -850,6 +755,25 @@ export async function runSearchTarget(params: {
? "results were found but none could be judged; results may be incomplete"
: null;
+ // Soft signal (NOT judgeDegraded): the run produced verdicts, but a majority of
+ // the attempted deep scrapes failed, so it likely missed real developments.
+ // Surfaced for observability; does not fail the check.
+ const partialScrapeLoss =
+ resultsJudged > 0 &&
+ judgeScrapeFailures > 0 &&
+ judgeScrapeFailures / (resultsJudged + judgeScrapeFailures) >= 0.5;
+ if (partialScrapeLoss) {
+ logger.warn(
+ "search monitor run had heavy partial scrape loss: judged some results but most scrapes failed; coverage may be incomplete",
+ {
+ monitorId: params.monitor.id,
+ targetId: target.id,
+ resultsJudged,
+ scrapeFailures: judgeScrapeFailures,
+ },
+ );
+ }
+
const isZDR = params.zeroDataRetention;
const searchCredits = searchCreditsForResultCount(searchResultsBilled, isZDR);
const judgeCredits = judgeCreditsForJudgedCount(resultsJudged);
@@ -865,6 +789,8 @@ export async function runSearchTarget(params: {
summary,
judgeDegraded,
degradedReason,
+ scrapeFailures: judgeScrapeFailures,
+ partialScrapeLoss,
sources,
searchCredits,
judgeCredits,
diff --git a/apps/api/src/services/monitoring/search/verify.test.ts b/apps/api/src/services/monitoring/search/verify.test.ts
deleted file mode 100644
index 220ae2d68..000000000
--- a/apps/api/src/services/monitoring/search/verify.test.ts
+++ /dev/null
@@ -1,204 +0,0 @@
-import {
- compileGoalCriteria,
- mergeCompiledCriteria,
- tokenizeContent,
- type GoalCriteria,
-} from "./criteria";
-import { verifyAlertCandidate } from "./verify";
-
-const baseCriteria = (over: Partial<GoalCriteria> = {}): GoalCriteria => ({
- goalVersion: "v1",
- generatedBy: "llm",
- subjectAliases: ["Firecrawl"],
- mustConcern: ["launch", "launches", "launched", "product"],
- excludedSubjects: ["Parallel"],
- ownedHosts: ["firecrawl.dev"],
- thirdPartyOnly: false,
- ...over,
-});
-
-const evidence = (
- over: Partial<Parameters<typeof verifyAlertCandidate>[0]["evidence"]> = {},
-) => ({
- url: "https://news.example.com/story",
- titleText: "Firecrawl launches a new product",
- claimText: "Firecrawl announced a new product today.",
- pageText: "Firecrawl announced a new product today. Details follow in prose.",
- ...over,
-});
-
-describe("criteria", () => {
- it("merge drops an excluded subject that is also an alias (alias wins)", () => {
- const deterministic = compileGoalCriteria({
- goal: "Alert me when Firecrawl launches a new product",
- subject: "Firecrawl",
- goalVersion: "v1",
- });
- const merged = mergeCompiledCriteria(deterministic, {
- subjectAliases: ["Firecrawl", "FIRE-1"],
- mustConcern: ["launch"],
- excludedSubjects: ["firecrawl", "Parallel"],
- ownedHosts: ["https://www.firecrawl.dev/blog"],
- thirdPartyOnly: false,
- });
- expect(merged.generatedBy).toBe("llm");
- expect(merged.excludedSubjects).toEqual(["Parallel"]);
- expect(merged.ownedHosts).toEqual(["firecrawl.dev"]);
- expect(merged.subjectAliases).toContain("FIRE-1");
- });
-
- it("strips ports when normalizing ownedHosts", () => {
- const deterministic = compileGoalCriteria({
- goal: "Alert me when Firecrawl launches a new product",
- subject: "Firecrawl",
- goalVersion: "v1",
- });
- const merged = mergeCompiledCriteria(deterministic, {
- subjectAliases: ["Firecrawl"],
- mustConcern: [],
- excludedSubjects: [],
- ownedHosts: ["https://firecrawl.dev:3000/blog", "localhost:8080"],
- thirdPartyOnly: true,
- });
- expect(merged.ownedHosts).toEqual(["firecrawl.dev", "localhost"]);
- });
-
- it("tokenizeContent keeps 2-char acronyms (AI, EU, Go) and drops 1-char noise", () => {
- expect(
- tokenizeContent("Go 1.0 released in the EU under the AI Act"),
- ).toEqual([
- "go",
- "released",
- "in",
- "the",
- "eu",
- "under",
- "the",
- "ai",
- "act",
- ]);
- });
-});
-
-describe("verifyAlertCandidate", () => {
- it("passes a clean alert", () => {
- const result = verifyAlertCandidate({
- criteria: baseCriteria(),
- concept: "Firecrawl product launch",
- evidence: evidence(),
- });
- expect(result.pass).toBe(true);
- expect(result.failures).toEqual([]);
- });
-
- it("fails subject containment when the subject never appears in page-backed evidence", () => {
- const result = verifyAlertCandidate({
- criteria: baseCriteria(),
- concept: "Firecrawl product launch",
- evidence: evidence({
- titleText: "A company shipped something",
- claimText: "Some launch happened.",
- pageText: "Some company shipped something today. Details follow.",
- }),
- });
- expect(result.failures.map(f => f.check)).toContain("subject_missing");
- });
-
- it("passes subject containment when the subject appears only in the page body", () => {
- const result = verifyAlertCandidate({
- criteria: baseCriteria(),
- concept: "Firecrawl product launch",
- evidence: evidence({
- titleText: "A scraping startup shipped something",
- claimText: "A new product launched today.",
- pageText: "The company behind Firecrawl announced a new product.",
- }),
- });
- expect(result.failures.map(f => f.check)).not.toContain("subject_missing");
- });
-
- it("skips subject containment for snippet-only results (no page content)", () => {
- const result = verifyAlertCandidate({
- criteria: baseCriteria(),
- concept: "Firecrawl product launch",
- evidence: evidence({
- titleText: "A company shipped something",
- claimText: "Some launch happened.",
- pageText: "",
- }),
- });
- expect(result.failures.map(f => f.check)).not.toContain("subject_missing");
- });
-
- it("fails excluded-subject dominance when a competitor owns the story", () => {
- const result = verifyAlertCandidate({
- criteria: baseCriteria(),
- concept: "Parallel launches Monitor API",
- evidence: evidence({
- titleText: "Parallel launches Monitor API",
- claimText:
- "Parallel launched its Monitor API; Firecrawl is named as a competitor.",
- }),
- });
- expect(result.failures.map(f => f.check)).toContain("excluded_subject");
- });
-
- it("fails concept relevance when the concept shares no goal terms (llm criteria only)", () => {
- const offGoal = verifyAlertCandidate({
- criteria: baseCriteria(),
- concept: "quarterly earnings beat",
- evidence: evidence(),
- });
- expect(offGoal.failures.map(f => f.check)).toContain("concept_off_goal");
-
- const deterministic = verifyAlertCandidate({
- criteria: baseCriteria({ generatedBy: "deterministic" }),
- concept: "quarterly earnings beat",
- evidence: evidence(),
- });
- expect(deterministic.failures.map(f => f.check)).not.toContain(
- "concept_off_goal",
- );
- });
-
- it("downgrades a subject-only concept that misses the topic (funding vs launch)", () => {
- // "Firecrawl funding" names the subject but not the goal's topic (launch);
- // a subject-alias token must NOT satisfy the must-concern gate.
- const result = verifyAlertCandidate({
- criteria: baseCriteria(),
- concept: "Firecrawl Series B funding",
- evidence: evidence(),
- });
- expect(result.failures.map(f => f.check)).toContain("concept_off_goal");
- });
-
- it("fails owned-surface only when the goal wants third-party coverage", () => {
- const owned = verifyAlertCandidate({
- criteria: baseCriteria({ thirdPartyOnly: true }),
- concept: "Firecrawl product launch",
- evidence: evidence({ url: "https://www.firecrawl.dev/blog/launch" }),
- });
- expect(owned.failures.map(f => f.check)).toContain("owned_surface");
-
- const eventGoal = verifyAlertCandidate({
- criteria: baseCriteria({ thirdPartyOnly: false }),
- concept: "Firecrawl product launch",
- evidence: evidence({ url: "https://www.firecrawl.dev/blog/launch" }),
- });
- expect(eventGoal.failures.map(f => f.check)).not.toContain("owned_surface");
- });
-
- it("empty criteria disables every check (fail-open for unknown goal shapes)", () => {
- const result = verifyAlertCandidate({
- criteria: baseCriteria({
- subjectAliases: [],
- mustConcern: [],
- excludedSubjects: [],
- ownedHosts: [],
- }),
- concept: "anything at all",
- evidence: evidence({ titleText: "unrelated", claimText: "unrelated" }),
- });
- expect(result.pass).toBe(true);
- });
-});
diff --git a/apps/api/src/services/monitoring/search/verify.ts b/apps/api/src/services/monitoring/search/verify.ts
deleted file mode 100644
index 82edae5ae..000000000
--- a/apps/api/src/services/monitoring/search/verify.ts
+++ /dev/null
@@ -1,124 +0,0 @@
-import { containsAlias, tokenizeContent, type GoalCriteria } from "./criteria";
-
-export type VerifyEvidence = {
- url: string;
- titleText: string;
- claimText: string;
- pageText: string;
-};
-
-type VerificationFailure = { check: string; detail: string };
-
-export type VerificationResult = {
- pass: boolean;
- failures: VerificationFailure[];
- criteriaSource: GoalCriteria["generatedBy"];
-};
-
-export function verifyAlertCandidate(params: {
- criteria: GoalCriteria;
- evidence: VerifyEvidence;
- concept: string;
-}): VerificationResult {
- const { criteria, evidence } = params;
- const failures: VerificationFailure[] = [];
-
- const pageBackedText = [
- evidence.titleText,
- evidence.claimText,
- String(evidence.pageText ?? "").slice(0, 20000),
- ]
- .filter(Boolean)
- .join(" ");
- const conceptText = String(params.concept ?? "");
- const hasPageContent = Boolean(String(evidence.pageText ?? "").trim());
-
- if (
- criteria.subjectAliases.length > 0 &&
- hasPageContent &&
- pageBackedText.trim()
- ) {
- const present = criteria.subjectAliases.some(
- alias =>
- containsAlias(pageBackedText, alias) ||
- tokenizeContent(alias).some(token =>
- containsAlias(pageBackedText, token),
- ),
- );
- if (!present) {
- failures.push({
- check: "subject_missing",
- detail: `none of [${criteria.subjectAliases.join(", ")}] appear in the page evidence`,
- });
- }
- }
-
- if (criteria.excludedSubjects.length > 0 && conceptText) {
- const conceptNamesAlias = criteria.subjectAliases.some(alias =>
- containsAlias(conceptText, alias),
- );
- const excludedInConcept = criteria.excludedSubjects.find(
- excluded =>
- containsAlias(conceptText, excluded) ||
- containsAlias(evidence.titleText, excluded),
- );
- if (excludedInConcept && !conceptNamesAlias) {
- failures.push({
- check: "excluded_subject",
- detail: `the story is about "${excludedInConcept}", which this monitor excludes`,
- });
- }
- }
-
- if (
- criteria.generatedBy === "llm" &&
- criteria.mustConcern.length > 0 &&
- conceptText
- ) {
- const conceptTokens = new Set(tokenizeContent(conceptText));
- const topicPresent = criteria.mustConcern.some(token =>
- conceptTokens.has(token),
- );
- if (conceptTokens.size > 0 && !topicPresent) {
- failures.push({
- check: "concept_off_goal",
- detail: `alert concept "${conceptText.slice(0, 80)}" shares no goal terms with the monitor`,
- });
- }
- }
-
- if (criteria.thirdPartyOnly === true && criteria.ownedHosts.length > 0) {
- const host = hostOf(evidence.url);
- const owned =
- host &&
- criteria.ownedHosts.find(
- ownedHost => host === ownedHost || host.endsWith(`.${ownedHost}`),
- );
- if (owned) {
- failures.push({
- check: "owned_surface",
- detail: `${host} is operated by the monitored subject`,
- });
- }
- }
-
- // `subject_missing` is a fragile literal alias-in-page check, so keep it as a
- // diagnostic but don't let it alone suppress an alert — only the stronger checks block.
- const blockingFailures = failures.filter(f => f.check !== "subject_missing");
-
- return {
- pass: blockingFailures.length === 0,
- failures,
- criteriaSource: criteria.generatedBy,
- };
-}
-
-function hostOf(url: string): string {
- try {
- return new URL(String(url ?? "")).hostname
- .toLowerCase()
- .replace(/^www\./, "");
- } catch {
- return "";
- }
-}
diff --git a/apps/api/src/services/monitoring/stale.ts b/apps/api/src/services/monitoring/stale.ts
index 73d217eb5..97c86775c 100644
--- a/apps/api/src/services/monitoring/stale.ts
+++ b/apps/api/src/services/monitoring/stale.ts
@@ -8,11 +8,14 @@ const MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS = 10 * 60 * 1000;
export const MONITOR_CHECK_STALE_ERROR =
"Monitor check exceeded the running timeout.";
-// A check is a search check if any of its target runs is of type "search".
-function isSearchCheck(check: { target_results?: unknown }): boolean {
+// The shorter search timeout only applies when EVERY target is a search target.
+// A mixed monitor (e.g. search + crawl) must keep the long timeout — its scrape
+// fan-out / crawl legitimately runs for many minutes and would otherwise be
+// falsely reaped at the 10-minute search cutoff.
+function isSearchOnlyCheck(check: { target_results?: unknown }): boolean {
const targetResults = check.target_results;
- if (!Array.isArray(targetResults)) return false;
- return targetResults.some(
+ if (!Array.isArray(targetResults) || targetResults.length === 0) return false;
+ return targetResults.every(
tr =>
tr != null &&
typeof tr === "object" &&
@@ -23,7 +26,7 @@ function isSearchCheck(check: { target_results?: unknown }): boolean {
export function monitorCheckStaleTimeoutMs(check: {
target_results?: unknown;
}): number {
- return isSearchCheck(check)
+ return isSearchOnlyCheck(check)
? MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS
: MONITOR_CHECK_STALE_TIMEOUT_MS;
}
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index eecb9e102..2c3469148 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -7,6 +7,7 @@ import {
estimateMonitorCreditsPerRun,
flatSearchTargetCredits,
} from "./store";
+import { judgeCreditsForJudgedCount } from "./search/billing";
import type { MonitorTarget } from "./types";
describe("monitoring store credit helpers", () => {
@@ -255,17 +256,17 @@ describe("FLAT deterministic search-monitor billing", () => {
describe("flatSearchTargetCredits (the deterministic actual)", () => {
it("sums searchCredits + judgeCredits across search target_results", () => {
- // The headline trace: search 2 + judge 5*3 = 17.
+ // The headline trace: search 2 + judge 1*3 = 5.
const targetResults = [
{
targetId: "search-1",
type: "search",
searchCredits: 2,
- judgeCredits: 15,
+ judgeCredits: 3,
resultsJudged: 3,
},
];
- expect(flatSearchTargetCredits(targetResults)).toBe(17);
+ expect(flatSearchTargetCredits(targetResults)).toBe(5);
});
it("raw / judge-off: just the 2 search credits, 0 judge", () => {
@@ -301,7 +302,7 @@ describe("FLAT deterministic search-monitor billing", () => {
});
});
- // Flat billing contract: actual = 2*ceil(totalResults/10) + 5*resultsJudged.
+ // Flat billing contract: actual = 2*ceil(totalResults/10) + 1*resultsJudged.
describe("flat search billing contract (eval H regression guard)", () => {
const searchCreditsFor = (totalResults: number) =>
2 * Math.ceil(totalResults / 10);
@@ -311,19 +312,19 @@ describe("FLAT deterministic search-monitor billing", () => {
searchCompleted: true,
resultCount: totalResults,
searchCredits: searchCreditsFor(totalResults),
- judgeCredits: resultsJudged * 5,
+ judgeCredits: judgeCreditsForJudgedCount(resultsJudged),
resultsJudged,
});
- it("is the exact deterministic flat sum (the headline 17-credit trace)", () => {
- // results=6 -> search 2; judged=3 -> 15; total 17.
- expect(flatSearchTargetCredits([stampedSearchRun(6, 3)])).toBe(17);
+ it("is the exact deterministic flat sum (the headline 5-credit trace)", () => {
+ // results=6 -> search 2; judged=3 -> 3; total 5.
+ expect(flatSearchTargetCredits([stampedSearchRun(6, 3)])).toBe(5);
});
it("never bills 0 on a check that returned results and judged them", () => {
- // results=6, judged=4 -> 2 + 20 = 22 (the judged#2 case that landed on 0).
+ // results=6, judged=4 -> 2 + 4 = 6 (the judged#2 case that landed on 0).
const credits = flatSearchTargetCredits([stampedSearchRun(6, 4)]);
- expect(credits).toBe(22);
+ expect(credits).toBe(6);
expect(credits).toBeGreaterThan(0);
});
@@ -333,7 +334,7 @@ describe("FLAT deterministic search-monitor billing", () => {
// searchCompleted=true guard ensures credits are stamped before summing.
const retried = stampedSearchRun(6, 4);
expect(retried.searchCompleted).toBe(true);
- expect(flatSearchTargetCredits([retried])).toBe(22);
+ expect(flatSearchTargetCredits([retried])).toBe(6);
});
it("search-only (judge off) bills just the search portion, never the judge", () => {
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 8cf931314..e391c8558 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -502,8 +502,15 @@ export async function updateMonitor(params: {
if (params.input.status !== undefined) patch.status = params.input.status;
if (params.input.webhook !== undefined)
patch.webhook = params.input.webhook ?? null;
- if (params.input.notification !== undefined) {
- patch.notification = params.input.notification ?? null;
+ // Defense-in-depth against a wiped notification: only write when the caller
+ // actually sent config. Treat an empty object (the legacy default-materialized
+ // {}) as "leave unchanged" rather than clobbering the stored email settings.
+ if (
+ params.input.notification !== undefined &&
+ params.input.notification !== null &&
+ Object.keys(params.input.notification).length > 0
+ ) {
+ patch.notification = params.input.notification;
}
if (params.input.retentionDays !== undefined) {
patch.retention_days = params.input.retentionDays;
diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.ts
index 8122aa6b6..44c7623ef 100644
--- a/apps/api/src/services/monitoring/types.ts
+++ b/apps/api/src/services/monitoring/types.ts
@@ -129,16 +129,19 @@ const monitorScheduleSchema = z
timezone: schedule.timezone,
}));
-const monitorNotificationSchema = z
- .strictObject({
- email: z
- .strictObject({
- enabled: z.boolean().optional().default(false),
- recipients: z.array(z.email()).max(25).optional().default([]),
- includeDiffs: z.boolean().optional().default(false),
- })
- .optional(),
- })
+const monitorNotificationInner = z.strictObject({
+ email: z
+ .strictObject({
+ enabled: z.boolean().optional().default(false),
+ recipients: z.array(z.email()).max(25).optional().default([]),
+ includeDiffs: z.boolean().optional().default(false),
+ })
+ .optional(),
+});
+// Create defaults a missing notification to {}. The UPDATE schema deliberately
+// does NOT (see updateMonitorSchema) — a default there would materialize {} on
+// any PATCH that omits notification and wipe the stored email config.
+const monitorNotificationSchema = monitorNotificationInner
.optional()
.default({});
@@ -223,6 +226,10 @@ export const updateMonitorSchema = createMonitorBaseSchema
.partial()
.extend({
status: z.enum(["active", "paused"]).optional(),
+ // Override the create-path notification (which defaults to {}): on update a
+ // default would materialize {} for a PATCH that omits notification and wipe
+ // the stored email config. No default => omitted means "leave unchanged".
+ notification: monitorNotificationInner.optional(),
})
.superRefine(rejectGoalClearedWithSearchTargets)
.refine(x => Object.keys(x).length > 0, "Update body cannot be empty");