Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Polish comments and docs in monitoring services
Developers Digest79eaf1f63aba
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.tsindex bd141f9fe..01dee9b74 100644--- a/apps/api/src/controllers/v2/monitor.ts+++ b/apps/api/src/controllers/v2/monitor.ts@@ -376,7 +376,7 @@ export async function updateMonitorController(}),);- // Only re-sync when notification config actually changed.+ // Only re-sync when notification config changed.let subscriptions: Awaited<ReturnType<typeof loadEmailRecipientSubscriptions>> = [];@@ -594,12 +594,11 @@ export async function getMonitorCheckController(});}-// Unauthenticated POST endpoints β the token is the credential. Backs the-// firecrawl-web confirm/unsubscribe pages; POST-only so passive GET scanners-// can't consume tokens against the dashboard URL.+// Unauthenticated: the token is the credential. POST-only so passive GET+// scanners can't consume tokens via the dashboard URL.const emailActionBodySchema = z.object({- // 32-byte base64url is 43 chars; range leaves room for future formats.+ // 32-byte base64url is 43 chars; range leaves room for other formats.token: z.string().min(16).max(64),});@@ -653,8 +652,7 @@ export async function confirmMonitorEmailController(const monitorName = await getMonitorNameById(row.monitor_id);- // Heuristic: if confirmed_at is older than 5s, this call was a no-op- // (already confirmed). Used only to flavor the rendered page.+ // confirmed_at older than 5s means this call was a no-op (already confirmed).let result: "confirmed" | "already_confirmed" | "already_unsubscribed";if (row.status === "unsubscribed") {result = "already_unsubscribed";diff --git a/apps/api/src/services/monitoring/queue.test.ts b/apps/api/src/services/monitoring/queue.test.tsindex ab8191803..3211071ec 100644--- a/apps/api/src/services/monitoring/queue.test.ts+++ b/apps/api/src/services/monitoring/queue.test.ts@@ -1,7 +1,3 @@-// Unit coverage for the branch's core RabbitMQ rework: prefetch(1), the-// subscribe dedup guard, the message ack/nack contract, and β most importantly β-// the reconnect/resubscribe wrapper (the A4 fix: a dropped consumer must-// re-subscribe and never go permanently deaf).import type { Mock } from "vitest";const { connectMock, channelMock, connectionMock } = vi.hoisted(() => {@@ -40,7 +36,6 @@ vi.mock("../../lib/logger", () => {return { logger: mk() };});-// Pull the close handler that createChannel registered via ch.on("close", cb).function consumeChannelCloseHandler(): () => void {const call = (channelMock.on as Mock).mock.calls.find(c => c[0] === "close");return call?.[1] as () => void;@@ -75,7 +70,6 @@ describe("monitoring queue (RabbitMQ wrapper)", () => {await q.consumeMonitorCheckJobs(vi.fn());await q.consumeMonitorCheckJobs(vi.fn());- // subscribedQueues guard => consume attached exactly once for the queue.expect(channelMock.consume).toHaveBeenCalledTimes(1);});@@ -100,7 +94,7 @@ describe("monitoring queue (RabbitMQ wrapper)", () => {expect(channelMock.nack).not.toHaveBeenCalled();const boom = vi.fn().mockRejectedValue(new Error("handler boom"));- const q2mod = q; // same module instance, second queue consumer+ const q2mod = q;await q2mod.consumeMonitorSearchCheckJobs(boom);const onSearchMessage = channelMock.consume.mock.calls.at(-1)![1];const msg2 = {@@ -120,15 +114,13 @@ describe("monitoring queue (RabbitMQ wrapper)", () => {await q.consumeMonitorCheckJobs(vi.fn());expect(channelMock.consume).toHaveBeenCalledTimes(1);- // Simulate the consume channel closing (a RabbitMQ blip).const onClose = consumeChannelCloseHandler();expect(onClose).toBeTypeOf("function");onClose();- // The reconnect is scheduled on a backoff timer; advance past it.+ // Reconnect is on a backoff timer; advance past it.await vi.advanceTimersByTimeAsync(1500);- // The registered consumer was re-attached on the fresh channel.expect(channelMock.consume.mock.calls.length).toBeGreaterThanOrEqual(2);expect(channelMock.consume.mock.calls.at(-1)![0]).toBe("monitor.checks");} finally {diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.tsindex bd50efc67..caed11082 100644--- a/apps/api/src/services/monitoring/queue.ts+++ b/apps/api/src/services/monitoring/queue.ts@@ -6,8 +6,7 @@ const MONITOR_CHECK_QUEUE = "monitor.checks";const MONITOR_CHECK_DLX = "monitor.checks.dlx";const MONITOR_CHECK_DLQ = "monitor.checks.dlq";-// Search checks are far heavier than scrape/crawl checks, so they get a dedicated-// queue + consumer β a burst of them can't starve the page/site/batch checks.+// Dedicated queue so a burst of heavy search checks can't starve page/site/batch checks.const MONITOR_SEARCH_CHECK_QUEUE = "monitor.checks.search";const MONITOR_SEARCH_CHECK_DLX = "monitor.checks.search.dlx";const MONITOR_SEARCH_CHECK_DLQ = "monitor.checks.search.dlq";@@ -23,33 +22,26 @@ export type MonitorCheckJobData = {type ConsumerHandler = (data: MonitorCheckJobData) => Promise<void>;let connection: amqp.ChannelModel | null = null;-// Publish and consume use separate channels: a channel exception on one side-// (a failed publish, or a PRECONDITION_FAILED on assert) must not tear down the-// other.+// Separate publish/consume channels so an exception on one can't tear down the other.let publishChannel: amqp.Channel | null = null;let consumeChannel: amqp.Channel | null = null;-// In-flight creation promises. Channel creation suspends on async asserts before-// the channel is assigned, so two concurrent callers (e.g. the two consume-// registrations started together at worker boot) would each create a channel and-// orphan one. Memoizing the promise serializes concurrent callers onto one.+// Memoized creation promises serialize concurrent callers onto one channel; without+// this, two callers racing during the async asserts would each create one and orphan one.let publishChannelPromise: Promise<amqp.Channel> | null = null;let consumeChannelPromise: Promise<amqp.Channel> | null = null;-// Registry of every consumer registered this process, so we can re-attach them-// after a connection/channel drop. Without this the worker β which never-// produces and so never re-triggers channel creation β goes permanently deaf-// after any RabbitMQ blip, and only a restart heals it.+// Registry of consumers so a reconnect can re-attach them; the worker never produces,+// so without this it goes permanently deaf after any RabbitMQ blip until restart.const registeredConsumers: Array<{ queue: string; handler: ConsumerHandler }> =[];-// Queues with a live consumer on the CURRENT consume channel. Cleared whenever-// the channel drops, so a reconnect re-subscribes exactly once per queue and-// repeated drops can't pile up duplicate consumers.+// Queues with a live consumer on the CURRENT consume channel; cleared on drop so a+// reconnect re-subscribes exactly once per queue and can't pile up duplicates.const subscribedQueues = new Set<string>();let reconnectInFlight = false;const RECONNECT_BASE_DELAY_MS = 1000;const RECONNECT_MAX_DELAY_MS = 30000;-// Declare a check queue + its dead-letter exchange/queue (same topology for both).+// Declare a check queue plus its dead-letter exchange/queue.async function assertCheckQueue(ch: amqp.Channel,queue: string,@@ -90,8 +82,7 @@ async function assertTopology(ch: amqp.Channel): Promise<void> {);}-// Drop all cached state and, if any consumers were registered, kick off a-// backoff reconnect that re-asserts topology and re-subscribes every consumer.+// Drop cached state and, if any consumers are registered, kick off a backoff reconnect.function handleConnectionDrop(reason: string, error?: unknown): void {if (connection || publishChannel || consumeChannel) {logger.warn("Monitor queue dropped β will reconnect", { reason, error });@@ -128,11 +119,9 @@ function scheduleReconnect(delayMs: number): void {error,nextDelayMs: nextDelay,});- // Best-effort close the half-set-up channel, then reset ALL cached state β- // including subscribedQueues. If a mid-loop subscribe() failed on an- // otherwise-healthy channel (so its "close" never fires), leaving- // subscribedQueues populated would make the next reconnect skip that queue- // (subscribe() early-returns on subscribedQueues.has) and deafen it forever.+ // Reset ALL cached state including subscribedQueues: a mid-loop subscribe()+ // failure on a healthy channel never fires "close", so a stale entry would+ // make the next reconnect skip (subscribe early-returns) and deafen that queue.await consumeChannel?.close().catch(() => {});connection = null;publishChannel = null;@@ -153,8 +142,7 @@ async function getConnection(): Promise<amqp.ChannelModel> {throw new Error("NUQ_RABBITMQ_URL is not configured");}- // Name the connection so operators can identify (and isolate) monitoring- // traffic in `rabbitmqctl list_connections` / the management UI.+ // Name the connection so operators can identify monitoring traffic in the management UI.const conn = await amqp.connect(url, {clientProperties: { connection_name: "monitoring-queue" },});@@ -170,9 +158,8 @@ async function createChannel(label: string): Promise<amqp.Channel> {const conn = await getConnection();const ch = await conn.createChannel();await assertTopology(ch);- // Channels can close independently of the connection (e.g. a- // PRECONDITION_FAILED from assertQueue). Without these handlers the cached- // channel would stay non-null and every send/consume would throw forever.+ // Channels can close independently of the connection (e.g. PRECONDITION_FAILED);+ // without this the cached channel stays non-null and every send/consume throws forever.ch.on("close", () => {if (label === "publish" && publishChannel === ch) {publishChannel = null;@@ -254,7 +241,6 @@ async function consumeQueue(queue: string,handler: ConsumerHandler,): Promise<void> {- // Record the registration so a reconnect can re-attach this consumer.if (!registeredConsumers.some(c => c.queue === queue)) {registeredConsumers.push({ queue, handler });}@@ -267,11 +253,9 @@ async function subscribe(queue: string,handler: ConsumerHandler,): Promise<void> {- // Already consuming this queue on the current channel β don't add a duplicate- // (a reconnect could otherwise re-subscribe an already-subscribed queue).+ // Don't add a duplicate consumer if a reconnect re-subscribes an already-subscribed queue.if (subscribedQueues.has(queue)) return;- // Per-consumer prefetch (amqplib default): each consumer gets its own in-flight- // slot, so the search and default consumers don't block each other.+ // Per-consumer prefetch (amqplib default) so search and default consumers don't block each other.await ch.prefetch(1);await ch.consume(diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.tsindex 292dd0f9a..ed78c69d7 100644--- a/apps/api/src/services/monitoring/runner.test.ts+++ b/apps/api/src/services/monitoring/runner.test.ts@@ -69,7 +69,7 @@ describe("monitoring runner", () => {});it("uses the shorter search timeout when the check has a search target", () => {- // 11 minutes old: past the 10-min search cutoff but well under 1 hour.+ // 11 min: past the 10-min search cutoff, under the 1-hour scrape cutoff.const elevenMinAgo = new Date(now.getTime() - 11 * 60 * 1000,).toISOString();@@ -78,22 +78,20 @@ describe("monitoring runner", () => {updated_at: now.toISOString(),created_at: now.toISOString(),};- // Scrape check: not stale yet (1-hour timeout).expect(isMonitorCheckStale({ ...base, target_results: [{ type: "scrape", targetId: "t1" }] },now,),).toBe(false);- // Search check: stale (10-minute timeout).expect(isMonitorCheckStale({ ...base, target_results: [{ type: "search", targetId: "t1" }] },now,),).toBe(true);- // MIXED search+crawl: keeps the 1-hour timeout (the crawl can legitimately- // run for many minutes) β must NOT be reaped at 11 minutes.+ // Mixed search+crawl keeps the 1-hour timeout: the crawl can legitimately+ // run for many minutes and must not be reaped at 11 minutes.expect(isMonitorCheckStale({@@ -110,10 +108,8 @@ describe("monitoring runner", () => {});});-// The redelivery-idempotency guard: on a re-delivered check, the runner must-// recognize a search target that ALREADY completed (searchCompleted:true) and-// restore its persisted figures instead of re-running the search + re-scraping +-// re-billing. findCompletedSearchTargetRun is that decision.+// On a re-delivered check, an already-completed search target (searchCompleted)+// must restore persisted figures instead of re-running/re-scraping/re-billing.describe("findCompletedSearchTargetRun (redelivery idempotency)", () => {const completed = {type: "search",@@ -169,9 +165,8 @@ describe("findCompletedSearchTargetRun (redelivery idempotency)", () => {null,"garbage",];- // t1's first (incomplete) entry must not match; the completed t1 does β but- // since find() returns the first match and the incomplete one is filtered by- // searchCompleted, the completed entry (also t1) is the one returned.+ // t1's first (incomplete) entry is filtered by searchCompleted, so the+ // completed t1 entry is the one returned.const found = findCompletedSearchTargetRun(results, "t1");expect(found).toMatchObject({ searchCredits: 2 });});diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 89386bd56..070c450c4 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -120,8 +120,7 @@ type MonitorTargetRun =| {targetId: string;type: "search";- // Set true only after the inline search stamps its credits; the reconciler- // waits on this so it never finalizes with credits still at 0.+ // Set only after the inline search stamps credits; reconciler waits on this so it never finalizes with credits at 0.searchCompleted?: boolean;resultCount?: number;matches?: number;@@ -260,10 +259,9 @@ export function estimateActualCredits(doc: any, options?: any): number {return creditsUsed;}const formats = Array.isArray(options?.formats) ? options.formats : [];- // Only charge the JSON-extraction premium when extraction actually produced a- // document.json. When it failed, the page was still scraped, so fall back to- // the base scrape credit rather than billing for extraction that never ran.- // Deterministic JSON costs 7 (it generates a reusable extractor); plain JSON 5.+ // Only charge the JSON-extraction premium when extraction produced a json; a+ // failed extraction still scraped the page, so fall back to base credit.+ // Deterministic JSON costs 7 (reusable extractor); plain JSON 5.const producedJson = doc?.json != null;if (!producedJson) return 1;if (includesFormat(formats, "deterministicJson")) return 7;@@ -332,11 +330,9 @@ async function runSingleScrape(params: {};}-// Deep-mode search-monitor page scrape. Runs inline via processJobInternal-// (skipNuq:true, bypassBilling) β it is NOT routed through NuQ, so it bypasses-// per-team/global scrape concurrency; the caller bounds the fan-out via-// SEARCH_SCRAPE_CONCURRENCY (search/run.ts). Never billed per-page (search-// monitors bill flat at the check level).+// Deep-mode search-monitor page scrape, inline (skipNuq) so it bypasses scrape+// concurrency; caller bounds fan-out via SEARCH_SCRAPE_CONCURRENCY. Never billed+// per-page β search monitors bill flat at the check level.async function scrapeSearchMonitorPage(params: {teamId: string;checkId: string;@@ -610,7 +606,7 @@ async function runCrawlTarget(params: {body.scrapeOptions.skipTlsVerification,);} catch {- // Crawls tolerate robots fetch failures in the public controller too.+ // Non-fatal robots fetch failure, same as the public crawl controller.}sc.queueBackend = await resolveNewGroupBackend(sc.team_id);@@ -833,9 +829,8 @@ async function sendNotifications(params: {}const nonSamePages = params.pages.filter(page => page.status !== "same");- // Pull the unified-diff text for up to 5 meaningful changed pages so the- // email leads with the actual diff. Cheap GCS reads, parallelised. Errors- // are swallowed per-page so a single GCS hiccup doesn't drop the alert.+ // Pull diff text for up to 5 meaningful changed pages so the email leads with+ // the diff. Errors swallowed per-page so one GCS hiccup doesn't drop the alert.const diffEligible = nonSamePages.filter(p => p.status === "changed" && (!p.judgment || p.judgment.meaningful),@@ -1001,7 +996,7 @@ async function enqueueMonitorCrawlTarget(params: {body.scrapeOptions.skipTlsVerification,);} catch {- // Non-fatal, same as the public crawl controller.+ // Non-fatal robots fetch failure, same as the public crawl controller.}sc.queueBackend = await resolveNewGroupBackend(sc.team_id);@@ -1041,7 +1036,7 @@ async function enqueueMonitorCrawlTarget(params: {return params.targetRun;}-// Runs inline, then persists onto the same monitor_pages / monitor_check_pages+// Runs inline, persisting onto the same monitor_pages / monitor_check_pages// tables the reconciler tallies.async function runMonitorSearchTarget(params: {monitor: MonitorRow;@@ -1054,7 +1049,7 @@ async function runMonitorSearchTarget(params: {summary: string;judgeDegraded: boolean;degradedReason: string | null;- // Flat, deterministic credits, recorded onto target_results by the caller.+ // Flat credits, recorded onto target_results by the caller.searchCredits: number;judgeCredits: number;resultsJudged: number;@@ -1102,8 +1097,7 @@ async function runMonitorSearchTarget(params: {id: target.id,queries: target.queries,searchWindow: target.searchWindow,- // depth/alertMode aren't settable via the API, but stored targets may- // still carry them (back-compat), so pass them through.+ // depth/alertMode aren't API-settable but stored targets may carry them (back-compat); pass through.alertMode: target.alertMode ?? "first_match",includeDomains: target.includeDomains,excludeDomains: target.excludeDomains,@@ -1130,7 +1124,6 @@ async function runMonitorSearchTarget(params: {}),});- // Flat credits recorded onto target_results (summed there at finalization).const searchCredits = result.searchCredits;const judgeCredits = result.judgeCredits;@@ -1166,8 +1159,7 @@ async function runMonitorSearchTarget(params: {});}- // Clear any rows from a prior (crashed/redelivered) run of this same check so- // the insert is effectively a replace β a redelivery can't duplicate pages.+ // Clear rows from a prior (crashed/redelivered) run so the insert is a replace, not a duplicate.await deleteMonitorCheckPages({ checkId: check.id, targetId: target.id });await insertMonitorCheckPages(pages);@@ -1197,9 +1189,8 @@ async function runMonitorSearchTarget(params: {};}-// Find a persisted search target run that already completed, so a redelivered-// check can restore its figures instead of re-running (and re-billing) it.-// Exported for unit coverage of the redelivery-idempotency decision.+// Find a completed search target run so a redelivered check can restore its+// figures instead of re-running (and re-billing) it.export function findCompletedSearchTargetRun(targetResults: unknown,targetId: string,@@ -1308,10 +1299,8 @@ export async function processMonitorCheckJob(} else if (target.type === "crawl" && targetRun.type === "crawl") {await enqueueMonitorCrawlTarget({ monitor, check, target, targetRun });} else if (target.type === "search" && targetRun.type === "search") {- // If this check was already run to completion for this target (a- // redelivery after the inline work finished but before the ack),- // restore the persisted figures instead of re-running β re-running- // would re-bill the search and re-scrape every page.+ // Redelivery after inline work finished but before ack: restore persisted+ // figures instead of re-running, which would re-bill and re-scrape.const priorRun = findCompletedSearchTargetRun(initialCheck.target_results,target.id,@@ -1335,14 +1324,11 @@ export async function processMonitorCheckJob(targetRun.searchCredits = searchResult.searchCredits;targetRun.judgeCredits = searchResult.judgeCredits;targetRun.resultsJudged = searchResult.resultsJudged;- // Set LAST, after credits are stamped, so the reconciler never finalizes- // with credits still at 0.+ // Set last, after credits are stamped, so the reconciler never finalizes with credits at 0.targetRun.searchCompleted = true;- // Persist immediately (not just at the end of the loop): the search work- // β external search calls + per-page scrapes β already happened. If we- // crash before the end-of-loop flush, a redelivery would re-run and- // re-scrape everything. Flushing searchCompleted now lets- // findCompletedSearchTargetRun short-circuit the redelivery.+ // Persist now, not just at end-of-loop: the search work already happened,+ // so a crash before the final flush would re-run and re-scrape everything;+ // flushing searchCompleted lets findCompletedSearchTargetRun short-circuit.await updateMonitorCheck(check.id, {target_results: targetResults,});diff --git a/apps/api/src/services/monitoring/search/billing.ts b/apps/api/src/services/monitoring/search/billing.tsindex cca5cf7e0..a4d3c2f77 100644--- a/apps/api/src/services/monitoring/search/billing.ts+++ b/apps/api/src/services/monitoring/search/billing.ts@@ -1,8 +1,6 @@const SEARCH_CREDITS_PER_TEN_RESULTS = 2;const SEARCH_CREDITS_PER_TEN_RESULTS_ZDR = 10;-// The single source of truth for the per-judged-result judge credit rate.-// Tests MUST import this rather than re-declaring a literal (which previously-// drifted to 5 and silently misrepresented billing).+// Single source of truth for the judge credit rate; import it rather than re-declaring (a duplicate literal previously drifted to 5).export const SEARCH_JUDGE_CREDITS_PER_RESULT = 1;export function searchCreditsForResultCount(diff --git a/apps/api/src/services/monitoring/search/cost.test.ts b/apps/api/src/services/monitoring/search/cost.test.tsindex 88ab71569..fc094c90b 100644--- a/apps/api/src/services/monitoring/search/cost.test.ts+++ b/apps/api/src/services/monitoring/search/cost.test.ts@@ -44,14 +44,14 @@ describe("search-monitor FLAT judge billing rate", () => {});it("computes judge credits as 1 * resultsJudged", () => {- expect(judgeCreditsForJudgedCount(0)).toBe(0); // raw / judge-off- expect(judgeCreditsForJudgedCount(3)).toBe(3); // deep, 3 results judged+ expect(judgeCreditsForJudgedCount(0)).toBe(0);+ expect(judgeCreditsForJudgedCount(3)).toBe(3);expect(judgeCreditsForJudgedCount(10)).toBe(10);});});-// Billed count == pages with judgedThisRun=true (not every page, not every page-// with a concept β reused/skipped pages keep a stale concept but aren't judged).+// Billed count == pages with judgedThisRun=true: reused/skipped pages keep a+// stale concept but aren't judged, so concept count overcounts.describe("canonical judged-result count (billing == persisted signal)", () => {type Page = {searchStatus: string;@@ -71,10 +71,10 @@ describe("canonical judged-result count (billing == persisted signal)", () => {concept: "stale-1",},{ searchStatus: "watching", judgedThisRun: false, concept: "stale-2" },- { searchStatus: "skipped" }, // no verdict, no judgedThisRun+ { searchStatus: "skipped" },];- expect(pages.filter(p => p.concept).length).toBe(3); // the misleading count- expect(billedJudgedCount(pages)).toBe(1); // the canonical, billed count+ expect(pages.filter(p => p.concept).length).toBe(3); // misleading count+ expect(billedJudgedCount(pages)).toBe(1); // canonical billed countexpect(judgeCreditsForJudgedCount(billedJudgedCount(pages))).toBe(1);});@@ -86,6 +86,6 @@ describe("canonical judged-result count (billing == persisted signal)", () => {{ searchStatus: "already_seen", judgedThisRun: true },{ searchStatus: "skipped" }, // verdict failed -> not billed];- expect(billedJudgedCount(pages)).toBe(4); // -> 1*4 = 4 judge credits+ expect(billedJudgedCount(pages)).toBe(4); // -> 4 judge credits});});diff --git a/apps/api/src/services/monitoring/search/dedupe.ts b/apps/api/src/services/monitoring/search/dedupe.tsindex ed2e55885..fd76f9c30 100644--- a/apps/api/src/services/monitoring/search/dedupe.ts+++ b/apps/api/src/services/monitoring/search/dedupe.ts@@ -22,10 +22,7 @@ export function stableSerpFingerprint(src: {snippet?: string;description?: string;}): string {- // A search result is identified by its canonical URL. Snippets and titles drift- // between searches (engines reword/truncate them), so hashing that text would- // re-flag the same URL as "changed" every check β causing the same result to be- // re-judged and re-billed. Keying on the URL keeps already-seen results free.+ // Key on canonical URL only: snippets/titles drift between searches, so hashing them would re-flag, re-judge and re-bill the same URL.return createHash("sha256").update(canonicalizeUrl(src.url ?? "")).digest("hex")diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.tsindex 7bb332da6..85531b365 100644--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts@@ -2,10 +2,7 @@ import type { Logger } from "winston";import type { SearchVerdict } from "./judge";import { canonicalizeUrl } from "./dedupe";-// vi.mock is hoisted above declarations, so the mocks its factories reference-// are created in vi.hoisted() (also hoisted) to avoid any TDZ surprises.-// The lean pipeline has a single LLM stage: the deep-mode per-page JSON verdict,-// produced by the injected scrapePage.+// Mocks live in vi.hoisted() so the hoisted vi.mock factories can reference them.const { searchMock, scrapeURLMock } = vi.hoisted(() => ({searchMock: vi.fn(),scrapeURLMock: vi.fn(),@@ -13,8 +10,7 @@ const { searchMock, scrapeURLMock } = vi.hoisted(() => ({vi.mock("uuid", () => ({ v7: () => "00000000-0000-7000-8000-000000000000" }));vi.mock("../../../search/v2", () => ({- // Tests set the mock to resolve a bare result array; the real search() returns- // a SearchV2Response ({ web: [...] }), so wrap arrays to match that contract.+ // Tests resolve a bare array; wrap it to match search()'s SearchV2Response shape.search: async (...a: unknown[]) => {const r = await searchMock(...a);return Array.isArray(r) ? { web: r } : r;@@ -63,8 +59,7 @@ function runParams(...targetOver,},monitorCheckId: "check-1",- // scrapeURLMock keeps the legacy { success, document } shape; adapt it to the- // ScrapeSearchResult the injected scrapePage now returns.+ // Adapt scrapeURLMock's legacy { success, document } shape to ScrapeSearchResult.scrapePage: async (...a: unknown[]) => {const r = (await scrapeURLMock(...a)) as {success?: boolean;@@ -223,24 +218,22 @@ describe("event state stamps + judgment on alert pages", () => {describe("judgeEnabled gates the LLM judge", () => {it("judge OFF behaves like raw: no LLM stages, no LLM credits, no concept/rationale", async () => {- // Monitor still has a goal and depth:"deep" β but judgeEnabled=false must- // collapse to raw and skip every LLM stage.+ // judgeEnabled=false collapses to raw and skips every LLM stage, even with goal + depth:"deep".searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);const result = await runSearchTarget(runParams({ depth: "deep" }, { judgeEnabled: false }),);- // No judge stage ran: raw mode does not scrape.+ // Raw mode does not scrape.expect(scrapeURLMock).not.toHaveBeenCalled();- // No judge credits billed (raw β nothing judged); search still ran and was billed.+ // Nothing judged, but search still ran and was billed.expect(result.judgeCredits).toBe(0);expect(result.resultsJudged).toBe(0);expect(result.resultCount).toBe(2);- // Raw alerts: deterministic searchStatus from dedup, but no LLM- // concept/rationale on any upsert.+ // Deterministic searchStatus from dedup, but no LLM concept/rationale.for (const upsert of result.pageUpserts) {expect(upsert.scraped).toBe(false);expect(upsert.metadata.concept).toBeUndefined();@@ -286,7 +279,6 @@ describe("judgeEnabled gates the LLM judge", () => {describe("deep-path scrape failures mark the check degraded (no silent empty)", () => {beforeEach(() => {- // Two candidates scraped; scrape outcome varies per test.searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);});@@ -295,15 +287,15 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",const result = await runSearchTarget(runParams({ depth: "deep" }));- // The false-negative we must surface: 0 judged / 0 alerts, but degraded.+ // 0 judged / 0 alerts but degraded β the false-negative we must surface.expect(result.resultsJudged).toBe(0);expect(result.matches).toBe(0);expect(result.judgeDegraded).toBe(true);expect(result.degradedReason).toMatch(/judged|incomplete/i);- // A non-empty summary so the check never reads as a clean "nothing new".+ // Non-empty summary so the check never reads as a clean "nothing new".expect(result.summary).not.toBe("");- // Each unscrapeable result is persisted as a "skipped" page so it surfaces- // as an error check page (skipped -> error) instead of silently vanishing.+ // Each unscrapeable result is persisted as "skipped" so it surfaces as an+ // error check page instead of silently vanishing.const skippedUpserts = result.pageUpserts.filter(u => u.status === "skipped",);@@ -338,8 +330,8 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",const result = await runSearchTarget(runParams({ depth: "deep" }));- // One result was actually judged (ignored) β that's a real evaluation, not a- // failure, so the check is NOT degraded even though a sibling scrape failed.+ // One result judged (ignored) is a real evaluation, so NOT degraded even+ // though a sibling scrape failed.expect(result.resultsJudged).toBe(1);expect(result.judgeDegraded).toBe(false);expect(result.degradedReason).toBeNull();@@ -357,7 +349,7 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",const result = await runSearchTarget(runParams({ depth: "deep" }));- // Everything scraped fine and was evaluated; judge ignored it all. Normal.+ // Everything scraped and was evaluated; judge ignored it all. Normal.expect(result.resultsJudged).toBe(2);expect(result.matches).toBe(0);expect(result.judgeDegraded).toBe(false);@@ -373,9 +365,8 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",expect(result.judgeDegraded).toBe(false);expect(result.degradedReason).toBeNull();expect(scrapeURLMock).not.toHaveBeenCalled();- // An empty SERP hedges with a BOUNDED retry (1 initial + 2 quick retries),- // NOT the old 5-attempt / ~11s growing backoff. Lock the budget so a genuinely- // empty query concludes fast instead of stalling the whole check.+ // An empty SERP hedges with a bounded retry (1 initial + 2 quick), so a+ // genuinely empty query concludes fast instead of stalling the check.expect(searchMock).toHaveBeenCalledTimes(3);});@@ -386,7 +377,7 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",serpRow(3),serpRow(4),]);- // Only result 1 scrapes + judges; results 2-4 all fail to scrape.+ // Only result 1 scrapes + judges; 2-4 fail to scrape.scrapeURLMock.mockImplementation(({ url }: { url: string }) =>url === serpRow(1).url? Promise.resolve({@@ -404,8 +395,8 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",expect(result.resultsJudged).toBe(1);expect(result.scrapeFailures).toBe(3);- // 3/(1+3) = 0.75 >= 0.5 -> soft partial-loss signal, but the run still- // produced a verdict so it is NOT a hard degrade.+ // 3/(1+3) = 0.75 >= 0.5 -> soft partial-loss signal, but a verdict was+ // produced so it is NOT a hard degrade.expect(result.partialScrapeLoss).toBe(true);expect(result.judgeDegraded).toBe(false);});@@ -432,7 +423,7 @@ describe("run time budget (a single check can't wedge the consumer)", () => {vi.useFakeTimers();try {searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);- // Scrapes never resolve β the pre-scrape can't finish on its own.+ // Scrapes never resolve, so the pre-scrape can't finish on its own.scrapeURLMock.mockReturnValue(new Promise(() => {}));const pending = runSearchTarget(runParams({ depth: "deep" }));diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 3083c196a..e5bea732d 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -16,8 +16,7 @@ import {judgeCreditsForJudgedCount,} from "./billing";-// Shared dedup decision for a search result, used by both the pre-scrape pass and-// the per-result loop so they never disagree on which results to scrape/judge.+// Shared dedup decision so the pre-scrape pass and per-result loop never disagree.function evaluateSerpCandidate(c: { url: string; title: string; description: string },knownPages: Map<string, KnownPage>,@@ -58,31 +57,20 @@ function windowToTbs(window: string): string {return "qdr:w";}-// A successful-but-empty SERP almost always means a genuine "no results" for this-// query+window β NOT a transient failure. The backend can occasionally return 0 for-// a query that succeeds a moment later, so hedge with a couple of QUICK retries, but-// don't burn ~11s of growing backoff (the old [400,1200,3000,6000]) on a query that-// simply has no matches β that latency stacks across queries and slows every check-// whose results legitimately don't exist.+// Empty SERPs are usually genuine "no results", not transient β hedge with a couple+// of quick retries, but don't burn long backoff on queries that simply have no matches.const SEARCH_EMPTY_RETRY_BACKOFF_MS = [500, 1000] as const;-// Hard wall-clock budget for a single inline search run. The deep-mode page-// scrapes are the only unbounded-ish step (each is capped at 20s, but a large-// candidate set could still add up); if the concurrent pre-scrape blows past-// this, we stop waiting and let the remaining candidates fall through as-// skipped/degraded rather than holding the consumer. Kept well under the-// 10-minute search stale timeout so a run always returns before the reaper.+// Wall-clock budget per run; if the pre-scrape blows past it, remaining candidates+// fall through as skipped/degraded. Kept under the 10-min stale timeout so a run+// always returns before the reaper.const SEARCH_RUN_BUDGET_MS = 4 * 60 * 1000;-// Deep-mode page scrapes run INLINE in the worker process (skipNuq), so they are-// NOT bounded by NuQ's per-team/global concurrency. Cap the pre-scrape fan-out-// ourselves so a single maxResults=50 check can't launch 50 simultaneous-// fetch+LLM-extraction scrapes and starve the worker that owns the search consumer.+// Deep-mode scrapes run inline (skipNuq), so NuQ concurrency doesn't bound them; cap+// the fan-out ourselves so one maxResults=50 check can't starve the search consumer.const SEARCH_SCRAPE_CONCURRENCY = 6;-// Run fn over items with at most `limit` in flight at once. Never rejects per the-// caller's needs β fn is expected to swallow its own errors (it writes outcomes-// into a shared cache).+// Never rejects: fn is expected to swallow its own errors into a shared cache.async function mapWithConcurrency<T>(items: T[],limit: number,@@ -99,9 +87,8 @@ async function mapWithConcurrency<T>(await Promise.all(pool);}-// Race a promise against a wall-clock deadline. Resolves false if the work-// finishes first, true if the deadline hits. Never rejects; the underlying work-// keeps running (bounded by its own per-scrape timeout) and populates its cache.+// Resolves true if the deadline hits first, false if the work finishes. Never+// rejects; the underlying work keeps running and populates its cache.async function raceDeadline(work: Promise<unknown>,budgetMs: number,@@ -198,9 +185,7 @@ export type KnownEvent = {alertCount?: number;};-// Deep-mode page scrape. Injected by the caller so it can route through the-// shared monitor scrape path (concurrency queue + bypassBilling), keeping this-// module free of worker/queue dependencies. Resolves to null on scrape failure.+// Injected by the caller so this module stays free of worker/queue dependencies.export type ScrapeSearchResult = {json: unknown;markdown: string;@@ -233,9 +218,8 @@ type SearchTargetRunResult = {summary: string;judgeDegraded: boolean;degradedReason: string | null;- // Per-page scrape failures in deep mode. partialScrapeLoss is a SOFT signal- // (distinct from judgeDegraded): some results judged, but a majority of the- // attempted scrapes failed β the run is usable but likely missed developments.+ // partialScrapeLoss is a soft signal (distinct from judgeDegraded): some results+ // judged, but most scrapes failed β usable but likely missed developments.scrapeFailures: number;partialScrapeLoss: boolean;sources: SearchSource[];@@ -345,8 +329,7 @@ export async function runSearchTarget(params: {}resultCount = candidates.length;- // An empty retrieval is indistinguishable from "nothing matched", so log it to- // make a genuine miss diagnosable.+ // An empty retrieval is indistinguishable from "nothing matched"; log to diagnose.if (resultCount === 0) {logger.warn("search monitor: all queries returned no results after retries",@@ -359,9 +342,7 @@ export async function runSearchTarget(params: {}const llmStagesAvailable = hasLlmProvider();- // Judge depth is raw (no LLM) or deep (per-page scrape + verdict). Standard- // (snippet-only judging) was removed: it was unreachable via the API (depth is- // stripped) and used a divergent LLM provider; deep is the single judged path.+ // raw (no LLM) or deep (per-page scrape + verdict); the former "standard" path was removed.const depth: "raw" | "deep" =!judgeEnabled || target.depth === "raw" ? "raw" : "deep";@@ -371,17 +352,15 @@ export async function runSearchTarget(params: {const recheckMs = target.recheckAfter ? windowToMs(target.recheckAfter) : 0;- // Pre-scrape deep-mode pages (bounded concurrency) and cache them, so the loop- // below isn't a serial scrapeβjudgeβscrape chain. Only the I/O is parallel; all- // dedup/event state stays in the single-threaded loop. These scrapes run inline- // (skipNuq), so the fan-out is bounded by SEARCH_SCRAPE_CONCURRENCY, not NuQ.+ // Pre-scrape deep-mode pages (bounded concurrency) into a cache so the loop isn't+ // a serial scrapeβjudge chain. Only the I/O is parallel; dedup/event state stays+ // in the single-threaded loop.const deepDocs = new Map<string,{ doc: ScrapeSearchResult | null; error?: unknown }>();- // Set if the concurrent pre-scrape exceeds the run budget: any candidate not- // yet cached is then treated as a scrape failure instead of being inline-scraped,- // so the loop returns promptly rather than holding the consumer.+ // If the pre-scrape exceeds the run budget, uncached candidates are treated as+ // scrape failures rather than inline-scraped, so the loop returns promptly.let budgetExceeded = false;if (depth === "deep" && llmStagesAvailable) {const judgePrompt = buildJudgePrompt(@@ -533,8 +512,7 @@ export async function runSearchTarget(params: {eventAlertCount,eventLastAlertAt: nowIso,},- // Raw mode does not run the judge, so it carries no judgment β these are- // surfaced as new results, not evaluated as "meaningful".+ // Raw mode runs no judge: surfaced as new results, not evaluated as "meaningful".});continue;}@@ -542,7 +520,7 @@ export async function runSearchTarget(params: {let verdict: SearchVerdict | null = null;let realDate: string | null = null;{- // Deep mode: read the doc from the pre-scrape cache; scrape inline only as a fallback.+ // Read from the pre-scrape cache; scrape inline only as a fallback.let doc: ScrapeSearchResult | null;const cached = deepDocs.get(canonical);if (cached?.error !== undefined) {@@ -556,9 +534,8 @@ export async function runSearchTarget(params: {);continue;}- // The pre-scrape ran out of time before caching this page. Treat it as a- // scrape failure rather than inline-scraping (which would re-incur the time- // we just budgeted out of), so the run returns promptly.+ // Out of budget before caching this page: treat as a scrape failure rather+ // than inline-scraping (which would re-incur the time we just budgeted out of).if (!cached && budgetExceeded) {judgeScrapeFailures += 1;pushUnevaluatedPage("run budget exceeded before scrape");@@ -654,8 +631,7 @@ export async function runSearchTarget(params: {const eventKey = canonical;const eventLabel = verdict.concept || canonical;- // Re-alert on every new result only in every_new_result mode; otherwise the- // canonical URL is deduped so we don't re-alert the same page.+ // Re-alert on every new result only in every_new_result mode; otherwise dedup.const alreadySatisfied =satisfied.has(eventKey) && target.alertMode !== "every_new_result";const eventMeta = { ...baseMeta, eventKey, eventLabel };@@ -759,9 +735,8 @@ export async function runSearchTarget(params: {? "results were found but none could be judged; results may be incomplete": null;- // Soft signal (NOT judgeDegraded): the run produced verdicts, but a majority of- // the attempted deep scrapes failed, so it likely missed real developments.- // Surfaced for observability; does not fail the check.+ // Soft signal (not judgeDegraded): verdicts produced but most scrapes failed, so+ // it likely missed real developments. Surfaced for observability; doesn't fail the check.const partialScrapeLoss =resultsJudged > 0 &&judgeScrapeFailures > 0 &&diff --git a/apps/api/src/services/monitoring/stale.ts b/apps/api/src/services/monitoring/stale.tsindex 97c86775c..a56065b84 100644--- a/apps/api/src/services/monitoring/stale.ts+++ b/apps/api/src/services/monitoring/stale.ts@@ -1,17 +1,12 @@import type { MonitorCheckRow } from "./types";export const MONITOR_CHECK_STALE_TIMEOUT_MS = 60 * 60 * 1000;-// Search checks run their whole search+scrape+judge inline and realistically-// finish in seconds-to-low-minutes. A stranded one (crash between the inline-// work and the ack) should self-heal quickly rather than look dead for an hour.+// Search checks run inline and finish in minutes; a stranded one should self-heal quickly, not look dead for an hour.const MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS = 10 * 60 * 1000;export const MONITOR_CHECK_STALE_ERROR ="Monitor check exceeded the running timeout.";-// The shorter search timeout only applies when EVERY target is a search target.-// A mixed monitor (e.g. search + crawl) must keep the long timeout β its scrape-// fan-out / crawl legitimately runs for many minutes and would otherwise be-// falsely reaped at the 10-minute search cutoff.+// Short timeout only when EVERY target is search; a mixed monitor's crawl/scrape fan-out legitimately runs for many minutes.function isSearchOnlyCheck(check: { target_results?: unknown }): boolean {const targetResults = check.target_results;if (!Array.isArray(targetResults) || targetResults.length === 0) return false;diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex 2c3469148..50910a6c3 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -233,7 +233,7 @@ describe("FLAT deterministic search-monitor billing", () => {});it("judge ON: search call + flat 1 per judged result (capped at maxResults)", () => {- // search = ceil(10/10)*2*1 = 2; judge = 1 * min(maxResults, 10*1) = 10.+ // search 2 + judge min(maxResults, 10) = 10.expect(estimateMonitorCreditsPerRun([searchTarget()], true)).toBe(12);});@@ -248,15 +248,14 @@ describe("FLAT deterministic search-monitor billing", () => {maxResults: 20,queries: ["a", "b", "c"],});- // search = ceil((20*3)/10)*2 = 12; judge = 1 * 20 (maxResults cap, NOT *queries)- // = 20 β 32. The runner judges at most maxResults candidates total.+ // search ceil((20*3)/10)*2 = 12; judge caps at maxResults (20, not *queries) -> 32.expect(estimateMonitorCreditsPerRun([t], true)).toBe(32);});});describe("flatSearchTargetCredits (the deterministic actual)", () => {it("sums searchCredits + judgeCredits across search target_results", () => {- // The headline trace: search 2 + judge 1*3 = 5.+ // search 2 + judge 1*3 = 5.const targetResults = [{targetId: "search-1",@@ -290,7 +289,7 @@ describe("FLAT deterministic search-monitor billing", () => {{ targetId: "search-1", type: "search", searchCredits: 4 },null,"garbage",- { type: "search" }, // missing credits β contributes 0+ { type: "search" }, // missing credits -> 0]),).toBe(4);});@@ -322,15 +321,14 @@ describe("FLAT deterministic search-monitor billing", () => {});it("never bills 0 on a check that returned results and judged them", () => {- // results=6, judged=4 -> 2 + 4 = 6 (the judged#2 case that landed on 0).+ // results=6, judged=4 -> 2 + 4 = 6 (regression: this case once landed on 0).const credits = flatSearchTargetCredits([stampedSearchRun(6, 4)]);expect(credits).toBe(6);expect(credits).toBeGreaterThan(0);});it("the RETRY path still bills: rate-limited-then-succeeded run is summed", () => {- // A run that hit rate-limiting, retried, and finally returned 6 results +- // judged 4 must record the SAME deterministic figure as a clean run β the+ // A retried run must record the same figure as a clean run; the// searchCompleted=true guard ensures credits are stamped before summing.const retried = stampedSearchRun(6, 4);expect(retried.searchCompleted).toBe(true);@@ -338,15 +336,14 @@ describe("FLAT deterministic search-monitor billing", () => {});it("search-only (judge off) bills just the search portion, never the judge", () => {- // results=6, judged=0 -> 2 + 0 = 2 (the raw case, which is already correct).+ // results=6, judged=0 -> 2 + 0 = 2 (raw case).expect(flatSearchTargetCredits([stampedSearchRun(6, 0)])).toBe(2);});it("a still-running search (no credits stamped yet) sums to 0 β the reconciler must NOT finalize here", () => {- // This is the pre-stamp shape the dispatcher writes before the inline- // search finishes. flatSearchTargetCredits sees 0; the never-zero- // guarantee instead relies on isMonitorCheckComplete refusing to finalize- // a run whose searchCompleted flag is not yet true (see runner.ts).+ // Pre-stamp shape the dispatcher writes before the inline search finishes.+ // The never-zero guarantee relies on isMonitorCheckComplete refusing to+ // finalize a run whose searchCompleted flag is not yet true (see runner.ts).const pending = { targetId: "search-1", type: "search" as const };expect((pending as any).searchCompleted).toBeUndefined();expect(flatSearchTargetCredits([pending])).toBe(0);@@ -355,8 +352,8 @@ describe("FLAT deterministic search-monitor billing", () => {it("page-sum never bills search-target pages (cost is at the check level)", () => {const targets: MonitorTarget[] = [searchTarget()];- // Even with judgment + status, a search page contributes 0 to the page sum;- // its credits live in target_results via flatSearchTargetCredits.+ // A search page contributes 0 to the page sum; its credits live in+ // target_results via flatSearchTargetCredits.expect(calculateMonitorCheckActualCreditsFromPages([diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex e391c8558..bbeefcf64 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -108,8 +108,7 @@ function estimateBaseCreditsPerPage(credits += SCRAPE_OPTION_CREDIT_BONUS;}- // Deterministic JSON generates a reusable extractor and costs more than plain- // JSON; both override the base scrape credit (mirrors estimateActualCredits).+ // Deterministic JSON costs more than plain JSON; both override the base scrape credit.if (usesDeterministicJson) {credits = DETERMINISTIC_JSON_SCRAPE_CREDITS_PER_PAGE;} else if (usesJsonCredits) {@@ -206,7 +205,7 @@ export function estimateMonitorCreditsPerRun((sum, target) => sum + estimateTargetBaseCredits(target, judgeEnabled),0,);- // Per-page judge allowance is scrape/crawl only (search judging is folded in above).+ // Per-page judge allowance is scrape/crawl only; search judging is folded in above.const judgeCredits = judgeEnabled? targets.reduce((sum, target) =>@@ -253,9 +252,8 @@ export function calculateMonitorCheckActualCreditsFromPages(baseCreditsByTarget.get(page.target_id ?? "") ??BASE_SCRAPE_CREDITS_PER_PAGE;- // Monitor-specific fallback only: new rows should prefer metadata.creditsUsed- // when the scrape path provides it. If it is missing, use retained monitor- // metadata to avoid obvious undercounts for PDFs and special postprocessors.+ // Fallback when metadata.creditsUsed is missing: use retained metadata to avoid+ // undercounting PDFs and special postprocessors.if (target &&shouldParsePDF(target.scrapeOptions?.parsers as any) &&@@ -301,7 +299,7 @@ export function calculateMonitorCheckActualCreditsFromPages(}return pages.reduce((total, page) => {- // Search pages carry no per-page credit β billed at check level.+ // Search pages carry no per-page credit; billed at check level.const target = targetsById.get(page.target_id ?? "");if (target?.type === "search") {return total;@@ -323,7 +321,6 @@ export function calculateMonitorCheckActualCreditsFromPages(}, 0);}-// Sum the flat searchCredits + judgeCredits recorded on a check's target_results.export function flatSearchTargetCredits(targetResults: unknown): number {if (!Array.isArray(targetResults)) return 0;return targetResults.reduce((total: number, run: unknown) => {@@ -390,8 +387,7 @@ export async function createMonitor(params: {const estimatedCreditsPerMonth =estimatedCreditsPerRun * estimateRunsPerMonth(params.intervalMs);- // Omit goal/judge_enabled keys when undefined so a pre-migration DB- // doesn't reject the insert. Migration lives in a separate repo.+ // Omit goal/judge_enabled when undefined so a pre-migration DB doesn't reject the insert.const insert: typeof schema.monitors.$inferInsert = {id: uuidv7(),team_id: params.teamId,@@ -502,9 +498,8 @@ export async function updateMonitor(params: {if (params.input.status !== undefined) patch.status = params.input.status;if (params.input.webhook !== undefined)patch.webhook = params.input.webhook ?? null;- // Defense-in-depth against a wiped notification: only write when the caller- // actually sent config. Treat an empty object (the legacy default-materialized- // {}) as "leave unchanged" rather than clobbering the stored email settings.+ // Only write when the caller sent config; treat empty {} (legacy default) as+ // "leave unchanged" rather than clobbering stored email settings.if (params.input.notification !== undefined &¶ms.input.notification !== null &&@@ -530,10 +525,8 @@ export async function updateMonitor(params: {patch.next_run_at = params.nextRunAt?.toISOString() ?? null;}- // Re-estimate whenever any cost input changed. Merge the patch with the- // current monitor row so a goal/judge-only update still recalculates- // against the existing targets + schedule, and a targets-only update- // preserves an already-enabled judge.+ // Re-estimate whenever any cost input changed, merging the patch with the current+ // row so partial updates recalculate against existing targets/schedule/judge.const costInputsChanged =params.input.targets !== undefined ||params.input.judgeEnabled !== undefined ||@@ -865,10 +858,9 @@ export async function insertMonitorCheckPages();}-// Delete all check-page rows for a (check, target). Used to make the inline-// search write idempotent: a redelivered search check clears its prior rows-// before re-inserting, so a crash-and-redeliver can't leave duplicate pages.-// Partition-safe (no unique constraint needed on the partitioned table).+// Makes the inline search write idempotent: a redelivered check clears its prior+// (check, target) rows before re-inserting, so crash-and-redeliver can't duplicate+// pages. Partition-safe (no unique constraint needed).export async function deleteMonitorCheckPages(params: {checkId: string;targetId: string;diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsindex a6e5837b7..20838ae37 100644--- a/apps/api/src/services/monitoring/types.ts+++ b/apps/api/src/services/monitoring/types.ts@@ -48,8 +48,8 @@ const monitorDomainSchema = z"Domain must be a valid hostname without protocol or path",);-// depth/alertMode/recheckAfter are internal-only; scrapeOptions is unused for-// search. Strip them before validation so older clients sending them don't 400.+// Strip internal-only depth/alertMode/recheckAfter (and search-unused scrapeOptions)+// before validation so older clients sending them don't 400.const searchTargetSchema = z.preprocess(value => {if (value && typeof value === "object" && !Array.isArray(value)) {@@ -138,9 +138,8 @@ const monitorNotificationInner = z.strictObject({}).optional(),});-// Create defaults a missing notification to {}. The UPDATE schema deliberately-// does NOT (see updateMonitorSchema) β a default there would materialize {} on-// any PATCH that omits notification and wipe the stored email config.+// Create defaults a missing notification to {}; the UPDATE schema deliberately does+// not (a default there would materialize {} on a PATCH and wipe the stored email config).const monitorNotificationSchema = monitorNotificationInner.optional().default({});@@ -196,9 +195,8 @@ export const createMonitorSchema = createMonitorBaseSchema.superRefine(requireGoalForSearchTargets).transform(applyJudgeEnabledDefault);-// Patches are partial and may rely on the already-stored goal, so only reject-// when the patch includes search targets AND explicitly clears the goal. The-// controller re-validates the merged monitor.+// Patches may rely on the already-stored goal, so only reject when the patch has+// search targets AND explicitly clears the goal; the controller re-validates the merge.function rejectGoalClearedWithSearchTargets(input: { targets?: unknown; goal?: unknown; judgeEnabled?: unknown },ctx: z.RefinementCtx,@@ -226,13 +224,10 @@ export const updateMonitorSchema = createMonitorBaseSchema.partial().extend({status: z.enum(["active", "paused"]).optional(),- // Override the create-path notification (which defaults to {}): on update a- // default would materialize {} for a PATCH that omits notification and wipe- // the stored email config. No default => omitted means "leave unchanged".+ // No default => omitted means "leave unchanged"; a default would materialize {} on+ // a PATCH and wipe the stored email config.notification: monitorNotificationInner.optional(),- // Same default-materialization trap: createMonitorBaseSchema gives- // retentionDays .default(30), so a partial PATCH that omits it would parse to- // 30 and silently reset a user's configured retention. Drop the default here.+ // Drop the create-path .default(30): otherwise an omitting PATCH resets configured retention.retentionDays: z.number().int().positive().max(365).optional(),}).superRefine(rejectGoalClearedWithSearchTargets)@@ -265,16 +260,15 @@ export const monitorCheckDetailQuerySchema = z.object({status: z.enum(["same", "new", "changed", "removed", "error"]).optional(),});-// Stripped from API input (see searchTargetSchema), but stored targets may-// still carry them; surface as optional internal-only fields so runner/store-// can read stored values without type errors.+// Stripped from API input but stored targets may carry them; surface as optional+// internal-only fields so runner/store can read stored values without type errors.export type MonitorTarget = z.infer<typeof monitorTargetSchema> & {id: string;} & {depth?: "raw" | "standard" | "deep";alertMode?: "first_match" | "every_new_result" | "material_dev";recheckAfter?: "1h" | "6h" | "24h" | "7d";- // Present on scrape/crawl, absent on search; optional so union reads compile.+ // Absent on search; optional so union reads compile.scrapeOptions?: z.infer<typeof scrapeOptionsSchema>;};export type CreateMonitorRequest = z.infer<typeof createMonitorSchema>;diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex f85753f84..fa2366f59 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -87,11 +87,9 @@ const processDeepResearchJobInternal = async (});if (result.success) {- // Move job to completed state in Redis and update research statusawait job.moveToCompleted(result, token, false);return result;} else {- // If the deep research failed but didn't throw an errorconst error = new Error("Deep research failed without specific error");await updateDeepResearch(job.data.researchId, {status: "failed",@@ -104,7 +102,7 @@ const processDeepResearchJobInternal = async (} catch (error) {logger.error(`π« Job errored ${job.id} - ${error}`, { error });- // Filter out TransportableErrors (flow control)+ // Skip TransportableErrors: they're flow control, not failures.if (!(error instanceof TransportableError)) {Sentry.captureException(error, {data: {@@ -114,7 +112,6 @@ const processDeepResearchJobInternal = async (}try {- // Move job to failed state in Redisawait job.moveToFailed(error, token, false);} catch (e) {logger.error("Failed to move job to failed state in Redis", { error });@@ -182,7 +179,7 @@ const processGenerateLlmsTxtJobInternal = async (} catch (error) {logger.error(`π« Job errored ${job.id} - ${error}`, { error });- // Filter out TransportableErrors (flow control)+ // Skip TransportableErrors: they're flow control, not failures.if (!(error instanceof TransportableError)) {Sentry.captureException(error, {data: {@@ -263,9 +260,9 @@ const workerFun = async (const worker = new Worker(queue.name, null, {connection: getRedisConnection(),- lockDuration: 60 * 1000, // 60 seconds- stalledInterval: 60 * 1000, // 60 seconds- maxStalledCount: 10, // 10 times+ lockDuration: 60 * 1000,+ stalledInterval: 60 * 1000,+ maxStalledCount: 10,});worker.startStalledCheckTimer();@@ -293,7 +290,7 @@ const workerFun = async (});}- await sleep(cantAcceptConnectionInterval); // more sleep+ await sleep(cantAcceptConnectionInterval);continue;} else if (!currentLiveness) {logger.info("Not accepting jobs because the liveness check failed");@@ -410,7 +407,6 @@ const crawlFinishWorker = async () => {}};-// Start all workersconst app = Express();let currentLiveness: boolean = true;