Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Polish comments and docs in monitoring services

Developers Digest79eaf1f63aba
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.ts
index bd141f9fe..01dee9b74 100644
--- a/apps/api/src/controllers/v2/monitor.ts
+++ b/apps/api/src/controllers/v2/monitor.ts
@@ -376,7 +376,7 @@ export async function updateMonitorController(
}),
);
- // Only re-sync when notification config actually changed.
+ // Only re-sync when notification config changed.
let subscriptions: Awaited<
ReturnType<typeof loadEmailRecipientSubscriptions>
> = [];
@@ -594,12 +594,11 @@ export async function getMonitorCheckController(
});
}
-// Unauthenticated POST endpoints β€” the token is the credential. Backs the
-// firecrawl-web confirm/unsubscribe pages; POST-only so passive GET scanners
-// can't consume tokens against the dashboard URL.
+// Unauthenticated: the token is the credential. POST-only so passive GET
+// scanners can't consume tokens via the dashboard URL.
const emailActionBodySchema = z.object({
- // 32-byte base64url is 43 chars; range leaves room for future formats.
+ // 32-byte base64url is 43 chars; range leaves room for other formats.
token: z.string().min(16).max(64),
});
@@ -653,8 +652,7 @@ export async function confirmMonitorEmailController(
const monitorName = await getMonitorNameById(row.monitor_id);
- // Heuristic: if confirmed_at is older than 5s, this call was a no-op
- // (already confirmed). Used only to flavor the rendered page.
+ // confirmed_at older than 5s means this call was a no-op (already confirmed).
let result: "confirmed" | "already_confirmed" | "already_unsubscribed";
if (row.status === "unsubscribed") {
result = "already_unsubscribed";
diff --git a/apps/api/src/services/monitoring/queue.test.ts b/apps/api/src/services/monitoring/queue.test.ts
index ab8191803..3211071ec 100644
--- a/apps/api/src/services/monitoring/queue.test.ts
+++ b/apps/api/src/services/monitoring/queue.test.ts
@@ -1,7 +1,3 @@
-// Unit coverage for the branch's core RabbitMQ rework: prefetch(1), the
-// subscribe dedup guard, the message ack/nack contract, and β€” most importantly β€”
-// the reconnect/resubscribe wrapper (the A4 fix: a dropped consumer must
-// re-subscribe and never go permanently deaf).
import type { Mock } from "vitest";
const { connectMock, channelMock, connectionMock } = vi.hoisted(() => {
@@ -40,7 +36,6 @@ vi.mock("../../lib/logger", () => {
return { logger: mk() };
});
-// Pull the close handler that createChannel registered via ch.on("close", cb).
function consumeChannelCloseHandler(): () => void {
const call = (channelMock.on as Mock).mock.calls.find(c => c[0] === "close");
return call?.[1] as () => void;
@@ -75,7 +70,6 @@ describe("monitoring queue (RabbitMQ wrapper)", () => {
await q.consumeMonitorCheckJobs(vi.fn());
await q.consumeMonitorCheckJobs(vi.fn());
- // subscribedQueues guard => consume attached exactly once for the queue.
expect(channelMock.consume).toHaveBeenCalledTimes(1);
});
@@ -100,7 +94,7 @@ describe("monitoring queue (RabbitMQ wrapper)", () => {
expect(channelMock.nack).not.toHaveBeenCalled();
const boom = vi.fn().mockRejectedValue(new Error("handler boom"));
- const q2mod = q; // same module instance, second queue consumer
+ const q2mod = q;
await q2mod.consumeMonitorSearchCheckJobs(boom);
const onSearchMessage = channelMock.consume.mock.calls.at(-1)![1];
const msg2 = {
@@ -120,15 +114,13 @@ describe("monitoring queue (RabbitMQ wrapper)", () => {
await q.consumeMonitorCheckJobs(vi.fn());
expect(channelMock.consume).toHaveBeenCalledTimes(1);
- // Simulate the consume channel closing (a RabbitMQ blip).
const onClose = consumeChannelCloseHandler();
expect(onClose).toBeTypeOf("function");
onClose();
- // The reconnect is scheduled on a backoff timer; advance past it.
+ // Reconnect is on a backoff timer; advance past it.
await vi.advanceTimersByTimeAsync(1500);
- // The registered consumer was re-attached on the fresh channel.
expect(channelMock.consume.mock.calls.length).toBeGreaterThanOrEqual(2);
expect(channelMock.consume.mock.calls.at(-1)![0]).toBe("monitor.checks");
} finally {
diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.ts
index bd50efc67..caed11082 100644
--- a/apps/api/src/services/monitoring/queue.ts
+++ b/apps/api/src/services/monitoring/queue.ts
@@ -6,8 +6,7 @@ const MONITOR_CHECK_QUEUE = "monitor.checks";
const MONITOR_CHECK_DLX = "monitor.checks.dlx";
const MONITOR_CHECK_DLQ = "monitor.checks.dlq";
-// Search checks are far heavier than scrape/crawl checks, so they get a dedicated
-// queue + consumer β€” a burst of them can't starve the page/site/batch checks.
+// Dedicated queue so a burst of heavy search checks can't starve page/site/batch checks.
const MONITOR_SEARCH_CHECK_QUEUE = "monitor.checks.search";
const MONITOR_SEARCH_CHECK_DLX = "monitor.checks.search.dlx";
const MONITOR_SEARCH_CHECK_DLQ = "monitor.checks.search.dlq";
@@ -23,33 +22,26 @@ export type MonitorCheckJobData = {
type ConsumerHandler = (data: MonitorCheckJobData) => Promise<void>;
let connection: amqp.ChannelModel | null = null;
-// Publish and consume use separate channels: a channel exception on one side
-// (a failed publish, or a PRECONDITION_FAILED on assert) must not tear down the
-// other.
+// Separate publish/consume channels so an exception on one can't tear down the other.
let publishChannel: amqp.Channel | null = null;
let consumeChannel: amqp.Channel | null = null;
-// In-flight creation promises. Channel creation suspends on async asserts before
-// the channel is assigned, so two concurrent callers (e.g. the two consume
-// registrations started together at worker boot) would each create a channel and
-// orphan one. Memoizing the promise serializes concurrent callers onto one.
+// Memoized creation promises serialize concurrent callers onto one channel; without
+// this, two callers racing during the async asserts would each create one and orphan one.
let publishChannelPromise: Promise<amqp.Channel> | null = null;
let consumeChannelPromise: Promise<amqp.Channel> | null = null;
-// Registry of every consumer registered this process, so we can re-attach them
-// after a connection/channel drop. Without this the worker β€” which never
-// produces and so never re-triggers channel creation β€” goes permanently deaf
-// after any RabbitMQ blip, and only a restart heals it.
+// Registry of consumers so a reconnect can re-attach them; the worker never produces,
+// so without this it goes permanently deaf after any RabbitMQ blip until restart.
const registeredConsumers: Array<{ queue: string; handler: ConsumerHandler }> =
[];
-// Queues with a live consumer on the CURRENT consume channel. Cleared whenever
-// the channel drops, so a reconnect re-subscribes exactly once per queue and
-// repeated drops can't pile up duplicate consumers.
+// Queues with a live consumer on the CURRENT consume channel; cleared on drop so a
+// reconnect re-subscribes exactly once per queue and can't pile up duplicates.
const subscribedQueues = new Set<string>();
let reconnectInFlight = false;
const RECONNECT_BASE_DELAY_MS = 1000;
const RECONNECT_MAX_DELAY_MS = 30000;
-// Declare a check queue + its dead-letter exchange/queue (same topology for both).
+// Declare a check queue plus its dead-letter exchange/queue.
async function assertCheckQueue(
ch: amqp.Channel,
queue: string,
@@ -90,8 +82,7 @@ async function assertTopology(ch: amqp.Channel): Promise<void> {
);
}
-// Drop all cached state and, if any consumers were registered, kick off a
-// backoff reconnect that re-asserts topology and re-subscribes every consumer.
+// Drop cached state and, if any consumers are registered, kick off a backoff reconnect.
function handleConnectionDrop(reason: string, error?: unknown): void {
if (connection || publishChannel || consumeChannel) {
logger.warn("Monitor queue dropped β€” will reconnect", { reason, error });
@@ -128,11 +119,9 @@ function scheduleReconnect(delayMs: number): void {
error,
nextDelayMs: nextDelay,
});
- // Best-effort close the half-set-up channel, then reset ALL cached state β€”
- // including subscribedQueues. If a mid-loop subscribe() failed on an
- // otherwise-healthy channel (so its "close" never fires), leaving
- // subscribedQueues populated would make the next reconnect skip that queue
- // (subscribe() early-returns on subscribedQueues.has) and deafen it forever.
+ // Reset ALL cached state including subscribedQueues: a mid-loop subscribe()
+ // failure on a healthy channel never fires "close", so a stale entry would
+ // make the next reconnect skip (subscribe early-returns) and deafen that queue.
await consumeChannel?.close().catch(() => {});
connection = null;
publishChannel = null;
@@ -153,8 +142,7 @@ async function getConnection(): Promise<amqp.ChannelModel> {
throw new Error("NUQ_RABBITMQ_URL is not configured");
}
- // Name the connection so operators can identify (and isolate) monitoring
- // traffic in `rabbitmqctl list_connections` / the management UI.
+ // Name the connection so operators can identify monitoring traffic in the management UI.
const conn = await amqp.connect(url, {
clientProperties: { connection_name: "monitoring-queue" },
});
@@ -170,9 +158,8 @@ async function createChannel(label: string): Promise<amqp.Channel> {
const conn = await getConnection();
const ch = await conn.createChannel();
await assertTopology(ch);
- // Channels can close independently of the connection (e.g. a
- // PRECONDITION_FAILED from assertQueue). Without these handlers the cached
- // channel would stay non-null and every send/consume would throw forever.
+ // Channels can close independently of the connection (e.g. PRECONDITION_FAILED);
+ // without this the cached channel stays non-null and every send/consume throws forever.
ch.on("close", () => {
if (label === "publish" && publishChannel === ch) {
publishChannel = null;
@@ -254,7 +241,6 @@ async function consumeQueue(
queue: string,
handler: ConsumerHandler,
): Promise<void> {
- // Record the registration so a reconnect can re-attach this consumer.
if (!registeredConsumers.some(c => c.queue === queue)) {
registeredConsumers.push({ queue, handler });
}
@@ -267,11 +253,9 @@ async function subscribe(
queue: string,
handler: ConsumerHandler,
): Promise<void> {
- // Already consuming this queue on the current channel β€” don't add a duplicate
- // (a reconnect could otherwise re-subscribe an already-subscribed queue).
+ // Don't add a duplicate consumer if a reconnect re-subscribes an already-subscribed queue.
if (subscribedQueues.has(queue)) return;
- // Per-consumer prefetch (amqplib default): each consumer gets its own in-flight
- // slot, so the search and default consumers don't block each other.
+ // Per-consumer prefetch (amqplib default) so search and default consumers don't block each other.
await ch.prefetch(1);
await ch.consume(
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.ts
index 292dd0f9a..ed78c69d7 100644
--- a/apps/api/src/services/monitoring/runner.test.ts
+++ b/apps/api/src/services/monitoring/runner.test.ts
@@ -69,7 +69,7 @@ describe("monitoring runner", () => {
});
it("uses the shorter search timeout when the check has a search target", () => {
- // 11 minutes old: past the 10-min search cutoff but well under 1 hour.
+ // 11 min: past the 10-min search cutoff, under the 1-hour scrape cutoff.
const elevenMinAgo = new Date(
now.getTime() - 11 * 60 * 1000,
).toISOString();
@@ -78,22 +78,20 @@ describe("monitoring runner", () => {
updated_at: now.toISOString(),
created_at: now.toISOString(),
};
- // Scrape check: not stale yet (1-hour timeout).
expect(
isMonitorCheckStale(
{ ...base, target_results: [{ type: "scrape", targetId: "t1" }] },
now,
),
).toBe(false);
- // Search check: stale (10-minute timeout).
expect(
isMonitorCheckStale(
{ ...base, target_results: [{ type: "search", targetId: "t1" }] },
now,
),
).toBe(true);
- // MIXED search+crawl: keeps the 1-hour timeout (the crawl can legitimately
- // run for many minutes) β€” must NOT be reaped at 11 minutes.
+ // Mixed search+crawl keeps the 1-hour timeout: the crawl can legitimately
+ // run for many minutes and must not be reaped at 11 minutes.
expect(
isMonitorCheckStale(
{
@@ -110,10 +108,8 @@ describe("monitoring runner", () => {
});
});
-// The redelivery-idempotency guard: on a re-delivered check, the runner must
-// recognize a search target that ALREADY completed (searchCompleted:true) and
-// restore its persisted figures instead of re-running the search + re-scraping +
-// re-billing. findCompletedSearchTargetRun is that decision.
+// On a re-delivered check, an already-completed search target (searchCompleted)
+// must restore persisted figures instead of re-running/re-scraping/re-billing.
describe("findCompletedSearchTargetRun (redelivery idempotency)", () => {
const completed = {
type: "search",
@@ -169,9 +165,8 @@ describe("findCompletedSearchTargetRun (redelivery idempotency)", () => {
null,
"garbage",
];
- // t1's first (incomplete) entry must not match; the completed t1 does β€” but
- // since find() returns the first match and the incomplete one is filtered by
- // searchCompleted, the completed entry (also t1) is the one returned.
+ // t1's first (incomplete) entry is filtered by searchCompleted, so the
+ // completed t1 entry is the one returned.
const found = findCompletedSearchTargetRun(results, "t1");
expect(found).toMatchObject({ searchCredits: 2 });
});
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 89386bd56..070c450c4 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -120,8 +120,7 @@ type MonitorTargetRun =
| {
targetId: string;
type: "search";
- // Set true only after the inline search stamps its credits; the reconciler
- // waits on this so it never finalizes with credits still at 0.
+ // Set only after the inline search stamps credits; reconciler waits on this so it never finalizes with credits at 0.
searchCompleted?: boolean;
resultCount?: number;
matches?: number;
@@ -260,10 +259,9 @@ export function estimateActualCredits(doc: any, options?: any): number {
return creditsUsed;
}
const formats = Array.isArray(options?.formats) ? options.formats : [];
- // Only charge the JSON-extraction premium when extraction actually produced a
- // document.json. When it failed, the page was still scraped, so fall back to
- // the base scrape credit rather than billing for extraction that never ran.
- // Deterministic JSON costs 7 (it generates a reusable extractor); plain JSON 5.
+ // Only charge the JSON-extraction premium when extraction produced a json; a
+ // failed extraction still scraped the page, so fall back to base credit.
+ // Deterministic JSON costs 7 (reusable extractor); plain JSON 5.
const producedJson = doc?.json != null;
if (!producedJson) return 1;
if (includesFormat(formats, "deterministicJson")) return 7;
@@ -332,11 +330,9 @@ async function runSingleScrape(params: {
};
}
-// Deep-mode search-monitor page scrape. Runs inline via processJobInternal
-// (skipNuq:true, bypassBilling) β€” it is NOT routed through NuQ, so it bypasses
-// per-team/global scrape concurrency; the caller bounds the fan-out via
-// SEARCH_SCRAPE_CONCURRENCY (search/run.ts). Never billed per-page (search
-// monitors bill flat at the check level).
+// Deep-mode search-monitor page scrape, inline (skipNuq) so it bypasses scrape
+// concurrency; caller bounds fan-out via SEARCH_SCRAPE_CONCURRENCY. Never billed
+// per-page β€” search monitors bill flat at the check level.
async function scrapeSearchMonitorPage(params: {
teamId: string;
checkId: string;
@@ -610,7 +606,7 @@ async function runCrawlTarget(params: {
body.scrapeOptions.skipTlsVerification,
);
} catch {
- // Crawls tolerate robots fetch failures in the public controller too.
+ // Non-fatal robots fetch failure, same as the public crawl controller.
}
sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
@@ -833,9 +829,8 @@ async function sendNotifications(params: {
}
const nonSamePages = params.pages.filter(page => page.status !== "same");
- // Pull the unified-diff text for up to 5 meaningful changed pages so the
- // email leads with the actual diff. Cheap GCS reads, parallelised. Errors
- // are swallowed per-page so a single GCS hiccup doesn't drop the alert.
+ // Pull diff text for up to 5 meaningful changed pages so the email leads with
+ // the diff. Errors swallowed per-page so one GCS hiccup doesn't drop the alert.
const diffEligible = nonSamePages
.filter(
p => p.status === "changed" && (!p.judgment || p.judgment.meaningful),
@@ -1001,7 +996,7 @@ async function enqueueMonitorCrawlTarget(params: {
body.scrapeOptions.skipTlsVerification,
);
} catch {
- // Non-fatal, same as the public crawl controller.
+ // Non-fatal robots fetch failure, same as the public crawl controller.
}
sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
@@ -1041,7 +1036,7 @@ async function enqueueMonitorCrawlTarget(params: {
return params.targetRun;
}
-// Runs inline, then persists onto the same monitor_pages / monitor_check_pages
+// Runs inline, persisting onto the same monitor_pages / monitor_check_pages
// tables the reconciler tallies.
async function runMonitorSearchTarget(params: {
monitor: MonitorRow;
@@ -1054,7 +1049,7 @@ async function runMonitorSearchTarget(params: {
summary: string;
judgeDegraded: boolean;
degradedReason: string | null;
- // Flat, deterministic credits, recorded onto target_results by the caller.
+ // Flat credits, recorded onto target_results by the caller.
searchCredits: number;
judgeCredits: number;
resultsJudged: number;
@@ -1102,8 +1097,7 @@ async function runMonitorSearchTarget(params: {
id: target.id,
queries: target.queries,
searchWindow: target.searchWindow,
- // depth/alertMode aren't settable via the API, but stored targets may
- // still carry them (back-compat), so pass them through.
+ // depth/alertMode aren't API-settable but stored targets may carry them (back-compat); pass through.
alertMode: target.alertMode ?? "first_match",
includeDomains: target.includeDomains,
excludeDomains: target.excludeDomains,
@@ -1130,7 +1124,6 @@ async function runMonitorSearchTarget(params: {
}),
});
- // Flat credits recorded onto target_results (summed there at finalization).
const searchCredits = result.searchCredits;
const judgeCredits = result.judgeCredits;
@@ -1166,8 +1159,7 @@ async function runMonitorSearchTarget(params: {
});
}
- // Clear any rows from a prior (crashed/redelivered) run of this same check so
- // the insert is effectively a replace β€” a redelivery can't duplicate pages.
+ // Clear rows from a prior (crashed/redelivered) run so the insert is a replace, not a duplicate.
await deleteMonitorCheckPages({ checkId: check.id, targetId: target.id });
await insertMonitorCheckPages(pages);
@@ -1197,9 +1189,8 @@ async function runMonitorSearchTarget(params: {
};
}
-// Find a persisted search target run that already completed, so a redelivered
-// check can restore its figures instead of re-running (and re-billing) it.
-// Exported for unit coverage of the redelivery-idempotency decision.
+// Find a completed search target run so a redelivered check can restore its
+// figures instead of re-running (and re-billing) it.
export function findCompletedSearchTargetRun(
targetResults: unknown,
targetId: string,
@@ -1308,10 +1299,8 @@ export async function processMonitorCheckJob(
} else if (target.type === "crawl" && targetRun.type === "crawl") {
await enqueueMonitorCrawlTarget({ monitor, check, target, targetRun });
} else if (target.type === "search" && targetRun.type === "search") {
- // If this check was already run to completion for this target (a
- // redelivery after the inline work finished but before the ack),
- // restore the persisted figures instead of re-running β€” re-running
- // would re-bill the search and re-scrape every page.
+ // Redelivery after inline work finished but before ack: restore persisted
+ // figures instead of re-running, which would re-bill and re-scrape.
const priorRun = findCompletedSearchTargetRun(
initialCheck.target_results,
target.id,
@@ -1335,14 +1324,11 @@ export async function processMonitorCheckJob(
targetRun.searchCredits = searchResult.searchCredits;
targetRun.judgeCredits = searchResult.judgeCredits;
targetRun.resultsJudged = searchResult.resultsJudged;
- // Set LAST, after credits are stamped, so the reconciler never finalizes
- // with credits still at 0.
+ // Set last, after credits are stamped, so the reconciler never finalizes with credits at 0.
targetRun.searchCompleted = true;
- // Persist immediately (not just at the end of the loop): the search work
- // β€” external search calls + per-page scrapes β€” already happened. If we
- // crash before the end-of-loop flush, a redelivery would re-run and
- // re-scrape everything. Flushing searchCompleted now lets
- // findCompletedSearchTargetRun short-circuit the redelivery.
+ // Persist now, not just at end-of-loop: the search work already happened,
+ // so a crash before the final flush would re-run and re-scrape everything;
+ // flushing searchCompleted lets findCompletedSearchTargetRun short-circuit.
await updateMonitorCheck(check.id, {
target_results: targetResults,
});
diff --git a/apps/api/src/services/monitoring/search/billing.ts b/apps/api/src/services/monitoring/search/billing.ts
index cca5cf7e0..a4d3c2f77 100644
--- a/apps/api/src/services/monitoring/search/billing.ts
+++ b/apps/api/src/services/monitoring/search/billing.ts
@@ -1,8 +1,6 @@
const SEARCH_CREDITS_PER_TEN_RESULTS = 2;
const SEARCH_CREDITS_PER_TEN_RESULTS_ZDR = 10;
-// The single source of truth for the per-judged-result judge credit rate.
-// Tests MUST import this rather than re-declaring a literal (which previously
-// drifted to 5 and silently misrepresented billing).
+// Single source of truth for the judge credit rate; import it rather than re-declaring (a duplicate literal previously drifted to 5).
export const SEARCH_JUDGE_CREDITS_PER_RESULT = 1;
export function searchCreditsForResultCount(
diff --git a/apps/api/src/services/monitoring/search/cost.test.ts b/apps/api/src/services/monitoring/search/cost.test.ts
index 88ab71569..fc094c90b 100644
--- a/apps/api/src/services/monitoring/search/cost.test.ts
+++ b/apps/api/src/services/monitoring/search/cost.test.ts
@@ -44,14 +44,14 @@ describe("search-monitor FLAT judge billing rate", () => {
});
it("computes judge credits as 1 * resultsJudged", () => {
- expect(judgeCreditsForJudgedCount(0)).toBe(0); // raw / judge-off
- expect(judgeCreditsForJudgedCount(3)).toBe(3); // deep, 3 results judged
+ expect(judgeCreditsForJudgedCount(0)).toBe(0);
+ expect(judgeCreditsForJudgedCount(3)).toBe(3);
expect(judgeCreditsForJudgedCount(10)).toBe(10);
});
});
-// Billed count == pages with judgedThisRun=true (not every page, not every page
-// with a concept β€” reused/skipped pages keep a stale concept but aren't judged).
+// Billed count == pages with judgedThisRun=true: reused/skipped pages keep a
+// stale concept but aren't judged, so concept count overcounts.
describe("canonical judged-result count (billing == persisted signal)", () => {
type Page = {
searchStatus: string;
@@ -71,10 +71,10 @@ describe("canonical judged-result count (billing == persisted signal)", () => {
concept: "stale-1",
},
{ searchStatus: "watching", judgedThisRun: false, concept: "stale-2" },
- { searchStatus: "skipped" }, // no verdict, no judgedThisRun
+ { searchStatus: "skipped" },
];
- expect(pages.filter(p => p.concept).length).toBe(3); // the misleading count
- expect(billedJudgedCount(pages)).toBe(1); // the canonical, billed count
+ expect(pages.filter(p => p.concept).length).toBe(3); // misleading count
+ expect(billedJudgedCount(pages)).toBe(1); // canonical billed count
expect(judgeCreditsForJudgedCount(billedJudgedCount(pages))).toBe(1);
});
@@ -86,6 +86,6 @@ describe("canonical judged-result count (billing == persisted signal)", () => {
{ searchStatus: "already_seen", judgedThisRun: true },
{ searchStatus: "skipped" }, // verdict failed -> not billed
];
- expect(billedJudgedCount(pages)).toBe(4); // -> 1*4 = 4 judge credits
+ expect(billedJudgedCount(pages)).toBe(4); // -> 4 judge credits
});
});
diff --git a/apps/api/src/services/monitoring/search/dedupe.ts b/apps/api/src/services/monitoring/search/dedupe.ts
index ed2e55885..fd76f9c30 100644
--- a/apps/api/src/services/monitoring/search/dedupe.ts
+++ b/apps/api/src/services/monitoring/search/dedupe.ts
@@ -22,10 +22,7 @@ export function stableSerpFingerprint(src: {
snippet?: string;
description?: string;
}): string {
- // A search result is identified by its canonical URL. Snippets and titles drift
- // between searches (engines reword/truncate them), so hashing that text would
- // re-flag the same URL as "changed" every check β€” causing the same result to be
- // re-judged and re-billed. Keying on the URL keeps already-seen results free.
+ // Key on canonical URL only: snippets/titles drift between searches, so hashing them would re-flag, re-judge and re-bill the same URL.
return createHash("sha256")
.update(canonicalizeUrl(src.url ?? ""))
.digest("hex")
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
index 7bb332da6..85531b365 100644
--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -2,10 +2,7 @@ import type { Logger } from "winston";
import type { SearchVerdict } from "./judge";
import { canonicalizeUrl } from "./dedupe";
-// vi.mock is hoisted above declarations, so the mocks its factories reference
-// are created in vi.hoisted() (also hoisted) to avoid any TDZ surprises.
-// The lean pipeline has a single LLM stage: the deep-mode per-page JSON verdict,
-// produced by the injected scrapePage.
+// Mocks live in vi.hoisted() so the hoisted vi.mock factories can reference them.
const { searchMock, scrapeURLMock } = vi.hoisted(() => ({
searchMock: vi.fn(),
scrapeURLMock: vi.fn(),
@@ -13,8 +10,7 @@ const { searchMock, scrapeURLMock } = vi.hoisted(() => ({
vi.mock("uuid", () => ({ v7: () => "00000000-0000-7000-8000-000000000000" }));
vi.mock("../../../search/v2", () => ({
- // Tests set the mock to resolve a bare result array; the real search() returns
- // a SearchV2Response ({ web: [...] }), so wrap arrays to match that contract.
+ // Tests resolve a bare array; wrap it to match search()'s SearchV2Response shape.
search: async (...a: unknown[]) => {
const r = await searchMock(...a);
return Array.isArray(r) ? { web: r } : r;
@@ -63,8 +59,7 @@ function runParams(
...targetOver,
},
monitorCheckId: "check-1",
- // scrapeURLMock keeps the legacy { success, document } shape; adapt it to the
- // ScrapeSearchResult the injected scrapePage now returns.
+ // Adapt scrapeURLMock's legacy { success, document } shape to ScrapeSearchResult.
scrapePage: async (...a: unknown[]) => {
const r = (await scrapeURLMock(...a)) as {
success?: boolean;
@@ -223,24 +218,22 @@ describe("event state stamps + judgment on alert pages", () => {
describe("judgeEnabled gates the LLM judge", () => {
it("judge OFF behaves like raw: no LLM stages, no LLM credits, no concept/rationale", async () => {
- // Monitor still has a goal and depth:"deep" β€” but judgeEnabled=false must
- // collapse to raw and skip every LLM stage.
+ // judgeEnabled=false collapses to raw and skips every LLM stage, even with goal + depth:"deep".
searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
const result = await runSearchTarget(
runParams({ depth: "deep" }, { judgeEnabled: false }),
);
- // No judge stage ran: raw mode does not scrape.
+ // Raw mode does not scrape.
expect(scrapeURLMock).not.toHaveBeenCalled();
- // No judge credits billed (raw β†’ nothing judged); search still ran and was billed.
+ // Nothing judged, but search still ran and was billed.
expect(result.judgeCredits).toBe(0);
expect(result.resultsJudged).toBe(0);
expect(result.resultCount).toBe(2);
- // Raw alerts: deterministic searchStatus from dedup, but no LLM
- // concept/rationale on any upsert.
+ // Deterministic searchStatus from dedup, but no LLM concept/rationale.
for (const upsert of result.pageUpserts) {
expect(upsert.scraped).toBe(false);
expect(upsert.metadata.concept).toBeUndefined();
@@ -286,7 +279,6 @@ describe("judgeEnabled gates the LLM judge", () => {
describe("deep-path scrape failures mark the check degraded (no silent empty)", () => {
beforeEach(() => {
- // Two candidates scraped; scrape outcome varies per test.
searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
});
@@ -295,15 +287,15 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",
const result = await runSearchTarget(runParams({ depth: "deep" }));
- // The false-negative we must surface: 0 judged / 0 alerts, but degraded.
+ // 0 judged / 0 alerts but degraded β€” the false-negative we must surface.
expect(result.resultsJudged).toBe(0);
expect(result.matches).toBe(0);
expect(result.judgeDegraded).toBe(true);
expect(result.degradedReason).toMatch(/judged|incomplete/i);
- // A non-empty summary so the check never reads as a clean "nothing new".
+ // Non-empty summary so the check never reads as a clean "nothing new".
expect(result.summary).not.toBe("");
- // Each unscrapeable result is persisted as a "skipped" page so it surfaces
- // as an error check page (skipped -> error) instead of silently vanishing.
+ // Each unscrapeable result is persisted as "skipped" so it surfaces as an
+ // error check page instead of silently vanishing.
const skippedUpserts = result.pageUpserts.filter(
u => u.status === "skipped",
);
@@ -338,8 +330,8 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",
const result = await runSearchTarget(runParams({ depth: "deep" }));
- // One result was actually judged (ignored) β€” that's a real evaluation, not a
- // failure, so the check is NOT degraded even though a sibling scrape failed.
+ // One result judged (ignored) is a real evaluation, so NOT degraded even
+ // though a sibling scrape failed.
expect(result.resultsJudged).toBe(1);
expect(result.judgeDegraded).toBe(false);
expect(result.degradedReason).toBeNull();
@@ -357,7 +349,7 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",
const result = await runSearchTarget(runParams({ depth: "deep" }));
- // Everything scraped fine and was evaluated; judge ignored it all. Normal.
+ // Everything scraped and was evaluated; judge ignored it all. Normal.
expect(result.resultsJudged).toBe(2);
expect(result.matches).toBe(0);
expect(result.judgeDegraded).toBe(false);
@@ -373,9 +365,8 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",
expect(result.judgeDegraded).toBe(false);
expect(result.degradedReason).toBeNull();
expect(scrapeURLMock).not.toHaveBeenCalled();
- // An empty SERP hedges with a BOUNDED retry (1 initial + 2 quick retries),
- // NOT the old 5-attempt / ~11s growing backoff. Lock the budget so a genuinely
- // empty query concludes fast instead of stalling the whole check.
+ // An empty SERP hedges with a bounded retry (1 initial + 2 quick), so a
+ // genuinely empty query concludes fast instead of stalling the check.
expect(searchMock).toHaveBeenCalledTimes(3);
});
@@ -386,7 +377,7 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",
serpRow(3),
serpRow(4),
]);
- // Only result 1 scrapes + judges; results 2-4 all fail to scrape.
+ // Only result 1 scrapes + judges; 2-4 fail to scrape.
scrapeURLMock.mockImplementation(({ url }: { url: string }) =>
url === serpRow(1).url
? Promise.resolve({
@@ -404,8 +395,8 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",
expect(result.resultsJudged).toBe(1);
expect(result.scrapeFailures).toBe(3);
- // 3/(1+3) = 0.75 >= 0.5 -> soft partial-loss signal, but the run still
- // produced a verdict so it is NOT a hard degrade.
+ // 3/(1+3) = 0.75 >= 0.5 -> soft partial-loss signal, but a verdict was
+ // produced so it is NOT a hard degrade.
expect(result.partialScrapeLoss).toBe(true);
expect(result.judgeDegraded).toBe(false);
});
@@ -432,7 +423,7 @@ describe("run time budget (a single check can't wedge the consumer)", () => {
vi.useFakeTimers();
try {
searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
- // Scrapes never resolve β€” the pre-scrape can't finish on its own.
+ // Scrapes never resolve, so the pre-scrape can't finish on its own.
scrapeURLMock.mockReturnValue(new Promise(() => {}));
const pending = runSearchTarget(runParams({ depth: "deep" }));
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index 3083c196a..e5bea732d 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -16,8 +16,7 @@ import {
judgeCreditsForJudgedCount,
} from "./billing";
-// Shared dedup decision for a search result, used by both the pre-scrape pass and
-// the per-result loop so they never disagree on which results to scrape/judge.
+// Shared dedup decision so the pre-scrape pass and per-result loop never disagree.
function evaluateSerpCandidate(
c: { url: string; title: string; description: string },
knownPages: Map<string, KnownPage>,
@@ -58,31 +57,20 @@ function windowToTbs(window: string): string {
return "qdr:w";
}
-// A successful-but-empty SERP almost always means a genuine "no results" for this
-// query+window β€” NOT a transient failure. The backend can occasionally return 0 for
-// a query that succeeds a moment later, so hedge with a couple of QUICK retries, but
-// don't burn ~11s of growing backoff (the old [400,1200,3000,6000]) on a query that
-// simply has no matches β€” that latency stacks across queries and slows every check
-// whose results legitimately don't exist.
+// Empty SERPs are usually genuine "no results", not transient β€” hedge with a couple
+// of quick retries, but don't burn long backoff on queries that simply have no matches.
const SEARCH_EMPTY_RETRY_BACKOFF_MS = [500, 1000] as const;
-// Hard wall-clock budget for a single inline search run. The deep-mode page
-// scrapes are the only unbounded-ish step (each is capped at 20s, but a large
-// candidate set could still add up); if the concurrent pre-scrape blows past
-// this, we stop waiting and let the remaining candidates fall through as
-// skipped/degraded rather than holding the consumer. Kept well under the
-// 10-minute search stale timeout so a run always returns before the reaper.
+// Wall-clock budget per run; if the pre-scrape blows past it, remaining candidates
+// fall through as skipped/degraded. Kept under the 10-min stale timeout so a run
+// always returns before the reaper.
const SEARCH_RUN_BUDGET_MS = 4 * 60 * 1000;
-// Deep-mode page scrapes run INLINE in the worker process (skipNuq), so they are
-// NOT bounded by NuQ's per-team/global concurrency. Cap the pre-scrape fan-out
-// ourselves so a single maxResults=50 check can't launch 50 simultaneous
-// fetch+LLM-extraction scrapes and starve the worker that owns the search consumer.
+// Deep-mode scrapes run inline (skipNuq), so NuQ concurrency doesn't bound them; cap
+// the fan-out ourselves so one maxResults=50 check can't starve the search consumer.
const SEARCH_SCRAPE_CONCURRENCY = 6;
-// Run fn over items with at most `limit` in flight at once. Never rejects per the
-// caller's needs β€” fn is expected to swallow its own errors (it writes outcomes
-// into a shared cache).
+// Never rejects: fn is expected to swallow its own errors into a shared cache.
async function mapWithConcurrency<T>(
items: T[],
limit: number,
@@ -99,9 +87,8 @@ async function mapWithConcurrency<T>(
await Promise.all(pool);
}
-// Race a promise against a wall-clock deadline. Resolves false if the work
-// finishes first, true if the deadline hits. Never rejects; the underlying work
-// keeps running (bounded by its own per-scrape timeout) and populates its cache.
+// Resolves true if the deadline hits first, false if the work finishes. Never
+// rejects; the underlying work keeps running and populates its cache.
async function raceDeadline(
work: Promise<unknown>,
budgetMs: number,
@@ -198,9 +185,7 @@ export type KnownEvent = {
alertCount?: number;
};
-// Deep-mode page scrape. Injected by the caller so it can route through the
-// shared monitor scrape path (concurrency queue + bypassBilling), keeping this
-// module free of worker/queue dependencies. Resolves to null on scrape failure.
+// Injected by the caller so this module stays free of worker/queue dependencies.
export type ScrapeSearchResult = {
json: unknown;
markdown: string;
@@ -233,9 +218,8 @@ type SearchTargetRunResult = {
summary: string;
judgeDegraded: boolean;
degradedReason: string | null;
- // Per-page scrape failures in deep mode. partialScrapeLoss is a SOFT signal
- // (distinct from judgeDegraded): some results judged, but a majority of the
- // attempted scrapes failed β€” the run is usable but likely missed developments.
+ // partialScrapeLoss is a soft signal (distinct from judgeDegraded): some results
+ // judged, but most scrapes failed β€” usable but likely missed developments.
scrapeFailures: number;
partialScrapeLoss: boolean;
sources: SearchSource[];
@@ -345,8 +329,7 @@ export async function runSearchTarget(params: {
}
resultCount = candidates.length;
- // An empty retrieval is indistinguishable from "nothing matched", so log it to
- // make a genuine miss diagnosable.
+ // An empty retrieval is indistinguishable from "nothing matched"; log to diagnose.
if (resultCount === 0) {
logger.warn(
"search monitor: all queries returned no results after retries",
@@ -359,9 +342,7 @@ export async function runSearchTarget(params: {
}
const llmStagesAvailable = hasLlmProvider();
- // Judge depth is raw (no LLM) or deep (per-page scrape + verdict). Standard
- // (snippet-only judging) was removed: it was unreachable via the API (depth is
- // stripped) and used a divergent LLM provider; deep is the single judged path.
+ // raw (no LLM) or deep (per-page scrape + verdict); the former "standard" path was removed.
const depth: "raw" | "deep" =
!judgeEnabled || target.depth === "raw" ? "raw" : "deep";
@@ -371,17 +352,15 @@ export async function runSearchTarget(params: {
const recheckMs = target.recheckAfter ? windowToMs(target.recheckAfter) : 0;
- // Pre-scrape deep-mode pages (bounded concurrency) and cache them, so the loop
- // below isn't a serial scrape→judge→scrape chain. Only the I/O is parallel; all
- // dedup/event state stays in the single-threaded loop. These scrapes run inline
- // (skipNuq), so the fan-out is bounded by SEARCH_SCRAPE_CONCURRENCY, not NuQ.
+ // Pre-scrape deep-mode pages (bounded concurrency) into a cache so the loop isn't
+ // a serial scrape→judge chain. Only the I/O is parallel; dedup/event state stays
+ // in the single-threaded loop.
const deepDocs = new Map<
string,
{ doc: ScrapeSearchResult | null; error?: unknown }
>();
- // Set if the concurrent pre-scrape exceeds the run budget: any candidate not
- // yet cached is then treated as a scrape failure instead of being inline-scraped,
- // so the loop returns promptly rather than holding the consumer.
+ // If the pre-scrape exceeds the run budget, uncached candidates are treated as
+ // scrape failures rather than inline-scraped, so the loop returns promptly.
let budgetExceeded = false;
if (depth === "deep" && llmStagesAvailable) {
const judgePrompt = buildJudgePrompt(
@@ -533,8 +512,7 @@ export async function runSearchTarget(params: {
eventAlertCount,
eventLastAlertAt: nowIso,
},
- // Raw mode does not run the judge, so it carries no judgment β€” these are
- // surfaced as new results, not evaluated as "meaningful".
+ // Raw mode runs no judge: surfaced as new results, not evaluated as "meaningful".
});
continue;
}
@@ -542,7 +520,7 @@ export async function runSearchTarget(params: {
let verdict: SearchVerdict | null = null;
let realDate: string | null = null;
{
- // Deep mode: read the doc from the pre-scrape cache; scrape inline only as a fallback.
+ // Read from the pre-scrape cache; scrape inline only as a fallback.
let doc: ScrapeSearchResult | null;
const cached = deepDocs.get(canonical);
if (cached?.error !== undefined) {
@@ -556,9 +534,8 @@ export async function runSearchTarget(params: {
);
continue;
}
- // The pre-scrape ran out of time before caching this page. Treat it as a
- // scrape failure rather than inline-scraping (which would re-incur the time
- // we just budgeted out of), so the run returns promptly.
+ // Out of budget before caching this page: treat as a scrape failure rather
+ // than inline-scraping (which would re-incur the time we just budgeted out of).
if (!cached && budgetExceeded) {
judgeScrapeFailures += 1;
pushUnevaluatedPage("run budget exceeded before scrape");
@@ -654,8 +631,7 @@ export async function runSearchTarget(params: {
const eventKey = canonical;
const eventLabel = verdict.concept || canonical;
- // Re-alert on every new result only in every_new_result mode; otherwise the
- // canonical URL is deduped so we don't re-alert the same page.
+ // Re-alert on every new result only in every_new_result mode; otherwise dedup.
const alreadySatisfied =
satisfied.has(eventKey) && target.alertMode !== "every_new_result";
const eventMeta = { ...baseMeta, eventKey, eventLabel };
@@ -759,9 +735,8 @@ export async function runSearchTarget(params: {
? "results were found but none could be judged; results may be incomplete"
: null;
- // Soft signal (NOT judgeDegraded): the run produced verdicts, but a majority of
- // the attempted deep scrapes failed, so it likely missed real developments.
- // Surfaced for observability; does not fail the check.
+ // Soft signal (not judgeDegraded): verdicts produced but most scrapes failed, so
+ // it likely missed real developments. Surfaced for observability; doesn't fail the check.
const partialScrapeLoss =
resultsJudged > 0 &&
judgeScrapeFailures > 0 &&
diff --git a/apps/api/src/services/monitoring/stale.ts b/apps/api/src/services/monitoring/stale.ts
index 97c86775c..a56065b84 100644
--- a/apps/api/src/services/monitoring/stale.ts
+++ b/apps/api/src/services/monitoring/stale.ts
@@ -1,17 +1,12 @@
import type { MonitorCheckRow } from "./types";
export const MONITOR_CHECK_STALE_TIMEOUT_MS = 60 * 60 * 1000;
-// Search checks run their whole search+scrape+judge inline and realistically
-// finish in seconds-to-low-minutes. A stranded one (crash between the inline
-// work and the ack) should self-heal quickly rather than look dead for an hour.
+// Search checks run inline and finish in minutes; a stranded one should self-heal quickly, not look dead for an hour.
const MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS = 10 * 60 * 1000;
export const MONITOR_CHECK_STALE_ERROR =
"Monitor check exceeded the running timeout.";
-// The shorter search timeout only applies when EVERY target is a search target.
-// A mixed monitor (e.g. search + crawl) must keep the long timeout β€” its scrape
-// fan-out / crawl legitimately runs for many minutes and would otherwise be
-// falsely reaped at the 10-minute search cutoff.
+// Short timeout only when EVERY target is search; a mixed monitor's crawl/scrape fan-out legitimately runs for many minutes.
function isSearchOnlyCheck(check: { target_results?: unknown }): boolean {
const targetResults = check.target_results;
if (!Array.isArray(targetResults) || targetResults.length === 0) return false;
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index 2c3469148..50910a6c3 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -233,7 +233,7 @@ describe("FLAT deterministic search-monitor billing", () => {
});
it("judge ON: search call + flat 1 per judged result (capped at maxResults)", () => {
- // search = ceil(10/10)*2*1 = 2; judge = 1 * min(maxResults, 10*1) = 10.
+ // search 2 + judge min(maxResults, 10) = 10.
expect(estimateMonitorCreditsPerRun([searchTarget()], true)).toBe(12);
});
@@ -248,15 +248,14 @@ describe("FLAT deterministic search-monitor billing", () => {
maxResults: 20,
queries: ["a", "b", "c"],
});
- // search = ceil((20*3)/10)*2 = 12; judge = 1 * 20 (maxResults cap, NOT *queries)
- // = 20 β†’ 32. The runner judges at most maxResults candidates total.
+ // search ceil((20*3)/10)*2 = 12; judge caps at maxResults (20, not *queries) -> 32.
expect(estimateMonitorCreditsPerRun([t], true)).toBe(32);
});
});
describe("flatSearchTargetCredits (the deterministic actual)", () => {
it("sums searchCredits + judgeCredits across search target_results", () => {
- // The headline trace: search 2 + judge 1*3 = 5.
+ // search 2 + judge 1*3 = 5.
const targetResults = [
{
targetId: "search-1",
@@ -290,7 +289,7 @@ describe("FLAT deterministic search-monitor billing", () => {
{ targetId: "search-1", type: "search", searchCredits: 4 },
null,
"garbage",
- { type: "search" }, // missing credits β†’ contributes 0
+ { type: "search" }, // missing credits -> 0
]),
).toBe(4);
});
@@ -322,15 +321,14 @@ describe("FLAT deterministic search-monitor billing", () => {
});
it("never bills 0 on a check that returned results and judged them", () => {
- // results=6, judged=4 -> 2 + 4 = 6 (the judged#2 case that landed on 0).
+ // results=6, judged=4 -> 2 + 4 = 6 (regression: this case once landed on 0).
const credits = flatSearchTargetCredits([stampedSearchRun(6, 4)]);
expect(credits).toBe(6);
expect(credits).toBeGreaterThan(0);
});
it("the RETRY path still bills: rate-limited-then-succeeded run is summed", () => {
- // A run that hit rate-limiting, retried, and finally returned 6 results +
- // judged 4 must record the SAME deterministic figure as a clean run β€” the
+ // A retried run must record the same figure as a clean run; the
// searchCompleted=true guard ensures credits are stamped before summing.
const retried = stampedSearchRun(6, 4);
expect(retried.searchCompleted).toBe(true);
@@ -338,15 +336,14 @@ describe("FLAT deterministic search-monitor billing", () => {
});
it("search-only (judge off) bills just the search portion, never the judge", () => {
- // results=6, judged=0 -> 2 + 0 = 2 (the raw case, which is already correct).
+ // results=6, judged=0 -> 2 + 0 = 2 (raw case).
expect(flatSearchTargetCredits([stampedSearchRun(6, 0)])).toBe(2);
});
it("a still-running search (no credits stamped yet) sums to 0 β€” the reconciler must NOT finalize here", () => {
- // This is the pre-stamp shape the dispatcher writes before the inline
- // search finishes. flatSearchTargetCredits sees 0; the never-zero
- // guarantee instead relies on isMonitorCheckComplete refusing to finalize
- // a run whose searchCompleted flag is not yet true (see runner.ts).
+ // Pre-stamp shape the dispatcher writes before the inline search finishes.
+ // The never-zero guarantee relies on isMonitorCheckComplete refusing to
+ // finalize a run whose searchCompleted flag is not yet true (see runner.ts).
const pending = { targetId: "search-1", type: "search" as const };
expect((pending as any).searchCompleted).toBeUndefined();
expect(flatSearchTargetCredits([pending])).toBe(0);
@@ -355,8 +352,8 @@ describe("FLAT deterministic search-monitor billing", () => {
it("page-sum never bills search-target pages (cost is at the check level)", () => {
const targets: MonitorTarget[] = [searchTarget()];
- // Even with judgment + status, a search page contributes 0 to the page sum;
- // its credits live in target_results via flatSearchTargetCredits.
+ // A search page contributes 0 to the page sum; its credits live in
+ // target_results via flatSearchTargetCredits.
expect(
calculateMonitorCheckActualCreditsFromPages(
[
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index e391c8558..bbeefcf64 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -108,8 +108,7 @@ function estimateBaseCreditsPerPage(
credits += SCRAPE_OPTION_CREDIT_BONUS;
}
- // Deterministic JSON generates a reusable extractor and costs more than plain
- // JSON; both override the base scrape credit (mirrors estimateActualCredits).
+ // Deterministic JSON costs more than plain JSON; both override the base scrape credit.
if (usesDeterministicJson) {
credits = DETERMINISTIC_JSON_SCRAPE_CREDITS_PER_PAGE;
} else if (usesJsonCredits) {
@@ -206,7 +205,7 @@ export function estimateMonitorCreditsPerRun(
(sum, target) => sum + estimateTargetBaseCredits(target, judgeEnabled),
0,
);
- // Per-page judge allowance is scrape/crawl only (search judging is folded in above).
+ // Per-page judge allowance is scrape/crawl only; search judging is folded in above.
const judgeCredits = judgeEnabled
? targets.reduce(
(sum, target) =>
@@ -253,9 +252,8 @@ export function calculateMonitorCheckActualCreditsFromPages(
baseCreditsByTarget.get(page.target_id ?? "") ??
BASE_SCRAPE_CREDITS_PER_PAGE;
- // Monitor-specific fallback only: new rows should prefer metadata.creditsUsed
- // when the scrape path provides it. If it is missing, use retained monitor
- // metadata to avoid obvious undercounts for PDFs and special postprocessors.
+ // Fallback when metadata.creditsUsed is missing: use retained metadata to avoid
+ // undercounting PDFs and special postprocessors.
if (
target &&
shouldParsePDF(target.scrapeOptions?.parsers as any) &&
@@ -301,7 +299,7 @@ export function calculateMonitorCheckActualCreditsFromPages(
}
return pages.reduce((total, page) => {
- // Search pages carry no per-page credit β€” billed at check level.
+ // Search pages carry no per-page credit; billed at check level.
const target = targetsById.get(page.target_id ?? "");
if (target?.type === "search") {
return total;
@@ -323,7 +321,6 @@ export function calculateMonitorCheckActualCreditsFromPages(
}, 0);
}
-// Sum the flat searchCredits + judgeCredits recorded on a check's target_results.
export function flatSearchTargetCredits(targetResults: unknown): number {
if (!Array.isArray(targetResults)) return 0;
return targetResults.reduce((total: number, run: unknown) => {
@@ -390,8 +387,7 @@ export async function createMonitor(params: {
const estimatedCreditsPerMonth =
estimatedCreditsPerRun * estimateRunsPerMonth(params.intervalMs);
- // Omit goal/judge_enabled keys when undefined so a pre-migration DB
- // doesn't reject the insert. Migration lives in a separate repo.
+ // Omit goal/judge_enabled when undefined so a pre-migration DB doesn't reject the insert.
const insert: typeof schema.monitors.$inferInsert = {
id: uuidv7(),
team_id: params.teamId,
@@ -502,9 +498,8 @@ export async function updateMonitor(params: {
if (params.input.status !== undefined) patch.status = params.input.status;
if (params.input.webhook !== undefined)
patch.webhook = params.input.webhook ?? null;
- // Defense-in-depth against a wiped notification: only write when the caller
- // actually sent config. Treat an empty object (the legacy default-materialized
- // {}) as "leave unchanged" rather than clobbering the stored email settings.
+ // Only write when the caller sent config; treat empty {} (legacy default) as
+ // "leave unchanged" rather than clobbering stored email settings.
if (
params.input.notification !== undefined &&
params.input.notification !== null &&
@@ -530,10 +525,8 @@ export async function updateMonitor(params: {
patch.next_run_at = params.nextRunAt?.toISOString() ?? null;
}
- // Re-estimate whenever any cost input changed. Merge the patch with the
- // current monitor row so a goal/judge-only update still recalculates
- // against the existing targets + schedule, and a targets-only update
- // preserves an already-enabled judge.
+ // Re-estimate whenever any cost input changed, merging the patch with the current
+ // row so partial updates recalculate against existing targets/schedule/judge.
const costInputsChanged =
params.input.targets !== undefined ||
params.input.judgeEnabled !== undefined ||
@@ -865,10 +858,9 @@ export async function insertMonitorCheckPages(
);
}
-// Delete all check-page rows for a (check, target). Used to make the inline
-// search write idempotent: a redelivered search check clears its prior rows
-// before re-inserting, so a crash-and-redeliver can't leave duplicate pages.
-// Partition-safe (no unique constraint needed on the partitioned table).
+// Makes the inline search write idempotent: a redelivered check clears its prior
+// (check, target) rows before re-inserting, so crash-and-redeliver can't duplicate
+// pages. Partition-safe (no unique constraint needed).
export async function deleteMonitorCheckPages(params: {
checkId: string;
targetId: string;
diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.ts
index a6e5837b7..20838ae37 100644
--- a/apps/api/src/services/monitoring/types.ts
+++ b/apps/api/src/services/monitoring/types.ts
@@ -48,8 +48,8 @@ const monitorDomainSchema = z
"Domain must be a valid hostname without protocol or path",
);
-// depth/alertMode/recheckAfter are internal-only; scrapeOptions is unused for
-// search. Strip them before validation so older clients sending them don't 400.
+// Strip internal-only depth/alertMode/recheckAfter (and search-unused scrapeOptions)
+// before validation so older clients sending them don't 400.
const searchTargetSchema = z.preprocess(
value => {
if (value && typeof value === "object" && !Array.isArray(value)) {
@@ -138,9 +138,8 @@ const monitorNotificationInner = z.strictObject({
})
.optional(),
});
-// Create defaults a missing notification to {}. The UPDATE schema deliberately
-// does NOT (see updateMonitorSchema) β€” a default there would materialize {} on
-// any PATCH that omits notification and wipe the stored email config.
+// Create defaults a missing notification to {}; the UPDATE schema deliberately does
+// not (a default there would materialize {} on a PATCH and wipe the stored email config).
const monitorNotificationSchema = monitorNotificationInner
.optional()
.default({});
@@ -196,9 +195,8 @@ export const createMonitorSchema = createMonitorBaseSchema
.superRefine(requireGoalForSearchTargets)
.transform(applyJudgeEnabledDefault);
-// Patches are partial and may rely on the already-stored goal, so only reject
-// when the patch includes search targets AND explicitly clears the goal. The
-// controller re-validates the merged monitor.
+// Patches may rely on the already-stored goal, so only reject when the patch has
+// search targets AND explicitly clears the goal; the controller re-validates the merge.
function rejectGoalClearedWithSearchTargets(
input: { targets?: unknown; goal?: unknown; judgeEnabled?: unknown },
ctx: z.RefinementCtx,
@@ -226,13 +224,10 @@ export const updateMonitorSchema = createMonitorBaseSchema
.partial()
.extend({
status: z.enum(["active", "paused"]).optional(),
- // Override the create-path notification (which defaults to {}): on update a
- // default would materialize {} for a PATCH that omits notification and wipe
- // the stored email config. No default => omitted means "leave unchanged".
+ // No default => omitted means "leave unchanged"; a default would materialize {} on
+ // a PATCH and wipe the stored email config.
notification: monitorNotificationInner.optional(),
- // Same default-materialization trap: createMonitorBaseSchema gives
- // retentionDays .default(30), so a partial PATCH that omits it would parse to
- // 30 and silently reset a user's configured retention. Drop the default here.
+ // Drop the create-path .default(30): otherwise an omitting PATCH resets configured retention.
retentionDays: z.number().int().positive().max(365).optional(),
})
.superRefine(rejectGoalClearedWithSearchTargets)
@@ -265,16 +260,15 @@ export const monitorCheckDetailQuerySchema = z.object({
status: z.enum(["same", "new", "changed", "removed", "error"]).optional(),
});
-// Stripped from API input (see searchTargetSchema), but stored targets may
-// still carry them; surface as optional internal-only fields so runner/store
-// can read stored values without type errors.
+// Stripped from API input but stored targets may carry them; surface as optional
+// internal-only fields so runner/store can read stored values without type errors.
export type MonitorTarget = z.infer<typeof monitorTargetSchema> & {
id: string;
} & {
depth?: "raw" | "standard" | "deep";
alertMode?: "first_match" | "every_new_result" | "material_dev";
recheckAfter?: "1h" | "6h" | "24h" | "7d";
- // Present on scrape/crawl, absent on search; optional so union reads compile.
+ // Absent on search; optional so union reads compile.
scrapeOptions?: z.infer<typeof scrapeOptionsSchema>;
};
export type CreateMonitorRequest = z.infer<typeof createMonitorSchema>;
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index f85753f84..fa2366f59 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -87,11 +87,9 @@ const processDeepResearchJobInternal = async (
});
if (result.success) {
- // Move job to completed state in Redis and update research status
await job.moveToCompleted(result, token, false);
return result;
} else {
- // If the deep research failed but didn't throw an error
const error = new Error("Deep research failed without specific error");
await updateDeepResearch(job.data.researchId, {
status: "failed",
@@ -104,7 +102,7 @@ const processDeepResearchJobInternal = async (
} catch (error) {
logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });
- // Filter out TransportableErrors (flow control)
+ // Skip TransportableErrors: they're flow control, not failures.
if (!(error instanceof TransportableError)) {
Sentry.captureException(error, {
data: {
@@ -114,7 +112,6 @@ const processDeepResearchJobInternal = async (
}
try {
- // Move job to failed state in Redis
await job.moveToFailed(error, token, false);
} catch (e) {
logger.error("Failed to move job to failed state in Redis", { error });
@@ -182,7 +179,7 @@ const processGenerateLlmsTxtJobInternal = async (
} catch (error) {
logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });
- // Filter out TransportableErrors (flow control)
+ // Skip TransportableErrors: they're flow control, not failures.
if (!(error instanceof TransportableError)) {
Sentry.captureException(error, {
data: {
@@ -263,9 +260,9 @@ const workerFun = async (
const worker = new Worker(queue.name, null, {
connection: getRedisConnection(),
- lockDuration: 60 * 1000, // 60 seconds
- stalledInterval: 60 * 1000, // 60 seconds
- maxStalledCount: 10, // 10 times
+ lockDuration: 60 * 1000,
+ stalledInterval: 60 * 1000,
+ maxStalledCount: 10,
});
worker.startStalledCheckTimer();
@@ -293,7 +290,7 @@ const workerFun = async (
});
}
- await sleep(cantAcceptConnectionInterval); // more sleep
+ await sleep(cantAcceptConnectionInterval);
continue;
} else if (!currentLiveness) {
logger.info("Not accepting jobs because the liveness check failed");
@@ -410,7 +407,6 @@ const crawlFinishWorker = async () => {
}
};
-// Start all workers
const app = Express();
let currentLiveness: boolean = true;