Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(monitor): mark search check degraded when deep-path scrapes fail (no silent empty)

Developers Digest7b0f9a518fd8
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 719edcf9a..367e1b7ed 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -124,6 +124,12 @@ type MonitorTargetRun =
// True when a query failed at the provider after retries (not legitimately
// empty); persisted so an empty-but-degraded run isn't read as "no changes".
searchDegraded?: boolean;
+ // True when the deep (scrape+judge) path was expected to evaluate results
+ // but evaluated ~none because the per-result scrapes failed; persisted so a
+ // judged check that scraped nothing isn't read as a clean "no changes".
+ judgeDegraded?: boolean;
+ // Human-readable explanation for whichever degraded flag is set.
+ degradedReason?: string | null;
// Flat, deterministic credits recorded onto target_results when the search
// completes; actual_credits is summed from these, not reconstructed from
// per-page metadata, so a missing page / reconciler race can't zero them.
@@ -971,6 +977,8 @@ async function runMonitorSearchTarget(params: {
matches: number;
summary: string;
searchDegraded: boolean;
+ judgeDegraded: boolean;
+ degradedReason: string | null;
// Flat, deterministic credits, recorded onto target_results by the caller.
searchCredits: number;
judgeCredits: number;
@@ -983,6 +991,8 @@ async function runMonitorSearchTarget(params: {
matches: 0,
summary: "",
searchDegraded: false,
+ judgeDegraded: false,
+ degradedReason: null,
searchCredits: 0,
judgeCredits: 0,
resultsJudged: 0,
@@ -1095,6 +1105,8 @@ async function runMonitorSearchTarget(params: {
matches: result.matches,
summary: result.summary,
searchDegraded: result.searchDegraded,
+ judgeDegraded: result.judgeDegraded,
+ degradedReason: result.degradedReason,
searchCredits,
judgeCredits,
resultsJudged: result.resultsJudged,
@@ -1204,6 +1216,8 @@ export async function processMonitorCheckJob(
targetRun.matches = searchResult.matches;
targetRun.summary = searchResult.summary;
targetRun.searchDegraded = searchResult.searchDegraded;
+ targetRun.judgeDegraded = searchResult.judgeDegraded;
+ targetRun.degradedReason = searchResult.degradedReason;
// Persisted onto target_results below so they survive to finalization.
targetRun.searchCredits = searchResult.searchCredits;
targetRun.judgeCredits = searchResult.judgeCredits;
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
index 245c37468..2ccc7706d 100644
--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -419,3 +419,94 @@ describe("judgeEnabled gates the LLM judge", () => {
expect(alertUpsert.metadata.concept).toBeDefined();
});
});
+
+describe("deep-path scrape failures mark the check degraded (no silent empty)", () => {
+ beforeEach(() => {
+ // Two candidates routed to scrape; scrape outcome varies per test.
+ searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);
+ routeMock.mockResolvedValue([
+ { id: "result_1", decision: "scrape", priority: 2, reason: "" },
+ { id: "result_2", decision: "scrape", priority: 1, reason: "" },
+ ]);
+ });
+
+ it("degraded=true when EVERY deep-path scrape fails (judging expected, nothing judged)", async () => {
+ scrapeURLMock.mockResolvedValue({ success: false });
+
+ const result = await runSearchTarget(runParams({ depth: "deep" }));
+
+ // The false-negative we must surface: 0 judged / 0 alerts, but degraded.
+ expect(result.resultsJudged).toBe(0);
+ expect(result.matches).toBe(0);
+ expect(result.judgeDegraded).toBe(true);
+ expect(result.degradedReason).toMatch(/scrape/i);
+ // searchDegraded is the provider-level signal and stays false here.
+ expect(result.searchDegraded).toBe(false);
+ // A non-empty summary so the check never reads as a clean "nothing new".
+ expect(result.summary).not.toBe("");
+ });
+
+ it("degraded=true when scrapes throw", async () => {
+ scrapeURLMock.mockRejectedValue(new Error("scrape timed out"));
+
+ const result = await runSearchTarget(runParams({ depth: "deep" }));
+
+ expect(result.resultsJudged).toBe(0);
+ expect(result.judgeDegraded).toBe(true);
+ expect(result.degradedReason).toMatch(/scrape/i);
+ });
+
+ it("NOT degraded when at least one scrape succeeds and is judged", async () => {
+ scrapeURLMock.mockImplementation((_id: string, url: string) =>
+ url === serpRow(1).url
+ ? Promise.resolve({
+ success: true,
+ document: {
+ json: verdict({ alertAction: "ignore" }),
+ markdown: "prose",
+ metadata: {},
+ },
+ })
+ : Promise.resolve({ success: false }),
+ );
+
+ const result = await runSearchTarget(runParams({ depth: "deep" }));
+
+ // One result was actually judged (ignored) β€” that's a real evaluation, not a
+ // failure, so the check is NOT degraded even though a sibling scrape failed.
+ expect(result.resultsJudged).toBe(1);
+ expect(result.judgeDegraded).toBe(false);
+ expect(result.degradedReason).toBeNull();
+ });
+
+ it("NOT degraded when the judge legitimately ignores every (scraped) result", async () => {
+ scrapeURLMock.mockResolvedValue({
+ success: true,
+ document: {
+ json: verdict({ alertAction: "ignore" }),
+ markdown: "prose",
+ metadata: {},
+ },
+ });
+
+ const result = await runSearchTarget(runParams({ depth: "deep" }));
+
+ // Everything scraped fine and was evaluated; judge ignored it all. Normal.
+ expect(result.resultsJudged).toBe(2);
+ expect(result.matches).toBe(0);
+ expect(result.judgeDegraded).toBe(false);
+ expect(result.degradedReason).toBeNull();
+ });
+
+ it("NOT degraded when search legitimately returns zero results", async () => {
+ searchMock.mockResolvedValue([]);
+
+ const result = await runSearchTarget(runParams({ depth: "deep" }));
+
+ expect(result.resultCount).toBe(0);
+ expect(result.judgeDegraded).toBe(false);
+ expect(result.searchDegraded).toBe(false);
+ expect(result.degradedReason).toBeNull();
+ expect(scrapeURLMock).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index 833a044c5..c11360211 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -180,6 +180,17 @@ type SearchTargetRunResult = {
// not confirm", not a confident "no new results" β€” avoids a silent false
// negative on a swallowed rate-limit.
searchDegraded: boolean;
+ // True when the deep (scrape+judge) path was expected to evaluate results but
+ // evaluated ~none BECAUSE the per-result scrapes failed/timed out β€” the
+ // false-negative twin of searchDegraded, one layer deeper. A judged check that
+ // completes with 0 pages / 0 judged because every scrape errored otherwise
+ // looks identical to "nothing new"; this flags it so it isn't read as clean.
+ // Conservative: NOT set when search legitimately returned 0 results, and NOT
+ // set when the judge legitimately ignored everything (resultsJudged > 0).
+ judgeDegraded: boolean;
+ // Human-readable explanation for whichever degraded flag is set (search OR
+ // judge), surfaced onto the check/summary the same way searchDegraded is.
+ degradedReason: string | null;
sources: SearchSource[];
// Search() call credits (β‰ˆ2 per 10 results), matching executeSearch's math.
searchCredits: number;
@@ -250,6 +261,12 @@ export async function runSearchTarget(params: {
// Queries that failed at the provider after retries (vs. legitimately empty);
// used to mark the run degraded.
let searchFailures = 0;
+ // Deep-path per-result scrape/judge failures (scrape threw, scrape !success,
+ // or verdict unparseable) β€” the silently-swallowed errors that hit `continue`
+ // BEFORE resultsJudged is incremented. Counted to mark the check degraded when
+ // judging was expected but produced ~nothing DUE TO these failures, so an
+ // all-scrapes-failed deep check can't masquerade as a clean "nothing new".
+ let judgeScrapeFailures = 0;
const seenThisRun = new Map<string, number>();
const candidates: Array<{
@@ -609,6 +626,7 @@ export async function runSearchTarget(params: {
);
} catch (error) {
logger.warn("search monitor scrape threw", { url: c.url, error });
+ judgeScrapeFailures += 1;
skipped += 1;
sources.push({ url: c.url, title: c.title, status: "skipped" });
continue;
@@ -616,6 +634,7 @@ export async function runSearchTarget(params: {
if (!res.success) {
logger.warn("search monitor scrape failed", { url: c.url });
+ judgeScrapeFailures += 1;
skipped += 1;
sources.push({ url: c.url, title: c.title, status: "skipped" });
continue;
@@ -624,6 +643,10 @@ export async function runSearchTarget(params: {
verdict = parseVerdict(res.document.json);
if (!verdict) {
+ // Scrape succeeded but the AI verdict was unparseable β€” the result was
+ // fetched yet never actually evaluated, so it counts as a judge-path
+ // failure for the degraded signal (not a legitimate "judge ignored it").
+ judgeScrapeFailures += 1;
skipped += 1;
sources.push({ url: c.url, title: c.title, status: "skipped" });
continue;
@@ -920,6 +943,47 @@ export async function runSearchTarget(params: {
}
}
+ // Deep-path degraded signal: judging WAS expected (depth deep + search
+ // returned candidates) but produced ~nothing (resultsJudged === 0) BECAUSE the
+ // per-result scrapes failed (judgeScrapeFailures > 0). This is the conservative
+ // trigger β€” it is deliberately NOT set when:
+ // - search legitimately returned 0 results (judgeExpected is false, no
+ // candidates), or
+ // - the judge legitimately evaluated results and ignored them all
+ // (resultsJudged > 0), or
+ // - only SOME scrapes failed but at least one was judged (resultsJudged > 0).
+ // Without this, a deep check whose every scrape timed out completes "clean"
+ // with 0 pages / 0 judged / 0 alerts β€” indistinguishable from "nothing new".
+ const judgeExpected = depth === "deep" && candidates.length > 0;
+ const judgeDegraded =
+ judgeExpected && resultsJudged === 0 && judgeScrapeFailures > 0;
+ if (judgeDegraded) {
+ logger.warn(
+ "search monitor run is degraded: deep-path scrapes failed for every candidate so nothing was judged; results may be incomplete (NOT a clean no-results)",
+ {
+ monitorId: params.monitor.id,
+ targetId: target.id,
+ candidates: candidates.length,
+ scrapeFailures: judgeScrapeFailures,
+ resultsJudged,
+ },
+ );
+ if (!summary) {
+ summary =
+ "Results were found but could not be fetched for evaluation (scrapes failed or timed out); this check is incomplete and may have missed new developments.";
+ }
+ }
+
+ // Single reason surfaced onto the check the same way searchDegraded is. Search
+ // failure takes precedence (it's upstream β€” a failed query means we never even
+ // got results to scrape).
+ const degradedReason: string | null =
+ searchDegraded && matches === 0
+ ? "search provider unavailable for one or more queries after retries; results may be incomplete"
+ : judgeDegraded
+ ? "deep-path scrapes failed for every candidate so nothing was judged; results may be incomplete"
+ : null;
+
const isZDR = params.zeroDataRetention;
const searchCredits =
Math.ceil(searchResultsBilled / 10) * (isZDR ? 10 : 2);
@@ -935,6 +999,8 @@ export async function runSearchTarget(params: {
matches,
summary,
searchDegraded,
+ judgeDegraded,
+ degradedReason,
sources,
searchCredits,
judgeCredits,