Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(monitor): mark search check degraded when deep-path scrapes fail (no silent empty)
Developers Digest7b0f9a518fd8
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 719edcf9a..367e1b7ed 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -124,6 +124,12 @@ type MonitorTargetRun =// True when a query failed at the provider after retries (not legitimately// empty); persisted so an empty-but-degraded run isn't read as "no changes".searchDegraded?: boolean;+ // True when the deep (scrape+judge) path was expected to evaluate results+ // but evaluated ~none because the per-result scrapes failed; persisted so a+ // judged check that scraped nothing isn't read as a clean "no changes".+ judgeDegraded?: boolean;+ // Human-readable explanation for whichever degraded flag is set.+ degradedReason?: string | null;// Flat, deterministic credits recorded onto target_results when the search// completes; actual_credits is summed from these, not reconstructed from// per-page metadata, so a missing page / reconciler race can't zero them.@@ -971,6 +977,8 @@ async function runMonitorSearchTarget(params: {matches: number;summary: string;searchDegraded: boolean;+ judgeDegraded: boolean;+ degradedReason: string | null;// Flat, deterministic credits, recorded onto target_results by the caller.searchCredits: number;judgeCredits: number;@@ -983,6 +991,8 @@ async function runMonitorSearchTarget(params: {matches: 0,summary: "",searchDegraded: false,+ judgeDegraded: false,+ degradedReason: null,searchCredits: 0,judgeCredits: 0,resultsJudged: 0,@@ -1095,6 +1105,8 @@ async function runMonitorSearchTarget(params: {matches: result.matches,summary: result.summary,searchDegraded: result.searchDegraded,+ judgeDegraded: result.judgeDegraded,+ degradedReason: result.degradedReason,searchCredits,judgeCredits,resultsJudged: result.resultsJudged,@@ -1204,6 +1216,8 @@ export async function processMonitorCheckJob(targetRun.matches = searchResult.matches;targetRun.summary = searchResult.summary;targetRun.searchDegraded = searchResult.searchDegraded;+ targetRun.judgeDegraded = searchResult.judgeDegraded;+ targetRun.degradedReason = searchResult.degradedReason;// Persisted onto target_results below so they survive to finalization.targetRun.searchCredits = searchResult.searchCredits;targetRun.judgeCredits = searchResult.judgeCredits;diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.tsindex 245c37468..2ccc7706d 100644--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts@@ -419,3 +419,94 @@ describe("judgeEnabled gates the LLM judge", () => {expect(alertUpsert.metadata.concept).toBeDefined();});});++describe("deep-path scrape failures mark the check degraded (no silent empty)", () => {+ beforeEach(() => {+ // Two candidates routed to scrape; scrape outcome varies per test.+ searchMock.mockResolvedValue([serpRow(1), serpRow(2)]);+ routeMock.mockResolvedValue([+ { id: "result_1", decision: "scrape", priority: 2, reason: "" },+ { id: "result_2", decision: "scrape", priority: 1, reason: "" },+ ]);+ });++ it("degraded=true when EVERY deep-path scrape fails (judging expected, nothing judged)", async () => {+ scrapeURLMock.mockResolvedValue({ success: false });++ const result = await runSearchTarget(runParams({ depth: "deep" }));++ // The false-negative we must surface: 0 judged / 0 alerts, but degraded.+ expect(result.resultsJudged).toBe(0);+ expect(result.matches).toBe(0);+ expect(result.judgeDegraded).toBe(true);+ expect(result.degradedReason).toMatch(/scrape/i);+ // searchDegraded is the provider-level signal and stays false here.+ expect(result.searchDegraded).toBe(false);+ // A non-empty summary so the check never reads as a clean "nothing new".+ expect(result.summary).not.toBe("");+ });++ it("degraded=true when scrapes throw", async () => {+ scrapeURLMock.mockRejectedValue(new Error("scrape timed out"));++ const result = await runSearchTarget(runParams({ depth: "deep" }));++ expect(result.resultsJudged).toBe(0);+ expect(result.judgeDegraded).toBe(true);+ expect(result.degradedReason).toMatch(/scrape/i);+ });++ it("NOT degraded when at least one scrape succeeds and is judged", async () => {+ scrapeURLMock.mockImplementation((_id: string, url: string) =>+ url === serpRow(1).url+ ? Promise.resolve({+ success: true,+ document: {+ json: verdict({ alertAction: "ignore" }),+ markdown: "prose",+ metadata: {},+ },+ })+ : Promise.resolve({ success: false }),+ );++ const result = await runSearchTarget(runParams({ depth: "deep" }));++ // One result was actually judged (ignored) β that's a real evaluation, not a+ // failure, so the check is NOT degraded even though a sibling scrape failed.+ expect(result.resultsJudged).toBe(1);+ expect(result.judgeDegraded).toBe(false);+ expect(result.degradedReason).toBeNull();+ });++ it("NOT degraded when the judge legitimately ignores every (scraped) result", async () => {+ scrapeURLMock.mockResolvedValue({+ success: true,+ document: {+ json: verdict({ alertAction: "ignore" }),+ markdown: "prose",+ metadata: {},+ },+ });++ const result = await runSearchTarget(runParams({ depth: "deep" }));++ // Everything scraped fine and was evaluated; judge ignored it all. Normal.+ expect(result.resultsJudged).toBe(2);+ expect(result.matches).toBe(0);+ expect(result.judgeDegraded).toBe(false);+ expect(result.degradedReason).toBeNull();+ });++ it("NOT degraded when search legitimately returns zero results", async () => {+ searchMock.mockResolvedValue([]);++ const result = await runSearchTarget(runParams({ depth: "deep" }));++ expect(result.resultCount).toBe(0);+ expect(result.judgeDegraded).toBe(false);+ expect(result.searchDegraded).toBe(false);+ expect(result.degradedReason).toBeNull();+ expect(scrapeURLMock).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 833a044c5..c11360211 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -180,6 +180,17 @@ type SearchTargetRunResult = {// not confirm", not a confident "no new results" β avoids a silent false// negative on a swallowed rate-limit.searchDegraded: boolean;+ // True when the deep (scrape+judge) path was expected to evaluate results but+ // evaluated ~none BECAUSE the per-result scrapes failed/timed out β the+ // false-negative twin of searchDegraded, one layer deeper. A judged check that+ // completes with 0 pages / 0 judged because every scrape errored otherwise+ // looks identical to "nothing new"; this flags it so it isn't read as clean.+ // Conservative: NOT set when search legitimately returned 0 results, and NOT+ // set when the judge legitimately ignored everything (resultsJudged > 0).+ judgeDegraded: boolean;+ // Human-readable explanation for whichever degraded flag is set (search OR+ // judge), surfaced onto the check/summary the same way searchDegraded is.+ degradedReason: string | null;sources: SearchSource[];// Search() call credits (β2 per 10 results), matching executeSearch's math.searchCredits: number;@@ -250,6 +261,12 @@ export async function runSearchTarget(params: {// Queries that failed at the provider after retries (vs. legitimately empty);// used to mark the run degraded.let searchFailures = 0;+ // Deep-path per-result scrape/judge failures (scrape threw, scrape !success,+ // or verdict unparseable) β the silently-swallowed errors that hit `continue`+ // BEFORE resultsJudged is incremented. Counted to mark the check degraded when+ // judging was expected but produced ~nothing DUE TO these failures, so an+ // all-scrapes-failed deep check can't masquerade as a clean "nothing new".+ let judgeScrapeFailures = 0;const seenThisRun = new Map<string, number>();const candidates: Array<{@@ -609,6 +626,7 @@ export async function runSearchTarget(params: {);} catch (error) {logger.warn("search monitor scrape threw", { url: c.url, error });+ judgeScrapeFailures += 1;skipped += 1;sources.push({ url: c.url, title: c.title, status: "skipped" });continue;@@ -616,6 +634,7 @@ export async function runSearchTarget(params: {if (!res.success) {logger.warn("search monitor scrape failed", { url: c.url });+ judgeScrapeFailures += 1;skipped += 1;sources.push({ url: c.url, title: c.title, status: "skipped" });continue;@@ -624,6 +643,10 @@ export async function runSearchTarget(params: {verdict = parseVerdict(res.document.json);if (!verdict) {+ // Scrape succeeded but the AI verdict was unparseable β the result was+ // fetched yet never actually evaluated, so it counts as a judge-path+ // failure for the degraded signal (not a legitimate "judge ignored it").+ judgeScrapeFailures += 1;skipped += 1;sources.push({ url: c.url, title: c.title, status: "skipped" });continue;@@ -920,6 +943,47 @@ export async function runSearchTarget(params: {}}+ // Deep-path degraded signal: judging WAS expected (depth deep + search+ // returned candidates) but produced ~nothing (resultsJudged === 0) BECAUSE the+ // per-result scrapes failed (judgeScrapeFailures > 0). This is the conservative+ // trigger β it is deliberately NOT set when:+ // - search legitimately returned 0 results (judgeExpected is false, no+ // candidates), or+ // - the judge legitimately evaluated results and ignored them all+ // (resultsJudged > 0), or+ // - only SOME scrapes failed but at least one was judged (resultsJudged > 0).+ // Without this, a deep check whose every scrape timed out completes "clean"+ // with 0 pages / 0 judged / 0 alerts β indistinguishable from "nothing new".+ const judgeExpected = depth === "deep" && candidates.length > 0;+ const judgeDegraded =+ judgeExpected && resultsJudged === 0 && judgeScrapeFailures > 0;+ if (judgeDegraded) {+ logger.warn(+ "search monitor run is degraded: deep-path scrapes failed for every candidate so nothing was judged; results may be incomplete (NOT a clean no-results)",+ {+ monitorId: params.monitor.id,+ targetId: target.id,+ candidates: candidates.length,+ scrapeFailures: judgeScrapeFailures,+ resultsJudged,+ },+ );+ if (!summary) {+ summary =+ "Results were found but could not be fetched for evaluation (scrapes failed or timed out); this check is incomplete and may have missed new developments.";+ }+ }++ // Single reason surfaced onto the check the same way searchDegraded is. Search+ // failure takes precedence (it's upstream β a failed query means we never even+ // got results to scrape).+ const degradedReason: string | null =+ searchDegraded && matches === 0+ ? "search provider unavailable for one or more queries after retries; results may be incomplete"+ : judgeDegraded+ ? "deep-path scrapes failed for every candidate so nothing was judged; results may be incomplete"+ : null;+const isZDR = params.zeroDataRetention;const searchCredits =Math.ceil(searchResultsBilled / 10) * (isZDR ? 10 : 2);@@ -935,6 +999,8 @@ export async function runSearchTarget(params: {matches,summary,searchDegraded,+ judgeDegraded,+ degradedReason,sources,searchCredits,judgeCredits,