Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(monitor): retry transient empty/rate-limited searches; surface real search failures instead of silent no-changes
Developers Digest229141a46fb6
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex 708574054..60e86142a 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -49,6 +49,11 @@ export async function fire_engine_search_v2(page?: number;type?: SearchResultType | SearchResultType[];enterprise?: ("default" | "anon" | "zdr")[];+ // Opt-in observability hook. Invoked with `true` when the request ultimately+ // failed (HTTP error / rate-limit / exhausted retries / thrown) and produced+ // no usable response. Stays undefined-safe for existing callers that don't+ // care to distinguish a real failure from a genuinely-empty result set.+ onFailure?: (reason: string) => void;},abort?: AbortSignal,): Promise<SearchV2Response> {@@ -81,5 +86,15 @@ export async function fire_engine_search_v2(abort,);+ // `attemptRequest` returns null on any non-OK HTTP response (e.g. 429+ // rate-limit) or thrown error, and `executeWithRetry` only returns null once+ // every attempt has been exhausted. A null here therefore means a real+ // failure β NOT a successful search that legitimately found nothing (which+ // comes back as a non-null `{}` / `{ web: [] }`). Surface that to opt-in+ // callers so a swallowed rate-limit isn't mistaken for "no results".+ if (result === null) {+ options.onFailure?.("fire_engine_search request failed after retries");+ }+return result ?? {};}diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.tsindex 3dca1a848..feea56989 100644--- a/apps/api/src/search/v2/index.ts+++ b/apps/api/src/search/v2/index.ts@@ -20,6 +20,7 @@ export async function search({timeout = 5000,type = undefined,enterprise = undefined,+ onFailure = undefined,}: {query: string;logger: Logger;@@ -35,6 +36,13 @@ export async function search({timeout?: number;type?: SearchResultType | SearchResultType[];enterprise?: ("default" | "anon" | "zdr")[];+ // Opt-in soft-failure hook. Called with a reason when the search provider+ // genuinely failed (rate-limit / HTTP error / thrown) rather than legitimately+ // returning zero results. Lets callers (e.g. the search monitor) tell a real+ // failure apart from a clean empty so they can retry or report degraded+ // instead of silently treating it as "no changes". Optional β existing+ // callers that omit it keep the exact same behavior and return type.+ onFailure?: (reason: string) => void;}): Promise<SearchV2Response> {try {if (config.FIRE_ENGINE_BETA_URL) {@@ -48,6 +56,7 @@ export async function search({location,type,enterprise,+ onFailure,});return results;@@ -80,6 +89,9 @@ export async function search({return {};} catch (error) {logger.error(`Error in search function`, { error });+ // Surface the thrown failure to opt-in callers so it isn't mistaken for a+ // genuinely-empty result. Return contract is unchanged (still `{}`).+ onFailure?.(error instanceof Error ? error.message : String(error));return {};}}diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex f68fa9d82..c8d7cf322 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -117,6 +117,11 @@ type MonitorTargetRun =resultCount?: number;matches?: number;summary?: string;+ // True when a query failed at the search provider (rate-limit / HTTP+ // error) after retries rather than legitimately returning no results.+ // Persisted into target_results so an empty-but-degraded run is visible+ // and not mistaken for a confident "no changes".+ searchDegraded?: boolean;};function createMonitorTargetRun(target: MonitorTarget): MonitorTargetRun {@@ -957,9 +962,16 @@ async function runMonitorSearchTarget(params: {resultCount: number;matches: number;summary: string;+ searchDegraded: boolean;}> {if (params.target.type !== "search") {- return { pages: [], resultCount: 0, matches: 0, summary: "" };+ return {+ pages: [],+ resultCount: 0,+ matches: 0,+ summary: "",+ searchDegraded: false,+ };}const { monitor, check, target } = params;const goalVersion = computeGoalVersion(monitor.goal, target.queries);@@ -1087,6 +1099,7 @@ async function runMonitorSearchTarget(params: {resultCount: result.resultCount,matches: result.matches,summary: result.summary,+ searchDegraded: result.searchDegraded,};}@@ -1192,6 +1205,7 @@ export async function processMonitorCheckJob(targetRun.resultCount = searchResult.resultCount;targetRun.matches = searchResult.matches;targetRun.summary = searchResult.summary;+ targetRun.searchDegraded = searchResult.searchDegraded;}}diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex fc7de7df0..494977728 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -47,6 +47,77 @@ function windowToTbs(window: string): string {return "qdr:w";}+// Bounded retry for monitor searches. The upstream provider rate-limits+// near-simultaneous identical queries and can answer with either a thrown/HTTP+// error OR a clean HTTP 200 carrying an empty body β both of which otherwise+// look identical to a genuinely-empty result and silently drop real hits.+//+// Strategy: retry on a hard failure (search() reported it via onFailure) AND on+// an empty result, with exponential backoff + jitter so we don't hammer the+// rate-limiter. A genuinely-empty query simply runs out of attempts and returns+// empty cleanly (no failure flagged), so its correct "no results" behavior is+// preserved. `failed` is only true when the LAST attempt was a real failure,+// letting the caller mark the check degraded instead of confidently empty.+const SEARCH_RETRY_BACKOFF_MS = [400, 1200] as const;++async function searchWithRetry(+ args: Parameters<typeof search>[0],+ logger: Logger,+): Promise<{ response: Awaited<ReturnType<typeof search>>; failed: boolean }> {+ const maxAttempts = SEARCH_RETRY_BACKOFF_MS.length + 1;+ let lastResponse: Awaited<ReturnType<typeof search>> = {};+ let lastFailureReason: string | null = null;++ for (let attempt = 0; attempt < maxAttempts; attempt++) {+ let failureReason: string | null = null;+ const response = await search({+ ...args,+ onFailure: reason => {+ failureReason = reason;+ },+ });+ lastResponse = response;+ lastFailureReason = failureReason;++ const hasResults = (response.web?.length ?? 0) > 0;+ // Success: real results came back. Return immediately.+ if (hasResults) return { response, failed: false };++ // No results. Could be a genuine empty OR a transient rate-limit (error or+ // empty 200). Retry with backoff while attempts remain.+ const isLastAttempt = attempt === maxAttempts - 1;+ if (!isLastAttempt) {+ const base = SEARCH_RETRY_BACKOFF_MS[attempt];+ const jitter = Math.floor(Math.random() * 250);+ logger.info("search monitor query empty/failed; retrying", {+ query: args.query,+ attempt: attempt + 1,+ failureReason,+ backoffMs: base + jitter,+ });+ await new Promise(r => setTimeout(r, base + jitter));+ continue;+ }+ }++ // Exhausted attempts. If the final attempt was a real provider failure,+ // surface it loudly (degraded) β this is the false-negative we must not hide.+ // If it was just empty with no failure, it is a legitimate no-results.+ if (lastFailureReason) {+ logger.warn(+ "search monitor query failed after retries; reporting degraded (NOT clean no-results)",+ {+ query: args.query,+ reason: lastFailureReason,+ attempts: maxAttempts,+ },+ );+ return { response: lastResponse, failed: true };+ }++ return { response: lastResponse, failed: false };+}+function isExcludedDomain(url: string,excludeDomains: string[] | undefined,@@ -111,6 +182,12 @@ type SearchTargetRunResult = {meaningful: number;matches: number;summary: string;+ // True when one or more of this target's queries failed at the search+ // provider (rate-limit / HTTP error) after retries, rather than legitimately+ // returning zero results. Callers should treat an empty run with+ // searchDegraded=true as "could not confirm" rather than a confident+ // "no new results", so a swallowed rate-limit isn't a silent false negative.+ searchDegraded: boolean;sources: SearchSource[];// Credits attributable to the search() calls themselves (β2 per 10 results),// matching executeSearch's search-credit math.@@ -169,6 +246,11 @@ export async function runSearchTarget(params: {const costTracking = new CostTracking();// Search calls bill β2 credits / 10 results, mirroring executeSearch().let searchResultsBilled = 0;+ // Count of queries that ultimately FAILED at the provider (rate-limit / HTTP+ // error) after retries, as opposed to legitimately returning zero results.+ // Used to mark the run degraded so an empty check isn't reported as a+ // confident "no changes" when it was really a swallowed search failure.+ let searchFailures = 0;const seenThisRun = new Map<string, number>();const candidates: Array<{@@ -183,12 +265,20 @@ export async function runSearchTarget(params: {includeDomains: target.includeDomains,excludeDomains: target.excludeDomains,});- const response = await search({- query: scopedQuery,+ const { response, failed } = await searchWithRetry(+ {+ query: scopedQuery,+ logger,+ num_results: target.maxResults,+ tbs,+ },logger,- num_results: target.maxResults,- tbs,- });+ );+ if (failed) {+ // A real provider failure (not a genuine empty). Record it so the run is+ // reported as degraded rather than a clean "no changes" for this query.+ searchFailures += 1;+ }// v2 search returns { web, news, images }; the monitor consumes web hits.const results = response.web ?? [];// Bill the search call at the platform rate (executeSearch: ~2 credits / 10@@ -807,6 +897,24 @@ export async function runSearchTarget(params: {}}+ const searchDegraded = searchFailures > 0;+ if (searchDegraded && matches === 0) {+ // Don't let a swallowed provider failure masquerade as a clean no-results.+ logger.warn(+ "search monitor run is degraded: one or more queries failed at the provider after retries; results may be incomplete",+ {+ monitorId: params.monitor.id,+ targetId: target.id,+ failedQueries: searchFailures,+ totalQueries: target.queries.length,+ },+ );+ if (!summary) {+ summary =+ "Search provider was unavailable for one or more queries (rate-limited or errored after retries); results may be incomplete.";+ }+ }+const isZDR = params.zeroDataRetention;const searchCredits =Math.ceil(searchResultsBilled / 10) * (isZDR ? 10 : 2);@@ -821,6 +929,7 @@ export async function runSearchTarget(params: {meaningful,matches,summary,+ searchDegraded,sources,searchCredits,llmCredits,