Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(monitor): retry transient empty/rate-limited searches; surface real search failures instead of silent no-changes

Developers Digest229141a46fb6
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.ts
index 708574054..60e86142a 100644
--- a/apps/api/src/search/v2/fireEngine-v2.ts
+++ b/apps/api/src/search/v2/fireEngine-v2.ts
@@ -49,6 +49,11 @@ export async function fire_engine_search_v2(
page?: number;
type?: SearchResultType | SearchResultType[];
enterprise?: ("default" | "anon" | "zdr")[];
+ // Opt-in observability hook. Invoked with `true` when the request ultimately
+ // failed (HTTP error / rate-limit / exhausted retries / thrown) and produced
+ // no usable response. Stays undefined-safe for existing callers that don't
+ // care to distinguish a real failure from a genuinely-empty result set.
+ onFailure?: (reason: string) => void;
},
abort?: AbortSignal,
): Promise<SearchV2Response> {
@@ -81,5 +86,15 @@ export async function fire_engine_search_v2(
abort,
);
+ // `attemptRequest` returns null on any non-OK HTTP response (e.g. 429
+ // rate-limit) or thrown error, and `executeWithRetry` only returns null once
+ // every attempt has been exhausted. A null here therefore means a real
+ // failure β€” NOT a successful search that legitimately found nothing (which
+ // comes back as a non-null `{}` / `{ web: [] }`). Surface that to opt-in
+ // callers so a swallowed rate-limit isn't mistaken for "no results".
+ if (result === null) {
+ options.onFailure?.("fire_engine_search request failed after retries");
+ }
+
return result ?? {};
}
diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.ts
index 3dca1a848..feea56989 100644
--- a/apps/api/src/search/v2/index.ts
+++ b/apps/api/src/search/v2/index.ts
@@ -20,6 +20,7 @@ export async function search({
timeout = 5000,
type = undefined,
enterprise = undefined,
+ onFailure = undefined,
}: {
query: string;
logger: Logger;
@@ -35,6 +36,13 @@ export async function search({
timeout?: number;
type?: SearchResultType | SearchResultType[];
enterprise?: ("default" | "anon" | "zdr")[];
+ // Opt-in soft-failure hook. Called with a reason when the search provider
+ // genuinely failed (rate-limit / HTTP error / thrown) rather than legitimately
+ // returning zero results. Lets callers (e.g. the search monitor) tell a real
+ // failure apart from a clean empty so they can retry or report degraded
+ // instead of silently treating it as "no changes". Optional β€” existing
+ // callers that omit it keep the exact same behavior and return type.
+ onFailure?: (reason: string) => void;
}): Promise<SearchV2Response> {
try {
if (config.FIRE_ENGINE_BETA_URL) {
@@ -48,6 +56,7 @@ export async function search({
location,
type,
enterprise,
+ onFailure,
});
return results;
@@ -80,6 +89,9 @@ export async function search({
return {};
} catch (error) {
logger.error(`Error in search function`, { error });
+ // Surface the thrown failure to opt-in callers so it isn't mistaken for a
+ // genuinely-empty result. Return contract is unchanged (still `{}`).
+ onFailure?.(error instanceof Error ? error.message : String(error));
return {};
}
}
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index f68fa9d82..c8d7cf322 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -117,6 +117,11 @@ type MonitorTargetRun =
resultCount?: number;
matches?: number;
summary?: string;
+ // True when a query failed at the search provider (rate-limit / HTTP
+ // error) after retries rather than legitimately returning no results.
+ // Persisted into target_results so an empty-but-degraded run is visible
+ // and not mistaken for a confident "no changes".
+ searchDegraded?: boolean;
};
function createMonitorTargetRun(target: MonitorTarget): MonitorTargetRun {
@@ -957,9 +962,16 @@ async function runMonitorSearchTarget(params: {
resultCount: number;
matches: number;
summary: string;
+ searchDegraded: boolean;
}> {
if (params.target.type !== "search") {
- return { pages: [], resultCount: 0, matches: 0, summary: "" };
+ return {
+ pages: [],
+ resultCount: 0,
+ matches: 0,
+ summary: "",
+ searchDegraded: false,
+ };
}
const { monitor, check, target } = params;
const goalVersion = computeGoalVersion(monitor.goal, target.queries);
@@ -1087,6 +1099,7 @@ async function runMonitorSearchTarget(params: {
resultCount: result.resultCount,
matches: result.matches,
summary: result.summary,
+ searchDegraded: result.searchDegraded,
};
}
@@ -1192,6 +1205,7 @@ export async function processMonitorCheckJob(
targetRun.resultCount = searchResult.resultCount;
targetRun.matches = searchResult.matches;
targetRun.summary = searchResult.summary;
+ targetRun.searchDegraded = searchResult.searchDegraded;
}
}
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index fc7de7df0..494977728 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -47,6 +47,77 @@ function windowToTbs(window: string): string {
return "qdr:w";
}
+// Bounded retry for monitor searches. The upstream provider rate-limits
+// near-simultaneous identical queries and can answer with either a thrown/HTTP
+// error OR a clean HTTP 200 carrying an empty body β€” both of which otherwise
+// look identical to a genuinely-empty result and silently drop real hits.
+//
+// Strategy: retry on a hard failure (search() reported it via onFailure) AND on
+// an empty result, with exponential backoff + jitter so we don't hammer the
+// rate-limiter. A genuinely-empty query simply runs out of attempts and returns
+// empty cleanly (no failure flagged), so its correct "no results" behavior is
+// preserved. `failed` is only true when the LAST attempt was a real failure,
+// letting the caller mark the check degraded instead of confidently empty.
+const SEARCH_RETRY_BACKOFF_MS = [400, 1200] as const;
+
+async function searchWithRetry(
+ args: Parameters<typeof search>[0],
+ logger: Logger,
+): Promise<{ response: Awaited<ReturnType<typeof search>>; failed: boolean }> {
+ const maxAttempts = SEARCH_RETRY_BACKOFF_MS.length + 1;
+ let lastResponse: Awaited<ReturnType<typeof search>> = {};
+ let lastFailureReason: string | null = null;
+
+ for (let attempt = 0; attempt < maxAttempts; attempt++) {
+ let failureReason: string | null = null;
+ const response = await search({
+ ...args,
+ onFailure: reason => {
+ failureReason = reason;
+ },
+ });
+ lastResponse = response;
+ lastFailureReason = failureReason;
+
+ const hasResults = (response.web?.length ?? 0) > 0;
+ // Success: real results came back. Return immediately.
+ if (hasResults) return { response, failed: false };
+
+ // No results. Could be a genuine empty OR a transient rate-limit (error or
+ // empty 200). Retry with backoff while attempts remain.
+ const isLastAttempt = attempt === maxAttempts - 1;
+ if (!isLastAttempt) {
+ const base = SEARCH_RETRY_BACKOFF_MS[attempt];
+ const jitter = Math.floor(Math.random() * 250);
+ logger.info("search monitor query empty/failed; retrying", {
+ query: args.query,
+ attempt: attempt + 1,
+ failureReason,
+ backoffMs: base + jitter,
+ });
+ await new Promise(r => setTimeout(r, base + jitter));
+ continue;
+ }
+ }
+
+ // Exhausted attempts. If the final attempt was a real provider failure,
+ // surface it loudly (degraded) β€” this is the false-negative we must not hide.
+ // If it was just empty with no failure, it is a legitimate no-results.
+ if (lastFailureReason) {
+ logger.warn(
+ "search monitor query failed after retries; reporting degraded (NOT clean no-results)",
+ {
+ query: args.query,
+ reason: lastFailureReason,
+ attempts: maxAttempts,
+ },
+ );
+ return { response: lastResponse, failed: true };
+ }
+
+ return { response: lastResponse, failed: false };
+}
+
function isExcludedDomain(
url: string,
excludeDomains: string[] | undefined,
@@ -111,6 +182,12 @@ type SearchTargetRunResult = {
meaningful: number;
matches: number;
summary: string;
+ // True when one or more of this target's queries failed at the search
+ // provider (rate-limit / HTTP error) after retries, rather than legitimately
+ // returning zero results. Callers should treat an empty run with
+ // searchDegraded=true as "could not confirm" rather than a confident
+ // "no new results", so a swallowed rate-limit isn't a silent false negative.
+ searchDegraded: boolean;
sources: SearchSource[];
// Credits attributable to the search() calls themselves (β‰ˆ2 per 10 results),
// matching executeSearch's search-credit math.
@@ -169,6 +246,11 @@ export async function runSearchTarget(params: {
const costTracking = new CostTracking();
// Search calls bill β‰ˆ2 credits / 10 results, mirroring executeSearch().
let searchResultsBilled = 0;
+ // Count of queries that ultimately FAILED at the provider (rate-limit / HTTP
+ // error) after retries, as opposed to legitimately returning zero results.
+ // Used to mark the run degraded so an empty check isn't reported as a
+ // confident "no changes" when it was really a swallowed search failure.
+ let searchFailures = 0;
const seenThisRun = new Map<string, number>();
const candidates: Array<{
@@ -183,12 +265,20 @@ export async function runSearchTarget(params: {
includeDomains: target.includeDomains,
excludeDomains: target.excludeDomains,
});
- const response = await search({
- query: scopedQuery,
+ const { response, failed } = await searchWithRetry(
+ {
+ query: scopedQuery,
+ logger,
+ num_results: target.maxResults,
+ tbs,
+ },
logger,
- num_results: target.maxResults,
- tbs,
- });
+ );
+ if (failed) {
+ // A real provider failure (not a genuine empty). Record it so the run is
+ // reported as degraded rather than a clean "no changes" for this query.
+ searchFailures += 1;
+ }
// v2 search returns { web, news, images }; the monitor consumes web hits.
const results = response.web ?? [];
// Bill the search call at the platform rate (executeSearch: ~2 credits / 10
@@ -807,6 +897,24 @@ export async function runSearchTarget(params: {
}
}
+ const searchDegraded = searchFailures > 0;
+ if (searchDegraded && matches === 0) {
+ // Don't let a swallowed provider failure masquerade as a clean no-results.
+ logger.warn(
+ "search monitor run is degraded: one or more queries failed at the provider after retries; results may be incomplete",
+ {
+ monitorId: params.monitor.id,
+ targetId: target.id,
+ failedQueries: searchFailures,
+ totalQueries: target.queries.length,
+ },
+ );
+ if (!summary) {
+ summary =
+ "Search provider was unavailable for one or more queries (rate-limited or errored after retries); results may be incomplete.";
+ }
+ }
+
const isZDR = params.zeroDataRetention;
const searchCredits =
Math.ceil(searchResultsBilled / 10) * (isZDR ? 10 : 2);
@@ -821,6 +929,7 @@ export async function runSearchTarget(params: {
meaningful,
matches,
summary,
+ searchDegraded,
sources,
searchCredits,
llmCredits,