Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

refactor(monitor): isolate search monitor from core

Developers Digest2fc4123794e7
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index e9ef087b8..e581b2ccd 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -259,11 +259,6 @@ export function calculateCost(
output_cost: 2.19,
},
"google/gemini-2.5-flash-lite": { input_cost: 0.1, output_cost: 0.4 },
- // Search-monitor models. flash-lite mirrors gemini-2.5-flash-lite pricing;
- // gemini-3-flash-preview priced at published Gemini Flash input/output rates.
- "gemini-flash-lite-latest": { input_cost: 0.1, output_cost: 0.4 },
- "gemini-2.5-flash-lite": { input_cost: 0.1, output_cost: 0.4 },
- "gemini-3-flash-preview": { input_cost: 0.3, output_cost: 2.5 },
};
let modelCost = modelCosts[model] || { input_cost: 0, output_cost: 0 };
//gemini-2.5-pro-exp-03-25 pricing
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.ts
index 60e86142a..708574054 100644
--- a/apps/api/src/search/v2/fireEngine-v2.ts
+++ b/apps/api/src/search/v2/fireEngine-v2.ts
@@ -49,11 +49,6 @@ export async function fire_engine_search_v2(
page?: number;
type?: SearchResultType | SearchResultType[];
enterprise?: ("default" | "anon" | "zdr")[];
- // Opt-in observability hook. Invoked with `true` when the request ultimately
- // failed (HTTP error / rate-limit / exhausted retries / thrown) and produced
- // no usable response. Stays undefined-safe for existing callers that don't
- // care to distinguish a real failure from a genuinely-empty result set.
- onFailure?: (reason: string) => void;
},
abort?: AbortSignal,
): Promise<SearchV2Response> {
@@ -86,15 +81,5 @@ export async function fire_engine_search_v2(
abort,
);
- // `attemptRequest` returns null on any non-OK HTTP response (e.g. 429
- // rate-limit) or thrown error, and `executeWithRetry` only returns null once
- // every attempt has been exhausted. A null here therefore means a real
- // failure β€” NOT a successful search that legitimately found nothing (which
- // comes back as a non-null `{}` / `{ web: [] }`). Surface that to opt-in
- // callers so a swallowed rate-limit isn't mistaken for "no results".
- if (result === null) {
- options.onFailure?.("fire_engine_search request failed after retries");
- }
-
return result ?? {};
}
diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.ts
index feea56989..3dca1a848 100644
--- a/apps/api/src/search/v2/index.ts
+++ b/apps/api/src/search/v2/index.ts
@@ -20,7 +20,6 @@ export async function search({
timeout = 5000,
type = undefined,
enterprise = undefined,
- onFailure = undefined,
}: {
query: string;
logger: Logger;
@@ -36,13 +35,6 @@ export async function search({
timeout?: number;
type?: SearchResultType | SearchResultType[];
enterprise?: ("default" | "anon" | "zdr")[];
- // Opt-in soft-failure hook. Called with a reason when the search provider
- // genuinely failed (rate-limit / HTTP error / thrown) rather than legitimately
- // returning zero results. Lets callers (e.g. the search monitor) tell a real
- // failure apart from a clean empty so they can retry or report degraded
- // instead of silently treating it as "no changes". Optional β€” existing
- // callers that omit it keep the exact same behavior and return type.
- onFailure?: (reason: string) => void;
}): Promise<SearchV2Response> {
try {
if (config.FIRE_ENGINE_BETA_URL) {
@@ -56,7 +48,6 @@ export async function search({
location,
type,
enterprise,
- onFailure,
});
return results;
@@ -89,9 +80,6 @@ export async function search({
return {};
} catch (error) {
logger.error(`Error in search function`, { error });
- // Surface the thrown failure to opt-in callers so it isn't mistaken for a
- // genuinely-empty result. Return contract is unchanged (still `{}`).
- onFailure?.(error instanceof Error ? error.message : String(error));
return {};
}
}
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index c11360211..fafb640a9 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -50,43 +50,31 @@ function windowToTbs(window: string): string {
}
// The provider rate-limits near-simultaneous identical queries and may answer
-// with an error OR a clean HTTP 200 with an empty body β€” indistinguishable from
-// a genuinely-empty result. Retry both empty and failed responses with backoff;
-// a real empty query just exhausts attempts and returns empty cleanly. `failed`
-// is true only when the LAST attempt was a real failure, so the caller can mark
-// the check degraded rather than confidently empty.
+// with a clean HTTP 200 empty body. Retry empty responses with backoff; a
+// genuinely-empty query just exhausts attempts and returns empty cleanly.
const SEARCH_RETRY_BACKOFF_MS = [400, 1200] as const;
async function searchWithRetry(
args: Parameters<typeof search>[0],
logger: Logger,
-): Promise<{ response: Awaited<ReturnType<typeof search>>; failed: boolean }> {
+): Promise<Awaited<ReturnType<typeof search>>> {
const maxAttempts = SEARCH_RETRY_BACKOFF_MS.length + 1;
let lastResponse: Awaited<ReturnType<typeof search>> = {};
- let lastFailureReason: string | null = null;
for (let attempt = 0; attempt < maxAttempts; attempt++) {
- let failureReason: string | null = null;
- const response = await search({
- ...args,
- onFailure: reason => {
- failureReason = reason;
- },
- });
+ const response = await search(args);
lastResponse = response;
- lastFailureReason = failureReason;
const hasResults = (response.web?.length ?? 0) > 0;
- if (hasResults) return { response, failed: false };
+ if (hasResults) return response;
const isLastAttempt = attempt === maxAttempts - 1;
if (!isLastAttempt) {
const base = SEARCH_RETRY_BACKOFF_MS[attempt];
const jitter = Math.floor(Math.random() * 250);
- logger.info("search monitor query empty/failed; retrying", {
+ logger.info("search monitor query empty; retrying", {
query: args.query,
attempt: attempt + 1,
- failureReason,
backoffMs: base + jitter,
});
await new Promise(r => setTimeout(r, base + jitter));
@@ -94,21 +82,7 @@ async function searchWithRetry(
}
}
- // Exhausted attempts: a real final failure is the false-negative we must not
- // hide; an empty with no failure is a legitimate no-results.
- if (lastFailureReason) {
- logger.warn(
- "search monitor query failed after retries; reporting degraded (NOT clean no-results)",
- {
- query: args.query,
- reason: lastFailureReason,
- attempts: maxAttempts,
- },
- );
- return { response: lastResponse, failed: true };
- }
-
- return { response: lastResponse, failed: false };
+ return lastResponse;
}
function isExcludedDomain(
@@ -175,21 +149,20 @@ type SearchTargetRunResult = {
meaningful: number;
matches: number;
summary: string;
- // True when a query failed at the provider after retries rather than
- // legitimately returning zero results. An empty-but-degraded run means "could
- // not confirm", not a confident "no new results" β€” avoids a silent false
- // negative on a swallowed rate-limit.
+ // Reserved provider-level degraded signal. The public search() helper swallows
+ // provider errors into an empty result, so the monitor cannot tell a real
+ // failure from a genuine no-results without reaching into core search; this
+ // therefore stays false. Kept for the check shape and possible future use.
searchDegraded: boolean;
// True when the deep (scrape+judge) path was expected to evaluate results but
- // evaluated ~none BECAUSE the per-result scrapes failed/timed out β€” the
- // false-negative twin of searchDegraded, one layer deeper. A judged check that
+ // evaluated ~none BECAUSE the per-result scrapes failed/timed out. A judged check that
// completes with 0 pages / 0 judged because every scrape errored otherwise
// looks identical to "nothing new"; this flags it so it isn't read as clean.
// Conservative: NOT set when search legitimately returned 0 results, and NOT
// set when the judge legitimately ignored everything (resultsJudged > 0).
judgeDegraded: boolean;
- // Human-readable explanation for whichever degraded flag is set (search OR
- // judge), surfaced onto the check/summary the same way searchDegraded is.
+ // Human-readable explanation when judgeDegraded is set, surfaced onto the
+ // check/summary.
degradedReason: string | null;
sources: SearchSource[];
// Search() call credits (β‰ˆ2 per 10 results), matching executeSearch's math.
@@ -258,9 +231,6 @@ export async function runSearchTarget(params: {
let searchResultsBilled = 0;
// Denominator for flat judge billing: judgeCredits = 5 * this.
let resultsJudged = 0;
- // Queries that failed at the provider after retries (vs. legitimately empty);
- // used to mark the run degraded.
- let searchFailures = 0;
// Deep-path per-result scrape/judge failures (scrape threw, scrape !success,
// or verdict unparseable) β€” the silently-swallowed errors that hit `continue`
// BEFORE resultsJudged is incremented. Counted to mark the check degraded when
@@ -281,7 +251,7 @@ export async function runSearchTarget(params: {
includeDomains: target.includeDomains,
excludeDomains: target.excludeDomains,
});
- const { response, failed } = await searchWithRetry(
+ const response = await searchWithRetry(
{
query: scopedQuery,
logger,
@@ -290,9 +260,6 @@ export async function runSearchTarget(params: {
},
logger,
);
- if (failed) {
- searchFailures += 1;
- }
const results = response.web ?? [];
// Bill on raw results, before dedupe/trim.
searchResultsBilled += results.length;
@@ -925,23 +892,12 @@ export async function runSearchTarget(params: {
}
}
- const searchDegraded = searchFailures > 0;
- if (searchDegraded && matches === 0) {
- // Don't let a swallowed provider failure masquerade as clean no-results.
- logger.warn(
- "search monitor run is degraded: one or more queries failed at the provider after retries; results may be incomplete",
- {
- monitorId: params.monitor.id,
- targetId: target.id,
- failedQueries: searchFailures,
- totalQueries: target.queries.length,
- },
- );
- if (!summary) {
- summary =
- "Search provider was unavailable for one or more queries (rate-limited or errored after retries); results may be incomplete.";
- }
- }
+ // The public search() helper swallows provider errors and returns an empty
+ // result, so the monitor can't distinguish a real provider failure from a
+ // genuine no-results without reaching into core search. We deliberately don't:
+ // empty-after-retries is treated as a clean no-results. The field is kept for
+ // the check shape (and the deep-path judge-degraded signal below still fires).
+ const searchDegraded = false;
// Deep-path degraded signal: judging WAS expected (depth deep + search
// returned candidates) but produced ~nothing (resultsJudged === 0) BECAUSE the
@@ -974,15 +930,11 @@ export async function runSearchTarget(params: {
}
}
- // Single reason surfaced onto the check the same way searchDegraded is. Search
- // failure takes precedence (it's upstream β€” a failed query means we never even
- // got results to scrape).
- const degradedReason: string | null =
- searchDegraded && matches === 0
- ? "search provider unavailable for one or more queries after retries; results may be incomplete"
- : judgeDegraded
- ? "deep-path scrapes failed for every candidate so nothing was judged; results may be incomplete"
- : null;
+ // Single reason surfaced onto the check when the deep path could not evaluate
+ // the results it found.
+ const degradedReason: string | null = judgeDegraded
+ ? "deep-path scrapes failed for every candidate so nothing was judged; results may be incomplete"
+ : null;
const isZDR = params.zeroDataRetention;
const searchCredits =