Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor(monitor): isolate search monitor from core
Developers Digest2fc4123794e7
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex e9ef087b8..e581b2ccd 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -259,11 +259,6 @@ export function calculateCost(output_cost: 2.19,},"google/gemini-2.5-flash-lite": { input_cost: 0.1, output_cost: 0.4 },- // Search-monitor models. flash-lite mirrors gemini-2.5-flash-lite pricing;- // gemini-3-flash-preview priced at published Gemini Flash input/output rates.- "gemini-flash-lite-latest": { input_cost: 0.1, output_cost: 0.4 },- "gemini-2.5-flash-lite": { input_cost: 0.1, output_cost: 0.4 },- "gemini-3-flash-preview": { input_cost: 0.3, output_cost: 2.5 },};let modelCost = modelCosts[model] || { input_cost: 0, output_cost: 0 };//gemini-2.5-pro-exp-03-25 pricingdiff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex 60e86142a..708574054 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -49,11 +49,6 @@ export async function fire_engine_search_v2(page?: number;type?: SearchResultType | SearchResultType[];enterprise?: ("default" | "anon" | "zdr")[];- // Opt-in observability hook. Invoked with `true` when the request ultimately- // failed (HTTP error / rate-limit / exhausted retries / thrown) and produced- // no usable response. Stays undefined-safe for existing callers that don't- // care to distinguish a real failure from a genuinely-empty result set.- onFailure?: (reason: string) => void;},abort?: AbortSignal,): Promise<SearchV2Response> {@@ -86,15 +81,5 @@ export async function fire_engine_search_v2(abort,);- // `attemptRequest` returns null on any non-OK HTTP response (e.g. 429- // rate-limit) or thrown error, and `executeWithRetry` only returns null once- // every attempt has been exhausted. A null here therefore means a real- // failure β NOT a successful search that legitimately found nothing (which- // comes back as a non-null `{}` / `{ web: [] }`). Surface that to opt-in- // callers so a swallowed rate-limit isn't mistaken for "no results".- if (result === null) {- options.onFailure?.("fire_engine_search request failed after retries");- }-return result ?? {};}diff --git a/apps/api/src/search/v2/index.ts b/apps/api/src/search/v2/index.tsindex feea56989..3dca1a848 100644--- a/apps/api/src/search/v2/index.ts+++ b/apps/api/src/search/v2/index.ts@@ -20,7 +20,6 @@ export async function search({timeout = 5000,type = undefined,enterprise = undefined,- onFailure = undefined,}: {query: string;logger: Logger;@@ -36,13 +35,6 @@ export async function search({timeout?: number;type?: SearchResultType | SearchResultType[];enterprise?: ("default" | "anon" | "zdr")[];- // Opt-in soft-failure hook. Called with a reason when the search provider- // genuinely failed (rate-limit / HTTP error / thrown) rather than legitimately- // returning zero results. Lets callers (e.g. the search monitor) tell a real- // failure apart from a clean empty so they can retry or report degraded- // instead of silently treating it as "no changes". Optional β existing- // callers that omit it keep the exact same behavior and return type.- onFailure?: (reason: string) => void;}): Promise<SearchV2Response> {try {if (config.FIRE_ENGINE_BETA_URL) {@@ -56,7 +48,6 @@ export async function search({location,type,enterprise,- onFailure,});return results;@@ -89,9 +80,6 @@ export async function search({return {};} catch (error) {logger.error(`Error in search function`, { error });- // Surface the thrown failure to opt-in callers so it isn't mistaken for a- // genuinely-empty result. Return contract is unchanged (still `{}`).- onFailure?.(error instanceof Error ? error.message : String(error));return {};}}diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex c11360211..fafb640a9 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -50,43 +50,31 @@ function windowToTbs(window: string): string {}// The provider rate-limits near-simultaneous identical queries and may answer-// with an error OR a clean HTTP 200 with an empty body β indistinguishable from-// a genuinely-empty result. Retry both empty and failed responses with backoff;-// a real empty query just exhausts attempts and returns empty cleanly. `failed`-// is true only when the LAST attempt was a real failure, so the caller can mark-// the check degraded rather than confidently empty.+// with a clean HTTP 200 empty body. Retry empty responses with backoff; a+// genuinely-empty query just exhausts attempts and returns empty cleanly.const SEARCH_RETRY_BACKOFF_MS = [400, 1200] as const;async function searchWithRetry(args: Parameters<typeof search>[0],logger: Logger,-): Promise<{ response: Awaited<ReturnType<typeof search>>; failed: boolean }> {+): Promise<Awaited<ReturnType<typeof search>>> {const maxAttempts = SEARCH_RETRY_BACKOFF_MS.length + 1;let lastResponse: Awaited<ReturnType<typeof search>> = {};- let lastFailureReason: string | null = null;for (let attempt = 0; attempt < maxAttempts; attempt++) {- let failureReason: string | null = null;- const response = await search({- ...args,- onFailure: reason => {- failureReason = reason;- },- });+ const response = await search(args);lastResponse = response;- lastFailureReason = failureReason;const hasResults = (response.web?.length ?? 0) > 0;- if (hasResults) return { response, failed: false };+ if (hasResults) return response;const isLastAttempt = attempt === maxAttempts - 1;if (!isLastAttempt) {const base = SEARCH_RETRY_BACKOFF_MS[attempt];const jitter = Math.floor(Math.random() * 250);- logger.info("search monitor query empty/failed; retrying", {+ logger.info("search monitor query empty; retrying", {query: args.query,attempt: attempt + 1,- failureReason,backoffMs: base + jitter,});await new Promise(r => setTimeout(r, base + jitter));@@ -94,21 +82,7 @@ async function searchWithRetry(}}- // Exhausted attempts: a real final failure is the false-negative we must not- // hide; an empty with no failure is a legitimate no-results.- if (lastFailureReason) {- logger.warn(- "search monitor query failed after retries; reporting degraded (NOT clean no-results)",- {- query: args.query,- reason: lastFailureReason,- attempts: maxAttempts,- },- );- return { response: lastResponse, failed: true };- }-- return { response: lastResponse, failed: false };+ return lastResponse;}function isExcludedDomain(@@ -175,21 +149,20 @@ type SearchTargetRunResult = {meaningful: number;matches: number;summary: string;- // True when a query failed at the provider after retries rather than- // legitimately returning zero results. An empty-but-degraded run means "could- // not confirm", not a confident "no new results" β avoids a silent false- // negative on a swallowed rate-limit.+ // Reserved provider-level degraded signal. The public search() helper swallows+ // provider errors into an empty result, so the monitor cannot tell a real+ // failure from a genuine no-results without reaching into core search; this+ // therefore stays false. Kept for the check shape and possible future use.searchDegraded: boolean;// True when the deep (scrape+judge) path was expected to evaluate results but- // evaluated ~none BECAUSE the per-result scrapes failed/timed out β the- // false-negative twin of searchDegraded, one layer deeper. A judged check that+ // evaluated ~none BECAUSE the per-result scrapes failed/timed out. A judged check that// completes with 0 pages / 0 judged because every scrape errored otherwise// looks identical to "nothing new"; this flags it so it isn't read as clean.// Conservative: NOT set when search legitimately returned 0 results, and NOT// set when the judge legitimately ignored everything (resultsJudged > 0).judgeDegraded: boolean;- // Human-readable explanation for whichever degraded flag is set (search OR- // judge), surfaced onto the check/summary the same way searchDegraded is.+ // Human-readable explanation when judgeDegraded is set, surfaced onto the+ // check/summary.degradedReason: string | null;sources: SearchSource[];// Search() call credits (β2 per 10 results), matching executeSearch's math.@@ -258,9 +231,6 @@ export async function runSearchTarget(params: {let searchResultsBilled = 0;// Denominator for flat judge billing: judgeCredits = 5 * this.let resultsJudged = 0;- // Queries that failed at the provider after retries (vs. legitimately empty);- // used to mark the run degraded.- let searchFailures = 0;// Deep-path per-result scrape/judge failures (scrape threw, scrape !success,// or verdict unparseable) β the silently-swallowed errors that hit `continue`// BEFORE resultsJudged is incremented. Counted to mark the check degraded when@@ -281,7 +251,7 @@ export async function runSearchTarget(params: {includeDomains: target.includeDomains,excludeDomains: target.excludeDomains,});- const { response, failed } = await searchWithRetry(+ const response = await searchWithRetry({query: scopedQuery,logger,@@ -290,9 +260,6 @@ export async function runSearchTarget(params: {},logger,);- if (failed) {- searchFailures += 1;- }const results = response.web ?? [];// Bill on raw results, before dedupe/trim.searchResultsBilled += results.length;@@ -925,23 +892,12 @@ export async function runSearchTarget(params: {}}- const searchDegraded = searchFailures > 0;- if (searchDegraded && matches === 0) {- // Don't let a swallowed provider failure masquerade as clean no-results.- logger.warn(- "search monitor run is degraded: one or more queries failed at the provider after retries; results may be incomplete",- {- monitorId: params.monitor.id,- targetId: target.id,- failedQueries: searchFailures,- totalQueries: target.queries.length,- },- );- if (!summary) {- summary =- "Search provider was unavailable for one or more queries (rate-limited or errored after retries); results may be incomplete.";- }- }+ // The public search() helper swallows provider errors and returns an empty+ // result, so the monitor can't distinguish a real provider failure from a+ // genuine no-results without reaching into core search. We deliberately don't:+ // empty-after-retries is treated as a clean no-results. The field is kept for+ // the check shape (and the deep-path judge-degraded signal below still fires).+ const searchDegraded = false;// Deep-path degraded signal: judging WAS expected (depth deep + search// returned candidates) but produced ~nothing (resultsJudged === 0) BECAUSE the@@ -974,15 +930,11 @@ export async function runSearchTarget(params: {}}- // Single reason surfaced onto the check the same way searchDegraded is. Search- // failure takes precedence (it's upstream β a failed query means we never even- // got results to scrape).- const degradedReason: string | null =- searchDegraded && matches === 0- ? "search provider unavailable for one or more queries after retries; results may be incomplete"- : judgeDegraded- ? "deep-path scrapes failed for every candidate so nothing was judged; results may be incomplete"- : null;+ // Single reason surfaced onto the check when the deep path could not evaluate+ // the results it found.+ const degradedReason: string | null = judgeDegraded+ ? "deep-path scrapes failed for every candidate so nothing was judged; results may be incomplete"+ : null;const isZDR = params.zeroDataRetention;const searchCredits =