Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas102a73408ff7
diff --git a/apps/api/src/lib/retry-utils.ts b/apps/api/src/lib/retry-utils.tsindex bc9f01a98..eada3ceec 100644--- a/apps/api/src/lib/retry-utils.ts+++ b/apps/api/src/lib/retry-utils.ts@@ -4,6 +4,52 @@ import { logger } from "./logger";const RETRY_DELAYS = [500, 1500, 3000] as const;const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;+/**+ * Generic HTTP request function for Fire Engine API calls+ */+export async function attemptRequest<T>(+ url: string,+ data: string,+ abort?: AbortSignal,+): Promise<T | null> {+ try {+ const response = await fetch(url, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "X-Disable-Cache": "true",+ },+ body: data,+ signal: abort,+ });++ if (response.ok) {+ return await response.json();+ } else {+ // Log non-OK responses for better observability+ const statusText = response.statusText || "Unknown Error";+ let bodySnippet = "";+ try {+ const body = await response.text();+ bodySnippet = body.length > 200 ? body.substring(0, 200) + "..." : body;+ } catch {+ bodySnippet = "[Unable to read response body]";+ }++ logger.warn(`Fire Engine API returned ${response.status} ${statusText}`, {+ url,+ status: response.status,+ statusText,+ bodySnippet,+ });+ }+ } catch (error) {+ logger.error("Fire Engine API request failed:", error);+ Sentry.captureException(error);+ }+ return null;+}+/*** Abortable sleep function that resolves immediately if the signal is aborted*/@@ -53,6 +99,11 @@ export async function executeWithRetry<T>(return result;}} catch (error) {+ // Don't log or report expected abort errors to reduce noise+ if (error instanceof Error && error.name === "AbortError") {+ break;+ }+logger.error(`Attempt ${attempt + 1} failed:`, error);Sentry.captureException(error);}diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.tsindex 8793e330e..f7264756d 100644--- a/apps/api/src/search/fireEngine.ts+++ b/apps/api/src/search/fireEngine.ts@@ -2,40 +2,18 @@ import dotenv from "dotenv";import { SearchResult } from "../../src/lib/entities";import * as Sentry from "@sentry/node";import { logger } from "../lib/logger";-import { executeWithRetry } from "../lib/retry-utils";+import { executeWithRetry, attemptRequest } from "../lib/retry-utils";dotenv.config();+const useFireEngine =+ process.env.FIRE_ENGINE_BETA_URL !== "" &&+ process.env.FIRE_ENGINE_BETA_URL !== undefined;+function hasResults(results: unknown): results is SearchResult[] {return Array.isArray(results) && results.length > 0;}-async function attemptRequest(- url: string,- data: string,- abort?: AbortSignal,-): Promise<SearchResult[] | null> {- try {- const response = await fetch(url, {- method: "POST",- headers: {- "Content-Type": "application/json",- "X-Disable-Cache": "true",- },- body: data,- signal: abort,- });-- if (response.ok) {- return await response.json();- }- } catch (error) {- logger.error("Request attempt failed:", error);- Sentry.captureException(error);- }- return null;-}-export async function fire_engine_search(q: string,options: {@@ -49,7 +27,7 @@ export async function fire_engine_search(},abort?: AbortSignal,): Promise<SearchResult[]> {- if (!process.env.FIRE_ENGINE_BETA_URL) {+ if (!useFireEngine) {return [];}@@ -67,7 +45,7 @@ export async function fire_engine_search(const data = JSON.stringify(payload);const result = await executeWithRetry<SearchResult[]>(- () => attemptRequest(url, data, abort),+ () => attemptRequest<SearchResult[]>(url, data, abort),hasResults,abort,);@@ -88,7 +66,7 @@ export async function fireEngineMap(},abort?: AbortSignal,): Promise<SearchResult[]> {- if (!process.env.FIRE_ENGINE_BETA_URL) {+ if (!useFireEngine) {logger.warn("(v1/map Beta) Results might differ from cloud offering currently.",);@@ -109,7 +87,7 @@ export async function fireEngineMap(const data = JSON.stringify(payload);const result = await executeWithRetry<SearchResult[]>(- () => attemptRequest(url, data, abort),+ () => attemptRequest<SearchResult[]>(url, data, abort),hasResults,abort,);diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex 4336b3d20..564ac2448 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -6,10 +6,14 @@ import {} from "../../lib/entities";import * as Sentry from "@sentry/node";import { logger } from "../../lib/logger";-import { executeWithRetry } from "../../lib/retry-utils";+import { executeWithRetry, attemptRequest } from "../../lib/retry-utils";dotenv.config();+const useFireEngine =+ process.env.FIRE_ENGINE_BETA_URL !== "" &&+ process.env.FIRE_ENGINE_BETA_URL !== undefined;+function normalizeSearchTypes(type?: SearchResultType | SearchResultType[],): SearchResultType[] {@@ -27,32 +31,6 @@ function hasCompleteResults(});}-async function attemptSearch(- url: string,- data: string,- abort?: AbortSignal,-): Promise<SearchV2Response | null> {- try {- const response = await fetch(url, {- method: "POST",- headers: {- "Content-Type": "application/json",- "X-Disable-Cache": "true",- },- body: data,- signal: abort,- });-- if (response.ok) {- return await response.json();- }- } catch (error) {- logger.error("Search attempt failed:", error);- Sentry.captureException(error);- }- return null;-}-export async function fire_engine_search_v2(q: string,options: {@@ -67,7 +45,10 @@ export async function fire_engine_search_v2(},abort?: AbortSignal,): Promise<SearchV2Response> {- if (!process.env.FIRE_ENGINE_BETA_URL) {+ if (!useFireEngine) {+ logger.warn(+ "FIRE_ENGINE_BETA_URL is not configured, returning empty search results",+ );return {};}@@ -87,7 +68,7 @@ export async function fire_engine_search_v2(const data = JSON.stringify(payload);const result = await executeWithRetry<SearchV2Response>(- () => attemptSearch(url, data, abort),+ () => attemptRequest<SearchV2Response>(url, data, abort),(response): response is SearchV2Response =>response !== null && hasCompleteResults(response, requestedTypes),abort,