Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: refactored
Nicolas66d76159b568
diff --git a/apps/api/src/lib/retry-utils.ts b/apps/api/src/lib/retry-utils.tsnew file mode 100644index 000000000..bc9f01a98--- /dev/null+++ b/apps/api/src/lib/retry-utils.ts@@ -0,0 +1,67 @@+import * as Sentry from "@sentry/node";+import { logger } from "./logger";++const RETRY_DELAYS = [500, 1500, 3000] as const;+const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;++/**+ * Abortable sleep function that resolves immediately if the signal is aborted+ */+function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {+ return new Promise((resolve, reject) => {+ if (signal?.aborted) {+ resolve();+ return;+ }++ const timeoutId = setTimeout(() => {+ resolve();+ }, ms);++ const abortHandler = () => {+ clearTimeout(timeoutId);+ resolve();+ };++ signal?.addEventListener("abort", abortHandler, { once: true });+ });+}++/**+ * Generic retry utility that executes an operation with exponential backoff+ * @param operation - The async operation to retry+ * @param hasValidResult - Function to check if the result is valid/complete+ * @param signal - Optional AbortSignal to cancel the operation+ * @param maxAttempts - Maximum number of attempts (defaults to 4)+ * @param retryDelays - Array of delay times in ms between retries+ * @returns The result of the operation or null if all attempts fail+ */+export async function executeWithRetry<T>(+ operation: () => Promise<T | null>,+ hasValidResult: (result: T | null) => result is T,+ signal?: AbortSignal,+ maxAttempts: number = MAX_ATTEMPTS,+ retryDelays: readonly number[] = RETRY_DELAYS,+): Promise<T | null> {+ for (let attempt = 0; attempt < maxAttempts; attempt++) {+ if (signal?.aborted) break;++ try {+ const result = await operation();++ if (hasValidResult(result)) {+ return result;+ }+ } catch (error) {+ logger.error(`Attempt ${attempt + 1} failed:`, error);+ Sentry.captureException(error);+ }++ // Wait before retry (except on last attempt)+ if (attempt < retryDelays.length) {+ await abortableSleep(retryDelays[attempt], signal);+ }+ }++ return null;+}diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.tsindex ac65ccca4..8793e330e 100644--- a/apps/api/src/search/fireEngine.ts+++ b/apps/api/src/search/fireEngine.ts@@ -2,14 +2,10 @@ import dotenv from "dotenv";import { SearchResult } from "../../src/lib/entities";import * as Sentry from "@sentry/node";import { logger } from "../lib/logger";+import { executeWithRetry } from "../lib/retry-utils";dotenv.config();-const RETRY_DELAYS = [500, 1500, 3000] as const;-const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;--const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));-function hasResults(results: unknown): results is SearchResult[] {return Array.isArray(results) && results.length > 0;}@@ -17,7 +13,7 @@ function hasResults(results: unknown): results is SearchResult[] {async function attemptRequest(url: string,data: string,- abort?: AbortSignal+ abort?: AbortSignal,): Promise<SearchResult[] | null> {try {const response = await fetch(url, {@@ -40,31 +36,6 @@ async function attemptRequest(return null;}-async function executeWithRetry(- url: string,- payload: Record<string, any>,- abort?: AbortSignal-): Promise<SearchResult[]> {- const data = JSON.stringify(payload);-- for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {- if (abort?.aborted) break;-- const responseData = await attemptRequest(url, data, abort);-- if (hasResults(responseData)) {- return responseData;- }-- // Wait before retry (except on last attempt)- if (attempt < RETRY_DELAYS.length) {- await sleep(RETRY_DELAYS[attempt]);- }- }-- return [];-}-export async function fire_engine_search(q: string,options: {@@ -93,7 +64,15 @@ export async function fire_engine_search(};const url = `${process.env.FIRE_ENGINE_BETA_URL}/search`;- return executeWithRetry(url, payload, abort);+ const data = JSON.stringify(payload);++ const result = await executeWithRetry<SearchResult[]>(+ () => attemptRequest(url, data, abort),+ hasResults,+ abort,+ );++ return result ?? [];}export async function fireEngineMap(@@ -127,5 +106,13 @@ export async function fireEngineMap(};const url = `${process.env.FIRE_ENGINE_BETA_URL}/map`;- return executeWithRetry(url, payload, abort);-}\ No newline at end of file+ const data = JSON.stringify(payload);++ const result = await executeWithRetry<SearchResult[]>(+ () => attemptRequest(url, data, abort),+ hasResults,+ abort,+ );++ return result ?? [];+}diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex 546ffe45b..4336b3d20 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -6,16 +6,12 @@ import {} from "../../lib/entities";import * as Sentry from "@sentry/node";import { logger } from "../../lib/logger";+import { executeWithRetry } from "../../lib/retry-utils";dotenv.config();-const RETRY_DELAYS = [500, 1500, 3000] as const;-const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;--const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));-function normalizeSearchTypes(- type?: SearchResultType | SearchResultType[]+ type?: SearchResultType | SearchResultType[],): SearchResultType[] {if (!type) return ["web"];return Array.isArray(type) ? type : [type];@@ -23,9 +19,9 @@ function normalizeSearchTypes(function hasCompleteResults(response: SearchV2Response,- requestedTypes: SearchResultType[]+ requestedTypes: SearchResultType[],): boolean {- return requestedTypes.every((type) => {+ return requestedTypes.every(type => {const results = response[type];return Array.isArray(results) && results.length > 0;});@@ -34,7 +30,7 @@ function hasCompleteResults(async function attemptSearch(url: string,data: string,- abort?: AbortSignal+ abort?: AbortSignal,): Promise<SearchV2Response | null> {try {const response = await fetch(url, {@@ -85,25 +81,17 @@ export async function fire_engine_search_v2(page: options.page ?? 1,type: options.type || "web",};-+const requestedTypes = normalizeSearchTypes(options.type);const url = `${process.env.FIRE_ENGINE_BETA_URL}/v2/search`;const data = JSON.stringify(payload);- for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {- if (abort?.aborted) break;-- const responseData = await attemptSearch(url, data, abort);-- if (responseData && hasCompleteResults(responseData, requestedTypes)) {- return responseData;- }+ const result = await executeWithRetry<SearchV2Response>(+ () => attemptSearch(url, data, abort),+ (response): response is SearchV2Response =>+ response !== null && hasCompleteResults(response, requestedTypes),+ abort,+ );- // Wait before retry (except on last attempt)- if (attempt < RETRY_DELAYS.length) {- await sleep(RETRY_DELAYS[attempt]);- }- }-- return {};-}\ No newline at end of file+ return result ?? {};+}