Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: refactored

Nicolas66d76159b568
diff --git a/apps/api/src/lib/retry-utils.ts b/apps/api/src/lib/retry-utils.ts
new file mode 100644
index 000000000..bc9f01a98
--- /dev/null
+++ b/apps/api/src/lib/retry-utils.ts
@@ -0,0 +1,67 @@
+import * as Sentry from "@sentry/node";
+import { logger } from "./logger";
+
+const RETRY_DELAYS = [500, 1500, 3000] as const;
+const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;
+
+/**
+ * Abortable sleep function that resolves immediately if the signal is aborted
+ */
+function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise((resolve, reject) => {
+ if (signal?.aborted) {
+ resolve();
+ return;
+ }
+
+ const timeoutId = setTimeout(() => {
+ resolve();
+ }, ms);
+
+ const abortHandler = () => {
+ clearTimeout(timeoutId);
+ resolve();
+ };
+
+ signal?.addEventListener("abort", abortHandler, { once: true });
+ });
+}
+
+/**
+ * Generic retry utility that executes an operation with exponential backoff
+ * @param operation - The async operation to retry
+ * @param hasValidResult - Function to check if the result is valid/complete
+ * @param signal - Optional AbortSignal to cancel the operation
+ * @param maxAttempts - Maximum number of attempts (defaults to 4)
+ * @param retryDelays - Array of delay times in ms between retries
+ * @returns The result of the operation or null if all attempts fail
+ */
+export async function executeWithRetry<T>(
+ operation: () => Promise<T | null>,
+ hasValidResult: (result: T | null) => result is T,
+ signal?: AbortSignal,
+ maxAttempts: number = MAX_ATTEMPTS,
+ retryDelays: readonly number[] = RETRY_DELAYS,
+): Promise<T | null> {
+ for (let attempt = 0; attempt < maxAttempts; attempt++) {
+ if (signal?.aborted) break;
+
+ try {
+ const result = await operation();
+
+ if (hasValidResult(result)) {
+ return result;
+ }
+ } catch (error) {
+ logger.error(`Attempt ${attempt + 1} failed:`, error);
+ Sentry.captureException(error);
+ }
+
+ // Wait before retry (except on last attempt)
+ if (attempt < retryDelays.length) {
+ await abortableSleep(retryDelays[attempt], signal);
+ }
+ }
+
+ return null;
+}
diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.ts
index ac65ccca4..8793e330e 100644
--- a/apps/api/src/search/fireEngine.ts
+++ b/apps/api/src/search/fireEngine.ts
@@ -2,14 +2,10 @@ import dotenv from "dotenv";
import { SearchResult } from "../../src/lib/entities";
import * as Sentry from "@sentry/node";
import { logger } from "../lib/logger";
+import { executeWithRetry } from "../lib/retry-utils";
dotenv.config();
-const RETRY_DELAYS = [500, 1500, 3000] as const;
-const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;
-
-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
function hasResults(results: unknown): results is SearchResult[] {
return Array.isArray(results) && results.length > 0;
}
@@ -17,7 +13,7 @@ function hasResults(results: unknown): results is SearchResult[] {
async function attemptRequest(
url: string,
data: string,
- abort?: AbortSignal
+ abort?: AbortSignal,
): Promise<SearchResult[] | null> {
try {
const response = await fetch(url, {
@@ -40,31 +36,6 @@ async function attemptRequest(
return null;
}
-async function executeWithRetry(
- url: string,
- payload: Record<string, any>,
- abort?: AbortSignal
-): Promise<SearchResult[]> {
- const data = JSON.stringify(payload);
-
- for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
- if (abort?.aborted) break;
-
- const responseData = await attemptRequest(url, data, abort);
-
- if (hasResults(responseData)) {
- return responseData;
- }
-
- // Wait before retry (except on last attempt)
- if (attempt < RETRY_DELAYS.length) {
- await sleep(RETRY_DELAYS[attempt]);
- }
- }
-
- return [];
-}
-
export async function fire_engine_search(
q: string,
options: {
@@ -93,7 +64,15 @@ export async function fire_engine_search(
};
const url = `${process.env.FIRE_ENGINE_BETA_URL}/search`;
- return executeWithRetry(url, payload, abort);
+ const data = JSON.stringify(payload);
+
+ const result = await executeWithRetry<SearchResult[]>(
+ () => attemptRequest(url, data, abort),
+ hasResults,
+ abort,
+ );
+
+ return result ?? [];
}
export async function fireEngineMap(
@@ -127,5 +106,13 @@ export async function fireEngineMap(
};
const url = `${process.env.FIRE_ENGINE_BETA_URL}/map`;
- return executeWithRetry(url, payload, abort);
-}
\ No newline at end of file
+ const data = JSON.stringify(payload);
+
+ const result = await executeWithRetry<SearchResult[]>(
+ () => attemptRequest(url, data, abort),
+ hasResults,
+ abort,
+ );
+
+ return result ?? [];
+}
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.ts
index 546ffe45b..4336b3d20 100644
--- a/apps/api/src/search/v2/fireEngine-v2.ts
+++ b/apps/api/src/search/v2/fireEngine-v2.ts
@@ -6,16 +6,12 @@ import {
} from "../../lib/entities";
import * as Sentry from "@sentry/node";
import { logger } from "../../lib/logger";
+import { executeWithRetry } from "../../lib/retry-utils";
dotenv.config();
-const RETRY_DELAYS = [500, 1500, 3000] as const;
-const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;
-
-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
function normalizeSearchTypes(
- type?: SearchResultType | SearchResultType[]
+ type?: SearchResultType | SearchResultType[],
): SearchResultType[] {
if (!type) return ["web"];
return Array.isArray(type) ? type : [type];
@@ -23,9 +19,9 @@ function normalizeSearchTypes(
function hasCompleteResults(
response: SearchV2Response,
- requestedTypes: SearchResultType[]
+ requestedTypes: SearchResultType[],
): boolean {
- return requestedTypes.every((type) => {
+ return requestedTypes.every(type => {
const results = response[type];
return Array.isArray(results) && results.length > 0;
});
@@ -34,7 +30,7 @@ function hasCompleteResults(
async function attemptSearch(
url: string,
data: string,
- abort?: AbortSignal
+ abort?: AbortSignal,
): Promise<SearchV2Response | null> {
try {
const response = await fetch(url, {
@@ -85,25 +81,17 @@ export async function fire_engine_search_v2(
page: options.page ?? 1,
type: options.type || "web",
};
-
+
const requestedTypes = normalizeSearchTypes(options.type);
const url = `${process.env.FIRE_ENGINE_BETA_URL}/v2/search`;
const data = JSON.stringify(payload);
- for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
- if (abort?.aborted) break;
-
- const responseData = await attemptSearch(url, data, abort);
-
- if (responseData && hasCompleteResults(responseData, requestedTypes)) {
- return responseData;
- }
+ const result = await executeWithRetry<SearchV2Response>(
+ () => attemptSearch(url, data, abort),
+ (response): response is SearchV2Response =>
+ response !== null && hasCompleteResults(response, requestedTypes),
+ abort,
+ );
- // Wait before retry (except on last attempt)
- if (attempt < RETRY_DELAYS.length) {
- await sleep(RETRY_DELAYS[attempt]);
- }
- }
-
- return {};
-}
\ No newline at end of file
+ return result ?? {};
+}