Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolas85cd7b380cf9
diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.ts
index d8d040cec..ac65ccca4 100644
--- a/apps/api/src/search/fireEngine.ts
+++ b/apps/api/src/search/fireEngine.ts
@@ -5,35 +5,22 @@ import { logger } from "../lib/logger";
dotenv.config();
-export async function fire_engine_search(
- q: string,
- options: {
- tbs?: string;
- filter?: string;
- lang?: string;
- country?: string;
- location?: string;
- numResults: number;
- page?: number;
- },
- abort?: AbortSignal,
-): Promise<SearchResult[]> {
- try {
- let data = JSON.stringify({
- query: q,
- lang: options.lang,
- country: options.country,
- location: options.location,
- tbs: options.tbs,
- numResults: options.numResults,
- page: options.page ?? 1,
- });
+const RETRY_DELAYS = [500, 1500, 3000] as const;
+const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;
- if (!process.env.FIRE_ENGINE_BETA_URL) {
- return [];
- }
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+function hasResults(results: unknown): results is SearchResult[] {
+ return Array.isArray(results) && results.length > 0;
+}
- const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/search`, {
+async function attemptRequest(
+ url: string,
+ data: string,
+ abort?: AbortSignal
+): Promise<SearchResult[] | null> {
+ try {
+ const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
@@ -44,19 +31,41 @@ export async function fire_engine_search(
});
if (response.ok) {
- const responseData = await response.json();
- return responseData;
- } else {
- return [];
+ return await response.json();
}
} catch (error) {
- logger.error(error);
+ logger.error("Request attempt failed:", error);
Sentry.captureException(error);
- return [];
}
+ return null;
}
-export async function fireEngineMap(
+async function executeWithRetry(
+ url: string,
+ payload: Record<string, any>,
+ abort?: AbortSignal
+): Promise<SearchResult[]> {
+ const data = JSON.stringify(payload);
+
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+ if (abort?.aborted) break;
+
+ const responseData = await attemptRequest(url, data, abort);
+
+ if (hasResults(responseData)) {
+ return responseData;
+ }
+
+ // Wait before retry (except on last attempt)
+ if (attempt < RETRY_DELAYS.length) {
+ await sleep(RETRY_DELAYS[attempt]);
+ }
+ }
+
+ return [];
+}
+
+export async function fire_engine_search(
q: string,
options: {
tbs?: string;
@@ -69,43 +78,54 @@ export async function fireEngineMap(
},
abort?: AbortSignal,
): Promise<SearchResult[]> {
- try {
- let data = JSON.stringify({
- query: q,
- lang: options.lang,
- country: options.country,
- location: options.location,
- tbs: options.tbs,
- numResults: options.numResults,
- page: options.page ?? 1,
- });
+ if (!process.env.FIRE_ENGINE_BETA_URL) {
+ return [];
+ }
- if (!process.env.FIRE_ENGINE_BETA_URL) {
- logger.warn(
- "(v1/map Beta) Results might differ from cloud offering currently.",
- );
- return [];
- }
+ const payload = {
+ query: q,
+ lang: options.lang,
+ country: options.country,
+ location: options.location,
+ tbs: options.tbs,
+ numResults: options.numResults,
+ page: options.page ?? 1,
+ };
- const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/map`, {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- "X-Disable-Cache": "true",
- },
- body: data,
- signal: abort,
- });
+ const url = `${process.env.FIRE_ENGINE_BETA_URL}/search`;
+ return executeWithRetry(url, payload, abort);
+}
- if (response.ok) {
- const responseData = await response.json();
- return responseData;
- } else {
- return [];
- }
- } catch (error) {
- logger.error(error);
- Sentry.captureException(error);
+export async function fireEngineMap(
+ q: string,
+ options: {
+ tbs?: string;
+ filter?: string;
+ lang?: string;
+ country?: string;
+ location?: string;
+ numResults: number;
+ page?: number;
+ },
+ abort?: AbortSignal,
+): Promise<SearchResult[]> {
+ if (!process.env.FIRE_ENGINE_BETA_URL) {
+ logger.warn(
+ "(v1/map Beta) Results might differ from cloud offering currently.",
+ );
return [];
}
-}
+
+ const payload = {
+ query: q,
+ lang: options.lang,
+ country: options.country,
+ location: options.location,
+ tbs: options.tbs,
+ numResults: options.numResults,
+ page: options.page ?? 1,
+ };
+
+ const url = `${process.env.FIRE_ENGINE_BETA_URL}/map`;
+ return executeWithRetry(url, payload, abort);
+}
\ No newline at end of file
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.ts
index 32417376c..546ffe45b 100644
--- a/apps/api/src/search/v2/fireEngine-v2.ts
+++ b/apps/api/src/search/v2/fireEngine-v2.ts
@@ -9,6 +9,54 @@ import { logger } from "../../lib/logger";
dotenv.config();
+const RETRY_DELAYS = [500, 1500, 3000] as const;
+const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+function normalizeSearchTypes(
+ type?: SearchResultType | SearchResultType[]
+): SearchResultType[] {
+ if (!type) return ["web"];
+ return Array.isArray(type) ? type : [type];
+}
+
+function hasCompleteResults(
+ response: SearchV2Response,
+ requestedTypes: SearchResultType[]
+): boolean {
+ return requestedTypes.every((type) => {
+ const results = response[type];
+ return Array.isArray(results) && results.length > 0;
+ });
+}
+
+async function attemptSearch(
+ url: string,
+ data: string,
+ abort?: AbortSignal
+): Promise<SearchV2Response | null> {
+ try {
+ const response = await fetch(url, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ "X-Disable-Cache": "true",
+ },
+ body: data,
+ signal: abort,
+ });
+
+ if (response.ok) {
+ return await response.json();
+ }
+ } catch (error) {
+ logger.error("Search attempt failed:", error);
+ Sentry.captureException(error);
+ }
+ return null;
+}
+
export async function fire_engine_search_v2(
q: string,
options: {
@@ -23,44 +71,39 @@ export async function fire_engine_search_v2(
},
abort?: AbortSignal,
): Promise<SearchV2Response> {
- try {
- let data = JSON.stringify({
- query: q,
- lang: options.lang,
- country: options.country,
- location: options.location,
- tbs: options.tbs,
- numResults: options.numResults,
- page: options.page ?? 1,
- type: options.type || "web",
- });
+ if (!process.env.FIRE_ENGINE_BETA_URL) {
+ return {};
+ }
- if (!process.env.FIRE_ENGINE_BETA_URL) {
- return {};
- }
+ const payload = {
+ query: q,
+ lang: options.lang,
+ country: options.country,
+ location: options.location,
+ tbs: options.tbs,
+ numResults: options.numResults,
+ page: options.page ?? 1,
+ type: options.type || "web",
+ };
+
+ const requestedTypes = normalizeSearchTypes(options.type);
+ const url = `${process.env.FIRE_ENGINE_BETA_URL}/v2/search`;
+ const data = JSON.stringify(payload);
- const response = await fetch(
- `${process.env.FIRE_ENGINE_BETA_URL}/v2/search`,
- {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- "X-Disable-Cache": "true",
- },
- body: data,
- signal: abort,
- },
- );
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+ if (abort?.aborted) break;
- if (response.ok) {
- const responseData = await response.json();
+ const responseData = await attemptSearch(url, data, abort);
+
+ if (responseData && hasCompleteResults(responseData, requestedTypes)) {
return responseData;
- } else {
- return {};
}
- } catch (error) {
- logger.error(error);
- Sentry.captureException(error);
- return {};
+
+ // Wait before retry (except on last attempt)
+ if (attempt < RETRY_DELAYS.length) {
+ await sleep(RETRY_DELAYS[attempt]);
+ }
}
-}
+
+ return {};
+}
\ No newline at end of file