Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas85cd7b380cf9
diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.tsindex d8d040cec..ac65ccca4 100644--- a/apps/api/src/search/fireEngine.ts+++ b/apps/api/src/search/fireEngine.ts@@ -5,35 +5,22 @@ import { logger } from "../lib/logger";dotenv.config();-export async function fire_engine_search(- q: string,- options: {- tbs?: string;- filter?: string;- lang?: string;- country?: string;- location?: string;- numResults: number;- page?: number;- },- abort?: AbortSignal,-): Promise<SearchResult[]> {- try {- let data = JSON.stringify({- query: q,- lang: options.lang,- country: options.country,- location: options.location,- tbs: options.tbs,- numResults: options.numResults,- page: options.page ?? 1,- });+const RETRY_DELAYS = [500, 1500, 3000] as const;+const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;- if (!process.env.FIRE_ENGINE_BETA_URL) {- return [];- }+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++function hasResults(results: unknown): results is SearchResult[] {+ return Array.isArray(results) && results.length > 0;+}- const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/search`, {+async function attemptRequest(+ url: string,+ data: string,+ abort?: AbortSignal+): Promise<SearchResult[] | null> {+ try {+ const response = await fetch(url, {method: "POST",headers: {"Content-Type": "application/json",@@ -44,19 +31,41 @@ export async function fire_engine_search(});if (response.ok) {- const responseData = await response.json();- return responseData;- } else {- return [];+ return await response.json();}} catch (error) {- logger.error(error);+ logger.error("Request attempt failed:", error);Sentry.captureException(error);- return [];}+ return null;}-export async function fireEngineMap(+async function executeWithRetry(+ url: string,+ payload: Record<string, any>,+ abort?: AbortSignal+): Promise<SearchResult[]> {+ const data = JSON.stringify(payload);++ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {+ if (abort?.aborted) break;++ const responseData = await attemptRequest(url, data, abort);++ if (hasResults(responseData)) {+ return responseData;+ }++ // Wait before retry (except on last attempt)+ if (attempt < RETRY_DELAYS.length) {+ await sleep(RETRY_DELAYS[attempt]);+ }+ }++ return [];+}++export async function fire_engine_search(q: string,options: {tbs?: string;@@ -69,43 +78,54 @@ export async function fireEngineMap(},abort?: AbortSignal,): Promise<SearchResult[]> {- try {- let data = JSON.stringify({- query: q,- lang: options.lang,- country: options.country,- location: options.location,- tbs: options.tbs,- numResults: options.numResults,- page: options.page ?? 1,- });+ if (!process.env.FIRE_ENGINE_BETA_URL) {+ return [];+ }- if (!process.env.FIRE_ENGINE_BETA_URL) {- logger.warn(- "(v1/map Beta) Results might differ from cloud offering currently.",- );- return [];- }+ const payload = {+ query: q,+ lang: options.lang,+ country: options.country,+ location: options.location,+ tbs: options.tbs,+ numResults: options.numResults,+ page: options.page ?? 1,+ };- const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/map`, {- method: "POST",- headers: {- "Content-Type": "application/json",- "X-Disable-Cache": "true",- },- body: data,- signal: abort,- });+ const url = `${process.env.FIRE_ENGINE_BETA_URL}/search`;+ return executeWithRetry(url, payload, abort);+}- if (response.ok) {- const responseData = await response.json();- return responseData;- } else {- return [];- }- } catch (error) {- logger.error(error);- Sentry.captureException(error);+export async function fireEngineMap(+ q: string,+ options: {+ tbs?: string;+ filter?: string;+ lang?: string;+ country?: string;+ location?: string;+ numResults: number;+ page?: number;+ },+ abort?: AbortSignal,+): Promise<SearchResult[]> {+ if (!process.env.FIRE_ENGINE_BETA_URL) {+ logger.warn(+ "(v1/map Beta) Results might differ from cloud offering currently.",+ );return [];}-}++ const payload = {+ query: q,+ lang: options.lang,+ country: options.country,+ location: options.location,+ tbs: options.tbs,+ numResults: options.numResults,+ page: options.page ?? 1,+ };++ const url = `${process.env.FIRE_ENGINE_BETA_URL}/map`;+ return executeWithRetry(url, payload, abort);+}\ No newline at end of filediff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex 32417376c..546ffe45b 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -9,6 +9,54 @@ import { logger } from "../../lib/logger";dotenv.config();+const RETRY_DELAYS = [500, 1500, 3000] as const;+const MAX_ATTEMPTS = RETRY_DELAYS.length + 1;++const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));++function normalizeSearchTypes(+ type?: SearchResultType | SearchResultType[]+): SearchResultType[] {+ if (!type) return ["web"];+ return Array.isArray(type) ? type : [type];+}++function hasCompleteResults(+ response: SearchV2Response,+ requestedTypes: SearchResultType[]+): boolean {+ return requestedTypes.every((type) => {+ const results = response[type];+ return Array.isArray(results) && results.length > 0;+ });+}++async function attemptSearch(+ url: string,+ data: string,+ abort?: AbortSignal+): Promise<SearchV2Response | null> {+ try {+ const response = await fetch(url, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "X-Disable-Cache": "true",+ },+ body: data,+ signal: abort,+ });++ if (response.ok) {+ return await response.json();+ }+ } catch (error) {+ logger.error("Search attempt failed:", error);+ Sentry.captureException(error);+ }+ return null;+}+export async function fire_engine_search_v2(q: string,options: {@@ -23,44 +71,39 @@ export async function fire_engine_search_v2(},abort?: AbortSignal,): Promise<SearchV2Response> {- try {- let data = JSON.stringify({- query: q,- lang: options.lang,- country: options.country,- location: options.location,- tbs: options.tbs,- numResults: options.numResults,- page: options.page ?? 1,- type: options.type || "web",- });+ if (!process.env.FIRE_ENGINE_BETA_URL) {+ return {};+ }- if (!process.env.FIRE_ENGINE_BETA_URL) {- return {};- }+ const payload = {+ query: q,+ lang: options.lang,+ country: options.country,+ location: options.location,+ tbs: options.tbs,+ numResults: options.numResults,+ page: options.page ?? 1,+ type: options.type || "web",+ };++ const requestedTypes = normalizeSearchTypes(options.type);+ const url = `${process.env.FIRE_ENGINE_BETA_URL}/v2/search`;+ const data = JSON.stringify(payload);- const response = await fetch(- `${process.env.FIRE_ENGINE_BETA_URL}/v2/search`,- {- method: "POST",- headers: {- "Content-Type": "application/json",- "X-Disable-Cache": "true",- },- body: data,- signal: abort,- },- );+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {+ if (abort?.aborted) break;- if (response.ok) {- const responseData = await response.json();+ const responseData = await attemptSearch(url, data, abort);++ if (responseData && hasCompleteResults(responseData, requestedTypes)) {return responseData;- } else {- return {};}- } catch (error) {- logger.error(error);- Sentry.captureException(error);- return {};++ // Wait before retry (except on last attempt)+ if (attempt < RETRY_DELAYS.length) {+ await sleep(RETRY_DELAYS[attempt]);+ }}-}++ return {};+}\ No newline at end of file