Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: max retries with axios-retry

Nicolas975f0575b4e0
diff --git a/apps/api/package.json b/apps/api/package.json
index 770828c18..a0f9cf8e5 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -67,6 +67,7 @@
"async": "^3.2.5",
"async-mutex": "^0.5.0",
"axios": "^1.3.4",
+ "axios-retry": "^4.5.0",
"bottleneck": "^2.19.5",
"bullmq": "^5.11.0",
"cacheable-lookup": "^6.1.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index b8f876a86..095b507c3 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -65,6 +65,9 @@ importers:
axios:
specifier: ^1.3.4
version: 1.7.2
+ axios-retry:
+ specifier: ^4.5.0
+ version: 4.5.0([email protected])
bottleneck:
specifier: ^2.19.5
version: 2.19.5
@@ -1903,6 +1906,11 @@ packages:
resolution: {integrity: sha512-8PJDLJv7qTTMMwdnbMvrLYuvB47M81wRtxQmEdV5w4rgbTXTt+vtPkXwajOfOdSyv/wZICJOC+/UhXH4aQ/R+w==}
+ resolution: {integrity: sha512-aR99oXhpEDGo0UuAlYcn2iGRds30k366Zfa05XWScR9QaQD4JYiP3/1Qt1u7YlefUOK+cn0CcwoL1oefavQUlQ==}
+ peerDependencies:
+ axios: 0.x || 1.x
+
resolution: {integrity: sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==}
@@ -4518,8 +4526,8 @@ packages:
engines: {node: '>=14.17'}
hasBin: true
- resolution: {integrity: sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==}
+ resolution: {integrity: sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==}
engines: {node: '>=14.17'}
hasBin: true
@@ -6950,6 +6958,11 @@ snapshots:
'@babel/runtime': 7.24.6
is-retry-allowed: 2.2.0
+ dependencies:
+ axios: 1.7.2
+ is-retry-allowed: 2.2.0
+
dependencies:
follow-redirects: 1.15.6
@@ -9195,7 +9208,7 @@ snapshots:
csv-parse: 5.5.6
gpt3-tokenizer: 1.1.5
openai: 3.3.0
- typescript: 5.5.4
+ typescript: 5.6.2
uuid: 9.0.1
zod: 3.23.8
transitivePeerDependencies:
@@ -9793,7 +9806,7 @@ snapshots:
dependencies:
diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
index dd3577b07..f715c4272 100644
--- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
+++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts
@@ -6,7 +6,11 @@ import { fetchAndProcessPdf } from "../utils/pdfProcessor";
import { universalTimeout } from "../global";
import { Logger } from "../../../lib/logger";
import * as Sentry from "@sentry/node";
+import axiosRetry from 'axios-retry';
+axiosRetry(axios, { retries: 3 , onRetry:()=>{
+ console.log("Retrying (fire-engine)...");
+}, retryDelay: axiosRetry.exponentialDelay});
/**
* Scrapes a URL with Fire-Engine
* @param url The URL to scrape
@@ -203,10 +207,10 @@ export async function scrapWithFireEngine({
}
} catch (error) {
if (error.code === "ECONNABORTED") {
- Logger.debug(`⛏️ Fire-Engine: Request timed out for ${url}`);
+ Logger.debug(`⛏️ Fire-Engine (catch block): Request timed out for ${url}`);
logParams.error_message = "Request timed out";
} else {
- Logger.debug(`⛏️ Fire-Engine: Failed to fetch url: ${url} | Error: ${error}`);
+ Logger.debug(`⛏️ Fire-Engine(catch block): Failed to fetch url: ${url} | Error: ${error}`);
logParams.error_message = error.message || error;
}
return { html: "", pageStatusCode: null, pageError: logParams.error_message };
diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
index 27bdc3a84..db8c46b1e 100644
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -424,7 +424,7 @@ export async function scrapSingleUrl(
Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with text length >= 100 or screenshot, breaking`);
break;
}
- if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 500)) {
+ if (pageStatusCode && (pageStatusCode == 404)) {
Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with status code 404, breaking`);
break;
}