Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL/proxy/auto): retry on 401

Gergő Móricz0e591dec49df
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 672d858a1..04e23b43e 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -315,7 +315,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
(engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||
engineResult.statusCode === 304;
const hasNoPageError = engineResult.error === undefined;
- const isLikelyProxyError = [403, 429].includes(engineResult.statusCode);
+ const isLikelyProxyError = [401, 403, 429].includes(engineResult.statusCode);
meta.results[engine] = {
state: "success",