Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL/proxy/auto): retry on 401
Gergő Móricz0e591dec49df
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 672d858a1..04e23b43e 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -315,7 +315,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {(engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||engineResult.statusCode === 304;const hasNoPageError = engineResult.error === undefined;- const isLikelyProxyError = [403, 429].includes(engineResult.statusCode);+ const isLikelyProxyError = [401, 403, 429].includes(engineResult.statusCode);meta.results[engine] = {state: "success",