Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update fetch.ts
Nicolas668ff3c71b4d
diff --git a/apps/api/src/scraper/WebScraper/scrapers/fetch.ts b/apps/api/src/scraper/WebScraper/scrapers/fetch.tsindex c9ddf93a8..88659797d 100644--- a/apps/api/src/scraper/WebScraper/scrapers/fetch.ts+++ b/apps/api/src/scraper/WebScraper/scrapers/fetch.ts@@ -51,7 +51,7 @@ export async function scrapWithFetch(const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);logParams.response_code = pageStatusCode;logParams.error_message = pageError;- return { content, pageStatusCode, pageError };+ return { content, pageStatusCode: response.status, pageError };} else {const text = response.data;logParams.success = true;@@ -67,7 +67,7 @@ export async function scrapWithFetch(logParams.error_message = error.message || error;Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} | Error: ${error}`);}- return { content: "", pageStatusCode: null, pageError: logParams.error_message };+ return { content: "", pageStatusCode: error.response?.status ?? null, pageError: logParams.error_message };} finally {const endTime = Date.now();logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;