Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update fetch.ts

Nicolas668ff3c71b4d
diff --git a/apps/api/src/scraper/WebScraper/scrapers/fetch.ts b/apps/api/src/scraper/WebScraper/scrapers/fetch.ts
index c9ddf93a8..88659797d 100644
--- a/apps/api/src/scraper/WebScraper/scrapers/fetch.ts
+++ b/apps/api/src/scraper/WebScraper/scrapers/fetch.ts
@@ -51,7 +51,7 @@ export async function scrapWithFetch(
const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);
logParams.response_code = pageStatusCode;
logParams.error_message = pageError;
- return { content, pageStatusCode, pageError };
+ return { content, pageStatusCode: response.status, pageError };
} else {
const text = response.data;
logParams.success = true;
@@ -67,7 +67,7 @@ export async function scrapWithFetch(
logParams.error_message = error.message || error;
Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} | Error: ${error}`);
}
- return { content: "", pageStatusCode: null, pageError: logParams.error_message };
+ return { content: "", pageStatusCode: error.response?.status ?? null, pageError: logParams.error_message };
} finally {
const endTime = Date.now();
logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;