Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update fetch.ts

Nicolasdba96998e35a
diff --git a/apps/api/src/scraper/WebScraper/scrapers/fetch.ts b/apps/api/src/scraper/WebScraper/scrapers/fetch.ts
index 88659797d..0df3be724 100644
--- a/apps/api/src/scraper/WebScraper/scrapers/fetch.ts
+++ b/apps/api/src/scraper/WebScraper/scrapers/fetch.ts
@@ -35,7 +35,9 @@ export async function scrapWithFetch(
});
if (response.status !== 200) {
- Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} with status: ${response.status}`);
+ Logger.debug(
+ `⛏️ Axios: Failed to fetch url: ${url} with status: ${response.status}`
+ );
logParams.error_message = response.statusText;
logParams.response_code = response.status;
return {
@@ -48,7 +50,10 @@ export async function scrapWithFetch(
const contentType = response.headers["content-type"];
if (contentType && contentType.includes("application/pdf")) {
logParams.success = true;
- const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);
+ const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(
+ url,
+ pageOptions?.parsePDF
+ );
logParams.response_code = pageStatusCode;
logParams.error_message = pageError;
return { content, pageStatusCode: response.status, pageError };
@@ -57,7 +62,11 @@ export async function scrapWithFetch(
logParams.success = true;
logParams.html = text;
logParams.response_code = response.status;
- return { content: text, pageStatusCode: response.status, pageError: null };
+ return {
+ content: text,
+ pageStatusCode: response.status,
+ pageError: null,
+ };
}
} catch (error) {
if (error.code === "ECONNABORTED") {
@@ -67,7 +76,11 @@ export async function scrapWithFetch(
logParams.error_message = error.message || error;
Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} | Error: ${error}`);
}
- return { content: "", pageStatusCode: error.response?.status ?? null, pageError: logParams.error_message };
+ return {
+ content: "",
+ pageStatusCode: error.response?.status ?? null,
+ pageError: logParams.error_message,
+ };
} finally {
const endTime = Date.now();
logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;