Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update fetch.ts
Nicolasdba96998e35a
diff --git a/apps/api/src/scraper/WebScraper/scrapers/fetch.ts b/apps/api/src/scraper/WebScraper/scrapers/fetch.tsindex 88659797d..0df3be724 100644--- a/apps/api/src/scraper/WebScraper/scrapers/fetch.ts+++ b/apps/api/src/scraper/WebScraper/scrapers/fetch.ts@@ -35,7 +35,9 @@ export async function scrapWithFetch(});if (response.status !== 200) {- Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} with status: ${response.status}`);+ Logger.debug(+ `⛏️ Axios: Failed to fetch url: ${url} with status: ${response.status}`+ );logParams.error_message = response.statusText;logParams.response_code = response.status;return {@@ -48,7 +50,10 @@ export async function scrapWithFetch(const contentType = response.headers["content-type"];if (contentType && contentType.includes("application/pdf")) {logParams.success = true;- const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(url, pageOptions?.parsePDF);+ const { content, pageStatusCode, pageError } = await fetchAndProcessPdf(+ url,+ pageOptions?.parsePDF+ );logParams.response_code = pageStatusCode;logParams.error_message = pageError;return { content, pageStatusCode: response.status, pageError };@@ -57,7 +62,11 @@ export async function scrapWithFetch(logParams.success = true;logParams.html = text;logParams.response_code = response.status;- return { content: text, pageStatusCode: response.status, pageError: null };+ return {+ content: text,+ pageStatusCode: response.status,+ pageError: null,+ };}} catch (error) {if (error.code === "ECONNABORTED") {@@ -67,7 +76,11 @@ export async function scrapWithFetch(logParams.error_message = error.message || error;Logger.debug(`⛏️ Axios: Failed to fetch url: ${url} | Error: ${error}`);}- return { content: "", pageStatusCode: error.response?.status ?? null, pageError: logParams.error_message };+ return {+ content: "",+ pageStatusCode: error.response?.status ?? null,+ pageError: logParams.error_message,+ };} finally {const endTime = Date.now();logParams.time_taken_seconds = (endTime - logParams.startTime) / 1000;