Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fixed file blocking process
rafaelsideguide2bf7b433e2c7
diff --git a/apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts b/apps/api/src/scraper/WebScraper/utils/pdfProcessor.tsindex b27db99a5..4d75df338 100644--- a/apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts+++ b/apps/api/src/scraper/WebScraper/utils/pdfProcessor.ts@@ -1,5 +1,5 @@import axios, { AxiosResponse } from "axios";-import fs from "fs";+import fs from "fs/promises";import { createReadStream, createWriteStream } from "node:fs";import FormData from "form-data";import dotenv from "dotenv";@@ -15,7 +15,7 @@ export async function fetchAndProcessPdf(url: string, parsePDF: boolean): Promistry {const { tempFilePath, pageStatusCode, pageError } = await downloadPdf(url);const content = await processPdfToText(tempFilePath, parsePDF);- fs.unlinkSync(tempFilePath); // Clean up the temporary file+ await fs.unlink(tempFilePath); // Clean up the temporary filereturn { content, pageStatusCode, pageError };} catch (error) {Logger.error(`Failed to fetch and process PDF: ${error.message}`);@@ -120,7 +120,7 @@ export async function processPdfToText(filePath: string, parsePDF: boolean): Pro}} else {try {- content = fs.readFileSync(filePath, "utf-8");+ content = await fs.readFile(filePath, "utf-8");} catch (error) {Logger.error(`Failed to read PDF file: ${error}`);content = "";@@ -131,7 +131,7 @@ export async function processPdfToText(filePath: string, parsePDF: boolean): Proasync function processPdf(file: string) {try {- const fileContent = fs.readFileSync(file);+ const fileContent = await fs.readFile(file);const data = await pdf(fileContent);return data.text;} catch (error) {