Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL/pdf/mu): add timeout and created_at (#1570)
Gergő Móricz192d056bef82
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 29a07a5dc..733b06df3 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -27,6 +27,8 @@ async function scrapePDFWithRunPodMU(tempFilePath,});+ const preCacheCheckStartTime = Date.now();+try {const cachedResult = await getPdfResultFromCache(base64Content);@@ -54,6 +56,8 @@ async function scrapePDFWithRunPodMU(input: {file_content: base64Content,filename: path.basename(tempFilePath) + ".pdf",+ timeout: timeToRun ? timeToRun - (Date.now() - preCacheCheckStartTime) : undefined,+ created_at: Date.now(),},},logger: meta.logger.child({@@ -103,6 +107,8 @@ export async function scrapePDF(meta: Meta,timeToRun: number | undefined,): Promise<EngineScrapeResult> {+ const startTime = Date.now();+if (!meta.options.parsePDF) {if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {const content = (await readFile(meta.pdfPrefetch.filePath)).toString("base64");@@ -167,7 +173,7 @@ export async function scrapePDF(}),},tempFilePath,- timeToRun,+ timeToRun ? (timeToRun - (Date.now() - startTime)) : undefined,base64Content,);} catch (error) {