Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL/pdf): support PDF prefetch when parsePDF is off
Gergő Móricz11ed679274a1
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 77905e78f..727e12c95 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -76,17 +76,34 @@ export async function scrapePDF(timeToRun: number | undefined,): Promise<EngineScrapeResult> {if (!meta.options.parsePDF) {- const file = await fetchFileToBuffer(meta.url, {- headers: meta.options.headers,- });- const content = file.buffer.toString("base64");- return {- url: file.response.url,- statusCode: file.response.status,-- html: content,- markdown: content,- };+ if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {+ const content = (await readFile(meta.pdfPrefetch.filePath)).toString("base64");+ return {+ url: meta.pdfPrefetch.url ?? meta.url,+ statusCode: meta.pdfPrefetch.status,++ html: content,+ markdown: content,+ };+ } else {+ const file = await fetchFileToBuffer(meta.url, {+ headers: meta.options.headers,+ });++ const ct = file.response.headers.get("Content-Type");+ if (ct && !ct.includes("application/pdf")) { // if downloaded file wasn't a PDF+ throw new PDFAntibotError();+ }++ const content = file.buffer.toString("base64");+ return {+ url: file.response.url,+ statusCode: file.response.status,++ html: content,+ markdown: content,+ };+ }}const { response, tempFilePath } = (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null)