Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/scrapeURL/pdf): raise PDF download limit to 50MB
mogery1d6ae3ebc95d
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 522deef45..e2b442ce2 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -26,6 +26,7 @@ import {FIRE_PDF_MAX_FILE_SIZE,MAX_FILE_SIZE,MILLISECONDS_PER_PAGE,+ PDF_DOWNLOAD_MAX_FILE_SIZE,} from "./types";import type { PDFProcessorResult } from "./types";import {@@ -81,7 +82,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {headers: meta.options.headers,signal: meta.abort.asSignal(),},- FIRE_PDF_MAX_FILE_SIZE,+ PDF_DOWNLOAD_MAX_FILE_SIZE,);if (!isPdfBuffer(file.buffer)) {@@ -123,7 +124,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {headers: meta.options.headers,signal: meta.abort.asSignal(),},- FIRE_PDF_MAX_FILE_SIZE,+ PDF_DOWNLOAD_MAX_FILE_SIZE,);try {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.tsindex 11f206a2f..9284dd9d5 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts@@ -15,4 +15,5 @@ export type PdfMetadata = { numPages: number; title?: string };export const MAX_FILE_SIZE = 19 * 1024 * 1024; // 19MBexport const FIRE_PDF_MAX_FILE_SIZE = 30 * 1024 * 1024; // 30MB+export const PDF_DOWNLOAD_MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MBexport const MILLISECONDS_PER_PAGE = 150;