Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf): add MINERU_PERCENT config for direct MinerU routing (#3354)
Abimael Martellff871d5fb3c4
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex ccc59ac19..7caea74cd 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -162,6 +162,9 @@ const configSchema = z.object({PDF_MU_V2_EXPERIMENT: z.string().optional(),PDF_MU_V2_EXPERIMENT_PERCENT: z.coerce.number().default(100),+ // MinerU direct routing (bypass Rust extraction for a % of traffic)+ MINERU_PERCENT: z.coerce.number().min(0).max(100).default(0),+// Fire PDF (replaces MinerU for a % of traffic)FIRE_PDF_ENABLE: z.stringbool().optional(),FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 8640924d3..53c96fa73 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -161,7 +161,21 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;const logger = meta.logger.child({ method: "scrapePDF/processPdf" });- if (!rustEnabled || mode === "ocr" || forceFirePDF) {+ // Route a percentage of traffic directly to MinerU, bypassing Rust extraction.+ // Forced Fire PDF takes precedence β don't divert those requests.+ const routeToMinerU =+ !forceFirePDF &&+ config.MINERU_PERCENT > 0 &&+ Math.random() * 100 < config.MINERU_PERCENT;++ if (routeToMinerU) {+ logger.info("Routing to MinerU via MINERU_PERCENT", {+ mineruPercent: config.MINERU_PERCENT,+ url: meta.rewrittenUrl ?? meta.url,+ });+ }++ if (!rustEnabled || mode === "ocr" || forceFirePDF || routeToMinerU) {// Legacy / OCR path: detect metadata only, skip Rust extraction.// When PDF_RUST_EXTRACT_ENABLE is off this is the only path taken,// matching current prod behaviour (detectPdf β MinerU β pdfParse).@@ -343,15 +357,19 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {}// OCR / MU fallback.- // Skipped only when Rust extraction is enabled AND mode is "fast".- const skipOCR = rustEnabled && mode === "fast";+ // Skipped only when Rust extraction is enabled AND mode is "fast",+ // unless we explicitly routed to MinerU via MINERU_PERCENT.+ const skipOCR = rustEnabled && mode === "fast" && !routeToMinerU;if (!result && !skipOCR) {const base64Content = (await readFile(tempFilePath)).toString("base64");- // Route a percentage of traffic to Fire PDF instead of MinerU+ // Route a percentage of traffic to Fire PDF instead of MinerU.+ // forceFirePDF always wins; skip percentage-based Fire PDF when+ // we explicitly routed to MinerU via MINERU_PERCENT.const useFirePDF =forceFirePDF ||- (config.FIRE_PDF_ENABLE &&+ (!routeToMinerU &&+ config.FIRE_PDF_ENABLE &&config.FIRE_PDF_BASE_URL &&base64Content.length < MAX_FILE_SIZE &&Math.random() * 100 < config.FIRE_PDF_PERCENT);