Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(pdf): add MINERU_PERCENT config for direct MinerU routing (#3354)

Abimael Martellff871d5fb3c4
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index ccc59ac19..7caea74cd 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -162,6 +162,9 @@ const configSchema = z.object({
PDF_MU_V2_EXPERIMENT: z.string().optional(),
PDF_MU_V2_EXPERIMENT_PERCENT: z.coerce.number().default(100),
+ // MinerU direct routing (bypass Rust extraction for a % of traffic)
+ MINERU_PERCENT: z.coerce.number().min(0).max(100).default(0),
+
// Fire PDF (replaces MinerU for a % of traffic)
FIRE_PDF_ENABLE: z.stringbool().optional(),
FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 8640924d3..53c96fa73 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -161,7 +161,21 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;
const logger = meta.logger.child({ method: "scrapePDF/processPdf" });
- if (!rustEnabled || mode === "ocr" || forceFirePDF) {
+ // Route a percentage of traffic directly to MinerU, bypassing Rust extraction.
+ // Forced Fire PDF takes precedence β€” don't divert those requests.
+ const routeToMinerU =
+ !forceFirePDF &&
+ config.MINERU_PERCENT > 0 &&
+ Math.random() * 100 < config.MINERU_PERCENT;
+
+ if (routeToMinerU) {
+ logger.info("Routing to MinerU via MINERU_PERCENT", {
+ mineruPercent: config.MINERU_PERCENT,
+ url: meta.rewrittenUrl ?? meta.url,
+ });
+ }
+
+ if (!rustEnabled || mode === "ocr" || forceFirePDF || routeToMinerU) {
// Legacy / OCR path: detect metadata only, skip Rust extraction.
// When PDF_RUST_EXTRACT_ENABLE is off this is the only path taken,
// matching current prod behaviour (detectPdf β†’ MinerU β†’ pdfParse).
@@ -343,15 +357,19 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
}
// OCR / MU fallback.
- // Skipped only when Rust extraction is enabled AND mode is "fast".
- const skipOCR = rustEnabled && mode === "fast";
+ // Skipped only when Rust extraction is enabled AND mode is "fast",
+ // unless we explicitly routed to MinerU via MINERU_PERCENT.
+ const skipOCR = rustEnabled && mode === "fast" && !routeToMinerU;
if (!result && !skipOCR) {
const base64Content = (await readFile(tempFilePath)).toString("base64");
- // Route a percentage of traffic to Fire PDF instead of MinerU
+ // Route a percentage of traffic to Fire PDF instead of MinerU.
+ // forceFirePDF always wins; skip percentage-based Fire PDF when
+ // we explicitly routed to MinerU via MINERU_PERCENT.
const useFirePDF =
forceFirePDF ||
- (config.FIRE_PDF_ENABLE &&
+ (!routeToMinerU &&
+ config.FIRE_PDF_ENABLE &&
config.FIRE_PDF_BASE_URL &&
base64Content.length < MAX_FILE_SIZE &&
Math.random() * 100 < config.FIRE_PDF_PERCENT);