Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: route configurable % of PDF traffic to Fire PDF (#3312)

Abimael Martell34e69082dca5
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 37a21a042..ccc59ac19 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -162,11 +162,11 @@ const configSchema = z.object({
PDF_MU_V2_EXPERIMENT: z.string().optional(),
PDF_MU_V2_EXPERIMENT_PERCENT: z.coerce.number().default(100),
- // Self-Hosted OCR Experiment
- PDF_OCR_EXPERIMENT_ENABLE: z.stringbool().optional(),
- PDF_OCR_EXPERIMENT_PERCENT: z.coerce.number().min(0).max(100).default(10),
- PDF_OCR_BASE_URL: z.string().optional(),
- PDF_OCR_API_KEY: z.string().optional(),
+ // Fire PDF (replaces MinerU for a % of traffic)
+ FIRE_PDF_ENABLE: z.stringbool().optional(),
+ FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),
+ FIRE_PDF_BASE_URL: z.string().optional(),
+ FIRE_PDF_API_KEY: z.string().optional(),
// RunPod
RUNPOD_MU_API_KEY: z.string().optional(),
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 4310555f0..e336ffb6d 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -527,6 +527,7 @@ const baseScrapeOptions = z.strictObject({
__searchPreviewToken: z.string().optional(),
__experimental_omce: z.boolean().prefault(false).optional(),
__experimental_omceDomain: z.string().optional(),
+ __forceFirePDF: z.boolean().prefault(false).optional(),
});
const fire1RefineOpts = {
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 1c936de53..8796e0a12 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -586,6 +586,7 @@ const baseScrapeOptions = z.strictObject({
__experimental_omce: z.boolean().prefault(false).optional(),
__experimental_omceDomain: z.string().optional(),
__experimental_engpicker: z.boolean().prefault(false).optional(),
+ __forceFirePDF: z.boolean().prefault(false).optional(),
});
type ScrapeOptionsBase = z.infer<typeof baseScrapeOptions>;
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index 6b6acd0d7..ec9c7651b 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -4,45 +4,46 @@ import { config } from "../config";
import crypto from "crypto";
import { storage } from "./gcs-jobs";
-const PDF_CACHE_PREFIX = "pdf-cache-v2/";
+type PdfCacheProvider = "runpod" | "firepdf";
+
+const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {
+ runpod: "pdf-cache-v2/",
+ firepdf: "pdf-cache-firepdf/",
+};
-/**
- * Creates a SHA-256 hash of the PDF content to use as a cache key
- * Directly hashes the content without any conversion
- */
function createPdfCacheKey(pdfContent: string | Buffer): string {
return crypto.createHash("sha256").update(pdfContent).digest("hex");
}
-/**
- * Save RunPod markdown results to GCS cache
- */
export async function savePdfResultToCache(
pdfContent: string,
result: { markdown: string; html: string },
+ provider: PdfCacheProvider = "runpod",
): Promise<string | null> {
try {
if (!config.GCS_BUCKET_NAME) {
return null;
}
+ const prefix = PROVIDER_PREFIXES[provider];
const cacheKey = createPdfCacheKey(pdfContent);
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
+ const blob = bucket.file(`${prefix}${cacheKey}.json`);
for (let i = 0; i < 3; i++) {
try {
await blob.save(JSON.stringify(result), {
contentType: "application/json",
metadata: {
- source: "runpod_pdf_conversion",
+ source: `${provider}_pdf_conversion`,
cache_type: "pdf_markdown",
created_at: new Date().toISOString(),
},
});
- logger.info(`Saved PDF RunPod result to GCS cache`, {
+ logger.info(`Saved PDF result to GCS cache`, {
cacheKey,
+ provider,
});
return cacheKey;
@@ -50,47 +51,46 @@ export async function savePdfResultToCache(
if (i === 2) {
throw error;
} else {
- logger.error(
- `Error saving PDF RunPod result to GCS cache, retrying`,
- {
- error,
- cacheKey,
- i,
- },
- );
+ logger.error(`Error saving PDF result to GCS cache, retrying`, {
+ error,
+ cacheKey,
+ provider,
+ i,
+ });
}
}
}
return cacheKey;
} catch (error) {
- logger.error(`Error saving PDF RunPod result to GCS cache`, {
+ logger.error(`Error saving PDF result to GCS cache`, {
error,
+ provider,
});
return null;
}
}
-/**
- * Get cached RunPod markdown results from GCS
- */
export async function getPdfResultFromCache(
pdfContent: string,
+ provider: PdfCacheProvider = "runpod",
): Promise<{ markdown: string; html: string } | null> {
try {
if (!config.GCS_BUCKET_NAME) {
return null;
}
+ const prefix = PROVIDER_PREFIXES[provider];
const cacheKey = createPdfCacheKey(pdfContent);
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
- const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);
+ const blob = bucket.file(`${prefix}${cacheKey}.json`);
const [content] = await blob.download();
const result = JSON.parse(content.toString());
- logger.info(`Retrieved PDF RunPod result from GCS cache`, {
+ logger.info(`Retrieved PDF result from GCS cache`, {
cacheKey,
+ provider,
});
return {
@@ -105,8 +105,9 @@ export async function getPdfResultFromCache(
return null;
}
- logger.error(`Error retrieving PDF RunPod result from GCS cache`, {
+ logger.error(`Error retrieving PDF result from GCS cache`, {
error,
+ provider,
});
return null;
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
new file mode 100644
index 000000000..6cbd92be4
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -0,0 +1,93 @@
+import { Meta } from "../..";
+import { config } from "../../../../config";
+import { robustFetch } from "../../lib/fetch";
+import { z } from "zod";
+import * as marked from "marked";
+import type { PDFProcessorResult } from "./types";
+import {
+ getPdfResultFromCache,
+ savePdfResultToCache,
+} from "../../../../lib/gcs-pdf-cache";
+
+export async function scrapePDFWithFirePDF(
+ meta: Meta,
+ base64Content: string,
+ maxPages?: number,
+ pagesProcessed?: number,
+): Promise<PDFProcessorResult> {
+ const logger = meta.logger.child({ method: "scrapePDF/firePDF" });
+
+ if (!maxPages && !meta.internalOptions.zeroDataRetention) {
+ try {
+ const cached = await getPdfResultFromCache(base64Content, "firepdf");
+ if (cached) {
+ logger.info("Using cached Fire PDF result", {
+ scrapeId: meta.id,
+ });
+ return cached;
+ }
+ } catch (error) {
+ logger.warn("Error checking Fire PDF cache, proceeding", { error });
+ }
+ }
+
+ meta.abort.throwIfAborted();
+
+ const startedAt = Date.now();
+
+ logger.info("Fire PDF started", {
+ scrapeId: meta.id,
+ url: meta.rewrittenUrl ?? meta.url,
+ maxPages,
+ pagesProcessed,
+ });
+
+ const resp = await robustFetch({
+ url: `${config.FIRE_PDF_BASE_URL}/ocr`,
+ method: "POST",
+ headers: config.FIRE_PDF_API_KEY
+ ? { Authorization: `Bearer ${config.FIRE_PDF_API_KEY}` }
+ : undefined,
+ body: {
+ pdf: base64Content,
+ scrape_id: meta.id,
+ ...(maxPages !== undefined && { max_pages: maxPages }),
+ },
+ logger,
+ schema: z.object({
+ markdown: z.string(),
+ failed_pages: z.array(z.number()).nullable(),
+ pages_processed: z.number().optional(),
+ }),
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ });
+
+ const durationMs = Date.now() - startedAt;
+ const pages = resp.pages_processed ?? pagesProcessed;
+
+ logger.info("Fire PDF completed", {
+ scrapeId: meta.id,
+ url: meta.rewrittenUrl ?? meta.url,
+ durationMs,
+ markdownLength: resp.markdown.length,
+ failedPages: resp.failed_pages,
+ pagesProcessed: pages,
+ perPageMs: pages ? Math.round(durationMs / pages) : undefined,
+ });
+
+ const processorResult = {
+ markdown: resp.markdown,
+ html: await marked.parse(resp.markdown, { async: true }),
+ };
+
+ if (!maxPages && !meta.internalOptions.zeroDataRetention) {
+ try {
+ await savePdfResultToCache(base64Content, processorResult, "firepdf");
+ } catch (error) {
+ logger.warn("Error saving Fire PDF result to cache", { error });
+ }
+ }
+
+ return processorResult;
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 397207996..d52cffe96 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -29,6 +29,7 @@ import {
} from "../../../../lib/native-logging";
import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";
import { scrapePDFWithRunPodMU } from "./runpodMU";
+import { scrapePDFWithFirePDF } from "./firePDF";
import { scrapePDFWithParsePDF } from "./pdfParse";
import { captureExceptionWithZdrCheck } from "../../../../services/sentry";
import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";
@@ -155,10 +156,12 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
let shadowIneligibleReason: string | null | undefined;
let shadowPagesNeedingOcr: number[] | undefined;
+ const forceFirePDF =
+ !!meta.options.__forceFirePDF && !!config.FIRE_PDF_BASE_URL;
const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;
const logger = meta.logger.child({ method: "scrapePDF/processPdf" });
- if (!rustEnabled || mode === "ocr") {
+ if (!rustEnabled || mode === "ocr" || forceFirePDF) {
// Legacy / OCR path: detect metadata only, skip Rust extraction.
// When PDF_RUST_EXTRACT_ENABLE is off this is the only path taken,
// matching current prod behaviour (detectPdf → MinerU → pdfParse).
@@ -341,7 +344,46 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
if (!result && !skipOCR) {
const base64Content = (await readFile(tempFilePath)).toString("base64");
+ // Route a percentage of traffic to Fire PDF instead of MinerU
+ const useFirePDF =
+ forceFirePDF ||
+ (config.FIRE_PDF_ENABLE &&
+ config.FIRE_PDF_BASE_URL &&
+ base64Content.length < MAX_FILE_SIZE &&
+ Math.random() * 100 < config.FIRE_PDF_PERCENT);
+
+ if (useFirePDF) {
+ try {
+ result = await scrapePDFWithFirePDF(
+ {
+ ...meta,
+ logger: meta.logger.child({
+ method: "scrapePDF/firePDF",
+ }),
+ },
+ base64Content,
+ maxPages,
+ effectivePageCount,
+ );
+ } catch (error) {
+ if (
+ error instanceof RemoveFeatureError ||
+ error instanceof AbortManagerThrownError
+ ) {
+ throw error;
+ }
+ if (forceFirePDF) {
+ throw error;
+ }
+ meta.logger.warn("Fire PDF failed -- falling back to MinerU", {
+ error,
+ });
+ }
+ }
+
if (
+ !result &&
+ !forceFirePDF &&
base64Content.length < MAX_FILE_SIZE &&
config.RUNPOD_MU_API_KEY &&
config.RUNPOD_MU_POD_ID
@@ -441,8 +483,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
}
}
- // Final fallback to PdfParse.
- if (!result) {
+ // Final fallback to PdfParse (skipped when Fire PDF is forced).
+ if (!result && !forceFirePDF) {
result = await scrapePDFWithParsePDF(
{
...meta,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
index 1934dc171..cd2d889ef 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
@@ -4,7 +4,6 @@ import * as marked from "marked";
import { robustFetch } from "../../lib/fetch";
import { z } from "zod";
import path from "node:path";
-import { runSelfHostedOCRExperiment } from "./selfHostedOCR";
import {
getPdfResultFromCache,
savePdfResultToCache,
@@ -210,15 +209,6 @@ export async function scrapePDFWithRunPodMU(
url: meta.rewrittenUrl ?? meta.url,
pagesProcessed,
});
- if (!meta.internalOptions.zeroDataRetention) {
- runSelfHostedOCRExperiment(
- meta,
- base64Content,
- { markdown: result.markdown, durationMs },
- maxPages,
- pagesProcessed,
- );
- }
}
return processorResult;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts
deleted file mode 100644
index ffb68dd34..000000000
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts
+++ /dev/null
@@ -1,116 +0,0 @@
-import { Meta } from "../..";
-import { config } from "../../../../config";
-import { robustFetch } from "../../lib/fetch";
-import { z } from "zod";
-
-function computeSimilarityMetrics(a: string, b: string): {
- jaccard: number;
- precision: number;
- recall: number;
- f1: number;
-} {
- const normalise = (s: string) =>
- s
- .replace(/[#*_`\[\]()>|~\-]/g, " ")
- .replace(/\s+/g, " ")
- .trim()
- .toLowerCase();
-
- const wordsA = new Set(normalise(a).split(" ").filter(Boolean));
- const wordsB = new Set(normalise(b).split(" ").filter(Boolean));
-
- if (wordsA.size === 0 && wordsB.size === 0) {
- return { jaccard: 1, precision: 1, recall: 1, f1: 1 };
- }
- if (wordsA.size === 0 || wordsB.size === 0) {
- return { jaccard: 0, precision: 0, recall: 0, f1: 0 };
- }
-
- let intersection = 0;
- for (const w of wordsA) {
- if (wordsB.has(w)) intersection++;
- }
-
- const jaccard = intersection / (wordsA.size + wordsB.size - intersection);
- const precision = intersection / wordsA.size; // % of our words in MinerU
- const recall = intersection / wordsB.size; // % of MinerU words in ours
- const f1 = (2 * precision * recall) / (precision + recall) || 0;
-
- return {
- jaccard: Math.round(jaccard * 1000) / 1000,
- precision: Math.round(precision * 1000) / 1000,
- recall: Math.round(recall * 1000) / 1000,
- f1: Math.round(f1 * 1000) / 1000,
- };
-}
-
-export function runSelfHostedOCRExperiment(
- meta: Meta,
- base64Content: string,
- muV1Result: { markdown: string; durationMs: number },
- maxPages?: number,
- pagesProcessed?: number,
-): void {
- if (
- !config.PDF_OCR_EXPERIMENT_ENABLE ||
- !config.PDF_OCR_BASE_URL ||
- Math.random() * 100 >= config.PDF_OCR_EXPERIMENT_PERCENT
- ) {
- return;
- }
-
- (async () => {
- const startedAt = Date.now();
- const logger = meta.logger.child({ method: "scrapePDF/selfHostedOCR" });
- try {
- const resp = await robustFetch({
- url: `${config.PDF_OCR_BASE_URL}/ocr`,
- method: "POST",
- headers: config.PDF_OCR_API_KEY
- ? { Authorization: `Bearer ${config.PDF_OCR_API_KEY}` }
- : undefined,
- body: {
- pdf: base64Content,
- scrape_id: meta.id,
- ...(maxPages !== undefined && { max_pages: maxPages }),
- },
- logger,
- schema: z.object({
- markdown: z.string(),
- failed_pages: z.array(z.number()).nullable(),
- pages_processed: z.number().optional(),
- }),
- mock: meta.mock,
- abort: meta.abort.asSignal(),
- });
- const ocrDurationMs = Date.now() - startedAt;
- const similarity = computeSimilarityMetrics(resp.markdown, muV1Result.markdown);
- const pages = resp.pages_processed ?? pagesProcessed;
- const timeDiffMs = muV1Result.durationMs - ocrDurationMs;
- const speedup = muV1Result.durationMs > 0 && ocrDurationMs > 0
- ? Math.round((muV1Result.durationMs / ocrDurationMs) * 100) / 100
- : undefined;
-
- logger.info("Self-hosted OCR experiment completed", {
- scrapeId: meta.id,
- url: meta.rewrittenUrl ?? meta.url,
- ocrDurationMs,
- muV1DurationMs: muV1Result.durationMs,
- timeDiffMs,
- speedup,
- ocrMarkdownLength: resp.markdown.length,
- muV1MarkdownLength: muV1Result.markdown.length,
- wordSimilarity: similarity.jaccard,
- wordSimilarityPrecision: similarity.precision,
- wordSimilarityRecall: similarity.recall,
- wordSimilarityF1: similarity.f1,
- failedPages: resp.failed_pages,
- pagesProcessed: pages,
- ocrPerPageMs: pages ? Math.round(ocrDurationMs / pages) : undefined,
- muV1PerPageMs: pages ? Math.round(muV1Result.durationMs / pages) : undefined,
- });
- } catch {
- // Non-blocking: instance may be down at any time, silently skip
- }
- })();
-}