Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: route configurable % of PDF traffic to Fire PDF (#3312)
Abimael Martell34e69082dca5
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 37a21a042..ccc59ac19 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -162,11 +162,11 @@ const configSchema = z.object({PDF_MU_V2_EXPERIMENT: z.string().optional(),PDF_MU_V2_EXPERIMENT_PERCENT: z.coerce.number().default(100),- // Self-Hosted OCR Experiment- PDF_OCR_EXPERIMENT_ENABLE: z.stringbool().optional(),- PDF_OCR_EXPERIMENT_PERCENT: z.coerce.number().min(0).max(100).default(10),- PDF_OCR_BASE_URL: z.string().optional(),- PDF_OCR_API_KEY: z.string().optional(),+ // Fire PDF (replaces MinerU for a % of traffic)+ FIRE_PDF_ENABLE: z.stringbool().optional(),+ FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),+ FIRE_PDF_BASE_URL: z.string().optional(),+ FIRE_PDF_API_KEY: z.string().optional(),// RunPodRUNPOD_MU_API_KEY: z.string().optional(),diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 4310555f0..e336ffb6d 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -527,6 +527,7 @@ const baseScrapeOptions = z.strictObject({__searchPreviewToken: z.string().optional(),__experimental_omce: z.boolean().prefault(false).optional(),__experimental_omceDomain: z.string().optional(),+ __forceFirePDF: z.boolean().prefault(false).optional(),});const fire1RefineOpts = {diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 1c936de53..8796e0a12 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -586,6 +586,7 @@ const baseScrapeOptions = z.strictObject({__experimental_omce: z.boolean().prefault(false).optional(),__experimental_omceDomain: z.string().optional(),__experimental_engpicker: z.boolean().prefault(false).optional(),+ __forceFirePDF: z.boolean().prefault(false).optional(),});type ScrapeOptionsBase = z.infer<typeof baseScrapeOptions>;diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex 6b6acd0d7..ec9c7651b 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -4,45 +4,46 @@ import { config } from "../config";import crypto from "crypto";import { storage } from "./gcs-jobs";-const PDF_CACHE_PREFIX = "pdf-cache-v2/";+type PdfCacheProvider = "runpod" | "firepdf";++const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {+ runpod: "pdf-cache-v2/",+ firepdf: "pdf-cache-firepdf/",+};-/**- * Creates a SHA-256 hash of the PDF content to use as a cache key- * Directly hashes the content without any conversion- */function createPdfCacheKey(pdfContent: string | Buffer): string {return crypto.createHash("sha256").update(pdfContent).digest("hex");}-/**- * Save RunPod markdown results to GCS cache- */export async function savePdfResultToCache(pdfContent: string,result: { markdown: string; html: string },+ provider: PdfCacheProvider = "runpod",): Promise<string | null> {try {if (!config.GCS_BUCKET_NAME) {return null;}+ const prefix = PROVIDER_PREFIXES[provider];const cacheKey = createPdfCacheKey(pdfContent);const bucket = storage.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);+ const blob = bucket.file(`${prefix}${cacheKey}.json`);for (let i = 0; i < 3; i++) {try {await blob.save(JSON.stringify(result), {contentType: "application/json",metadata: {- source: "runpod_pdf_conversion",+ source: `${provider}_pdf_conversion`,cache_type: "pdf_markdown",created_at: new Date().toISOString(),},});- logger.info(`Saved PDF RunPod result to GCS cache`, {+ logger.info(`Saved PDF result to GCS cache`, {cacheKey,+ provider,});return cacheKey;@@ -50,47 +51,46 @@ export async function savePdfResultToCache(if (i === 2) {throw error;} else {- logger.error(- `Error saving PDF RunPod result to GCS cache, retrying`,- {- error,- cacheKey,- i,- },- );+ logger.error(`Error saving PDF result to GCS cache, retrying`, {+ error,+ cacheKey,+ provider,+ i,+ });}}}return cacheKey;} catch (error) {- logger.error(`Error saving PDF RunPod result to GCS cache`, {+ logger.error(`Error saving PDF result to GCS cache`, {error,+ provider,});return null;}}-/**- * Get cached RunPod markdown results from GCS- */export async function getPdfResultFromCache(pdfContent: string,+ provider: PdfCacheProvider = "runpod",): Promise<{ markdown: string; html: string } | null> {try {if (!config.GCS_BUCKET_NAME) {return null;}+ const prefix = PROVIDER_PREFIXES[provider];const cacheKey = createPdfCacheKey(pdfContent);const bucket = storage.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(`${PDF_CACHE_PREFIX}${cacheKey}.json`);+ const blob = bucket.file(`${prefix}${cacheKey}.json`);const [content] = await blob.download();const result = JSON.parse(content.toString());- logger.info(`Retrieved PDF RunPod result from GCS cache`, {+ logger.info(`Retrieved PDF result from GCS cache`, {cacheKey,+ provider,});return {@@ -105,8 +105,9 @@ export async function getPdfResultFromCache(return null;}- logger.error(`Error retrieving PDF RunPod result from GCS cache`, {+ logger.error(`Error retrieving PDF result from GCS cache`, {error,+ provider,});return null;}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsnew file mode 100644index 000000000..6cbd92be4--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -0,0 +1,93 @@+import { Meta } from "../..";+import { config } from "../../../../config";+import { robustFetch } from "../../lib/fetch";+import { z } from "zod";+import * as marked from "marked";+import type { PDFProcessorResult } from "./types";+import {+ getPdfResultFromCache,+ savePdfResultToCache,+} from "../../../../lib/gcs-pdf-cache";++export async function scrapePDFWithFirePDF(+ meta: Meta,+ base64Content: string,+ maxPages?: number,+ pagesProcessed?: number,+): Promise<PDFProcessorResult> {+ const logger = meta.logger.child({ method: "scrapePDF/firePDF" });++ if (!maxPages && !meta.internalOptions.zeroDataRetention) {+ try {+ const cached = await getPdfResultFromCache(base64Content, "firepdf");+ if (cached) {+ logger.info("Using cached Fire PDF result", {+ scrapeId: meta.id,+ });+ return cached;+ }+ } catch (error) {+ logger.warn("Error checking Fire PDF cache, proceeding", { error });+ }+ }++ meta.abort.throwIfAborted();++ const startedAt = Date.now();++ logger.info("Fire PDF started", {+ scrapeId: meta.id,+ url: meta.rewrittenUrl ?? meta.url,+ maxPages,+ pagesProcessed,+ });++ const resp = await robustFetch({+ url: `${config.FIRE_PDF_BASE_URL}/ocr`,+ method: "POST",+ headers: config.FIRE_PDF_API_KEY+ ? { Authorization: `Bearer ${config.FIRE_PDF_API_KEY}` }+ : undefined,+ body: {+ pdf: base64Content,+ scrape_id: meta.id,+ ...(maxPages !== undefined && { max_pages: maxPages }),+ },+ logger,+ schema: z.object({+ markdown: z.string(),+ failed_pages: z.array(z.number()).nullable(),+ pages_processed: z.number().optional(),+ }),+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ });++ const durationMs = Date.now() - startedAt;+ const pages = resp.pages_processed ?? pagesProcessed;++ logger.info("Fire PDF completed", {+ scrapeId: meta.id,+ url: meta.rewrittenUrl ?? meta.url,+ durationMs,+ markdownLength: resp.markdown.length,+ failedPages: resp.failed_pages,+ pagesProcessed: pages,+ perPageMs: pages ? Math.round(durationMs / pages) : undefined,+ });++ const processorResult = {+ markdown: resp.markdown,+ html: await marked.parse(resp.markdown, { async: true }),+ };++ if (!maxPages && !meta.internalOptions.zeroDataRetention) {+ try {+ await savePdfResultToCache(base64Content, processorResult, "firepdf");+ } catch (error) {+ logger.warn("Error saving Fire PDF result to cache", { error });+ }+ }++ return processorResult;+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 397207996..d52cffe96 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -29,6 +29,7 @@ import {} from "../../../../lib/native-logging";import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";import { scrapePDFWithRunPodMU } from "./runpodMU";+import { scrapePDFWithFirePDF } from "./firePDF";import { scrapePDFWithParsePDF } from "./pdfParse";import { captureExceptionWithZdrCheck } from "../../../../services/sentry";import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";@@ -155,10 +156,12 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {let shadowIneligibleReason: string | null | undefined;let shadowPagesNeedingOcr: number[] | undefined;+ const forceFirePDF =+ !!meta.options.__forceFirePDF && !!config.FIRE_PDF_BASE_URL;const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;const logger = meta.logger.child({ method: "scrapePDF/processPdf" });- if (!rustEnabled || mode === "ocr") {+ if (!rustEnabled || mode === "ocr" || forceFirePDF) {// Legacy / OCR path: detect metadata only, skip Rust extraction.// When PDF_RUST_EXTRACT_ENABLE is off this is the only path taken,// matching current prod behaviour (detectPdf → MinerU → pdfParse).@@ -341,7 +344,46 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (!result && !skipOCR) {const base64Content = (await readFile(tempFilePath)).toString("base64");+ // Route a percentage of traffic to Fire PDF instead of MinerU+ const useFirePDF =+ forceFirePDF ||+ (config.FIRE_PDF_ENABLE &&+ config.FIRE_PDF_BASE_URL &&+ base64Content.length < MAX_FILE_SIZE &&+ Math.random() * 100 < config.FIRE_PDF_PERCENT);++ if (useFirePDF) {+ try {+ result = await scrapePDFWithFirePDF(+ {+ ...meta,+ logger: meta.logger.child({+ method: "scrapePDF/firePDF",+ }),+ },+ base64Content,+ maxPages,+ effectivePageCount,+ );+ } catch (error) {+ if (+ error instanceof RemoveFeatureError ||+ error instanceof AbortManagerThrownError+ ) {+ throw error;+ }+ if (forceFirePDF) {+ throw error;+ }+ meta.logger.warn("Fire PDF failed -- falling back to MinerU", {+ error,+ });+ }+ }+if (+ !result &&+ !forceFirePDF &&base64Content.length < MAX_FILE_SIZE &&config.RUNPOD_MU_API_KEY &&config.RUNPOD_MU_POD_ID@@ -441,8 +483,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {}}- // Final fallback to PdfParse.- if (!result) {+ // Final fallback to PdfParse (skipped when Fire PDF is forced).+ if (!result && !forceFirePDF) {result = await scrapePDFWithParsePDF({...meta,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.tsindex 1934dc171..cd2d889ef 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts@@ -4,7 +4,6 @@ import * as marked from "marked";import { robustFetch } from "../../lib/fetch";import { z } from "zod";import path from "node:path";-import { runSelfHostedOCRExperiment } from "./selfHostedOCR";import {getPdfResultFromCache,savePdfResultToCache,@@ -210,15 +209,6 @@ export async function scrapePDFWithRunPodMU(url: meta.rewrittenUrl ?? meta.url,pagesProcessed,});- if (!meta.internalOptions.zeroDataRetention) {- runSelfHostedOCRExperiment(- meta,- base64Content,- { markdown: result.markdown, durationMs },- maxPages,- pagesProcessed,- );- }}return processorResult;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.tsdeleted file mode 100644index ffb68dd34..000000000--- a/apps/api/src/scraper/scrapeURL/engines/pdf/selfHostedOCR.ts+++ /dev/null@@ -1,116 +0,0 @@-import { Meta } from "../..";-import { config } from "../../../../config";-import { robustFetch } from "../../lib/fetch";-import { z } from "zod";--function computeSimilarityMetrics(a: string, b: string): {- jaccard: number;- precision: number;- recall: number;- f1: number;-} {- const normalise = (s: string) =>- s- .replace(/[#*_`\[\]()>|~\-]/g, " ")- .replace(/\s+/g, " ")- .trim()- .toLowerCase();-- const wordsA = new Set(normalise(a).split(" ").filter(Boolean));- const wordsB = new Set(normalise(b).split(" ").filter(Boolean));-- if (wordsA.size === 0 && wordsB.size === 0) {- return { jaccard: 1, precision: 1, recall: 1, f1: 1 };- }- if (wordsA.size === 0 || wordsB.size === 0) {- return { jaccard: 0, precision: 0, recall: 0, f1: 0 };- }-- let intersection = 0;- for (const w of wordsA) {- if (wordsB.has(w)) intersection++;- }-- const jaccard = intersection / (wordsA.size + wordsB.size - intersection);- const precision = intersection / wordsA.size; // % of our words in MinerU- const recall = intersection / wordsB.size; // % of MinerU words in ours- const f1 = (2 * precision * recall) / (precision + recall) || 0;-- return {- jaccard: Math.round(jaccard * 1000) / 1000,- precision: Math.round(precision * 1000) / 1000,- recall: Math.round(recall * 1000) / 1000,- f1: Math.round(f1 * 1000) / 1000,- };-}--export function runSelfHostedOCRExperiment(- meta: Meta,- base64Content: string,- muV1Result: { markdown: string; durationMs: number },- maxPages?: number,- pagesProcessed?: number,-): void {- if (- !config.PDF_OCR_EXPERIMENT_ENABLE ||- !config.PDF_OCR_BASE_URL ||- Math.random() * 100 >= config.PDF_OCR_EXPERIMENT_PERCENT- ) {- return;- }-- (async () => {- const startedAt = Date.now();- const logger = meta.logger.child({ method: "scrapePDF/selfHostedOCR" });- try {- const resp = await robustFetch({- url: `${config.PDF_OCR_BASE_URL}/ocr`,- method: "POST",- headers: config.PDF_OCR_API_KEY- ? { Authorization: `Bearer ${config.PDF_OCR_API_KEY}` }- : undefined,- body: {- pdf: base64Content,- scrape_id: meta.id,- ...(maxPages !== undefined && { max_pages: maxPages }),- },- logger,- schema: z.object({- markdown: z.string(),- failed_pages: z.array(z.number()).nullable(),- pages_processed: z.number().optional(),- }),- mock: meta.mock,- abort: meta.abort.asSignal(),- });- const ocrDurationMs = Date.now() - startedAt;- const similarity = computeSimilarityMetrics(resp.markdown, muV1Result.markdown);- const pages = resp.pages_processed ?? pagesProcessed;- const timeDiffMs = muV1Result.durationMs - ocrDurationMs;- const speedup = muV1Result.durationMs > 0 && ocrDurationMs > 0- ? Math.round((muV1Result.durationMs / ocrDurationMs) * 100) / 100- : undefined;-- logger.info("Self-hosted OCR experiment completed", {- scrapeId: meta.id,- url: meta.rewrittenUrl ?? meta.url,- ocrDurationMs,- muV1DurationMs: muV1Result.durationMs,- timeDiffMs,- speedup,- ocrMarkdownLength: resp.markdown.length,- muV1MarkdownLength: muV1Result.markdown.length,- wordSimilarity: similarity.jaccard,- wordSimilarityPrecision: similarity.precision,- wordSimilarityRecall: similarity.recall,- wordSimilarityF1: similarity.f1,- failedPages: resp.failed_pages,- pagesProcessed: pages,- ocrPerPageMs: pages ? Math.round(ocrDurationMs / pages) : undefined,- muV1PerPageMs: pages ? Math.round(muV1Result.durationMs / pages) : undefined,- });- } catch {- // Non-blocking: instance may be down at any time, silently skip- }- })();-}