Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(pdf): Implement PDF Detection, and Text Extraction (#2860)

Abimael Martell60c93d9585ed
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.toml
index 18833603f..ccbb5c452 100644
--- a/apps/api/native/Cargo.toml
+++ b/apps/api/native/Cargo.toml
@@ -12,7 +12,7 @@ crate-type = ["cdylib"]
chrono = { version = "0.4", features = ["serde"] }
kuchikiki = "0.8.2"
lol_html = "2.6.0"
-lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "6ee84c30f9226d6752dc8e1892298c9d46ac3eb6" }
+pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector", rev = "f37c911" }
maud = "0.27.0"
napi = { version = "3.0.0", features = ["serde-json", "tokio_rt"] }
napi-derive = "3.0.0"
diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rs
index 1678292dd..14a680185 100644
--- a/apps/api/native/src/pdf.rs
+++ b/apps/api/native/src/pdf.rs
@@ -1,36 +1,67 @@
use napi::bindgen_prelude::*;
use napi_derive::napi;
-use serde::Serialize;
+use pdf_inspector::{
+ PdfOptions, PdfType,
+ process_pdf_with_options as rust_process_pdf,
+};
-#[derive(Debug, Clone, Serialize)]
#[napi(object)]
-pub struct PDFMetadata {
- pub num_pages: i32,
- #[serde(skip_serializing_if = "Option::is_none")]
+pub struct PdfProcessResult {
+ pub pdf_type: String,
+ pub markdown: Option<String>,
+ pub page_count: i32,
+ pub processing_time_ms: f64,
+ pub pages_needing_ocr: Vec<i32>,
pub title: Option<String>,
+ pub confidence: f64,
+ pub is_complex: bool,
}
-fn _get_pdf_metadata(path: &str) -> std::result::Result<PDFMetadata, String> {
- let metadata = match lopdf::Document::load_metadata(path) {
- Ok(m) => m,
- Err(e) => {
- return Err(format!("Failed to load PDF metadata: {}", e));
- }
- };
+fn pdf_type_str(t: PdfType) -> &'static str {
+ match t {
+ PdfType::TextBased => "TextBased",
+ PdfType::Scanned => "Scanned",
+ PdfType::ImageBased => "ImageBased",
+ PdfType::Mixed => "Mixed",
+ }
+}
- Ok(PDFMetadata {
- num_pages: metadata.page_count as i32,
- title: metadata.title,
- })
+fn to_napi_result(result: pdf_inspector::PdfProcessResult) -> PdfProcessResult {
+ PdfProcessResult {
+ pdf_type: pdf_type_str(result.pdf_type).to_string(),
+ markdown: result.markdown,
+ page_count: result.page_count as i32,
+ processing_time_ms: result.processing_time_ms as f64,
+ pages_needing_ocr: result.pages_needing_ocr.iter().map(|&p| p as i32).collect(),
+ title: result.title,
+ confidence: result.confidence as f64,
+ is_complex: result.layout.is_complex,
+ }
}
-/// Extract metadata from PDF file.
+/// Process a PDF file: detect type, extract text + markdown if text-based.
#[napi]
-pub fn get_pdf_metadata(path: String) -> Result<PDFMetadata> {
- _get_pdf_metadata(&path).map_err(|e| {
+pub fn process_pdf(path: String) -> Result<PdfProcessResult> {
+ let result = rust_process_pdf(&path, PdfOptions::new()).map_err(|e| {
Error::new(
Status::GenericFailure,
- format!("Failed to get PDF metadata: {e}"),
+ format!("Failed to process PDF: {e}"),
)
- })
+ })?;
+
+ Ok(to_napi_result(result))
+}
+
+/// Fast metadata-only detection: page count, title, type, confidence.
+/// Skips text extraction, markdown generation, and layout analysis.
+#[napi]
+pub fn detect_pdf(path: String) -> Result<PdfProcessResult> {
+ let result = rust_process_pdf(&path, PdfOptions::detect_only()).map_err(|e| {
+ Error::new(
+ Status::GenericFailure,
+ format!("Failed to detect PDF: {e}"),
+ )
+ })?;
+
+ Ok(to_napi_result(result))
}
diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.ts
index a9c908f0d..bedc30d95 100644
--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts
@@ -147,6 +147,80 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {
);
});
+ describe("Mode - object format", () => {
+ it.concurrent(
+ "accepts mode: 'fast' and parses PDF with Rust parser",
+ async () => {
+ const response = await scrape(
+ {
+ url: pdfUrl,
+ parsers: [{ type: "pdf", mode: "fast" }],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.numPages).toBeGreaterThan(0);
+ },
+ scrapeTimeout * 2,
+ );
+
+ it.concurrent(
+ "accepts mode: 'auto' and parses PDF",
+ async () => {
+ const response = await scrape(
+ {
+ url: pdfUrl,
+ parsers: [{ type: "pdf", mode: "auto" }],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.numPages).toBeGreaterThan(0);
+ },
+ scrapeTimeout * 2,
+ );
+
+ it.concurrent(
+ "accepts mode: 'ocr' and parses PDF via OCR",
+ async () => {
+ const response = await scrape(
+ {
+ url: pdfUrl,
+ parsers: [{ type: "pdf", mode: "ocr" }],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.numPages).toBeGreaterThan(0);
+ },
+ scrapeTimeout * 2,
+ );
+
+ it.concurrent(
+ "accepts mode with maxPages combined",
+ async () => {
+ const response = await scrape(
+ {
+ url: pdfUrl,
+ parsers: [{ type: "pdf", mode: "fast", maxPages: 1 }],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.numPages).toBe(1);
+ },
+ scrapeTimeout * 2,
+ );
+ });
+
describe("Default behavior", () => {
it.concurrent(
"parses PDF by default when parsers not specified",
@@ -256,6 +330,42 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {
},
scrapeTimeout,
);
+
+ it.concurrent(
+ "rejects invalid mode in object format",
+ async () => {
+ const raw = await scrapeRaw(
+ {
+ url: pdfUrl,
+ parsers: [{ type: "pdf", mode: "invalid" } as any],
+ },
+ identity,
+ );
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "rejects colon-separated shorthand strings",
+ async () => {
+ const raw = await scrapeRaw(
+ {
+ url: pdfUrl,
+ parsers: ["pdf:fast" as any],
+ },
+ identity,
+ );
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ },
+ scrapeTimeout,
+ );
});
describe("Billing implications", () => {
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 9dd187cba..925d58002 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -155,6 +155,9 @@ const configSchema = z.object({
RUNPOD_MU_API_KEY: z.string().optional(),
RUNPOD_MU_POD_ID: z.string().optional(),
+ // PDF Rust Extraction (pdf-inspector)
+ PDF_RUST_EXTRACT_ENABLE: z.stringbool().optional(),
+
// Webhooks
SELF_HOSTED_WEBHOOK_URL: z.string().optional(),
SELF_HOSTED_WEBHOOK_HMAC_SECRET: z.string().optional(),
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index db0aa83d4..5f461da21 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -413,8 +413,13 @@ export type FormatObject =
| AttributesFormatWithOptions
| { type: "branding" };
+const pdfModeSchema = z.enum(["fast", "auto", "ocr"]);
+
+export type PDFMode = z.infer<typeof pdfModeSchema>;
+
const pdfParserWithOptions = z.strictObject({
type: z.literal("pdf"),
+ mode: pdfModeSchema.optional(),
maxPages: z.int().positive().finite().max(10000).optional(),
});
@@ -449,6 +454,17 @@ export function getPDFMaxPages(parsers?: Parsers): number | undefined {
return undefined;
}
+export function getPDFMode(parsers?: Parsers): PDFMode {
+ if (!parsers) return "auto";
+ for (const parser of parsers) {
+ if (parser === "pdf") return "auto";
+ if (typeof parser === "object" && parser.type === "pdf") {
+ return parser.mode ?? "auto";
+ }
+ }
+ return "auto";
+}
+
function transformIframeSelector(selector: string): string {
return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, match => {
const prefix = match.match(/^[\s,]/)?.[0] || "";
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 2300a6d43..da33785a4 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -16,6 +16,7 @@ import {
PDFAntibotError,
DocumentAntibotError,
PDFInsufficientTimeError,
+ PDFOCRRequiredError,
NoEnginesLeftError,
ZDRViolationError,
PDFPrefetchFailed,
@@ -45,6 +46,7 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_DNS_RESOLUTION_ERROR: DNSResolutionError,
SCRAPE_PDF_INSUFFICIENT_TIME_ERROR: PDFInsufficientTimeError,
SCRAPE_PDF_ANTIBOT_ERROR: PDFAntibotError,
+ SCRAPE_PDF_OCR_REQUIRED: PDFOCRRequiredError,
SCRAPE_DOCUMENT_ANTIBOT_ERROR: DocumentAntibotError,
SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,
SCRAPE_NO_CACHED_DATA: NoCachedDataError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index dc0157753..146900f64 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -14,6 +14,7 @@ export type ErrorCodes =
| "SCRAPE_DNS_RESOLUTION_ERROR"
| "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR"
| "SCRAPE_PDF_ANTIBOT_ERROR"
+ | "SCRAPE_PDF_OCR_REQUIRED"
| "SCRAPE_DOCUMENT_ANTIBOT_ERROR"
| "SCRAPE_UNSUPPORTED_FILE_ERROR"
| "SCRAPE_ACTION_ERROR"
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 7a897cc1b..8f561ff5d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -18,7 +18,7 @@ import { indexMaxReasonableTime, scrapeURLWithIndex } from "./index/index";
import { queryEngpickerVerdict, useIndex } from "../../../services";
import { hasFormatOfType } from "../../../lib/format-utils";
import { getPDFMaxPages } from "../../../controllers/v2/types";
-import { PdfMetadata } from "@mendable/firecrawl-rs";
+import type { PdfMetadata } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
export type Engine =
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 636e7ba54..60c6f101a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -1,294 +1,48 @@
import { Meta } from "../..";
import { config } from "../../../../config";
import { EngineScrapeResult } from "..";
-import * as marked from "marked";
-import { robustFetch } from "../../lib/fetch";
-import { z } from "zod";
import * as Sentry from "@sentry/node";
-import escapeHtml from "escape-html";
-import PdfParse from "pdf-parse";
+import * as marked from "marked";
import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";
import {
PDFAntibotError,
PDFInsufficientTimeError,
+ PDFOCRRequiredError,
PDFPrefetchFailed,
RemoveFeatureError,
EngineUnsuccessfulError,
} from "../../error";
import { readFile, unlink } from "node:fs/promises";
-import path from "node:path";
import type { Response } from "undici";
-import {
- getPdfResultFromCache,
- savePdfResultToCache,
-} from "../../../../lib/gcs-pdf-cache";
import { AbortManagerThrownError } from "../../lib/abortManager";
import {
shouldParsePDF,
getPDFMaxPages,
+ getPDFMode,
} from "../../../../controllers/v2/types";
-import { getPdfMetadata } from "@mendable/firecrawl-rs";
-import { captureExceptionWithZdrCheck } from "../../../../services/sentry";
-
-type PDFProcessorResult = { html: string; markdown?: string };
-
-const MAX_FILE_SIZE = 19 * 1024 * 1024; // 19MB
-const MILLISECONDS_PER_PAGE = 150;
-
-async function scrapePDFWithRunPodMU(
- meta: Meta,
- tempFilePath: string,
- base64Content: string,
- maxPages?: number,
- pagesProcessed?: number,
-): Promise<PDFProcessorResult> {
- meta.logger.debug("Processing PDF document with RunPod MU", {
- tempFilePath,
- });
-
- if (!maxPages) {
- try {
- const cachedResult = await getPdfResultFromCache(base64Content);
- if (cachedResult) {
- meta.logger.info("Using cached RunPod MU result for PDF", {
- tempFilePath,
- });
- return cachedResult;
- }
- } catch (error) {
- meta.logger.warn("Error checking PDF cache, proceeding with RunPod MU", {
- error,
- tempFilePath,
- });
- }
- }
-
- meta.abort.throwIfAborted();
-
- meta.logger.info("Max Pdf pages", {
- tempFilePath,
- maxPages,
- });
-
- if (
- config.PDF_MU_V2_EXPERIMENT === "true" &&
- config.PDF_MU_V2_BASE_URL &&
- Math.random() * 100 < config.PDF_MU_V2_EXPERIMENT_PERCENT
- ) {
- (async () => {
- const pdfParseId = crypto.randomUUID();
- const startedAt = Date.now();
- const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });
- logger.info("MU v2 experiment started", {
- scrapeId: meta.id,
- pdfParseId,
- url: meta.rewrittenUrl ?? meta.url,
- maxPages,
- });
- try {
- const resp = await robustFetch({
- url: config.PDF_MU_V2_BASE_URL ?? "",
- method: "POST",
- headers: config.PDF_MU_V2_API_KEY
- ? { Authorization: `Bearer ${config.PDF_MU_V2_API_KEY}` }
- : undefined,
- body: {
- input: {
- file_content: base64Content,
- filename: path.basename(tempFilePath) + ".pdf",
- timeout: meta.abort.scrapeTimeout(),
- created_at: Date.now(),
- id: pdfParseId,
- ...(maxPages !== undefined && { max_pages: maxPages }),
- },
- },
- logger,
- schema: z.any(),
- mock: meta.mock,
- abort: meta.abort.asSignal(),
- });
- const body: any = resp as any;
- const tokensIn = body?.metadata?.["total-input-tokens"];
- const tokensOut = body?.metadata?.["total-output-tokens"];
- const pages = body?.metadata?.["pdf-total-pages"];
- const durationMs = Date.now() - startedAt;
- logger.info("MU v2 experiment completed", {
- durationMs,
- url: meta.rewrittenUrl ?? meta.url,
- tokensIn,
- tokensOut,
- pages,
- });
- } catch (error) {
- const durationMs = Date.now() - startedAt;
- logger.warn("MU v2 experiment failed", { error, durationMs });
- }
- })();
- }
-
- const muV1StartedAt = Date.now();
- const podStart = await robustFetch({
- url: "https://api.runpod.ai/v2/" + config.RUNPOD_MU_POD_ID + "/runsync",
- method: "POST",
- headers: {
- Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,
- },
- body: {
- input: {
- file_content: base64Content,
- filename: path.basename(tempFilePath) + ".pdf",
- timeout: meta.abort.scrapeTimeout(),
- created_at: Date.now(),
- ...(maxPages !== undefined && { max_pages: maxPages }),
- },
- },
- logger: meta.logger.child({
- method: "scrapePDFWithRunPodMU/runsync/robustFetch",
- }),
- schema: z.object({
- id: z.string(),
- status: z.string(),
- error: z.string().optional(),
- output: z
- .object({
- markdown: z.string().optional(),
- status: z.string().optional(),
- })
- .optional(),
- }),
- mock: meta.mock,
- abort: meta.abort.asSignal(),
- });
-
- let status: string = podStart.status;
- let result: { markdown?: string } | undefined =
- podStart.output?.markdown !== undefined
- ? { markdown: podStart.output.markdown }
- : undefined;
- let lastRunPodError: string | undefined = podStart.error;
-
- if (status === "IN_QUEUE" || status === "IN_PROGRESS") {
- do {
- meta.abort.throwIfAborted();
- await new Promise(resolve => setTimeout(resolve, 2500));
- meta.abort.throwIfAborted();
- const podStatus = await robustFetch({
- url: `https://api.runpod.ai/v2/${config.RUNPOD_MU_POD_ID}/status/${podStart.id}`,
- method: "GET",
- headers: {
- Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,
- },
- logger: meta.logger.child({
- method: "scrapePDFWithRunPodMU/status/robustFetch",
- }),
- schema: z.object({
- status: z.string(),
- error: z.string().optional(),
- output: z
- .object({
- markdown: z.string().optional(),
- status: z.string().optional(),
- })
- .optional(),
- }),
- mock: meta.mock,
- abort: meta.abort.asSignal(),
- });
- status = podStatus.status;
- result =
- podStatus.output?.markdown !== undefined
- ? { markdown: podStatus.output.markdown }
- : undefined;
- if (podStatus.error !== undefined) {
- lastRunPodError = podStatus.error;
- }
- } while (status !== "COMPLETED" && status !== "FAILED");
- }
-
- if (status === "FAILED") {
- const durationMs = Date.now() - muV1StartedAt;
- const errorMessage =
- lastRunPodError ?? "RunPod MU failed to parse PDF (no error message)";
- const err = new Error(`RunPod MU failed to parse PDF: ${errorMessage}`);
- meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {
- durationMs,
- url: meta.rewrittenUrl ?? meta.url,
- mu_id: podStart.id,
- runpodError: lastRunPodError,
- ...(pagesProcessed !== undefined && { pagesProcessed }),
- });
- captureExceptionWithZdrCheck(err, {
- extra: {
- zeroDataRetention: meta.internalOptions.zeroDataRetention ?? false,
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId,
- url: meta.rewrittenUrl ?? meta.url,
- runpodId: podStart.id,
- runpodError: lastRunPodError,
- durationMs,
- },
- });
- throw err;
- }
-
- if (!result?.markdown) {
- const durationMs = Date.now() - muV1StartedAt;
- meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {
- durationMs,
- url: meta.rewrittenUrl ?? meta.url,
- mu_id: podStart.id,
- ...(pagesProcessed !== undefined && { pagesProcessed }),
- });
- throw new Error("RunPod MU returned no result");
- }
-
- const processorResult = {
- markdown: result.markdown,
- html: await marked.parse(result.markdown, { async: true }),
- } satisfies PDFProcessorResult;
-
- if (!meta.internalOptions.zeroDataRetention) {
- try {
- await savePdfResultToCache(base64Content, processorResult);
- } catch (error) {
- meta.logger.warn("Error saving PDF to cache", {
- error,
- tempFilePath,
- });
- }
- }
-
- {
- const durationMs = Date.now() - muV1StartedAt;
- meta.logger.child({ method: "scrapePDF/MUv1" }).info("MU v1 completed", {
- durationMs,
- url: meta.rewrittenUrl ?? meta.url,
- mu_id: podStart.id,
- ...(pagesProcessed !== undefined && { pagesProcessed }),
- });
- }
-
- return processorResult;
-}
-
-async function scrapePDFWithParsePDF(
- meta: Meta,
- tempFilePath: string,
-): Promise<PDFProcessorResult> {
- meta.logger.debug("Processing PDF document with parse-pdf", { tempFilePath });
-
- const result = await PdfParse(await readFile(tempFilePath));
- const escaped = escapeHtml(result.text);
-
- return {
- markdown: escaped,
- html: escaped,
- };
+import type { PDFMode } from "../../../../controllers/v2/types";
+import { processPdf, detectPdf } from "@mendable/firecrawl-rs";
+import { MAX_FILE_SIZE, MILLISECONDS_PER_PAGE } from "./types";
+import type { PDFProcessorResult } from "./types";
+import { scrapePDFWithRunPodMU } from "./runpodMU";
+import { scrapePDFWithParsePDF } from "./pdfParse";
+
+/** Check if the PDF is eligible for Rust extraction, returning a rejection reason or null. */
+function getIneligibleReason(
+ result: ReturnType<typeof processPdf>,
+): string | null {
+ if (result.pdfType !== "TextBased") return `pdfType=${result.pdfType}`;
+ if (result.confidence < 0.95) return `confidence=${result.confidence}`;
+ if (result.isComplex) return "complex layout (tables/columns)";
+ if (!result.markdown?.length)
+ return "empty markdown (unexpected for TextBased)";
+ return null;
}
export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
const shouldParse = shouldParsePDF(meta.options.parsers);
const maxPages = getPDFMaxPages(meta.options.parsers);
+ const mode: PDFMode = getPDFMode(meta.options.parsers);
if (!shouldParse) {
if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {
@@ -375,14 +129,113 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
}
}
- const pdfMetadata = await getPdfMetadata(tempFilePath);
- const effectivePageCount = maxPages
- ? Math.min(pdfMetadata.numPages, maxPages)
- : pdfMetadata.numPages;
+ let result: PDFProcessorResult | null = null;
+ let effectivePageCount: number = 0;
+ let metadataTitle: string | undefined;
+
+ const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;
+ const logger = meta.logger.child({ method: "scrapePDF/processPdf" });
+
+ if (!rustEnabled || mode === "ocr") {
+ // Legacy / OCR path: detect metadata only, skip Rust extraction.
+ // When PDF_RUST_EXTRACT_ENABLE is off this is the only path taken,
+ // matching current prod behaviour (detectPdf → MinerU → pdfParse).
+ try {
+ const startedAt = Date.now();
+ const detection = detectPdf(tempFilePath);
+ const durationMs = Date.now() - startedAt;
+
+ logger.info("detectPdf completed", {
+ durationMs,
+ pdfType: detection.pdfType,
+ pageCount: detection.pageCount,
+ url: meta.rewrittenUrl ?? meta.url,
+ rustEnabled,
+ mode,
+ });
+
+ effectivePageCount = maxPages
+ ? Math.min(detection.pageCount, maxPages)
+ : detection.pageCount;
+ metadataTitle = detection.title ?? undefined;
+ } catch (error) {
+ logger.warn("detectPdf failed", {
+ error,
+ url: meta.rewrittenUrl ?? meta.url,
+ });
+ Sentry.captureException(error);
+ }
+ } else {
+ // Rust extraction enabled (fast / auto modes).
+ try {
+ const startedAt = Date.now();
+ const pdfResult = processPdf(tempFilePath);
+ const durationMs = Date.now() - startedAt;
+
+ logger.info("processPdf completed", {
+ durationMs,
+ pdfType: pdfResult.pdfType,
+ pageCount: pdfResult.pageCount,
+ confidence: pdfResult.confidence,
+ isComplex: pdfResult.isComplex,
+ markdownLength: pdfResult.markdown?.length ?? 0,
+ url: meta.rewrittenUrl ?? meta.url,
+ mode,
+ });
+ effectivePageCount = maxPages
+ ? Math.min(pdfResult.pageCount, maxPages)
+ : pdfResult.pageCount;
+ metadataTitle = pdfResult.title ?? undefined;
+
+ const ineligibleReason = getIneligibleReason(pdfResult);
+ const eligible = !ineligibleReason;
+
+ logger.info("Rust PDF eligibility", {
+ rust_pdf_eligible: eligible,
+ reason: ineligibleReason ?? "eligible",
+ url: meta.rewrittenUrl ?? meta.url,
+ pdfType: pdfResult.pdfType,
+ isComplex: pdfResult.isComplex,
+ pageCount: pdfResult.pageCount,
+ confidence: pdfResult.confidence,
+ mode,
+ });
+
+ // In fast mode, if the PDF requires OCR, fail immediately with a
+ // clear error instead of returning empty content.
+ if (
+ mode === "fast" &&
+ (pdfResult.pdfType === "Scanned" ||
+ pdfResult.pdfType === "ImageBased")
+ ) {
+ throw new PDFOCRRequiredError(pdfResult.pdfType);
+ }
+
+ if (eligible && pdfResult.markdown) {
+ const html = await marked.parse(pdfResult.markdown, { async: true });
+ result = { markdown: pdfResult.markdown, html };
+ }
+ } catch (error) {
+ if (error instanceof PDFOCRRequiredError) {
+ throw error;
+ }
+ logger.warn("processPdf failed, falling back to MU/PdfParse", {
+ error,
+ url: meta.rewrittenUrl ?? meta.url,
+ });
+ Sentry.captureException(error);
+ // effectivePageCount stays 0 — skip time budget check
+ }
+ }
+
+ // Only enforce the per-page time budget when we need MU/fallback.
+ // Rust extraction is fast enough that the constraint doesn't apply.
if (
+ !result &&
+ effectivePageCount > 0 &&
effectivePageCount * MILLISECONDS_PER_PAGE >
- (meta.abort.scrapeTimeout() ?? Infinity)
+ (meta.abort.scrapeTimeout() ?? Infinity)
) {
throw new PDFInsufficientTimeError(
effectivePageCount,
@@ -390,64 +243,65 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
);
}
- let result: PDFProcessorResult | null = null;
-
- const base64Content = (await readFile(tempFilePath)).toString("base64");
-
- // First try RunPod MU if conditions are met
- if (
- base64Content.length < MAX_FILE_SIZE &&
- config.RUNPOD_MU_API_KEY &&
- config.RUNPOD_MU_POD_ID
- ) {
- const muV1StartedAt = Date.now();
- try {
- result = await scrapePDFWithRunPodMU(
- {
- ...meta,
- logger: meta.logger.child({
- method: "scrapePDF/scrapePDFWithRunPodMU",
- }),
- },
- tempFilePath,
- base64Content,
- maxPages,
- effectivePageCount,
- );
- const muV1DurationMs = Date.now() - muV1StartedAt;
- meta.logger
- .child({ method: "scrapePDF/MUv1Experiment" })
- .info("MU v1 completed", {
- durationMs: muV1DurationMs,
- url: meta.rewrittenUrl ?? meta.url,
- pages: effectivePageCount,
- success: true,
- });
- } catch (error) {
- if (
- error instanceof RemoveFeatureError ||
- error instanceof AbortManagerThrownError
- ) {
- throw error;
+ // OCR / MU fallback.
+ // Skipped only when Rust extraction is enabled AND mode is "fast".
+ const skipOCR = rustEnabled && mode === "fast";
+ if (!result && !skipOCR) {
+ const base64Content = (await readFile(tempFilePath)).toString("base64");
+
+ if (
+ base64Content.length < MAX_FILE_SIZE &&
+ config.RUNPOD_MU_API_KEY &&
+ config.RUNPOD_MU_POD_ID
+ ) {
+ const muV1StartedAt = Date.now();
+ try {
+ result = await scrapePDFWithRunPodMU(
+ {
+ ...meta,
+ logger: meta.logger.child({
+ method: "scrapePDF/scrapePDFWithRunPodMU",
+ }),
+ },
+ tempFilePath,
+ base64Content,
+ maxPages,
+ );
+ const muV1DurationMs = Date.now() - muV1StartedAt;
+ meta.logger
+ .child({ method: "scrapePDF/MUv1Experiment" })
+ .info("MU v1 completed", {
+ durationMs: muV1DurationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ pages: effectivePageCount,
+ success: true,
+ });
+ } catch (error) {
+ if (
+ error instanceof RemoveFeatureError ||
+ error instanceof AbortManagerThrownError
+ ) {
+ throw error;
+ }
+ meta.logger.warn(
+ "RunPod MU failed to parse PDF (could be due to timeout) -- falling back to parse-pdf",
+ { error },
+ );
+ Sentry.captureException(error);
+ const muV1DurationMs = Date.now() - muV1StartedAt;
+ meta.logger
+ .child({ method: "scrapePDF/MUv1Experiment" })
+ .info("MU v1 failed", {
+ durationMs: muV1DurationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ pages: effectivePageCount,
+ success: false,
+ });
}
- meta.logger.warn(
- "RunPod MU failed to parse PDF (could be due to timeout) -- falling back to parse-pdf",
- { error },
- );
- Sentry.captureException(error);
- const muV1DurationMs = Date.now() - muV1StartedAt;
- meta.logger
- .child({ method: "scrapePDF/MUv1Experiment" })
- .info("MU v1 failed", {
- durationMs: muV1DurationMs,
- url: meta.rewrittenUrl ?? meta.url,
- pages: effectivePageCount,
- success: false,
- });
}
}
- // If RunPod MU failed or wasn't attempted, use PdfParse
+ // Final fallback to PdfParse.
if (!result) {
result = await scrapePDFWithParsePDF(
{
@@ -466,10 +320,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
html: result?.html ?? "",
markdown: result?.markdown ?? "",
pdfMetadata: {
- // Rust parser gets the metadata incorrectly, so we overwrite the page count here with the effective page count
- // TODO: fix this later
numPages: effectivePageCount,
- title: pdfMetadata.title,
+ title: metadataTitle,
},
proxyUsed: "basic",
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/pdfParse.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/pdfParse.ts
new file mode 100644
index 000000000..9cf8a1207
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/pdfParse.ts
@@ -0,0 +1,20 @@
+import { Meta } from "../..";
+import escapeHtml from "escape-html";
+import PdfParse from "pdf-parse";
+import { readFile } from "node:fs/promises";
+import type { PDFProcessorResult } from "./types";
+
+export async function scrapePDFWithParsePDF(
+ meta: Meta,
+ tempFilePath: string,
+): Promise<PDFProcessorResult> {
+ meta.logger.debug("Processing PDF document with parse-pdf", { tempFilePath });
+
+ const result = await PdfParse(await readFile(tempFilePath));
+ const escaped = escapeHtml(result.text);
+
+ return {
+ markdown: escaped,
+ html: escaped,
+ };
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
new file mode 100644
index 000000000..ae9e04e19
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
@@ -0,0 +1,211 @@
+import { Meta } from "../..";
+import { config } from "../../../../config";
+import * as marked from "marked";
+import { robustFetch } from "../../lib/fetch";
+import { z } from "zod";
+import path from "node:path";
+import {
+ getPdfResultFromCache,
+ savePdfResultToCache,
+} from "../../../../lib/gcs-pdf-cache";
+import type { PDFProcessorResult } from "./types";
+
+export async function scrapePDFWithRunPodMU(
+ meta: Meta,
+ tempFilePath: string,
+ base64Content: string,
+ maxPages?: number,
+): Promise<PDFProcessorResult> {
+ meta.logger.debug("Processing PDF document with RunPod MU", {
+ tempFilePath,
+ });
+
+ if (!maxPages) {
+ try {
+ const cachedResult = await getPdfResultFromCache(base64Content);
+ if (cachedResult) {
+ meta.logger.info("Using cached RunPod MU result for PDF", {
+ tempFilePath,
+ });
+ return cachedResult;
+ }
+ } catch (error) {
+ meta.logger.warn("Error checking PDF cache, proceeding with RunPod MU", {
+ error,
+ tempFilePath,
+ });
+ }
+ }
+
+ meta.abort.throwIfAborted();
+
+ meta.logger.info("Max Pdf pages", {
+ tempFilePath,
+ maxPages,
+ });
+
+ if (
+ config.PDF_MU_V2_EXPERIMENT === "true" &&
+ config.PDF_MU_V2_BASE_URL &&
+ Math.random() * 100 < config.PDF_MU_V2_EXPERIMENT_PERCENT
+ ) {
+ (async () => {
+ const pdfParseId = crypto.randomUUID();
+ const startedAt = Date.now();
+ const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });
+ logger.info("MU v2 experiment started", {
+ scrapeId: meta.id,
+ pdfParseId,
+ url: meta.rewrittenUrl ?? meta.url,
+ maxPages,
+ });
+ try {
+ const resp = await robustFetch({
+ url: config.PDF_MU_V2_BASE_URL ?? "",
+ method: "POST",
+ headers: config.PDF_MU_V2_API_KEY
+ ? { Authorization: `Bearer ${config.PDF_MU_V2_API_KEY}` }
+ : undefined,
+ body: {
+ input: {
+ file_content: base64Content,
+ filename: path.basename(tempFilePath) + ".pdf",
+ timeout: meta.abort.scrapeTimeout(),
+ created_at: Date.now(),
+ id: pdfParseId,
+ ...(maxPages !== undefined && { max_pages: maxPages }),
+ },
+ },
+ logger,
+ schema: z.any(),
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ });
+ const body: any = resp as any;
+ const tokensIn = body?.metadata?.["total-input-tokens"];
+ const tokensOut = body?.metadata?.["total-output-tokens"];
+ const pages = body?.metadata?.["pdf-total-pages"];
+ const durationMs = Date.now() - startedAt;
+ logger.info("MU v2 experiment completed", {
+ durationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ tokensIn,
+ tokensOut,
+ pages,
+ });
+ } catch (error) {
+ const durationMs = Date.now() - startedAt;
+ logger.warn("MU v2 experiment failed", { error, durationMs });
+ }
+ })();
+ }
+
+ const muV1StartedAt = Date.now();
+ const podStart = await robustFetch({
+ url: "https://api.runpod.ai/v2/" + config.RUNPOD_MU_POD_ID + "/runsync",
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,
+ },
+ body: {
+ input: {
+ file_content: base64Content,
+ filename: path.basename(tempFilePath) + ".pdf",
+ timeout: meta.abort.scrapeTimeout(),
+ created_at: Date.now(),
+ ...(maxPages !== undefined && { max_pages: maxPages }),
+ },
+ },
+ logger: meta.logger.child({
+ method: "scrapePDFWithRunPodMU/runsync/robustFetch",
+ }),
+ schema: z.object({
+ id: z.string(),
+ status: z.string(),
+ output: z
+ .object({
+ markdown: z.string(),
+ })
+ .optional(),
+ }),
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ });
+
+ let status: string = podStart.status;
+ let result: { markdown: string } | undefined = podStart.output;
+
+ if (status === "IN_QUEUE" || status === "IN_PROGRESS") {
+ do {
+ meta.abort.throwIfAborted();
+ await new Promise(resolve => setTimeout(resolve, 2500));
+ meta.abort.throwIfAborted();
+ const podStatus = await robustFetch({
+ url: `https://api.runpod.ai/v2/${config.RUNPOD_MU_POD_ID}/status/${podStart.id}`,
+ method: "GET",
+ headers: {
+ Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,
+ },
+ logger: meta.logger.child({
+ method: "scrapePDFWithRunPodMU/status/robustFetch",
+ }),
+ schema: z.object({
+ status: z.string(),
+ output: z
+ .object({
+ markdown: z.string(),
+ })
+ .optional(),
+ }),
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ });
+ status = podStatus.status;
+ result = podStatus.output;
+ } while (status !== "COMPLETED" && status !== "FAILED");
+ }
+
+ if (status === "FAILED") {
+ const durationMs = Date.now() - muV1StartedAt;
+ meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {
+ durationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ });
+ throw new Error("RunPod MU failed to parse PDF");
+ }
+
+ if (!result) {
+ const durationMs = Date.now() - muV1StartedAt;
+ meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {
+ durationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ });
+ throw new Error("RunPod MU returned no result");
+ }
+
+ const processorResult = {
+ markdown: result.markdown,
+ html: await marked.parse(result.markdown, { async: true }),
+ };
+
+ if (!meta.internalOptions.zeroDataRetention) {
+ try {
+ await savePdfResultToCache(base64Content, processorResult);
+ } catch (error) {
+ meta.logger.warn("Error saving PDF to cache", {
+ error,
+ tempFilePath,
+ });
+ }
+ }
+
+ {
+ const durationMs = Date.now() - muV1StartedAt;
+ meta.logger.child({ method: "scrapePDF/MUv1" }).info("MU v1 completed", {
+ durationMs,
+ url: meta.rewrittenUrl ?? meta.url,
+ });
+ }
+
+ return processorResult;
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
new file mode 100644
index 000000000..d9311cbaf
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
@@ -0,0 +1,6 @@
+export type PDFProcessorResult = { html: string; markdown?: string };
+
+export type PdfMetadata = { numPages: number; title?: string };
+
+export const MAX_FILE_SIZE = 19 * 1024 * 1024; // 19MB
+export const MILLISECONDS_PER_PAGE = 150;
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 24e5662e6..4fcdb1365 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -354,6 +354,31 @@ export class ZDRViolationError extends TransportableError {
}
}
+export class PDFOCRRequiredError extends TransportableError {
+ constructor(public pdfType: string) {
+ super(
+ "SCRAPE_PDF_OCR_REQUIRED",
+ `This PDF is ${pdfType === "Scanned" ? "scanned" : "image-based"} and requires OCR for text extraction, but the requested PDF mode is "fast" which only supports text-based PDFs. To process this PDF, use mode "auto" (which falls back to OCR automatically) or mode "ocr" (which forces OCR processing). Example: parsers: [{ type: "pdf", mode: "auto" }]`,
+ );
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ pdfType: this.pdfType,
+ };
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new PDFOCRRequiredError(data.pdfType);
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class PDFPrefetchFailed extends TransportableError {
constructor() {
const message = isSelfHosted()
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index e5ddd7842..9102b9b1e 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -36,6 +36,7 @@ import {
UnsupportedFileError,
SSLError,
PDFInsufficientTimeError,
+ PDFOCRRequiredError,
IndexMissError,
NoCachedDataError,
DNSResolutionError,
@@ -646,6 +647,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof ActionError ||
error.error instanceof UnsupportedFileError ||
error.error instanceof PDFAntibotError ||
+ error.error instanceof PDFOCRRequiredError ||
error.error instanceof DocumentAntibotError ||
error.error instanceof PDFInsufficientTimeError ||
error.error instanceof ProxySelectionError ||
@@ -1197,6 +1199,14 @@ export async function scrapeURL(
meta.logger.warn("scrapeURL: Insufficient time to process PDF", {
error,
});
+ } else if (error instanceof PDFOCRRequiredError) {
+ errorType = "PDFOCRRequiredError";
+ meta.logger.warn(
+ "scrapeURL: PDF requires OCR but fast mode was requested",
+ {
+ error,
+ },
+ );
} else if (error instanceof PDFPrefetchFailed) {
errorType = "PDFPrefetchFailed";
meta.logger.warn(
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index a49c73e9d..4d289bf09 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -5,7 +5,11 @@ import { htmlTransform } from "../lib/removeUnwantedElements";
import { extractLinks } from "../lib/extractLinks";
import { extractImages } from "../lib/extractImages";
import { extractMetadata } from "../lib/extractMetadata";
-import { performLLMExtract, performSummary, performCleanContent } from "./llmExtract";
+import {
+ performLLMExtract,
+ performSummary,
+ performCleanContent,
+} from "./llmExtract";
import { uploadScreenshot } from "./uploadScreenshot";
import { removeBase64Images } from "./removeBase64Images";
import { performAgent } from "./agent";
@@ -85,7 +89,13 @@ async function deriveMarkdownFromHTML(
const hasJson = hasFormatOfType(meta.options.formats, "json");
const hasSummary = hasFormatOfType(meta.options.formats, "summary");
- if (!hasMarkdown && !hasChangeTracking && !hasJson && !hasSummary && !meta.options.onlyCleanContent) {
+ if (
+ !hasMarkdown &&
+ !hasChangeTracking &&
+ !hasJson &&
+ !hasSummary &&
+ !meta.options.onlyCleanContent
+ ) {
return document;
}
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 16630c3a1..9c957cb67 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -7,7 +7,7 @@ import { redisEvictConnection } from "./redis";
import type { Logger } from "winston";
import psl from "psl";
import { MapDocument } from "../controllers/v2/types";
-import { PdfMetadata } from "@mendable/firecrawl-rs";
+import type { PdfMetadata } from "../scraper/scrapeURL/engines/pdf/types";
import { storage } from "../lib/gcs-jobs";
import { withSpan, setSpanAttributes } from "../lib/otel-tracer";
import { config } from "../config";