Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf): Implement PDF Detection, and Text Extraction (#2860)
Abimael Martell60c93d9585ed
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex 18833603f..ccbb5c452 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -12,7 +12,7 @@ crate-type = ["cdylib"]chrono = { version = "0.4", features = ["serde"] }kuchikiki = "0.8.2"lol_html = "2.6.0"-lopdf = { git = "https://github.com/J-F-Liu/lopdf", rev = "6ee84c30f9226d6752dc8e1892298c9d46ac3eb6" }+pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector", rev = "f37c911" }maud = "0.27.0"napi = { version = "3.0.0", features = ["serde-json", "tokio_rt"] }napi-derive = "3.0.0"diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rsindex 1678292dd..14a680185 100644--- a/apps/api/native/src/pdf.rs+++ b/apps/api/native/src/pdf.rs@@ -1,36 +1,67 @@use napi::bindgen_prelude::*;use napi_derive::napi;-use serde::Serialize;+use pdf_inspector::{+ PdfOptions, PdfType,+ process_pdf_with_options as rust_process_pdf,+};-#[derive(Debug, Clone, Serialize)]#[napi(object)]-pub struct PDFMetadata {- pub num_pages: i32,- #[serde(skip_serializing_if = "Option::is_none")]+pub struct PdfProcessResult {+ pub pdf_type: String,+ pub markdown: Option<String>,+ pub page_count: i32,+ pub processing_time_ms: f64,+ pub pages_needing_ocr: Vec<i32>,pub title: Option<String>,+ pub confidence: f64,+ pub is_complex: bool,}-fn _get_pdf_metadata(path: &str) -> std::result::Result<PDFMetadata, String> {- let metadata = match lopdf::Document::load_metadata(path) {- Ok(m) => m,- Err(e) => {- return Err(format!("Failed to load PDF metadata: {}", e));- }- };+fn pdf_type_str(t: PdfType) -> &'static str {+ match t {+ PdfType::TextBased => "TextBased",+ PdfType::Scanned => "Scanned",+ PdfType::ImageBased => "ImageBased",+ PdfType::Mixed => "Mixed",+ }+}- Ok(PDFMetadata {- num_pages: metadata.page_count as i32,- title: metadata.title,- })+fn to_napi_result(result: pdf_inspector::PdfProcessResult) -> PdfProcessResult {+ PdfProcessResult {+ pdf_type: pdf_type_str(result.pdf_type).to_string(),+ markdown: result.markdown,+ page_count: result.page_count as i32,+ processing_time_ms: result.processing_time_ms as f64,+ pages_needing_ocr: result.pages_needing_ocr.iter().map(|&p| p as i32).collect(),+ title: result.title,+ confidence: result.confidence as f64,+ is_complex: result.layout.is_complex,+ }}-/// Extract metadata from PDF file.+/// Process a PDF file: detect type, extract text + markdown if text-based.#[napi]-pub fn get_pdf_metadata(path: String) -> Result<PDFMetadata> {- _get_pdf_metadata(&path).map_err(|e| {+pub fn process_pdf(path: String) -> Result<PdfProcessResult> {+ let result = rust_process_pdf(&path, PdfOptions::new()).map_err(|e| {Error::new(Status::GenericFailure,- format!("Failed to get PDF metadata: {e}"),+ format!("Failed to process PDF: {e}"),)- })+ })?;++ Ok(to_napi_result(result))+}++/// Fast metadata-only detection: page count, title, type, confidence.+/// Skips text extraction, markdown generation, and layout analysis.+#[napi]+pub fn detect_pdf(path: String) -> Result<PdfProcessResult> {+ let result = rust_process_pdf(&path, PdfOptions::detect_only()).map_err(|e| {+ Error::new(+ Status::GenericFailure,+ format!("Failed to detect PDF: {e}"),+ )+ })?;++ Ok(to_napi_result(result))}diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.tsindex a9c908f0d..bedc30d95 100644--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts@@ -147,6 +147,80 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {);});+ describe("Mode - object format", () => {+ it.concurrent(+ "accepts mode: 'fast' and parses PDF with Rust parser",+ async () => {+ const response = await scrape(+ {+ url: pdfUrl,+ parsers: [{ type: "pdf", mode: "fast" }],+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.numPages).toBeGreaterThan(0);+ },+ scrapeTimeout * 2,+ );++ it.concurrent(+ "accepts mode: 'auto' and parses PDF",+ async () => {+ const response = await scrape(+ {+ url: pdfUrl,+ parsers: [{ type: "pdf", mode: "auto" }],+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.numPages).toBeGreaterThan(0);+ },+ scrapeTimeout * 2,+ );++ it.concurrent(+ "accepts mode: 'ocr' and parses PDF via OCR",+ async () => {+ const response = await scrape(+ {+ url: pdfUrl,+ parsers: [{ type: "pdf", mode: "ocr" }],+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.numPages).toBeGreaterThan(0);+ },+ scrapeTimeout * 2,+ );++ it.concurrent(+ "accepts mode with maxPages combined",+ async () => {+ const response = await scrape(+ {+ url: pdfUrl,+ parsers: [{ type: "pdf", mode: "fast", maxPages: 1 }],+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.numPages).toBe(1);+ },+ scrapeTimeout * 2,+ );+ });+describe("Default behavior", () => {it.concurrent("parses PDF by default when parsers not specified",@@ -256,6 +330,42 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {},scrapeTimeout,);++ it.concurrent(+ "rejects invalid mode in object format",+ async () => {+ const raw = await scrapeRaw(+ {+ url: pdfUrl,+ parsers: [{ type: "pdf", mode: "invalid" } as any],+ },+ identity,+ );++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ },+ scrapeTimeout,+ );++ it.concurrent(+ "rejects colon-separated shorthand strings",+ async () => {+ const raw = await scrapeRaw(+ {+ url: pdfUrl,+ parsers: ["pdf:fast" as any],+ },+ identity,+ );++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ },+ scrapeTimeout,+ );});describe("Billing implications", () => {diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 9dd187cba..925d58002 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -155,6 +155,9 @@ const configSchema = z.object({RUNPOD_MU_API_KEY: z.string().optional(),RUNPOD_MU_POD_ID: z.string().optional(),+ // PDF Rust Extraction (pdf-inspector)+ PDF_RUST_EXTRACT_ENABLE: z.stringbool().optional(),+// WebhooksSELF_HOSTED_WEBHOOK_URL: z.string().optional(),SELF_HOSTED_WEBHOOK_HMAC_SECRET: z.string().optional(),diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex db0aa83d4..5f461da21 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -413,8 +413,13 @@ export type FormatObject =| AttributesFormatWithOptions| { type: "branding" };+const pdfModeSchema = z.enum(["fast", "auto", "ocr"]);++export type PDFMode = z.infer<typeof pdfModeSchema>;+const pdfParserWithOptions = z.strictObject({type: z.literal("pdf"),+ mode: pdfModeSchema.optional(),maxPages: z.int().positive().finite().max(10000).optional(),});@@ -449,6 +454,17 @@ export function getPDFMaxPages(parsers?: Parsers): number | undefined {return undefined;}+export function getPDFMode(parsers?: Parsers): PDFMode {+ if (!parsers) return "auto";+ for (const parser of parsers) {+ if (parser === "pdf") return "auto";+ if (typeof parser === "object" && parser.type === "pdf") {+ return parser.mode ?? "auto";+ }+ }+ return "auto";+}+function transformIframeSelector(selector: string): string {return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, match => {const prefix = match.match(/^[\s,]/)?.[0] || "";diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 2300a6d43..da33785a4 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -16,6 +16,7 @@ import {PDFAntibotError,DocumentAntibotError,PDFInsufficientTimeError,+ PDFOCRRequiredError,NoEnginesLeftError,ZDRViolationError,PDFPrefetchFailed,@@ -45,6 +46,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_DNS_RESOLUTION_ERROR: DNSResolutionError,SCRAPE_PDF_INSUFFICIENT_TIME_ERROR: PDFInsufficientTimeError,SCRAPE_PDF_ANTIBOT_ERROR: PDFAntibotError,+ SCRAPE_PDF_OCR_REQUIRED: PDFOCRRequiredError,SCRAPE_DOCUMENT_ANTIBOT_ERROR: DocumentAntibotError,SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,SCRAPE_NO_CACHED_DATA: NoCachedDataError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex dc0157753..146900f64 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -14,6 +14,7 @@ export type ErrorCodes =| "SCRAPE_DNS_RESOLUTION_ERROR"| "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR"| "SCRAPE_PDF_ANTIBOT_ERROR"+ | "SCRAPE_PDF_OCR_REQUIRED"| "SCRAPE_DOCUMENT_ANTIBOT_ERROR"| "SCRAPE_UNSUPPORTED_FILE_ERROR"| "SCRAPE_ACTION_ERROR"diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 7a897cc1b..8f561ff5d 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -18,7 +18,7 @@ import { indexMaxReasonableTime, scrapeURLWithIndex } from "./index/index";import { queryEngpickerVerdict, useIndex } from "../../../services";import { hasFormatOfType } from "../../../lib/format-utils";import { getPDFMaxPages } from "../../../controllers/v2/types";-import { PdfMetadata } from "@mendable/firecrawl-rs";+import type { PdfMetadata } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";export type Engine =diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 636e7ba54..60c6f101a 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -1,294 +1,48 @@import { Meta } from "../..";import { config } from "../../../../config";import { EngineScrapeResult } from "..";-import * as marked from "marked";-import { robustFetch } from "../../lib/fetch";-import { z } from "zod";import * as Sentry from "@sentry/node";-import escapeHtml from "escape-html";-import PdfParse from "pdf-parse";+import * as marked from "marked";import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";import {PDFAntibotError,PDFInsufficientTimeError,+ PDFOCRRequiredError,PDFPrefetchFailed,RemoveFeatureError,EngineUnsuccessfulError,} from "../../error";import { readFile, unlink } from "node:fs/promises";-import path from "node:path";import type { Response } from "undici";-import {- getPdfResultFromCache,- savePdfResultToCache,-} from "../../../../lib/gcs-pdf-cache";import { AbortManagerThrownError } from "../../lib/abortManager";import {shouldParsePDF,getPDFMaxPages,+ getPDFMode,} from "../../../../controllers/v2/types";-import { getPdfMetadata } from "@mendable/firecrawl-rs";-import { captureExceptionWithZdrCheck } from "../../../../services/sentry";--type PDFProcessorResult = { html: string; markdown?: string };--const MAX_FILE_SIZE = 19 * 1024 * 1024; // 19MB-const MILLISECONDS_PER_PAGE = 150;--async function scrapePDFWithRunPodMU(- meta: Meta,- tempFilePath: string,- base64Content: string,- maxPages?: number,- pagesProcessed?: number,-): Promise<PDFProcessorResult> {- meta.logger.debug("Processing PDF document with RunPod MU", {- tempFilePath,- });-- if (!maxPages) {- try {- const cachedResult = await getPdfResultFromCache(base64Content);- if (cachedResult) {- meta.logger.info("Using cached RunPod MU result for PDF", {- tempFilePath,- });- return cachedResult;- }- } catch (error) {- meta.logger.warn("Error checking PDF cache, proceeding with RunPod MU", {- error,- tempFilePath,- });- }- }-- meta.abort.throwIfAborted();-- meta.logger.info("Max Pdf pages", {- tempFilePath,- maxPages,- });-- if (- config.PDF_MU_V2_EXPERIMENT === "true" &&- config.PDF_MU_V2_BASE_URL &&- Math.random() * 100 < config.PDF_MU_V2_EXPERIMENT_PERCENT- ) {- (async () => {- const pdfParseId = crypto.randomUUID();- const startedAt = Date.now();- const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });- logger.info("MU v2 experiment started", {- scrapeId: meta.id,- pdfParseId,- url: meta.rewrittenUrl ?? meta.url,- maxPages,- });- try {- const resp = await robustFetch({- url: config.PDF_MU_V2_BASE_URL ?? "",- method: "POST",- headers: config.PDF_MU_V2_API_KEY- ? { Authorization: `Bearer ${config.PDF_MU_V2_API_KEY}` }- : undefined,- body: {- input: {- file_content: base64Content,- filename: path.basename(tempFilePath) + ".pdf",- timeout: meta.abort.scrapeTimeout(),- created_at: Date.now(),- id: pdfParseId,- ...(maxPages !== undefined && { max_pages: maxPages }),- },- },- logger,- schema: z.any(),- mock: meta.mock,- abort: meta.abort.asSignal(),- });- const body: any = resp as any;- const tokensIn = body?.metadata?.["total-input-tokens"];- const tokensOut = body?.metadata?.["total-output-tokens"];- const pages = body?.metadata?.["pdf-total-pages"];- const durationMs = Date.now() - startedAt;- logger.info("MU v2 experiment completed", {- durationMs,- url: meta.rewrittenUrl ?? meta.url,- tokensIn,- tokensOut,- pages,- });- } catch (error) {- const durationMs = Date.now() - startedAt;- logger.warn("MU v2 experiment failed", { error, durationMs });- }- })();- }-- const muV1StartedAt = Date.now();- const podStart = await robustFetch({- url: "https://api.runpod.ai/v2/" + config.RUNPOD_MU_POD_ID + "/runsync",- method: "POST",- headers: {- Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,- },- body: {- input: {- file_content: base64Content,- filename: path.basename(tempFilePath) + ".pdf",- timeout: meta.abort.scrapeTimeout(),- created_at: Date.now(),- ...(maxPages !== undefined && { max_pages: maxPages }),- },- },- logger: meta.logger.child({- method: "scrapePDFWithRunPodMU/runsync/robustFetch",- }),- schema: z.object({- id: z.string(),- status: z.string(),- error: z.string().optional(),- output: z- .object({- markdown: z.string().optional(),- status: z.string().optional(),- })- .optional(),- }),- mock: meta.mock,- abort: meta.abort.asSignal(),- });-- let status: string = podStart.status;- let result: { markdown?: string } | undefined =- podStart.output?.markdown !== undefined- ? { markdown: podStart.output.markdown }- : undefined;- let lastRunPodError: string | undefined = podStart.error;-- if (status === "IN_QUEUE" || status === "IN_PROGRESS") {- do {- meta.abort.throwIfAborted();- await new Promise(resolve => setTimeout(resolve, 2500));- meta.abort.throwIfAborted();- const podStatus = await robustFetch({- url: `https://api.runpod.ai/v2/${config.RUNPOD_MU_POD_ID}/status/${podStart.id}`,- method: "GET",- headers: {- Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,- },- logger: meta.logger.child({- method: "scrapePDFWithRunPodMU/status/robustFetch",- }),- schema: z.object({- status: z.string(),- error: z.string().optional(),- output: z- .object({- markdown: z.string().optional(),- status: z.string().optional(),- })- .optional(),- }),- mock: meta.mock,- abort: meta.abort.asSignal(),- });- status = podStatus.status;- result =- podStatus.output?.markdown !== undefined- ? { markdown: podStatus.output.markdown }- : undefined;- if (podStatus.error !== undefined) {- lastRunPodError = podStatus.error;- }- } while (status !== "COMPLETED" && status !== "FAILED");- }-- if (status === "FAILED") {- const durationMs = Date.now() - muV1StartedAt;- const errorMessage =- lastRunPodError ?? "RunPod MU failed to parse PDF (no error message)";- const err = new Error(`RunPod MU failed to parse PDF: ${errorMessage}`);- meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {- durationMs,- url: meta.rewrittenUrl ?? meta.url,- mu_id: podStart.id,- runpodError: lastRunPodError,- ...(pagesProcessed !== undefined && { pagesProcessed }),- });- captureExceptionWithZdrCheck(err, {- extra: {- zeroDataRetention: meta.internalOptions.zeroDataRetention ?? false,- scrapeId: meta.id,- teamId: meta.internalOptions.teamId,- url: meta.rewrittenUrl ?? meta.url,- runpodId: podStart.id,- runpodError: lastRunPodError,- durationMs,- },- });- throw err;- }-- if (!result?.markdown) {- const durationMs = Date.now() - muV1StartedAt;- meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {- durationMs,- url: meta.rewrittenUrl ?? meta.url,- mu_id: podStart.id,- ...(pagesProcessed !== undefined && { pagesProcessed }),- });- throw new Error("RunPod MU returned no result");- }-- const processorResult = {- markdown: result.markdown,- html: await marked.parse(result.markdown, { async: true }),- } satisfies PDFProcessorResult;-- if (!meta.internalOptions.zeroDataRetention) {- try {- await savePdfResultToCache(base64Content, processorResult);- } catch (error) {- meta.logger.warn("Error saving PDF to cache", {- error,- tempFilePath,- });- }- }-- {- const durationMs = Date.now() - muV1StartedAt;- meta.logger.child({ method: "scrapePDF/MUv1" }).info("MU v1 completed", {- durationMs,- url: meta.rewrittenUrl ?? meta.url,- mu_id: podStart.id,- ...(pagesProcessed !== undefined && { pagesProcessed }),- });- }-- return processorResult;-}--async function scrapePDFWithParsePDF(- meta: Meta,- tempFilePath: string,-): Promise<PDFProcessorResult> {- meta.logger.debug("Processing PDF document with parse-pdf", { tempFilePath });-- const result = await PdfParse(await readFile(tempFilePath));- const escaped = escapeHtml(result.text);-- return {- markdown: escaped,- html: escaped,- };+import type { PDFMode } from "../../../../controllers/v2/types";+import { processPdf, detectPdf } from "@mendable/firecrawl-rs";+import { MAX_FILE_SIZE, MILLISECONDS_PER_PAGE } from "./types";+import type { PDFProcessorResult } from "./types";+import { scrapePDFWithRunPodMU } from "./runpodMU";+import { scrapePDFWithParsePDF } from "./pdfParse";++/** Check if the PDF is eligible for Rust extraction, returning a rejection reason or null. */+function getIneligibleReason(+ result: ReturnType<typeof processPdf>,+): string | null {+ if (result.pdfType !== "TextBased") return `pdfType=${result.pdfType}`;+ if (result.confidence < 0.95) return `confidence=${result.confidence}`;+ if (result.isComplex) return "complex layout (tables/columns)";+ if (!result.markdown?.length)+ return "empty markdown (unexpected for TextBased)";+ return null;}export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {const shouldParse = shouldParsePDF(meta.options.parsers);const maxPages = getPDFMaxPages(meta.options.parsers);+ const mode: PDFMode = getPDFMode(meta.options.parsers);if (!shouldParse) {if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {@@ -375,14 +129,113 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {}}- const pdfMetadata = await getPdfMetadata(tempFilePath);- const effectivePageCount = maxPages- ? Math.min(pdfMetadata.numPages, maxPages)- : pdfMetadata.numPages;+ let result: PDFProcessorResult | null = null;+ let effectivePageCount: number = 0;+ let metadataTitle: string | undefined;++ const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;+ const logger = meta.logger.child({ method: "scrapePDF/processPdf" });++ if (!rustEnabled || mode === "ocr") {+ // Legacy / OCR path: detect metadata only, skip Rust extraction.+ // When PDF_RUST_EXTRACT_ENABLE is off this is the only path taken,+ // matching current prod behaviour (detectPdf → MinerU → pdfParse).+ try {+ const startedAt = Date.now();+ const detection = detectPdf(tempFilePath);+ const durationMs = Date.now() - startedAt;++ logger.info("detectPdf completed", {+ durationMs,+ pdfType: detection.pdfType,+ pageCount: detection.pageCount,+ url: meta.rewrittenUrl ?? meta.url,+ rustEnabled,+ mode,+ });++ effectivePageCount = maxPages+ ? Math.min(detection.pageCount, maxPages)+ : detection.pageCount;+ metadataTitle = detection.title ?? undefined;+ } catch (error) {+ logger.warn("detectPdf failed", {+ error,+ url: meta.rewrittenUrl ?? meta.url,+ });+ Sentry.captureException(error);+ }+ } else {+ // Rust extraction enabled (fast / auto modes).+ try {+ const startedAt = Date.now();+ const pdfResult = processPdf(tempFilePath);+ const durationMs = Date.now() - startedAt;++ logger.info("processPdf completed", {+ durationMs,+ pdfType: pdfResult.pdfType,+ pageCount: pdfResult.pageCount,+ confidence: pdfResult.confidence,+ isComplex: pdfResult.isComplex,+ markdownLength: pdfResult.markdown?.length ?? 0,+ url: meta.rewrittenUrl ?? meta.url,+ mode,+ });+ effectivePageCount = maxPages+ ? Math.min(pdfResult.pageCount, maxPages)+ : pdfResult.pageCount;+ metadataTitle = pdfResult.title ?? undefined;++ const ineligibleReason = getIneligibleReason(pdfResult);+ const eligible = !ineligibleReason;++ logger.info("Rust PDF eligibility", {+ rust_pdf_eligible: eligible,+ reason: ineligibleReason ?? "eligible",+ url: meta.rewrittenUrl ?? meta.url,+ pdfType: pdfResult.pdfType,+ isComplex: pdfResult.isComplex,+ pageCount: pdfResult.pageCount,+ confidence: pdfResult.confidence,+ mode,+ });++ // In fast mode, if the PDF requires OCR, fail immediately with a+ // clear error instead of returning empty content.+ if (+ mode === "fast" &&+ (pdfResult.pdfType === "Scanned" ||+ pdfResult.pdfType === "ImageBased")+ ) {+ throw new PDFOCRRequiredError(pdfResult.pdfType);+ }++ if (eligible && pdfResult.markdown) {+ const html = await marked.parse(pdfResult.markdown, { async: true });+ result = { markdown: pdfResult.markdown, html };+ }+ } catch (error) {+ if (error instanceof PDFOCRRequiredError) {+ throw error;+ }+ logger.warn("processPdf failed, falling back to MU/PdfParse", {+ error,+ url: meta.rewrittenUrl ?? meta.url,+ });+ Sentry.captureException(error);+ // effectivePageCount stays 0 — skip time budget check+ }+ }++ // Only enforce the per-page time budget when we need MU/fallback.+ // Rust extraction is fast enough that the constraint doesn't apply.if (+ !result &&+ effectivePageCount > 0 &&effectivePageCount * MILLISECONDS_PER_PAGE >- (meta.abort.scrapeTimeout() ?? Infinity)+ (meta.abort.scrapeTimeout() ?? Infinity)) {throw new PDFInsufficientTimeError(effectivePageCount,@@ -390,64 +243,65 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {);}- let result: PDFProcessorResult | null = null;-- const base64Content = (await readFile(tempFilePath)).toString("base64");-- // First try RunPod MU if conditions are met- if (- base64Content.length < MAX_FILE_SIZE &&- config.RUNPOD_MU_API_KEY &&- config.RUNPOD_MU_POD_ID- ) {- const muV1StartedAt = Date.now();- try {- result = await scrapePDFWithRunPodMU(- {- ...meta,- logger: meta.logger.child({- method: "scrapePDF/scrapePDFWithRunPodMU",- }),- },- tempFilePath,- base64Content,- maxPages,- effectivePageCount,- );- const muV1DurationMs = Date.now() - muV1StartedAt;- meta.logger- .child({ method: "scrapePDF/MUv1Experiment" })- .info("MU v1 completed", {- durationMs: muV1DurationMs,- url: meta.rewrittenUrl ?? meta.url,- pages: effectivePageCount,- success: true,- });- } catch (error) {- if (- error instanceof RemoveFeatureError ||- error instanceof AbortManagerThrownError- ) {- throw error;+ // OCR / MU fallback.+ // Skipped only when Rust extraction is enabled AND mode is "fast".+ const skipOCR = rustEnabled && mode === "fast";+ if (!result && !skipOCR) {+ const base64Content = (await readFile(tempFilePath)).toString("base64");++ if (+ base64Content.length < MAX_FILE_SIZE &&+ config.RUNPOD_MU_API_KEY &&+ config.RUNPOD_MU_POD_ID+ ) {+ const muV1StartedAt = Date.now();+ try {+ result = await scrapePDFWithRunPodMU(+ {+ ...meta,+ logger: meta.logger.child({+ method: "scrapePDF/scrapePDFWithRunPodMU",+ }),+ },+ tempFilePath,+ base64Content,+ maxPages,+ );+ const muV1DurationMs = Date.now() - muV1StartedAt;+ meta.logger+ .child({ method: "scrapePDF/MUv1Experiment" })+ .info("MU v1 completed", {+ durationMs: muV1DurationMs,+ url: meta.rewrittenUrl ?? meta.url,+ pages: effectivePageCount,+ success: true,+ });+ } catch (error) {+ if (+ error instanceof RemoveFeatureError ||+ error instanceof AbortManagerThrownError+ ) {+ throw error;+ }+ meta.logger.warn(+ "RunPod MU failed to parse PDF (could be due to timeout) -- falling back to parse-pdf",+ { error },+ );+ Sentry.captureException(error);+ const muV1DurationMs = Date.now() - muV1StartedAt;+ meta.logger+ .child({ method: "scrapePDF/MUv1Experiment" })+ .info("MU v1 failed", {+ durationMs: muV1DurationMs,+ url: meta.rewrittenUrl ?? meta.url,+ pages: effectivePageCount,+ success: false,+ });}- meta.logger.warn(- "RunPod MU failed to parse PDF (could be due to timeout) -- falling back to parse-pdf",- { error },- );- Sentry.captureException(error);- const muV1DurationMs = Date.now() - muV1StartedAt;- meta.logger- .child({ method: "scrapePDF/MUv1Experiment" })- .info("MU v1 failed", {- durationMs: muV1DurationMs,- url: meta.rewrittenUrl ?? meta.url,- pages: effectivePageCount,- success: false,- });}}- // If RunPod MU failed or wasn't attempted, use PdfParse+ // Final fallback to PdfParse.if (!result) {result = await scrapePDFWithParsePDF({@@ -466,10 +320,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {html: result?.html ?? "",markdown: result?.markdown ?? "",pdfMetadata: {- // Rust parser gets the metadata incorrectly, so we overwrite the page count here with the effective page count- // TODO: fix this laternumPages: effectivePageCount,- title: pdfMetadata.title,+ title: metadataTitle,},proxyUsed: "basic",diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/pdfParse.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/pdfParse.tsnew file mode 100644index 000000000..9cf8a1207--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/pdfParse.ts@@ -0,0 +1,20 @@+import { Meta } from "../..";+import escapeHtml from "escape-html";+import PdfParse from "pdf-parse";+import { readFile } from "node:fs/promises";+import type { PDFProcessorResult } from "./types";++export async function scrapePDFWithParsePDF(+ meta: Meta,+ tempFilePath: string,+): Promise<PDFProcessorResult> {+ meta.logger.debug("Processing PDF document with parse-pdf", { tempFilePath });++ const result = await PdfParse(await readFile(tempFilePath));+ const escaped = escapeHtml(result.text);++ return {+ markdown: escaped,+ html: escaped,+ };+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.tsnew file mode 100644index 000000000..ae9e04e19--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts@@ -0,0 +1,211 @@+import { Meta } from "../..";+import { config } from "../../../../config";+import * as marked from "marked";+import { robustFetch } from "../../lib/fetch";+import { z } from "zod";+import path from "node:path";+import {+ getPdfResultFromCache,+ savePdfResultToCache,+} from "../../../../lib/gcs-pdf-cache";+import type { PDFProcessorResult } from "./types";++export async function scrapePDFWithRunPodMU(+ meta: Meta,+ tempFilePath: string,+ base64Content: string,+ maxPages?: number,+): Promise<PDFProcessorResult> {+ meta.logger.debug("Processing PDF document with RunPod MU", {+ tempFilePath,+ });++ if (!maxPages) {+ try {+ const cachedResult = await getPdfResultFromCache(base64Content);+ if (cachedResult) {+ meta.logger.info("Using cached RunPod MU result for PDF", {+ tempFilePath,+ });+ return cachedResult;+ }+ } catch (error) {+ meta.logger.warn("Error checking PDF cache, proceeding with RunPod MU", {+ error,+ tempFilePath,+ });+ }+ }++ meta.abort.throwIfAborted();++ meta.logger.info("Max Pdf pages", {+ tempFilePath,+ maxPages,+ });++ if (+ config.PDF_MU_V2_EXPERIMENT === "true" &&+ config.PDF_MU_V2_BASE_URL &&+ Math.random() * 100 < config.PDF_MU_V2_EXPERIMENT_PERCENT+ ) {+ (async () => {+ const pdfParseId = crypto.randomUUID();+ const startedAt = Date.now();+ const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });+ logger.info("MU v2 experiment started", {+ scrapeId: meta.id,+ pdfParseId,+ url: meta.rewrittenUrl ?? meta.url,+ maxPages,+ });+ try {+ const resp = await robustFetch({+ url: config.PDF_MU_V2_BASE_URL ?? "",+ method: "POST",+ headers: config.PDF_MU_V2_API_KEY+ ? { Authorization: `Bearer ${config.PDF_MU_V2_API_KEY}` }+ : undefined,+ body: {+ input: {+ file_content: base64Content,+ filename: path.basename(tempFilePath) + ".pdf",+ timeout: meta.abort.scrapeTimeout(),+ created_at: Date.now(),+ id: pdfParseId,+ ...(maxPages !== undefined && { max_pages: maxPages }),+ },+ },+ logger,+ schema: z.any(),+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ });+ const body: any = resp as any;+ const tokensIn = body?.metadata?.["total-input-tokens"];+ const tokensOut = body?.metadata?.["total-output-tokens"];+ const pages = body?.metadata?.["pdf-total-pages"];+ const durationMs = Date.now() - startedAt;+ logger.info("MU v2 experiment completed", {+ durationMs,+ url: meta.rewrittenUrl ?? meta.url,+ tokensIn,+ tokensOut,+ pages,+ });+ } catch (error) {+ const durationMs = Date.now() - startedAt;+ logger.warn("MU v2 experiment failed", { error, durationMs });+ }+ })();+ }++ const muV1StartedAt = Date.now();+ const podStart = await robustFetch({+ url: "https://api.runpod.ai/v2/" + config.RUNPOD_MU_POD_ID + "/runsync",+ method: "POST",+ headers: {+ Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,+ },+ body: {+ input: {+ file_content: base64Content,+ filename: path.basename(tempFilePath) + ".pdf",+ timeout: meta.abort.scrapeTimeout(),+ created_at: Date.now(),+ ...(maxPages !== undefined && { max_pages: maxPages }),+ },+ },+ logger: meta.logger.child({+ method: "scrapePDFWithRunPodMU/runsync/robustFetch",+ }),+ schema: z.object({+ id: z.string(),+ status: z.string(),+ output: z+ .object({+ markdown: z.string(),+ })+ .optional(),+ }),+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ });++ let status: string = podStart.status;+ let result: { markdown: string } | undefined = podStart.output;++ if (status === "IN_QUEUE" || status === "IN_PROGRESS") {+ do {+ meta.abort.throwIfAborted();+ await new Promise(resolve => setTimeout(resolve, 2500));+ meta.abort.throwIfAborted();+ const podStatus = await robustFetch({+ url: `https://api.runpod.ai/v2/${config.RUNPOD_MU_POD_ID}/status/${podStart.id}`,+ method: "GET",+ headers: {+ Authorization: `Bearer ${config.RUNPOD_MU_API_KEY}`,+ },+ logger: meta.logger.child({+ method: "scrapePDFWithRunPodMU/status/robustFetch",+ }),+ schema: z.object({+ status: z.string(),+ output: z+ .object({+ markdown: z.string(),+ })+ .optional(),+ }),+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ });+ status = podStatus.status;+ result = podStatus.output;+ } while (status !== "COMPLETED" && status !== "FAILED");+ }++ if (status === "FAILED") {+ const durationMs = Date.now() - muV1StartedAt;+ meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {+ durationMs,+ url: meta.rewrittenUrl ?? meta.url,+ });+ throw new Error("RunPod MU failed to parse PDF");+ }++ if (!result) {+ const durationMs = Date.now() - muV1StartedAt;+ meta.logger.child({ method: "scrapePDF/MUv1" }).warn("MU v1 failed", {+ durationMs,+ url: meta.rewrittenUrl ?? meta.url,+ });+ throw new Error("RunPod MU returned no result");+ }++ const processorResult = {+ markdown: result.markdown,+ html: await marked.parse(result.markdown, { async: true }),+ };++ if (!meta.internalOptions.zeroDataRetention) {+ try {+ await savePdfResultToCache(base64Content, processorResult);+ } catch (error) {+ meta.logger.warn("Error saving PDF to cache", {+ error,+ tempFilePath,+ });+ }+ }++ {+ const durationMs = Date.now() - muV1StartedAt;+ meta.logger.child({ method: "scrapePDF/MUv1" }).info("MU v1 completed", {+ durationMs,+ url: meta.rewrittenUrl ?? meta.url,+ });+ }++ return processorResult;+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.tsnew file mode 100644index 000000000..d9311cbaf--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts@@ -0,0 +1,6 @@+export type PDFProcessorResult = { html: string; markdown?: string };++export type PdfMetadata = { numPages: number; title?: string };++export const MAX_FILE_SIZE = 19 * 1024 * 1024; // 19MB+export const MILLISECONDS_PER_PAGE = 150;diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 24e5662e6..4fcdb1365 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -354,6 +354,31 @@ export class ZDRViolationError extends TransportableError {}}+export class PDFOCRRequiredError extends TransportableError {+ constructor(public pdfType: string) {+ super(+ "SCRAPE_PDF_OCR_REQUIRED",+ `This PDF is ${pdfType === "Scanned" ? "scanned" : "image-based"} and requires OCR for text extraction, but the requested PDF mode is "fast" which only supports text-based PDFs. To process this PDF, use mode "auto" (which falls back to OCR automatically) or mode "ocr" (which forces OCR processing). Example: parsers: [{ type: "pdf", mode: "auto" }]`,+ );+ }++ serialize() {+ return {+ ...super.serialize(),+ pdfType: this.pdfType,+ };+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new PDFOCRRequiredError(data.pdfType);+ x.stack = data.stack;+ return x;+ }+}+export class PDFPrefetchFailed extends TransportableError {constructor() {const message = isSelfHosted()diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex e5ddd7842..9102b9b1e 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -36,6 +36,7 @@ import {UnsupportedFileError,SSLError,PDFInsufficientTimeError,+ PDFOCRRequiredError,IndexMissError,NoCachedDataError,DNSResolutionError,@@ -646,6 +647,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof ActionError ||error.error instanceof UnsupportedFileError ||error.error instanceof PDFAntibotError ||+ error.error instanceof PDFOCRRequiredError ||error.error instanceof DocumentAntibotError ||error.error instanceof PDFInsufficientTimeError ||error.error instanceof ProxySelectionError ||@@ -1197,6 +1199,14 @@ export async function scrapeURL(meta.logger.warn("scrapeURL: Insufficient time to process PDF", {error,});+ } else if (error instanceof PDFOCRRequiredError) {+ errorType = "PDFOCRRequiredError";+ meta.logger.warn(+ "scrapeURL: PDF requires OCR but fast mode was requested",+ {+ error,+ },+ );} else if (error instanceof PDFPrefetchFailed) {errorType = "PDFPrefetchFailed";meta.logger.warn(diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex a49c73e9d..4d289bf09 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -5,7 +5,11 @@ import { htmlTransform } from "../lib/removeUnwantedElements";import { extractLinks } from "../lib/extractLinks";import { extractImages } from "../lib/extractImages";import { extractMetadata } from "../lib/extractMetadata";-import { performLLMExtract, performSummary, performCleanContent } from "./llmExtract";+import {+ performLLMExtract,+ performSummary,+ performCleanContent,+} from "./llmExtract";import { uploadScreenshot } from "./uploadScreenshot";import { removeBase64Images } from "./removeBase64Images";import { performAgent } from "./agent";@@ -85,7 +89,13 @@ async function deriveMarkdownFromHTML(const hasJson = hasFormatOfType(meta.options.formats, "json");const hasSummary = hasFormatOfType(meta.options.formats, "summary");- if (!hasMarkdown && !hasChangeTracking && !hasJson && !hasSummary && !meta.options.onlyCleanContent) {+ if (+ !hasMarkdown &&+ !hasChangeTracking &&+ !hasJson &&+ !hasSummary &&+ !meta.options.onlyCleanContent+ ) {return document;}diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 16630c3a1..9c957cb67 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -7,7 +7,7 @@ import { redisEvictConnection } from "./redis";import type { Logger } from "winston";import psl from "psl";import { MapDocument } from "../controllers/v2/types";-import { PdfMetadata } from "@mendable/firecrawl-rs";+import type { PdfMetadata } from "../scraper/scrapeURL/engines/pdf/types";import { storage } from "../lib/gcs-jobs";import { withSpan, setSpanAttributes } from "../lib/otel-tracer";import { config } from "../config";