Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(api): run native PDF extraction off the Node.js event loop

mogerye33e1f6c15c6
diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rs
index 400f010f2..a4f019f07 100644
--- a/apps/api/native/src/pdf.rs
+++ b/apps/api/native/src/pdf.rs
@@ -43,76 +43,101 @@ fn to_napi_result(result: pdf_inspector::PdfProcessResult) -> PdfProcessResult {
/// Process a PDF file: detect type, extract text + markdown if text-based.
/// When `max_pages` is provided, only the first N pages are extracted.
/// Pass `ctx` (NativeContext) for structured tracing with scrape_id/url.
+///
+/// Runs on tokio's blocking thread pool: extraction is CPU-bound and can take
+/// tens of seconds on large PDFs, which would otherwise freeze the Node.js
+/// event loop (and every in-flight request on the process) for the duration.
#[napi]
-pub fn process_pdf(
+pub async fn process_pdf(
path: String,
max_pages: Option<u32>,
ctx: Option<NativeContext>,
) -> Result<PdfProcessResult> {
- let traced = with_native_tracing(ctx.as_ref(), "pdf", || {
- tracing::info!(max_pages = ?max_pages, "starting PDF processing");
+ tokio::task::spawn_blocking(move || {
+ let traced = with_native_tracing(ctx.as_ref(), "pdf", || {
+ tracing::info!(max_pages = ?max_pages, "starting PDF processing");
- let opts = match max_pages {
- Some(n) if n > 0 => PdfOptions::new().pages(1..=n),
- _ => PdfOptions::new(),
- };
+ let opts = match max_pages {
+ Some(n) if n > 0 => PdfOptions::new().pages(1..=n),
+ _ => PdfOptions::new(),
+ };
- let result = rust_process_pdf(&path, opts).map_err(|e| {
- tracing::error!(error = %e, "PDF processing failed");
- Error::new(
- Status::GenericFailure,
- format!("Failed to process PDF: {e}"),
- )
- })?;
+ let result = rust_process_pdf(&path, opts).map_err(|e| {
+ tracing::error!(error = %e, "PDF processing failed");
+ Error::new(
+ Status::GenericFailure,
+ format!("Failed to process PDF: {e}"),
+ )
+ })?;
- tracing::info!(
- pdf_type = pdf_type_str(result.pdf_type),
- page_count = result.page_count,
- confidence = %result.confidence,
- is_complex = result.layout.is_complex,
- "PDF processing complete"
- );
+ tracing::info!(
+ pdf_type = pdf_type_str(result.pdf_type),
+ page_count = result.page_count,
+ confidence = %result.confidence,
+ is_complex = result.layout.is_complex,
+ "PDF processing complete"
+ );
- Ok(to_napi_result(result))
- });
+ Ok(to_napi_result(result))
+ });
- match traced.value {
- Ok(mut result) => {
- result.logs = traced.logs;
- Ok(result)
+ match traced.value {
+ Ok(mut result) => {
+ result.logs = traced.logs;
+ Ok(result)
+ }
+ Err(err) => Err(embed_logs_in_error(err, &traced.logs)),
}
- Err(err) => Err(embed_logs_in_error(err, &traced.logs)),
- }
+ })
+ .await
+ .map_err(|e| {
+ Error::new(
+ Status::GenericFailure,
+ format!("PDF processing task failed to complete: {e}"),
+ )
+ })?
}
/// Fast metadata-only detection: page count, title, type, confidence.
/// Skips text extraction, markdown generation, and layout analysis.
/// Pass `ctx` (NativeContext) for structured tracing with scrape_id/url.
+///
+/// Offloaded to tokio's blocking pool for the same reason as `process_pdf`:
+/// even metadata-only parsing can be slow on huge or malformed PDFs.
#[napi]
-pub fn detect_pdf(path: String, ctx: Option<NativeContext>) -> Result<PdfProcessResult> {
- let traced = with_native_tracing(ctx.as_ref(), "pdf", || {
- tracing::info!("starting PDF detection");
+pub async fn detect_pdf(path: String, ctx: Option<NativeContext>) -> Result<PdfProcessResult> {
+ tokio::task::spawn_blocking(move || {
+ let traced = with_native_tracing(ctx.as_ref(), "pdf", || {
+ tracing::info!("starting PDF detection");
- let result = rust_process_pdf(&path, PdfOptions::detect_only()).map_err(|e| {
- tracing::error!(error = %e, "PDF detection failed");
- Error::new(Status::GenericFailure, format!("Failed to detect PDF: {e}"))
- })?;
+ let result = rust_process_pdf(&path, PdfOptions::detect_only()).map_err(|e| {
+ tracing::error!(error = %e, "PDF detection failed");
+ Error::new(Status::GenericFailure, format!("Failed to detect PDF: {e}"))
+ })?;
- tracing::info!(
- pdf_type = pdf_type_str(result.pdf_type),
- page_count = result.page_count,
- confidence = %result.confidence,
- "PDF detection complete"
- );
+ tracing::info!(
+ pdf_type = pdf_type_str(result.pdf_type),
+ page_count = result.page_count,
+ confidence = %result.confidence,
+ "PDF detection complete"
+ );
- Ok(to_napi_result(result))
- });
+ Ok(to_napi_result(result))
+ });
- match traced.value {
- Ok(mut result) => {
- result.logs = traced.logs;
- Ok(result)
+ match traced.value {
+ Ok(mut result) => {
+ result.logs = traced.logs;
+ Ok(result)
+ }
+ Err(err) => Err(embed_logs_in_error(err, &traced.logs)),
}
- Err(err) => Err(embed_logs_in_error(err, &traced.logs)),
- }
+ })
+ .await
+ .map_err(|e| {
+ Error::new(
+ Status::GenericFailure,
+ format!("PDF detection task failed to complete: {e}"),
+ )
+ })?
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/nativeBinding.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/nativeBinding.test.ts
new file mode 100644
index 000000000..33f591370
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/nativeBinding.test.ts
@@ -0,0 +1,136 @@
+import { mkdtempSync, writeFileSync, rmSync } from "fs";
+import { tmpdir } from "os";
+import path from "path";
+import { processPdf, detectPdf } from "@mendable/firecrawl-rs";
+
+/**
+ * Tests for the native (Rust) PDF binding's interaction with the Node event
+ * loop. Regression: processPdf/detectPdf used to be synchronous napi calls β€”
+ * a 181-page complex PDF once blocked an app pod's event loop for ~25s,
+ * failing liveness probes and timing out every unrelated in-flight scrape on
+ * the process. They must run off the main thread (tokio spawn_blocking).
+ */
+
+/** Minimal valid PDF writer β€” no fixtures checked into the repo. */
+function makePdf(pages: number, linesPerPage: number): Buffer {
+ const kids: string[] = [];
+ const objects: [number, string][] = [];
+ let nextId = 4;
+
+ for (let i = 0; i < pages; i++) {
+ const pageId = nextId;
+ const contentId = nextId + 1;
+ nextId += 2;
+ kids.push(`${pageId} 0 R`);
+ // NB: keep the page/line numbers at the END of the sentence β€” a leading
+ // "Page N line M:" prefix trips the extractor's header/boilerplate filter
+ // and the line gets dropped.
+ const lines: string[] = [
+ `(The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs. Page ${i + 1}, line 0.) Tj`,
+ ];
+ for (let j = 1; j < linesPerPage; j++) {
+ lines.push(
+ `0 -14 Td (The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs. Page ${i + 1}, line ${j}.) Tj`,
+ );
+ }
+ const text = `BT /F1 12 Tf 72 720 Td ${lines.join(" ")} ET`;
+ objects.push([
+ pageId,
+ `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents ${contentId} 0 R >>`,
+ ]);
+ objects.push([
+ contentId,
+ `<< /Length ${text.length} >>\nstream\n${text}\nendstream`,
+ ]);
+ }
+
+ objects.unshift(
+ [1, "<< /Type /Catalog /Pages 2 0 R >>"],
+ [2, `<< /Type /Pages /Kids [${kids.join(" ")}] /Count ${pages} >>`],
+ [3, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"],
+ );
+ objects.sort((a, b) => a[0] - b[0]);
+
+ let out = "%PDF-1.4\n";
+ const offsets = new Map<number, number>();
+ for (const [id, body] of objects) {
+ offsets.set(id, out.length);
+ out += `${id} 0 obj\n${body}\nendobj\n`;
+ }
+ const xrefPos = out.length;
+ const maxId = nextId - 1;
+ out += `xref\n0 ${maxId + 1}\n0000000000 65535 f \n`;
+ for (let id = 1; id <= maxId; id++) {
+ out += `${offsets.get(id)}`.padStart(10, "0") + " 00000 n \n";
+ }
+ out += `trailer\n<< /Size ${maxId + 1} /Root 1 0 R >>\nstartxref\n${xrefPos}\n%%EOF\n`;
+ return Buffer.from(out, "latin1");
+}
+
+describe("native PDF binding", () => {
+ let dir: string;
+ let smallPdf: string;
+ let heavyPdf: string;
+
+ beforeAll(() => {
+ dir = mkdtempSync(path.join(tmpdir(), "firecrawl-native-pdf-"));
+ smallPdf = path.join(dir, "small.pdf");
+ heavyPdf = path.join(dir, "heavy.pdf");
+ writeFileSync(smallPdf, makePdf(1, 1));
+ // Big enough that extraction takes ~1s of CPU: with the old synchronous
+ // binding the event loop would be frozen for that entire time.
+ writeFileSync(heavyPdf, makePdf(2000, 50));
+ });
+
+ afterAll(() => {
+ rmSync(dir, { recursive: true, force: true });
+ });
+
+ it("detectPdf returns metadata", async () => {
+ const result = await detectPdf(smallPdf, {
+ scrapeId: "test",
+ url: "file://small.pdf",
+ });
+ expect(result.pageCount).toBe(1);
+ expect(result.pdfType).toBe("TextBased");
+ });
+
+ it("processPdf extracts markdown and native logs", async () => {
+ const result = await processPdf(smallPdf, null, {
+ scrapeId: "test",
+ url: "file://small.pdf",
+ });
+ expect(result.pageCount).toBe(1);
+ expect(result.markdown).toContain("quick brown fox");
+ expect(result.logs.map(l => l.message)).toContain(
+ "starting PDF processing",
+ );
+ });
+
+ it("does not block the event loop during large extractions", async () => {
+ let maxLag = 0;
+ let last = Date.now();
+ const interval = setInterval(() => {
+ const now = Date.now();
+ maxLag = Math.max(maxLag, now - last - 50);
+ last = now;
+ }, 50);
+
+ try {
+ const result = await processPdf(heavyPdf, null, {
+ scrapeId: "test",
+ url: "file://heavy.pdf",
+ });
+ expect(result.pageCount).toBe(2000);
+ } finally {
+ // Let any overdue interval tick fire before clearing: the await
+ // continuation is a microtask and would otherwise beat the timer,
+ // hiding the lag a synchronous binding would have caused.
+ await new Promise(resolve => setTimeout(resolve, 100));
+ clearInterval(interval);
+ }
+
+ // Synchronous binding: lag ~= full extraction time (~1s+ for this file).
+ expect(maxLag).toBeLessThan(250);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 07505a3c8..2484f784c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -58,7 +58,7 @@ import { comparePdfOutputs } from "./shadowComparison";
/** Check if the PDF is eligible for Rust extraction, returning a rejection reason or null. */
function getIneligibleReason(
- result: ReturnType<typeof processPdf>,
+ result: Awaited<ReturnType<typeof processPdf>>,
): string | null {
if (result.pdfType !== "TextBased") return `pdfType=${result.pdfType}`;
if (result.confidence < 0.95) return `confidence=${result.confidence}`;
@@ -260,7 +260,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
};
const startedAt = Date.now();
const detection = await withSpan("native.pdf.detect", async span => {
- const result = detectPdf(tempFilePath, nativeCtx);
+ const result = await detectPdf(tempFilePath, nativeCtx);
setSpanAttributes(span, {
"native.module": "pdf",
"native.pdf_type": result.pdfType,
@@ -309,7 +309,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
};
const startedAt = Date.now();
const pdfResult = await withSpan("native.pdf.process", async span => {
- const result = processPdf(
+ const result = await processPdf(
tempFilePath,
maxPages ?? undefined,
nativeCtx,