Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): run native PDF extraction off the Node.js event loop
mogerye33e1f6c15c6
diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rsindex 400f010f2..a4f019f07 100644--- a/apps/api/native/src/pdf.rs+++ b/apps/api/native/src/pdf.rs@@ -43,76 +43,101 @@ fn to_napi_result(result: pdf_inspector::PdfProcessResult) -> PdfProcessResult {/// Process a PDF file: detect type, extract text + markdown if text-based./// When `max_pages` is provided, only the first N pages are extracted./// Pass `ctx` (NativeContext) for structured tracing with scrape_id/url.+///+/// Runs on tokio's blocking thread pool: extraction is CPU-bound and can take+/// tens of seconds on large PDFs, which would otherwise freeze the Node.js+/// event loop (and every in-flight request on the process) for the duration.#[napi]-pub fn process_pdf(+pub async fn process_pdf(path: String,max_pages: Option<u32>,ctx: Option<NativeContext>,) -> Result<PdfProcessResult> {- let traced = with_native_tracing(ctx.as_ref(), "pdf", || {- tracing::info!(max_pages = ?max_pages, "starting PDF processing");+ tokio::task::spawn_blocking(move || {+ let traced = with_native_tracing(ctx.as_ref(), "pdf", || {+ tracing::info!(max_pages = ?max_pages, "starting PDF processing");- let opts = match max_pages {- Some(n) if n > 0 => PdfOptions::new().pages(1..=n),- _ => PdfOptions::new(),- };+ let opts = match max_pages {+ Some(n) if n > 0 => PdfOptions::new().pages(1..=n),+ _ => PdfOptions::new(),+ };- let result = rust_process_pdf(&path, opts).map_err(|e| {- tracing::error!(error = %e, "PDF processing failed");- Error::new(- Status::GenericFailure,- format!("Failed to process PDF: {e}"),- )- })?;+ let result = rust_process_pdf(&path, opts).map_err(|e| {+ tracing::error!(error = %e, "PDF processing failed");+ Error::new(+ Status::GenericFailure,+ format!("Failed to process PDF: {e}"),+ )+ })?;- tracing::info!(- pdf_type = pdf_type_str(result.pdf_type),- page_count = result.page_count,- confidence = %result.confidence,- is_complex = result.layout.is_complex,- "PDF processing complete"- );+ tracing::info!(+ pdf_type = pdf_type_str(result.pdf_type),+ page_count = result.page_count,+ confidence = %result.confidence,+ is_complex = result.layout.is_complex,+ "PDF processing complete"+ );- Ok(to_napi_result(result))- });+ Ok(to_napi_result(result))+ });- match traced.value {- Ok(mut result) => {- result.logs = traced.logs;- Ok(result)+ match traced.value {+ Ok(mut result) => {+ result.logs = traced.logs;+ Ok(result)+ }+ Err(err) => Err(embed_logs_in_error(err, &traced.logs)),}- Err(err) => Err(embed_logs_in_error(err, &traced.logs)),- }+ })+ .await+ .map_err(|e| {+ Error::new(+ Status::GenericFailure,+ format!("PDF processing task failed to complete: {e}"),+ )+ })?}/// Fast metadata-only detection: page count, title, type, confidence./// Skips text extraction, markdown generation, and layout analysis./// Pass `ctx` (NativeContext) for structured tracing with scrape_id/url.+///+/// Offloaded to tokio's blocking pool for the same reason as `process_pdf`:+/// even metadata-only parsing can be slow on huge or malformed PDFs.#[napi]-pub fn detect_pdf(path: String, ctx: Option<NativeContext>) -> Result<PdfProcessResult> {- let traced = with_native_tracing(ctx.as_ref(), "pdf", || {- tracing::info!("starting PDF detection");+pub async fn detect_pdf(path: String, ctx: Option<NativeContext>) -> Result<PdfProcessResult> {+ tokio::task::spawn_blocking(move || {+ let traced = with_native_tracing(ctx.as_ref(), "pdf", || {+ tracing::info!("starting PDF detection");- let result = rust_process_pdf(&path, PdfOptions::detect_only()).map_err(|e| {- tracing::error!(error = %e, "PDF detection failed");- Error::new(Status::GenericFailure, format!("Failed to detect PDF: {e}"))- })?;+ let result = rust_process_pdf(&path, PdfOptions::detect_only()).map_err(|e| {+ tracing::error!(error = %e, "PDF detection failed");+ Error::new(Status::GenericFailure, format!("Failed to detect PDF: {e}"))+ })?;- tracing::info!(- pdf_type = pdf_type_str(result.pdf_type),- page_count = result.page_count,- confidence = %result.confidence,- "PDF detection complete"- );+ tracing::info!(+ pdf_type = pdf_type_str(result.pdf_type),+ page_count = result.page_count,+ confidence = %result.confidence,+ "PDF detection complete"+ );- Ok(to_napi_result(result))- });+ Ok(to_napi_result(result))+ });- match traced.value {- Ok(mut result) => {- result.logs = traced.logs;- Ok(result)+ match traced.value {+ Ok(mut result) => {+ result.logs = traced.logs;+ Ok(result)+ }+ Err(err) => Err(embed_logs_in_error(err, &traced.logs)),}- Err(err) => Err(embed_logs_in_error(err, &traced.logs)),- }+ })+ .await+ .map_err(|e| {+ Error::new(+ Status::GenericFailure,+ format!("PDF detection task failed to complete: {e}"),+ )+ })?}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/nativeBinding.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/nativeBinding.test.tsnew file mode 100644index 000000000..33f591370--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/nativeBinding.test.ts@@ -0,0 +1,136 @@+import { mkdtempSync, writeFileSync, rmSync } from "fs";+import { tmpdir } from "os";+import path from "path";+import { processPdf, detectPdf } from "@mendable/firecrawl-rs";++/**+ * Tests for the native (Rust) PDF binding's interaction with the Node event+ * loop. Regression: processPdf/detectPdf used to be synchronous napi calls β+ * a 181-page complex PDF once blocked an app pod's event loop for ~25s,+ * failing liveness probes and timing out every unrelated in-flight scrape on+ * the process. They must run off the main thread (tokio spawn_blocking).+ */++/** Minimal valid PDF writer β no fixtures checked into the repo. */+function makePdf(pages: number, linesPerPage: number): Buffer {+ const kids: string[] = [];+ const objects: [number, string][] = [];+ let nextId = 4;++ for (let i = 0; i < pages; i++) {+ const pageId = nextId;+ const contentId = nextId + 1;+ nextId += 2;+ kids.push(`${pageId} 0 R`);+ // NB: keep the page/line numbers at the END of the sentence β a leading+ // "Page N line M:" prefix trips the extractor's header/boilerplate filter+ // and the line gets dropped.+ const lines: string[] = [+ `(The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs. Page ${i + 1}, line 0.) Tj`,+ ];+ for (let j = 1; j < linesPerPage; j++) {+ lines.push(+ `0 -14 Td (The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs. Page ${i + 1}, line ${j}.) Tj`,+ );+ }+ const text = `BT /F1 12 Tf 72 720 Td ${lines.join(" ")} ET`;+ objects.push([+ pageId,+ `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents ${contentId} 0 R >>`,+ ]);+ objects.push([+ contentId,+ `<< /Length ${text.length} >>\nstream\n${text}\nendstream`,+ ]);+ }++ objects.unshift(+ [1, "<< /Type /Catalog /Pages 2 0 R >>"],+ [2, `<< /Type /Pages /Kids [${kids.join(" ")}] /Count ${pages} >>`],+ [3, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"],+ );+ objects.sort((a, b) => a[0] - b[0]);++ let out = "%PDF-1.4\n";+ const offsets = new Map<number, number>();+ for (const [id, body] of objects) {+ offsets.set(id, out.length);+ out += `${id} 0 obj\n${body}\nendobj\n`;+ }+ const xrefPos = out.length;+ const maxId = nextId - 1;+ out += `xref\n0 ${maxId + 1}\n0000000000 65535 f \n`;+ for (let id = 1; id <= maxId; id++) {+ out += `${offsets.get(id)}`.padStart(10, "0") + " 00000 n \n";+ }+ out += `trailer\n<< /Size ${maxId + 1} /Root 1 0 R >>\nstartxref\n${xrefPos}\n%%EOF\n`;+ return Buffer.from(out, "latin1");+}++describe("native PDF binding", () => {+ let dir: string;+ let smallPdf: string;+ let heavyPdf: string;++ beforeAll(() => {+ dir = mkdtempSync(path.join(tmpdir(), "firecrawl-native-pdf-"));+ smallPdf = path.join(dir, "small.pdf");+ heavyPdf = path.join(dir, "heavy.pdf");+ writeFileSync(smallPdf, makePdf(1, 1));+ // Big enough that extraction takes ~1s of CPU: with the old synchronous+ // binding the event loop would be frozen for that entire time.+ writeFileSync(heavyPdf, makePdf(2000, 50));+ });++ afterAll(() => {+ rmSync(dir, { recursive: true, force: true });+ });++ it("detectPdf returns metadata", async () => {+ const result = await detectPdf(smallPdf, {+ scrapeId: "test",+ url: "file://small.pdf",+ });+ expect(result.pageCount).toBe(1);+ expect(result.pdfType).toBe("TextBased");+ });++ it("processPdf extracts markdown and native logs", async () => {+ const result = await processPdf(smallPdf, null, {+ scrapeId: "test",+ url: "file://small.pdf",+ });+ expect(result.pageCount).toBe(1);+ expect(result.markdown).toContain("quick brown fox");+ expect(result.logs.map(l => l.message)).toContain(+ "starting PDF processing",+ );+ });++ it("does not block the event loop during large extractions", async () => {+ let maxLag = 0;+ let last = Date.now();+ const interval = setInterval(() => {+ const now = Date.now();+ maxLag = Math.max(maxLag, now - last - 50);+ last = now;+ }, 50);++ try {+ const result = await processPdf(heavyPdf, null, {+ scrapeId: "test",+ url: "file://heavy.pdf",+ });+ expect(result.pageCount).toBe(2000);+ } finally {+ // Let any overdue interval tick fire before clearing: the await+ // continuation is a microtask and would otherwise beat the timer,+ // hiding the lag a synchronous binding would have caused.+ await new Promise(resolve => setTimeout(resolve, 100));+ clearInterval(interval);+ }++ // Synchronous binding: lag ~= full extraction time (~1s+ for this file).+ expect(maxLag).toBeLessThan(250);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 07505a3c8..2484f784c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -58,7 +58,7 @@ import { comparePdfOutputs } from "./shadowComparison";/** Check if the PDF is eligible for Rust extraction, returning a rejection reason or null. */function getIneligibleReason(- result: ReturnType<typeof processPdf>,+ result: Awaited<ReturnType<typeof processPdf>>,): string | null {if (result.pdfType !== "TextBased") return `pdfType=${result.pdfType}`;if (result.confidence < 0.95) return `confidence=${result.confidence}`;@@ -260,7 +260,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {};const startedAt = Date.now();const detection = await withSpan("native.pdf.detect", async span => {- const result = detectPdf(tempFilePath, nativeCtx);+ const result = await detectPdf(tempFilePath, nativeCtx);setSpanAttributes(span, {"native.module": "pdf","native.pdf_type": result.pdfType,@@ -309,7 +309,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {};const startedAt = Date.now();const pdfResult = await withSpan("native.pdf.process", async span => {- const result = processPdf(+ const result = await processPdf(tempFilePath,maxPages ?? undefined,nativeCtx,