Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(native): structured tracing and panic safety for Rust PDF processing (#3139)
Abimael Martell14c9d2776a1d
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex f16908ac9..c4e96214d 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -29,10 +29,12 @@ zip = "5.0.0"calamine = "0.26"cfb = "0.10"tokio = "1.48.0"+tracing = "0.1"+tracing-subscriber = { version = "0.3", default-features = false, features = ["registry"] }[build-dependencies]napi-build = "2"[profile.release]lto = true-strip = "symbols"+strip = "debuginfo"diff --git a/apps/api/native/src/lib.rs b/apps/api/native/src/lib.rsindex 731c4f2b4..f440be0cf 100644--- a/apps/api/native/src/lib.rs+++ b/apps/api/native/src/lib.rs@@ -3,6 +3,7 @@pub use crate::crawler::*;pub use crate::engpicker::*;pub use crate::html::*;+pub use crate::logging::*;pub use crate::pdf::*;pub use crate::utils::*;@@ -12,6 +13,7 @@ mod crawler;mod document;mod engpicker;mod html;+mod logging;mod pdf;mod utils;diff --git a/apps/api/native/src/logging.rs b/apps/api/native/src/logging.rsnew file mode 100644index 000000000..bf444e895--- /dev/null+++ b/apps/api/native/src/logging.rs@@ -0,0 +1,269 @@+use napi_derive::napi;+use serde::Serialize;+use serde_json::Value;+use std::sync::{Arc, Mutex};+use tracing::field::{Field, Visit};+use tracing::Level;+use tracing_subscriber::layer::SubscriberExt;+use tracing_subscriber::Layer;++/// Context passed from TypeScript to continue the trace.+#[derive(Clone)]+#[napi(object)]+pub struct NativeContext {+ pub scrape_id: String,+ pub url: String,+}++/// A single log entry captured during Rust execution.+#[derive(Clone, Debug, Serialize)]+#[napi(object)]+pub struct NativeLogEntry {+ pub level: String,+ pub target: String,+ pub message: String,+ pub fields: Value,+ pub timestamp_ms: f64,+}++struct LogCollector {+ logs: Arc<Mutex<Vec<NativeLogEntry>>>,+}++struct FieldVisitor {+ fields: serde_json::Map<String, Value>,+ message: Option<String>,+}++impl Visit for FieldVisitor {+ fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {+ if field.name() == "message" {+ self.message = Some(format!("{:?}", value));+ } else {+ self+ .fields+ .insert(field.name().to_string(), Value::String(format!("{:?}", value)));+ }+ }++ fn record_str(&mut self, field: &Field, value: &str) {+ if field.name() == "message" {+ self.message = Some(value.to_string());+ } else {+ self+ .fields+ .insert(field.name().to_string(), Value::String(value.to_string()));+ }+ }++ fn record_i64(&mut self, field: &Field, value: i64) {+ self+ .fields+ .insert(field.name().to_string(), Value::Number(value.into()));+ }++ fn record_u64(&mut self, field: &Field, value: u64) {+ self+ .fields+ .insert(field.name().to_string(), Value::Number(value.into()));+ }++ fn record_f64(&mut self, field: &Field, value: f64) {+ if let Some(n) = serde_json::Number::from_f64(value) {+ self+ .fields+ .insert(field.name().to_string(), Value::Number(n));+ }+ }++ fn record_bool(&mut self, field: &Field, value: bool) {+ self+ .fields+ .insert(field.name().to_string(), Value::Bool(value));+ }+}++impl<S: tracing::Subscriber> Layer<S> for LogCollector {+ fn on_event(&self, event: &tracing::Event<'_>, _ctx: tracing_subscriber::layer::Context<'_, S>) {+ let mut visitor = FieldVisitor {+ fields: serde_json::Map::new(),+ message: None,+ };+ event.record(&mut visitor);++ let level = match *event.metadata().level() {+ Level::ERROR => "error",+ Level::WARN => "warn",+ Level::INFO => "info",+ Level::DEBUG => "debug",+ Level::TRACE => "trace",+ };++ let entry = NativeLogEntry {+ level: level.to_string(),+ target: event.metadata().target().to_string(),+ message: visitor.message.unwrap_or_default(),+ fields: Value::Object(visitor.fields),+ timestamp_ms: std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .map(|d| d.as_secs_f64() * 1000.0)+ .unwrap_or(0.0),+ };++ if let Ok(mut logs) = self.logs.lock() {+ logs.push(entry);+ }+ }+}++#[derive(Debug)]+pub struct TracingResult<T> {+ pub value: T,+ pub logs: Vec<NativeLogEntry>,+}++/// Run a closure with tracing enabled, capturing all log events.+/// Wraps the closure in `catch_unwind` for panic safety.+///+/// Returns `TracingResult<napi::Result<T>>` so that logs are **always**+/// available β even when the closure returns `Err` or panics.+pub fn with_native_tracing<T, F>(+ ctx: Option<&NativeContext>,+ module: &str,+ f: F,+) -> TracingResult<napi::Result<T>>+where+ F: FnOnce() -> napi::Result<T>,+{+ let logs = Arc::new(Mutex::new(Vec::new()));+ let collector = LogCollector { logs: logs.clone() };+ let subscriber = tracing_subscriber::Registry::default().with(collector);++ let result = tracing::subscriber::with_default(subscriber, || {+ let _span = match ctx {+ Some(c) => tracing::info_span!(+ "native",+ scrape_id = %c.scrape_id,+ url = %c.url,+ module = %module,+ )+ .entered(),+ None => tracing::info_span!("native", module = %module).entered(),+ };++ match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {+ Ok(result) => result,+ Err(panic_info) => {+ let msg = if let Some(s) = panic_info.downcast_ref::<&str>() {+ s.to_string()+ } else if let Some(s) = panic_info.downcast_ref::<String>() {+ s.clone()+ } else {+ "unknown panic".to_string()+ };+ let backtrace = std::backtrace::Backtrace::force_capture();+ tracing::error!(+ panic = true,+ backtrace = %backtrace,+ "native panic in {}: {}", module, msg,+ );++ Err(napi::Error::new(+ napi::Status::GenericFailure,+ format!("Rust panic in {module}: {msg}\nBacktrace:\n{backtrace}"),+ ))+ }+ }+ });++ let collected = logs.lock().map(|l| l.clone()).unwrap_or_default();++ TracingResult {+ value: result,+ logs: collected,+ }+}++/// Append serialized logs to a NAPI error so they survive the FFI boundary.+/// The TS side can extract them from `error.message` via `extractNativeLogs`.+pub fn embed_logs_in_error(err: napi::Error, logs: &[NativeLogEntry]) -> napi::Error {+ if logs.is_empty() {+ return err;+ }+ if let Ok(logs_json) = serde_json::to_string(logs) {+ napi::Error::new(+ err.status,+ format!("{}\n__native_logs__:{logs_json}", err.reason),+ )+ } else {+ err+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_collects_logs() {+ let traced = with_native_tracing(None, "test", || {+ tracing::info!("hello from rust");+ Ok(42)+ });++ let value = traced.value.unwrap();+ assert_eq!(value, 42);+ assert_eq!(traced.logs.len(), 1);+ assert_eq!(traced.logs[0].level, "info");+ assert!(traced.logs[0].message.contains("hello from rust"));+ }++ #[test]+ fn test_with_context() {+ let ctx = NativeContext {+ scrape_id: "test-123".to_string(),+ url: "https://example.com".to_string(),+ };+ let traced = with_native_tracing(Some(&ctx), "pdf", || {+ tracing::warn!("something odd");+ Ok("ok")+ });++ assert_eq!(traced.value.unwrap(), "ok");+ assert_eq!(traced.logs.len(), 1);+ assert_eq!(traced.logs[0].level, "warn");+ }++ #[test]+ fn test_captures_panic_with_logs() {+ let traced: TracingResult<napi::Result<()>> = with_native_tracing(None, "test", || {+ panic!("test panic");+ });++ assert!(traced.value.is_err());+ let err = traced.value.unwrap_err();+ assert!(err.reason.contains("test panic"));+ assert!(err.reason.contains("Backtrace"));+ // Panic log is preserved even though the closure failed+ assert!(!traced.logs.is_empty());+ assert_eq!(traced.logs[0].level, "error");+ assert!(traced.logs[0].message.contains("test panic"));+ }++ #[test]+ fn test_error_preserves_logs() {+ let traced: TracingResult<napi::Result<()>> = with_native_tracing(None, "test", || {+ tracing::info!("before error");+ Err(napi::Error::new(+ napi::Status::GenericFailure,+ "test error",+ ))+ });++ assert!(traced.value.is_err());+ // Logs are preserved even on error paths+ assert_eq!(traced.logs.len(), 1);+ assert_eq!(traced.logs[0].level, "info");+ assert!(traced.logs[0].message.contains("before error"));+ }+}diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rsindex 412821b00..f97fc2743 100644--- a/apps/api/native/src/pdf.rs+++ b/apps/api/native/src/pdf.rs@@ -1,9 +1,8 @@use napi::bindgen_prelude::*;use napi_derive::napi;-use pdf_inspector::{- PdfOptions, PdfType,- process_pdf_with_options as rust_process_pdf,-};+use pdf_inspector::{PdfOptions, PdfType, process_pdf_with_options as rust_process_pdf};++use crate::logging::{embed_logs_in_error, with_native_tracing, NativeContext, NativeLogEntry};#[napi(object)]pub struct PdfProcessResult {@@ -15,6 +14,7 @@ pub struct PdfProcessResult {pub title: Option<String>,pub confidence: f64,pub is_complex: bool,+ pub logs: Vec<NativeLogEntry>,}fn pdf_type_str(t: PdfType) -> &'static str {@@ -36,38 +36,83 @@ fn to_napi_result(result: pdf_inspector::PdfProcessResult) -> PdfProcessResult {title: result.title,confidence: result.confidence as f64,is_complex: result.layout.is_complex,+ logs: Vec::new(),}}/// Process a PDF file: detect type, extract text + markdown if text-based./// When `max_pages` is provided, only the first N pages are extracted.+/// Pass `ctx` (NativeContext) for structured tracing with scrape_id/url.#[napi]-pub fn process_pdf(path: String, max_pages: Option<u32>) -> Result<PdfProcessResult> {- let opts = match max_pages {- Some(n) if n > 0 => PdfOptions::new().pages(1..=n),- _ => PdfOptions::new(),- };+pub fn process_pdf(+ path: String,+ max_pages: Option<u32>,+ ctx: Option<NativeContext>,+) -> Result<PdfProcessResult> {+ let traced = with_native_tracing(ctx.as_ref(), "pdf", || {+ tracing::info!(max_pages = ?max_pages, "starting PDF processing");++ let opts = match max_pages {+ Some(n) if n > 0 => PdfOptions::new().pages(1..=n),+ _ => PdfOptions::new(),+ };++ let result = rust_process_pdf(&path, opts).map_err(|e| {+ tracing::error!(error = %e, "PDF processing failed");+ Error::new(Status::GenericFailure, format!("Failed to process PDF: {e}"))+ })?;- let result = rust_process_pdf(&path, opts).map_err(|e| {- Error::new(- Status::GenericFailure,- format!("Failed to process PDF: {e}"),- )- })?;+ tracing::info!(+ pdf_type = pdf_type_str(result.pdf_type),+ page_count = result.page_count,+ confidence = %result.confidence,+ is_complex = result.layout.is_complex,+ "PDF processing complete"+ );- Ok(to_napi_result(result))+ Ok(to_napi_result(result))+ });++ match traced.value {+ Ok(mut result) => {+ result.logs = traced.logs;+ Ok(result)+ }+ Err(err) => Err(embed_logs_in_error(err, &traced.logs)),+ }}/// Fast metadata-only detection: page count, title, type, confidence./// Skips text extraction, markdown generation, and layout analysis.+/// Pass `ctx` (NativeContext) for structured tracing with scrape_id/url.#[napi]-pub fn detect_pdf(path: String) -> Result<PdfProcessResult> {- let result = rust_process_pdf(&path, PdfOptions::detect_only()).map_err(|e| {- Error::new(- Status::GenericFailure,- format!("Failed to detect PDF: {e}"),- )- })?;+pub fn detect_pdf(+ path: String,+ ctx: Option<NativeContext>,+) -> Result<PdfProcessResult> {+ let traced = with_native_tracing(ctx.as_ref(), "pdf", || {+ tracing::info!("starting PDF detection");- Ok(to_napi_result(result))+ let result = rust_process_pdf(&path, PdfOptions::detect_only()).map_err(|e| {+ tracing::error!(error = %e, "PDF detection failed");+ Error::new(Status::GenericFailure, format!("Failed to detect PDF: {e}"))+ })?;++ tracing::info!(+ pdf_type = pdf_type_str(result.pdf_type),+ page_count = result.page_count,+ confidence = %result.confidence,+ "PDF detection complete"+ );++ Ok(to_napi_result(result))+ });++ match traced.value {+ Ok(mut result) => {+ result.logs = traced.logs;+ Ok(result)+ }+ Err(err) => Err(embed_logs_in_error(err, &traced.logs)),+ }}diff --git a/apps/api/src/lib/native-logging.ts b/apps/api/src/lib/native-logging.tsnew file mode 100644index 000000000..a643ab313--- /dev/null+++ b/apps/api/src/lib/native-logging.ts@@ -0,0 +1,77 @@+import type { Logger } from "winston";++const NATIVE_LOGS_SEPARATOR = "\n__native_logs__:";++/** Matches the NativeLogEntry struct from Rust (@mendable/firecrawl-rs). */+export interface NativeLogEntry {+ level: string;+ target: string;+ message: string;+ fields: Record<string, unknown>;+ timestampMs: number;+}++/**+ * Extract native logs embedded in a NAPI error message by `embed_logs_in_error`.+ * Emits them through the logger and returns the cleaned error message.+ */+export function extractAndEmitNativeLogs(+ error: unknown,+ parentLogger: Logger,+ module: string,+): void {+ if (!(error instanceof Error)) return;+ const idx = error.message.indexOf(NATIVE_LOGS_SEPARATOR);+ if (idx === -1) return;++ const logsJson = error.message.slice(idx + NATIVE_LOGS_SEPARATOR.length);++ try {+ const logs: NativeLogEntry[] = JSON.parse(logsJson);+ // Only strip after successful parse so we don't lose data on failure+ error.message = error.message.slice(0, idx);+ emitNativeLogs(logs, parentLogger, module);+ } catch {+ // JSON parse failed β leave the original error message intact+ }+}++/**+ * Emit log entries captured inside the Rust native module through a Winston+ * logger, preserving trace context (scrape_id / url via the parent logger)+ * and adding `source: "native"` + the Rust module name as labels.+ */+export function emitNativeLogs(+ logs: NativeLogEntry[] | undefined,+ parentLogger: Logger,+ module: string,+): void {+ if (!logs || logs.length === 0) return;++ const childLogger = parentLogger.child({ source: "native", module });++ for (const entry of logs) {+ const meta = {+ rustTarget: entry.target,+ ...entry.fields,+ };++ switch (entry.level) {+ case "error":+ childLogger.error(entry.message, meta);+ break;+ case "warn":+ childLogger.warn(entry.message, meta);+ break;+ case "info":+ childLogger.info(entry.message, meta);+ break;+ case "debug":+ case "trace":+ childLogger.debug(entry.message, meta);+ break;+ default:+ childLogger.info(entry.message, meta);+ }+ }+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex caf1d8170..9561f3029 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -23,6 +23,8 @@ import type { PDFMode } from "../../../../controllers/v2/types";import { processPdf, detectPdf } from "@mendable/firecrawl-rs";import { MAX_FILE_SIZE, MILLISECONDS_PER_PAGE } from "./types";import type { PDFProcessorResult } from "./types";+import { emitNativeLogs, extractAndEmitNativeLogs } from "../../../../lib/native-logging";+import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";import { scrapePDFWithRunPodMU } from "./runpodMU";import { scrapePDFWithParsePDF } from "./pdfParse";import { captureExceptionWithZdrCheck } from "../../../../services/sentry";@@ -155,8 +157,18 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// When PDF_RUST_EXTRACT_ENABLE is off this is the only path taken,// matching current prod behaviour (detectPdf β MinerU β pdfParse).try {+ const nativeCtx = { scrapeId: meta.id, url: meta.rewrittenUrl ?? meta.url };const startedAt = Date.now();- const detection = detectPdf(tempFilePath);+ const detection = await withSpan("native.pdf.detect", async (span) => {+ const result = detectPdf(tempFilePath, nativeCtx);+ setSpanAttributes(span, {+ "native.module": "pdf",+ "native.pdf_type": result.pdfType,+ "native.page_count": result.pageCount,+ });+ emitNativeLogs(result.logs, meta.logger, "pdf.detect");+ return result;+ });const durationMs = Date.now() - startedAt;logger.info("detectPdf completed", {@@ -173,6 +185,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {: detection.pageCount;metadataTitle = detection.title ?? undefined;} catch (error) {+ extractAndEmitNativeLogs(error, meta.logger, "pdf.detect");logger.warn("detectPdf failed", {error,url: meta.rewrittenUrl ?? meta.url,@@ -189,8 +202,20 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {} else {// Rust extraction enabled (fast / auto modes).try {+ const nativeCtx = { scrapeId: meta.id, url: meta.rewrittenUrl ?? meta.url };const startedAt = Date.now();- const pdfResult = processPdf(tempFilePath, maxPages ?? undefined);+ const pdfResult = await withSpan("native.pdf.process", async (span) => {+ const result = processPdf(tempFilePath, maxPages ?? undefined, nativeCtx);+ setSpanAttributes(span, {+ "native.module": "pdf",+ "native.pdf_type": result.pdfType,+ "native.page_count": result.pageCount,+ "native.confidence": result.confidence,+ "native.is_complex": result.isComplex,+ });+ emitNativeLogs(result.logs, meta.logger, "pdf.process");+ return result;+ });const durationMs = Date.now() - startedAt;logger.info("processPdf completed", {@@ -258,6 +283,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (error instanceof PDFOCRRequiredError) {throw error;}+ extractAndEmitNativeLogs(error, meta.logger, "pdf.process");logger.warn("processPdf failed, falling back to MU/PdfParse", {error,url: meta.rewrittenUrl ?? meta.url,