Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update pdf-inspector to dcdb0a3, expand shadow comparison to Mixed PDFs (#3177)
Abimael Martell050e918dc773
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex 17eb878ef..999cc199a 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -12,7 +12,7 @@ crate-type = ["cdylib"]chrono = { version = "0.4", features = ["serde"] }kuchikiki = "0.8.2"lol_html = "2.6.0"-pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector", rev = "442a169" }+pdf-inspector = { git = "https://github.com/firecrawl/pdf-inspector", rev = "dcdb0a3" }maud = "0.27.0"napi = { version = "3.0.0", features = ["serde-json", "tokio_rt"] }napi-derive = "3.0.0"diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 9561f3029..d34a4e7b4 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -23,7 +23,10 @@ import type { PDFMode } from "../../../../controllers/v2/types";import { processPdf, detectPdf } from "@mendable/firecrawl-rs";import { MAX_FILE_SIZE, MILLISECONDS_PER_PAGE } from "./types";import type { PDFProcessorResult } from "./types";-import { emitNativeLogs, extractAndEmitNativeLogs } from "../../../../lib/native-logging";+import {+ emitNativeLogs,+ extractAndEmitNativeLogs,+} from "../../../../lib/native-logging";import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";import { scrapePDFWithRunPodMU } from "./runpodMU";import { scrapePDFWithParsePDF } from "./pdfParse";@@ -157,9 +160,12 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// When PDF_RUST_EXTRACT_ENABLE is off this is the only path taken,// matching current prod behaviour (detectPdf β MinerU β pdfParse).try {- const nativeCtx = { scrapeId: meta.id, url: meta.rewrittenUrl ?? meta.url };+ const nativeCtx = {+ scrapeId: meta.id,+ url: meta.rewrittenUrl ?? meta.url,+ };const startedAt = Date.now();- const detection = await withSpan("native.pdf.detect", async (span) => {+ const detection = await withSpan("native.pdf.detect", async span => {const result = detectPdf(tempFilePath, nativeCtx);setSpanAttributes(span, {"native.module": "pdf",@@ -202,10 +208,17 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {} else {// Rust extraction enabled (fast / auto modes).try {- const nativeCtx = { scrapeId: meta.id, url: meta.rewrittenUrl ?? meta.url };+ const nativeCtx = {+ scrapeId: meta.id,+ url: meta.rewrittenUrl ?? meta.url,+ };const startedAt = Date.now();- const pdfResult = await withSpan("native.pdf.process", async (span) => {- const result = processPdf(tempFilePath, maxPages ?? undefined, nativeCtx);+ const pdfResult = await withSpan("native.pdf.process", async span => {+ const result = processPdf(+ tempFilePath,+ maxPages ?? undefined,+ nativeCtx,+ );setSpanAttributes(span, {"native.module": "pdf","native.pdf_type": result.pdfType,@@ -248,14 +261,18 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {mode,});- // Only shadow-compare when Rust had a real chance at extraction.- // Scanned/ImageBased/Mixed PDFs are expected to produce near-zero- // Rust output β comparing them just adds noise to the metrics.+ // Shadow-compare when Rust produced meaningful output but wasn't+ // eligible for direct serving. Includes:+ // - Ineligible TextBased (complex layouts, lower confidence)+ // - Mixed PDFs with substantial extracted text (invisible OCR layers)+ const charsPerPage =+ (pdfResult.markdown?.length ?? 0) / Math.max(pdfResult.pageCount, 1);const shadowEligible =!eligible &&pdfResult.markdown &&config.PDF_SHADOW_COMPARISON_ENABLE &&- pdfResult.pdfType === "TextBased";+ (pdfResult.pdfType === "TextBased" ||+ (pdfResult.pdfType === "Mixed" && charsPerPage >= 200));rustMarkdownForShadow = shadowEligible ? pdfResult.markdown : undefined;if (shadowEligible) {