Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): skip shadow comparison for scanned/image-based PDFs (#2968)
Abimael Martell5b76e0adfa90
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 4eb788503..a75e144c5 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -142,6 +142,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {let effectivePageCount: number = 0;let metadataTitle: string | undefined;let rustMarkdownForShadow: string | undefined;+ let shadowPdfType: string | undefined;+ let shadowConfidence: number | undefined;+ let shadowIsComplex: boolean | undefined;+ let shadowIneligibleReason: string | null | undefined;const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;const logger = meta.logger.child({ method: "scrapePDF/processPdf" });@@ -219,10 +223,23 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {mode,});- rustMarkdownForShadow =- !eligible && pdfResult.markdown && config.PDF_SHADOW_COMPARISON_ENABLE- ? pdfResult.markdown- : undefined;+ // Only shadow-compare when Rust had a real chance at extraction.+ // Scanned/ImageBased PDFs are expected to fail — comparing them+ // just adds noise to the metrics.+ const shadowEligible =+ !eligible &&+ pdfResult.markdown &&+ config.PDF_SHADOW_COMPARISON_ENABLE &&+ pdfResult.pdfType !== "Scanned" &&+ pdfResult.pdfType !== "ImageBased";++ rustMarkdownForShadow = shadowEligible ? pdfResult.markdown : undefined;+ if (shadowEligible) {+ shadowPdfType = pdfResult.pdfType;+ shadowConfidence = pdfResult.confidence;+ shadowIsComplex = pdfResult.isComplex;+ shadowIneligibleReason = ineligibleReason;+ }// In fast mode, if the PDF requires OCR, fail immediately with a// clear error instead of returning empty content.@@ -307,10 +324,16 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {success: true,});- if (rustMarkdownForShadow && result?.markdown && config.PDF_SHADOW_COMPARISON_ENABLE) {+ if (+ rustMarkdownForShadow &&+ result?.markdown &&+ config.PDF_SHADOW_COMPARISON_ENABLE+ ) {const shadowRust = rustMarkdownForShadow;const shadowMu = result.markdown;- const shadowLogger = meta.logger.child({ method: "scrapePDF/shadowComparison" });+ const shadowLogger = meta.logger.child({+ method: "scrapePDF/shadowComparison",+ });const isZdr = !!meta.internalOptions.zeroDataRetention;(async () => {@@ -320,6 +343,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {scrapeId: meta.id,url: isZdr ? undefined : (meta.rewrittenUrl ?? meta.url),pageCount: effectivePageCount,+ pdfType: shadowPdfType,+ confidence: shadowConfidence,+ isComplex: shadowIsComplex,+ ineligibleReason: shadowIneligibleReason,...metrics.overall,});} catch (error) {