Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
skip shadow comparison for non-TextBased PDFs (#2969)
Abimael Martellde8d73ecb946
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex a75e144c5..f98a021e0 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -224,14 +224,13 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {});// Only shadow-compare when Rust had a real chance at extraction.- // Scanned/ImageBased PDFs are expected to fail — comparing them- // just adds noise to the metrics.+ // Scanned/ImageBased/Mixed PDFs are expected to produce near-zero+ // Rust output — comparing them just adds noise to the metrics.const shadowEligible =!eligible &&pdfResult.markdown &&config.PDF_SHADOW_COMPARISON_ENABLE &&- pdfResult.pdfType !== "Scanned" &&- pdfResult.pdfType !== "ImageBased";+ pdfResult.pdfType === "TextBased";rustMarkdownForShadow = shadowEligible ? pdfResult.markdown : undefined;if (shadowEligible) {