Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: prevent PDF scrapes with parsePDF:false from being indexed (#1933)
Gergő Móriczbfbff8967849
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 031dab909..b9db25833 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -594,6 +594,73 @@ describe("Scrape tests", () => {expect(response.metadata.cacheState).toBe("hit");}, scrapeTimeout * 2 + 1 * indexCooldown);++ it.concurrent("does not index PDF scrapes with parsePDF:false", async () => {+ const id = crypto.randomUUID();+ const url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf?testId=" + id;++ // First scrape with parsePDF:false and maxAge:0 to force a fresh scrape+ const response1 = await scrape({+ url,+ parsePDF: false,+ maxAge: 0,+ timeout: scrapeTimeout,+ }, identity);++ // Verify we got base64 content+ expect(response1.metadata.cacheState).not.toBe("hit");+ expect(response1.markdown).toBeDefined();+ // Base64 content should start with JVBERi (which is "%PDF" base64 encoded)+ expect(response1.markdown!.startsWith("JVBERi")).toBe(true);++ // Wait for indexing to potentially happen (it shouldn't)+ await new Promise(resolve => setTimeout(resolve, indexCooldown));++ // Now scrape with parsePDF:true and a high maxAge value+ const response2 = await scrape({+ url,+ parsePDF: true,+ maxAge: scrapeTimeout * 10,+ timeout: scrapeTimeout,+ }, identity);++ // Should NOT hit cache (because parsePDF:false shouldn't have indexed)+ expect(response2.metadata.cacheState).not.toBe("hit");+ // Should get parsed text content, not base64+ expect(response2.markdown).toBeDefined();+ expect(response2.markdown!.startsWith("JVBERi")).toBe(false);+ // PDF should contain actual text content+ expect(response2.markdown!.toLowerCase()).toContain("dummy");++ // Wait for this one to be indexed+ await new Promise(resolve => setTimeout(resolve, indexCooldown));++ // Now scrape again with parsePDF:true and high maxAge - should hit cache+ const response3 = await scrape({+ url,+ parsePDF: true,+ maxAge: scrapeTimeout * 10,+ timeout: scrapeTimeout,+ }, identity);++ expect(response3.metadata.cacheState).toBe("hit");+ expect(response3.metadata.cachedAt).toBeDefined();+ // Should still be parsed text, not base64+ expect(response3.markdown!.startsWith("JVBERi")).toBe(false);+ expect(response3.markdown!.toLowerCase()).toContain("dummy");++ // Verify that scraping with parsePDF:false still doesn't hit cache+ const response4 = await scrape({+ url,+ parsePDF: false,+ maxAge: scrapeTimeout * 10,+ timeout: scrapeTimeout,+ }, identity);++ // Should not hit cache since parsePDF:false results are not indexed+ expect(response4.metadata.cacheState).not.toBe("hit");+ expect(response4.markdown!.startsWith("JVBERi")).toBe(true);+ }, scrapeTimeout * 4 + 3 * indexCooldown);});describe("Change Tracking format", () => {diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 1718fbc02..9f1fe14db 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -88,10 +88,11 @@ function execForward(fancyName: string, command: string): Promise<void> {})(),]);- console.log("=== Starting API and Worker...");+ console.log("=== Starting API, Worker, and Index Worker...");const api = execForward("api", "pnpm start:production:nobuild");const worker = execForward("worker", "pnpm worker:production");+ const indexWorker = execForward("index-worker", "pnpm index-worker:production");try {await Promise.race([@@ -107,14 +108,17 @@ function execForward(fancyName: string, command: string): Promise<void> {cmd,api,worker,+ indexWorker,]);} finally {- console.log("=== Tearing down API and Worker...");+ console.log("=== Tearing down API, Worker, and Index Worker...");exec("pkill -f 'queue-worker.js'");exec("pkill -f 'index.js'");+ exec("pkill -f 'index-worker.js'");await Promise.all([api,worker,+ indexWorker,]);}diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex c5024101b..3fe3ffc8d 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -12,6 +12,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {&& !meta.internalOptions.zeroDataRetention&& meta.winnerEngine !== "index"&& meta.winnerEngine !== "index;documents"+ && !(meta.winnerEngine === "pdf" && meta.options.parsePDF === false)&& (meta.internalOptions.teamId === "sitemap"|| (@@ -207,6 +208,25 @@ export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefinthrow new EngineError("Document not found in GCS");}+ // Check if the cached content is a PDF base64 (starts with JVBERi)+ const isCachedPdfBase64 = doc.html && doc.html.startsWith("JVBERi");++ // If the cached content is base64 PDF but we want parsed PDF (parsePDF:true or default)+ if (isCachedPdfBase64 && meta.options.parsePDF !== false) {+ // Cached content is unparsed PDF, but we want parsed - report cache miss+ throw new IndexMissError();+ }++ // If the cached content is NOT base64 PDF but we want unparsed PDF (parsePDF:false)+ if (!isCachedPdfBase64 && meta.options.parsePDF === false) {+ // Check if URL looks like a PDF+ const isPdfUrl = meta.url.toLowerCase().endsWith(".pdf") || meta.url.includes(".pdf?");+ if (isPdfUrl) {+ // This is likely a parsed PDF cached, but we want unparsed - report cache miss+ throw new IndexMissError();+ }+ }+return {url: doc.url,html: doc.html,