Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: prevent PDF scrapes with parsePDF:false from being indexed (#1933)

Gergő Móriczbfbff8967849
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index 031dab909..b9db25833 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -594,6 +594,73 @@ describe("Scrape tests", () => {
expect(response.metadata.cacheState).toBe("hit");
}, scrapeTimeout * 2 + 1 * indexCooldown);
+
+ it.concurrent("does not index PDF scrapes with parsePDF:false", async () => {
+ const id = crypto.randomUUID();
+ const url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf?testId=" + id;
+
+ // First scrape with parsePDF:false and maxAge:0 to force a fresh scrape
+ const response1 = await scrape({
+ url,
+ parsePDF: false,
+ maxAge: 0,
+ timeout: scrapeTimeout,
+ }, identity);
+
+ // Verify we got base64 content
+ expect(response1.metadata.cacheState).not.toBe("hit");
+ expect(response1.markdown).toBeDefined();
+ // Base64 content should start with JVBERi (which is "%PDF" base64 encoded)
+ expect(response1.markdown!.startsWith("JVBERi")).toBe(true);
+
+ // Wait for indexing to potentially happen (it shouldn't)
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
+
+ // Now scrape with parsePDF:true and a high maxAge value
+ const response2 = await scrape({
+ url,
+ parsePDF: true,
+ maxAge: scrapeTimeout * 10,
+ timeout: scrapeTimeout,
+ }, identity);
+
+ // Should NOT hit cache (because parsePDF:false shouldn't have indexed)
+ expect(response2.metadata.cacheState).not.toBe("hit");
+ // Should get parsed text content, not base64
+ expect(response2.markdown).toBeDefined();
+ expect(response2.markdown!.startsWith("JVBERi")).toBe(false);
+ // PDF should contain actual text content
+ expect(response2.markdown!.toLowerCase()).toContain("dummy");
+
+ // Wait for this one to be indexed
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
+
+ // Now scrape again with parsePDF:true and high maxAge - should hit cache
+ const response3 = await scrape({
+ url,
+ parsePDF: true,
+ maxAge: scrapeTimeout * 10,
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(response3.metadata.cacheState).toBe("hit");
+ expect(response3.metadata.cachedAt).toBeDefined();
+ // Should still be parsed text, not base64
+ expect(response3.markdown!.startsWith("JVBERi")).toBe(false);
+ expect(response3.markdown!.toLowerCase()).toContain("dummy");
+
+ // Verify that scraping with parsePDF:false still doesn't hit cache
+ const response4 = await scrape({
+ url,
+ parsePDF: false,
+ maxAge: scrapeTimeout * 10,
+ timeout: scrapeTimeout,
+ }, identity);
+
+ // Should not hit cache since parsePDF:false results are not indexed
+ expect(response4.metadata.cacheState).not.toBe("hit");
+ expect(response4.markdown!.startsWith("JVBERi")).toBe(true);
+ }, scrapeTimeout * 4 + 3 * indexCooldown);
});
describe("Change Tracking format", () => {
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 1718fbc02..9f1fe14db 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -88,10 +88,11 @@ function execForward(fancyName: string, command: string): Promise<void> {
})(),
]);
- console.log("=== Starting API and Worker...");
+ console.log("=== Starting API, Worker, and Index Worker...");
const api = execForward("api", "pnpm start:production:nobuild");
const worker = execForward("worker", "pnpm worker:production");
+ const indexWorker = execForward("index-worker", "pnpm index-worker:production");
try {
await Promise.race([
@@ -107,14 +108,17 @@ function execForward(fancyName: string, command: string): Promise<void> {
cmd,
api,
worker,
+ indexWorker,
]);
} finally {
- console.log("=== Tearing down API and Worker...");
+ console.log("=== Tearing down API, Worker, and Index Worker...");
exec("pkill -f 'queue-worker.js'");
exec("pkill -f 'index.js'");
+ exec("pkill -f 'index-worker.js'");
await Promise.all([
api,
worker,
+ indexWorker,
]);
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index c5024101b..3fe3ffc8d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -12,6 +12,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
&& !meta.internalOptions.zeroDataRetention
&& meta.winnerEngine !== "index"
&& meta.winnerEngine !== "index;documents"
+ && !(meta.winnerEngine === "pdf" && meta.options.parsePDF === false)
&& (
meta.internalOptions.teamId === "sitemap"
|| (
@@ -207,6 +208,25 @@ export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefin
throw new EngineError("Document not found in GCS");
}
+ // Check if the cached content is a PDF base64 (starts with JVBERi)
+ const isCachedPdfBase64 = doc.html && doc.html.startsWith("JVBERi");
+
+ // If the cached content is base64 PDF but we want parsed PDF (parsePDF:true or default)
+ if (isCachedPdfBase64 && meta.options.parsePDF !== false) {
+ // Cached content is unparsed PDF, but we want parsed - report cache miss
+ throw new IndexMissError();
+ }
+
+ // If the cached content is NOT base64 PDF but we want unparsed PDF (parsePDF:false)
+ if (!isCachedPdfBase64 && meta.options.parsePDF === false) {
+ // Check if URL looks like a PDF
+ const isPdfUrl = meta.url.toLowerCase().endsWith(".pdf") || meta.url.includes(".pdf?");
+ if (isPdfUrl) {
+ // This is likely a parsed PDF cached, but we want unparsed - report cache miss
+ throw new IndexMissError();
+ }
+ }
+
return {
url: doc.url,
html: doc.html,