Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): write index entries for maxPages requests when the PDF was not truncated (#4698)

Gergő Móricz7b0b0b996008
diff --git a/apps/api/src/__tests__/snips/v2/parsers-maxpages-index-cache.test.ts b/apps/api/src/__tests__/snips/v2/parsers-maxpages-index-cache.test.ts
new file mode 100644
index 000000000..83e9cc872
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/parsers-maxpages-index-cache.test.ts
@@ -0,0 +1,89 @@
+import { describeIf, TEST_PRODUCTION, TEST_SUITE_WEBSITE } from "../lib";
+import { Identity, idmux, scrapeTimeout, scrape, indexCooldown } from "./lib";
+import type { ScrapeRequestInput } from "../../../controllers/v2/types";
+import crypto from "crypto";
+
+describeIf(TEST_PRODUCTION)(
+ "V2 Scrape parsers[pdf].maxPages index caching",
+ () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "v2-parsers-maxpages-index-cache",
+ concurrency: 100,
+ credits: 1000000,
+ });
+ }, 10000);
+
+ test(
+ "caches a PDF whose page count fits under maxPages",
+ async () => {
+ // Unique query param: other suites index /example.pdf without
+ // maxPages, and index lookup keys ignore parser settings.
+ const url = `${TEST_SUITE_WEBSITE}/example.pdf?testId=${crypto.randomUUID()}`;
+ const body = {
+ url,
+ parsers: [{ type: "pdf", maxPages: 10000 }],
+ maxAge: scrapeTimeout * 2,
+ timeout: scrapeTimeout,
+ } satisfies ScrapeRequestInput;
+
+ const response1 = await scrape(body, identity);
+ expect(response1.metadata.cacheState).toBe("miss");
+
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
+
+ const response2 = await scrape(body, identity);
+ expect(response2.metadata.cacheState).toBe("hit");
+ },
+ scrapeTimeout * 2 + indexCooldown + 10000,
+ );
+
+ test(
+ "does not cache a PDF truncated by maxPages",
+ async () => {
+ const url = `${TEST_SUITE_WEBSITE}/example-long.pdf?testId=${crypto.randomUUID()}`;
+ const body = {
+ url,
+ parsers: [{ type: "pdf", maxPages: 3 }],
+ maxAge: scrapeTimeout * 2,
+ timeout: scrapeTimeout,
+ } satisfies ScrapeRequestInput;
+
+ const response1 = await scrape(body, identity);
+ expect(response1.metadata.cacheState).toBe("miss");
+ expect(response1.metadata.totalPages).toBeGreaterThan(3);
+ expect(response1.metadata.numPages).toBe(3);
+
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
+
+ const response2 = await scrape(body, identity);
+ expect(response2.metadata.cacheState).toBe("miss");
+ },
+ scrapeTimeout * 2 + indexCooldown + 10000,
+ );
+
+ test(
+ "caches HTML scrapes with a maxPages parser set",
+ async () => {
+ const url = `${TEST_SUITE_WEBSITE}?testId=${crypto.randomUUID()}`;
+ const body = {
+ url,
+ parsers: [{ type: "pdf", maxPages: 100 }],
+ maxAge: scrapeTimeout * 2,
+ timeout: scrapeTimeout,
+ } satisfies ScrapeRequestInput;
+
+ const response1 = await scrape(body, identity);
+ expect(response1.metadata.cacheState).toBe("miss");
+
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
+
+ const response2 = await scrape(body, identity);
+ expect(response2.metadata.cacheState).toBe("hit");
+ },
+ scrapeTimeout * 2 + indexCooldown + 10000,
+ );
+ },
+);
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index c0bbfd5e3..d2e652bb1 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -53,6 +53,15 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
screenshotFormat?.viewport !== undefined ||
screenshotFormat?.quality !== undefined;
+ // A PDF capped by maxPages is truncated, but one whose true page count
+ // fits under the cap is identical to an unlimited scrape and safe to cache.
+ const pdfMaxPages = getPDFMaxPages(meta.options.parsers);
+ const isTruncatedPdf =
+ pdfMaxPages !== undefined &&
+ document.metadata.contentType === "application/pdf" &&
+ (document.metadata.totalPages === undefined ||
+ document.metadata.totalPages > pdfMaxPages);
+
const shouldCache =
meta.options.storeInCache &&
!meta.internalOptions.isParse &&
@@ -70,16 +79,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
!getPDFPageMarkdown(meta.options.parsers) &&
!getPDFBlocks(meta.options.parsers) &&
!getPDFPageMarkers(meta.options.parsers) &&
- !meta.options.parsers?.some(parser => {
- if (
- typeof parser === "object" &&
- parser !== null &&
- "maxPages" in parser
- ) {
- return true;
- }
- return false;
- }) &&
+ !isTruncatedPdf &&
(meta.internalOptions.teamId === "sitemap" ||
(meta.winnerEngine !== "fire-engine;tlsclient" &&
meta.winnerEngine !== "fire-engine;tlsclient;stealth" &&