Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): write index entries for maxPages requests when the PDF was not truncated (#4698)
Gergő Móricz7b0b0b996008
diff --git a/apps/api/src/__tests__/snips/v2/parsers-maxpages-index-cache.test.ts b/apps/api/src/__tests__/snips/v2/parsers-maxpages-index-cache.test.tsnew file mode 100644index 000000000..83e9cc872--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/parsers-maxpages-index-cache.test.ts@@ -0,0 +1,89 @@+import { describeIf, TEST_PRODUCTION, TEST_SUITE_WEBSITE } from "../lib";+import { Identity, idmux, scrapeTimeout, scrape, indexCooldown } from "./lib";+import type { ScrapeRequestInput } from "../../../controllers/v2/types";+import crypto from "crypto";++describeIf(TEST_PRODUCTION)(+ "V2 Scrape parsers[pdf].maxPages index caching",+ () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "v2-parsers-maxpages-index-cache",+ concurrency: 100,+ credits: 1000000,+ });+ }, 10000);++ test(+ "caches a PDF whose page count fits under maxPages",+ async () => {+ // Unique query param: other suites index /example.pdf without+ // maxPages, and index lookup keys ignore parser settings.+ const url = `${TEST_SUITE_WEBSITE}/example.pdf?testId=${crypto.randomUUID()}`;+ const body = {+ url,+ parsers: [{ type: "pdf", maxPages: 10000 }],+ maxAge: scrapeTimeout * 2,+ timeout: scrapeTimeout,+ } satisfies ScrapeRequestInput;++ const response1 = await scrape(body, identity);+ expect(response1.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, indexCooldown));++ const response2 = await scrape(body, identity);+ expect(response2.metadata.cacheState).toBe("hit");+ },+ scrapeTimeout * 2 + indexCooldown + 10000,+ );++ test(+ "does not cache a PDF truncated by maxPages",+ async () => {+ const url = `${TEST_SUITE_WEBSITE}/example-long.pdf?testId=${crypto.randomUUID()}`;+ const body = {+ url,+ parsers: [{ type: "pdf", maxPages: 3 }],+ maxAge: scrapeTimeout * 2,+ timeout: scrapeTimeout,+ } satisfies ScrapeRequestInput;++ const response1 = await scrape(body, identity);+ expect(response1.metadata.cacheState).toBe("miss");+ expect(response1.metadata.totalPages).toBeGreaterThan(3);+ expect(response1.metadata.numPages).toBe(3);++ await new Promise(resolve => setTimeout(resolve, indexCooldown));++ const response2 = await scrape(body, identity);+ expect(response2.metadata.cacheState).toBe("miss");+ },+ scrapeTimeout * 2 + indexCooldown + 10000,+ );++ test(+ "caches HTML scrapes with a maxPages parser set",+ async () => {+ const url = `${TEST_SUITE_WEBSITE}?testId=${crypto.randomUUID()}`;+ const body = {+ url,+ parsers: [{ type: "pdf", maxPages: 100 }],+ maxAge: scrapeTimeout * 2,+ timeout: scrapeTimeout,+ } satisfies ScrapeRequestInput;++ const response1 = await scrape(body, identity);+ expect(response1.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, indexCooldown));++ const response2 = await scrape(body, identity);+ expect(response2.metadata.cacheState).toBe("hit");+ },+ scrapeTimeout * 2 + indexCooldown + 10000,+ );+ },+);diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex c0bbfd5e3..d2e652bb1 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -53,6 +53,15 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {screenshotFormat?.viewport !== undefined ||screenshotFormat?.quality !== undefined;+ // A PDF capped by maxPages is truncated, but one whose true page count+ // fits under the cap is identical to an unlimited scrape and safe to cache.+ const pdfMaxPages = getPDFMaxPages(meta.options.parsers);+ const isTruncatedPdf =+ pdfMaxPages !== undefined &&+ document.metadata.contentType === "application/pdf" &&+ (document.metadata.totalPages === undefined ||+ document.metadata.totalPages > pdfMaxPages);+const shouldCache =meta.options.storeInCache &&!meta.internalOptions.isParse &&@@ -70,16 +79,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {!getPDFPageMarkdown(meta.options.parsers) &&!getPDFBlocks(meta.options.parsers) &&!getPDFPageMarkers(meta.options.parsers) &&- !meta.options.parsers?.some(parser => {- if (- typeof parser === "object" &&- parser !== null &&- "maxPages" in parser- ) {- return true;- }- return false;- }) &&+ !isTruncatedPdf &&(meta.internalOptions.teamId === "sitemap" ||(meta.winnerEngine !== "fire-engine;tlsclient" &&meta.winnerEngine !== "fire-engine;tlsclient;stealth" &&