Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/index): allow index lookup with parsers[pdf].maxPages set (#4300)
Gergő Móricz37d0db5f28a1
diff --git a/.gitignore b/.gitignoreindex 47dcc2df7..7d2396ed9 100644--- a/.gitignore+++ b/.gitignore@@ -60,4 +60,5 @@ target/_local/-apps/api/src/lib/scrape-interact/*.md\ No newline at end of file+apps/api/src/lib/scrape-interact/*.md+.pnpm-storediff --git a/.pnpm-store/v11/.pnpm-needs-build-marker b/.pnpm-store/v11/.pnpm-needs-build-markerdeleted file mode 100644index e69de29bb..000000000diff --git a/.pnpm-store/v11/index.db b/.pnpm-store/v11/index.dbdeleted file mode 100644index d583c087b..000000000Binary files a/.pnpm-store/v11/index.db and /dev/null differdiff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 5d1563a59..224efaf76 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -27,10 +27,7 @@ import {} from "./x-twitter";import { queryEngpickerVerdict, useIndex } from "../../../services";import { hasFormatOfType } from "../../../lib/format-utils";-import {- getPDFMaxPages,- getPDFPageMarkdown,-} from "../../../controllers/v2/types";+import { getPDFPageMarkdown } from "../../../controllers/v2/types";import type { PdfMetadata } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";import { BrandingNotSupportedError } from "../error";@@ -565,8 +562,6 @@ export function shouldUseIndex(meta: Meta) {config.FIRECRAWL_INDEX_WRITE_ONLY !== true &&!hasFormatOfType(meta.options.formats, "changeTracking") &&!hasFormatOfType(meta.options.formats, "branding") &&- // Skip index if a non-default PDF maxPages is specified- getPDFMaxPages(meta.options.parsers) === undefined &&// The URL index does not yet persist physical-page capability metadata.!getPDFPageMarkdown(meta.options.parsers) &&!hasCustomScreenshotSettings &&diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex f282f9f33..08f2ba413 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -37,6 +37,7 @@ import {NoCachedDataError,} from "../../error";import {+ getPDFMaxPages,getPDFPageMarkdown,shouldParsePDF,} from "../../../../controllers/v2/types";@@ -549,6 +550,18 @@ export async function scrapeURLWithIndex(}}+ // Check if returned PDF has a higher numPages than what the user's parsers[pdf].maxPages config allows.+ let numPages = doc.pdfMetadata?.numPages ?? doc.numPages;+ if (numPages !== undefined) {+ let maxPages = getPDFMaxPages(meta.options.parsers);+ if (maxPages !== undefined && numPages > maxPages) {+ logLookup("debug", "hit", {+ pdfMismatch: "cached_pdf_overflows_parsers_max_pages",+ });+ throw new IndexMissError();+ }+ }+logLookup("debug", "hit", {age: Date.now() - new Date(selectedRow.created_at).getTime(),status: selectedRow.status,