Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrapeURL/index): allow index lookup with parsers[pdf].maxPages set (#4300)

Gergő Móricz37d0db5f28a1
diff --git a/.gitignore b/.gitignore
index 47dcc2df7..7d2396ed9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -60,4 +60,5 @@ target/
_local/
-apps/api/src/lib/scrape-interact/*.md
\ No newline at end of file
+apps/api/src/lib/scrape-interact/*.md
+.pnpm-store
diff --git a/.pnpm-store/v11/.pnpm-needs-build-marker b/.pnpm-store/v11/.pnpm-needs-build-marker
deleted file mode 100644
index e69de29bb..000000000
diff --git a/.pnpm-store/v11/index.db b/.pnpm-store/v11/index.db
deleted file mode 100644
index d583c087b..000000000
Binary files a/.pnpm-store/v11/index.db and /dev/null differ
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 5d1563a59..224efaf76 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -27,10 +27,7 @@ import {
} from "./x-twitter";
import { queryEngpickerVerdict, useIndex } from "../../../services";
import { hasFormatOfType } from "../../../lib/format-utils";
-import {
- getPDFMaxPages,
- getPDFPageMarkdown,
-} from "../../../controllers/v2/types";
+import { getPDFPageMarkdown } from "../../../controllers/v2/types";
import type { PdfMetadata } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
import { BrandingNotSupportedError } from "../error";
@@ -565,8 +562,6 @@ export function shouldUseIndex(meta: Meta) {
config.FIRECRAWL_INDEX_WRITE_ONLY !== true &&
!hasFormatOfType(meta.options.formats, "changeTracking") &&
!hasFormatOfType(meta.options.formats, "branding") &&
- // Skip index if a non-default PDF maxPages is specified
- getPDFMaxPages(meta.options.parsers) === undefined &&
// The URL index does not yet persist physical-page capability metadata.
!getPDFPageMarkdown(meta.options.parsers) &&
!hasCustomScreenshotSettings &&
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index f282f9f33..08f2ba413 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -37,6 +37,7 @@ import {
NoCachedDataError,
} from "../../error";
import {
+ getPDFMaxPages,
getPDFPageMarkdown,
shouldParsePDF,
} from "../../../../controllers/v2/types";
@@ -549,6 +550,18 @@ export async function scrapeURLWithIndex(
}
}
+ // Check if returned PDF has a higher numPages than what the user's parsers[pdf].maxPages config allows.
+ let numPages = doc.pdfMetadata?.numPages ?? doc.numPages;
+ if (numPages !== undefined) {
+ let maxPages = getPDFMaxPages(meta.options.parsers);
+ if (maxPages !== undefined && numPages > maxPages) {
+ logLookup("debug", "hit", {
+ pdfMismatch: "cached_pdf_overflows_parsers_max_pages",
+ });
+ throw new IndexMissError();
+ }
+ }
+
logLookup("debug", "hit", {
age: Date.now() - new Date(selectedRow.created_at).getTime(),
status: selectedRow.status,