Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf): report totalPages for truncated PDF scrapes (#3913)
firecrawl-spring[bot]81215679d69a
diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.tsindex bedc30d95..bd7b20681 100644--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts@@ -17,6 +17,7 @@ beforeAll(async () => {describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {const pdfUrl = `${TEST_SUITE_WEBSITE}/example.pdf`;+ const longPdfUrl = `${TEST_SUITE_WEBSITE}/example-long.pdf`;const htmlUrl = TEST_SUITE_WEBSITE;describe("Array format", () => {@@ -142,9 +143,46 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {expect(response.markdown).toContain("PDF Test File");expect(response.metadata.numPages).toBeGreaterThan(0);expect(response.metadata.numPages).toBeLessThan(10000);+ // When the cap exceeds the real page count nothing is truncated, so+ // totalPages should be reported and equal numPages.+ expect(response.metadata.totalPages).toBe(response.metadata.numPages);},scrapeTimeout * 2,);++ it.concurrent(+ "reports totalPages when a PDF is truncated by maxPages",+ async () => {+ // example-long.pdf has many pages; cap well below the real count.+ const truncated = await scrape(+ {+ url: longPdfUrl,+ parsers: [{ type: "pdf", maxPages: 5 }],+ },+ identity,+ );++ expect(truncated.metadata.numPages).toBe(5);+ // The true total is reported even though only 5 pages were parsed.+ expect(truncated.metadata.totalPages).toBeDefined();+ expect(truncated.metadata.totalPages!).toBeGreaterThan(+ truncated.metadata.numPages!,+ );++ // The reported total should match an uncapped scrape's page count.+ const full = await scrape(+ {+ url: longPdfUrl,+ parsers: [{ type: "pdf" }],+ },+ identity,+ );++ expect(full.metadata.totalPages).toBe(full.metadata.numPages);+ expect(truncated.metadata.totalPages).toBe(full.metadata.numPages);+ },+ scrapeTimeout * 10,+ );});describe("Mode - object format", () => {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex c1d1758e9..5a2f0eb69 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1073,6 +1073,7 @@ export type Document = {scrapeId?: string;error?: string;numPages?: number;+ totalPages?: number;contentType?: string;timezone?: string;proxyUsed: "basic" | "stealth";diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 2111b486d..48894bd63 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1302,6 +1302,7 @@ export type Document = {scrapeId?: string;error?: string;numPages?: number;+ totalPages?: number;contentType?: string;timezone?: string;proxyUsed: "basic" | "stealth";diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 784ab1e63..53012521b 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -112,8 +112,9 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {pdfMetadata:document.metadata.numPages !== undefined? {- // reconstruct pdfMetadata from numPages and title+ // reconstruct pdfMetadata from numPages, totalPages and titlenumPages: document.metadata.numPages,+ totalPages: document.metadata.totalPages ?? undefined,title: document.metadata.title ?? undefined,}: undefined,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex e2b442ce2..dec5be080 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -157,6 +157,9 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {let result: PDFProcessorResult | null = null;let effectivePageCount: number = 0;+ // True page count of the document before maxPages capping. Stays undefined+ // when native detection fails, so it can be omitted from the response.+ let totalPageCount: number | undefined;let metadataTitle: string | undefined;let rustMarkdownForShadow: string | undefined;let shadowPdfType: string | undefined;@@ -215,6 +218,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {mode,});+ totalPageCount = detection.pageCount;effectivePageCount = maxPages? Math.min(detection.pageCount, maxPages): detection.pageCount;@@ -271,6 +275,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {mode,});+ totalPageCount = pdfResult.pageCount;effectivePageCount = maxPages? Math.min(pdfResult.pageCount, maxPages): pdfResult.pageCount;@@ -606,6 +611,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {markdown: result?.markdown ?? "",pdfMetadata: {numPages: effectivePageCount,+ totalPages: totalPageCount,title: metadataTitle,},diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.tsindex 9284dd9d5..b04067b4c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts@@ -11,7 +11,18 @@ export type PDFProcessorResult = {pagesProcessed?: number;};-export type PdfMetadata = { numPages: number; title?: string };+export type PdfMetadata = {+ /** Pages actually parsed for this request (capped by maxPages). */+ numPages: number;+ /**+ * True page count of the document, before any maxPages capping. Omitted when+ * the underlying engine couldn't determine it (e.g. native detection failed),+ * so consumers must treat undefined as "no signal" rather than "not truncated".+ * When present, `totalPages > numPages` means the result was truncated.+ */+ totalPages?: number;+ title?: string;+};export const MAX_FILE_SIZE = 19 * 1024 * 1024; // 19MBexport const FIRE_PDF_MAX_FILE_SIZE = 30 * 1024 * 1024; // 30MBdiff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 207747925..72d624b6c 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -992,6 +992,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {statusCode: engineResult.statusCode,error: engineResult.error,numPages: engineResult.pdfMetadata?.numPages,+ ...(engineResult.pdfMetadata?.totalPages !== undefined+ ? { totalPages: engineResult.pdfMetadata.totalPages }+ : {}),...(engineResult.pdfMetadata?.title? { title: engineResult.pdfMetadata.title }: {}),diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex ff06fbdc8..5e58086f7 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.28.3",+ "version": "4.28.4","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 9e39360d2..d7daab3df 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -570,6 +570,7 @@ export interface DocumentMetadata {statusCode?: number;scrapeId?: string;numPages?: number;+ totalPages?: number;contentType?: string;timezone?: string;proxyUsed?: "basic" | "stealth";diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 2d29226b0..ec1e9e86b 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.30.3"+__version__ = "4.30.4"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 816ea8f7c..4b0c31f86 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -123,6 +123,7 @@ class DocumentMetadata(BaseModel):status_code: Optional[int] = Nonescrape_id: Optional[str] = Nonenum_pages: Optional[int] = None+ total_pages: Optional[int] = Nonecontent_type: Optional[str] = Noneproxy_used: Optional[Literal["basic", "stealth"]] = Nonetimezone: Optional[str] = None@@ -211,6 +212,7 @@ class DocumentMetadata(BaseModel):if isinstance(v, list) and k in {"status_code","num_pages",+ "total_pages","credits_used",}:first = v[0] if v else Nonediff --git a/apps/python-sdk/firecrawl/v2/utils/normalize.py b/apps/python-sdk/firecrawl/v2/utils/normalize.pyindex 19e849034..192f4be30 100644--- a/apps/python-sdk/firecrawl/v2/utils/normalize.py+++ b/apps/python-sdk/firecrawl/v2/utils/normalize.py@@ -43,6 +43,7 @@ def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:"statusCode": "status_code","scrapeId": "scrape_id","numPages": "num_pages",+ "totalPages": "total_pages","contentType": "content_type","proxyUsed": "proxy_used","cacheState": "cache_state",