Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(pdf): report totalPages for truncated PDF scrapes (#3913)

firecrawl-spring[bot]81215679d69a
diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.ts
index bedc30d95..bd7b20681 100644
--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts
@@ -17,6 +17,7 @@ beforeAll(async () => {
describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {
const pdfUrl = `${TEST_SUITE_WEBSITE}/example.pdf`;
+ const longPdfUrl = `${TEST_SUITE_WEBSITE}/example-long.pdf`;
const htmlUrl = TEST_SUITE_WEBSITE;
describe("Array format", () => {
@@ -142,9 +143,46 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Parsers parameter tests", () => {
expect(response.markdown).toContain("PDF Test File");
expect(response.metadata.numPages).toBeGreaterThan(0);
expect(response.metadata.numPages).toBeLessThan(10000);
+ // When the cap exceeds the real page count nothing is truncated, so
+ // totalPages should be reported and equal numPages.
+ expect(response.metadata.totalPages).toBe(response.metadata.numPages);
},
scrapeTimeout * 2,
);
+
+ it.concurrent(
+ "reports totalPages when a PDF is truncated by maxPages",
+ async () => {
+ // example-long.pdf has many pages; cap well below the real count.
+ const truncated = await scrape(
+ {
+ url: longPdfUrl,
+ parsers: [{ type: "pdf", maxPages: 5 }],
+ },
+ identity,
+ );
+
+ expect(truncated.metadata.numPages).toBe(5);
+ // The true total is reported even though only 5 pages were parsed.
+ expect(truncated.metadata.totalPages).toBeDefined();
+ expect(truncated.metadata.totalPages!).toBeGreaterThan(
+ truncated.metadata.numPages!,
+ );
+
+ // The reported total should match an uncapped scrape's page count.
+ const full = await scrape(
+ {
+ url: longPdfUrl,
+ parsers: [{ type: "pdf" }],
+ },
+ identity,
+ );
+
+ expect(full.metadata.totalPages).toBe(full.metadata.numPages);
+ expect(truncated.metadata.totalPages).toBe(full.metadata.numPages);
+ },
+ scrapeTimeout * 10,
+ );
});
describe("Mode - object format", () => {
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index c1d1758e9..5a2f0eb69 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1073,6 +1073,7 @@ export type Document = {
scrapeId?: string;
error?: string;
numPages?: number;
+ totalPages?: number;
contentType?: string;
timezone?: string;
proxyUsed: "basic" | "stealth";
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 2111b486d..48894bd63 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1302,6 +1302,7 @@ export type Document = {
scrapeId?: string;
error?: string;
numPages?: number;
+ totalPages?: number;
contentType?: string;
timezone?: string;
proxyUsed: "basic" | "stealth";
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 784ab1e63..53012521b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -112,8 +112,9 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
pdfMetadata:
document.metadata.numPages !== undefined
? {
- // reconstruct pdfMetadata from numPages and title
+ // reconstruct pdfMetadata from numPages, totalPages and title
numPages: document.metadata.numPages,
+ totalPages: document.metadata.totalPages ?? undefined,
title: document.metadata.title ?? undefined,
}
: undefined,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index e2b442ce2..dec5be080 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -157,6 +157,9 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
let result: PDFProcessorResult | null = null;
let effectivePageCount: number = 0;
+ // True page count of the document before maxPages capping. Stays undefined
+ // when native detection fails, so it can be omitted from the response.
+ let totalPageCount: number | undefined;
let metadataTitle: string | undefined;
let rustMarkdownForShadow: string | undefined;
let shadowPdfType: string | undefined;
@@ -215,6 +218,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
mode,
});
+ totalPageCount = detection.pageCount;
effectivePageCount = maxPages
? Math.min(detection.pageCount, maxPages)
: detection.pageCount;
@@ -271,6 +275,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
mode,
});
+ totalPageCount = pdfResult.pageCount;
effectivePageCount = maxPages
? Math.min(pdfResult.pageCount, maxPages)
: pdfResult.pageCount;
@@ -606,6 +611,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
markdown: result?.markdown ?? "",
pdfMetadata: {
numPages: effectivePageCount,
+ totalPages: totalPageCount,
title: metadataTitle,
},
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
index 9284dd9d5..b04067b4c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
@@ -11,7 +11,18 @@ export type PDFProcessorResult = {
pagesProcessed?: number;
};
-export type PdfMetadata = { numPages: number; title?: string };
+export type PdfMetadata = {
+ /** Pages actually parsed for this request (capped by maxPages). */
+ numPages: number;
+ /**
+ * True page count of the document, before any maxPages capping. Omitted when
+ * the underlying engine couldn't determine it (e.g. native detection failed),
+ * so consumers must treat undefined as "no signal" rather than "not truncated".
+ * When present, `totalPages > numPages` means the result was truncated.
+ */
+ totalPages?: number;
+ title?: string;
+};
export const MAX_FILE_SIZE = 19 * 1024 * 1024; // 19MB
export const FIRE_PDF_MAX_FILE_SIZE = 30 * 1024 * 1024; // 30MB
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 207747925..72d624b6c 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -992,6 +992,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
statusCode: engineResult.statusCode,
error: engineResult.error,
numPages: engineResult.pdfMetadata?.numPages,
+ ...(engineResult.pdfMetadata?.totalPages !== undefined
+ ? { totalPages: engineResult.pdfMetadata.totalPages }
+ : {}),
...(engineResult.pdfMetadata?.title
? { title: engineResult.pdfMetadata.title }
: {}),
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index ff06fbdc8..5e58086f7 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.28.3",
+ "version": "4.28.4",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 9e39360d2..d7daab3df 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -570,6 +570,7 @@ export interface DocumentMetadata {
statusCode?: number;
scrapeId?: string;
numPages?: number;
+ totalPages?: number;
contentType?: string;
timezone?: string;
proxyUsed?: "basic" | "stealth";
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 2d29226b0..ec1e9e86b 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.30.3"
+__version__ = "4.30.4"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 816ea8f7c..4b0c31f86 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -123,6 +123,7 @@ class DocumentMetadata(BaseModel):
status_code: Optional[int] = None
scrape_id: Optional[str] = None
num_pages: Optional[int] = None
+ total_pages: Optional[int] = None
content_type: Optional[str] = None
proxy_used: Optional[Literal["basic", "stealth"]] = None
timezone: Optional[str] = None
@@ -211,6 +212,7 @@ class DocumentMetadata(BaseModel):
if isinstance(v, list) and k in {
"status_code",
"num_pages",
+ "total_pages",
"credits_used",
}:
first = v[0] if v else None
diff --git a/apps/python-sdk/firecrawl/v2/utils/normalize.py b/apps/python-sdk/firecrawl/v2/utils/normalize.py
index 19e849034..192f4be30 100644
--- a/apps/python-sdk/firecrawl/v2/utils/normalize.py
+++ b/apps/python-sdk/firecrawl/v2/utils/normalize.py
@@ -43,6 +43,7 @@ def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:
"statusCode": "status_code",
"scrapeId": "scrape_id",
"numPages": "num_pages",
+ "totalPages": "total_pages",
"contentType": "content_type",
"proxyUsed": "proxy_used",
"cacheState": "cache_state",