Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: add maxPages parameter to PDF parser in v2 scrape API (#2047)

devin-ai-integration[bot]7bea613ec093
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 2e2d9b182..5f121056b 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -272,6 +272,7 @@ describe("Crawl tests", () => {
}, 8 * scrapeTimeout);
});
}
+
});
describe("Robots.txt FFI Integration tests", () => {
diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.ts
index b9257f5cf..303172a6f 100644
--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts
@@ -57,6 +57,42 @@ describe("Parsers parameter tests", () => {
}, scrapeTimeout);
});
+ describe("Object format", () => {
+ it.concurrent("accepts parsers: [{type: 'pdf'}] and parses PDF", async () => {
+ const response = await scrape({
+ url: pdfUrl,
+ parsers: [{ type: "pdf" }],
+ }, identity);
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.numPages).toBeGreaterThan(0);
+ }, scrapeTimeout * 2);
+
+ it.concurrent("accepts parsers: [{type: 'pdf', maxPages: 1}] and limits pages", async () => {
+ const response = await scrape({
+ url: pdfUrl,
+ parsers: [{ type: "pdf", maxPages: 1 }],
+ }, identity);
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.numPages).toBe(1);
+ }, scrapeTimeout * 2);
+
+ it.concurrent("handles maxPages larger than actual pages", async () => {
+ const response = await scrape({
+ url: pdfUrl,
+ parsers: [{ type: "pdf", maxPages: 10000 }],
+ }, identity);
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.numPages).toBeGreaterThan(0);
+ expect(response.metadata.numPages).toBeLessThan(10000);
+ }, scrapeTimeout * 2);
+ });
+
describe("Default behavior", () => {
it.concurrent("parses PDF by default when parsers not specified", async () => {
@@ -103,6 +139,28 @@ describe("Parsers parameter tests", () => {
expect(raw.body.success).toBe(false);
expect(raw.body.error).toBe("Bad Request");
}, scrapeTimeout);
+
+ it.concurrent("rejects negative maxPages", async () => {
+ const raw = await scrapeRaw({
+ url: pdfUrl,
+ parsers: [{ type: "pdf", maxPages: -1 }],
+ }, identity);
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ }, scrapeTimeout);
+
+ it.concurrent("rejects maxPages over limit", async () => {
+ const raw = await scrapeRaw({
+ url: pdfUrl,
+ parsers: [{ type: "pdf", maxPages: 10001 }],
+ }, identity);
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ }, scrapeTimeout);
});
describe("Billing implications", () => {
@@ -125,5 +183,16 @@ describe("Parsers parameter tests", () => {
// Should bill flat rate (1 credit) when PDF parsing is disabled
expect(response.metadata.creditsUsed).toBe(1);
}, scrapeTimeout * 2);
+
+ it.concurrent("bills based on limited pages with maxPages", async () => {
+ const response = await scrape({
+ url: pdfUrl,
+ parsers: [{ type: "pdf", maxPages: 1 }],
+ }, identity);
+
+ // Should bill based on limited pages (1 page = 1 credit)
+ expect(response.metadata.creditsUsed).toBe(1);
+ expect(response.metadata.numPages).toBe(1);
+ }, scrapeTimeout * 2);
});
-});
\ No newline at end of file
+});
diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.ts
index d1a880b23..ca61d49e5 100644
--- a/apps/api/src/__tests__/snips/v2/search.test.ts
+++ b/apps/api/src/__tests__/snips/v2/search.test.ts
@@ -33,4 +33,5 @@ describe("Search tests", () => {
expect(doc.markdown).toBeDefined();
}
}, 125000);
+
});
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index f7057df56..e670df22b 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -235,10 +235,47 @@ export type FormatObject =
| ScreenshotFormatWithOptions
| AttributesFormatWithOptions
-export const parsersSchema = z.array(z.enum(["pdf"])).default(["pdf"]);
+export const pdfParserWithOptions = z.object({
+ type: z.literal("pdf"),
+ maxPages: z.number().int().positive().finite().max(10000).optional(),
+}).strict();
+
+export const parsersSchema = z
+ .array(
+ z.union([
+ z.literal("pdf"),
+ pdfParserWithOptions,
+ ])
+ )
+ .default(["pdf"]);
export type Parsers = z.infer<typeof parsersSchema>;
+export function shouldParsePDF(parsers?: Parsers): boolean {
+ if (!parsers) return true;
+ return parsers.some(parser => {
+ if (parser === "pdf") return true;
+ if (typeof parser === "object" && parser !== null && "type" in parser) {
+ return (parser as any).type === "pdf";
+ }
+ return false;
+ });
+}
+
+export function getPDFMaxPages(parsers?: Parsers): number | undefined {
+ if (!parsers) return undefined;
+ const pdfParser = parsers.find(parser => {
+ if (typeof parser === "object" && parser !== null && "type" in parser) {
+ return (parser as any).type === "pdf";
+ }
+ return false;
+ });
+ if (pdfParser && typeof pdfParser === "object" && "maxPages" in pdfParser) {
+ return (pdfParser as any).maxPages;
+ }
+ return undefined;
+}
+
function transformIframeSelector(selector: string): string {
return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, (match) => {
const prefix = match.match(/^[\s,]/)?.[0] || '';
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index ec47a82c5..2e82754e6 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -1,5 +1,5 @@
import { InternalOptions } from "src/scraper/scrapeURL";
-import { Document, ScrapeOptions, TeamFlags } from "../controllers/v2/types";
+import { Document, ScrapeOptions, TeamFlags, shouldParsePDF } from "../controllers/v2/types";
import { CostTracking } from "./extract/extraction-service";
import { hasFormatOfType } from "./format-utils";
@@ -34,8 +34,8 @@ export async function calculateCreditsToBeBilled(options: ScrapeOptions, interna
creditsToBeBilled += (flags?.zdrCost ?? 1);
}
- const shouldParsePDF = options.parsers?.includes("pdf") ?? true;
- if (shouldParsePDF && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) {
+ const shouldParse = shouldParsePDF(options.parsers);
+ if (shouldParse && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) {
creditsToBeBilled += creditsPerPDFPage * (document.metadata.numPages - 1);
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index b076da894..fbd7ec8bf 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -3,6 +3,7 @@ import { EngineScrapeResult } from "..";
import { Meta } from "../..";
import { getIndexFromGCS, hashURL, index_supabase_service, normalizeURLForIndex, saveIndexToGCS, generateURLSplits, addIndexInsertJob, generateDomainSplits, addOMCEJob, addDomainFrequencyJob } from "../../../../services";
import { EngineError, IndexMissError } from "../../error";
+import { shouldParsePDF } from "../../../../controllers/v2/types";
import crypto from "crypto";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
@@ -10,7 +11,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
&& !meta.internalOptions.zeroDataRetention
&& meta.winnerEngine !== "index"
&& meta.winnerEngine !== "index;documents"
- && !(meta.winnerEngine === "pdf" && meta.options.parsers?.includes("pdf") === false)
+ && !(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers))
&& (
meta.internalOptions.teamId === "sitemap"
|| (
@@ -249,13 +250,13 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult
const isCachedPdfBase64 = doc.html && doc.html.startsWith("JVBERi");
// If the cached content is base64 PDF but we want parsed PDF (parsePDF:true or default)
- if (isCachedPdfBase64 && meta.options.parsers?.includes("pdf") !== false) {
+ if (isCachedPdfBase64 && shouldParsePDF(meta.options.parsers)) {
// Cached content is unparsed PDF, but we want parsed - report cache miss
throw new IndexMissError();
}
// If the cached content is NOT base64 PDF but we want unparsed PDF (parsePDF:false)
- if (!isCachedPdfBase64 && meta.options.parsers?.includes("pdf") === false) {
+ if (!isCachedPdfBase64 && !shouldParsePDF(meta.options.parsers)) {
// Check if URL looks like a PDF
const isPdfUrl = meta.url.toLowerCase().endsWith(".pdf") || meta.url.includes(".pdf?");
if (isPdfUrl) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 4f726e62c..85ae95673 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -19,6 +19,7 @@ import type { Response } from "undici";
import { getPDFMetadata } from "../../../../lib/pdf-parser";
import { getPdfResultFromCache, savePdfResultToCache } from "../../../../lib/gcs-pdf-cache";
import { AbortManagerThrownError } from "../../lib/abortManager";
+import { shouldParsePDF, getPDFMaxPages } from "../../../../controllers/v2/types";
type PDFProcessorResult = { html: string; markdown?: string };
@@ -29,6 +30,7 @@ async function scrapePDFWithRunPodMU(
meta: Meta,
tempFilePath: string,
base64Content: string,
+ maxPages?: number,
): Promise<PDFProcessorResult> {
meta.logger.debug("Processing PDF document with RunPod MU", {
tempFilePath,
@@ -66,6 +68,7 @@ async function scrapePDFWithRunPodMU(
filename: path.basename(tempFilePath) + ".pdf",
timeout: meta.abort.scrapeTimeout(),
created_at: Date.now(),
+ ...(maxPages !== undefined && { max_pages: maxPages }),
},
},
logger: meta.logger.child({
@@ -162,9 +165,10 @@ async function scrapePDFWithParsePDF(
export async function scrapePDF(
meta: Meta,
): Promise<EngineScrapeResult> {
- const shouldParsePDF = meta.options.parsers?.includes("pdf") ?? true;
+ const shouldParse = shouldParsePDF(meta.options.parsers);
+ const maxPages = getPDFMaxPages(meta.options.parsers);
- if (!shouldParsePDF) {
+ if (!shouldParse) {
if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {
const content = (await readFile(meta.pdfPrefetch.filePath)).toString(
"base64",
@@ -228,11 +232,12 @@ export async function scrapePDF(
}
const pdfMetadata = await getPDFMetadata(tempFilePath);
+ const effectivePageCount = maxPages ? Math.min(pdfMetadata.numPages, maxPages) : pdfMetadata.numPages;
- if (pdfMetadata.numPages * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) {
+ if (effectivePageCount * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) {
throw new PDFInsufficientTimeError(
- pdfMetadata.numPages,
- pdfMetadata.numPages * MILLISECONDS_PER_PAGE + 5000,
+ effectivePageCount,
+ effectivePageCount * MILLISECONDS_PER_PAGE + 5000,
);
}
@@ -256,6 +261,7 @@ export async function scrapePDF(
},
tempFilePath,
base64Content,
+ maxPages,
);
} catch (error) {
if (
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index f9abd453c..313af37ff 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.0.0",
+ "version": "4.1.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 56d83cfd6..1139df6f0 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -133,7 +133,7 @@ export interface ScrapeOptions {
timeout?: number;
waitFor?: number;
mobile?: boolean;
- parsers?: string[];
+ parsers?: Array<string | { type: "pdf"; maxPages?: number }>;
actions?: ActionOption[];
location?: LocationConfig;
skipTlsVerification?: boolean;
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 54b89235e..55e843736 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.0.0"
+__version__ = "4.1.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 3c37afc0e..a4bc2f60b 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -18,6 +18,7 @@ from .types import (
CrawlResponse,
CrawlJob,
CrawlParamsRequest,
+ PDFParser,
CrawlParamsData,
WebhookConfig,
CrawlErrorsResponse,
@@ -105,7 +106,7 @@ class FirecrawlClient:
timeout: Optional[int] = None,
wait_for: Optional[int] = None,
mobile: Optional[bool] = None,
- parsers: Optional[List[str]] = None,
+ parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,
actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,
location: Optional['Location'] = None,
skip_tls_verification: Optional[bool] = None,
@@ -571,7 +572,7 @@ class FirecrawlClient:
timeout: Optional[int] = None,
wait_for: Optional[int] = None,
mobile: Optional[bool] = None,
- parsers: Optional[List[str]] = None,
+ parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,
actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,
location: Optional['Location'] = None,
skip_tls_verification: Optional[bool] = None,
@@ -759,7 +760,7 @@ class FirecrawlClient:
timeout: Optional[int] = None,
wait_for: Optional[int] = None,
mobile: Optional[bool] = None,
- parsers: Optional[List[str]] = None,
+ parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,
actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,
location: Optional['Location'] = None,
skip_tls_verification: Optional[bool] = None,
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 319162e14..411295d8e 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -278,7 +278,7 @@ class ScrapeOptions(BaseModel):
timeout: Optional[int] = None
wait_for: Optional[int] = None
mobile: Optional[bool] = None
- parsers: Optional[List[str]] = None
+ parsers: Optional[Union[List[str], List[Union[str, 'PDFParser']]]] = None
actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None
location: Optional['Location'] = None
skip_tls_verification: Optional[bool] = None
@@ -536,6 +536,11 @@ class PDFAction(BaseModel):
landscape: Optional[bool] = None
scale: Optional[float] = None
+class PDFParser(BaseModel):
+ """PDF parser configuration with optional page limit."""
+ type: Literal["pdf"] = "pdf"
+ max_pages: Optional[int] = None
+
# Location types
class Location(BaseModel):
"""Location configuration for scraping."""
@@ -594,6 +599,26 @@ class SearchRequest(BaseModel):
return normalized_categories
+ @field_validator('parsers')
+ @classmethod
+ def validate_parsers(cls, v):
+ """Validate and normalize parsers input."""
+ if v is None:
+ return v
+
+ normalized_parsers = []
+ for parser in v:
+ if isinstance(parser, str):
+ normalized_parsers.append(parser)
+ elif isinstance(parser, dict):
+ normalized_parsers.append(PDFParser(**parser))
+ elif isinstance(parser, PDFParser):
+ normalized_parsers.append(parser)
+ else:
+ raise ValueError(f"Invalid parser format: {parser}")
+
+ return normalized_parsers
+
class LinkResult(BaseModel):
"""A generic link result with optional metadata (used by search and map)."""
url: str
@@ -686,4 +711,4 @@ AnyResponse = Union[
MapResponse,
SearchResponse,
ErrorResponse,
-]
\ No newline at end of file
+]
diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.py
index f380a02cf..075ad9a2a 100644
--- a/apps/python-sdk/firecrawl/v2/utils/validation.py
+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py
@@ -311,6 +311,20 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
converted_action[action_key] = action_value
converted_actions.append(converted_action)
scrape_data["actions"] = converted_actions
+ elif key == "parsers":
+ converted_parsers = []
+ for parser in value:
+ if isinstance(parser, str):
+ converted_parsers.append(parser)
+ elif isinstance(parser, dict):
+ converted_parsers.append(parser)
+ else:
+ parser_data = parser.model_dump(exclude_none=True)
+ # Convert snake_case to camelCase for API
+ if "max_pages" in parser_data:
+ parser_data["maxPages"] = parser_data.pop("max_pages")
+ converted_parsers.append(parser_data)
+ scrape_data["parsers"] = converted_parsers
elif key == "location":
# Handle location conversion
if isinstance(value, dict):
@@ -321,4 +335,4 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
# For fields that don't need conversion, use as-is
scrape_data[key] = value
- return scrape_data
\ No newline at end of file
+ return scrape_data
\ No newline at end of file