Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: add maxPages parameter to PDF parser in v2 scrape API (#2047)
devin-ai-integration[bot]7bea613ec093
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 2e2d9b182..5f121056b 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -272,6 +272,7 @@ describe("Crawl tests", () => {}, 8 * scrapeTimeout);});}+});describe("Robots.txt FFI Integration tests", () => {diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.tsindex b9257f5cf..303172a6f 100644--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts@@ -57,6 +57,42 @@ describe("Parsers parameter tests", () => {}, scrapeTimeout);});+ describe("Object format", () => {+ it.concurrent("accepts parsers: [{type: 'pdf'}] and parses PDF", async () => {+ const response = await scrape({+ url: pdfUrl,+ parsers: [{ type: "pdf" }],+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.numPages).toBeGreaterThan(0);+ }, scrapeTimeout * 2);++ it.concurrent("accepts parsers: [{type: 'pdf', maxPages: 1}] and limits pages", async () => {+ const response = await scrape({+ url: pdfUrl,+ parsers: [{ type: "pdf", maxPages: 1 }],+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.numPages).toBe(1);+ }, scrapeTimeout * 2);++ it.concurrent("handles maxPages larger than actual pages", async () => {+ const response = await scrape({+ url: pdfUrl,+ parsers: [{ type: "pdf", maxPages: 10000 }],+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.numPages).toBeGreaterThan(0);+ expect(response.metadata.numPages).toBeLessThan(10000);+ }, scrapeTimeout * 2);+ });+describe("Default behavior", () => {it.concurrent("parses PDF by default when parsers not specified", async () => {@@ -103,6 +139,28 @@ describe("Parsers parameter tests", () => {expect(raw.body.success).toBe(false);expect(raw.body.error).toBe("Bad Request");}, scrapeTimeout);++ it.concurrent("rejects negative maxPages", async () => {+ const raw = await scrapeRaw({+ url: pdfUrl,+ parsers: [{ type: "pdf", maxPages: -1 }],+ }, identity);++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ }, scrapeTimeout);++ it.concurrent("rejects maxPages over limit", async () => {+ const raw = await scrapeRaw({+ url: pdfUrl,+ parsers: [{ type: "pdf", maxPages: 10001 }],+ }, identity);++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ }, scrapeTimeout);});describe("Billing implications", () => {@@ -125,5 +183,16 @@ describe("Parsers parameter tests", () => {// Should bill flat rate (1 credit) when PDF parsing is disabledexpect(response.metadata.creditsUsed).toBe(1);}, scrapeTimeout * 2);++ it.concurrent("bills based on limited pages with maxPages", async () => {+ const response = await scrape({+ url: pdfUrl,+ parsers: [{ type: "pdf", maxPages: 1 }],+ }, identity);++ // Should bill based on limited pages (1 page = 1 credit)+ expect(response.metadata.creditsUsed).toBe(1);+ expect(response.metadata.numPages).toBe(1);+ }, scrapeTimeout * 2);});-});\ No newline at end of file+});diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.tsindex d1a880b23..ca61d49e5 100644--- a/apps/api/src/__tests__/snips/v2/search.test.ts+++ b/apps/api/src/__tests__/snips/v2/search.test.ts@@ -33,4 +33,5 @@ describe("Search tests", () => {expect(doc.markdown).toBeDefined();}}, 125000);+});diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex f7057df56..e670df22b 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -235,10 +235,47 @@ export type FormatObject =| ScreenshotFormatWithOptions| AttributesFormatWithOptions-export const parsersSchema = z.array(z.enum(["pdf"])).default(["pdf"]);+export const pdfParserWithOptions = z.object({+ type: z.literal("pdf"),+ maxPages: z.number().int().positive().finite().max(10000).optional(),+}).strict();++export const parsersSchema = z+ .array(+ z.union([+ z.literal("pdf"),+ pdfParserWithOptions,+ ])+ )+ .default(["pdf"]);export type Parsers = z.infer<typeof parsersSchema>;+export function shouldParsePDF(parsers?: Parsers): boolean {+ if (!parsers) return true;+ return parsers.some(parser => {+ if (parser === "pdf") return true;+ if (typeof parser === "object" && parser !== null && "type" in parser) {+ return (parser as any).type === "pdf";+ }+ return false;+ });+}++export function getPDFMaxPages(parsers?: Parsers): number | undefined {+ if (!parsers) return undefined;+ const pdfParser = parsers.find(parser => {+ if (typeof parser === "object" && parser !== null && "type" in parser) {+ return (parser as any).type === "pdf";+ }+ return false;+ });+ if (pdfParser && typeof pdfParser === "object" && "maxPages" in pdfParser) {+ return (pdfParser as any).maxPages;+ }+ return undefined;+}+function transformIframeSelector(selector: string): string {return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, (match) => {const prefix = match.match(/^[\s,]/)?.[0] || '';diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex ec47a82c5..2e82754e6 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -1,5 +1,5 @@import { InternalOptions } from "src/scraper/scrapeURL";-import { Document, ScrapeOptions, TeamFlags } from "../controllers/v2/types";+import { Document, ScrapeOptions, TeamFlags, shouldParsePDF } from "../controllers/v2/types";import { CostTracking } from "./extract/extraction-service";import { hasFormatOfType } from "./format-utils";@@ -34,8 +34,8 @@ export async function calculateCreditsToBeBilled(options: ScrapeOptions, internacreditsToBeBilled += (flags?.zdrCost ?? 1);}- const shouldParsePDF = options.parsers?.includes("pdf") ?? true;- if (shouldParsePDF && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) {+ const shouldParse = shouldParsePDF(options.parsers);+ if (shouldParse && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) {creditsToBeBilled += creditsPerPDFPage * (document.metadata.numPages - 1);}diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex b076da894..fbd7ec8bf 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -3,6 +3,7 @@ import { EngineScrapeResult } from "..";import { Meta } from "../..";import { getIndexFromGCS, hashURL, index_supabase_service, normalizeURLForIndex, saveIndexToGCS, generateURLSplits, addIndexInsertJob, generateDomainSplits, addOMCEJob, addDomainFrequencyJob } from "../../../../services";import { EngineError, IndexMissError } from "../../error";+import { shouldParsePDF } from "../../../../controllers/v2/types";import crypto from "crypto";export async function sendDocumentToIndex(meta: Meta, document: Document) {@@ -10,7 +11,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {&& !meta.internalOptions.zeroDataRetention&& meta.winnerEngine !== "index"&& meta.winnerEngine !== "index;documents"- && !(meta.winnerEngine === "pdf" && meta.options.parsers?.includes("pdf") === false)+ && !(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers))&& (meta.internalOptions.teamId === "sitemap"|| (@@ -249,13 +250,13 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResultconst isCachedPdfBase64 = doc.html && doc.html.startsWith("JVBERi");// If the cached content is base64 PDF but we want parsed PDF (parsePDF:true or default)- if (isCachedPdfBase64 && meta.options.parsers?.includes("pdf") !== false) {+ if (isCachedPdfBase64 && shouldParsePDF(meta.options.parsers)) {// Cached content is unparsed PDF, but we want parsed - report cache missthrow new IndexMissError();}// If the cached content is NOT base64 PDF but we want unparsed PDF (parsePDF:false)- if (!isCachedPdfBase64 && meta.options.parsers?.includes("pdf") === false) {+ if (!isCachedPdfBase64 && !shouldParsePDF(meta.options.parsers)) {// Check if URL looks like a PDFconst isPdfUrl = meta.url.toLowerCase().endsWith(".pdf") || meta.url.includes(".pdf?");if (isPdfUrl) {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 4f726e62c..85ae95673 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -19,6 +19,7 @@ import type { Response } from "undici";import { getPDFMetadata } from "../../../../lib/pdf-parser";import { getPdfResultFromCache, savePdfResultToCache } from "../../../../lib/gcs-pdf-cache";import { AbortManagerThrownError } from "../../lib/abortManager";+import { shouldParsePDF, getPDFMaxPages } from "../../../../controllers/v2/types";type PDFProcessorResult = { html: string; markdown?: string };@@ -29,6 +30,7 @@ async function scrapePDFWithRunPodMU(meta: Meta,tempFilePath: string,base64Content: string,+ maxPages?: number,): Promise<PDFProcessorResult> {meta.logger.debug("Processing PDF document with RunPod MU", {tempFilePath,@@ -66,6 +68,7 @@ async function scrapePDFWithRunPodMU(filename: path.basename(tempFilePath) + ".pdf",timeout: meta.abort.scrapeTimeout(),created_at: Date.now(),+ ...(maxPages !== undefined && { max_pages: maxPages }),},},logger: meta.logger.child({@@ -162,9 +165,10 @@ async function scrapePDFWithParsePDF(export async function scrapePDF(meta: Meta,): Promise<EngineScrapeResult> {- const shouldParsePDF = meta.options.parsers?.includes("pdf") ?? true;+ const shouldParse = shouldParsePDF(meta.options.parsers);+ const maxPages = getPDFMaxPages(meta.options.parsers);- if (!shouldParsePDF) {+ if (!shouldParse) {if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {const content = (await readFile(meta.pdfPrefetch.filePath)).toString("base64",@@ -228,11 +232,12 @@ export async function scrapePDF(}const pdfMetadata = await getPDFMetadata(tempFilePath);+ const effectivePageCount = maxPages ? Math.min(pdfMetadata.numPages, maxPages) : pdfMetadata.numPages;- if (pdfMetadata.numPages * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) {+ if (effectivePageCount * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) {throw new PDFInsufficientTimeError(- pdfMetadata.numPages,- pdfMetadata.numPages * MILLISECONDS_PER_PAGE + 5000,+ effectivePageCount,+ effectivePageCount * MILLISECONDS_PER_PAGE + 5000,);}@@ -256,6 +261,7 @@ export async function scrapePDF(},tempFilePath,base64Content,+ maxPages,);} catch (error) {if (diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex f9abd453c..313af37ff 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.0.0",+ "version": "4.1.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 56d83cfd6..1139df6f0 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -133,7 +133,7 @@ export interface ScrapeOptions {timeout?: number;waitFor?: number;mobile?: boolean;- parsers?: string[];+ parsers?: Array<string | { type: "pdf"; maxPages?: number }>;actions?: ActionOption[];location?: LocationConfig;skipTlsVerification?: boolean;diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 54b89235e..55e843736 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.0.0"+__version__ = "4.1.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 3c37afc0e..a4bc2f60b 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -18,6 +18,7 @@ from .types import (CrawlResponse,CrawlJob,CrawlParamsRequest,+ PDFParser,CrawlParamsData,WebhookConfig,CrawlErrorsResponse,@@ -105,7 +106,7 @@ class FirecrawlClient:timeout: Optional[int] = None,wait_for: Optional[int] = None,mobile: Optional[bool] = None,- parsers: Optional[List[str]] = None,+ parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,location: Optional['Location'] = None,skip_tls_verification: Optional[bool] = None,@@ -571,7 +572,7 @@ class FirecrawlClient:timeout: Optional[int] = None,wait_for: Optional[int] = None,mobile: Optional[bool] = None,- parsers: Optional[List[str]] = None,+ parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,location: Optional['Location'] = None,skip_tls_verification: Optional[bool] = None,@@ -759,7 +760,7 @@ class FirecrawlClient:timeout: Optional[int] = None,wait_for: Optional[int] = None,mobile: Optional[bool] = None,- parsers: Optional[List[str]] = None,+ parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,location: Optional['Location'] = None,skip_tls_verification: Optional[bool] = None,diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 319162e14..411295d8e 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -278,7 +278,7 @@ class ScrapeOptions(BaseModel):timeout: Optional[int] = Nonewait_for: Optional[int] = Nonemobile: Optional[bool] = None- parsers: Optional[List[str]] = None+ parsers: Optional[Union[List[str], List[Union[str, 'PDFParser']]]] = Noneactions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = Nonelocation: Optional['Location'] = Noneskip_tls_verification: Optional[bool] = None@@ -536,6 +536,11 @@ class PDFAction(BaseModel):landscape: Optional[bool] = Nonescale: Optional[float] = None+class PDFParser(BaseModel):+ """PDF parser configuration with optional page limit."""+ type: Literal["pdf"] = "pdf"+ max_pages: Optional[int] = None+# Location typesclass Location(BaseModel):"""Location configuration for scraping."""@@ -594,6 +599,26 @@ class SearchRequest(BaseModel):return normalized_categories+ @field_validator('parsers')+ @classmethod+ def validate_parsers(cls, v):+ """Validate and normalize parsers input."""+ if v is None:+ return v++ normalized_parsers = []+ for parser in v:+ if isinstance(parser, str):+ normalized_parsers.append(parser)+ elif isinstance(parser, dict):+ normalized_parsers.append(PDFParser(**parser))+ elif isinstance(parser, PDFParser):+ normalized_parsers.append(parser)+ else:+ raise ValueError(f"Invalid parser format: {parser}")++ return normalized_parsers+class LinkResult(BaseModel):"""A generic link result with optional metadata (used by search and map)."""url: str@@ -686,4 +711,4 @@ AnyResponse = Union[MapResponse,SearchResponse,ErrorResponse,-]\ No newline at end of file+]diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.pyindex f380a02cf..075ad9a2a 100644--- a/apps/python-sdk/firecrawl/v2/utils/validation.py+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py@@ -311,6 +311,20 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[stconverted_action[action_key] = action_valueconverted_actions.append(converted_action)scrape_data["actions"] = converted_actions+ elif key == "parsers":+ converted_parsers = []+ for parser in value:+ if isinstance(parser, str):+ converted_parsers.append(parser)+ elif isinstance(parser, dict):+ converted_parsers.append(parser)+ else:+ parser_data = parser.model_dump(exclude_none=True)+ # Convert snake_case to camelCase for API+ if "max_pages" in parser_data:+ parser_data["maxPages"] = parser_data.pop("max_pages")+ converted_parsers.append(parser_data)+ scrape_data["parsers"] = converted_parserselif key == "location":# Handle location conversionif isinstance(value, dict):@@ -321,4 +335,4 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st# For fields that don't need conversion, use as-isscrape_data[key] = value- return scrape_data\ No newline at end of file+ return scrape_data\ No newline at end of file