Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(python-sdk): scrape is done!
rafaelmmiller08c8f4209196
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.pynew file mode 100644index 000000000..8c9ca7ebc--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py@@ -0,0 +1,65 @@+import os+import pytest+from dotenv import load_dotenv+from firecrawl import Firecrawl+from firecrawl.v2.types import Viewport, ScreenshotAction, Document++load_dotenv()++if not os.getenv("API_KEY"):+ raise ValueError("API_KEY is not set")++if not os.getenv("API_URL"):+ raise ValueError("API_URL is not set")+++class TestScrapeE2E:+ """End-to-end tests for scrape functionality (v2)."""++ def setup_method(self):+ self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))++ def _assert_valid_document(self, doc: Document):+ assert isinstance(doc, Document)+ # At least one main content field should be present+ assert (+ (doc.markdown is not None and len(doc.markdown) > 0)+ or (doc.html is not None and len(doc.html) > 0)+ or (doc.raw_html is not None and len(doc.raw_html) > 0)+ )+ # Metadata should exist with a source URL or title when available+ assert doc.metadata is not None++ def test_scrape_minimal(self):+ """Scrape a URL with minimal parameters and return a document."""+ doc = self.client.scrape("https://docs.firecrawl.dev")+ self._assert_valid_document(doc)++ def test_scrape_with_options_markdown(self):+ """Scrape with simple markdown format and options passed as kwargs."""+ doc = self.client.scrape(+ "https://docs.firecrawl.dev",+ formats=["markdown"],+ only_main_content=False,+ mobile=False,+ )+ self._assert_valid_document(doc)++ def test_scrape_with_screenshot_action_viewport(self):+ """Scrape with a screenshot action including viewport passed as kwargs."""+ viewport = Viewport(width=800, height=600)+ action = ScreenshotAction(full_page=False, quality=80, viewport=viewport)+ doc = self.client.scrape(+ "https://docs.firecrawl.dev",+ formats=["markdown"],+ actions=[action],+ )+ self._assert_valid_document(doc)++ def test_scrape_invalid_url(self):+ """Scrape should fail with empty or invalid URLs."""+ with pytest.raises(ValueError, match="URL cannot be empty"):+ self.client.scrape("")++ with pytest.raises(ValueError, match="URL cannot be empty"):+ self.client.scrape(" ")\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/scrape/test_scrape_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/scrape/test_scrape_request_preparation.pynew file mode 100644index 000000000..822a1432d--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/scrape/test_scrape_request_preparation.py@@ -0,0 +1,92 @@+import pytest+from firecrawl.v2.types import ScrapeOptions, Viewport, ScreenshotAction+from firecrawl.v2.methods.scrape import _prepare_scrape_request+++class TestScrapeRequestPreparation:+ """Unit tests for scrape request preparation."""++ def test_basic_request_preparation(self):+ """Test basic request preparation with minimal fields."""+ data = _prepare_scrape_request("https://example.com")++ # Check basic fields+ assert data["url"] == "https://example.com"++ # Check that no options are present+ assert "formats" not in data+ assert "headers" not in data++ def test_scrape_options_conversion(self):+ """Test that ScrapeOptions fields are converted to camelCase."""+ options = ScrapeOptions(+ formats=["markdown", "html"],+ headers={"User-Agent": "Test"},+ include_tags=["h1", "h2"],+ exclude_tags=["nav"],+ only_main_content=False,+ timeout=15000,+ wait_for=2000,+ mobile=True,+ skip_tls_verification=True,+ remove_base64_images=False+ )++ data = _prepare_scrape_request("https://example.com", options)++ # Check basic field+ assert data["url"] == "https://example.com"++ # Check snake_case to camelCase conversions+ assert "includeTags" in data+ assert data["includeTags"] == ["h1", "h2"]+ assert "excludeTags" in data+ assert data["excludeTags"] == ["nav"]+ assert "onlyMainContent" in data+ assert data["onlyMainContent"] is False+ assert "waitFor" in data+ assert data["waitFor"] == 2000+ assert "skipTlsVerification" in data+ assert data["skipTlsVerification"] is True+ assert "removeBase64Images" in data+ assert data["removeBase64Images"] is False++ # Check that snake_case fields are not present+ assert "include_tags" not in data+ assert "exclude_tags" not in data+ assert "only_main_content" not in data++ def test_actions_conversion(self):+ """Test that actions are converted to camelCase."""+ viewport = Viewport(width=800, height=600)+ action = ScreenshotAction(full_page=False, quality=80, viewport=viewport)++ options = ScrapeOptions(actions=[action])+ data = _prepare_scrape_request("https://example.com", options)++ assert "actions" in data+ assert len(data["actions"]) == 1++ action_data = data["actions"][0]+ assert action_data["type"] == "screenshot"+ assert action_data["fullPage"] is False+ assert action_data["quality"] == 80+ assert "viewport" in action_data++ def test_none_options_handling(self):+ """Test handling of None options."""+ data = _prepare_scrape_request("https://example.com", None)++ assert data["url"] == "https://example.com"+ # Should not have any option fields+ assert len(data) == 1++ def test_empty_url_validation(self):+ """Test validation with empty URL."""+ with pytest.raises(ValueError, match="URL cannot be empty"):+ _prepare_scrape_request("")++ def test_whitespace_url_validation(self):+ """Test validation with whitespace-only URL."""+ with pytest.raises(ValueError, match="URL cannot be empty"):+ _prepare_scrape_request(" ")\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.pyindex ed7253816..653506af0 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py@@ -227,10 +227,64 @@ class TestPrepareScrapeOptions:schema={"type": "object", "properties": {"name": {"type": "string"}}})- print(f"DEBUG: format_obj.schema = {format_obj.schema}")- print(f"DEBUG: format_obj.model_dump() = {format_obj.model_dump()}")-- # Check that model_dump() shows schemadumped = format_obj.model_dump()assert "schema" in dumped- assert dumped["schema"] == {"type": "object", "properties": {"name": {"type": "string"}}}\ No newline at end of file+ assert dumped["schema"] == {"type": "object", "properties": {"name": {"type": "string"}}}++ def test_prepare_new_v2_fields(self):+ """Test preparation with new v2 fields."""+ from firecrawl.v2.types import Viewport, ScreenshotAction++ viewport = Viewport(width=1920, height=1080)+ screenshot_action = ScreenshotAction(+ type="screenshot",+ full_page=True,+ quality=90,+ viewport=viewport+ )++ options = ScrapeOptions(+ fast_mode=True,+ use_mock="test-mock",+ block_ads=False,+ store_in_cache=False,+ max_age=7200000, # 2 hours+ actions=[screenshot_action],+ parsers=["pdf"]+ )++ result = prepare_scrape_options(options)++ # Check new field conversions+ assert "fastMode" in result+ assert result["fastMode"] is True+ assert "useMock" in result+ assert result["useMock"] == "test-mock"+ assert "blockAds" in result+ assert result["blockAds"] is False+ assert "storeInCache" in result+ assert result["storeInCache"] is False+ assert "maxAge" in result+ assert result["maxAge"] == 7200000++ # Check actions conversion+ assert "actions" in result+ assert len(result["actions"]) == 1+ action = result["actions"][0]+ assert action["type"] == "screenshot"+ assert action["fullPage"] is True+ assert action["quality"] == 90+ assert "viewport" in action+ assert action["viewport"]["width"] == 1920+ assert action["viewport"]["height"] == 1080++ # Check parsers+ assert "parsers" in result+ assert result["parsers"] == ["pdf"]++ # Check that snake_case fields are not present+ assert "fast_mode" not in result+ assert "use_mock" not in result+ assert "block_ads" not in result+ assert "store_in_cache" not in result+ assert "max_age" not in result\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex 2bd5f1125..b812481f7 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -131,7 +131,7 @@ class Firecrawl:self.v2 = V2Proxy(self._v2_client)- # self.scrape = self._v2_client.scrape+ self.scrape = self._v2_client.scrapeself.crawl = self._v2_client.crawlself.start_crawl = self._v2_client.start_crawlself.crawl_params_preview = self._v2_client.crawl_params_previewdiff --git a/apps/python-sdk/firecrawl/v2/__init__.py b/apps/python-sdk/firecrawl/v2/__init__.pyindex 01ce7a5a7..bc45080fd 100644--- a/apps/python-sdk/firecrawl/v2/__init__.py+++ b/apps/python-sdk/firecrawl/v2/__init__.py@@ -1,8 +1,9 @@-from typing import Optional, List, Union, Dict, Callable, Literal, Any+from typing import Optional, List, Union, Dict, Anyfrom ..types import (- SearchData, SearchResult, Document, ScrapeOptions, SearchRequest,+ SearchData, Document, ScrapeOptions, SearchRequest,SourceOption, FormatOption, CrawlRequest, CrawlJob, CrawlResponse,- CrawlParamsRequest, CrawlParamsData, CrawlErrorsResponse, ActiveCrawlsResponse+ CrawlParamsRequest, CrawlParamsData, CrawlErrorsResponse, ActiveCrawlsResponse,+ WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction, Location)from .methods.search import search as search_methodfrom .methods.crawl import (@@ -14,6 +15,7 @@ from .methods.crawl import (get_crawl_errors as get_crawl_errors_method,get_active_crawls as get_active_crawls_method)+from .methods.scrape import scrape as scrape_methodfrom .utils.http_client import HttpClientclass FirecrawlClient:@@ -26,11 +28,52 @@ class FirecrawlClient:self.api_url = api_urlself._client = HttpClient(api_key=self.api_key, api_url=self.api_url)- # scrapedef scrape(- self- ):- pass+ self,+ url: str,+ formats: Optional[List[FormatOption]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ timeout: Optional[int] = None,+ wait_for: Optional[int] = None,+ mobile: Optional[bool] = None,+ parsers: Optional[List[str]] = None,+ actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]] = None,+ location: Optional[Location] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ fast_mode: Optional[bool] = None,+ use_mock: Optional[str] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[str] = None,+ max_age: Optional[int] = None,+ store_in_cache: Optional[bool] = None,+ ) -> Document:+ """Scrape a single URL and return the document."""+ options = ScrapeOptions(+ formats=formats,+ headers=headers,+ include_tags=include_tags,+ exclude_tags=exclude_tags,+ only_main_content=only_main_content,+ timeout=timeout,+ wait_for=wait_for,+ mobile=mobile,+ parsers=parsers,+ actions=actions,+ location=location,+ skip_tls_verification=skip_tls_verification,+ remove_base64_images=remove_base64_images,+ fast_mode=fast_mode,+ use_mock=use_mock,+ block_ads=block_ads,+ proxy=proxy,+ max_age=max_age,+ store_in_cache=store_in_cache+ )+ return scrape_method(self._client, url, options)# batch-scrapedef batch_scrape(@@ -61,7 +104,6 @@ class FirecrawlClient:):pass- # crawldef crawl(self,url: str,@@ -104,7 +146,6 @@ class FirecrawlClient:)return crawl_method(self._client, request, poll_interval, timeout)- # start-crawldef start_crawl(self,url: str,@@ -213,8 +254,8 @@ class FirecrawlClient:# credit-usagedef credit_usage(- self+ self):- pass+ pass__all__ = ['FirecrawlClient']\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 718281aef..c06007de4 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -10,7 +10,8 @@ from .types import (ClientConfig, ScrapeOptions, CrawlOptions, MapOptions, ExtractOptions,ScrapeResponse, CrawlResponse, CrawlStatusResponse, BatchScrapeResponse,BatchScrapeStatusResponse, MapResponse, ExtractResponse, Document,- SearchRequest, SearchResponse, CrawlRequest, WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse+ SearchRequest, SearchResponse, CrawlRequest, WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse,+ FormatOption, WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction, Location,)from .utils.http_client import HttpClientfrom .utils.error_handler import FirecrawlError@@ -63,24 +64,80 @@ class FirecrawlClient:self.http_client = HttpClient(api_key, api_url)- # def scrape(- # self,- # url: str,- # options: Optional[ScrapeOptions] = None- # ) -> ScrapeResponse:- # """- # Scrape a single URL.-- # Args:- # url: URL to scrape- # options: Scraping options-- # Returns:- # ScrapeResponse containing the scraped document- # """-- # return scrape_module.scrape(self.http_client, url, options)-+ def scrape(+ self,+ url: str,+ *,+ formats: Optional[List['FormatOption']] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ timeout: Optional[int] = None,+ wait_for: Optional[int] = None,+ mobile: Optional[bool] = None,+ parsers: Optional[List[str]] = None,+ actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,+ location: Optional['Location'] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ fast_mode: Optional[bool] = None,+ use_mock: Optional[str] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[str] = None,+ max_age: Optional[int] = None,+ store_in_cache: Optional[bool] = None,+ ) -> Document:+ """+ Scrape a single URL and return the document.+ Args:+ url: URL to scrape+ formats: List of formats to scrape+ headers: Dictionary of headers to use+ include_tags: List of tags to include+ exclude_tags: List of tags to exclude+ only_main_content: Whether to only scrape the main content+ timeout: Timeout in seconds+ wait_for: Wait for a specific element to be present+ mobile: Whether to use mobile mode+ parsers: List of parsers to use+ actions: List of actions to perform+ location: Location to scrape+ skip_tls_verification: Whether to skip TLS verification+ remove_base64_images: Whether to remove base64 images+ fast_mode: Whether to use fast mode+ use_mock: Whether to use mock mode+ block_ads: Whether to block ads+ proxy: Proxy to use+ max_age: Maximum age of the cache+ store_in_cache: Whether to store the result in the cache+ Returns:+ Document+ """+ options = ScrapeOptions(+ **{k: v for k, v in dict(+ formats=formats,+ headers=headers,+ include_tags=include_tags,+ exclude_tags=exclude_tags,+ only_main_content=only_main_content,+ timeout=timeout,+ wait_for=wait_for,+ mobile=mobile,+ parsers=parsers,+ actions=actions,+ location=location,+ skip_tls_verification=skip_tls_verification,+ remove_base64_images=remove_base64_images,+ fast_mode=fast_mode,+ use_mock=use_mock,+ block_ads=block_ads,+ proxy=proxy,+ max_age=max_age,+ store_in_cache=store_in_cache,+ ).items() if v is not None}+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache]) else None+ return scrape_module.scrape(self.http_client, url, options)def search(self,diff --git a/apps/python-sdk/firecrawl/v2/methods/scrape.py b/apps/python-sdk/firecrawl/v2/methods/scrape.pyindex eacaf8bf0..0eaf7e720 100644--- a/apps/python-sdk/firecrawl/v2/methods/scrape.py+++ b/apps/python-sdk/firecrawl/v2/methods/scrape.py@@ -2,189 +2,61 @@Scraping functionality for Firecrawl v2 API."""-import jsonfrom typing import Optional, Dict, Any-from .types import ScrapeRequest, ScrapeResponse, ScrapeOptions, Document-from .utils import HttpClient, handle_response_error, validate_scrape_options+from ..types import ScrapeOptions, Document+from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options-def scrape(- client: HttpClient,- url: str,- options: Optional[ScrapeOptions] = None-) -> ScrapeResponse:+def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:"""- Scrape a single URL and return the content.+ Prepare a scrape request payload for v2 API.Args:- client: HTTP client instanceurl: URL to scrape- options: Scraping options+ options: ScrapeOptions (snake_case) to convert and includeReturns:- ScrapeResponse containing the scraped document-- Raises:- FirecrawlError: If the scraping operation fails+ Request payload dictionary with camelCase fields"""- # Prepare request data- request_data = {"url": url}-- if options:- request_data["options"] = options.dict(exclude_none=True, by_alias=True)-- # Make the API request- response = client.post("/v1/scrape", request_data)-- # Handle errors- if not response.ok:- handle_response_error(response, "scrape URL")-- # Parse response- response_data = response.json()-- # Create document from response- if response_data.get("success"):- document_data = response_data.get("data", {})- document = Document(**document_data)-- return ScrapeResponse(- success=True,- data=document,- warning=response_data.get("warning")- )- else:- return ScrapeResponse(- success=False,- error=response_data.get("error", "Unknown error occurred")- )+ if not url or not url.strip():+ raise ValueError("URL cannot be empty")+ request_data: Dict[str, Any] = {"url": url.strip()}-def scrape_with_extract(- client: HttpClient,- url: str,- schema: Dict[str, Any],- options: Optional[ScrapeOptions] = None,- system_prompt: Optional[str] = None,- prompt: Optional[str] = None-) -> ScrapeResponse:- """- Scrape a URL and extract structured data using a schema.-- Args:- client: HTTP client instance- url: URL to scrape- schema: JSON schema for data extraction- options: Scraping options- system_prompt: System prompt for extraction- prompt: User prompt for extraction-- Returns:- ScrapeResponse with extracted data-- Raises:- FirecrawlError: If the operation fails- """- # Prepare request data- request_data = {- "url": url,- "extractorOptions": {- "extractionSchema": schema- }- }-- if system_prompt:- request_data["extractorOptions"]["extractionPrompt"] = system_prompt-- if prompt:- request_data["extractorOptions"]["userPrompt"] = prompt-- if options:- request_data["pageOptions"] = options.dict(exclude_none=True, by_alias=True)-- # Make the API request- response = client.post("/v1/scrape", request_data)-- # Handle errors- if not response.ok:- handle_response_error(response, "scrape and extract from URL")-- # Parse response- response_data = response.json()-- # Create document from response- if response_data.get("success"):- document_data = response_data.get("data", {})- document = Document(**document_data)-- return ScrapeResponse(- success=True,- data=document,- warning=response_data.get("warning")- )- else:- return ScrapeResponse(- success=False,- error=response_data.get("error", "Unknown error occurred")- )+ if options is not None:+ validated = validate_scrape_options(options)+ if validated is not None:+ opts = prepare_scrape_options(validated)+ if opts:+ request_data.update(opts)+ return request_data-def scrape_multiple(- client: HttpClient,- urls: list[str],- options: Optional[ScrapeOptions] = None-) -> list[ScrapeResponse]:+def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:"""- Scrape multiple URLs concurrently.+ Scrape a single URL and return the document.++ The v2 API returns: { success: boolean, data: Document }+ We surface just the Document to callers.Args:client: HTTP client instance- urls: List of URLs to scrape- options: Scraping options+ url: URL to scrape+ options: Scraping options (snake_case)Returns:- List of ScrapeResponse objects-- Note:- This function makes individual requests for each URL.- For large batches, consider using batch_scrape instead.+ Document"""- results = []-- for url in urls:- try:- result = scrape(client, url, options)- results.append(result)- except Exception as e:- # Create error response for failed scrapes- error_response = ScrapeResponse(- success=False,- error=f"Failed to scrape {url}: {str(e)}"- )- results.append(error_response)-- return results-+ payload = _prepare_scrape_request(url, options)+ response = client.post("/v2/scrape", payload)+ if not response.ok:+ handle_response_error(response, "scrape")+ body = response.json()+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))-def prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:- """- Prepare a scrape request payload.-- Args:- url: URL to scrape- options: Scraping options-- Returns:- Request payload dictionary- """- request_data = {"url": url}-- if options:- validated_options = validate_scrape_options(options)- if validated_options:- request_data["options"] = validated_options.dict(exclude_none=True, by_alias=True)-- return request_data\ No newline at end of file+ document_data = body.get("data", {})+ return Document(**document_data)\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex b4262314f..dc9a15cb9 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -79,11 +79,16 @@ SourceOption = Union[str, Source]FormatString = Literal[# camelCase versions (API format)- "markdown", "html", "rawHtml", "links", "screenshot", "json", "changeTracking",+ "markdown", "html", "rawHtml", "links", "screenshot", "summary", "changeTracking", "json",# snake_case versions (user-friendly)"raw_html", "change_tracking"]+class Viewport(BaseModel):+ """Viewport configuration for screenshots."""+ width: int+ height: int+class Format(BaseModel):"""Configuration for a format."""type: FormatString@@ -100,7 +105,14 @@ class ChangeTrackingFormat(Format):prompt: Optional[str] = Nonetag: Optional[str] = None-FormatOption = Union[FormatString, Format, JsonFormat, ChangeTrackingFormat]+class ScreenshotFormat(BaseModel):+ """Configuration for screenshot format."""+ type: Literal["screenshot"] = "screenshot"+ full_page: Optional[bool] = None+ quality: Optional[int] = None+ viewport: Optional[Union[Dict[str, int], Viewport]] = None++FormatOption = Union[FormatString, Format, JsonFormat, ChangeTrackingFormat, ScreenshotFormat]# Scrape typesclass ScrapeFormats(BaseModel):@@ -136,22 +148,25 @@ class ScrapeFormats(BaseModel):class ScrapeOptions(BaseModel):"""Options for scraping operations."""- formats: Optional[Union[ScrapeFormats, List[FormatOption]]] = None+ formats: Optional[List[FormatOption]] = Noneheaders: Optional[Dict[str, str]] = Noneinclude_tags: Optional[List[str]] = Noneexclude_tags: Optional[List[str]] = None- only_main_content: bool = True+ only_main_content: Optional[bool] = Nonetimeout: Optional[int] = Nonewait_for: Optional[int] = None- mobile: bool = False- skip_tls_verification: bool = False- remove_base64_images: bool = True- block_ads: bool = False- proxy: Optional[str] = None- max_age: Optional[int] = None- store_in_cache: bool = False- location: Optional['Location'] = None+ mobile: Optional[bool] = None+ parsers: Optional[List[str]] = Noneactions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None+ location: Optional['Location'] = None+ skip_tls_verification: Optional[bool] = None+ remove_base64_images: Optional[bool] = None+ fast_mode: Optional[bool] = None+ use_mock: Optional[str] = None+ block_ads: Optional[bool] = None+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None+ max_age: Optional[int] = None+ store_in_cache: Optional[bool] = None@field_validator('formats')@classmethod@@ -313,8 +328,9 @@ class WaitAction(BaseModel):class ScreenshotAction(BaseModel):"""Screenshot action to perform during scraping."""type: Literal["screenshot"] = "screenshot"- full_page: Optional[bool] = Field(None, alias="fullPage")+ full_page: Optional[bool] = Nonequality: Optional[int] = None+ viewport: Optional[Union[Dict[str, int], Viewport]] = Noneclass ClickAction(BaseModel):"""Click action to perform during scraping."""diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.pyindex 4f85c19b9..39bb6acc7 100644--- a/apps/python-sdk/firecrawl/v2/utils/validation.py+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py@@ -97,116 +97,111 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[stif validated_options is None:return None+ # Apply default values for None fields+ default_values = {+ "only_main_content": True,+ "mobile": False,+ "skip_tls_verification": True,+ "remove_base64_images": True,+ "fast_mode": False,+ "block_ads": True,+ "max_age": 14400000,+ "store_in_cache": True+ }+# Convert to dict and handle manual snake_case to camelCase conversionoptions_data = validated_options.model_dump(exclude_none=True)++ # Apply defaults for None fields+ for field, default_value in default_values.items():+ if field not in options_data:+ options_data[field] = default_value+scrape_data = {}+ # Manual field mapping for snake_case to camelCase conversion+ field_mappings = {+ "include_tags": "includeTags",+ "exclude_tags": "excludeTags",+ "only_main_content": "onlyMainContent",+ "wait_for": "waitFor",+ "skip_tls_verification": "skipTlsVerification",+ "remove_base64_images": "removeBase64Images",+ "fast_mode": "fastMode",+ "use_mock": "useMock",+ "block_ads": "blockAds",+ "store_in_cache": "storeInCache",+ "max_age": "maxAge"+ }++ # Apply field mappings+ for snake_case, camel_case in field_mappings.items():+ if snake_case in options_data:+ scrape_data[camel_case] = options_data.pop(snake_case)++ # Handle special casesfor key, value in options_data.items():if value is not None:if key == "formats":- # Handle formats - if it's a ScrapeFormats object, convert boolean flags to format strings- if isinstance(value, dict) and any(key in value for key in ['markdown', 'html', 'raw_html', 'content', 'links', 'screenshot', 'screenshot_full_page']):- # Convert ScrapeFormats boolean flags to format strings- formats = []- if value.get('markdown', False):- formats.append("markdown")- if value.get('html', False):- formats.append("html")- if value.get('raw_html', False):- formats.append("rawHtml")- if value.get('content', False):- formats.append("content")- if value.get('links', False):- formats.append("links")- if value.get('screenshot', False):- formats.append("screenshot")- if value.get('screenshot_full_page', False):- formats.append("screenshot@fullPage")- scrape_data["formats"] = formats- elif hasattr(value, 'markdown') or hasattr(value, 'html') or hasattr(value, 'raw_html') or hasattr(value, 'content') or hasattr(value, 'links') or hasattr(value, 'screenshot') or hasattr(value, 'screenshot_full_page'):- # Convert ScrapeFormats boolean flags to format strings- formats = []- if getattr(value, 'markdown', False):- formats.append("markdown")- if getattr(value, 'html', False):- formats.append("html")- if getattr(value, 'raw_html', False):- formats.append("rawHtml")- if getattr(value, 'content', False):- formats.append("content")- if getattr(value, 'links', False):- formats.append("links")- if getattr(value, 'screenshot', False):- formats.append("screenshot")- if getattr(value, 'screenshot_full_page', False):- formats.append("screenshot@fullPage")- scrape_data["formats"] = formats- elif hasattr(value, 'formats') and value.formats is not None:- # Convert format strings from snake_case to camelCase- converted_formats = []- for fmt in value.formats:- if isinstance(fmt, str):- converted_formats.append(_convert_format_string(fmt))- elif hasattr(fmt, 'type'):- converted_formats.append(_convert_format_string(fmt.type))+ # Handle formats conversion+ converted_formats = []+ for fmt in value:+ if isinstance(fmt, str):+ # Handle format strings+ if fmt == "json":+ raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")+ converted_formats.append(_convert_format_string(fmt))+ elif isinstance(fmt, dict):+ # Handle format objects+ if fmt.get('type') == 'json':+ validated_json = _validate_json_format(fmt)+ converted_formats.append(validated_json)else:+ # Convert other format objects+ if 'type' in fmt:+ fmt['type'] = _convert_format_string(fmt['type'])converted_formats.append(fmt)- scrape_data["formats"] = converted_formats- elif isinstance(value, list):- # If it's already a list, convert format strings from snake_case to camelCase- converted_formats = []- for fmt in value:- if isinstance(fmt, str):- # Handle json format specially - it can't be a string- if fmt == "json":- raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")- converted_formats.append(_convert_format_string(fmt))- elif isinstance(fmt, dict):- # Handle format objects (like json format)- if fmt.get('type') == 'json':- validated_json = _validate_json_format(fmt)- converted_formats.append(validated_json)+ elif hasattr(fmt, 'type'):+ # Handle Format objects+ if fmt.type == 'json':+ # For json format, we need the full object+ converted_formats.append(fmt.model_dump())+ else:+ # For other formats, just convert the type+ converted_formats.append(_convert_format_string(fmt.type))+ else:+ converted_formats.append(fmt)+ scrape_data["formats"] = converted_formats+ elif key == "actions":+ # Handle actions conversion+ converted_actions = []+ for action in value:+ if isinstance(action, dict):+ # Convert action dict+ converted_action = {}+ for action_key, action_value in action.items():+ if action_key == "full_page":+ converted_action["fullPage"] = action_valueelse:- # Convert other format objects- if 'type' in fmt:- fmt['type'] = _convert_format_string(fmt['type'])- converted_formats.append(fmt)- elif hasattr(fmt, 'type'):- # Handle Format objects- if fmt.type == 'json':- # For json format, we need the full object- converted_formats.append(fmt.model_dump())+ converted_action[action_key] = action_value+ converted_actions.append(converted_action)+ else:+ # Handle action objects+ action_data = action.model_dump(exclude_none=True)+ converted_action = {}+ for action_key, action_value in action_data.items():+ if action_key == "full_page":+ converted_action["fullPage"] = action_valueelse:- # For other formats, just convert the type- converted_formats.append(_convert_format_string(fmt.type))- else:- converted_formats.append(fmt)- scrape_data["formats"] = converted_formats+ converted_action[action_key] = action_value+ converted_actions.append(converted_action)+ scrape_data["actions"] = converted_actions+ elif key == "location":+ # Handle location conversion+ if isinstance(value, dict):+ scrape_data["location"] = valueelse:- # Fallback - use the value as-is- scrape_data["formats"] = value- elif key == "include_tags":- scrape_data["includeTags"] = value- elif key == "exclude_tags":- scrape_data["excludeTags"] = value- elif key == "only_main_content":- scrape_data["onlyMainContent"] = value- elif key == "wait_for":- scrape_data["waitFor"] = value- elif key == "skip_tls_verification":- scrape_data["skipTlsVerification"] = value- elif key == "remove_base64_images":- scrape_data["removeBase64Images"] = value- elif key == "block_ads":- scrape_data["blockAds"] = value- elif key == "store_in_cache":- scrape_data["storeInCache"] = value- elif key == "max_age":- scrape_data["maxAge"] = value- # Note: raw_html should be in formats array, not as a separate field- # Note: screenshot_full_page is not supported by v2 API- # elif key == "screenshot_full_page":- # scrape_data["screenshot@fullPage"] = value+ scrape_data["location"] = value.model_dump(exclude_none=True)else:# For fields that don't need conversion, use as-isscrape_data[key] = value