Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(python-sdk): scrape is done!

rafaelmmiller08c8f4209196
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py
new file mode 100644
index 000000000..8c9ca7ebc
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py
@@ -0,0 +1,65 @@
+import os
+import pytest
+from dotenv import load_dotenv
+from firecrawl import Firecrawl
+from firecrawl.v2.types import Viewport, ScreenshotAction, Document
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+ raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+ raise ValueError("API_URL is not set")
+
+
+class TestScrapeE2E:
+ """End-to-end tests for scrape functionality (v2)."""
+
+ def setup_method(self):
+ self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+
+ def _assert_valid_document(self, doc: Document):
+ assert isinstance(doc, Document)
+ # At least one main content field should be present
+ assert (
+ (doc.markdown is not None and len(doc.markdown) > 0)
+ or (doc.html is not None and len(doc.html) > 0)
+ or (doc.raw_html is not None and len(doc.raw_html) > 0)
+ )
+ # Metadata should exist with a source URL or title when available
+ assert doc.metadata is not None
+
+ def test_scrape_minimal(self):
+ """Scrape a URL with minimal parameters and return a document."""
+ doc = self.client.scrape("https://docs.firecrawl.dev")
+ self._assert_valid_document(doc)
+
+ def test_scrape_with_options_markdown(self):
+ """Scrape with simple markdown format and options passed as kwargs."""
+ doc = self.client.scrape(
+ "https://docs.firecrawl.dev",
+ formats=["markdown"],
+ only_main_content=False,
+ mobile=False,
+ )
+ self._assert_valid_document(doc)
+
+ def test_scrape_with_screenshot_action_viewport(self):
+ """Scrape with a screenshot action including viewport passed as kwargs."""
+ viewport = Viewport(width=800, height=600)
+ action = ScreenshotAction(full_page=False, quality=80, viewport=viewport)
+ doc = self.client.scrape(
+ "https://docs.firecrawl.dev",
+ formats=["markdown"],
+ actions=[action],
+ )
+ self._assert_valid_document(doc)
+
+ def test_scrape_invalid_url(self):
+ """Scrape should fail with empty or invalid URLs."""
+ with pytest.raises(ValueError, match="URL cannot be empty"):
+ self.client.scrape("")
+
+ with pytest.raises(ValueError, match="URL cannot be empty"):
+ self.client.scrape(" ")
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/scrape/test_scrape_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/scrape/test_scrape_request_preparation.py
new file mode 100644
index 000000000..822a1432d
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/scrape/test_scrape_request_preparation.py
@@ -0,0 +1,92 @@
+import pytest
+from firecrawl.v2.types import ScrapeOptions, Viewport, ScreenshotAction
+from firecrawl.v2.methods.scrape import _prepare_scrape_request
+
+
+class TestScrapeRequestPreparation:
+ """Unit tests for scrape request preparation."""
+
+ def test_basic_request_preparation(self):
+ """Test basic request preparation with minimal fields."""
+ data = _prepare_scrape_request("https://example.com")
+
+ # Check basic fields
+ assert data["url"] == "https://example.com"
+
+ # Check that no options are present
+ assert "formats" not in data
+ assert "headers" not in data
+
+ def test_scrape_options_conversion(self):
+ """Test that ScrapeOptions fields are converted to camelCase."""
+ options = ScrapeOptions(
+ formats=["markdown", "html"],
+ headers={"User-Agent": "Test"},
+ include_tags=["h1", "h2"],
+ exclude_tags=["nav"],
+ only_main_content=False,
+ timeout=15000,
+ wait_for=2000,
+ mobile=True,
+ skip_tls_verification=True,
+ remove_base64_images=False
+ )
+
+ data = _prepare_scrape_request("https://example.com", options)
+
+ # Check basic field
+ assert data["url"] == "https://example.com"
+
+ # Check snake_case to camelCase conversions
+ assert "includeTags" in data
+ assert data["includeTags"] == ["h1", "h2"]
+ assert "excludeTags" in data
+ assert data["excludeTags"] == ["nav"]
+ assert "onlyMainContent" in data
+ assert data["onlyMainContent"] is False
+ assert "waitFor" in data
+ assert data["waitFor"] == 2000
+ assert "skipTlsVerification" in data
+ assert data["skipTlsVerification"] is True
+ assert "removeBase64Images" in data
+ assert data["removeBase64Images"] is False
+
+ # Check that snake_case fields are not present
+ assert "include_tags" not in data
+ assert "exclude_tags" not in data
+ assert "only_main_content" not in data
+
+ def test_actions_conversion(self):
+ """Test that actions are converted to camelCase."""
+ viewport = Viewport(width=800, height=600)
+ action = ScreenshotAction(full_page=False, quality=80, viewport=viewport)
+
+ options = ScrapeOptions(actions=[action])
+ data = _prepare_scrape_request("https://example.com", options)
+
+ assert "actions" in data
+ assert len(data["actions"]) == 1
+
+ action_data = data["actions"][0]
+ assert action_data["type"] == "screenshot"
+ assert action_data["fullPage"] is False
+ assert action_data["quality"] == 80
+ assert "viewport" in action_data
+
+ def test_none_options_handling(self):
+ """Test handling of None options."""
+ data = _prepare_scrape_request("https://example.com", None)
+
+ assert data["url"] == "https://example.com"
+ # Should not have any option fields
+ assert len(data) == 1
+
+ def test_empty_url_validation(self):
+ """Test validation with empty URL."""
+ with pytest.raises(ValueError, match="URL cannot be empty"):
+ _prepare_scrape_request("")
+
+ def test_whitespace_url_validation(self):
+ """Test validation with whitespace-only URL."""
+ with pytest.raises(ValueError, match="URL cannot be empty"):
+ _prepare_scrape_request(" ")
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
index ed7253816..653506af0 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
@@ -227,10 +227,64 @@ class TestPrepareScrapeOptions:
schema={"type": "object", "properties": {"name": {"type": "string"}}}
)
- print(f"DEBUG: format_obj.schema = {format_obj.schema}")
- print(f"DEBUG: format_obj.model_dump() = {format_obj.model_dump()}")
-
- # Check that model_dump() shows schema
dumped = format_obj.model_dump()
assert "schema" in dumped
- assert dumped["schema"] == {"type": "object", "properties": {"name": {"type": "string"}}}
\ No newline at end of file
+ assert dumped["schema"] == {"type": "object", "properties": {"name": {"type": "string"}}}
+
+ def test_prepare_new_v2_fields(self):
+ """Test preparation with new v2 fields."""
+ from firecrawl.v2.types import Viewport, ScreenshotAction
+
+ viewport = Viewport(width=1920, height=1080)
+ screenshot_action = ScreenshotAction(
+ type="screenshot",
+ full_page=True,
+ quality=90,
+ viewport=viewport
+ )
+
+ options = ScrapeOptions(
+ fast_mode=True,
+ use_mock="test-mock",
+ block_ads=False,
+ store_in_cache=False,
+ max_age=7200000, # 2 hours
+ actions=[screenshot_action],
+ parsers=["pdf"]
+ )
+
+ result = prepare_scrape_options(options)
+
+ # Check new field conversions
+ assert "fastMode" in result
+ assert result["fastMode"] is True
+ assert "useMock" in result
+ assert result["useMock"] == "test-mock"
+ assert "blockAds" in result
+ assert result["blockAds"] is False
+ assert "storeInCache" in result
+ assert result["storeInCache"] is False
+ assert "maxAge" in result
+ assert result["maxAge"] == 7200000
+
+ # Check actions conversion
+ assert "actions" in result
+ assert len(result["actions"]) == 1
+ action = result["actions"][0]
+ assert action["type"] == "screenshot"
+ assert action["fullPage"] is True
+ assert action["quality"] == 90
+ assert "viewport" in action
+ assert action["viewport"]["width"] == 1920
+ assert action["viewport"]["height"] == 1080
+
+ # Check parsers
+ assert "parsers" in result
+ assert result["parsers"] == ["pdf"]
+
+ # Check that snake_case fields are not present
+ assert "fast_mode" not in result
+ assert "use_mock" not in result
+ assert "block_ads" not in result
+ assert "store_in_cache" not in result
+ assert "max_age" not in result
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.py
index 2bd5f1125..b812481f7 100644
--- a/apps/python-sdk/firecrawl/client.py
+++ b/apps/python-sdk/firecrawl/client.py
@@ -131,7 +131,7 @@ class Firecrawl:
self.v2 = V2Proxy(self._v2_client)
- # self.scrape = self._v2_client.scrape
+ self.scrape = self._v2_client.scrape
self.crawl = self._v2_client.crawl
self.start_crawl = self._v2_client.start_crawl
self.crawl_params_preview = self._v2_client.crawl_params_preview
diff --git a/apps/python-sdk/firecrawl/v2/__init__.py b/apps/python-sdk/firecrawl/v2/__init__.py
index 01ce7a5a7..bc45080fd 100644
--- a/apps/python-sdk/firecrawl/v2/__init__.py
+++ b/apps/python-sdk/firecrawl/v2/__init__.py
@@ -1,8 +1,9 @@
-from typing import Optional, List, Union, Dict, Callable, Literal, Any
+from typing import Optional, List, Union, Dict, Any
from ..types import (
- SearchData, SearchResult, Document, ScrapeOptions, SearchRequest,
+ SearchData, Document, ScrapeOptions, SearchRequest,
SourceOption, FormatOption, CrawlRequest, CrawlJob, CrawlResponse,
- CrawlParamsRequest, CrawlParamsData, CrawlErrorsResponse, ActiveCrawlsResponse
+ CrawlParamsRequest, CrawlParamsData, CrawlErrorsResponse, ActiveCrawlsResponse,
+ WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction, Location
)
from .methods.search import search as search_method
from .methods.crawl import (
@@ -14,6 +15,7 @@ from .methods.crawl import (
get_crawl_errors as get_crawl_errors_method,
get_active_crawls as get_active_crawls_method
)
+from .methods.scrape import scrape as scrape_method
from .utils.http_client import HttpClient
class FirecrawlClient:
@@ -26,11 +28,52 @@ class FirecrawlClient:
self.api_url = api_url
self._client = HttpClient(api_key=self.api_key, api_url=self.api_url)
- # scrape
def scrape(
- self
- ):
- pass
+ self,
+ url: str,
+ formats: Optional[List[FormatOption]] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ timeout: Optional[int] = None,
+ wait_for: Optional[int] = None,
+ mobile: Optional[bool] = None,
+ parsers: Optional[List[str]] = None,
+ actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]] = None,
+ location: Optional[Location] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ fast_mode: Optional[bool] = None,
+ use_mock: Optional[str] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[str] = None,
+ max_age: Optional[int] = None,
+ store_in_cache: Optional[bool] = None,
+ ) -> Document:
+ """Scrape a single URL and return the document."""
+ options = ScrapeOptions(
+ formats=formats,
+ headers=headers,
+ include_tags=include_tags,
+ exclude_tags=exclude_tags,
+ only_main_content=only_main_content,
+ timeout=timeout,
+ wait_for=wait_for,
+ mobile=mobile,
+ parsers=parsers,
+ actions=actions,
+ location=location,
+ skip_tls_verification=skip_tls_verification,
+ remove_base64_images=remove_base64_images,
+ fast_mode=fast_mode,
+ use_mock=use_mock,
+ block_ads=block_ads,
+ proxy=proxy,
+ max_age=max_age,
+ store_in_cache=store_in_cache
+ )
+ return scrape_method(self._client, url, options)
# batch-scrape
def batch_scrape(
@@ -61,7 +104,6 @@ class FirecrawlClient:
):
pass
- # crawl
def crawl(
self,
url: str,
@@ -104,7 +146,6 @@ class FirecrawlClient:
)
return crawl_method(self._client, request, poll_interval, timeout)
- # start-crawl
def start_crawl(
self,
url: str,
@@ -213,8 +254,8 @@ class FirecrawlClient:
# credit-usage
def credit_usage(
- self
+ self
):
- pass
+ pass
__all__ = ['FirecrawlClient']
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 718281aef..c06007de4 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -10,7 +10,8 @@ from .types import (
ClientConfig, ScrapeOptions, CrawlOptions, MapOptions, ExtractOptions,
ScrapeResponse, CrawlResponse, CrawlStatusResponse, BatchScrapeResponse,
BatchScrapeStatusResponse, MapResponse, ExtractResponse, Document,
- SearchRequest, SearchResponse, CrawlRequest, WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse
+ SearchRequest, SearchResponse, CrawlRequest, WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse,
+ FormatOption, WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction, Location,
)
from .utils.http_client import HttpClient
from .utils.error_handler import FirecrawlError
@@ -63,24 +64,80 @@ class FirecrawlClient:
self.http_client = HttpClient(api_key, api_url)
- # def scrape(
- # self,
- # url: str,
- # options: Optional[ScrapeOptions] = None
- # ) -> ScrapeResponse:
- # """
- # Scrape a single URL.
-
- # Args:
- # url: URL to scrape
- # options: Scraping options
-
- # Returns:
- # ScrapeResponse containing the scraped document
- # """
-
- # return scrape_module.scrape(self.http_client, url, options)
-
+ def scrape(
+ self,
+ url: str,
+ *,
+ formats: Optional[List['FormatOption']] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ timeout: Optional[int] = None,
+ wait_for: Optional[int] = None,
+ mobile: Optional[bool] = None,
+ parsers: Optional[List[str]] = None,
+ actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,
+ location: Optional['Location'] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ fast_mode: Optional[bool] = None,
+ use_mock: Optional[str] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[str] = None,
+ max_age: Optional[int] = None,
+ store_in_cache: Optional[bool] = None,
+ ) -> Document:
+ """
+ Scrape a single URL and return the document.
+ Args:
+ url: URL to scrape
+ formats: List of formats to scrape
+ headers: Dictionary of headers to use
+ include_tags: List of tags to include
+ exclude_tags: List of tags to exclude
+ only_main_content: Whether to only scrape the main content
+ timeout: Timeout in seconds
+ wait_for: Wait for a specific element to be present
+ mobile: Whether to use mobile mode
+ parsers: List of parsers to use
+ actions: List of actions to perform
+ location: Location to scrape
+ skip_tls_verification: Whether to skip TLS verification
+ remove_base64_images: Whether to remove base64 images
+ fast_mode: Whether to use fast mode
+ use_mock: Whether to use mock mode
+ block_ads: Whether to block ads
+ proxy: Proxy to use
+ max_age: Maximum age of the cache
+ store_in_cache: Whether to store the result in the cache
+ Returns:
+ Document
+ """
+ options = ScrapeOptions(
+ **{k: v for k, v in dict(
+ formats=formats,
+ headers=headers,
+ include_tags=include_tags,
+ exclude_tags=exclude_tags,
+ only_main_content=only_main_content,
+ timeout=timeout,
+ wait_for=wait_for,
+ mobile=mobile,
+ parsers=parsers,
+ actions=actions,
+ location=location,
+ skip_tls_verification=skip_tls_verification,
+ remove_base64_images=remove_base64_images,
+ fast_mode=fast_mode,
+ use_mock=use_mock,
+ block_ads=block_ads,
+ proxy=proxy,
+ max_age=max_age,
+ store_in_cache=store_in_cache,
+ ).items() if v is not None}
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache]) else None
+ return scrape_module.scrape(self.http_client, url, options)
def search(
self,
diff --git a/apps/python-sdk/firecrawl/v2/methods/scrape.py b/apps/python-sdk/firecrawl/v2/methods/scrape.py
index eacaf8bf0..0eaf7e720 100644
--- a/apps/python-sdk/firecrawl/v2/methods/scrape.py
+++ b/apps/python-sdk/firecrawl/v2/methods/scrape.py
@@ -2,189 +2,61 @@
Scraping functionality for Firecrawl v2 API.
"""
-import json
from typing import Optional, Dict, Any
-from .types import ScrapeRequest, ScrapeResponse, ScrapeOptions, Document
-from .utils import HttpClient, handle_response_error, validate_scrape_options
+from ..types import ScrapeOptions, Document
+from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options
-def scrape(
- client: HttpClient,
- url: str,
- options: Optional[ScrapeOptions] = None
-) -> ScrapeResponse:
+def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
"""
- Scrape a single URL and return the content.
+ Prepare a scrape request payload for v2 API.
Args:
- client: HTTP client instance
url: URL to scrape
- options: Scraping options
+ options: ScrapeOptions (snake_case) to convert and include
Returns:
- ScrapeResponse containing the scraped document
-
- Raises:
- FirecrawlError: If the scraping operation fails
+ Request payload dictionary with camelCase fields
"""
- # Prepare request data
- request_data = {"url": url}
-
- if options:
- request_data["options"] = options.dict(exclude_none=True, by_alias=True)
-
- # Make the API request
- response = client.post("/v1/scrape", request_data)
-
- # Handle errors
- if not response.ok:
- handle_response_error(response, "scrape URL")
-
- # Parse response
- response_data = response.json()
-
- # Create document from response
- if response_data.get("success"):
- document_data = response_data.get("data", {})
- document = Document(**document_data)
-
- return ScrapeResponse(
- success=True,
- data=document,
- warning=response_data.get("warning")
- )
- else:
- return ScrapeResponse(
- success=False,
- error=response_data.get("error", "Unknown error occurred")
- )
+ if not url or not url.strip():
+ raise ValueError("URL cannot be empty")
+ request_data: Dict[str, Any] = {"url": url.strip()}
-def scrape_with_extract(
- client: HttpClient,
- url: str,
- schema: Dict[str, Any],
- options: Optional[ScrapeOptions] = None,
- system_prompt: Optional[str] = None,
- prompt: Optional[str] = None
-) -> ScrapeResponse:
- """
- Scrape a URL and extract structured data using a schema.
-
- Args:
- client: HTTP client instance
- url: URL to scrape
- schema: JSON schema for data extraction
- options: Scraping options
- system_prompt: System prompt for extraction
- prompt: User prompt for extraction
-
- Returns:
- ScrapeResponse with extracted data
-
- Raises:
- FirecrawlError: If the operation fails
- """
- # Prepare request data
- request_data = {
- "url": url,
- "extractorOptions": {
- "extractionSchema": schema
- }
- }
-
- if system_prompt:
- request_data["extractorOptions"]["extractionPrompt"] = system_prompt
-
- if prompt:
- request_data["extractorOptions"]["userPrompt"] = prompt
-
- if options:
- request_data["pageOptions"] = options.dict(exclude_none=True, by_alias=True)
-
- # Make the API request
- response = client.post("/v1/scrape", request_data)
-
- # Handle errors
- if not response.ok:
- handle_response_error(response, "scrape and extract from URL")
-
- # Parse response
- response_data = response.json()
-
- # Create document from response
- if response_data.get("success"):
- document_data = response_data.get("data", {})
- document = Document(**document_data)
-
- return ScrapeResponse(
- success=True,
- data=document,
- warning=response_data.get("warning")
- )
- else:
- return ScrapeResponse(
- success=False,
- error=response_data.get("error", "Unknown error occurred")
- )
+ if options is not None:
+ validated = validate_scrape_options(options)
+ if validated is not None:
+ opts = prepare_scrape_options(validated)
+ if opts:
+ request_data.update(opts)
+ return request_data
-def scrape_multiple(
- client: HttpClient,
- urls: list[str],
- options: Optional[ScrapeOptions] = None
-) -> list[ScrapeResponse]:
+def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
"""
- Scrape multiple URLs concurrently.
+ Scrape a single URL and return the document.
+
+ The v2 API returns: { success: boolean, data: Document }
+ We surface just the Document to callers.
Args:
client: HTTP client instance
- urls: List of URLs to scrape
- options: Scraping options
+ url: URL to scrape
+ options: Scraping options (snake_case)
Returns:
- List of ScrapeResponse objects
-
- Note:
- This function makes individual requests for each URL.
- For large batches, consider using batch_scrape instead.
+ Document
"""
- results = []
-
- for url in urls:
- try:
- result = scrape(client, url, options)
- results.append(result)
- except Exception as e:
- # Create error response for failed scrapes
- error_response = ScrapeResponse(
- success=False,
- error=f"Failed to scrape {url}: {str(e)}"
- )
- results.append(error_response)
-
- return results
-
+ payload = _prepare_scrape_request(url, options)
+ response = client.post("/v2/scrape", payload)
+ if not response.ok:
+ handle_response_error(response, "scrape")
+ body = response.json()
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
-def prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
- """
- Prepare a scrape request payload.
-
- Args:
- url: URL to scrape
- options: Scraping options
-
- Returns:
- Request payload dictionary
- """
- request_data = {"url": url}
-
- if options:
- validated_options = validate_scrape_options(options)
- if validated_options:
- request_data["options"] = validated_options.dict(exclude_none=True, by_alias=True)
-
- return request_data
\ No newline at end of file
+ document_data = body.get("data", {})
+ return Document(**document_data)
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index b4262314f..dc9a15cb9 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -79,11 +79,16 @@ SourceOption = Union[str, Source]
FormatString = Literal[
# camelCase versions (API format)
- "markdown", "html", "rawHtml", "links", "screenshot", "json", "changeTracking",
+ "markdown", "html", "rawHtml", "links", "screenshot", "summary", "changeTracking", "json",
# snake_case versions (user-friendly)
"raw_html", "change_tracking"
]
+class Viewport(BaseModel):
+ """Viewport configuration for screenshots."""
+ width: int
+ height: int
+
class Format(BaseModel):
"""Configuration for a format."""
type: FormatString
@@ -100,7 +105,14 @@ class ChangeTrackingFormat(Format):
prompt: Optional[str] = None
tag: Optional[str] = None
-FormatOption = Union[FormatString, Format, JsonFormat, ChangeTrackingFormat]
+class ScreenshotFormat(BaseModel):
+ """Configuration for screenshot format."""
+ type: Literal["screenshot"] = "screenshot"
+ full_page: Optional[bool] = None
+ quality: Optional[int] = None
+ viewport: Optional[Union[Dict[str, int], Viewport]] = None
+
+FormatOption = Union[FormatString, Format, JsonFormat, ChangeTrackingFormat, ScreenshotFormat]
# Scrape types
class ScrapeFormats(BaseModel):
@@ -136,22 +148,25 @@ class ScrapeFormats(BaseModel):
class ScrapeOptions(BaseModel):
"""Options for scraping operations."""
- formats: Optional[Union[ScrapeFormats, List[FormatOption]]] = None
+ formats: Optional[List[FormatOption]] = None
headers: Optional[Dict[str, str]] = None
include_tags: Optional[List[str]] = None
exclude_tags: Optional[List[str]] = None
- only_main_content: bool = True
+ only_main_content: Optional[bool] = None
timeout: Optional[int] = None
wait_for: Optional[int] = None
- mobile: bool = False
- skip_tls_verification: bool = False
- remove_base64_images: bool = True
- block_ads: bool = False
- proxy: Optional[str] = None
- max_age: Optional[int] = None
- store_in_cache: bool = False
- location: Optional['Location'] = None
+ mobile: Optional[bool] = None
+ parsers: Optional[List[str]] = None
actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None
+ location: Optional['Location'] = None
+ skip_tls_verification: Optional[bool] = None
+ remove_base64_images: Optional[bool] = None
+ fast_mode: Optional[bool] = None
+ use_mock: Optional[str] = None
+ block_ads: Optional[bool] = None
+ proxy: Optional[Literal["basic", "stealth", "auto"]] = None
+ max_age: Optional[int] = None
+ store_in_cache: Optional[bool] = None
@field_validator('formats')
@classmethod
@@ -313,8 +328,9 @@ class WaitAction(BaseModel):
class ScreenshotAction(BaseModel):
"""Screenshot action to perform during scraping."""
type: Literal["screenshot"] = "screenshot"
- full_page: Optional[bool] = Field(None, alias="fullPage")
+ full_page: Optional[bool] = None
quality: Optional[int] = None
+ viewport: Optional[Union[Dict[str, int], Viewport]] = None
class ClickAction(BaseModel):
"""Click action to perform during scraping."""
diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.py
index 4f85c19b9..39bb6acc7 100644
--- a/apps/python-sdk/firecrawl/v2/utils/validation.py
+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py
@@ -97,116 +97,111 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
if validated_options is None:
return None
+ # Apply default values for None fields
+ default_values = {
+ "only_main_content": True,
+ "mobile": False,
+ "skip_tls_verification": True,
+ "remove_base64_images": True,
+ "fast_mode": False,
+ "block_ads": True,
+ "max_age": 14400000,
+ "store_in_cache": True
+ }
+
# Convert to dict and handle manual snake_case to camelCase conversion
options_data = validated_options.model_dump(exclude_none=True)
+
+ # Apply defaults for None fields
+ for field, default_value in default_values.items():
+ if field not in options_data:
+ options_data[field] = default_value
+
scrape_data = {}
+ # Manual field mapping for snake_case to camelCase conversion
+ field_mappings = {
+ "include_tags": "includeTags",
+ "exclude_tags": "excludeTags",
+ "only_main_content": "onlyMainContent",
+ "wait_for": "waitFor",
+ "skip_tls_verification": "skipTlsVerification",
+ "remove_base64_images": "removeBase64Images",
+ "fast_mode": "fastMode",
+ "use_mock": "useMock",
+ "block_ads": "blockAds",
+ "store_in_cache": "storeInCache",
+ "max_age": "maxAge"
+ }
+
+ # Apply field mappings
+ for snake_case, camel_case in field_mappings.items():
+ if snake_case in options_data:
+ scrape_data[camel_case] = options_data.pop(snake_case)
+
+ # Handle special cases
for key, value in options_data.items():
if value is not None:
if key == "formats":
- # Handle formats - if it's a ScrapeFormats object, convert boolean flags to format strings
- if isinstance(value, dict) and any(key in value for key in ['markdown', 'html', 'raw_html', 'content', 'links', 'screenshot', 'screenshot_full_page']):
- # Convert ScrapeFormats boolean flags to format strings
- formats = []
- if value.get('markdown', False):
- formats.append("markdown")
- if value.get('html', False):
- formats.append("html")
- if value.get('raw_html', False):
- formats.append("rawHtml")
- if value.get('content', False):
- formats.append("content")
- if value.get('links', False):
- formats.append("links")
- if value.get('screenshot', False):
- formats.append("screenshot")
- if value.get('screenshot_full_page', False):
- formats.append("screenshot@fullPage")
- scrape_data["formats"] = formats
- elif hasattr(value, 'markdown') or hasattr(value, 'html') or hasattr(value, 'raw_html') or hasattr(value, 'content') or hasattr(value, 'links') or hasattr(value, 'screenshot') or hasattr(value, 'screenshot_full_page'):
- # Convert ScrapeFormats boolean flags to format strings
- formats = []
- if getattr(value, 'markdown', False):
- formats.append("markdown")
- if getattr(value, 'html', False):
- formats.append("html")
- if getattr(value, 'raw_html', False):
- formats.append("rawHtml")
- if getattr(value, 'content', False):
- formats.append("content")
- if getattr(value, 'links', False):
- formats.append("links")
- if getattr(value, 'screenshot', False):
- formats.append("screenshot")
- if getattr(value, 'screenshot_full_page', False):
- formats.append("screenshot@fullPage")
- scrape_data["formats"] = formats
- elif hasattr(value, 'formats') and value.formats is not None:
- # Convert format strings from snake_case to camelCase
- converted_formats = []
- for fmt in value.formats:
- if isinstance(fmt, str):
- converted_formats.append(_convert_format_string(fmt))
- elif hasattr(fmt, 'type'):
- converted_formats.append(_convert_format_string(fmt.type))
+ # Handle formats conversion
+ converted_formats = []
+ for fmt in value:
+ if isinstance(fmt, str):
+ # Handle format strings
+ if fmt == "json":
+ raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
+ converted_formats.append(_convert_format_string(fmt))
+ elif isinstance(fmt, dict):
+ # Handle format objects
+ if fmt.get('type') == 'json':
+ validated_json = _validate_json_format(fmt)
+ converted_formats.append(validated_json)
else:
+ # Convert other format objects
+ if 'type' in fmt:
+ fmt['type'] = _convert_format_string(fmt['type'])
converted_formats.append(fmt)
- scrape_data["formats"] = converted_formats
- elif isinstance(value, list):
- # If it's already a list, convert format strings from snake_case to camelCase
- converted_formats = []
- for fmt in value:
- if isinstance(fmt, str):
- # Handle json format specially - it can't be a string
- if fmt == "json":
- raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
- converted_formats.append(_convert_format_string(fmt))
- elif isinstance(fmt, dict):
- # Handle format objects (like json format)
- if fmt.get('type') == 'json':
- validated_json = _validate_json_format(fmt)
- converted_formats.append(validated_json)
+ elif hasattr(fmt, 'type'):
+ # Handle Format objects
+ if fmt.type == 'json':
+ # For json format, we need the full object
+ converted_formats.append(fmt.model_dump())
+ else:
+ # For other formats, just convert the type
+ converted_formats.append(_convert_format_string(fmt.type))
+ else:
+ converted_formats.append(fmt)
+ scrape_data["formats"] = converted_formats
+ elif key == "actions":
+ # Handle actions conversion
+ converted_actions = []
+ for action in value:
+ if isinstance(action, dict):
+ # Convert action dict
+ converted_action = {}
+ for action_key, action_value in action.items():
+ if action_key == "full_page":
+ converted_action["fullPage"] = action_value
else:
- # Convert other format objects
- if 'type' in fmt:
- fmt['type'] = _convert_format_string(fmt['type'])
- converted_formats.append(fmt)
- elif hasattr(fmt, 'type'):
- # Handle Format objects
- if fmt.type == 'json':
- # For json format, we need the full object
- converted_formats.append(fmt.model_dump())
+ converted_action[action_key] = action_value
+ converted_actions.append(converted_action)
+ else:
+ # Handle action objects
+ action_data = action.model_dump(exclude_none=True)
+ converted_action = {}
+ for action_key, action_value in action_data.items():
+ if action_key == "full_page":
+ converted_action["fullPage"] = action_value
else:
- # For other formats, just convert the type
- converted_formats.append(_convert_format_string(fmt.type))
- else:
- converted_formats.append(fmt)
- scrape_data["formats"] = converted_formats
+ converted_action[action_key] = action_value
+ converted_actions.append(converted_action)
+ scrape_data["actions"] = converted_actions
+ elif key == "location":
+ # Handle location conversion
+ if isinstance(value, dict):
+ scrape_data["location"] = value
else:
- # Fallback - use the value as-is
- scrape_data["formats"] = value
- elif key == "include_tags":
- scrape_data["includeTags"] = value
- elif key == "exclude_tags":
- scrape_data["excludeTags"] = value
- elif key == "only_main_content":
- scrape_data["onlyMainContent"] = value
- elif key == "wait_for":
- scrape_data["waitFor"] = value
- elif key == "skip_tls_verification":
- scrape_data["skipTlsVerification"] = value
- elif key == "remove_base64_images":
- scrape_data["removeBase64Images"] = value
- elif key == "block_ads":
- scrape_data["blockAds"] = value
- elif key == "store_in_cache":
- scrape_data["storeInCache"] = value
- elif key == "max_age":
- scrape_data["maxAge"] = value
- # Note: raw_html should be in formats array, not as a separate field
- # Note: screenshot_full_page is not supported by v2 API
- # elif key == "screenshot_full_page":
- # scrape_data["screenshot@fullPage"] = value
+ scrape_data["location"] = value.model_dump(exclude_none=True)
else:
# For fields that don't need conversion, use as-is
scrape_data[key] = value