Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(python-sdk): removed client duplication, bunch of type fixing, added map method + e2e/unit tests
rafaelmmillerdd6b46d373e8
diff --git a/apps/api/requests/v2/map.requests.http b/apps/api/requests/v2/map.requests.httpindex bbd7820e6..eb96552a5 100644--- a/apps/api/requests/v2/map.requests.http+++ b/apps/api/requests/v2/map.requests.http@@ -10,7 +10,7 @@ Authorization: Bearer {{$dotenv TEST_API_KEY}}content-type: application/json{- "url": "https://firecrawl.dev",+ "url": "https://firecrawl.dev"}# {diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.pyindex 848a55ef5..b9a53c537 100644--- a/apps/python-sdk/example.py+++ b/apps/python-sdk/example.py@@ -81,5 +81,9 @@ def main():print(search_response)+ # map example+ map_response = firecrawl.map("https://firecrawl.dev")+ print(map_response)+if __name__ == "__main__":main()\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/TODO.md b/apps/python-sdk/firecrawl/TODO.mdnew file mode 100644index 000000000..743e15e75--- /dev/null+++ b/apps/python-sdk/firecrawl/TODO.md@@ -0,0 +1,89 @@+- [ ] improve error handling for 500s+++============================================================================ FAILURES ============================================================================+_____________________________________________________ TestCrawlE2E.test_get_active_crawls_with_running_crawl _____________________________________________________++self = <test_crawl.TestCrawlE2E object at 0x1039a5810>++ def test_get_active_crawls_with_running_crawl(self):+ """Test getting active crawls when there's a running crawl."""+ # Start a crawl+ start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=5)+ assert start_job.id is not None++ # Get active crawls+> active_crawls_response = self.client.active_crawls()+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^++firecrawl/__tests__/e2e/v2/test_crawl.py:149:+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _+firecrawl/v2/client.py:386: in active_crawls+ return self.get_active_crawls()+ ^^^^^^^^^^^^^^^^^^^^^^^^+firecrawl/v2/client.py:377: in get_active_crawls+ return crawl_module.get_active_crawls(self.http_client)+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^+firecrawl/v2/methods/crawl.py:446: in get_active_crawls+ handle_response_error(response, "get active crawls")+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _++response = <Response [500]>, action = 'get active crawls'++ def handle_response_error(response: requests.Response, action: str) -> None:+ """+ Handle API response errors and raise appropriate exceptions.++ Args:+ response: The HTTP response object+ action: Description of the action being performed++ Raises:+ FirecrawlError: Appropriate error based on status code+ """+ try:+ response_json = response.json()+ error_message = response_json.get('error', 'No error message provided.')+ error_details = response_json.get('details', 'No additional error details provided.')+ except:+ # If we can't parse JSON, provide a helpful error message+ try:+ response_text = response.text[:500] # Limit to first 500 chars+ if response_text.strip():+ error_message = f"Server returned non-JSON response: {response_text}"+ error_details = f"Full response status: {response.status_code}"+ else:+ error_message = f"Server returned empty response with status {response.status_code}"+ error_details = "No additional details available"+ except:+ error_message = f"Server returned unreadable response with status {response.status_code}"+ error_details = "No additional details available"++ # Create appropriate error message+ if response.status_code == 400:+ message = f"Bad Request: Failed to {action}. {error_message} - {error_details}"+ raise BadRequestError(message, response.status_code, response)+ elif response.status_code == 401:+ message = f"Unauthorized: Failed to {action}. {error_message} - {error_details}"+ raise UnauthorizedError(message, response.status_code, response)+ elif response.status_code == 402:+ message = f"Payment Required: Failed to {action}. {error_message} - {error_details}"+ raise PaymentRequiredError(message, response.status_code, response)+ elif response.status_code == 403:+ message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"+ raise WebsiteNotSupportedError(message, response.status_code, response)+ elif response.status_code == 408:+ message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"+ raise RequestTimeoutError(message, response.status_code, response)+ elif response.status_code == 429:+ message = f"Rate Limit Exceeded: Failed to {action}. {error_message} - {error_details}"+ raise RateLimitError(message, response.status_code, response)+ elif response.status_code == 500:+ message = f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"+> raise InternalServerError(message, response.status_code, response)+E firecrawl.v2.utils.error_handler.InternalServerError: Internal Server Error: Failed to get active crawls. An unexpected error occurred. Please contact [email protected] for help. Your exception ID is c775af539e3a44f286664e55c488638e - No additional error details provided.++firecrawl/v2/utils/error_handler.py:104: InternalServerError+==================================================================== short test summary info =====================================================================+FAILED firecrawl/__tests__/e2e/v2/test_crawl.py::TestCrawlE2E::test_get_active_crawls_with_running_crawl - firecrawl.v2.utils.error_handler.InternalServerError: Internal Server Error: Failed to get active crawls. An unexpected error occurred. Please contact help@f...+======================================================================= 1 failed in 1.17s ========================================================================diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.pyindex 3a70e5e55..8f6d41c11 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py@@ -194,7 +194,7 @@ class TestCrawlE2E:def test_crawl_with_scrape_options(self):"""Test crawl with scrape options."""scrape_opts = ScrapeOptions(- formats=["markdown"],+ formats=["markdown", "links"],only_main_content=False,mobile=True,)@@ -207,6 +207,25 @@ class TestCrawlE2E:assert crawl_job.id is not None+ def test_crawl_with_json_format_object(self):+ """Crawl with scrape_options including a JSON format object (prompt + schema)."""+ json_schema = {+ "type": "object",+ "properties": {+ "title": {"type": "string"}+ },+ "required": ["title"],+ }+ scrape_opts = ScrapeOptions(+ formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]+ )+ crawl_job = self.client.start_crawl(+ "https://docs.firecrawl.dev",+ limit=2,+ scrape_options=scrape_opts+ )+ assert crawl_job.id is not None+def test_crawl_all_parameters(self):"""Test crawl with all possible parameters."""scrape_opts = ScrapeOptions(diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.pynew file mode 100644index 000000000..02f6f70bc--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.py@@ -0,0 +1,68 @@+import os+from dotenv import load_dotenv+from firecrawl import Firecrawl+++load_dotenv()++if not os.getenv("API_KEY"):+ raise ValueError("API_KEY is not set")++if not os.getenv("API_URL"):+ raise ValueError("API_URL is not set")+++import pytest+++class TestMapE2E:+ """End-to-end tests for map functionality (v2)."""++ def setup_method(self):+ self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))++ def test_map_minimal_request(self):+ resp = self.client.map("https://docs.firecrawl.dev")++ assert hasattr(resp, "success") and resp.success is True+ assert hasattr(resp, "data") and resp.data is not None+ assert hasattr(resp.data, "links")+ assert isinstance(resp.data.links, list)++ # Basic sanity checks on at least one link+ if len(resp.data.links) > 0:+ first = resp.data.links[0]+ assert hasattr(first, "url")+ assert isinstance(first.url, str) and first.url.startswith("http")++ @pytest.mark.parametrize(+ "sitemap_only,ignore_sitemap",+ [+ (True, None), # sitemap: only+ (None, True), # sitemap: skip+ (None, None), # sitemap: include (default)+ ],+ )+ def test_map_with_options(self, sitemap_only, ignore_sitemap):+ kwargs = {+ "search": "docs",+ "include_subdomains": True,+ "limit": 10,+ }+ if sitemap_only is not None:+ kwargs["sitemap_only"] = sitemap_only+ if ignore_sitemap is not None:+ kwargs["ignore_sitemap"] = ignore_sitemap++ resp = self.client.map("https://docs.firecrawl.dev", **kwargs)++ assert hasattr(resp, "success") and resp.success is True+ assert hasattr(resp, "data") and resp.data is not None+ assert isinstance(resp.data.links, list)++ # Limit should be respected (server-side)+ assert len(resp.data.links) <= 10++ for link in resp.data.links:+ assert hasattr(link, "url")+ assert isinstance(link.url, str) and link.url.startswith("http")diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.pyindex 8c9ca7ebc..7acaa9d75 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py@@ -2,6 +2,8 @@ import osimport pytestfrom dotenv import load_dotenvfrom firecrawl import Firecrawl+import json as _json+import pytestfrom firecrawl.v2.types import Viewport, ScreenshotAction, Documentload_dotenv()@@ -56,6 +58,55 @@ class TestScrapeE2E:)self._assert_valid_document(doc)+ @pytest.mark.parametrize("fmt,expect_field", [+ ("markdown", "markdown"),+ ("html", "html"),+ ("raw_html", "raw_html"),+ ("links", "links"),+ ("screenshot", "screenshot"),+ ])+ def test_scrape_basic_formats(self, fmt, expect_field):+ """Verify basic formats request succeeds and expected fields are present when applicable."""+ doc = self.client.scrape(+ "https://docs.firecrawl.dev",+ formats=[fmt],+ )+ # For formats that are not content (links/screenshot/json), skip main-content assertion+ if expect_field not in {"links", "screenshot"}:+ self._assert_valid_document(doc)+ if expect_field == "markdown":+ assert doc.markdown is not None+ elif expect_field == "html":+ assert doc.html is not None+ elif expect_field == "raw_html":+ assert doc.raw_html is not None+ elif expect_field == "screenshot":+ assert doc.screenshot is not None+ elif expect_field == "links":+ assert isinstance(doc.links, list)+ assert len(doc.links) > 0++ def test_scrape_with_json_format_object(self):+ """Scrape with JSON format object (requires prompt and schema)."""+ json_schema = {+ "type": "object",+ "properties": {+ "title": {"type": "string"}+ },+ "required": ["title"],+ }+ doc = self.client.scrape(+ "https://docs.firecrawl.dev",+ formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}],+ only_main_content=True,+ )+ # JSON format may not include main content fields; ensure request succeeded+ assert isinstance(doc, Document)+ # If backend returns extracted json content, it should be present under `json`+ # (Do not fail if backend omits it; existence depends on implementation)+ # if hasattr(doc, 'json'):+ # assert doc.json is not None+def test_scrape_invalid_url(self):"""Scrape should fail with empty or invalid URLs."""with pytest.raises(ValueError, match="URL cannot be empty"):diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.pyindex 7dbbc54f4..deb309c1b 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py@@ -117,7 +117,15 @@ def test_search_all_parameters():ignore_invalid_urls=True,timeout=60000,scrape_options=ScrapeOptions(- formats=["markdown", "html"],+ formats=[+ "markdown",+ "html",+ {+ "type": "json",+ "prompt": "Extract the title and description from the page",+ "schema": schema+ }+ ],headers={"User-Agent": "Firecrawl-Test/1.0"},include_tags=["h1", "h2", "p"],exclude_tags=["nav", "footer"],@@ -208,4 +216,23 @@ def test_search_formats_flexibility():assert isinstance(results1, SearchData)assert isinstance(results2, SearchData)assert results1.web is not None- assert results2.web is not None\ No newline at end of file+ assert results2.web is not None++def test_search_with_json_format_object():+ """Search with scrape_options including a JSON format object (prompt + schema)."""+ json_schema = {+ "type": "object",+ "properties": {+ "title": {"type": "string"}+ },+ "required": ["title"],+ }+ results = firecrawl.search(+ query="site:docs.firecrawl.dev",+ limit=1,+ scrape_options=ScrapeOptions(+ formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]+ ),+ )+ assert isinstance(results, SearchData)+ assert results.web is not None and len(results.web) >= 0\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/map/test_map_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/map/test_map_request_preparation.pynew file mode 100644index 000000000..d4f8a6a4e--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/map/test_map_request_preparation.py@@ -0,0 +1,51 @@+import pytest+from firecrawl.v2.types import MapOptions+from firecrawl.v2.methods.map import _prepare_map_request+++class TestMapRequestPreparation:+ """Unit tests for map request preparation."""++ def test_basic_request_preparation(self):+ data = _prepare_map_request("https://example.com")+ assert data["url"] == "https://example.com"+ # Default sitemap handling should be "include" when no flags provided+ assert "sitemap" not in data # we only send when options provided++ def test_sitemap_transformations(self):+ # sitemap_only -> sitemap: "only"+ opts = MapOptions(sitemap_only=True)+ data = _prepare_map_request("https://example.com", opts)+ assert data["sitemap"] == "only"++ # ignore_sitemap -> sitemap: "skip"+ opts = MapOptions(ignore_sitemap=True)+ data = _prepare_map_request("https://example.com", opts)+ assert data["sitemap"] == "skip"++ # default when options present but neither flag set -> include+ opts = MapOptions(search="docs")+ data = _prepare_map_request("https://example.com", opts)+ assert data["sitemap"] == "include"++ def test_field_conversions(self):+ opts = MapOptions(+ search="docs",+ include_subdomains=True,+ limit=25,+ sitemap_only=True,+ )+ data = _prepare_map_request("https://example.com", opts)++ assert data["url"] == "https://example.com"+ assert data["search"] == "docs"+ assert data["includeSubdomains"] is True+ assert data["limit"] == 25+ assert data["sitemap"] == "only"++ def test_invalid_url(self):+ with pytest.raises(ValueError):+ _prepare_map_request("")+ with pytest.raises(ValueError):+ _prepare_map_request(" ")+diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex b812481f7..31d04da3d 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -62,7 +62,7 @@ class V2Proxy:self.start_crawl = client_instance.start_crawlself.crawl_params_preview = client_instance.crawl_params_preview# self.batch_scrape = client_instance.batch_scrape- # self.map = client_instance.map+ self.map = client_instance.mapdef __getattr__(self, name):"""Forward attribute access to the underlying client."""@@ -141,7 +141,7 @@ class Firecrawl:self.active_crawls = self._v2_client.active_crawls# self.batch_scrape = self._v2_client.batch_scrape- # self.map = self._v2_client.map+ self.map = self._v2_client.mapself.search = self._v2_client.searchclass AsyncFirecrawl:diff --git a/apps/python-sdk/firecrawl/v2/__init__.py b/apps/python-sdk/firecrawl/v2/__init__.pyindex bc45080fd..0904fc44b 100644--- a/apps/python-sdk/firecrawl/v2/__init__.py+++ b/apps/python-sdk/firecrawl/v2/__init__.py@@ -1,261 +1,3 @@-from typing import Optional, List, Union, Dict, Any-from ..types import (- SearchData, Document, ScrapeOptions, SearchRequest,- SourceOption, FormatOption, CrawlRequest, CrawlJob, CrawlResponse,- CrawlParamsRequest, CrawlParamsData, CrawlErrorsResponse, ActiveCrawlsResponse,- WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction, Location-)-from .methods.search import search as search_method-from .methods.crawl import (- crawl as crawl_method,- start_crawl as start_crawl_method,- cancel_crawl as cancel_crawl_method,- get_crawl_status as get_crawl_status_method,- crawl_params_preview as crawl_params_preview_method,- get_crawl_errors as get_crawl_errors_method,- get_active_crawls as get_active_crawls_method-)-from .methods.scrape import scrape as scrape_method-from .utils.http_client import HttpClient+from .client import FirecrawlClient-class FirecrawlClient:- """- Firecrawl v2 API client.- """-- def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):- self.api_key = api_key- self.api_url = api_url- self._client = HttpClient(api_key=self.api_key, api_url=self.api_url)-- def scrape(- self,- url: str,- formats: Optional[List[FormatOption]] = None,- headers: Optional[Dict[str, str]] = None,- include_tags: Optional[List[str]] = None,- exclude_tags: Optional[List[str]] = None,- only_main_content: Optional[bool] = None,- timeout: Optional[int] = None,- wait_for: Optional[int] = None,- mobile: Optional[bool] = None,- parsers: Optional[List[str]] = None,- actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]] = None,- location: Optional[Location] = None,- skip_tls_verification: Optional[bool] = None,- remove_base64_images: Optional[bool] = None,- fast_mode: Optional[bool] = None,- use_mock: Optional[str] = None,- block_ads: Optional[bool] = None,- proxy: Optional[str] = None,- max_age: Optional[int] = None,- store_in_cache: Optional[bool] = None,- ) -> Document:- """Scrape a single URL and return the document."""- options = ScrapeOptions(- formats=formats,- headers=headers,- include_tags=include_tags,- exclude_tags=exclude_tags,- only_main_content=only_main_content,- timeout=timeout,- wait_for=wait_for,- mobile=mobile,- parsers=parsers,- actions=actions,- location=location,- skip_tls_verification=skip_tls_verification,- remove_base64_images=remove_base64_images,- fast_mode=fast_mode,- use_mock=use_mock,- block_ads=block_ads,- proxy=proxy,- max_age=max_age,- store_in_cache=store_in_cache- )- return scrape_method(self._client, url, options)-- # batch-scrape- def batch_scrape(- self- ):- pass-- # start-batch-scrape- def start_batch_scrape(- self- ):- pass-- # get-batch-scrape-status- def get_batch_scrape_status(- self- ):- pass-- # cancel-batch-scrape- def cancel_batch_scrape(- self- ):- pass- # get-batch-scrape-errors- def get_batch_scrape_errors(- self- ):- pass-- def crawl(- self,- url: str,- prompt: Optional[str] = None,- include_paths: Optional[List[str]] = None,- exclude_paths: Optional[List[str]] = None,- max_discovery_depth: Optional[int] = None,- ignore_sitemap: bool = False,- ignore_query_parameters: bool = False,- limit: Optional[int] = None,- crawl_entire_domain: bool = False,- allow_external_links: bool = False,- allow_subdomains: bool = False,- delay: Optional[int] = None,- max_concurrency: Optional[int] = None,- webhook: Optional[Dict[str, Any]] = None,- scrape_options: Optional[ScrapeOptions] = None,- zero_data_retention: bool = False,- poll_interval: int = 2,- timeout: Optional[int] = None- ) -> CrawlJob:- """Start a crawl job and wait for it to complete."""- request = CrawlRequest(- url=url,- prompt=prompt,- include_paths=include_paths,- exclude_paths=exclude_paths,- max_discovery_depth=max_discovery_depth,- ignore_sitemap=ignore_sitemap,- ignore_query_parameters=ignore_query_parameters,- limit=limit,- crawl_entire_domain=crawl_entire_domain,- allow_external_links=allow_external_links,- allow_subdomains=allow_subdomains,- delay=delay,- max_concurrency=max_concurrency,- webhook=webhook,- scrape_options=scrape_options,- zero_data_retention=zero_data_retention- )- return crawl_method(self._client, request, poll_interval, timeout)-- def start_crawl(- self,- url: str,- prompt: Optional[str] = None,- include_paths: Optional[List[str]] = None,- exclude_paths: Optional[List[str]] = None,- max_discovery_depth: Optional[int] = None,- ignore_sitemap: bool = False,- ignore_query_parameters: bool = False,- limit: Optional[int] = None,- crawl_entire_domain: bool = False,- allow_external_links: bool = False,- allow_subdomains: bool = False,- delay: Optional[int] = None,- max_concurrency: Optional[int] = None,- webhook: Optional[Dict[str, Any]] = None,- scrape_options: Optional[ScrapeOptions] = None,- zero_data_retention: bool = False- ) -> CrawlResponse:- """Start a crawl job and return immediately."""- request = CrawlRequest(- url=url,- prompt=prompt,- include_paths=include_paths,- exclude_paths=exclude_paths,- max_discovery_depth=max_discovery_depth,- ignore_sitemap=ignore_sitemap,- ignore_query_parameters=ignore_query_parameters,- limit=limit,- crawl_entire_domain=crawl_entire_domain,- allow_external_links=allow_external_links,- allow_subdomains=allow_subdomains,- delay=delay,- max_concurrency=max_concurrency,- webhook=webhook,- scrape_options=scrape_options,- zero_data_retention=zero_data_retention- )- return start_crawl_method(self._client, request)-- def get_crawl_status(- self,- job_id: str- ) -> CrawlJob:- """Get the status of a crawl job."""- return get_crawl_status_method(self._client, job_id)-- def cancel_crawl(- self,- job_id: str- ) -> bool:- """Cancel a running crawl job."""- return cancel_crawl_method(self._client, job_id)-- def crawl_params_preview(- self,- url: str,- prompt: str- ) -> CrawlParamsData:- """Get crawl parameters from LLM based on URL and prompt."""- request = CrawlParamsRequest(url=url, prompt=prompt)- return crawl_params_preview_method(self._client, request)-- def active_crawls(- self- ) -> ActiveCrawlsResponse:- """Get a list of active crawl jobs."""- return get_active_crawls_method(self._client)-- def get_crawl_errors(- self,- crawl_id: str- ) -> CrawlErrorsResponse:- """Get errors from a crawl job."""- return get_crawl_errors_method(self._client, crawl_id)-- # map- def map(- self- ):- pass-- def search(- self,- query: str,- sources: Optional[List[SourceOption]] = None,- limit: Optional[int] = 5,- tbs: Optional[str] = None,- location: Optional[str] = None,- ignore_invalid_urls: Optional[bool] = True,- timeout: Optional[int] = 60000,- scrape_options: Optional[ScrapeOptions] = None,- ) -> SearchData:- """Search for documents."""- request = SearchRequest(- query=query,- sources=sources,- limit=limit,- tbs=tbs,- location=location,- ignore_invalid_urls=ignore_invalid_urls,- timeout=timeout,- scrape_options=scrape_options,- )- return search_method(self._client, request)-- # credit-usage- def credit_usage(- self- ):- pass--__all__ = ['FirecrawlClient']\ No newline at end of file+__all__ = ["FirecrawlClient"]\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex c06007de4..af3f505b5 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -7,11 +7,33 @@ This module provides the main client class that orchestrates all v2 functionalitimport osfrom typing import Optional, List, Dict, Any, Callable, Unionfrom .types import (- ClientConfig, ScrapeOptions, CrawlOptions, MapOptions, ExtractOptions,- ScrapeResponse, CrawlResponse, CrawlStatusResponse, BatchScrapeResponse,- BatchScrapeStatusResponse, MapResponse, ExtractResponse, Document,- SearchRequest, SearchResponse, CrawlRequest, WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse,- FormatOption, WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction, Location,+ ClientConfig,+ ScrapeOptions,+ Document,+ SearchRequest,+ SearchData,+ SourceOption,+ CrawlRequest,+ CrawlResponse,+ CrawlJob,+ CrawlParamsRequest,+ CrawlParamsData,+ WebhookConfig,+ CrawlErrorsResponse,+ ActiveCrawlsResponse,+ MapOptions,+ MapResponse,+ FormatOption,+ WaitAction,+ ScreenshotAction,+ ClickAction,+ WriteAction,+ PressAction,+ ScrollAction,+ ScrapeAction,+ ExecuteJavascriptAction,+ PDFAction,+ Location,)from .utils.http_client import HttpClientfrom .utils.error_handler import FirecrawlError@@ -19,6 +41,7 @@ from .methods import scrape as scrape_modulefrom .methods import crawl as crawl_modulefrom .methods import batch as batch_modulefrom .methods import search as search_module+from .methods import map as map_moduleclass FirecrawlClient:"""@@ -143,12 +166,14 @@ class FirecrawlClient:self,query: str,*,+ sources: Optional[List[SourceOption]] = None,limit: Optional[int] = None,tbs: Optional[str] = None,location: Optional[str] = None,+ ignore_invalid_urls: Optional[bool] = None,timeout: Optional[int] = None,- page_options: Optional[ScrapeOptions] = None,- ) -> SearchResponse:+ scrape_options: Optional[ScrapeOptions] = None,+ ) -> SearchData:"""Search for documents.@@ -161,17 +186,20 @@ class FirecrawlClient:page_options: Options for scraping individual pagesReturns:- SearchResponse containing the search results+ SearchData containing the search results"""- options = SearchRequest(+ request = SearchRequest(+ query=query,+ sources=sources,limit=limit,tbs=tbs,location=location,+ ignore_invalid_urls=ignore_invalid_urls,timeout=timeout,- page_options=page_options+ scrape_options=scrape_options,)-- return search_module.search(self.http_client, query, options)++ return search_module.search(self.http_client, request)def crawl(self,@@ -194,7 +222,7 @@ class FirecrawlClient:zero_data_retention: bool = False,poll_interval: int = 2,timeout: Optional[int] = None- ) -> CrawlStatusResponse:+ ) -> CrawlJob:"""Start a crawl job and wait for it to complete.@@ -219,7 +247,7 @@ class FirecrawlClient:timeout: Maximum seconds to wait (None for no timeout)Returns:- CrawlStatusResponse when job completes+ CrawlJob when job completesRaises:ValueError: If request is invalid@@ -321,7 +349,7 @@ class FirecrawlClient:return crawl_module.start_crawl(self.http_client, request)- def get_crawl_status(self, job_id: str) -> CrawlStatusResponse:+ def get_crawl_status(self, job_id: str) -> CrawlJob:"""Get the status of a crawl job.@@ -329,24 +357,24 @@ class FirecrawlClient:job_id: ID of the crawl jobReturns:- CrawlStatusResponse with current status and data+ CrawlJob with current status and dataRaises:Exception: If the status check fails"""return crawl_module.get_crawl_status(self.http_client, job_id)- def check_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse:+ def get_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse:"""- Get errors from a crawl job.+ Retrieve error details and robots.txt blocks for a given crawl job.Args:crawl_id: The ID of the crawl job-+Returns:- CrawlErrorsResponse containing errors and robots blocked URLs+ CrawlErrorsResponse containing per-URL errors and robots-blocked URLs"""- return crawl_module.check_crawl_errors(self.http_client, crawl_id)+ return crawl_module.get_crawl_errors(self.http_client, crawl_id)def get_active_crawls(self) -> ActiveCrawlsResponse:"""@@ -357,6 +385,38 @@ class FirecrawlClient:"""return crawl_module.get_active_crawls(self.http_client)+ def active_crawls(self) -> ActiveCrawlsResponse:+ """+ List currently active crawl jobs for the authenticated team.++ Returns:+ ActiveCrawlsResponse containing the list of active crawl jobs+ """+ return self.get_active_crawls()++ def map(+ self,+ url: str,+ *,+ search: Optional[str] = None,+ include_subdomains: Optional[bool] = None,+ limit: Optional[int] = None,+ ignore_sitemap: Optional[bool] = None,+ sitemap_only: Optional[bool] = None,+ ) -> MapResponse:+ """+ Map a URL and return discovered links (with optional titles/descriptions).+ """+ options = MapOptions(+ search=search,+ include_subdomains=include_subdomains,+ limit=limit,+ ignore_sitemap=ignore_sitemap,+ sitemap_only=sitemap_only,+ ) if any(v is not None for v in [search, include_subdomains, limit, ignore_sitemap, sitemap_only]) else None++ return map_module.map(self.http_client, url, options)+def cancel_crawl(self, crawl_id: str) -> bool:"""Cancel a crawl job.@@ -368,4 +428,11 @@ class FirecrawlClient:bool: True if the crawl was cancelled, False otherwise"""return crawl_module.cancel_crawl(self.http_client, crawl_id)++ def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData:+ """+ Get crawl parameters from LLM based on URL and prompt.+ """+ request = CrawlParamsRequest(url=url, prompt=prompt)+ return crawl_module.crawl_params_preview(self.http_client, request)\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.pyindex 2a19eda02..b10ba770a 100644--- a/apps/python-sdk/firecrawl/v2/methods/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py@@ -4,11 +4,11 @@ Batch scraping functionality for Firecrawl v2 API.import timefrom typing import Optional, List, Callable-from .types import (+from ..types import (BatchScrapeRequest, BatchScrapeResponse,BatchScrapeJob, BatchScrapeData, ScrapeOptions, Document)-from .utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options+from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_optionsdef start_batch_scrape(@@ -34,7 +34,7 @@ def start_batch_scrape(request_data = prepare_batch_request(urls, options)# Make the API request- response = client.post("/v1/batch/scrape", request_data)+ response = client.post("/v2/batch/scrape", request_data)# Handle errorsif not response.ok:@@ -77,7 +77,7 @@ def get_batch_scrape_status(FirecrawlError: If the status check fails"""# Make the API request- response = client.get(f"/v1/batch/scrape/{job_id}")+ response = client.get(f"/v2/batch/scrape/{job_id}")# Handle errorsif not response.ok:@@ -132,7 +132,7 @@ def cancel_batch_scrape(FirecrawlError: If the cancellation fails"""# Make the API request- response = client.delete(f"/v1/batch/scrape/{job_id}")+ response = client.delete(f"/v2/batch/scrape/{job_id}")# Handle errorsif not response.ok:@@ -235,7 +235,7 @@ def batch_scrape_and_wait(batch_response = start_batch_scrape(client, urls, options)if not batch_response.success or not batch_response.data:- return BatchScrapeStatusResponse(+ return BatchScrapeResponse(success=False,error=batch_response.error or "Failed to start batch scrape")diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex 1894723f1..b6b2ee1ac 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -8,7 +8,7 @@ from ..types import (CrawlRequest,CrawlJob,CrawlResponse, Document, CrawlParamsRequest, CrawlParamsResponse, CrawlParamsData,- WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse+ WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, ActiveCrawl)from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options@@ -416,13 +416,19 @@ def get_crawl_errors(http_client: HttpClient, crawl_id: str) -> CrawlErrorsRespoException: If the request fails"""response = http_client.get(f"/v2/crawl/{crawl_id}/errors")-- if response.status_code != 200:++ if not response.ok:handle_response_error(response, "check crawl errors")-+try:- data = response.json()- return CrawlErrorsResponse(**data)+ body = response.json()+ payload = body.get("data", body)+ # Manual key normalization since we avoid Pydantic aliases+ normalized = {+ "errors": payload.get("errors", []),+ "robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),+ }+ return CrawlErrorsResponse(**normalized)except Exception as e:raise Exception(f"Failed to parse crawl errors response: {e}")@@ -441,13 +447,23 @@ def get_active_crawls(client: HttpClient) -> ActiveCrawlsResponse:Exception: If the request fails"""response = client.get("/v2/crawl/active")-+if not response.ok:handle_response_error(response, "get active crawls")-- response_data = response.json()-- if response_data.get("success"):- return ActiveCrawlsResponse(**response_data.get("data", {}))- else:- raise Exception(response_data.get("error", "Unknown error occurred"))++ body = response.json()+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))++ data = body.get("data", {})+ crawls_in = data.get("crawls", [])+ normalized_crawls = []+ for c in crawls_in:+ if isinstance(c, dict):+ normalized_crawls.append({+ "id": c.get("id"),+ "team_id": c.get("teamId", c.get("team_id")),+ "url": c.get("url"),+ "options": c.get("options"),+ })+ return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized_crawls])diff --git a/apps/python-sdk/firecrawl/v2/methods/map.py b/apps/python-sdk/firecrawl/v2/methods/map.pynew file mode 100644index 000000000..7daae4457--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/map.py@@ -0,0 +1,69 @@+"""+Mapping functionality for Firecrawl v2 API.+"""++from typing import Optional, Dict, Any+from ..types import MapOptions, MapResponse, MapData, LinkResult+from ..utils import HttpClient, handle_response_error+++def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict[str, Any]:+ if not url or not url.strip():+ raise ValueError("URL cannot be empty")++ payload: Dict[str, Any] = {"url": url.strip()}++ if options is not None:+ # Transform ignoreSitemap and sitemapOnly to the new unified sitemap parameter+ # sitemap: "only" | "include" | "skip" (default include)+ sitemap: str = "include"+ if options.sitemap_only:+ sitemap = "only"+ elif options.ignore_sitemap:+ sitemap = "skip"++ data: Dict[str, Any] = {}+ data["sitemap"] = sitemap++ if options.search is not None:+ data["search"] = options.search+ if options.include_subdomains is not None:+ data["includeSubdomains"] = options.include_subdomains+ if options.limit is not None:+ data["limit"] = options.limit+ payload.update(data)++ return payload+++def map(client: HttpClient, url: str, options: Optional[MapOptions] = None) -> MapResponse:+ """+ Map a URL and return a list of links with optional titles/descriptions.+ """+ request_data = _prepare_map_request(url, options)++ response = client.post("/v2/map", request_data)++ if not response.ok:+ handle_response_error(response, "map")++ body = response.json()+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))++ data = body.get("data", {})+ result_links: list[LinkResult] = []+ for item in data.get("links", []):+ if isinstance(item, dict):+ result_links.append(+ LinkResult(+ url=item.get("url", ""),+ title=item.get("title"),+ description=item.get("description"),+ )+ )+ elif isinstance(item, str):+ result_links.append(LinkResult(url=item))++ return MapResponse(success=True, data=MapData(links=result_links))+diff --git a/apps/python-sdk/firecrawl/v2/methods/scrape.py b/apps/python-sdk/firecrawl/v2/methods/scrape.pyindex 0eaf7e720..408d8d378 100644--- a/apps/python-sdk/firecrawl/v2/methods/scrape.py+++ b/apps/python-sdk/firecrawl/v2/methods/scrape.py@@ -59,4 +59,10 @@ def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = Noneraise Exception(body.get("error", "Unknown error occurred"))document_data = body.get("data", {})- return Document(**document_data)\ No newline at end of file+ # Normalize keys for Document (no Pydantic aliases)+ normalized = dict(document_data)+ if 'rawHtml' in normalized and 'raw_html' not in normalized:+ normalized['raw_html'] = normalized.pop('rawHtml')+ if 'changeTracking' in normalized and 'change_tracking' not in normalized:+ normalized['change_tracking'] = normalized.pop('changeTracking')+ return Document(**normalized)\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.pyindex 6ac1d76c8..1495b0f34 100644--- a/apps/python-sdk/firecrawl/v2/methods/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/search.py@@ -3,7 +3,7 @@ Search functionality for Firecrawl v2 API."""from typing import Optional, Dict, Any, Union-from ...types import SearchRequest, SearchData, SearchResult, Document+from ..types import SearchRequest, SearchData, SearchResult, Documentfrom ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options@@ -46,9 +46,19 @@ def search(results = []for doc_data in source_documents:if isinstance(doc_data, dict):- if any(key in doc_data for key in ['markdown', 'html', 'content', 'screenshot']):- results.append(Document(**doc_data))+ # If page scraping options were provided, API returns full Document objects+ if request.scrape_options is not None and any(+ key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'screenshot', 'changeTracking']+ ):+ # Normalize keys for Document (no Pydantic aliases)+ normalized = dict(doc_data)+ if 'rawHtml' in normalized and 'raw_html' not in normalized:+ normalized['raw_html'] = normalized.pop('rawHtml')+ if 'changeTracking' in normalized and 'change_tracking' not in normalized:+ normalized['change_tracking'] = normalized.pop('changeTracking')+ results.append(Document(**normalized))else:+ # Minimal search result shaperesults.append(SearchResult(url=doc_data.get('url', ''),title=doc_data.get('title'),@@ -56,8 +66,7 @@ def search())elif isinstance(doc_data, str):results.append(SearchResult(url=doc_data))-- # Set the appropriate field based on source type+if hasattr(search_data, source_type):setattr(search_data, source_type, results)diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex dc9a15cb9..f5a265193 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -15,6 +15,7 @@ warnings.filterwarnings("ignore", message="Field name \"schema\" in \"Format\" swarnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonFormat\" shadows an attribute in parent \"Format\"")warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ChangeTrackingFormat\" shadows an attribute in parent \"Format\"")warnings.filterwarnings("ignore", message="Field name \"json\" in \"ScrapeFormats\" shadows an attribute in parent \"BaseModel\"")+warnings.filterwarnings("ignore", message="Field name \"json\" in \"Document\" shadows an attribute in parent \"BaseModel\"")T = TypeVar('T')@@ -34,25 +35,26 @@ class DocumentMetadata(BaseModel):language: Optional[str] = Nonekeywords: Optional[Union[str, List[str]]] = Nonerobots: Optional[str] = None- og_title: Optional[str] = Field(None, alias="ogTitle")- og_description: Optional[str] = Field(None, alias="ogDescription")- og_url: Optional[str] = Field(None, alias="ogUrl")- og_image: Optional[str] = Field(None, alias="ogImage")- source_url: Optional[str] = Field(None, alias="sourceURL")- status_code: Optional[int] = Field(None, alias="statusCode")+ og_title: Optional[str] = None+ og_description: Optional[str] = None+ og_url: Optional[str] = None+ og_image: Optional[str] = None+ source_url: Optional[str] = None+ status_code: Optional[int] = Noneerror: Optional[str] = Noneclass Document(BaseModel):"""A scraped document."""markdown: Optional[str] = Nonehtml: Optional[str] = None- raw_html: Optional[str] = Field(None, alias="rawHtml")+ raw_html: Optional[str] = None+ json: Optional[Any] = Nonemetadata: Optional[DocumentMetadata] = Nonelinks: Optional[List[str]] = Nonescreenshot: Optional[str] = Noneactions: Optional[Dict[str, Any]] = Nonewarning: Optional[str] = None- change_tracking: Optional[Dict[str, Any]] = Field(None, alias="changeTracking")+ change_tracking: Optional[Dict[str, Any]] = None# Webhook typesclass WebhookConfig(BaseModel):@@ -64,7 +66,7 @@ class WebhookConfig(BaseModel):class WebhookData(BaseModel):"""Data sent to webhooks."""- job_id: str = Field(alias="jobId")+ job_id: strstatus: strcurrent: Optional[int] = Nonetotal: Optional[int] = None@@ -120,7 +122,7 @@ class ScrapeFormats(BaseModel):formats: Optional[List[FormatOption]] = Nonemarkdown: bool = Truehtml: bool = False- raw_html: bool = Field(False, alias="rawHtml")+ raw_html: bool = Falselinks: bool = Falsescreenshot: bool = Falsechange_tracking: bool = False@@ -148,7 +150,7 @@ class ScrapeFormats(BaseModel):class ScrapeOptions(BaseModel):"""Options for scraping operations."""- formats: Optional[List[FormatOption]] = None+ formats: Optional[Union['ScrapeFormats', List[FormatOption]]] = Noneheaders: Optional[Dict[str, str]] = Noneinclude_tags: Optional[List[str]] = Noneexclude_tags: Optional[List[str]] = None@@ -174,15 +176,10 @@ class ScrapeOptions(BaseModel):"""Validate and normalize formats input."""if v is None:return v-- # If it's already a ScrapeFormats object, return as isif isinstance(v, ScrapeFormats):return v-- # If it's a list, keep it as a list (don't convert to ScrapeFormats)if isinstance(v, list):return v-raise ValueError(f"Invalid formats type: {type(v)}. Expected ScrapeFormats or List[FormatOption]")class ScrapeRequest(BaseModel):@@ -236,6 +233,7 @@ class CrawlJob(BaseModel):class SearchDocument(Document):"""A document from a search operation with URL and description."""url: str+ title: Optional[str] = Nonedescription: Optional[str] = Noneclass MapDocument(Document):@@ -283,8 +281,8 @@ class BatchScrapeJob(BaseModel):status: Literal["scraping", "completed", "failed"]current: Optional[int] = Nonetotal: Optional[int] = None- created_at: Optional[datetime] = Field(None, alias="createdAt")- completed_at: Optional[datetime] = Field(None, alias="completedAt")+ created_at: Optional[datetime] = None+ completed_at: Optional[datetime] = Noneclass BatchScrapeData(BaseModel):"""Batch scrape results data."""@@ -301,8 +299,9 @@ class BatchScrapeResponse(BaseResponse[BatchScrapeData]):class MapOptions(BaseModel):"""Options for mapping operations."""search: Optional[str] = None- ignore_sitemap: bool = Field(False, alias="ignoreSitemap")- include_subdomains: bool = Field(False, alias="includeSubdomains")+ ignore_sitemap: Optional[bool] = None+ sitemap_only: Optional[bool] = None+ include_subdomains: Optional[bool] = Nonelimit: Optional[int] = Noneclass MapRequest(BaseModel):@@ -312,7 +311,7 @@ class MapRequest(BaseModel):class MapData(BaseModel):"""Map results data."""- links: List[str]+ links: List['SearchResult']class MapResponse(BaseResponse[MapData]):"""Response for map operations."""@@ -406,17 +405,20 @@ class SearchRequest(BaseModel):return normalized_sources-class SearchResult(BaseModel):- """A search result with basic information."""+class LinkResult(BaseModel):+ """A generic link result with optional metadata (used by search and map)."""url: strtitle: Optional[str] = Nonedescription: Optional[str] = None++# Backward-compatible alias for existing tests/usages+SearchResult = LinkResultclass SearchData(BaseModel):"""Search results grouped by source type."""- web: Optional[List[Union[SearchResult, SearchDocument]]] = None- news: Optional[List[Union[SearchResult, SearchDocument]]] = None- images: Optional[List[Union[SearchResult, SearchDocument]]] = None+ web: Optional[List[Union[LinkResult, SearchDocument]]] = None+ news: Optional[List[Union[LinkResult, SearchDocument]]] = None+ images: Optional[List[Union[LinkResult, SearchDocument]]] = Noneclass SearchResponse(BaseResponse[SearchData]):"""Response from search operation."""@@ -442,19 +444,19 @@ class JobStatus(BaseModel):status: Literal["pending", "scraping", "completed", "failed"]current: Optional[int] = Nonetotal: Optional[int] = None- created_at: Optional[datetime] = Field(None, alias="createdAt")- completed_at: Optional[datetime] = Field(None, alias="completedAt")- expires_at: Optional[datetime] = Field(None, alias="expiresAt")+ created_at: Optional[datetime] = None+ completed_at: Optional[datetime] = None+ expires_at: Optional[datetime] = Noneclass CrawlErrorsResponse(BaseModel):"""Response from crawl error monitoring."""- errors: List[Dict[str, str]] = Field(description="List of errors with fields: id, timestamp, url, error")- robots_blocked: List[str] = Field(alias="robotsBlocked", description="List of URLs blocked by robots.txt")+ errors: List[Dict[str, str]]+ robots_blocked: List[str]class ActiveCrawl(BaseModel):"""Information about an active crawl job."""id: str- team_id: str = Field(alias="teamId")+ team_id: strurl: stroptions: Optional[Dict[str, Any]] = Nonediff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.pyindex 39bb6acc7..655048104 100644--- a/apps/python-sdk/firecrawl/v2/utils/validation.py+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py@@ -2,8 +2,8 @@Shared validation functions for Firecrawl v2 API."""-from typing import Optional, Dict, Any-from ..types import ScrapeOptions+from typing import Optional, Dict, Any, List+from ..types import ScrapeOptions, ScrapeFormatsdef _convert_format_string(format_str: str) -> str:@@ -144,34 +144,112 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[stif value is not None:if key == "formats":# Handle formats conversion- converted_formats = []- for fmt in value:- if isinstance(fmt, str):- # Handle format strings- if fmt == "json":- raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")- converted_formats.append(_convert_format_string(fmt))- elif isinstance(fmt, dict):- # Handle format objects- if fmt.get('type') == 'json':- validated_json = _validate_json_format(fmt)- converted_formats.append(validated_json)+ converted_formats: List[Any] = []++ # Prefer using original object to detect ScrapeFormats vs list+ original_formats = getattr(options, 'formats', None)++ if isinstance(original_formats, ScrapeFormats):+ # Include explicit list first+ if original_formats.formats:+ for fmt in original_formats.formats:+ if isinstance(fmt, str):+ if fmt == "json":+ raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")+ converted_formats.append(_convert_format_string(fmt))+ elif isinstance(fmt, dict):+ fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None+ if fmt_type == 'json':+ validated_json = _validate_json_format({**fmt, 'type': 'json'})+ converted_formats.append(validated_json)+ elif fmt_type == 'screenshot':+ # Normalize screenshot options+ normalized = {**fmt, 'type': 'screenshot'}+ if 'full_page' in normalized:+ normalized['fullPage'] = normalized.pop('full_page')+ # Normalize viewport if it's a model instance+ vp = normalized.get('viewport')+ if hasattr(vp, 'model_dump'):+ normalized['viewport'] = vp.model_dump(exclude_none=True)+ converted_formats.append(normalized)+ else:+ if 'type' in fmt:+ fmt['type'] = fmt_type or fmt['type']+ converted_formats.append(fmt)+ elif hasattr(fmt, 'type'):+ if fmt.type == 'json':+ converted_formats.append(fmt.model_dump())+ else:+ converted_formats.append(_convert_format_string(fmt.type))+ else:+ converted_formats.append(fmt)++ # Add booleans from ScrapeFormats+ if original_formats.markdown:+ converted_formats.append("markdown")+ if original_formats.html:+ converted_formats.append("html")+ if original_formats.raw_html:+ converted_formats.append("rawHtml")+ if original_formats.links:+ converted_formats.append("links")+ if original_formats.screenshot:+ converted_formats.append("screenshot")+ if original_formats.change_tracking:+ converted_formats.append("changeTracking")+ # Note: We intentionally do not auto-include 'json' when boolean is set,+ # because JSON requires an object with schema/prompt. The caller must+ # supply the full json format object explicitly.+ elif isinstance(original_formats, list):+ for fmt in original_formats:+ if isinstance(fmt, str):+ if fmt == "json":+ raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")+ converted_formats.append(_convert_format_string(fmt))+ elif isinstance(fmt, dict):+ fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None+ if fmt_type == 'json':+ validated_json = _validate_json_format({**fmt, 'type': 'json'})+ converted_formats.append(validated_json)+ elif fmt_type == 'screenshot':+ normalized = {**fmt, 'type': 'screenshot'}+ if 'full_page' in normalized:+ normalized['fullPage'] = normalized.pop('full_page')+ vp = normalized.get('viewport')+ if hasattr(vp, 'model_dump'):+ normalized['viewport'] = vp.model_dump(exclude_none=True)+ converted_formats.append(normalized)+ else:+ if 'type' in fmt:+ fmt['type'] = fmt_type or fmt['type']+ converted_formats.append(fmt)+ elif hasattr(fmt, 'type'):+ if fmt.type == 'json':+ converted_formats.append(fmt.model_dump())+ elif fmt.type == 'screenshot':+ normalized = {'type': 'screenshot'}+ if getattr(fmt, 'full_page', None) is not None:+ normalized['fullPage'] = fmt.full_page+ if getattr(fmt, 'quality', None) is not None:+ normalized['quality'] = fmt.quality+ vp = getattr(fmt, 'viewport', None)+ if vp is not None:+ normalized['viewport'] = vp.model_dump(exclude_none=True) if hasattr(vp, 'model_dump') else vp+ converted_formats.append(normalized)+ else:+ converted_formats.append(_convert_format_string(fmt.type))else:- # Convert other format objects- if 'type' in fmt:- fmt['type'] = _convert_format_string(fmt['type'])converted_formats.append(fmt)- elif hasattr(fmt, 'type'):- # Handle Format objects- if fmt.type == 'json':- # For json format, we need the full object- converted_formats.append(fmt.model_dump())- else:- # For other formats, just convert the type- converted_formats.append(_convert_format_string(fmt.type))- else:- converted_formats.append(fmt)- scrape_data["formats"] = converted_formats+ else:+ # Fallback: try to iterate over value if it's a list-like+ try:+ for fmt in value:+ converted_formats.append(fmt)+ except TypeError:+ pass++ if converted_formats:+ scrape_data["formats"] = converted_formatselif key == "actions":# Handle actions conversionconverted_actions = []