Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(python-sdk): removed client duplication, bunch of type fixing, added map method + e2e/unit tests

rafaelmmillerdd6b46d373e8
diff --git a/apps/api/requests/v2/map.requests.http b/apps/api/requests/v2/map.requests.http
index bbd7820e6..eb96552a5 100644
--- a/apps/api/requests/v2/map.requests.http
+++ b/apps/api/requests/v2/map.requests.http
@@ -10,7 +10,7 @@ Authorization: Bearer {{$dotenv TEST_API_KEY}}
content-type: application/json
{
- "url": "https://firecrawl.dev",
+ "url": "https://firecrawl.dev"
}
# {
diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.py
index 848a55ef5..b9a53c537 100644
--- a/apps/python-sdk/example.py
+++ b/apps/python-sdk/example.py
@@ -81,5 +81,9 @@ def main():
print(search_response)
+ # map example
+ map_response = firecrawl.map("https://firecrawl.dev")
+ print(map_response)
+
if __name__ == "__main__":
main()
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/TODO.md b/apps/python-sdk/firecrawl/TODO.md
new file mode 100644
index 000000000..743e15e75
--- /dev/null
+++ b/apps/python-sdk/firecrawl/TODO.md
@@ -0,0 +1,89 @@
+- [ ] improve error handling for 500s
+
+
+============================================================================ FAILURES ============================================================================
+_____________________________________________________ TestCrawlE2E.test_get_active_crawls_with_running_crawl _____________________________________________________
+
+self = <test_crawl.TestCrawlE2E object at 0x1039a5810>
+
+ def test_get_active_crawls_with_running_crawl(self):
+ """Test getting active crawls when there's a running crawl."""
+ # Start a crawl
+ start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=5)
+ assert start_job.id is not None
+
+ # Get active crawls
+> active_crawls_response = self.client.active_crawls()
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+firecrawl/__tests__/e2e/v2/test_crawl.py:149:
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+firecrawl/v2/client.py:386: in active_crawls
+ return self.get_active_crawls()
+ ^^^^^^^^^^^^^^^^^^^^^^^^
+firecrawl/v2/client.py:377: in get_active_crawls
+ return crawl_module.get_active_crawls(self.http_client)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+firecrawl/v2/methods/crawl.py:446: in get_active_crawls
+ handle_response_error(response, "get active crawls")
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+
+response = <Response [500]>, action = 'get active crawls'
+
+ def handle_response_error(response: requests.Response, action: str) -> None:
+ """
+ Handle API response errors and raise appropriate exceptions.
+
+ Args:
+ response: The HTTP response object
+ action: Description of the action being performed
+
+ Raises:
+ FirecrawlError: Appropriate error based on status code
+ """
+ try:
+ response_json = response.json()
+ error_message = response_json.get('error', 'No error message provided.')
+ error_details = response_json.get('details', 'No additional error details provided.')
+ except:
+ # If we can't parse JSON, provide a helpful error message
+ try:
+ response_text = response.text[:500] # Limit to first 500 chars
+ if response_text.strip():
+ error_message = f"Server returned non-JSON response: {response_text}"
+ error_details = f"Full response status: {response.status_code}"
+ else:
+ error_message = f"Server returned empty response with status {response.status_code}"
+ error_details = "No additional details available"
+ except:
+ error_message = f"Server returned unreadable response with status {response.status_code}"
+ error_details = "No additional details available"
+
+ # Create appropriate error message
+ if response.status_code == 400:
+ message = f"Bad Request: Failed to {action}. {error_message} - {error_details}"
+ raise BadRequestError(message, response.status_code, response)
+ elif response.status_code == 401:
+ message = f"Unauthorized: Failed to {action}. {error_message} - {error_details}"
+ raise UnauthorizedError(message, response.status_code, response)
+ elif response.status_code == 402:
+ message = f"Payment Required: Failed to {action}. {error_message} - {error_details}"
+ raise PaymentRequiredError(message, response.status_code, response)
+ elif response.status_code == 403:
+ message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
+ raise WebsiteNotSupportedError(message, response.status_code, response)
+ elif response.status_code == 408:
+ message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
+ raise RequestTimeoutError(message, response.status_code, response)
+ elif response.status_code == 429:
+ message = f"Rate Limit Exceeded: Failed to {action}. {error_message} - {error_details}"
+ raise RateLimitError(message, response.status_code, response)
+ elif response.status_code == 500:
+ message = f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"
+> raise InternalServerError(message, response.status_code, response)
+E firecrawl.v2.utils.error_handler.InternalServerError: Internal Server Error: Failed to get active crawls. An unexpected error occurred. Please contact [email protected] for help. Your exception ID is c775af539e3a44f286664e55c488638e - No additional error details provided.
+
+firecrawl/v2/utils/error_handler.py:104: InternalServerError
+==================================================================== short test summary info =====================================================================
+FAILED firecrawl/__tests__/e2e/v2/test_crawl.py::TestCrawlE2E::test_get_active_crawls_with_running_crawl - firecrawl.v2.utils.error_handler.InternalServerError: Internal Server Error: Failed to get active crawls. An unexpected error occurred. Please contact help@f...
+======================================================================= 1 failed in 1.17s ========================================================================
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py
index 3a70e5e55..8f6d41c11 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py
@@ -194,7 +194,7 @@ class TestCrawlE2E:
def test_crawl_with_scrape_options(self):
"""Test crawl with scrape options."""
scrape_opts = ScrapeOptions(
- formats=["markdown"],
+ formats=["markdown", "links"],
only_main_content=False,
mobile=True,
)
@@ -207,6 +207,25 @@ class TestCrawlE2E:
assert crawl_job.id is not None
+ def test_crawl_with_json_format_object(self):
+ """Crawl with scrape_options including a JSON format object (prompt + schema)."""
+ json_schema = {
+ "type": "object",
+ "properties": {
+ "title": {"type": "string"}
+ },
+ "required": ["title"],
+ }
+ scrape_opts = ScrapeOptions(
+ formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]
+ )
+ crawl_job = self.client.start_crawl(
+ "https://docs.firecrawl.dev",
+ limit=2,
+ scrape_options=scrape_opts
+ )
+ assert crawl_job.id is not None
+
def test_crawl_all_parameters(self):
"""Test crawl with all possible parameters."""
scrape_opts = ScrapeOptions(
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.py
new file mode 100644
index 000000000..02f6f70bc
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.py
@@ -0,0 +1,68 @@
+import os
+from dotenv import load_dotenv
+from firecrawl import Firecrawl
+
+
+load_dotenv()
+
+if not os.getenv("API_KEY"):
+ raise ValueError("API_KEY is not set")
+
+if not os.getenv("API_URL"):
+ raise ValueError("API_URL is not set")
+
+
+import pytest
+
+
+class TestMapE2E:
+ """End-to-end tests for map functionality (v2)."""
+
+ def setup_method(self):
+ self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+
+ def test_map_minimal_request(self):
+ resp = self.client.map("https://docs.firecrawl.dev")
+
+ assert hasattr(resp, "success") and resp.success is True
+ assert hasattr(resp, "data") and resp.data is not None
+ assert hasattr(resp.data, "links")
+ assert isinstance(resp.data.links, list)
+
+ # Basic sanity checks on at least one link
+ if len(resp.data.links) > 0:
+ first = resp.data.links[0]
+ assert hasattr(first, "url")
+ assert isinstance(first.url, str) and first.url.startswith("http")
+
+ @pytest.mark.parametrize(
+ "sitemap_only,ignore_sitemap",
+ [
+ (True, None), # sitemap: only
+ (None, True), # sitemap: skip
+ (None, None), # sitemap: include (default)
+ ],
+ )
+ def test_map_with_options(self, sitemap_only, ignore_sitemap):
+ kwargs = {
+ "search": "docs",
+ "include_subdomains": True,
+ "limit": 10,
+ }
+ if sitemap_only is not None:
+ kwargs["sitemap_only"] = sitemap_only
+ if ignore_sitemap is not None:
+ kwargs["ignore_sitemap"] = ignore_sitemap
+
+ resp = self.client.map("https://docs.firecrawl.dev", **kwargs)
+
+ assert hasattr(resp, "success") and resp.success is True
+ assert hasattr(resp, "data") and resp.data is not None
+ assert isinstance(resp.data.links, list)
+
+ # Limit should be respected (server-side)
+ assert len(resp.data.links) <= 10
+
+ for link in resp.data.links:
+ assert hasattr(link, "url")
+ assert isinstance(link.url, str) and link.url.startswith("http")
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py
index 8c9ca7ebc..7acaa9d75 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_scrape.py
@@ -2,6 +2,8 @@ import os
import pytest
from dotenv import load_dotenv
from firecrawl import Firecrawl
+import json as _json
+import pytest
from firecrawl.v2.types import Viewport, ScreenshotAction, Document
load_dotenv()
@@ -56,6 +58,55 @@ class TestScrapeE2E:
)
self._assert_valid_document(doc)
+ @pytest.mark.parametrize("fmt,expect_field", [
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("raw_html", "raw_html"),
+ ("links", "links"),
+ ("screenshot", "screenshot"),
+ ])
+ def test_scrape_basic_formats(self, fmt, expect_field):
+ """Verify basic formats request succeeds and expected fields are present when applicable."""
+ doc = self.client.scrape(
+ "https://docs.firecrawl.dev",
+ formats=[fmt],
+ )
+ # For formats that are not content (links/screenshot/json), skip main-content assertion
+ if expect_field not in {"links", "screenshot"}:
+ self._assert_valid_document(doc)
+ if expect_field == "markdown":
+ assert doc.markdown is not None
+ elif expect_field == "html":
+ assert doc.html is not None
+ elif expect_field == "raw_html":
+ assert doc.raw_html is not None
+ elif expect_field == "screenshot":
+ assert doc.screenshot is not None
+ elif expect_field == "links":
+ assert isinstance(doc.links, list)
+ assert len(doc.links) > 0
+
+ def test_scrape_with_json_format_object(self):
+ """Scrape with JSON format object (requires prompt and schema)."""
+ json_schema = {
+ "type": "object",
+ "properties": {
+ "title": {"type": "string"}
+ },
+ "required": ["title"],
+ }
+ doc = self.client.scrape(
+ "https://docs.firecrawl.dev",
+ formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}],
+ only_main_content=True,
+ )
+ # JSON format may not include main content fields; ensure request succeeded
+ assert isinstance(doc, Document)
+ # If backend returns extracted json content, it should be present under `json`
+ # (Do not fail if backend omits it; existence depends on implementation)
+ # if hasattr(doc, 'json'):
+ # assert doc.json is not None
+
def test_scrape_invalid_url(self):
"""Scrape should fail with empty or invalid URLs."""
with pytest.raises(ValueError, match="URL cannot be empty"):
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
index 7dbbc54f4..deb309c1b 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
@@ -117,7 +117,15 @@ def test_search_all_parameters():
ignore_invalid_urls=True,
timeout=60000,
scrape_options=ScrapeOptions(
- formats=["markdown", "html"],
+ formats=[
+ "markdown",
+ "html",
+ {
+ "type": "json",
+ "prompt": "Extract the title and description from the page",
+ "schema": schema
+ }
+ ],
headers={"User-Agent": "Firecrawl-Test/1.0"},
include_tags=["h1", "h2", "p"],
exclude_tags=["nav", "footer"],
@@ -208,4 +216,23 @@ def test_search_formats_flexibility():
assert isinstance(results1, SearchData)
assert isinstance(results2, SearchData)
assert results1.web is not None
- assert results2.web is not None
\ No newline at end of file
+ assert results2.web is not None
+
+def test_search_with_json_format_object():
+ """Search with scrape_options including a JSON format object (prompt + schema)."""
+ json_schema = {
+ "type": "object",
+ "properties": {
+ "title": {"type": "string"}
+ },
+ "required": ["title"],
+ }
+ results = firecrawl.search(
+ query="site:docs.firecrawl.dev",
+ limit=1,
+ scrape_options=ScrapeOptions(
+ formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]
+ ),
+ )
+ assert isinstance(results, SearchData)
+ assert results.web is not None and len(results.web) >= 0
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/map/test_map_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/map/test_map_request_preparation.py
new file mode 100644
index 000000000..d4f8a6a4e
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/map/test_map_request_preparation.py
@@ -0,0 +1,51 @@
+import pytest
+from firecrawl.v2.types import MapOptions
+from firecrawl.v2.methods.map import _prepare_map_request
+
+
+class TestMapRequestPreparation:
+ """Unit tests for map request preparation."""
+
+ def test_basic_request_preparation(self):
+ data = _prepare_map_request("https://example.com")
+ assert data["url"] == "https://example.com"
+ # Default sitemap handling should be "include" when no flags provided
+ assert "sitemap" not in data # we only send when options provided
+
+ def test_sitemap_transformations(self):
+ # sitemap_only -> sitemap: "only"
+ opts = MapOptions(sitemap_only=True)
+ data = _prepare_map_request("https://example.com", opts)
+ assert data["sitemap"] == "only"
+
+ # ignore_sitemap -> sitemap: "skip"
+ opts = MapOptions(ignore_sitemap=True)
+ data = _prepare_map_request("https://example.com", opts)
+ assert data["sitemap"] == "skip"
+
+ # default when options present but neither flag set -> include
+ opts = MapOptions(search="docs")
+ data = _prepare_map_request("https://example.com", opts)
+ assert data["sitemap"] == "include"
+
+ def test_field_conversions(self):
+ opts = MapOptions(
+ search="docs",
+ include_subdomains=True,
+ limit=25,
+ sitemap_only=True,
+ )
+ data = _prepare_map_request("https://example.com", opts)
+
+ assert data["url"] == "https://example.com"
+ assert data["search"] == "docs"
+ assert data["includeSubdomains"] is True
+ assert data["limit"] == 25
+ assert data["sitemap"] == "only"
+
+ def test_invalid_url(self):
+ with pytest.raises(ValueError):
+ _prepare_map_request("")
+ with pytest.raises(ValueError):
+ _prepare_map_request(" ")
+
diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.py
index b812481f7..31d04da3d 100644
--- a/apps/python-sdk/firecrawl/client.py
+++ b/apps/python-sdk/firecrawl/client.py
@@ -62,7 +62,7 @@ class V2Proxy:
self.start_crawl = client_instance.start_crawl
self.crawl_params_preview = client_instance.crawl_params_preview
# self.batch_scrape = client_instance.batch_scrape
- # self.map = client_instance.map
+ self.map = client_instance.map
def __getattr__(self, name):
"""Forward attribute access to the underlying client."""
@@ -141,7 +141,7 @@ class Firecrawl:
self.active_crawls = self._v2_client.active_crawls
# self.batch_scrape = self._v2_client.batch_scrape
- # self.map = self._v2_client.map
+ self.map = self._v2_client.map
self.search = self._v2_client.search
class AsyncFirecrawl:
diff --git a/apps/python-sdk/firecrawl/v2/__init__.py b/apps/python-sdk/firecrawl/v2/__init__.py
index bc45080fd..0904fc44b 100644
--- a/apps/python-sdk/firecrawl/v2/__init__.py
+++ b/apps/python-sdk/firecrawl/v2/__init__.py
@@ -1,261 +1,3 @@
-from typing import Optional, List, Union, Dict, Any
-from ..types import (
- SearchData, Document, ScrapeOptions, SearchRequest,
- SourceOption, FormatOption, CrawlRequest, CrawlJob, CrawlResponse,
- CrawlParamsRequest, CrawlParamsData, CrawlErrorsResponse, ActiveCrawlsResponse,
- WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction, Location
-)
-from .methods.search import search as search_method
-from .methods.crawl import (
- crawl as crawl_method,
- start_crawl as start_crawl_method,
- cancel_crawl as cancel_crawl_method,
- get_crawl_status as get_crawl_status_method,
- crawl_params_preview as crawl_params_preview_method,
- get_crawl_errors as get_crawl_errors_method,
- get_active_crawls as get_active_crawls_method
-)
-from .methods.scrape import scrape as scrape_method
-from .utils.http_client import HttpClient
+from .client import FirecrawlClient
-class FirecrawlClient:
- """
- Firecrawl v2 API client.
- """
-
- def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):
- self.api_key = api_key
- self.api_url = api_url
- self._client = HttpClient(api_key=self.api_key, api_url=self.api_url)
-
- def scrape(
- self,
- url: str,
- formats: Optional[List[FormatOption]] = None,
- headers: Optional[Dict[str, str]] = None,
- include_tags: Optional[List[str]] = None,
- exclude_tags: Optional[List[str]] = None,
- only_main_content: Optional[bool] = None,
- timeout: Optional[int] = None,
- wait_for: Optional[int] = None,
- mobile: Optional[bool] = None,
- parsers: Optional[List[str]] = None,
- actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]] = None,
- location: Optional[Location] = None,
- skip_tls_verification: Optional[bool] = None,
- remove_base64_images: Optional[bool] = None,
- fast_mode: Optional[bool] = None,
- use_mock: Optional[str] = None,
- block_ads: Optional[bool] = None,
- proxy: Optional[str] = None,
- max_age: Optional[int] = None,
- store_in_cache: Optional[bool] = None,
- ) -> Document:
- """Scrape a single URL and return the document."""
- options = ScrapeOptions(
- formats=formats,
- headers=headers,
- include_tags=include_tags,
- exclude_tags=exclude_tags,
- only_main_content=only_main_content,
- timeout=timeout,
- wait_for=wait_for,
- mobile=mobile,
- parsers=parsers,
- actions=actions,
- location=location,
- skip_tls_verification=skip_tls_verification,
- remove_base64_images=remove_base64_images,
- fast_mode=fast_mode,
- use_mock=use_mock,
- block_ads=block_ads,
- proxy=proxy,
- max_age=max_age,
- store_in_cache=store_in_cache
- )
- return scrape_method(self._client, url, options)
-
- # batch-scrape
- def batch_scrape(
- self
- ):
- pass
-
- # start-batch-scrape
- def start_batch_scrape(
- self
- ):
- pass
-
- # get-batch-scrape-status
- def get_batch_scrape_status(
- self
- ):
- pass
-
- # cancel-batch-scrape
- def cancel_batch_scrape(
- self
- ):
- pass
- # get-batch-scrape-errors
- def get_batch_scrape_errors(
- self
- ):
- pass
-
- def crawl(
- self,
- url: str,
- prompt: Optional[str] = None,
- include_paths: Optional[List[str]] = None,
- exclude_paths: Optional[List[str]] = None,
- max_discovery_depth: Optional[int] = None,
- ignore_sitemap: bool = False,
- ignore_query_parameters: bool = False,
- limit: Optional[int] = None,
- crawl_entire_domain: bool = False,
- allow_external_links: bool = False,
- allow_subdomains: bool = False,
- delay: Optional[int] = None,
- max_concurrency: Optional[int] = None,
- webhook: Optional[Dict[str, Any]] = None,
- scrape_options: Optional[ScrapeOptions] = None,
- zero_data_retention: bool = False,
- poll_interval: int = 2,
- timeout: Optional[int] = None
- ) -> CrawlJob:
- """Start a crawl job and wait for it to complete."""
- request = CrawlRequest(
- url=url,
- prompt=prompt,
- include_paths=include_paths,
- exclude_paths=exclude_paths,
- max_discovery_depth=max_discovery_depth,
- ignore_sitemap=ignore_sitemap,
- ignore_query_parameters=ignore_query_parameters,
- limit=limit,
- crawl_entire_domain=crawl_entire_domain,
- allow_external_links=allow_external_links,
- allow_subdomains=allow_subdomains,
- delay=delay,
- max_concurrency=max_concurrency,
- webhook=webhook,
- scrape_options=scrape_options,
- zero_data_retention=zero_data_retention
- )
- return crawl_method(self._client, request, poll_interval, timeout)
-
- def start_crawl(
- self,
- url: str,
- prompt: Optional[str] = None,
- include_paths: Optional[List[str]] = None,
- exclude_paths: Optional[List[str]] = None,
- max_discovery_depth: Optional[int] = None,
- ignore_sitemap: bool = False,
- ignore_query_parameters: bool = False,
- limit: Optional[int] = None,
- crawl_entire_domain: bool = False,
- allow_external_links: bool = False,
- allow_subdomains: bool = False,
- delay: Optional[int] = None,
- max_concurrency: Optional[int] = None,
- webhook: Optional[Dict[str, Any]] = None,
- scrape_options: Optional[ScrapeOptions] = None,
- zero_data_retention: bool = False
- ) -> CrawlResponse:
- """Start a crawl job and return immediately."""
- request = CrawlRequest(
- url=url,
- prompt=prompt,
- include_paths=include_paths,
- exclude_paths=exclude_paths,
- max_discovery_depth=max_discovery_depth,
- ignore_sitemap=ignore_sitemap,
- ignore_query_parameters=ignore_query_parameters,
- limit=limit,
- crawl_entire_domain=crawl_entire_domain,
- allow_external_links=allow_external_links,
- allow_subdomains=allow_subdomains,
- delay=delay,
- max_concurrency=max_concurrency,
- webhook=webhook,
- scrape_options=scrape_options,
- zero_data_retention=zero_data_retention
- )
- return start_crawl_method(self._client, request)
-
- def get_crawl_status(
- self,
- job_id: str
- ) -> CrawlJob:
- """Get the status of a crawl job."""
- return get_crawl_status_method(self._client, job_id)
-
- def cancel_crawl(
- self,
- job_id: str
- ) -> bool:
- """Cancel a running crawl job."""
- return cancel_crawl_method(self._client, job_id)
-
- def crawl_params_preview(
- self,
- url: str,
- prompt: str
- ) -> CrawlParamsData:
- """Get crawl parameters from LLM based on URL and prompt."""
- request = CrawlParamsRequest(url=url, prompt=prompt)
- return crawl_params_preview_method(self._client, request)
-
- def active_crawls(
- self
- ) -> ActiveCrawlsResponse:
- """Get a list of active crawl jobs."""
- return get_active_crawls_method(self._client)
-
- def get_crawl_errors(
- self,
- crawl_id: str
- ) -> CrawlErrorsResponse:
- """Get errors from a crawl job."""
- return get_crawl_errors_method(self._client, crawl_id)
-
- # map
- def map(
- self
- ):
- pass
-
- def search(
- self,
- query: str,
- sources: Optional[List[SourceOption]] = None,
- limit: Optional[int] = 5,
- tbs: Optional[str] = None,
- location: Optional[str] = None,
- ignore_invalid_urls: Optional[bool] = True,
- timeout: Optional[int] = 60000,
- scrape_options: Optional[ScrapeOptions] = None,
- ) -> SearchData:
- """Search for documents."""
- request = SearchRequest(
- query=query,
- sources=sources,
- limit=limit,
- tbs=tbs,
- location=location,
- ignore_invalid_urls=ignore_invalid_urls,
- timeout=timeout,
- scrape_options=scrape_options,
- )
- return search_method(self._client, request)
-
- # credit-usage
- def credit_usage(
- self
- ):
- pass
-
-__all__ = ['FirecrawlClient']
\ No newline at end of file
+__all__ = ["FirecrawlClient"]
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index c06007de4..af3f505b5 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -7,11 +7,33 @@ This module provides the main client class that orchestrates all v2 functionalit
import os
from typing import Optional, List, Dict, Any, Callable, Union
from .types import (
- ClientConfig, ScrapeOptions, CrawlOptions, MapOptions, ExtractOptions,
- ScrapeResponse, CrawlResponse, CrawlStatusResponse, BatchScrapeResponse,
- BatchScrapeStatusResponse, MapResponse, ExtractResponse, Document,
- SearchRequest, SearchResponse, CrawlRequest, WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse,
- FormatOption, WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction, Location,
+ ClientConfig,
+ ScrapeOptions,
+ Document,
+ SearchRequest,
+ SearchData,
+ SourceOption,
+ CrawlRequest,
+ CrawlResponse,
+ CrawlJob,
+ CrawlParamsRequest,
+ CrawlParamsData,
+ WebhookConfig,
+ CrawlErrorsResponse,
+ ActiveCrawlsResponse,
+ MapOptions,
+ MapResponse,
+ FormatOption,
+ WaitAction,
+ ScreenshotAction,
+ ClickAction,
+ WriteAction,
+ PressAction,
+ ScrollAction,
+ ScrapeAction,
+ ExecuteJavascriptAction,
+ PDFAction,
+ Location,
)
from .utils.http_client import HttpClient
from .utils.error_handler import FirecrawlError
@@ -19,6 +41,7 @@ from .methods import scrape as scrape_module
from .methods import crawl as crawl_module
from .methods import batch as batch_module
from .methods import search as search_module
+from .methods import map as map_module
class FirecrawlClient:
"""
@@ -143,12 +166,14 @@ class FirecrawlClient:
self,
query: str,
*,
+ sources: Optional[List[SourceOption]] = None,
limit: Optional[int] = None,
tbs: Optional[str] = None,
location: Optional[str] = None,
+ ignore_invalid_urls: Optional[bool] = None,
timeout: Optional[int] = None,
- page_options: Optional[ScrapeOptions] = None,
- ) -> SearchResponse:
+ scrape_options: Optional[ScrapeOptions] = None,
+ ) -> SearchData:
"""
Search for documents.
@@ -161,17 +186,20 @@ class FirecrawlClient:
page_options: Options for scraping individual pages
Returns:
- SearchResponse containing the search results
+ SearchData containing the search results
"""
- options = SearchRequest(
+ request = SearchRequest(
+ query=query,
+ sources=sources,
limit=limit,
tbs=tbs,
location=location,
+ ignore_invalid_urls=ignore_invalid_urls,
timeout=timeout,
- page_options=page_options
+ scrape_options=scrape_options,
)
-
- return search_module.search(self.http_client, query, options)
+
+ return search_module.search(self.http_client, request)
def crawl(
self,
@@ -194,7 +222,7 @@ class FirecrawlClient:
zero_data_retention: bool = False,
poll_interval: int = 2,
timeout: Optional[int] = None
- ) -> CrawlStatusResponse:
+ ) -> CrawlJob:
"""
Start a crawl job and wait for it to complete.
@@ -219,7 +247,7 @@ class FirecrawlClient:
timeout: Maximum seconds to wait (None for no timeout)
Returns:
- CrawlStatusResponse when job completes
+ CrawlJob when job completes
Raises:
ValueError: If request is invalid
@@ -321,7 +349,7 @@ class FirecrawlClient:
return crawl_module.start_crawl(self.http_client, request)
- def get_crawl_status(self, job_id: str) -> CrawlStatusResponse:
+ def get_crawl_status(self, job_id: str) -> CrawlJob:
"""
Get the status of a crawl job.
@@ -329,24 +357,24 @@ class FirecrawlClient:
job_id: ID of the crawl job
Returns:
- CrawlStatusResponse with current status and data
+ CrawlJob with current status and data
Raises:
Exception: If the status check fails
"""
return crawl_module.get_crawl_status(self.http_client, job_id)
- def check_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse:
+ def get_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse:
"""
- Get errors from a crawl job.
+ Retrieve error details and robots.txt blocks for a given crawl job.
Args:
crawl_id: The ID of the crawl job
-
+
Returns:
- CrawlErrorsResponse containing errors and robots blocked URLs
+ CrawlErrorsResponse containing per-URL errors and robots-blocked URLs
"""
- return crawl_module.check_crawl_errors(self.http_client, crawl_id)
+ return crawl_module.get_crawl_errors(self.http_client, crawl_id)
def get_active_crawls(self) -> ActiveCrawlsResponse:
"""
@@ -357,6 +385,38 @@ class FirecrawlClient:
"""
return crawl_module.get_active_crawls(self.http_client)
+ def active_crawls(self) -> ActiveCrawlsResponse:
+ """
+ List currently active crawl jobs for the authenticated team.
+
+ Returns:
+ ActiveCrawlsResponse containing the list of active crawl jobs
+ """
+ return self.get_active_crawls()
+
+ def map(
+ self,
+ url: str,
+ *,
+ search: Optional[str] = None,
+ include_subdomains: Optional[bool] = None,
+ limit: Optional[int] = None,
+ ignore_sitemap: Optional[bool] = None,
+ sitemap_only: Optional[bool] = None,
+ ) -> MapResponse:
+ """
+ Map a URL and return discovered links (with optional titles/descriptions).
+ """
+ options = MapOptions(
+ search=search,
+ include_subdomains=include_subdomains,
+ limit=limit,
+ ignore_sitemap=ignore_sitemap,
+ sitemap_only=sitemap_only,
+ ) if any(v is not None for v in [search, include_subdomains, limit, ignore_sitemap, sitemap_only]) else None
+
+ return map_module.map(self.http_client, url, options)
+
def cancel_crawl(self, crawl_id: str) -> bool:
"""
Cancel a crawl job.
@@ -368,4 +428,11 @@ class FirecrawlClient:
bool: True if the crawl was cancelled, False otherwise
"""
return crawl_module.cancel_crawl(self.http_client, crawl_id)
+
+ def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData:
+ """
+ Get crawl parameters from LLM based on URL and prompt.
+ """
+ request = CrawlParamsRequest(url=url, prompt=prompt)
+ return crawl_module.crawl_params_preview(self.http_client, request)
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.py
index 2a19eda02..b10ba770a 100644
--- a/apps/python-sdk/firecrawl/v2/methods/batch.py
+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py
@@ -4,11 +4,11 @@ Batch scraping functionality for Firecrawl v2 API.
import time
from typing import Optional, List, Callable
-from .types import (
+from ..types import (
BatchScrapeRequest, BatchScrapeResponse,
BatchScrapeJob, BatchScrapeData, ScrapeOptions, Document
)
-from .utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
+from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
def start_batch_scrape(
@@ -34,7 +34,7 @@ def start_batch_scrape(
request_data = prepare_batch_request(urls, options)
# Make the API request
- response = client.post("/v1/batch/scrape", request_data)
+ response = client.post("/v2/batch/scrape", request_data)
# Handle errors
if not response.ok:
@@ -77,7 +77,7 @@ def get_batch_scrape_status(
FirecrawlError: If the status check fails
"""
# Make the API request
- response = client.get(f"/v1/batch/scrape/{job_id}")
+ response = client.get(f"/v2/batch/scrape/{job_id}")
# Handle errors
if not response.ok:
@@ -132,7 +132,7 @@ def cancel_batch_scrape(
FirecrawlError: If the cancellation fails
"""
# Make the API request
- response = client.delete(f"/v1/batch/scrape/{job_id}")
+ response = client.delete(f"/v2/batch/scrape/{job_id}")
# Handle errors
if not response.ok:
@@ -235,7 +235,7 @@ def batch_scrape_and_wait(
batch_response = start_batch_scrape(client, urls, options)
if not batch_response.success or not batch_response.data:
- return BatchScrapeStatusResponse(
+ return BatchScrapeResponse(
success=False,
error=batch_response.error or "Failed to start batch scrape"
)
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
index 1894723f1..b6b2ee1ac 100644
--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -8,7 +8,7 @@ from ..types import (
CrawlRequest,
CrawlJob,
CrawlResponse, Document, CrawlParamsRequest, CrawlParamsResponse, CrawlParamsData,
- WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse
+ WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, ActiveCrawl
)
from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
@@ -416,13 +416,19 @@ def get_crawl_errors(http_client: HttpClient, crawl_id: str) -> CrawlErrorsRespo
Exception: If the request fails
"""
response = http_client.get(f"/v2/crawl/{crawl_id}/errors")
-
- if response.status_code != 200:
+
+ if not response.ok:
handle_response_error(response, "check crawl errors")
-
+
try:
- data = response.json()
- return CrawlErrorsResponse(**data)
+ body = response.json()
+ payload = body.get("data", body)
+ # Manual key normalization since we avoid Pydantic aliases
+ normalized = {
+ "errors": payload.get("errors", []),
+ "robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
+ }
+ return CrawlErrorsResponse(**normalized)
except Exception as e:
raise Exception(f"Failed to parse crawl errors response: {e}")
@@ -441,13 +447,23 @@ def get_active_crawls(client: HttpClient) -> ActiveCrawlsResponse:
Exception: If the request fails
"""
response = client.get("/v2/crawl/active")
-
+
if not response.ok:
handle_response_error(response, "get active crawls")
-
- response_data = response.json()
-
- if response_data.get("success"):
- return ActiveCrawlsResponse(**response_data.get("data", {}))
- else:
- raise Exception(response_data.get("error", "Unknown error occurred"))
+
+ body = response.json()
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+
+ data = body.get("data", {})
+ crawls_in = data.get("crawls", [])
+ normalized_crawls = []
+ for c in crawls_in:
+ if isinstance(c, dict):
+ normalized_crawls.append({
+ "id": c.get("id"),
+ "team_id": c.get("teamId", c.get("team_id")),
+ "url": c.get("url"),
+ "options": c.get("options"),
+ })
+ return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized_crawls])
diff --git a/apps/python-sdk/firecrawl/v2/methods/map.py b/apps/python-sdk/firecrawl/v2/methods/map.py
new file mode 100644
index 000000000..7daae4457
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/methods/map.py
@@ -0,0 +1,69 @@
+"""
+Mapping functionality for Firecrawl v2 API.
+"""
+
+from typing import Optional, Dict, Any
+from ..types import MapOptions, MapResponse, MapData, LinkResult
+from ..utils import HttpClient, handle_response_error
+
+
+def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict[str, Any]:
+ if not url or not url.strip():
+ raise ValueError("URL cannot be empty")
+
+ payload: Dict[str, Any] = {"url": url.strip()}
+
+ if options is not None:
+ # Transform ignoreSitemap and sitemapOnly to the new unified sitemap parameter
+ # sitemap: "only" | "include" | "skip" (default include)
+ sitemap: str = "include"
+ if options.sitemap_only:
+ sitemap = "only"
+ elif options.ignore_sitemap:
+ sitemap = "skip"
+
+ data: Dict[str, Any] = {}
+ data["sitemap"] = sitemap
+
+ if options.search is not None:
+ data["search"] = options.search
+ if options.include_subdomains is not None:
+ data["includeSubdomains"] = options.include_subdomains
+ if options.limit is not None:
+ data["limit"] = options.limit
+ payload.update(data)
+
+ return payload
+
+
+def map(client: HttpClient, url: str, options: Optional[MapOptions] = None) -> MapResponse:
+ """
+ Map a URL and return a list of links with optional titles/descriptions.
+ """
+ request_data = _prepare_map_request(url, options)
+
+ response = client.post("/v2/map", request_data)
+
+ if not response.ok:
+ handle_response_error(response, "map")
+
+ body = response.json()
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+
+ data = body.get("data", {})
+ result_links: list[LinkResult] = []
+ for item in data.get("links", []):
+ if isinstance(item, dict):
+ result_links.append(
+ LinkResult(
+ url=item.get("url", ""),
+ title=item.get("title"),
+ description=item.get("description"),
+ )
+ )
+ elif isinstance(item, str):
+ result_links.append(LinkResult(url=item))
+
+ return MapResponse(success=True, data=MapData(links=result_links))
+
diff --git a/apps/python-sdk/firecrawl/v2/methods/scrape.py b/apps/python-sdk/firecrawl/v2/methods/scrape.py
index 0eaf7e720..408d8d378 100644
--- a/apps/python-sdk/firecrawl/v2/methods/scrape.py
+++ b/apps/python-sdk/firecrawl/v2/methods/scrape.py
@@ -59,4 +59,10 @@ def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None
raise Exception(body.get("error", "Unknown error occurred"))
document_data = body.get("data", {})
- return Document(**document_data)
\ No newline at end of file
+ # Normalize keys for Document (no Pydantic aliases)
+ normalized = dict(document_data)
+ if 'rawHtml' in normalized and 'raw_html' not in normalized:
+ normalized['raw_html'] = normalized.pop('rawHtml')
+ if 'changeTracking' in normalized and 'change_tracking' not in normalized:
+ normalized['change_tracking'] = normalized.pop('changeTracking')
+ return Document(**normalized)
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.py
index 6ac1d76c8..1495b0f34 100644
--- a/apps/python-sdk/firecrawl/v2/methods/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/search.py
@@ -3,7 +3,7 @@ Search functionality for Firecrawl v2 API.
"""
from typing import Optional, Dict, Any, Union
-from ...types import SearchRequest, SearchData, SearchResult, Document
+from ..types import SearchRequest, SearchData, SearchResult, Document
from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
@@ -46,9 +46,19 @@ def search(
results = []
for doc_data in source_documents:
if isinstance(doc_data, dict):
- if any(key in doc_data for key in ['markdown', 'html', 'content', 'screenshot']):
- results.append(Document(**doc_data))
+ # If page scraping options were provided, API returns full Document objects
+ if request.scrape_options is not None and any(
+ key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'screenshot', 'changeTracking']
+ ):
+ # Normalize keys for Document (no Pydantic aliases)
+ normalized = dict(doc_data)
+ if 'rawHtml' in normalized and 'raw_html' not in normalized:
+ normalized['raw_html'] = normalized.pop('rawHtml')
+ if 'changeTracking' in normalized and 'change_tracking' not in normalized:
+ normalized['change_tracking'] = normalized.pop('changeTracking')
+ results.append(Document(**normalized))
else:
+ # Minimal search result shape
results.append(SearchResult(
url=doc_data.get('url', ''),
title=doc_data.get('title'),
@@ -56,8 +66,7 @@ def search(
))
elif isinstance(doc_data, str):
results.append(SearchResult(url=doc_data))
-
- # Set the appropriate field based on source type
+
if hasattr(search_data, source_type):
setattr(search_data, source_type, results)
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index dc9a15cb9..f5a265193 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -15,6 +15,7 @@ warnings.filterwarnings("ignore", message="Field name \"schema\" in \"Format\" s
warnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonFormat\" shadows an attribute in parent \"Format\"")
warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ChangeTrackingFormat\" shadows an attribute in parent \"Format\"")
warnings.filterwarnings("ignore", message="Field name \"json\" in \"ScrapeFormats\" shadows an attribute in parent \"BaseModel\"")
+warnings.filterwarnings("ignore", message="Field name \"json\" in \"Document\" shadows an attribute in parent \"BaseModel\"")
T = TypeVar('T')
@@ -34,25 +35,26 @@ class DocumentMetadata(BaseModel):
language: Optional[str] = None
keywords: Optional[Union[str, List[str]]] = None
robots: Optional[str] = None
- og_title: Optional[str] = Field(None, alias="ogTitle")
- og_description: Optional[str] = Field(None, alias="ogDescription")
- og_url: Optional[str] = Field(None, alias="ogUrl")
- og_image: Optional[str] = Field(None, alias="ogImage")
- source_url: Optional[str] = Field(None, alias="sourceURL")
- status_code: Optional[int] = Field(None, alias="statusCode")
+ og_title: Optional[str] = None
+ og_description: Optional[str] = None
+ og_url: Optional[str] = None
+ og_image: Optional[str] = None
+ source_url: Optional[str] = None
+ status_code: Optional[int] = None
error: Optional[str] = None
class Document(BaseModel):
"""A scraped document."""
markdown: Optional[str] = None
html: Optional[str] = None
- raw_html: Optional[str] = Field(None, alias="rawHtml")
+ raw_html: Optional[str] = None
+ json: Optional[Any] = None
metadata: Optional[DocumentMetadata] = None
links: Optional[List[str]] = None
screenshot: Optional[str] = None
actions: Optional[Dict[str, Any]] = None
warning: Optional[str] = None
- change_tracking: Optional[Dict[str, Any]] = Field(None, alias="changeTracking")
+ change_tracking: Optional[Dict[str, Any]] = None
# Webhook types
class WebhookConfig(BaseModel):
@@ -64,7 +66,7 @@ class WebhookConfig(BaseModel):
class WebhookData(BaseModel):
"""Data sent to webhooks."""
- job_id: str = Field(alias="jobId")
+ job_id: str
status: str
current: Optional[int] = None
total: Optional[int] = None
@@ -120,7 +122,7 @@ class ScrapeFormats(BaseModel):
formats: Optional[List[FormatOption]] = None
markdown: bool = True
html: bool = False
- raw_html: bool = Field(False, alias="rawHtml")
+ raw_html: bool = False
links: bool = False
screenshot: bool = False
change_tracking: bool = False
@@ -148,7 +150,7 @@ class ScrapeFormats(BaseModel):
class ScrapeOptions(BaseModel):
"""Options for scraping operations."""
- formats: Optional[List[FormatOption]] = None
+ formats: Optional[Union['ScrapeFormats', List[FormatOption]]] = None
headers: Optional[Dict[str, str]] = None
include_tags: Optional[List[str]] = None
exclude_tags: Optional[List[str]] = None
@@ -174,15 +176,10 @@ class ScrapeOptions(BaseModel):
"""Validate and normalize formats input."""
if v is None:
return v
-
- # If it's already a ScrapeFormats object, return as is
if isinstance(v, ScrapeFormats):
return v
-
- # If it's a list, keep it as a list (don't convert to ScrapeFormats)
if isinstance(v, list):
return v
-
raise ValueError(f"Invalid formats type: {type(v)}. Expected ScrapeFormats or List[FormatOption]")
class ScrapeRequest(BaseModel):
@@ -236,6 +233,7 @@ class CrawlJob(BaseModel):
class SearchDocument(Document):
"""A document from a search operation with URL and description."""
url: str
+ title: Optional[str] = None
description: Optional[str] = None
class MapDocument(Document):
@@ -283,8 +281,8 @@ class BatchScrapeJob(BaseModel):
status: Literal["scraping", "completed", "failed"]
current: Optional[int] = None
total: Optional[int] = None
- created_at: Optional[datetime] = Field(None, alias="createdAt")
- completed_at: Optional[datetime] = Field(None, alias="completedAt")
+ created_at: Optional[datetime] = None
+ completed_at: Optional[datetime] = None
class BatchScrapeData(BaseModel):
"""Batch scrape results data."""
@@ -301,8 +299,9 @@ class BatchScrapeResponse(BaseResponse[BatchScrapeData]):
class MapOptions(BaseModel):
"""Options for mapping operations."""
search: Optional[str] = None
- ignore_sitemap: bool = Field(False, alias="ignoreSitemap")
- include_subdomains: bool = Field(False, alias="includeSubdomains")
+ ignore_sitemap: Optional[bool] = None
+ sitemap_only: Optional[bool] = None
+ include_subdomains: Optional[bool] = None
limit: Optional[int] = None
class MapRequest(BaseModel):
@@ -312,7 +311,7 @@ class MapRequest(BaseModel):
class MapData(BaseModel):
"""Map results data."""
- links: List[str]
+ links: List['SearchResult']
class MapResponse(BaseResponse[MapData]):
"""Response for map operations."""
@@ -406,17 +405,20 @@ class SearchRequest(BaseModel):
return normalized_sources
-class SearchResult(BaseModel):
- """A search result with basic information."""
+class LinkResult(BaseModel):
+ """A generic link result with optional metadata (used by search and map)."""
url: str
title: Optional[str] = None
description: Optional[str] = None
+
+# Backward-compatible alias for existing tests/usages
+SearchResult = LinkResult
class SearchData(BaseModel):
"""Search results grouped by source type."""
- web: Optional[List[Union[SearchResult, SearchDocument]]] = None
- news: Optional[List[Union[SearchResult, SearchDocument]]] = None
- images: Optional[List[Union[SearchResult, SearchDocument]]] = None
+ web: Optional[List[Union[LinkResult, SearchDocument]]] = None
+ news: Optional[List[Union[LinkResult, SearchDocument]]] = None
+ images: Optional[List[Union[LinkResult, SearchDocument]]] = None
class SearchResponse(BaseResponse[SearchData]):
"""Response from search operation."""
@@ -442,19 +444,19 @@ class JobStatus(BaseModel):
status: Literal["pending", "scraping", "completed", "failed"]
current: Optional[int] = None
total: Optional[int] = None
- created_at: Optional[datetime] = Field(None, alias="createdAt")
- completed_at: Optional[datetime] = Field(None, alias="completedAt")
- expires_at: Optional[datetime] = Field(None, alias="expiresAt")
+ created_at: Optional[datetime] = None
+ completed_at: Optional[datetime] = None
+ expires_at: Optional[datetime] = None
class CrawlErrorsResponse(BaseModel):
"""Response from crawl error monitoring."""
- errors: List[Dict[str, str]] = Field(description="List of errors with fields: id, timestamp, url, error")
- robots_blocked: List[str] = Field(alias="robotsBlocked", description="List of URLs blocked by robots.txt")
+ errors: List[Dict[str, str]]
+ robots_blocked: List[str]
class ActiveCrawl(BaseModel):
"""Information about an active crawl job."""
id: str
- team_id: str = Field(alias="teamId")
+ team_id: str
url: str
options: Optional[Dict[str, Any]] = None
diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.py
index 39bb6acc7..655048104 100644
--- a/apps/python-sdk/firecrawl/v2/utils/validation.py
+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py
@@ -2,8 +2,8 @@
Shared validation functions for Firecrawl v2 API.
"""
-from typing import Optional, Dict, Any
-from ..types import ScrapeOptions
+from typing import Optional, Dict, Any, List
+from ..types import ScrapeOptions, ScrapeFormats
def _convert_format_string(format_str: str) -> str:
@@ -144,34 +144,112 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
if value is not None:
if key == "formats":
# Handle formats conversion
- converted_formats = []
- for fmt in value:
- if isinstance(fmt, str):
- # Handle format strings
- if fmt == "json":
- raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
- converted_formats.append(_convert_format_string(fmt))
- elif isinstance(fmt, dict):
- # Handle format objects
- if fmt.get('type') == 'json':
- validated_json = _validate_json_format(fmt)
- converted_formats.append(validated_json)
+ converted_formats: List[Any] = []
+
+ # Prefer using original object to detect ScrapeFormats vs list
+ original_formats = getattr(options, 'formats', None)
+
+ if isinstance(original_formats, ScrapeFormats):
+ # Include explicit list first
+ if original_formats.formats:
+ for fmt in original_formats.formats:
+ if isinstance(fmt, str):
+ if fmt == "json":
+ raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
+ converted_formats.append(_convert_format_string(fmt))
+ elif isinstance(fmt, dict):
+ fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None
+ if fmt_type == 'json':
+ validated_json = _validate_json_format({**fmt, 'type': 'json'})
+ converted_formats.append(validated_json)
+ elif fmt_type == 'screenshot':
+ # Normalize screenshot options
+ normalized = {**fmt, 'type': 'screenshot'}
+ if 'full_page' in normalized:
+ normalized['fullPage'] = normalized.pop('full_page')
+ # Normalize viewport if it's a model instance
+ vp = normalized.get('viewport')
+ if hasattr(vp, 'model_dump'):
+ normalized['viewport'] = vp.model_dump(exclude_none=True)
+ converted_formats.append(normalized)
+ else:
+ if 'type' in fmt:
+ fmt['type'] = fmt_type or fmt['type']
+ converted_formats.append(fmt)
+ elif hasattr(fmt, 'type'):
+ if fmt.type == 'json':
+ converted_formats.append(fmt.model_dump())
+ else:
+ converted_formats.append(_convert_format_string(fmt.type))
+ else:
+ converted_formats.append(fmt)
+
+ # Add booleans from ScrapeFormats
+ if original_formats.markdown:
+ converted_formats.append("markdown")
+ if original_formats.html:
+ converted_formats.append("html")
+ if original_formats.raw_html:
+ converted_formats.append("rawHtml")
+ if original_formats.links:
+ converted_formats.append("links")
+ if original_formats.screenshot:
+ converted_formats.append("screenshot")
+ if original_formats.change_tracking:
+ converted_formats.append("changeTracking")
+ # Note: We intentionally do not auto-include 'json' when boolean is set,
+ # because JSON requires an object with schema/prompt. The caller must
+ # supply the full json format object explicitly.
+ elif isinstance(original_formats, list):
+ for fmt in original_formats:
+ if isinstance(fmt, str):
+ if fmt == "json":
+ raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")
+ converted_formats.append(_convert_format_string(fmt))
+ elif isinstance(fmt, dict):
+ fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else None
+ if fmt_type == 'json':
+ validated_json = _validate_json_format({**fmt, 'type': 'json'})
+ converted_formats.append(validated_json)
+ elif fmt_type == 'screenshot':
+ normalized = {**fmt, 'type': 'screenshot'}
+ if 'full_page' in normalized:
+ normalized['fullPage'] = normalized.pop('full_page')
+ vp = normalized.get('viewport')
+ if hasattr(vp, 'model_dump'):
+ normalized['viewport'] = vp.model_dump(exclude_none=True)
+ converted_formats.append(normalized)
+ else:
+ if 'type' in fmt:
+ fmt['type'] = fmt_type or fmt['type']
+ converted_formats.append(fmt)
+ elif hasattr(fmt, 'type'):
+ if fmt.type == 'json':
+ converted_formats.append(fmt.model_dump())
+ elif fmt.type == 'screenshot':
+ normalized = {'type': 'screenshot'}
+ if getattr(fmt, 'full_page', None) is not None:
+ normalized['fullPage'] = fmt.full_page
+ if getattr(fmt, 'quality', None) is not None:
+ normalized['quality'] = fmt.quality
+ vp = getattr(fmt, 'viewport', None)
+ if vp is not None:
+ normalized['viewport'] = vp.model_dump(exclude_none=True) if hasattr(vp, 'model_dump') else vp
+ converted_formats.append(normalized)
+ else:
+ converted_formats.append(_convert_format_string(fmt.type))
else:
- # Convert other format objects
- if 'type' in fmt:
- fmt['type'] = _convert_format_string(fmt['type'])
converted_formats.append(fmt)
- elif hasattr(fmt, 'type'):
- # Handle Format objects
- if fmt.type == 'json':
- # For json format, we need the full object
- converted_formats.append(fmt.model_dump())
- else:
- # For other formats, just convert the type
- converted_formats.append(_convert_format_string(fmt.type))
- else:
- converted_formats.append(fmt)
- scrape_data["formats"] = converted_formats
+ else:
+ # Fallback: try to iterate over value if it's a list-like
+ try:
+ for fmt in value:
+ converted_formats.append(fmt)
+ except TypeError:
+ pass
+
+ if converted_formats:
+ scrape_data["formats"] = converted_formats
elif key == "actions":
# Handle actions conversion
converted_actions = []