Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(python-sdk): get_crawl_errors and active_crawls, got rid of useless tests
rafaelmmillerd2b325f815b7
diff --git a/apps/api/requests/v2/crawl.requests.http b/apps/api/requests/v2/crawl.requests.httpindex 47d34fdd2..72be878ee 100644--- a/apps/api/requests/v2/crawl.requests.http+++ b/apps/api/requests/v2/crawl.requests.http@@ -48,3 +48,10 @@ content-type: application/json"url": "https://firecrawl.dev","includePaths": ["blog", "news"]}+++### Active Crawls+# @name activeCrawls+GET {{baseUrl}}/v2/crawl/active HTTP/1.1+Authorization: Bearer {{$dotenv TEST_API_KEY}}+content-type: application/json\ No newline at end of filediff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.pyindex 8cf3147dc..848a55ef5 100644--- a/apps/python-sdk/example.py+++ b/apps/python-sdk/example.py@@ -7,7 +7,7 @@ import osimport timefrom dotenv import load_dotenvfrom firecrawl import Firecrawl-from firecrawl.v2.types import ScrapeOptions, ScrapeFormats+from firecrawl.v2.types import ScrapeOptions, ScrapeFormats, WebhookConfigload_dotenv()@@ -22,18 +22,23 @@ def main():firecrawl = Firecrawl(api_key=api_key, api_url=api_url)- # crawl+ # crawlcrawl_response = firecrawl.crawl("docs.firecrawl.dev", limit=5)print(crawl_response)+ # start crawlcrawl_job = firecrawl.start_crawl('docs.firecrawl.dev', limit=5)print(crawl_job)- while (crawl_job.status != 'completed'):- crawl_job = firecrawl.get_crawl_status(crawl_job.id)+ crawl_response = firecrawl.get_crawl_status(crawl_job.id)+ print(crawl_response)++ while (crawl_response.status != 'completed'):+ print(f"Crawl status: {crawl_response.status}")+ crawl_response = firecrawl.get_crawl_status(crawl_job.id)time.sleep(2)- print(crawl_job)+ print(crawl_response)# crawl params previewparams_data = firecrawl.crawl_params_preview(@@ -42,6 +47,31 @@ def main():)print(params_data)+ # crawl with webhook example+ webhook_job = firecrawl.start_crawl(+ "docs.firecrawl.dev",+ limit=3,+ webhook="https://your-webhook-endpoint.com/firecrawl"+ )++ # advanced webhook with configuration+ webhook_config = WebhookConfig(+ url="https://your-webhook-endpoint.com/firecrawl",+ headers={"Authorization": "Bearer your-token"},+ events=["completed", "failed"]+ )++ webhook_job_advanced = firecrawl.start_crawl(+ "docs.firecrawl.dev",+ limit=2,+ webhook=webhook_config+ )++ # Check crawl errors+ errors = firecrawl.get_crawl_errors(crawl_job.id)+ print(f"Crawl errors: {errors.errors}")+ print(f"Robots blocked: {errors.robots_blocked}")+# search examplessearch_response = firecrawl.search(query="What is the capital of France?",diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.pyindex 47db5bded..3a70e5e55 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py@@ -77,6 +77,90 @@ class TestCrawlE2E:time.sleep(5)assert cancel_job == True+ def test_get_crawl_errors(self):+ """Test getting crawl errors."""+ # First start a crawl+ start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=3)+ assert start_job.id is not None++ job_id = start_job.id++ # Get errors (should work even if no errors exist)+ errors_response = self.client.get_crawl_errors(job_id)++ # Verify the response structure+ assert hasattr(errors_response, 'errors')+ assert hasattr(errors_response, 'robots_blocked')+ assert isinstance(errors_response.errors, list)+ assert isinstance(errors_response.robots_blocked, list)++ # Errors list should contain dictionaries with expected fields+ for error in errors_response.errors:+ assert isinstance(error, dict)+ assert 'id' in error+ assert 'timestamp' in error+ assert 'url' in error+ assert 'error' in error+ assert isinstance(error['id'], str)+ assert isinstance(error['timestamp'], str)+ assert isinstance(error['url'], str)+ assert isinstance(error['error'], str)++ # Robots blocked should be a list of strings+ for blocked_url in errors_response.robots_blocked:+ assert isinstance(blocked_url, str)++ def test_get_crawl_errors_with_invalid_job_id(self):+ """Test getting crawl errors with an invalid job ID."""+ with pytest.raises(Exception):+ self.client.get_crawl_errors("invalid-job-id-12345")++ def test_get_active_crawls(self):+ """Test getting active crawls."""+ # Get active crawls+ active_crawls_response = self.client.active_crawls()++ # Verify the response structure+ assert hasattr(active_crawls_response, 'success')+ assert hasattr(active_crawls_response, 'crawls')+ assert isinstance(active_crawls_response.success, bool)+ assert isinstance(active_crawls_response.crawls, list)++ # Each crawl should have the required fields+ for crawl in active_crawls_response.crawls:+ assert hasattr(crawl, 'id')+ assert hasattr(crawl, 'team_id')+ assert hasattr(crawl, 'url')+ assert isinstance(crawl.id, str)+ assert isinstance(crawl.team_id, str)+ assert isinstance(crawl.url, str)++ # Options field is optional but if present should be a dict+ if hasattr(crawl, 'options') and crawl.options is not None:+ assert isinstance(crawl.options, dict)++ def test_get_active_crawls_with_running_crawl(self):+ """Test getting active crawls when there's a running crawl."""+ # Start a crawl+ start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=5)+ assert start_job.id is not None++ # Get active crawls+ active_crawls_response = self.client.active_crawls()++ # Verify the response structure+ assert hasattr(active_crawls_response, 'success')+ assert hasattr(active_crawls_response, 'crawls')+ assert isinstance(active_crawls_response.success, bool)+ assert isinstance(active_crawls_response.crawls, list)++ # The started crawl should be in the active crawls list+ active_crawl_ids = [crawl.id for crawl in active_crawls_response.crawls]+ assert start_job.id in active_crawl_ids++ # Cancel the crawl to clean up+ self.client.cancel_crawl(start_job.id)+def test_crawl_with_wait(self):"""Test crawl with wait for completion."""crawl_job = self.client.crawl(diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/crawl/test_crawl_params.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/crawl/test_crawl_params.pyindex 388b259a0..039870fb8 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/crawl/test_crawl_params.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/crawl/test_crawl_params.py@@ -1,196 +1,70 @@+"""+Unit tests for crawl params functionality in Firecrawl v2 SDK.+"""+import pytest-from unittest.mock import Mock, patch-from firecrawl.v2.types import CrawlParamsRequest, CrawlParamsResponse, CrawlParamsData-from firecrawl.v2.methods.crawl import crawl_params_preview+from firecrawl.v2.types import CrawlParamsRequest, CrawlParamsData-class TestCrawlParams:- """Unit tests for crawl_params function."""+class TestCrawlParamsRequest:+ """Unit tests for CrawlParamsRequest."""- def test_crawl_params_success(self):- """Test successful crawl_params call."""- # Mock client and response- mock_client = Mock()- mock_response = Mock()- mock_response.ok = True- mock_response.json.return_value = {- "success": True,- "data": {- "limit": 10,- "maxDiscoveryDepth": 3,- "ignoreSitemap": False- },- "warning": None- }- mock_client.post.return_value = mock_response-- # Create request+ def test_crawl_params_request_creation(self):+ """Test creating CrawlParamsRequest with valid data."""request = CrawlParamsRequest(url="https://example.com",prompt="Extract all blog posts")- # Call function- result = crawl_params_preview(mock_client, request)-- # Verify client call- mock_client.post.assert_called_once_with("/v2/crawl-params", {- "url": "https://example.com",- "prompt": "Extract all blog posts"- })-- # Verify result- assert isinstance(result, CrawlParamsData)- assert result.limit == 10- assert result.max_discovery_depth == 3- assert result.ignore_sitemap is False- assert result.warning is None+ assert request.url == "https://example.com"+ assert request.prompt == "Extract all blog posts"- def test_crawl_params_api_error(self):- """Test crawl_params with API error."""- # Mock client and response- mock_client = Mock()- mock_response = Mock()- mock_response.ok = False- mock_response.status_code = 400- mock_response.text = "Bad Request"- mock_client.post.return_value = mock_response-- # Create request+ def test_crawl_params_request_serialization(self):+ """Test that CrawlParamsRequest serializes correctly."""request = CrawlParamsRequest(url="https://example.com",- prompt="Extract all blog posts"+ prompt="Extract all blog posts and documentation")- # Call function and expect exception- with pytest.raises(Exception, match="crawl params"):- crawl_params_preview(mock_client, request)-- def test_crawl_params_success_false(self):- """Test crawl_params with success=False in response."""- # Mock client and response- mock_client = Mock()- mock_response = Mock()- mock_response.ok = True- mock_response.json.return_value = {- "success": False,- "error": "Invalid URL provided"- }- mock_client.post.return_value = mock_response-- # Create request- request = CrawlParamsRequest(- url="https://example.com",- prompt="Extract all blog posts"- )+ data = request.model_dump()- # Call function and expect exception- with pytest.raises(Exception, match="Invalid URL provided"):- crawl_params_preview(mock_client, request)+ assert data["url"] == "https://example.com"+ assert data["prompt"] == "Extract all blog posts and documentation"- def test_crawl_params_empty_url(self):- """Test crawl_params with empty URL."""- # Create request with empty URL- request = CrawlParamsRequest(- url="",- prompt="Extract all blog posts"- )-- # Call function and expect exception- with pytest.raises(ValueError, match="URL cannot be empty"):- crawl_params_preview(Mock(), request)- def test_crawl_params_whitespace_url(self):- """Test crawl_params with whitespace-only URL."""- # Create request with whitespace URL- request = CrawlParamsRequest(- url=" ",- prompt="Extract all blog posts"- )-- # Call function and expect exception- with pytest.raises(ValueError, match="URL cannot be empty"):- crawl_params_preview(Mock(), request)+class TestCrawlParamsData:+ """Unit tests for CrawlParamsData."""- def test_crawl_params_empty_prompt(self):- """Test crawl_params with empty prompt."""- # Create request with empty prompt- request = CrawlParamsRequest(- url="https://example.com",- prompt=""- )-- # Call function and expect exception- with pytest.raises(ValueError, match="Prompt cannot be empty"):- crawl_params_preview(Mock(), request)+ def test_crawl_params_data_creation(self):+ """Test creating CrawlParamsData with minimal data."""+ data = CrawlParamsData()++ assert data.include_paths is None+ assert data.exclude_paths is None+ assert data.max_discovery_depth is None+ assert data.ignore_sitemap is False+ assert data.limit is None+ assert data.crawl_entire_domain is False+ assert data.allow_external_links is False+ assert data.scrape_options is None+ assert data.warning is None- def test_crawl_params_whitespace_prompt(self):- """Test crawl_params with whitespace-only prompt."""- # Create request with whitespace prompt- request = CrawlParamsRequest(- url="https://example.com",- prompt=" "+ def test_crawl_params_data_with_values(self):+ """Test creating CrawlParamsData with values."""+ data = CrawlParamsData(+ include_paths=["/blog/*"],+ exclude_paths=["/admin/*"],+ max_discovery_depth=3,+ limit=50,+ crawl_entire_domain=True,+ allow_external_links=False,+ warning="Test warning")- # Call function and expect exception- with pytest.raises(ValueError, match="Prompt cannot be empty"):- crawl_params_preview(Mock(), request)-- def test_crawl_params_complex_options(self):- """Test crawl_params with complex options in response."""- # Mock client and response- mock_client = Mock()- mock_response = Mock()- mock_response.ok = True- mock_response.json.return_value = {- "success": True,- "data": {- "includePaths": ["/blog/*", "/docs/*"],- "excludePaths": ["/admin/*"],- "maxDiscoveryDepth": 3,- "ignoreSitemap": False,- "limit": 50,- "crawlEntireDomain": True,- "allowExternalLinks": False,- "scrapeOptions": {- "formats": ["markdown"],- "onlyMainContent": False,- "mobile": True,- "timeout": None,- "waitFor": None,- "skipTlsVerification": False,- "removeBase64Images": True- }- }- }- mock_client.post.return_value = mock_response-- # Create request- request = CrawlParamsRequest(- url="https://example.com",- prompt="Extract all blog posts and documentation with mobile view"- )-- # Call function- result = crawl_params_preview(mock_client, request)-- # Verify result- assert result is not None-- # Check all fields- assert result.include_paths == ["/blog/*", "/docs/*"]- assert result.exclude_paths == ["/admin/*"]- assert result.max_discovery_depth == 3- assert result.ignore_sitemap is False- assert result.limit == 50- assert result.crawl_entire_domain is True- assert result.allow_external_links is False-- # Check nested scrape options- assert result.scrape_options is not None- assert result.scrape_options.formats is not None- # formats is a ScrapeFormats object, so we need to access its formats field- assert len(result.scrape_options.formats.formats) == 1- assert result.scrape_options.formats.formats[0].type == "markdown"- assert result.scrape_options.only_main_content is False- assert result.scrape_options.mobile is True\ No newline at end of file+ assert data.include_paths == ["/blog/*"]+ assert data.exclude_paths == ["/admin/*"]+ assert data.max_discovery_depth == 3+ assert data.limit == 50+ assert data.crawl_entire_domain is True+ assert data.allow_external_links is False+ assert data.warning == "Test warning"\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/crawl/test_webhook.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/crawl/test_webhook.pynew file mode 100644index 000000000..eca5d06fa--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/crawl/test_webhook.py@@ -0,0 +1,123 @@+"""+Unit tests for webhook functionality in Firecrawl v2 SDK.+"""++import pytest+from firecrawl.v2.types import WebhookConfig, CrawlRequest+from firecrawl.v2.methods.crawl import _prepare_crawl_request++class TestWebhookConfig:+ """Test WebhookConfig class functionality."""++ def test_webhook_config_creation_minimal(self):+ """Test creating WebhookConfig with minimal parameters."""+ webhook = WebhookConfig(url="https://example.com/webhook")+ assert webhook.url == "https://example.com/webhook"+ assert webhook.headers is None+ assert webhook.metadata is None+ assert webhook.events is None++ def test_webhook_config_creation_full(self):+ """Test creating WebhookConfig with all parameters."""+ webhook = WebhookConfig(+ url="https://example.com/webhook",+ headers={"Authorization": "Bearer token"},+ metadata={"project": "test"},+ events=["completed", "failed"]+ )+ assert webhook.url == "https://example.com/webhook"+ assert webhook.headers == {"Authorization": "Bearer token"}+ assert webhook.metadata == {"project": "test"}+ assert webhook.events == ["completed", "failed"]++ def test_webhook_config_validation(self):+ """Test WebhookConfig validation."""+ # URL is required+ with pytest.raises(Exception): # Pydantic validation error+ WebhookConfig()+++class TestCrawlRequestWebhook:+ """Test CrawlRequest webhook functionality."""++ def test_crawl_request_with_string_webhook(self):+ """Test CrawlRequest with string webhook."""+ request = CrawlRequest(+ url="https://example.com",+ webhook="https://example.com/webhook"+ )++ data = _prepare_crawl_request(request)+ assert data["webhook"] == "https://example.com/webhook"++ def test_crawl_request_with_webhook_config(self):+ """Test CrawlRequest with WebhookConfig object."""+ webhook_config = WebhookConfig(+ url="https://example.com/webhook",+ headers={"Authorization": "Bearer token"},+ events=["completed"]+ )++ request = CrawlRequest(+ url="https://example.com",+ webhook=webhook_config+ )++ data = _prepare_crawl_request(request)+ assert data["webhook"]["url"] == "https://example.com/webhook"+ assert data["webhook"]["headers"] == {"Authorization": "Bearer token"}+ assert data["webhook"]["events"] == ["completed"]++ def test_crawl_request_without_webhook(self):+ """Test CrawlRequest without webhook."""+ request = CrawlRequest(url="https://example.com")++ data = _prepare_crawl_request(request)+ assert "webhook" not in data++ def test_crawl_request_webhook_serialization(self):+ """Test that webhook config is properly serialized."""+ webhook_config = WebhookConfig(+ url="https://example.com/webhook",+ headers={"Content-Type": "application/json"},+ metadata={"test": "value"},+ events=["page", "completed"]+ )++ request = CrawlRequest(+ url="https://example.com",+ webhook=webhook_config+ )++ data = _prepare_crawl_request(request)+ webhook_data = data["webhook"]++ # Check that all fields are properly serialized+ assert webhook_data["url"] == "https://example.com/webhook"+ assert webhook_data["headers"] == {"Content-Type": "application/json"}+ assert webhook_data["metadata"] == {"test": "value"}+ assert webhook_data["events"] == ["page", "completed"]++ def test_crawl_request_webhook_with_none_values(self):+ """Test webhook config with None values are excluded from serialization."""+ webhook_config = WebhookConfig(+ url="https://example.com/webhook",+ headers=None,+ metadata=None,+ events=None+ )++ request = CrawlRequest(+ url="https://example.com",+ webhook=webhook_config+ )++ data = _prepare_crawl_request(request)+ webhook_data = data["webhook"]++ # Only url should be present, None values should be excluded+ assert webhook_data["url"] == "https://example.com/webhook"+ assert "headers" not in webhook_data+ assert "metadata" not in webhook_data+ assert "events" not in webhook_data+diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.pyindex 015c8f6b8..ed7253816 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py@@ -1,5 +1,5 @@import pytest-from firecrawl.v2.types import ScrapeOptions+from firecrawl.v2.types import JsonFormat, ScrapeOptionsfrom firecrawl.v2.utils.validation import validate_scrape_options, prepare_scrape_options@@ -220,10 +220,8 @@ class TestPrepareScrapeOptions:def test_format_schema_conversion(self):"""Test that Format schema is properly handled."""- from firecrawl.v2.types import Format-- # Create a Format object with schema- format_obj = Format(+ # Create a JsonFormat object with schema+ format_obj = JsonFormat(type="json",prompt="Extract product info",schema={"type": "object", "properties": {"name": {"type": "string"}}}diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex a99e9326d..2bd5f1125 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -137,6 +137,9 @@ class Firecrawl:self.crawl_params_preview = self._v2_client.crawl_params_previewself.get_crawl_status = self._v2_client.get_crawl_statusself.cancel_crawl = self._v2_client.cancel_crawl+ self.get_crawl_errors = self._v2_client.get_crawl_errors+ self.active_crawls = self._v2_client.active_crawls+# self.batch_scrape = self._v2_client.batch_scrape# self.map = self._v2_client.mapself.search = self._v2_client.searchdiff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.pyindex 755b8e40a..0d7323084 100644--- a/apps/python-sdk/firecrawl/types.py+++ b/apps/python-sdk/firecrawl/types.py@@ -27,6 +27,8 @@ from .v2.types import (CrawlParamsRequest,CrawlParamsData,CrawlParamsResponse,+ CrawlErrorsResponse,+ ActiveCrawlsResponse,# Batch scrape typesBatchScrapeRequest,@@ -44,6 +46,7 @@ from .v2.types import (Source,SourceOption,Format,+ JsonFormat,FormatOption,SearchRequest,SearchResult,@@ -63,7 +66,6 @@ from .v2.types import (# Location and format typesLocation,- JsonFormat,# Error typesErrorDetails,@@ -102,6 +104,8 @@ __all__ = ['CrawlParamsRequest','CrawlParamsData','CrawlParamsResponse',+ 'CrawlErrorsResponse',+ 'ActiveCrawlsResponse',# Batch scrape types'BatchScrapeRequest',@@ -119,6 +123,7 @@ __all__ = ['Source','SourceOption','Format',+ 'JsonFormat','FormatOption','SearchRequest','SearchResult',@@ -138,7 +143,6 @@ __all__ = [# Location and format types'Location',- 'JsonFormat',# Error types'ErrorDetails',diff --git a/apps/python-sdk/firecrawl/v2/__init__.py b/apps/python-sdk/firecrawl/v2/__init__.pyindex fad147bce..01ce7a5a7 100644--- a/apps/python-sdk/firecrawl/v2/__init__.py+++ b/apps/python-sdk/firecrawl/v2/__init__.py@@ -2,7 +2,7 @@ from typing import Optional, List, Union, Dict, Callable, Literal, Anyfrom ..types import (SearchData, SearchResult, Document, ScrapeOptions, SearchRequest,SourceOption, FormatOption, CrawlRequest, CrawlJob, CrawlResponse,- CrawlParamsRequest, CrawlParamsData+ CrawlParamsRequest, CrawlParamsData, CrawlErrorsResponse, ActiveCrawlsResponse)from .methods.search import search as search_methodfrom .methods.crawl import (@@ -10,7 +10,9 @@ from .methods.crawl import (start_crawl as start_crawl_method,cancel_crawl as cancel_crawl_method,get_crawl_status as get_crawl_status_method,- crawl_params_preview as crawl_params_preview_method+ crawl_params_preview as crawl_params_preview_method,+ get_crawl_errors as get_crawl_errors_method,+ get_active_crawls as get_active_crawls_method)from .utils.http_client import HttpClient@@ -20,7 +22,7 @@ class FirecrawlClient:"""def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):- self.api_key = api_key or "placeholder-key"+ self.api_key = api_keyself.api_url = api_urlself._client = HttpClient(api_key=self.api_key, api_url=self.api_url)@@ -143,7 +145,6 @@ class FirecrawlClient:)return start_crawl_method(self._client, request)- # get-crawl-statusdef get_crawl_status(self,job_id: str@@ -151,7 +152,6 @@ class FirecrawlClient:"""Get the status of a crawl job."""return get_crawl_status_method(self._client, job_id)- # cancel-crawldef cancel_crawl(self,job_id: str@@ -159,7 +159,6 @@ class FirecrawlClient:"""Cancel a running crawl job."""return cancel_crawl_method(self._client, job_id)- # crawl-params-previewdef crawl_params_preview(self,url: str,@@ -169,17 +168,18 @@ class FirecrawlClient:request = CrawlParamsRequest(url=url, prompt=prompt)return crawl_params_preview_method(self._client, request)- # get-active-crawls- def get_active_crawls(+ def active_crawls(self- ):- pass+ ) -> ActiveCrawlsResponse:+ """Get a list of active crawl jobs."""+ return get_active_crawls_method(self._client)- # get-crawl-errorsdef get_crawl_errors(- self- ):- pass+ self,+ crawl_id: str+ ) -> CrawlErrorsResponse:+ """Get errors from a crawl job."""+ return get_crawl_errors_method(self._client, crawl_id)# mapdef map(diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 113ca961e..718281aef 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -5,21 +5,19 @@ This module provides the main client class that orchestrates all v2 functionalit"""import os-from typing import Optional, List, Dict, Any, Callable+from typing import Optional, List, Dict, Any, Callable, Unionfrom .types import (ClientConfig, ScrapeOptions, CrawlOptions, MapOptions, ExtractOptions,ScrapeResponse, CrawlResponse, CrawlStatusResponse, BatchScrapeResponse,BatchScrapeStatusResponse, MapResponse, ExtractResponse, Document,- SearchRequest, SearchResponse+ SearchRequest, SearchResponse, CrawlRequest, WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse)from .utils.http_client import HttpClientfrom .utils.error_handler import FirecrawlError-# Import functions from modules-from . import scrape as scrape_module-from . import crawl as crawl_module-from . import batch as batch_module-from . import search as search_module-+from .methods import scrape as scrape_module+from .methods import crawl as crawl_module+from .methods import batch as batch_module+from .methods import search as search_moduleclass FirecrawlClient:"""@@ -108,7 +106,6 @@ class FirecrawlClient:Returns:SearchResponse containing the search results"""- # Build options object from individual parametersoptions = SearchRequest(limit=limit,tbs=tbs,@@ -118,4 +115,200 @@ class FirecrawlClient:)return search_module.search(self.http_client, query, options)++ def crawl(+ self,+ url: str,+ *,+ prompt: Optional[str] = None,+ exclude_paths: Optional[List[str]] = None,+ include_paths: Optional[List[str]] = None,+ max_discovery_depth: Optional[int] = None,+ ignore_sitemap: bool = False,+ ignore_query_parameters: bool = False,+ limit: Optional[int] = None,+ crawl_entire_domain: bool = False,+ allow_external_links: bool = False,+ allow_subdomains: bool = False,+ delay: Optional[int] = None,+ max_concurrency: Optional[int] = None,+ webhook: Optional[Union[str, WebhookConfig]] = None,+ scrape_options: Optional[ScrapeOptions] = None,+ zero_data_retention: bool = False,+ poll_interval: int = 2,+ timeout: Optional[int] = None+ ) -> CrawlStatusResponse:+ """+ Start a crawl job and wait for it to complete.++ Args:+ url: Target URL to start crawling from+ prompt: Optional prompt to guide the crawl+ exclude_paths: Patterns of URLs to exclude+ include_paths: Patterns of URLs to include+ max_discovery_depth: Maximum depth for finding new URLs+ ignore_sitemap: Skip sitemap.xml processing+ ignore_query_parameters: Ignore URL parameters+ limit: Maximum pages to crawl+ crawl_entire_domain: Follow parent directory links+ allow_external_links: Follow external domain links+ allow_subdomains: Follow subdomains+ delay: Delay in seconds between scrapes+ max_concurrency: Maximum number of concurrent scrapes+ webhook: Webhook configuration for notifications+ scrape_options: Page scraping configuration+ zero_data_retention: Whether to delete data after 24 hours+ poll_interval: Seconds between status checks+ timeout: Maximum seconds to wait (None for no timeout)++ Returns:+ CrawlStatusResponse when job completes++ Raises:+ ValueError: If request is invalid+ Exception: If the crawl fails to start or complete+ TimeoutError: If timeout is reached+ """+ request = CrawlRequest(+ url=url,+ prompt=prompt,+ exclude_paths=exclude_paths,+ include_paths=include_paths,+ max_discovery_depth=max_discovery_depth,+ ignore_sitemap=ignore_sitemap,+ ignore_query_parameters=ignore_query_parameters,+ limit=limit,+ crawl_entire_domain=crawl_entire_domain,+ allow_external_links=allow_external_links,+ allow_subdomains=allow_subdomains,+ delay=delay,+ max_concurrency=max_concurrency,+ webhook=webhook,+ scrape_options=scrape_options,+ zero_data_retention=zero_data_retention+ )++ return crawl_module.crawl(+ self.http_client,+ request,+ poll_interval=poll_interval,+ timeout=timeout+ )++ def start_crawl(+ self,+ url: str,+ *,+ prompt: Optional[str] = None,+ exclude_paths: Optional[List[str]] = None,+ include_paths: Optional[List[str]] = None,+ max_discovery_depth: Optional[int] = None,+ ignore_sitemap: bool = False,+ ignore_query_parameters: bool = False,+ limit: Optional[int] = None,+ crawl_entire_domain: bool = False,+ allow_external_links: bool = False,+ allow_subdomains: bool = False,+ delay: Optional[int] = None,+ max_concurrency: Optional[int] = None,+ webhook: Optional[Union[str, WebhookConfig]] = None,+ scrape_options: Optional[ScrapeOptions] = None,+ zero_data_retention: bool = False+ ) -> CrawlResponse:+ """+ Start an asynchronous crawl job.++ Args:+ url: Target URL to start crawling from+ prompt: Optional prompt to guide the crawl+ exclude_paths: Patterns of URLs to exclude+ include_paths: Patterns of URLs to include+ max_discovery_depth: Maximum depth for finding new URLs+ ignore_sitemap: Skip sitemap.xml processing+ ignore_query_parameters: Ignore URL parameters+ limit: Maximum pages to crawl+ crawl_entire_domain: Follow parent directory links+ allow_external_links: Follow external domain links+ allow_subdomains: Follow subdomains+ delay: Delay in seconds between scrapes+ max_concurrency: Maximum number of concurrent scrapes+ webhook: Webhook configuration for notifications+ scrape_options: Page scraping configuration+ zero_data_retention: Whether to delete data after 24 hours++ Returns:+ CrawlResponse with job information++ Raises:+ ValueError: If request is invalid+ Exception: If the crawl operation fails to start+ """+ request = CrawlRequest(+ url=url,+ prompt=prompt,+ exclude_paths=exclude_paths,+ include_paths=include_paths,+ max_discovery_depth=max_discovery_depth,+ ignore_sitemap=ignore_sitemap,+ ignore_query_parameters=ignore_query_parameters,+ limit=limit,+ crawl_entire_domain=crawl_entire_domain,+ allow_external_links=allow_external_links,+ allow_subdomains=allow_subdomains,+ delay=delay,+ max_concurrency=max_concurrency,+ webhook=webhook,+ scrape_options=scrape_options,+ zero_data_retention=zero_data_retention+ )++ return crawl_module.start_crawl(self.http_client, request)++ def get_crawl_status(self, job_id: str) -> CrawlStatusResponse:+ """+ Get the status of a crawl job.++ Args:+ job_id: ID of the crawl job++ Returns:+ CrawlStatusResponse with current status and data++ Raises:+ Exception: If the status check fails+ """+ return crawl_module.get_crawl_status(self.http_client, job_id)++ def check_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse:+ """+ Get errors from a crawl job.++ Args:+ crawl_id: The ID of the crawl job++ Returns:+ CrawlErrorsResponse containing errors and robots blocked URLs+ """+ return crawl_module.check_crawl_errors(self.http_client, crawl_id)++ def get_active_crawls(self) -> ActiveCrawlsResponse:+ """+ Get a list of currently active crawl jobs.++ Returns:+ ActiveCrawlsResponse containing a list of active crawl jobs.+ """+ return crawl_module.get_active_crawls(self.http_client)++ def cancel_crawl(self, crawl_id: str) -> bool:+ """+ Cancel a crawl job.++ Args:+ crawl_id: The ID of the crawl job to cancel++ Returns:+ bool: True if the crawl was cancelled, False otherwise+ """+ return crawl_module.cancel_crawl(self.http_client, crawl_id)\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex c86c85302..1894723f1 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -3,11 +3,12 @@ Crawling functionality for Firecrawl v2 API."""import time-from typing import Optional+from typing import Optional, Dict, Anyfrom ..types import (CrawlRequest,CrawlJob,- CrawlResponse, Document, CrawlParamsRequest, CrawlParamsResponse, CrawlParamsData+ CrawlResponse, Document, CrawlParamsRequest, CrawlParamsResponse, CrawlParamsData,+ WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse)from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options@@ -67,6 +68,14 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:request_data.pop("prompt", None)request_data.pop("scrape_options", None)+ # Handle webhook conversion first (before model_dump)+ if request.webhook is not None:+ if isinstance(request.webhook, str):+ data["webhook"] = request.webhook+ else:+ # Convert WebhookConfig to dict+ data["webhook"] = request.webhook.model_dump(exclude_none=True)+# Convert other snake_case fields to camelCasefield_mappings = {"include_paths": "includePaths",@@ -79,7 +88,6 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:"allow_subdomains": "allowSubdomains","delay": "delay","max_concurrency": "maxConcurrency",- "webhook": "webhook","zero_data_retention": "zeroDataRetention"}@@ -333,6 +341,14 @@ def crawl_params_preview(client: HttpClient, request: CrawlParamsRequest) -> Cra"zeroDataRetention": "zero_data_retention"}+ # Handle webhook conversion+ if "webhook" in params_data:+ webhook_data = params_data["webhook"]+ if isinstance(webhook_data, dict):+ converted_params["webhook"] = WebhookConfig(**webhook_data)+ else:+ converted_params["webhook"] = webhook_data+for camel_case, snake_case in field_mappings.items():if camel_case in params_data:if camel_case == "scrapeOptions" and params_data[camel_case] is not None:@@ -382,4 +398,56 @@ def crawl_params_preview(client: HttpClient, request: CrawlParamsRequest) -> Crareturn CrawlParamsData(**converted_params)else:- raise Exception(response_data.get("error", "Unknown error occurred"))\ No newline at end of file+ raise Exception(response_data.get("error", "Unknown error occurred"))+++def get_crawl_errors(http_client: HttpClient, crawl_id: str) -> CrawlErrorsResponse:+ """+ Get errors from a crawl job.++ Args:+ http_client: HTTP client for making requests+ crawl_id: The ID of the crawl job++ Returns:+ CrawlErrorsResponse containing errors and robots blocked URLs++ Raises:+ Exception: If the request fails+ """+ response = http_client.get(f"/v2/crawl/{crawl_id}/errors")++ if response.status_code != 200:+ handle_response_error(response, "check crawl errors")++ try:+ data = response.json()+ return CrawlErrorsResponse(**data)+ except Exception as e:+ raise Exception(f"Failed to parse crawl errors response: {e}")+++def get_active_crawls(client: HttpClient) -> ActiveCrawlsResponse:+ """+ Get a list of currently active crawl jobs.++ Args:+ client: HTTP client instance++ Returns:+ ActiveCrawlsResponse containing a list of active crawl jobs++ Raises:+ Exception: If the request fails+ """+ response = client.get("/v2/crawl/active")++ if not response.ok:+ handle_response_error(response, "get active crawls")++ response_data = response.json()++ if response_data.get("success"):+ return ActiveCrawlsResponse(**response_data.get("data", {}))+ else:+ raise Exception(response_data.get("error", "Unknown error occurred"))diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex c3e461c66..b4262314f 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -54,13 +54,29 @@ class Document(BaseModel):warning: Optional[str] = Nonechange_tracking: Optional[Dict[str, Any]] = Field(None, alias="changeTracking")+# Webhook types+class WebhookConfig(BaseModel):+ """Configuration for webhooks."""+ url: str+ headers: Optional[Dict[str, str]] = None+ metadata: Optional[Dict[str, str]] = None+ events: Optional[List[Literal["completed", "failed", "page", "started"]]] = None++class WebhookData(BaseModel):+ """Data sent to webhooks."""+ job_id: str = Field(alias="jobId")+ status: str+ current: Optional[int] = None+ total: Optional[int] = None+ data: Optional[List[Document]] = None+ error: Optional[str] = None+class Source(BaseModel):"""Configuration for a search source."""type: strSourceOption = Union[str, Source]-FormatString = Literal[# camelCase versions (API format)"markdown", "html", "rawHtml", "links", "screenshot", "json", "changeTracking",@@ -183,7 +199,7 @@ class CrawlRequest(BaseModel):allow_subdomains: bool = Falsedelay: Optional[int] = Nonemax_concurrency: Optional[int] = None- webhook: Optional[Dict[str, Any]] = None+ webhook: Optional[Union[str, WebhookConfig]] = Nonescrape_options: Optional[ScrapeOptions] = Nonezero_data_retention: bool = False@@ -231,7 +247,7 @@ class CrawlParamsData(BaseModel):allow_subdomains: bool = Falsedelay: Optional[int] = Nonemax_concurrency: Optional[int] = None- webhook: Optional[Dict[str, Any]] = None+ webhook: Optional[Union[str, WebhookConfig]] = Nonescrape_options: Optional[ScrapeOptions] = Nonezero_data_retention: bool = Falsewarning: Optional[str] = None@@ -343,12 +359,6 @@ class Location(BaseModel):country: Optional[str] = Nonelanguages: Optional[List[str]] = None-# JSON format types-class JsonFormat(BaseModel):- """Configuration for JSON extraction."""- prompt: Optional[str] = None- schema_field: Optional[Dict[str, Any]] = Field(None, alias="schema")-class SearchRequest(BaseModel):"""Request for search operations."""query: str@@ -420,15 +430,22 @@ class JobStatus(BaseModel):completed_at: Optional[datetime] = Field(None, alias="completedAt")expires_at: Optional[datetime] = Field(None, alias="expiresAt")-# Webhook types-class WebhookData(BaseModel):- """Data sent to webhooks."""- job_id: str = Field(alias="jobId")- status: str- current: Optional[int] = None- total: Optional[int] = None- data: Optional[List[Document]] = None- error: Optional[str] = None+class CrawlErrorsResponse(BaseModel):+ """Response from crawl error monitoring."""+ errors: List[Dict[str, str]] = Field(description="List of errors with fields: id, timestamp, url, error")+ robots_blocked: List[str] = Field(alias="robotsBlocked", description="List of URLs blocked by robots.txt")++class ActiveCrawl(BaseModel):+ """Information about an active crawl job."""+ id: str+ team_id: str = Field(alias="teamId")+ url: str+ options: Optional[Dict[str, Any]] = None++class ActiveCrawlsResponse(BaseModel):+ """Response from active crawls endpoint."""+ success: bool = True+ crawls: List[ActiveCrawl]# Configuration typesclass ClientConfig(BaseModel):@@ -446,5 +463,5 @@ AnyResponse = Union[BatchScrapeResponse,MapResponse,SearchResponse,- ErrorResponse+ ErrorResponse,]\ No newline at end of file