Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(python-sdk): expose job id and warning on CrawlJob (#4895)
claude[bot]c4873f545dfd
diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex a98e7c9f7..708521597 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -111,6 +111,17 @@ crawl_status = firecrawl.crawl(print(crawl_status)```+The result is a `CrawlJob`. It includes the job `id` and any `warning` from the API, so you can use the job after the crawl ends, for example with `get_crawl_errors(crawl_status.id)`. If the crawl does not finish within `timeout`, the SDK raises `CrawlJobTimeoutError`. It is a `TimeoutError` subclass with a `job_id` attribute, so you can check or cancel the job later.++```python+from firecrawl import CrawlJobTimeoutError++try:+ crawl_status = firecrawl.crawl('https://firecrawl.dev', limit=100, timeout=120)+except CrawlJobTimeoutError as e:+ firecrawl.cancel_crawl(e.job_id)+```+### Asynchronous Crawling<Tip>Looking for async operations? Check out the [Async Class](#async-class) section below.</Tip>diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 6ebce505e..0a3d967a4 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -19,6 +19,7 @@ from .v2.types import (AlexandriaScrapeResult,ExchangeSearchResult,)+from .v2.utils.error_handler import CrawlJobTimeoutErrorfrom .v1 import (V1FirecrawlApp,AsyncV1FirecrawlApp,@@ -27,7 +28,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.45.1"+__version__ = "4.46.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")@@ -97,6 +98,7 @@ __all__ = ['AlexandriaScrapeData','AlexandriaScrapeResult','ExchangeSearchResult',+ 'CrawlJobTimeoutError','V1FirecrawlApp','AsyncV1FirecrawlApp','V1JsonConfig',diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_job_fields.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_job_fields.pynew file mode 100644index 000000000..a7401cd9d--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_job_fields.py@@ -0,0 +1,228 @@+"""+Unit tests for the job id and warning on CrawlJob, and for CrawlJobTimeoutError.+"""++import itertools+from unittest.mock import AsyncMock, Mock, patch++import pytest++import firecrawl+from firecrawl.v2.client_async import AsyncFirecrawlClient+from firecrawl.v2.methods import crawl as crawl_module+from firecrawl.v2.methods.aio import crawl as async_crawl_module+from firecrawl.v2.types import CrawlJob, CrawlRequest, PaginationConfig+from firecrawl.v2.utils import CrawlJobTimeoutError++JOB_ID = "crawl-job-123"+WARNING = "Only 1 result(s) found. For broader coverage, try crawling with crawlEntireDomain=true"+SAMPLE_DOC = {"url": "https://example.com", "markdown": "# Hello", "metadata": {"statusCode": 200}}+++def _status_body(status="completed", next_url=None, warning=WARNING):+ body = {+ "success": True,+ "status": status,+ "completed": 1,+ "total": 1,+ "creditsUsed": 1,+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": next_url,+ "data": [SAMPLE_DOC],+ }+ if warning is not None:+ body["warning"] = warning+ return body+++def _sync_response(body):+ response = Mock()+ response.ok = True+ response.status_code = 200+ response.json.return_value = body+ return response+++def _async_response(body):+ response = Mock()+ response.status_code = 200+ response.json.return_value = body+ return response+++def _stepping_clock(step=10.0):+ counter = itertools.count()+ return lambda: next(counter) * step+++class TestCrawlJobModel:+ def test_id_and_warning_are_optional(self):+ job = CrawlJob(status="completed")+ assert job.id is None+ assert job.warning is None+++class TestSyncCrawlJobFields:+ def test_get_crawl_status_sets_id_and_warning(self):+ client = Mock()+ client.get.return_value = _sync_response(_status_body())++ job = crawl_module.get_crawl_status(client, JOB_ID)++ assert job.id == JOB_ID+ assert job.warning == WARNING++ def test_get_crawl_status_without_warning(self):+ client = Mock()+ client.get.return_value = _sync_response(_status_body(warning=None))++ job = crawl_module.get_crawl_status(client, JOB_ID)++ assert job.id == JOB_ID+ assert job.warning is None++ def test_get_crawl_status_keeps_id_and_warning_with_auto_pagination(self):+ client = Mock()+ client.get.side_effect = [+ _sync_response(_status_body(next_url=f"https://api.firecrawl.dev/v2/crawl/{JOB_ID}?skip=1")),+ _sync_response(_status_body(warning=None)),+ ]++ job = crawl_module.get_crawl_status(client, JOB_ID, PaginationConfig(auto_paginate=True))++ assert client.get.call_count == 2+ assert len(job.data) == 2+ assert job.id == JOB_ID+ assert job.warning == WARNING++ def test_get_crawl_status_page_reads_id_from_next_url(self):+ client = Mock()+ client.get.return_value = _sync_response(_status_body())++ job = crawl_module.get_crawl_status_page(+ client, f"https://api.firecrawl.dev/v2/crawl/{JOB_ID}?skip=10"+ )++ assert job.id == JOB_ID+ assert job.warning == WARNING++ def test_crawl_returns_job_with_id(self):+ client = Mock()+ start_response = _sync_response({"success": True, "id": JOB_ID, "url": f"https://api.firecrawl.dev/v2/crawl/{JOB_ID}"})+ client.post.return_value = start_response+ client.get.return_value = _sync_response(_status_body())++ job = crawl_module.crawl(client, CrawlRequest(url="https://example.com"))++ assert job.id == JOB_ID+ assert job.status == "completed"+ assert job.warning == WARNING++ def test_wait_timeout_raises_crawl_job_timeout_error(self):+ client = Mock()+ client.get.return_value = _sync_response(_status_body(status="scraping"))++ with patch.object(crawl_module.time, "monotonic", side_effect=_stepping_clock()), \+ patch.object(crawl_module.time, "sleep"):+ with pytest.raises(CrawlJobTimeoutError) as exc_info:+ crawl_module.wait_for_crawl_completion(client, JOB_ID, poll_interval=1, timeout=5)++ error = exc_info.value+ assert isinstance(error, TimeoutError)+ assert error.job_id == JOB_ID+ assert error.timeout == 5+ assert str(error) == f"Crawl job {JOB_ID} did not complete within 5 seconds"++ def test_existing_timeout_error_handlers_still_catch_it(self):+ client = Mock()+ client.get.return_value = _sync_response(_status_body(status="scraping"))++ with patch.object(crawl_module.time, "monotonic", side_effect=_stepping_clock()), \+ patch.object(crawl_module.time, "sleep"):+ with pytest.raises(TimeoutError):+ crawl_module.wait_for_crawl_completion(client, JOB_ID, poll_interval=1, timeout=5)++ def test_timeout_error_survives_pickle_and_deepcopy(self):+ import copy+ import pickle++ error = CrawlJobTimeoutError(JOB_ID, 5)+ for clone in (pickle.loads(pickle.dumps(error)), copy.deepcopy(error)):+ assert isinstance(clone, CrawlJobTimeoutError)+ assert clone.job_id == JOB_ID+ assert clone.timeout == 5+ assert str(clone) == str(error)++ def test_error_is_exported_from_package_root(self):+ assert firecrawl.CrawlJobTimeoutError is CrawlJobTimeoutError+++class TestAsyncCrawlJobFields:+ @pytest.mark.asyncio+ async def test_get_crawl_status_sets_id_and_warning(self):+ client = AsyncMock()+ client.get.return_value = _async_response(_status_body())++ job = await async_crawl_module.get_crawl_status(client, JOB_ID)++ assert job.id == JOB_ID+ assert job.warning == WARNING++ @pytest.mark.asyncio+ async def test_get_crawl_status_keeps_id_and_warning_with_auto_pagination(self):+ client = AsyncMock()+ client.get.side_effect = [+ _async_response(_status_body(next_url=f"https://api.firecrawl.dev/v2/crawl/{JOB_ID}?skip=1")),+ _async_response(_status_body(warning=None)),+ ]++ job = await async_crawl_module.get_crawl_status(client, JOB_ID)++ assert client.get.call_count == 2+ assert len(job.data) == 2+ assert job.id == JOB_ID+ assert job.warning == WARNING++ @pytest.mark.asyncio+ async def test_get_crawl_status_page_reads_id_from_next_url(self):+ client = AsyncMock()+ client.get.return_value = _async_response(_status_body())++ job = await async_crawl_module.get_crawl_status_page(+ client, f"https://api.firecrawl.dev/v2/crawl/{JOB_ID}?skip=10"+ )++ assert job.id == JOB_ID+ assert job.warning == WARNING++ @pytest.mark.asyncio+ async def test_client_crawl_returns_job_with_id(self):+ client = AsyncFirecrawlClient(api_key="fc-test")+ client.async_http_client = AsyncMock()+ client.async_http_client.post.return_value = _async_response(+ {"success": True, "id": JOB_ID, "url": f"https://api.firecrawl.dev/v2/crawl/{JOB_ID}"}+ )+ client.async_http_client.get.return_value = _async_response(_status_body())++ job = await client.crawl(url="https://example.com")++ assert job.id == JOB_ID+ assert job.warning == WARNING++ @pytest.mark.asyncio+ async def test_wait_crawl_timeout_raises_crawl_job_timeout_error(self):+ client = AsyncFirecrawlClient(api_key="fc-test")+ client.async_http_client = AsyncMock()+ client.async_http_client.get.return_value = _async_response(_status_body(status="scraping"))++ import firecrawl.v2.client_async as client_async_module++ with patch.object(client_async_module.time, "monotonic", side_effect=_stepping_clock()), \+ patch.object(client_async_module.asyncio, "sleep", new=AsyncMock()):+ with pytest.raises(CrawlJobTimeoutError) as exc_info:+ await client.wait_crawl(JOB_ID, poll_interval=1, timeout=5)++ error = exc_info.value+ assert isinstance(error, TimeoutError)+ assert error.job_id == JOB_ID+ assert error.timeout == 5diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 583343230..2c690d98f 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -907,7 +907,8 @@ class FirecrawlClient:CrawlJob in a terminal state ("completed", "failed", or "cancelled")Raises:- TimeoutError: If the job does not finish within timeout+ CrawlJobTimeoutError: If the job does not finish within timeout (a+ ``TimeoutError`` subclass that carries ``job_id`` and ``timeout``)"""return crawl_module.wait_for_crawl_completion(self.http_client,diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 1307a2117..6a6e44fa1 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -56,6 +56,7 @@ from .types import ()from .utils.http_client import HttpClientfrom .utils.http_client_async import AsyncHttpClient+from .utils.error_handler import CrawlJobTimeoutErrorfrom .methods.aio import scrape as async_scrape # type: ignore[attr-defined]from .methods.aio import parse as async_parse # type: ignore[attr-defined]@@ -391,7 +392,8 @@ class AsyncFirecrawlClient:CrawlJob: The final status of the crawl job when it reaches a terminal state.Raises:- TimeoutError: If the crawl does not reach a terminal state within the specified timeout.+ CrawlJobTimeoutError: If the crawl does not reach a terminal state within the specified+ timeout. It is a ``TimeoutError`` subclass that carries ``job_id`` and ``timeout``.Terminal states:- "completed": The crawl finished successfully.@@ -408,7 +410,7 @@ class AsyncFirecrawlClient:if status.status in ["completed", "failed", "cancelled"]:return statusif timeout and (time.monotonic() - start) > timeout:- raise TimeoutError("Crawl wait timed out")+ raise CrawlJobTimeoutError(job_id, timeout)await asyncio.sleep(poll_interval)async def crawl(self, **kwargs) -> CrawlJob:diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.pyindex 916649bf9..bdc286649 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py@@ -16,6 +16,7 @@ from ...utils.error_handler import handle_response_errorfrom ...utils.validation import prepare_scrape_optionsfrom ...utils.http_client_async import AsyncHttpClientfrom ...utils.normalize import normalize_document_input+from ..crawl import _job_id_from_next_urlimport time@@ -87,6 +88,7 @@ def _parse_crawl_status_response(body: Dict[str, Any]) -> Dict[str, Any]:"credits_used": body.get("creditsUsed", 0),"expires_at": body.get("expiresAt"),"next": body.get("next"),+ "warning": body.get("warning"),"data": _parse_crawl_documents(body.get("data", [])),}@@ -160,12 +162,14 @@ async def get_crawl_status()return CrawlJob(+ id=job_id,status=payload["status"],completed=payload["completed"],total=payload["total"],credits_used=payload["credits_used"],expires_at=payload["expires_at"],next=payload["next"] if not auto_paginate else None,+ warning=payload["warning"],data=documents,)@@ -196,12 +200,14 @@ async def get_crawl_status_page(body = response.json()payload = _parse_crawl_status_response(body)return CrawlJob(+ id=_job_id_from_next_url(next_url),status=payload["status"],completed=payload["completed"],total=payload["total"],credits_used=payload["credits_used"],expires_at=payload["expires_at"],next=payload["next"],+ warning=payload["warning"],data=payload["data"],)diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex 9f1a8b742..641ddcd6f 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -2,6 +2,7 @@Crawling functionality for Firecrawl v2 API."""+import reimport timefrom typing import Optional, Dict, Any, Listfrom ..types import (@@ -11,6 +12,7 @@ from ..types import (WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, ActiveCrawl, PaginationConfig)from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options+from ..utils.error_handler import CrawlJobTimeoutErrorfrom ..utils.normalize import normalize_document_input@@ -129,10 +131,20 @@ def _parse_crawl_status_response(response_data: Dict[str, Any]) -> Dict[str, Any"credits_used": response_data.get("creditsUsed", 0),"expires_at": response_data.get("expiresAt"),"next": response_data.get("next"),+ "warning": response_data.get("warning"),"data": _parse_crawl_documents(response_data.get("data", [])),}+_CRAWL_JOB_ID_IN_URL = re.compile(r"/v2/crawl/([^/?#]+)")+++def _job_id_from_next_url(next_url: str) -> Optional[str]:+ """Return the crawl job id from a status page URL, if the URL contains one."""+ match = _CRAWL_JOB_ID_IN_URL.search(next_url or "")+ return match.group(1) if match else None++def start_crawl(client: HttpClient, request: CrawlRequest) -> CrawlResponse:"""Start a crawl job for a website.@@ -223,12 +235,14 @@ def get_crawl_status(# Create CrawlJob with current status and datareturn CrawlJob(+ id=job_id,status=payload["status"],completed=payload["completed"],total=payload["total"],credits_used=payload["credits_used"],expires_at=payload["expires_at"],next=payload["next"] if not auto_paginate else None,+ warning=payload["warning"],data=documents,)@@ -262,12 +276,14 @@ def get_crawl_status_page(payload = _parse_crawl_status_response(response_data)return CrawlJob(+ id=_job_id_from_next_url(next_url),status=payload["status"],completed=payload["completed"],total=payload["total"],credits_used=payload["credits_used"],expires_at=payload["expires_at"],next=payload["next"],+ warning=payload["warning"],data=payload["data"],)@@ -393,7 +409,8 @@ def wait_for_crawl_completion(Raises:Exception: If the job fails- TimeoutError: If timeout is reached+ CrawlJobTimeoutError: If timeout is reached (a ``TimeoutError`` subclass+ that carries ``job_id`` and ``timeout``)"""start_time = time.monotonic()@@ -410,7 +427,7 @@ def wait_for_crawl_completion(# Check timeoutif timeout is not None and (time.monotonic() - start_time) > timeout:- raise TimeoutError(f"Crawl job {job_id} did not complete within {timeout} seconds")+ raise CrawlJobTimeoutError(job_id, timeout)# Wait before next polltime.sleep(poll_interval)@@ -441,7 +458,8 @@ def crawl(Raises:ValueError: If request is invalidException: If the crawl fails to start or complete- TimeoutError: If timeout is reached+ CrawlJobTimeoutError: If timeout is reached (a ``TimeoutError`` subclass+ that carries ``job_id`` and ``timeout``)"""# Start the crawlcrawl_job = start_crawl(client, request)diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex ecf2f4dab..8a2653000 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -1011,12 +1011,14 @@ class CrawlResponse(BaseModel):class CrawlJob(BaseModel):"""Crawl job status and progress data."""+ id: Optional[str] = Nonestatus: Literal["scraping", "completed", "failed", "cancelled"]total: int = 0completed: int = 0credits_used: int = 0expires_at: Optional[datetime] = Nonenext: Optional[str] = None+ warning: Optional[str] = Nonedata: List[Document] = []diff --git a/apps/python-sdk/firecrawl/v2/utils/__init__.py b/apps/python-sdk/firecrawl/v2/utils/__init__.pyindex 125ade15d..62b65dc26 100644--- a/apps/python-sdk/firecrawl/v2/utils/__init__.py+++ b/apps/python-sdk/firecrawl/v2/utils/__init__.py@@ -3,7 +3,7 @@ Utility modules for v2 API client."""from .http_client import HttpClient-from .error_handler import FirecrawlError, ProviderTermsRequiredError, RequiresAction, handle_response_error+from .error_handler import CrawlJobTimeoutError, FirecrawlError, ProviderTermsRequiredError, RequiresAction, handle_response_errorfrom .validation import validate_scrape_options, prepare_scrape_options-__all__ = ['HttpClient', 'FirecrawlError', 'ProviderTermsRequiredError', 'RequiresAction', 'handle_response_error', 'validate_scrape_options', 'prepare_scrape_options']\ No newline at end of file+__all__ = ['HttpClient', 'CrawlJobTimeoutError', 'FirecrawlError', 'ProviderTermsRequiredError', 'RequiresAction', 'handle_response_error', 'validate_scrape_options', 'prepare_scrape_options']\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/utils/error_handler.py b/apps/python-sdk/firecrawl/v2/utils/error_handler.pyindex fa90e20ca..b0739df7f 100644--- a/apps/python-sdk/firecrawl/v2/utils/error_handler.py+++ b/apps/python-sdk/firecrawl/v2/utils/error_handler.py@@ -77,6 +77,25 @@ class RequestTimeoutError(FirecrawlError):pass+class CrawlJobTimeoutError(TimeoutError):+ """Raised when a crawl job does not finish within the wait timeout.++ Subclasses the built-in ``TimeoutError``, so ``except TimeoutError`` still+ catches it. The ``job_id`` attribute lets callers keep using the job, for+ example to check its status later or to cancel it.+ """++ def __init__(self, job_id: str, timeout: float):+ super().__init__(f"Crawl job {job_id} did not complete within {timeout} seconds")+ self.job_id = job_id+ self.timeout = timeout++ def __reduce__(self):+ # self.args holds only the message, so rebuild from the constructor+ # arguments. This keeps pickle and copy.deepcopy working.+ return (type(self), (self.job_id, self.timeout))++class RateLimitError(FirecrawlError):"""Raised when the rate limit is exceeded (429)."""passdiff --git a/apps/test-site/pnpm-lock.yaml b/apps/test-site/pnpm-lock.yamlindex 13c60765a..f2362cc9e 100644--- a/apps/test-site/pnpm-lock.yaml+++ b/apps/test-site/pnpm-lock.yaml@@ -7,7 +7,7 @@ settings:overrides:fast-xml-builder: '>=1.1.6 <2.0.0'fast-xml-parser: ^5.7.0- devalue: '>=5.9.1 <6.0.0'+ devalue: '>=5.9.3 <6.0.0'h3: '>=1.15.9'picomatch@<4.0.4: '>=4.0.4'smol-toml@<1.7.1: 1.8.0@@ -916,8 +916,8 @@ packages:resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==}engines: {node: '>=8'}- resolution: {integrity: sha512-po4PAY5c53tw5XMocSnf8A/5OHhbbUftpr93aEN6BBoAdntUmK7vu7wOATqvt7cXO7m1Cl4gMVn6p7n6n4mj0w==}+ resolution: {integrity: sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==}resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==}@@ -2543,7 +2543,7 @@ snapshots:clsx: 2.1.1common-ancestor-path: 2.0.0cookie: 2.0.1- devalue: 5.9.2+ devalue: 5.9.4diff: 9.0.0dset: 3.1.4es-module-lexer: 2.1.0@@ -2698,7 +2698,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}dependencies:diff --git a/apps/test-site/pnpm-workspace.yaml b/apps/test-site/pnpm-workspace.yamlindex 676a690b7..ba2f3e1e0 100644--- a/apps/test-site/pnpm-workspace.yaml+++ b/apps/test-site/pnpm-workspace.yaml@@ -4,9 +4,11 @@ allowBuilds:overrides:fast-xml-builder: ">=1.1.6 <2.0.0"fast-xml-parser: "^5.7.0"- # GHSA-9rgm-9g3h-6x36: keep the 5.x line only. The- # <6.0.0 ceiling cannot resolve to 6.x.- devalue: ">=5.9.1 <6.0.0"+ # GHSA-9rgm-9g3h-6x36, GHSA-J22F-VQ7H-C4QM, GHSA-HX4R-W6WJ-J8FG,+ # GHSA-MCM9-63F2-9J32, GHSA-WF3X-273G-MVXV, GHSA-X5RW-Q4PP-HG5G,+ # GHSA-4Q55-J62X-FR9H: all fixed in >=5.9.3; keep the 5.x line only.+ # The <6.0.0 ceiling cannot resolve to 6.x.+ devalue: ">=5.9.3 <6.0.0"h3: ">=1.15.9""picomatch@<4.0.4": ">=4.0.4""smol-toml@<1.7.1": "1.8.0"