Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(python-sdk): cancel the crawl when async crawl() is cancelled; cancel_crawl returns False on 409 (#4897)
claude[bot]a73af1d5df0c
diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex 5cdf39ebc..0eac85c95 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -129,6 +129,8 @@ except CrawlJobTimeoutError as e:firecrawl.cancel_crawl(e.job_id)```+With `AsyncFirecrawl`, you can stop waiting for `crawl()` with `Task.cancel()`, `asyncio.timeout` or `asyncio.wait_for`. If the crawl already has a job id, the SDK then sends a best-effort cancel for it. If the cancellation comes before `start_crawl` returns the id, the SDK sends no cancel, and the crawl can keep running. After the cancel, `Task.cancel()` gives you `asyncio.CancelledError`. `asyncio.timeout` and `asyncio.wait_for` give you `TimeoutError`. To keep the crawl running after you stop waiting, use `start_crawl` and `wait_crawl`.+### Asynchronous Crawling<Tip>Looking for async operations? Check out the [Async Class](#async-class) section below.</Tip>@@ -181,7 +183,7 @@ if status.next:### Cancelling a Crawl-To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status.+To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status. It returns `False` if the crawl already completed.```pythoncancel_crawl = firecrawl.cancel_crawl(id)diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex e8afdf168..3e443fccd 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -41,7 +41,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.49.2"+__version__ = "4.49.3"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_cancel.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_cancel.pynew file mode 100644index 000000000..e7896b605--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_cancel.py@@ -0,0 +1,280 @@+"""+Unit tests for cancel_crawl on a finished crawl, and for async crawl()+cancelling its job when the caller's task is cancelled.+"""++import asyncio+import gc+import time+from unittest.mock import AsyncMock, Mock, patch++import pytest++import firecrawl.v2.client_async as client_async_module+from firecrawl.v2.client_async import AsyncFirecrawlClient+from firecrawl.v2.methods import crawl as crawl_module+from firecrawl.v2.methods.aio import crawl as async_crawl_module+from firecrawl.v2.utils.error_handler import FirecrawlError++JOB_ID = "crawl-job-123"+ALREADY_COMPLETED = {"success": False, "error": "Crawl is already completed"}+++def _sync_response(body, status_code=200):+ response = Mock()+ response.ok = status_code < 400+ response.status_code = status_code+ response.json.return_value = body+ return response+++def _async_response(body, status_code=200):+ response = Mock()+ response.status_code = status_code+ response.json.return_value = body+ return response+++def _status_body(status):+ return {+ "success": True,+ "status": status,+ "completed": 1,+ "total": 1,+ "creditsUsed": 1,+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": None,+ "data": [],+ }+++START_BODY = {"success": True, "id": JOB_ID, "url": f"https://api.firecrawl.dev/v2/crawl/{JOB_ID}"}+++class TestSyncCancelCrawl:+ def test_returns_true_when_cancelled(self):+ client = Mock()+ client.delete.return_value = _sync_response({"status": "cancelled"})++ assert crawl_module.cancel_crawl(client, JOB_ID) is True+ client.delete.assert_called_once_with(f"/v2/crawl/{JOB_ID}")++ def test_returns_false_when_crawl_already_completed(self):+ client = Mock()+ client.delete.return_value = _sync_response(ALREADY_COMPLETED, status_code=409)++ assert crawl_module.cancel_crawl(client, JOB_ID) is False++ @pytest.mark.parametrize("status_code", [401, 404, 500])+ def test_other_errors_still_raise(self, status_code):+ client = Mock()+ client.delete.return_value = _sync_response({"success": False, "error": "nope"}, status_code=status_code)++ with pytest.raises(FirecrawlError) as exc_info:+ crawl_module.cancel_crawl(client, JOB_ID)++ assert exc_info.value.status_code == status_code+++class TestAsyncCancelCrawl:+ @pytest.mark.asyncio+ async def test_returns_true_when_cancelled(self):+ client = AsyncMock()+ client.delete.return_value = _async_response({"status": "cancelled"})++ assert await async_crawl_module.cancel_crawl(client, JOB_ID) is True+ client.delete.assert_awaited_once_with(f"/v2/crawl/{JOB_ID}")++ @pytest.mark.asyncio+ async def test_returns_false_when_crawl_already_completed(self):+ client = AsyncMock()+ client.delete.return_value = _async_response(ALREADY_COMPLETED, status_code=409)++ assert await async_crawl_module.cancel_crawl(client, JOB_ID) is False++ @pytest.mark.asyncio+ @pytest.mark.parametrize("status_code", [401, 404, 500])+ async def test_other_errors_still_raise(self, status_code):+ client = AsyncMock()+ client.delete.return_value = _async_response({"success": False, "error": "nope"}, status_code=status_code)++ with pytest.raises(FirecrawlError) as exc_info:+ await async_crawl_module.cancel_crawl(client, JOB_ID)++ assert exc_info.value.status_code == status_code+++def _client_with_blocking_wait():+ """Return a client whose crawl starts, then blocks on the first status poll."""+ client = AsyncFirecrawlClient(api_key="fc-test")+ client.async_http_client = AsyncMock()+ client.async_http_client.post.return_value = _async_response(START_BODY)+ client.async_http_client.delete.return_value = _async_response({"status": "cancelled"})+ polling = asyncio.Event()++ async def blocking_get(*args, **kwargs):+ polling.set()+ await asyncio.Event().wait()++ client.async_http_client.get.side_effect = blocking_get+ return client, polling+++class TestAsyncCrawlCancellation:+ @pytest.mark.asyncio+ async def test_task_cancel_during_wait_cancels_the_crawl(self):+ client, polling = _client_with_blocking_wait()++ task = asyncio.ensure_future(client.crawl(url="https://example.com"))+ await polling.wait()+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await task++ client.async_http_client.delete.assert_awaited_once_with(f"/v2/crawl/{JOB_ID}")++ @pytest.mark.asyncio+ async def test_wait_for_timeout_cancels_the_crawl(self):+ client, _ = _client_with_blocking_wait()++ with pytest.raises(asyncio.TimeoutError):+ await asyncio.wait_for(client.crawl(url="https://example.com"), timeout=0.05)++ client.async_http_client.delete.assert_awaited_once_with(f"/v2/crawl/{JOB_ID}")++ @pytest.mark.asyncio+ async def test_cancel_error_does_not_mask_cancelled_error(self):+ client, polling = _client_with_blocking_wait()+ client.async_http_client.delete.side_effect = RuntimeError("network down")++ task = asyncio.ensure_future(client.crawl(url="https://example.com"))+ await polling.wait()+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await task++ client.async_http_client.delete.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_cancel_api_error_does_not_mask_cancelled_error(self):+ client, polling = _client_with_blocking_wait()+ client.async_http_client.delete.return_value = _async_response(+ {"success": False, "error": "boom"}, status_code=500+ )++ task = asyncio.ensure_future(client.crawl(url="https://example.com"))+ await polling.wait()+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await task++ client.async_http_client.delete.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_slow_cancel_is_bounded_by_a_timeout(self):+ client, polling = _client_with_blocking_wait()++ async def hanging_delete(*args, **kwargs):+ await asyncio.Event().wait()++ client.async_http_client.delete.side_effect = hanging_delete++ with patch.object(client_async_module, "_ABANDONED_CRAWL_CANCEL_TIMEOUT", 0.01):+ task = asyncio.ensure_future(client.crawl(url="https://example.com"))+ await polling.wait()+ started = time.monotonic()+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(task, timeout=5)+ elapsed = time.monotonic() - started++ client.async_http_client.delete.assert_awaited_once()+ assert elapsed < 1.0++ @pytest.mark.asyncio+ async def test_second_cancel_does_not_leave_an_unretrieved_error(self):+ client, polling = _client_with_blocking_wait()+ deleting = asyncio.Event()++ async def failing_delete(*args, **kwargs):+ deleting.set()+ await asyncio.sleep(0.05)+ raise RuntimeError("network down")++ client.async_http_client.delete.side_effect = failing_delete++ loop = asyncio.get_running_loop()+ reported = []+ previous_handler = loop.get_exception_handler()+ loop.set_exception_handler(lambda _loop, context: reported.append(context))+ try:+ task = asyncio.ensure_future(client.crawl(url="https://example.com"))+ await polling.wait()+ task.cancel()+ await deleting.wait()+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await task+ del task++ # Let the background cancel fail, then collect it.+ await asyncio.sleep(0.2)+ gc.collect()+ await asyncio.sleep(0)+ finally:+ loop.set_exception_handler(previous_handler)++ client.async_http_client.delete.assert_awaited_once()+ assert reported == []++ @pytest.mark.asyncio+ async def test_cancel_before_start_returns_sends_nothing(self):+ client = AsyncFirecrawlClient(api_key="fc-test")+ client.async_http_client = AsyncMock()+ starting = asyncio.Event()++ async def blocking_post(*args, **kwargs):+ starting.set()+ await asyncio.Event().wait()++ client.async_http_client.post.side_effect = blocking_post++ task = asyncio.ensure_future(client.crawl(url="https://example.com"))+ await starting.wait()+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await task++ client.async_http_client.delete.assert_not_called()+ client.async_http_client.get.assert_not_called()++ @pytest.mark.asyncio+ async def test_completed_crawl_sends_no_cancel(self):+ client = AsyncFirecrawlClient(api_key="fc-test")+ client.async_http_client = AsyncMock()+ client.async_http_client.post.return_value = _async_response(START_BODY)+ client.async_http_client.get.return_value = _async_response(_status_body("completed"))++ job = await client.crawl(url="https://example.com")++ assert job.id == JOB_ID+ client.async_http_client.delete.assert_not_called()++ @pytest.mark.asyncio+ async def test_cancelled_wait_crawl_sends_no_cancel(self):+ client, polling = _client_with_blocking_wait()++ task = asyncio.ensure_future(client.wait_crawl(JOB_ID))+ await polling.wait()+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await task++ client.async_http_client.delete.assert_not_called()diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 0cd74c680..d21a50116 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -1192,7 +1192,8 @@ class FirecrawlClient:crawl_id: The ID of the crawl job to cancelReturns:- bool: True if the crawl was cancelled, False otherwise+ bool: True if the crawl was cancelled. False if the crawl was not+ cancelled, for example because it already completed."""return crawl_module.cancel_crawl(self.http_client, crawl_id)diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 86ee3db00..7547c2480 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -86,6 +86,17 @@ from .methods.research_docs import (from .client import _SCRAPE_OPTION_KEYSfrom .watcher_async import AsyncWatcher+# Maximum seconds to wait for the cancel request that async crawl() sends when+# its caller is cancelled.+_ABANDONED_CRAWL_CANCEL_TIMEOUT = 10.0+++def _consume_task_outcome(task: "asyncio.Future[Any]") -> None:+ """Mark a finished task's exception as read, so asyncio does not log it."""+ if not task.cancelled():+ task.exception()++class AsyncFirecrawlClient:@staticmethoddef _is_cloud_service(url: str) -> bool:@@ -431,7 +442,27 @@ class AsyncFirecrawlClient:await asyncio.sleep(poll_interval)async def crawl(self, **kwargs) -> CrawlJob:- # wrapper combining start and wait+ """+ Start a crawl job and wait for it to complete.++ Takes the same arguments as ``start_crawl``, plus ``poll_interval``,+ ``timeout`` and ``request_timeout`` (see ``wait_crawl``).++ Returns:+ CrawlJob: The final status of the crawl job.++ Raises:+ CrawlJobTimeoutError: If the crawl does not reach a terminal state within+ ``timeout``. The crawl keeps running, and the error carries ``job_id``.+ asyncio.CancelledError: If the task that awaits this call is cancelled+ (``Task.cancel()``; ``asyncio.timeout`` and ``asyncio.wait_for``+ turn this into ``TimeoutError`` for their caller). If+ ``start_crawl`` already returned the job id, the SDK first sends a+ best-effort ``cancel_crawl`` for the job, so the crawl does not keep+ running and use credits. A cancellation before the id arrives sends+ no cancel. To keep control of the job, use ``start_crawl`` and+ ``wait_crawl`` instead.+ """resp = await self.start_crawl(**{k: v for k, v in kwargs.items() if k not in ("poll_interval", "timeout", "request_timeout")})@@ -439,12 +470,38 @@ class AsyncFirecrawlClient:timeout = kwargs.get("timeout")request_timeout = kwargs.get("request_timeout")effective_request_timeout = request_timeout if request_timeout is not None else timeout- return await self.wait_crawl(- resp.id,- poll_interval=poll_interval,- timeout=timeout,- request_timeout=effective_request_timeout,+ try:+ return await self.wait_crawl(+ resp.id,+ poll_interval=poll_interval,+ timeout=timeout,+ request_timeout=effective_request_timeout,+ )+ except asyncio.CancelledError:+ # The caller can never receive this job's result, so stop the crawl.+ await self._cancel_abandoned_crawl(resp.id)+ raise++ async def _cancel_abandoned_crawl(self, job_id: str) -> None:+ """Send a best-effort cancel for a crawl whose waiter was cancelled.++ The cancel runs in a shielded task, so a second cancellation of the+ caller does not stop the request. Errors are ignored, so they never+ replace the caller's ``CancelledError``.+ """+ cancel_task = asyncio.ensure_future(+ asyncio.wait_for(+ async_crawl.cancel_crawl(self.async_http_client, job_id),+ timeout=_ABANDONED_CRAWL_CANCEL_TIMEOUT,+ ))+ # If the caller is cancelled again, nothing awaits cancel_task. Read its+ # outcome when it finishes, so asyncio does not log an unretrieved error.+ cancel_task.add_done_callback(_consume_task_outcome)+ try:+ await asyncio.shield(cancel_task)+ except (Exception, asyncio.CancelledError):+ passasync def get_crawl_status(self,@@ -499,6 +556,16 @@ class AsyncFirecrawlClient:)async def cancel_crawl(self, job_id: str) -> bool:+ """+ Cancel a crawl job.++ Args:+ job_id: The ID of the crawl job to cancel++ Returns:+ bool: True if the crawl was cancelled. False if the crawl was not+ cancelled, for example because it already completed.+ """return await async_crawl.cancel_crawl(self.async_http_client, job_id)async def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData:diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.pyindex 2863fd601..2ffb5e0e2 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py@@ -295,12 +295,17 @@ async def cancel_crawl(client: AsyncHttpClient, job_id: str) -> bool:job_id: ID of the crawl jobReturns:- True if cancellation was successful+ True if the crawl was cancelled. False if the crawl was not+ cancelled, for example because it already completed (HTTP 409).Raises:- Exception: If the cancellation operation fails+ FirecrawlError: If the API returns any other error status+ httpx.HTTPError: If the request fails before a response arrives"""response = await client.delete(f"/v2/crawl/{job_id}")+ if response.status_code == 409:+ # The crawl already completed, so there is nothing to cancel.+ return Falseif response.status_code >= 400:handle_response_error(response, "cancel crawl")body = response.json()diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex e19e7f74d..6f88299c4 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -372,13 +372,19 @@ def cancel_crawl(client: HttpClient, job_id: str) -> bool:job_id: ID of the crawl job to cancelReturns:- bool: True if the crawl was cancelled, False otherwise+ bool: True if the crawl was cancelled. False if the crawl was not+ cancelled, for example because it already completed (HTTP 409).Raises:- Exception: If the cancellation fails+ FirecrawlError: If the API returns any other error status+ requests.RequestException: If the request fails before a response arrives"""response = client.delete(f"/v2/crawl/{job_id}")+ if response.status_code == 409:+ # The crawl already completed, so there is nothing to cancel.+ return False+if not response.ok:handle_response_error(response, "cancel crawl")