Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(python-sdk): cancel the crawl when async crawl() is cancelled; cancel_crawl returns False on 409 (#4897)

claude[bot]a73af1d5df0c
diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.md
index 5cdf39ebc..0eac85c95 100644
--- a/apps/python-sdk/README.md
+++ b/apps/python-sdk/README.md
@@ -129,6 +129,8 @@ except CrawlJobTimeoutError as e:
firecrawl.cancel_crawl(e.job_id)
```
+With `AsyncFirecrawl`, you can stop waiting for `crawl()` with `Task.cancel()`, `asyncio.timeout` or `asyncio.wait_for`. If the crawl already has a job id, the SDK then sends a best-effort cancel for it. If the cancellation comes before `start_crawl` returns the id, the SDK sends no cancel, and the crawl can keep running. After the cancel, `Task.cancel()` gives you `asyncio.CancelledError`. `asyncio.timeout` and `asyncio.wait_for` give you `TimeoutError`. To keep the crawl running after you stop waiting, use `start_crawl` and `wait_crawl`.
+
### Asynchronous Crawling
<Tip>Looking for async operations? Check out the [Async Class](#async-class) section below.</Tip>
@@ -181,7 +183,7 @@ if status.next:
### Cancelling a Crawl
-To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status.
+To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status. It returns `False` if the crawl already completed.
```python
cancel_crawl = firecrawl.cancel_crawl(id)
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index e8afdf168..3e443fccd 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -41,7 +41,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.49.2"
+__version__ = "4.49.3"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_cancel.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_cancel.py
new file mode 100644
index 000000000..e7896b605
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_cancel.py
@@ -0,0 +1,280 @@
+"""
+Unit tests for cancel_crawl on a finished crawl, and for async crawl()
+cancelling its job when the caller's task is cancelled.
+"""
+
+import asyncio
+import gc
+import time
+from unittest.mock import AsyncMock, Mock, patch
+
+import pytest
+
+import firecrawl.v2.client_async as client_async_module
+from firecrawl.v2.client_async import AsyncFirecrawlClient
+from firecrawl.v2.methods import crawl as crawl_module
+from firecrawl.v2.methods.aio import crawl as async_crawl_module
+from firecrawl.v2.utils.error_handler import FirecrawlError
+
+JOB_ID = "crawl-job-123"
+ALREADY_COMPLETED = {"success": False, "error": "Crawl is already completed"}
+
+
+def _sync_response(body, status_code=200):
+ response = Mock()
+ response.ok = status_code < 400
+ response.status_code = status_code
+ response.json.return_value = body
+ return response
+
+
+def _async_response(body, status_code=200):
+ response = Mock()
+ response.status_code = status_code
+ response.json.return_value = body
+ return response
+
+
+def _status_body(status):
+ return {
+ "success": True,
+ "status": status,
+ "completed": 1,
+ "total": 1,
+ "creditsUsed": 1,
+ "expiresAt": "2024-01-01T00:00:00Z",
+ "next": None,
+ "data": [],
+ }
+
+
+START_BODY = {"success": True, "id": JOB_ID, "url": f"https://api.firecrawl.dev/v2/crawl/{JOB_ID}"}
+
+
+class TestSyncCancelCrawl:
+ def test_returns_true_when_cancelled(self):
+ client = Mock()
+ client.delete.return_value = _sync_response({"status": "cancelled"})
+
+ assert crawl_module.cancel_crawl(client, JOB_ID) is True
+ client.delete.assert_called_once_with(f"/v2/crawl/{JOB_ID}")
+
+ def test_returns_false_when_crawl_already_completed(self):
+ client = Mock()
+ client.delete.return_value = _sync_response(ALREADY_COMPLETED, status_code=409)
+
+ assert crawl_module.cancel_crawl(client, JOB_ID) is False
+
+ @pytest.mark.parametrize("status_code", [401, 404, 500])
+ def test_other_errors_still_raise(self, status_code):
+ client = Mock()
+ client.delete.return_value = _sync_response({"success": False, "error": "nope"}, status_code=status_code)
+
+ with pytest.raises(FirecrawlError) as exc_info:
+ crawl_module.cancel_crawl(client, JOB_ID)
+
+ assert exc_info.value.status_code == status_code
+
+
+class TestAsyncCancelCrawl:
+ @pytest.mark.asyncio
+ async def test_returns_true_when_cancelled(self):
+ client = AsyncMock()
+ client.delete.return_value = _async_response({"status": "cancelled"})
+
+ assert await async_crawl_module.cancel_crawl(client, JOB_ID) is True
+ client.delete.assert_awaited_once_with(f"/v2/crawl/{JOB_ID}")
+
+ @pytest.mark.asyncio
+ async def test_returns_false_when_crawl_already_completed(self):
+ client = AsyncMock()
+ client.delete.return_value = _async_response(ALREADY_COMPLETED, status_code=409)
+
+ assert await async_crawl_module.cancel_crawl(client, JOB_ID) is False
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("status_code", [401, 404, 500])
+ async def test_other_errors_still_raise(self, status_code):
+ client = AsyncMock()
+ client.delete.return_value = _async_response({"success": False, "error": "nope"}, status_code=status_code)
+
+ with pytest.raises(FirecrawlError) as exc_info:
+ await async_crawl_module.cancel_crawl(client, JOB_ID)
+
+ assert exc_info.value.status_code == status_code
+
+
+def _client_with_blocking_wait():
+ """Return a client whose crawl starts, then blocks on the first status poll."""
+ client = AsyncFirecrawlClient(api_key="fc-test")
+ client.async_http_client = AsyncMock()
+ client.async_http_client.post.return_value = _async_response(START_BODY)
+ client.async_http_client.delete.return_value = _async_response({"status": "cancelled"})
+ polling = asyncio.Event()
+
+ async def blocking_get(*args, **kwargs):
+ polling.set()
+ await asyncio.Event().wait()
+
+ client.async_http_client.get.side_effect = blocking_get
+ return client, polling
+
+
+class TestAsyncCrawlCancellation:
+ @pytest.mark.asyncio
+ async def test_task_cancel_during_wait_cancels_the_crawl(self):
+ client, polling = _client_with_blocking_wait()
+
+ task = asyncio.ensure_future(client.crawl(url="https://example.com"))
+ await polling.wait()
+ task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ client.async_http_client.delete.assert_awaited_once_with(f"/v2/crawl/{JOB_ID}")
+
+ @pytest.mark.asyncio
+ async def test_wait_for_timeout_cancels_the_crawl(self):
+ client, _ = _client_with_blocking_wait()
+
+ with pytest.raises(asyncio.TimeoutError):
+ await asyncio.wait_for(client.crawl(url="https://example.com"), timeout=0.05)
+
+ client.async_http_client.delete.assert_awaited_once_with(f"/v2/crawl/{JOB_ID}")
+
+ @pytest.mark.asyncio
+ async def test_cancel_error_does_not_mask_cancelled_error(self):
+ client, polling = _client_with_blocking_wait()
+ client.async_http_client.delete.side_effect = RuntimeError("network down")
+
+ task = asyncio.ensure_future(client.crawl(url="https://example.com"))
+ await polling.wait()
+ task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ client.async_http_client.delete.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_cancel_api_error_does_not_mask_cancelled_error(self):
+ client, polling = _client_with_blocking_wait()
+ client.async_http_client.delete.return_value = _async_response(
+ {"success": False, "error": "boom"}, status_code=500
+ )
+
+ task = asyncio.ensure_future(client.crawl(url="https://example.com"))
+ await polling.wait()
+ task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ client.async_http_client.delete.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_slow_cancel_is_bounded_by_a_timeout(self):
+ client, polling = _client_with_blocking_wait()
+
+ async def hanging_delete(*args, **kwargs):
+ await asyncio.Event().wait()
+
+ client.async_http_client.delete.side_effect = hanging_delete
+
+ with patch.object(client_async_module, "_ABANDONED_CRAWL_CANCEL_TIMEOUT", 0.01):
+ task = asyncio.ensure_future(client.crawl(url="https://example.com"))
+ await polling.wait()
+ started = time.monotonic()
+ task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=5)
+ elapsed = time.monotonic() - started
+
+ client.async_http_client.delete.assert_awaited_once()
+ assert elapsed < 1.0
+
+ @pytest.mark.asyncio
+ async def test_second_cancel_does_not_leave_an_unretrieved_error(self):
+ client, polling = _client_with_blocking_wait()
+ deleting = asyncio.Event()
+
+ async def failing_delete(*args, **kwargs):
+ deleting.set()
+ await asyncio.sleep(0.05)
+ raise RuntimeError("network down")
+
+ client.async_http_client.delete.side_effect = failing_delete
+
+ loop = asyncio.get_running_loop()
+ reported = []
+ previous_handler = loop.get_exception_handler()
+ loop.set_exception_handler(lambda _loop, context: reported.append(context))
+ try:
+ task = asyncio.ensure_future(client.crawl(url="https://example.com"))
+ await polling.wait()
+ task.cancel()
+ await deleting.wait()
+ task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ del task
+
+ # Let the background cancel fail, then collect it.
+ await asyncio.sleep(0.2)
+ gc.collect()
+ await asyncio.sleep(0)
+ finally:
+ loop.set_exception_handler(previous_handler)
+
+ client.async_http_client.delete.assert_awaited_once()
+ assert reported == []
+
+ @pytest.mark.asyncio
+ async def test_cancel_before_start_returns_sends_nothing(self):
+ client = AsyncFirecrawlClient(api_key="fc-test")
+ client.async_http_client = AsyncMock()
+ starting = asyncio.Event()
+
+ async def blocking_post(*args, **kwargs):
+ starting.set()
+ await asyncio.Event().wait()
+
+ client.async_http_client.post.side_effect = blocking_post
+
+ task = asyncio.ensure_future(client.crawl(url="https://example.com"))
+ await starting.wait()
+ task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ client.async_http_client.delete.assert_not_called()
+ client.async_http_client.get.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_completed_crawl_sends_no_cancel(self):
+ client = AsyncFirecrawlClient(api_key="fc-test")
+ client.async_http_client = AsyncMock()
+ client.async_http_client.post.return_value = _async_response(START_BODY)
+ client.async_http_client.get.return_value = _async_response(_status_body("completed"))
+
+ job = await client.crawl(url="https://example.com")
+
+ assert job.id == JOB_ID
+ client.async_http_client.delete.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_cancelled_wait_crawl_sends_no_cancel(self):
+ client, polling = _client_with_blocking_wait()
+
+ task = asyncio.ensure_future(client.wait_crawl(JOB_ID))
+ await polling.wait()
+ task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ client.async_http_client.delete.assert_not_called()
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 0cd74c680..d21a50116 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -1192,7 +1192,8 @@ class FirecrawlClient:
crawl_id: The ID of the crawl job to cancel
Returns:
- bool: True if the crawl was cancelled, False otherwise
+ bool: True if the crawl was cancelled. False if the crawl was not
+ cancelled, for example because it already completed.
"""
return crawl_module.cancel_crawl(self.http_client, crawl_id)
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index 86ee3db00..7547c2480 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -86,6 +86,17 @@ from .methods.research_docs import (
from .client import _SCRAPE_OPTION_KEYS
from .watcher_async import AsyncWatcher
+# Maximum seconds to wait for the cancel request that async crawl() sends when
+# its caller is cancelled.
+_ABANDONED_CRAWL_CANCEL_TIMEOUT = 10.0
+
+
+def _consume_task_outcome(task: "asyncio.Future[Any]") -> None:
+ """Mark a finished task's exception as read, so asyncio does not log it."""
+ if not task.cancelled():
+ task.exception()
+
+
class AsyncFirecrawlClient:
@staticmethod
def _is_cloud_service(url: str) -> bool:
@@ -431,7 +442,27 @@ class AsyncFirecrawlClient:
await asyncio.sleep(poll_interval)
async def crawl(self, **kwargs) -> CrawlJob:
- # wrapper combining start and wait
+ """
+ Start a crawl job and wait for it to complete.
+
+ Takes the same arguments as ``start_crawl``, plus ``poll_interval``,
+ ``timeout`` and ``request_timeout`` (see ``wait_crawl``).
+
+ Returns:
+ CrawlJob: The final status of the crawl job.
+
+ Raises:
+ CrawlJobTimeoutError: If the crawl does not reach a terminal state within
+ ``timeout``. The crawl keeps running, and the error carries ``job_id``.
+ asyncio.CancelledError: If the task that awaits this call is cancelled
+ (``Task.cancel()``; ``asyncio.timeout`` and ``asyncio.wait_for``
+ turn this into ``TimeoutError`` for their caller). If
+ ``start_crawl`` already returned the job id, the SDK first sends a
+ best-effort ``cancel_crawl`` for the job, so the crawl does not keep
+ running and use credits. A cancellation before the id arrives sends
+ no cancel. To keep control of the job, use ``start_crawl`` and
+ ``wait_crawl`` instead.
+ """
resp = await self.start_crawl(
**{k: v for k, v in kwargs.items() if k not in ("poll_interval", "timeout", "request_timeout")}
)
@@ -439,12 +470,38 @@ class AsyncFirecrawlClient:
timeout = kwargs.get("timeout")
request_timeout = kwargs.get("request_timeout")
effective_request_timeout = request_timeout if request_timeout is not None else timeout
- return await self.wait_crawl(
- resp.id,
- poll_interval=poll_interval,
- timeout=timeout,
- request_timeout=effective_request_timeout,
+ try:
+ return await self.wait_crawl(
+ resp.id,
+ poll_interval=poll_interval,
+ timeout=timeout,
+ request_timeout=effective_request_timeout,
+ )
+ except asyncio.CancelledError:
+ # The caller can never receive this job's result, so stop the crawl.
+ await self._cancel_abandoned_crawl(resp.id)
+ raise
+
+ async def _cancel_abandoned_crawl(self, job_id: str) -> None:
+ """Send a best-effort cancel for a crawl whose waiter was cancelled.
+
+ The cancel runs in a shielded task, so a second cancellation of the
+ caller does not stop the request. Errors are ignored, so they never
+ replace the caller's ``CancelledError``.
+ """
+ cancel_task = asyncio.ensure_future(
+ asyncio.wait_for(
+ async_crawl.cancel_crawl(self.async_http_client, job_id),
+ timeout=_ABANDONED_CRAWL_CANCEL_TIMEOUT,
+ )
)
+ # If the caller is cancelled again, nothing awaits cancel_task. Read its
+ # outcome when it finishes, so asyncio does not log an unretrieved error.
+ cancel_task.add_done_callback(_consume_task_outcome)
+ try:
+ await asyncio.shield(cancel_task)
+ except (Exception, asyncio.CancelledError):
+ pass
async def get_crawl_status(
self,
@@ -499,6 +556,16 @@ class AsyncFirecrawlClient:
)
async def cancel_crawl(self, job_id: str) -> bool:
+ """
+ Cancel a crawl job.
+
+ Args:
+ job_id: The ID of the crawl job to cancel
+
+ Returns:
+ bool: True if the crawl was cancelled. False if the crawl was not
+ cancelled, for example because it already completed.
+ """
return await async_crawl.cancel_crawl(self.async_http_client, job_id)
async def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData:
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
index 2863fd601..2ffb5e0e2 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
@@ -295,12 +295,17 @@ async def cancel_crawl(client: AsyncHttpClient, job_id: str) -> bool:
job_id: ID of the crawl job
Returns:
- True if cancellation was successful
+ True if the crawl was cancelled. False if the crawl was not
+ cancelled, for example because it already completed (HTTP 409).
Raises:
- Exception: If the cancellation operation fails
+ FirecrawlError: If the API returns any other error status
+ httpx.HTTPError: If the request fails before a response arrives
"""
response = await client.delete(f"/v2/crawl/{job_id}")
+ if response.status_code == 409:
+ # The crawl already completed, so there is nothing to cancel.
+ return False
if response.status_code >= 400:
handle_response_error(response, "cancel crawl")
body = response.json()
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
index e19e7f74d..6f88299c4 100644
--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -372,13 +372,19 @@ def cancel_crawl(client: HttpClient, job_id: str) -> bool:
job_id: ID of the crawl job to cancel
Returns:
- bool: True if the crawl was cancelled, False otherwise
+ bool: True if the crawl was cancelled. False if the crawl was not
+ cancelled, for example because it already completed (HTTP 409).
Raises:
- Exception: If the cancellation fails
+ FirecrawlError: If the API returns any other error status
+ requests.RequestException: If the request fails before a response arrives
"""
response = client.delete(f"/v2/crawl/{job_id}")
+ if response.status_code == 409:
+ # The crawl already completed, so there is nothing to cancel.
+ return False
+
if not response.ok:
handle_response_error(response, "cancel crawl")