Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(python-sdk): stop status auto-pagination from re-polling a running job (#5220)
Erik Engervall99399112c496
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.pyindex a521d9b04..7b3415465 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py@@ -2,8 +2,10 @@ import osimport timeimport pytestfrom dotenv import load_dotenv+from unittest.mock import patchfrom firecrawl import Firecrawlfrom firecrawl.v2.types import ScrapeOptions, PaginationConfig+from firecrawl.v2.utils.http_client import HttpClientload_dotenv()@@ -49,6 +51,17 @@ class TestBatchScrapeE2E:assert job.status in ["scraping", "completed", "failed"]assert job.total >= 0+ def test_status_of_running_single_url_batch_does_not_repoll(self):+ """One status call on a just-started batch reads the current page instead of polling until it finishes."""+ start_resp = self.client.start_batch_scrape(["https://docs.firecrawl.dev"], formats=["markdown"])++ with patch.object(HttpClient, "get", autospec=True, side_effect=HttpClient.get) as get:+ job = self.client.get_batch_scrape_status(start_resp.id)++ if job.status != "scraping":+ pytest.skip("Batch finished before the status call; nothing running to page over.")+ assert get.call_count <= 2+def test_get_batch_scrape_status_page(self):"""Fetch a single batch scrape page using the next URL."""urls = [f"https://docs.firecrawl.dev?batch={i}" for i in range(15)]diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.pyindex a872db84c..cc7304d23 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py@@ -891,5 +891,114 @@ class TestPaginationEdgeCases:assert self.mock_client.get.call_count == 2+class TestRunningJobPagination:+ """A status call on a running job must not re-poll a cursor that only waits for new documents."""++ job_id = "running-job-123"+ sample_doc = {"url": "https://example.com", "markdown": "# Test", "metadata": {"title": "Test"}}++ def _page(self, status, data, next_url, *, async_client=False):+ response = Mock()+ if async_client:+ response.status_code = 200+ else:+ response.ok = True+ response.json.return_value = {+ "success": True,+ "status": status,+ "completed": len(data),+ "total": 1,+ "creditsUsed": len(data),+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": next_url,+ "data": data,+ }+ return response++ def _running_responses(self, kind, *, async_client=False, first_page_docs=0):+ url = f"https://api.firecrawl.dev/v2/{kind}/{self.job_id}?skip={first_page_docs}"+ docs = [self.sample_doc] * first_page_docs+ responses = [self._page("scraping", docs, url, async_client=async_client)]+ responses += [self._page("scraping", [], url, async_client=async_client) for _ in range(50)]+ responses.append(self._page("completed", [self.sample_doc], None, async_client=async_client))+ return responses++ @pytest.mark.parametrize(+ "kind,get_status",+ [("batch/scrape", get_batch_scrape_status), ("crawl", get_crawl_status)],+ )+ def test_running_job_with_empty_first_page_makes_one_request(self, kind, get_status):+ client = Mock()+ client.get.side_effect = self._running_responses(kind)++ result = get_status(client, self.job_id)++ assert client.get.call_count == 1+ assert result.status == "scraping"+ assert result.data == []++ @pytest.mark.parametrize(+ "kind,get_status",+ [("batch/scrape", get_batch_scrape_status), ("crawl", get_crawl_status)],+ )+ def test_running_job_stops_when_cursor_repeats(self, kind, get_status):+ client = Mock()+ client.get.side_effect = self._running_responses(kind, first_page_docs=1)++ result = get_status(client, self.job_id)++ assert client.get.call_count == 2+ assert result.status == "scraping"+ assert len(result.data) == 1++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "kind,get_status",+ [("batch/scrape", get_batch_scrape_status_async), ("crawl", get_crawl_status_async)],+ )+ async def test_running_job_with_empty_first_page_makes_one_request_async(self, kind, get_status):+ client = AsyncMock()+ client.get.side_effect = self._running_responses(kind, async_client=True)++ result = await get_status(client, self.job_id)++ assert client.get.call_count == 1+ assert result.status == "scraping"+ assert result.data == []++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "kind,get_status",+ [("batch/scrape", get_batch_scrape_status_async), ("crawl", get_crawl_status_async)],+ )+ async def test_running_job_stops_when_cursor_repeats_async(self, kind, get_status):+ client = AsyncMock()+ client.get.side_effect = self._running_responses(kind, async_client=True, first_page_docs=1)++ result = await get_status(client, self.job_id)++ assert client.get.call_count == 2+ assert result.status == "scraping"+ assert len(result.data) == 1++ @pytest.mark.parametrize(+ "kind,get_status",+ [("batch/scrape", get_batch_scrape_status), ("crawl", get_crawl_status)],+ )+ def test_completed_job_follows_empty_page_while_cursor_advances(self, kind, get_status):+ url = f"https://api.firecrawl.dev/v2/{kind}/{self.job_id}"+ client = Mock()+ client.get.side_effect = [+ self._page("completed", [], f"{url}?skip=1"),+ self._page("completed", [], f"{url}?skip=2"),+ self._page("completed", [self.sample_doc], None),+ ]++ result = get_status(client, self.job_id)++ assert client.get.call_count == 3+ assert len(result.data) == 1++if __name__ == "__main__":pytest.main([__file__, "-v"])diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py b/apps/python-sdk/firecrawl/v2/methods/aio/batch.pyindex af002d9f0..02c23f84f 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py@@ -98,7 +98,7 @@ async def get_batch_scrape_status(# Handle pagination if requestedauto_paginate = pagination_config.auto_paginate if pagination_config else True- if auto_paginate and payload["next"]:+ if auto_paginate and payload["next"] and (docs or payload["status"] == "completed"):docs = await _fetch_all_batch_pages_async(client,payload["next"],@@ -217,7 +217,9 @@ async def _fetch_all_batch_pages_async(if (max_results is not None) and (len(documents) >= max_results):break- # Get next URL+ # A running job repeats its cursor on an empty page until new documents finish.+ if not page_payload["data"] and page_payload["next"] == current_url:+ breakcurrent_url = page_payload["next"]page_count += 1diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.pyindex 2ffb5e0e2..bd742cfea 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py@@ -152,7 +152,11 @@ async def get_crawl_status(# Handle pagination if requestedauto_paginate = pagination_config.auto_paginate if pagination_config else True- if auto_paginate and payload["next"]:+ if auto_paginate and payload["next"] and (documents or payload["status"] == "completed") and not (+ pagination_config+ and pagination_config.max_results is not None+ and len(documents) >= pagination_config.max_results+ ):documents = await _fetch_all_pages_async(client,payload["next"],@@ -279,7 +283,9 @@ async def _fetch_all_pages_async(if (max_results is not None) and (len(documents) >= max_results):break- # Get next URL+ # A running job repeats its cursor on an empty page until new documents finish.+ if not page_payload["data"] and page_payload["next"] == current_url:+ breakcurrent_url = page_payload["next"]page_count += 1diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.pyindex c8dd46b3d..34ea20bb2 100644--- a/apps/python-sdk/firecrawl/v2/methods/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py@@ -136,7 +136,7 @@ def get_batch_scrape_status(# Handle pagination if requestedauto_paginate = pagination_config.auto_paginate if pagination_config else True- if auto_paginate and payload["next"]:+ if auto_paginate and payload["next"] and (documents or payload["status"] == "completed"):documents = _fetch_all_batch_pages(client,payload["next"],@@ -258,7 +258,9 @@ def _fetch_all_batch_pages(if max_results is not None and len(documents) >= max_results:break- # Get next URL+ # A running job repeats its cursor on an empty page until new documents finish.+ if not page_payload["data"] and page_payload["next"] == current_url:+ breakcurrent_url = page_payload["next"]page_count += 1diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex 6f88299c4..9094ea215 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -220,7 +220,7 @@ def get_crawl_status(# Handle pagination if requestedauto_paginate = pagination_config.auto_paginate if pagination_config else True- if auto_paginate and payload["next"] and not (+ if auto_paginate and payload["next"] and (documents or payload["status"] == "completed") and not (pagination_configand pagination_config.max_results is not Noneand len(documents) >= pagination_config.max_results@@ -356,7 +356,9 @@ def _fetch_all_pages(if max_results is not None and len(documents) >= max_results:break- # Get next URL+ # A running job repeats its cursor on an empty page until new documents finish.+ if not page_payload["data"] and page_payload["next"] == current_url:+ breakcurrent_url = page_payload["next"]page_count += 1