Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(python-sdk): stop status auto-pagination from re-polling a running job (#5220)

Erik Engervall99399112c496
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py
index a521d9b04..7b3415465 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py
@@ -2,8 +2,10 @@ import os
import time
import pytest
from dotenv import load_dotenv
+from unittest.mock import patch
from firecrawl import Firecrawl
from firecrawl.v2.types import ScrapeOptions, PaginationConfig
+from firecrawl.v2.utils.http_client import HttpClient
load_dotenv()
@@ -49,6 +51,17 @@ class TestBatchScrapeE2E:
assert job.status in ["scraping", "completed", "failed"]
assert job.total >= 0
+ def test_status_of_running_single_url_batch_does_not_repoll(self):
+ """One status call on a just-started batch reads the current page instead of polling until it finishes."""
+ start_resp = self.client.start_batch_scrape(["https://docs.firecrawl.dev"], formats=["markdown"])
+
+ with patch.object(HttpClient, "get", autospec=True, side_effect=HttpClient.get) as get:
+ job = self.client.get_batch_scrape_status(start_resp.id)
+
+ if job.status != "scraping":
+ pytest.skip("Batch finished before the status call; nothing running to page over.")
+ assert get.call_count <= 2
+
def test_get_batch_scrape_status_page(self):
"""Fetch a single batch scrape page using the next URL."""
urls = [f"https://docs.firecrawl.dev?batch={i}" for i in range(15)]
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py
index a872db84c..cc7304d23 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py
@@ -891,5 +891,114 @@ class TestPaginationEdgeCases:
assert self.mock_client.get.call_count == 2
+class TestRunningJobPagination:
+ """A status call on a running job must not re-poll a cursor that only waits for new documents."""
+
+ job_id = "running-job-123"
+ sample_doc = {"url": "https://example.com", "markdown": "# Test", "metadata": {"title": "Test"}}
+
+ def _page(self, status, data, next_url, *, async_client=False):
+ response = Mock()
+ if async_client:
+ response.status_code = 200
+ else:
+ response.ok = True
+ response.json.return_value = {
+ "success": True,
+ "status": status,
+ "completed": len(data),
+ "total": 1,
+ "creditsUsed": len(data),
+ "expiresAt": "2024-01-01T00:00:00Z",
+ "next": next_url,
+ "data": data,
+ }
+ return response
+
+ def _running_responses(self, kind, *, async_client=False, first_page_docs=0):
+ url = f"https://api.firecrawl.dev/v2/{kind}/{self.job_id}?skip={first_page_docs}"
+ docs = [self.sample_doc] * first_page_docs
+ responses = [self._page("scraping", docs, url, async_client=async_client)]
+ responses += [self._page("scraping", [], url, async_client=async_client) for _ in range(50)]
+ responses.append(self._page("completed", [self.sample_doc], None, async_client=async_client))
+ return responses
+
+ @pytest.mark.parametrize(
+ "kind,get_status",
+ [("batch/scrape", get_batch_scrape_status), ("crawl", get_crawl_status)],
+ )
+ def test_running_job_with_empty_first_page_makes_one_request(self, kind, get_status):
+ client = Mock()
+ client.get.side_effect = self._running_responses(kind)
+
+ result = get_status(client, self.job_id)
+
+ assert client.get.call_count == 1
+ assert result.status == "scraping"
+ assert result.data == []
+
+ @pytest.mark.parametrize(
+ "kind,get_status",
+ [("batch/scrape", get_batch_scrape_status), ("crawl", get_crawl_status)],
+ )
+ def test_running_job_stops_when_cursor_repeats(self, kind, get_status):
+ client = Mock()
+ client.get.side_effect = self._running_responses(kind, first_page_docs=1)
+
+ result = get_status(client, self.job_id)
+
+ assert client.get.call_count == 2
+ assert result.status == "scraping"
+ assert len(result.data) == 1
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "kind,get_status",
+ [("batch/scrape", get_batch_scrape_status_async), ("crawl", get_crawl_status_async)],
+ )
+ async def test_running_job_with_empty_first_page_makes_one_request_async(self, kind, get_status):
+ client = AsyncMock()
+ client.get.side_effect = self._running_responses(kind, async_client=True)
+
+ result = await get_status(client, self.job_id)
+
+ assert client.get.call_count == 1
+ assert result.status == "scraping"
+ assert result.data == []
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "kind,get_status",
+ [("batch/scrape", get_batch_scrape_status_async), ("crawl", get_crawl_status_async)],
+ )
+ async def test_running_job_stops_when_cursor_repeats_async(self, kind, get_status):
+ client = AsyncMock()
+ client.get.side_effect = self._running_responses(kind, async_client=True, first_page_docs=1)
+
+ result = await get_status(client, self.job_id)
+
+ assert client.get.call_count == 2
+ assert result.status == "scraping"
+ assert len(result.data) == 1
+
+ @pytest.mark.parametrize(
+ "kind,get_status",
+ [("batch/scrape", get_batch_scrape_status), ("crawl", get_crawl_status)],
+ )
+ def test_completed_job_follows_empty_page_while_cursor_advances(self, kind, get_status):
+ url = f"https://api.firecrawl.dev/v2/{kind}/{self.job_id}"
+ client = Mock()
+ client.get.side_effect = [
+ self._page("completed", [], f"{url}?skip=1"),
+ self._page("completed", [], f"{url}?skip=2"),
+ self._page("completed", [self.sample_doc], None),
+ ]
+
+ result = get_status(client, self.job_id)
+
+ assert client.get.call_count == 3
+ assert len(result.data) == 1
+
+
if __name__ == "__main__":
pytest.main([__file__, "-v"])
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
index af002d9f0..02c23f84f 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
@@ -98,7 +98,7 @@ async def get_batch_scrape_status(
# Handle pagination if requested
auto_paginate = pagination_config.auto_paginate if pagination_config else True
- if auto_paginate and payload["next"]:
+ if auto_paginate and payload["next"] and (docs or payload["status"] == "completed"):
docs = await _fetch_all_batch_pages_async(
client,
payload["next"],
@@ -217,7 +217,9 @@ async def _fetch_all_batch_pages_async(
if (max_results is not None) and (len(documents) >= max_results):
break
- # Get next URL
+ # A running job repeats its cursor on an empty page until new documents finish.
+ if not page_payload["data"] and page_payload["next"] == current_url:
+ break
current_url = page_payload["next"]
page_count += 1
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
index 2ffb5e0e2..bd742cfea 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
@@ -152,7 +152,11 @@ async def get_crawl_status(
# Handle pagination if requested
auto_paginate = pagination_config.auto_paginate if pagination_config else True
- if auto_paginate and payload["next"]:
+ if auto_paginate and payload["next"] and (documents or payload["status"] == "completed") and not (
+ pagination_config
+ and pagination_config.max_results is not None
+ and len(documents) >= pagination_config.max_results
+ ):
documents = await _fetch_all_pages_async(
client,
payload["next"],
@@ -279,7 +283,9 @@ async def _fetch_all_pages_async(
if (max_results is not None) and (len(documents) >= max_results):
break
- # Get next URL
+ # A running job repeats its cursor on an empty page until new documents finish.
+ if not page_payload["data"] and page_payload["next"] == current_url:
+ break
current_url = page_payload["next"]
page_count += 1
diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.py
index c8dd46b3d..34ea20bb2 100644
--- a/apps/python-sdk/firecrawl/v2/methods/batch.py
+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py
@@ -136,7 +136,7 @@ def get_batch_scrape_status(
# Handle pagination if requested
auto_paginate = pagination_config.auto_paginate if pagination_config else True
- if auto_paginate and payload["next"]:
+ if auto_paginate and payload["next"] and (documents or payload["status"] == "completed"):
documents = _fetch_all_batch_pages(
client,
payload["next"],
@@ -258,7 +258,9 @@ def _fetch_all_batch_pages(
if max_results is not None and len(documents) >= max_results:
break
- # Get next URL
+ # A running job repeats its cursor on an empty page until new documents finish.
+ if not page_payload["data"] and page_payload["next"] == current_url:
+ break
current_url = page_payload["next"]
page_count += 1
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
index 6f88299c4..9094ea215 100644
--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -220,7 +220,7 @@ def get_crawl_status(
# Handle pagination if requested
auto_paginate = pagination_config.auto_paginate if pagination_config else True
- if auto_paginate and payload["next"] and not (
+ if auto_paginate and payload["next"] and (documents or payload["status"] == "completed") and not (
pagination_config
and pagination_config.max_results is not None
and len(documents) >= pagination_config.max_results
@@ -356,7 +356,9 @@ def _fetch_all_pages(
if max_results is not None and len(documents) >= max_results:
break
- # Get next URL
+ # A running job repeats its cursor on an empty page until new documents finish.
+ if not page_payload["data"] and page_payload["next"] == current_url:
+ break
current_url = page_payload["next"]
page_count += 1