Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add manual pagination helpers for Python SDK (#2727)
firecrawl-spring[bot]5696a701d21f
diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex 253553b68..50aa82ca2 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -87,6 +87,32 @@ crawl_status = firecrawl.get_crawl_status("<crawl_id>")print(crawl_status)```+### Manual Pagination (v2)++Crawl and batch scrape status responses may include a `next` URL when more data is available. The SDK auto-paginates by default; to page manually, disable auto-pagination and pass the opaque `next` URL back to the SDK.++```python+from firecrawl.v2.types import PaginationConfig++# Crawl: fetch one page at a time+crawl_job = firecrawl.start_crawl("https://firecrawl.dev", limit=100)+status = firecrawl.get_crawl_status(+ crawl_job.id,+ pagination_config=PaginationConfig(auto_paginate=False),+)+if status.next:+ page2 = firecrawl.get_crawl_status_page(status.next)++# Batch scrape: fetch one page at a time+batch_job = firecrawl.start_batch_scrape(["https://firecrawl.dev"])+status = firecrawl.get_batch_scrape_status(+ batch_job.id,+ pagination_config=PaginationConfig(auto_paginate=False),+)+if status.next:+ page2 = firecrawl.get_batch_scrape_status_page(status.next)+```+### Cancelling a CrawlTo cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status.@@ -184,4 +210,4 @@ firecrawl = Firecrawl(api_key="YOUR_API_KEY")doc_v1 = firecrawl.v1.scrape_url('https://firecrawl.dev', formats=['markdown', 'html'])crawl_v1 = firecrawl.v1.crawl_url('https://firecrawl.dev', limit=100)map_v1 = firecrawl.v1.map_url('https://firecrawl.dev')-```\ No newline at end of file+```diff --git a/apps/python-sdk/example_pagination.py b/apps/python-sdk/example_pagination.pyindex 106d578ca..589ff9392 100644--- a/apps/python-sdk/example_pagination.py+++ b/apps/python-sdk/example_pagination.py@@ -18,7 +18,8 @@ print(f"Total documents fetched: {len(crawl_result.data)}")print(f"Next URL: {crawl_result.next}") # Should be None since auto-pagination is enabled# Example 2: Manual crawl with pagination control-# Use this when you need to control how many pages to fetch or want to process results incrementally+# Use this when you need to control how many pages to fetch or want to process results incrementally.+# The next URL is opaque; pass it back to the SDK to fetch the next page.print("\n=== Example 2: Manual crawl with pagination control ===")crawl_job = firecrawl.start_crawl("https://example.com", limit=100)@@ -27,6 +28,9 @@ pagination_config = PaginationConfig(auto_paginate=False)status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)print(f"Documents from first page: {len(status.data)}")print(f"Next URL: {status.next}") # Will show the next page URL+if status.next:+ next_page = firecrawl.get_crawl_status_page(status.next)+ print(f"Documents from next page: {len(next_page.data)}")# Example 3: Limited pagination - fetch only 3 pages# Useful for controlling memory usage or processing time@@ -68,11 +72,16 @@ batch_result = firecrawl.batch_scrape(urls)print(f"Batch scrape documents: {len(batch_result.data)}")# Example 8: Manual batch scrape with pagination control-# Use this when you need to control how many pages to fetch or want to process results incrementally+# Use this when you need to control how many pages to fetch or want to process results incrementally.+# The next URL is opaque; pass it back to the SDK to fetch the next page.print("\n=== Example 8: Manual batch scrape with pagination control ===")batch_job = firecrawl.start_batch_scrape(urls)-status = firecrawl.get_batch_scrape_status(batch_job.id)+status = firecrawl.get_batch_scrape_status(batch_job.id, pagination_config=PaginationConfig(auto_paginate=False))print(f"Batch scrape documents: {len(status.data)}")+print(f"Next URL: {status.next}")+if status.next:+ next_page = firecrawl.get_batch_scrape_status_page(status.next)+ print(f"Batch scrape next page documents: {len(next_page.data)}")# Example 9: Async usageprint("\n=== Example 9: Async pagination ===")@@ -94,6 +103,9 @@ async def async_example():pagination_config=pagination_config)print(f"Async crawl with pagination: {len(status.data)}")+ if status.next:+ next_page = await async_client.get_crawl_status_page(status.next)+ print(f"Async crawl next page documents: {len(next_page.data)}")# Run async example# asyncio.run(async_example())diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex fcda4c526..76deaa9be 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.13.2"+__version__ = "4.13.3"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.pyindex f5fb8ecbe..a521d9b04 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py@@ -1,8 +1,9 @@import os+import timeimport pytestfrom dotenv import load_dotenvfrom firecrawl import Firecrawl-from firecrawl.v2.types import ScrapeOptions+from firecrawl.v2.types import ScrapeOptions, PaginationConfigload_dotenv()@@ -48,6 +49,39 @@ class TestBatchScrapeE2E:assert job.status in ["scraping", "completed", "failed"]assert job.total >= 0+ def test_get_batch_scrape_status_page(self):+ """Fetch a single batch scrape page using the next URL."""+ urls = [f"https://docs.firecrawl.dev?batch={i}" for i in range(15)]++ start_resp = self.client.start_batch_scrape(+ urls,+ formats=["markdown"],+ ignore_invalid_urls=True,+ )+ assert start_resp.id is not None++ pagination_config = PaginationConfig(auto_paginate=False)+ deadline = time.time() + 120+ status_job = None+ while time.time() < deadline:+ status_job = self.client.get_batch_scrape_status(+ start_resp.id,+ pagination_config=pagination_config,+ )+ if status_job.next:+ break+ if status_job.status in ["completed", "failed", "cancelled"]:+ break+ time.sleep(2)++ assert status_job is not None+ if not status_job.next:+ pytest.skip("Batch scrape completed without pagination; skipping page fetch.")++ next_page = self.client.get_batch_scrape_status_page(status_job.next)+ assert isinstance(next_page.data, list)+ assert next_page.status in ["scraping", "completed", "failed", "cancelled"]+def test_wait_batch_with_all_params(self):"""Blocking waiter with JSON and changeTracking formats plus many options."""urls = [@@ -103,4 +137,3 @@ class TestBatchScrapeE2E:cancelled = self.client.cancel_batch_scrape(start_resp.id)assert cancelled is True-diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.pyindex 2544445af..6fa88780c 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py@@ -3,7 +3,7 @@ import timeimport osfrom dotenv import load_dotenvfrom firecrawl import Firecrawl-from firecrawl.v2.types import ScrapeOptions+from firecrawl.v2.types import ScrapeOptions, PaginationConfigload_dotenv()@@ -66,6 +66,33 @@ class TestCrawlE2E:assert status_job.next is Noneassert isinstance(status_job.data, list)+ def test_get_crawl_status_page(self):+ """Fetch a single crawl page using the next URL."""+ start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=25)+ assert start_job.id is not None++ pagination_config = PaginationConfig(auto_paginate=False)+ deadline = time.time() + 120+ status_job = None+ while time.time() < deadline:+ status_job = self.client.get_crawl_status(+ start_job.id,+ pagination_config=pagination_config,+ )+ if status_job.next:+ break+ if status_job.status in ["completed", "failed", "cancelled"]:+ break+ time.sleep(2)++ assert status_job is not None+ if not status_job.next:+ pytest.skip("Crawl completed without pagination; skipping page fetch.")++ next_page = self.client.get_crawl_status_page(status_job.next)+ assert isinstance(next_page.data, list)+ assert next_page.status in ["scraping", "completed", "failed", "cancelled"]+def test_cancel_crawl(self):"""Test canceling a crawl."""start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=3)@@ -275,4 +302,4 @@ class TestCrawlE2E:assert params_data is not Noneassert params_data.limit is not None or params_data.include_paths is not None or params_data.max_discovery_depth is not Noneassert 'blog/.*' in params_data.include_paths- assert 'docs/.*' in params_data.include_paths\ No newline at end of file+ assert 'docs/.*' in params_data.include_pathsdiff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.pyindex d05c562c7..a872db84c 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py@@ -14,10 +14,18 @@ from firecrawl.v2.types import (Document,DocumentMetadata)-from firecrawl.v2.methods.crawl import get_crawl_status, _fetch_all_pages-from firecrawl.v2.methods.batch import get_batch_scrape_status, _fetch_all_batch_pages-from firecrawl.v2.methods.aio.crawl import get_crawl_status as get_crawl_status_async, _fetch_all_pages_async-from firecrawl.v2.methods.aio.batch import get_batch_scrape_status as get_batch_scrape_status_async, _fetch_all_batch_pages_async+from firecrawl.v2.methods.crawl import get_crawl_status, get_crawl_status_page, _fetch_all_pages+from firecrawl.v2.methods.batch import get_batch_scrape_status, get_batch_scrape_status_page, _fetch_all_batch_pages+from firecrawl.v2.methods.aio.crawl import (+ get_crawl_status as get_crawl_status_async,+ get_crawl_status_page as get_crawl_status_page_async,+ _fetch_all_pages_async,+)+from firecrawl.v2.methods.aio.batch import (+ get_batch_scrape_status as get_batch_scrape_status_async,+ get_batch_scrape_status_page as get_batch_scrape_status_page_async,+ _fetch_all_batch_pages_async,+)class TestPaginationConfig:@@ -123,6 +131,59 @@ class TestCrawlPagination:self.mock_client.get.assert_called_with(f"/v2/crawl/{self.job_id}", timeout=timeout_seconds)++ def test_get_crawl_status_page(self):+ """Test get_crawl_status_page returns a single page."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "status": "completed",+ "completed": 10,+ "total": 20,+ "creditsUsed": 5,+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=3",+ "data": [self.sample_doc],+ }++ self.mock_client.get.return_value = mock_response+ next_url = "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2"++ result = get_crawl_status_page(self.mock_client, next_url)++ assert result.status == "completed"+ assert result.next == "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=3"+ assert len(result.data) == 1+ self.mock_client.get.assert_called_with(next_url, timeout=None)++ def test_get_crawl_status_page_propagates_request_timeout(self):+ """Ensure request_timeout is forwarded to crawl status page requests."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "status": "completed",+ "completed": 1,+ "total": 1,+ "creditsUsed": 1,+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": None,+ "data": [self.sample_doc],+ }++ self.mock_client.get.return_value = mock_response++ next_url = "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2"+ timeout_seconds = 4.2+ result = get_crawl_status_page(+ self.mock_client,+ next_url,+ request_timeout=timeout_seconds,+ )++ assert result.status == "completed"+ self.mock_client.get.assert_called_with(next_url, timeout=timeout_seconds)def test_get_crawl_status_with_pagination(self):"""Test get_crawl_status with auto_paginate=True."""@@ -326,6 +387,59 @@ class TestBatchScrapePagination:assert result.next == "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2"assert len(result.data) == 1assert isinstance(result.data[0], Document)++ def test_get_batch_scrape_status_page(self):+ """Test get_batch_scrape_status_page returns a single page."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "status": "completed",+ "completed": 10,+ "total": 20,+ "creditsUsed": 5,+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=3",+ "data": [self.sample_doc],+ }++ self.mock_client.get.return_value = mock_response+ next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2"++ result = get_batch_scrape_status_page(self.mock_client, next_url)++ assert result.status == "completed"+ assert result.next == "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=3"+ assert len(result.data) == 1+ self.mock_client.get.assert_called_with(next_url, timeout=None)++ def test_get_batch_scrape_status_page_propagates_request_timeout(self):+ """Ensure request_timeout is forwarded to batch status page requests."""+ mock_response = Mock()+ mock_response.ok = True+ mock_response.json.return_value = {+ "success": True,+ "status": "completed",+ "completed": 1,+ "total": 1,+ "creditsUsed": 1,+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": None,+ "data": [self.sample_doc],+ }++ self.mock_client.get.return_value = mock_response++ next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2"+ timeout_seconds = 2.7+ result = get_batch_scrape_status_page(+ self.mock_client,+ next_url,+ request_timeout=timeout_seconds,+ )++ assert result.status == "completed"+ self.mock_client.get.assert_called_with(next_url, timeout=timeout_seconds)def test_get_batch_scrape_status_with_pagination(self):"""Test get_batch_scrape_status with auto_paginate=True."""@@ -493,6 +607,61 @@ class TestAsyncPagination:f"/v2/crawl/{self.job_id}", timeout=timeout_seconds)+ @pytest.mark.asyncio+ async def test_get_crawl_status_page_async(self):+ """Test async get_crawl_status_page returns a single page."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "success": True,+ "status": "completed",+ "completed": 10,+ "total": 20,+ "creditsUsed": 5,+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": "https://api.firecrawl.dev/v2/crawl/test-async-123?page=3",+ "data": [self.sample_doc],+ }++ self.mock_client.get.return_value = mock_response+ next_url = "https://api.firecrawl.dev/v2/crawl/test-async-123?page=2"++ result = await get_crawl_status_page_async(self.mock_client, next_url)++ assert result.status == "completed"+ assert result.next == "https://api.firecrawl.dev/v2/crawl/test-async-123?page=3"+ assert len(result.data) == 1+ self.mock_client.get.assert_awaited_with(next_url, timeout=None)++ @pytest.mark.asyncio+ async def test_get_crawl_status_page_async_propagates_request_timeout(self):+ """Ensure async request_timeout is forwarded to crawl status page requests."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "success": True,+ "status": "completed",+ "completed": 1,+ "total": 1,+ "creditsUsed": 1,+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": None,+ "data": [self.sample_doc],+ }++ self.mock_client.get.return_value = mock_response++ next_url = "https://api.firecrawl.dev/v2/crawl/test-async-123?page=2"+ timeout_seconds = 6.1+ result = await get_crawl_status_page_async(+ self.mock_client,+ next_url,+ request_timeout=timeout_seconds,+ )++ assert result.status == "completed"+ self.mock_client.get.assert_awaited_with(next_url, timeout=timeout_seconds)+@pytest.mark.asyncioasync def test_get_batch_scrape_status_async_with_pagination(self):"""Test async get_batch_scrape_status with pagination."""@@ -534,6 +703,61 @@ class TestAsyncPagination:assert result.next is Noneassert len(result.data) == 2assert self.mock_client.get.call_count == 2++ @pytest.mark.asyncio+ async def test_get_batch_scrape_status_page_async(self):+ """Test async get_batch_scrape_status_page returns a single page."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "success": True,+ "status": "completed",+ "completed": 10,+ "total": 20,+ "creditsUsed": 5,+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=3",+ "data": [self.sample_doc],+ }++ self.mock_client.get.return_value = mock_response+ next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=2"++ result = await get_batch_scrape_status_page_async(self.mock_client, next_url)++ assert result.status == "completed"+ assert result.next == "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=3"+ assert len(result.data) == 1+ self.mock_client.get.assert_awaited_with(next_url, timeout=None)++ @pytest.mark.asyncio+ async def test_get_batch_scrape_status_page_async_propagates_request_timeout(self):+ """Ensure async request_timeout is forwarded to batch status page requests."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "success": True,+ "status": "completed",+ "completed": 1,+ "total": 1,+ "creditsUsed": 1,+ "expiresAt": "2024-01-01T00:00:00Z",+ "next": None,+ "data": [self.sample_doc],+ }++ self.mock_client.get.return_value = mock_response++ next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=2"+ timeout_seconds = 4.4+ result = await get_batch_scrape_status_page_async(+ self.mock_client,+ next_url,+ request_timeout=timeout_seconds,+ )++ assert result.status == "completed"+ self.mock_client.get.assert_awaited_with(next_url, timeout=timeout_seconds)@pytest.mark.asyncioasync def test_fetch_all_pages_async_limits(self):diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex db5c1c0a9..c15936fe0 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -61,6 +61,7 @@ class V2Proxy:self.crawl = client_instance.crawlself.start_crawl = client_instance.start_crawlself.get_crawl_status = client_instance.get_crawl_status+ self.get_crawl_status_page = client_instance.get_crawl_status_pageself.cancel_crawl = client_instance.cancel_crawlself.get_crawl_errors = client_instance.get_crawl_errorsself.get_active_crawls = client_instance.get_active_crawls@@ -78,6 +79,7 @@ class V2Proxy:self.start_batch_scrape = client_instance.start_batch_scrapeself.get_batch_scrape_status = client_instance.get_batch_scrape_status+ self.get_batch_scrape_status_page = client_instance.get_batch_scrape_status_pageself.cancel_batch_scrape = client_instance.cancel_batch_scrapeself.batch_scrape = client_instance.batch_scrapeself.get_batch_scrape_errors = client_instance.get_batch_scrape_errors@@ -127,6 +129,7 @@ class AsyncV2Proxy:self.start_crawl = client_instance.start_crawlself.wait_crawl = client_instance.wait_crawlself.get_crawl_status = client_instance.get_crawl_status+ self.get_crawl_status_page = client_instance.get_crawl_status_pageself.cancel_crawl = client_instance.cancel_crawlself.get_crawl_errors = client_instance.get_crawl_errorsself.get_active_crawls = client_instance.get_active_crawls@@ -144,6 +147,7 @@ class AsyncV2Proxy:self.start_batch_scrape = client_instance.start_batch_scrapeself.get_batch_scrape_status = client_instance.get_batch_scrape_status+ self.get_batch_scrape_status_page = client_instance.get_batch_scrape_status_pageself.cancel_batch_scrape = client_instance.cancel_batch_scrapeself.wait_batch_scrape = client_instance.wait_batch_scrapeself.batch_scrape = client_instance.batch_scrape@@ -198,6 +202,7 @@ class Firecrawl:self.start_crawl = self._v2_client.start_crawlself.crawl_params_preview = self._v2_client.crawl_params_previewself.get_crawl_status = self._v2_client.get_crawl_status+ self.get_crawl_status_page = self._v2_client.get_crawl_status_pageself.cancel_crawl = self._v2_client.cancel_crawlself.get_crawl_errors = self._v2_client.get_crawl_errorsself.get_active_crawls = self._v2_client.get_active_crawls@@ -205,6 +210,7 @@ class Firecrawl:self.start_batch_scrape = self._v2_client.start_batch_scrapeself.get_batch_scrape_status = self._v2_client.get_batch_scrape_status+ self.get_batch_scrape_status_page = self._v2_client.get_batch_scrape_status_pageself.cancel_batch_scrape = self._v2_client.cancel_batch_scrapeself.batch_scrape = self._v2_client.batch_scrapeself.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors@@ -248,6 +254,7 @@ class AsyncFirecrawl:self.start_crawl = self._v2_client.start_crawlself.get_crawl_status = self._v2_client.get_crawl_status+ self.get_crawl_status_page = self._v2_client.get_crawl_status_pageself.cancel_crawl = self._v2_client.cancel_crawlself.crawl = self._v2_client.crawlself.get_crawl_errors = self._v2_client.get_crawl_errors@@ -256,6 +263,7 @@ class AsyncFirecrawl:self.start_batch_scrape = self._v2_client.start_batch_scrapeself.get_batch_scrape_status = self._v2_client.get_batch_scrape_status+ self.get_batch_scrape_status_page = self._v2_client.get_batch_scrape_status_pageself.cancel_batch_scrape = self._v2_client.cancel_batch_scrapeself.batch_scrape = self._v2_client.batch_scrapeself.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors@@ -278,4 +286,4 @@ class AsyncFirecrawl:# Export Firecrawl as an alias for FirecrawlAppFirecrawlApp = Firecrawl-AsyncFirecrawlApp = AsyncFirecrawl\ No newline at end of file+AsyncFirecrawlApp = AsyncFirecrawldiff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex c5df6d9d8..8a476169e 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -423,6 +423,28 @@ class FirecrawlClient:pagination_config=pagination_config,request_timeout=request_timeout,)++ def get_crawl_status_page(+ self,+ next_url: str,+ *,+ request_timeout: Optional[float] = None,+ ) -> CrawlJob:+ """+ Fetch a single page of crawl results using a next URL.++ Args:+ next_url: Opaque next URL from a prior crawl status response+ request_timeout: Timeout (in seconds) for the HTTP request++ Returns:+ CrawlJob with the page data and next URL (if any)+ """+ return crawl_module.get_crawl_status_page(+ self.http_client,+ next_url,+ request_timeout=request_timeout,+ )def get_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse:"""@@ -741,6 +763,27 @@ class FirecrawlClient:pagination_config=pagination_config)+ def get_batch_scrape_status_page(+ self,+ next_url: str,+ *,+ request_timeout: Optional[float] = None,+ ):+ """Fetch a single page of batch scrape results using a next URL.++ Args:+ next_url: Opaque next URL from a prior batch scrape status response+ request_timeout: Timeout (in seconds) for the HTTP request++ Returns:+ BatchScrapeJob with the page data and next URL (if any)+ """+ return batch_module.get_batch_scrape_status_page(+ self.http_client,+ next_url,+ request_timeout=request_timeout,+ )+def cancel_batch_scrape(self, job_id: str) -> bool:"""Cancel a running batch scrape job.diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex c2493421f..5bd482977 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -178,6 +178,28 @@ class AsyncFirecrawlClient:request_timeout=request_timeout,)+ async def get_crawl_status_page(+ self,+ next_url: str,+ *,+ request_timeout: Optional[float] = None,+ ) -> CrawlJob:+ """+ Fetch a single page of crawl results using a next URL.++ Args:+ next_url: Opaque next URL from a prior crawl status response+ request_timeout: Timeout (in seconds) for the HTTP request++ Returns:+ CrawlJob with the page data and next URL (if any)+ """+ return await async_crawl.get_crawl_status_page(+ self.async_http_client,+ next_url,+ request_timeout=request_timeout,+ )+async def cancel_crawl(self, job_id: str) -> bool:return await async_crawl.cancel_crawl(self.async_http_client, job_id)@@ -248,6 +270,18 @@ class AsyncFirecrawlClient:pagination_config=pagination_config)+ async def get_batch_scrape_status_page(+ self,+ next_url: str,+ *,+ request_timeout: Optional[float] = None,+ ):+ return await async_batch.get_batch_scrape_status_page(+ self.async_http_client,+ next_url,+ request_timeout=request_timeout,+ )+async def cancel_batch_scrape(self, job_id: str) -> bool:return await async_batch.cancel_batch_scrape(self.async_http_client, job_id)diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py b/apps/python-sdk/firecrawl/v2/methods/aio/batch.pyindex 958cd78db..4585f4443 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py@@ -7,6 +7,29 @@ from ...utils.normalize import normalize_document_inputfrom ...methods.batch import validate_batch_urlsimport time+def _parse_batch_scrape_documents(data_list: Optional[List[Any]]) -> List[Document]:+ documents: List[Document] = []+ for doc in data_list or []:+ if isinstance(doc, dict):+ normalized = normalize_document_input(doc)+ documents.append(Document(**normalized))+ return documents+++def _parse_batch_scrape_status_response(body: Dict[str, Any]) -> Dict[str, Any]:+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))++ return {+ "status": body.get("status"),+ "completed": body.get("completed", 0),+ "total": body.get("total", 0),+ "credits_used": body.get("creditsUsed"),+ "expires_at": body.get("expiresAt"),+ "next": body.get("next"),+ "data": _parse_batch_scrape_documents(body.get("data", []) or []),+ }+def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwargs) -> Dict[str, Any]:if not urls:raise ValueError("URLs list cannot be empty")@@ -68,35 +91,66 @@ async def get_batch_scrape_status(if response.status_code >= 400:handle_response_error(response, "get batch scrape status")body = response.json()- if not body.get("success"):- raise Exception(body.get("error", "Unknown error occurred"))- docs: List[Document] = []- for doc in body.get("data", []) or []:- if isinstance(doc, dict):- normalized = normalize_document_input(doc)- docs.append(Document(**normalized))+ payload = _parse_batch_scrape_status_response(body)+ docs = payload["data"]# Handle pagination if requestedauto_paginate = pagination_config.auto_paginate if pagination_config else True- if auto_paginate and body.get("next"):+ if auto_paginate and payload["next"]:docs = await _fetch_all_batch_pages_async(client,- body.get("next"),+ payload["next"],docs,pagination_config)return BatchScrapeJob(- status=body.get("status"),- completed=body.get("completed", 0),- total=body.get("total", 0),- credits_used=body.get("creditsUsed"),- expires_at=body.get("expiresAt"),- next=body.get("next") if not auto_paginate else None,+ status=payload["status"],+ completed=payload["completed"],+ total=payload["total"],+ credits_used=payload["credits_used"],+ expires_at=payload["expires_at"],+ next=payload["next"] if not auto_paginate else None,data=docs,)+async def get_batch_scrape_status_page(+ client: AsyncHttpClient,+ next_url: str,+ *,+ request_timeout: Optional[float] = None,+) -> BatchScrapeJob:+ """+ Fetch a single page of batch scrape results using the provided next URL.++ Args:+ client: Async HTTP client instance+ next_url: Opaque next URL from a prior batch scrape status response+ request_timeout: Timeout (in seconds) for the HTTP request++ Returns:+ BatchScrapeJob with the page data and next URL (if any)++ Raises:+ Exception: If the request fails or returns an error response+ """+ response = await client.get(next_url, timeout=request_timeout)+ if response.status_code >= 400:+ handle_response_error(response, "get batch scrape status page")+ body = response.json()+ payload = _parse_batch_scrape_status_response(body)+ return BatchScrapeJob(+ status=payload["status"],+ completed=payload["completed"],+ total=payload["total"],+ credits_used=payload["credits_used"],+ expires_at=payload["expires_at"],+ next=payload["next"],+ data=payload["data"],+ )++async def _fetch_all_batch_pages_async(client: AsyncHttpClient,next_url: str,@@ -145,25 +199,24 @@ async def _fetch_all_batch_pages_async(breakpage_data = response.json()-- if not page_data.get("success"):+ try:+ page_payload = _parse_batch_scrape_status_response(page_data)+ except Exception:break# Add documents from this page- for doc in page_data.get("data", []) or []:- if isinstance(doc, dict):- # Check max_results limit- if (max_results is not None) and (len(documents) >= max_results):- break- normalized = normalize_document_input(doc)- documents.append(Document(**normalized))+ for document in page_payload["data"]:+ # Check max_results limit+ if (max_results is not None) and (len(documents) >= max_results):+ break+ documents.append(document)# Check if we hit max_results limitif (max_results is not None) and (len(documents) >= max_results):break# Get next URL- current_url = page_data.get("next")+ current_url = page_payload["next"]page_count += 1return documents@@ -185,4 +238,3 @@ async def get_batch_scrape_errors(client: AsyncHttpClient, job_id: str) -> Dict[if not body.get("success"):raise Exception(body.get("error", "Unknown error occurred"))return body-diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.pyindex c2a10d8d7..354735345 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py@@ -61,6 +61,30 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:return data+def _parse_crawl_documents(data_list: Optional[List[Any]]) -> List[Document]:+ documents: List[Document] = []+ for doc_data in data_list or []:+ if isinstance(doc_data, dict):+ normalized = normalize_document_input(doc_data)+ documents.append(Document(**normalized))+ return documents+++def _parse_crawl_status_response(body: Dict[str, Any]) -> Dict[str, Any]:+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))++ return {+ "status": body.get("status"),+ "completed": body.get("completed", 0),+ "total": body.get("total", 0),+ "credits_used": body.get("creditsUsed", 0),+ "expires_at": body.get("expiresAt"),+ "next": body.get("next"),+ "data": _parse_crawl_documents(body.get("data", [])),+ }++async def start_crawl(client: AsyncHttpClient, request: CrawlRequest) -> CrawlResponse:"""Start a crawl job for a website.@@ -114,34 +138,66 @@ async def get_crawl_status(if response.status_code >= 400:handle_response_error(response, "get crawl status")body = response.json()- if body.get("success"):- documents = []- for doc_data in body.get("data", []):- if isinstance(doc_data, dict):- normalized = normalize_document_input(doc_data)- documents.append(Document(**normalized))-- # Handle pagination if requested- auto_paginate = pagination_config.auto_paginate if pagination_config else True- if auto_paginate and body.get("next"):- documents = await _fetch_all_pages_async(- client,- body.get("next"),- documents,- pagination_config,- request_timeout=request_timeout,- )-- return CrawlJob(- status=body.get("status"),- completed=body.get("completed", 0),- total=body.get("total", 0),- credits_used=body.get("creditsUsed", 0),- expires_at=body.get("expiresAt"),- next=body.get("next") if not auto_paginate else None,- data=documents,+ payload = _parse_crawl_status_response(body)++ documents = payload["data"]++ # Handle pagination if requested+ auto_paginate = pagination_config.auto_paginate if pagination_config else True+ if auto_paginate and payload["next"]:+ documents = await _fetch_all_pages_async(+ client,+ payload["next"],+ documents,+ pagination_config,+ request_timeout=request_timeout,)- raise Exception(body.get("error", "Unknown error occurred"))++ return CrawlJob(+ status=payload["status"],+ completed=payload["completed"],+ total=payload["total"],+ credits_used=payload["credits_used"],+ expires_at=payload["expires_at"],+ next=payload["next"] if not auto_paginate else None,+ data=documents,+ )+++async def get_crawl_status_page(+ client: AsyncHttpClient,+ next_url: str,+ *,+ request_timeout: Optional[float] = None,+) -> CrawlJob:+ """+ Fetch a single page of crawl results using the provided next URL.++ Args:+ client: Async HTTP client instance+ next_url: Opaque next URL from a prior crawl status response+ request_timeout: Timeout (in seconds) for the HTTP request++ Returns:+ CrawlJob with the page data and next URL (if any)++ Raises:+ Exception: If the request fails or returns an error response+ """+ response = await client.get(next_url, timeout=request_timeout)+ if response.status_code >= 400:+ handle_response_error(response, "get crawl status page")+ body = response.json()+ payload = _parse_crawl_status_response(body)+ return CrawlJob(+ status=payload["status"],+ completed=payload["completed"],+ total=payload["total"],+ credits_used=payload["credits_used"],+ expires_at=payload["expires_at"],+ next=payload["next"],+ data=payload["data"],+ )async def _fetch_all_pages_async(@@ -195,25 +251,24 @@ async def _fetch_all_pages_async(breakpage_data = response.json()-- if not page_data.get("success"):+ try:+ page_payload = _parse_crawl_status_response(page_data)+ except Exception:break# Add documents from this page- for doc_data in page_data.get("data", []):- if isinstance(doc_data, dict):- # Check max_results limit- if (max_results is not None) and (len(documents) >= max_results):- break- normalized = normalize_document_input(doc_data)- documents.append(Document(**normalized))+ for document in page_payload["data"]:+ # Check max_results limit+ if (max_results is not None) and (len(documents) >= max_results):+ break+ documents.append(document)# Check if we hit max_results limitif (max_results is not None) and (len(documents) >= max_results):break# Get next URL- current_url = page_data.get("next")+ current_url = page_payload["next"]page_count += 1return documentsdiff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.pyindex 53d2ea44d..2b3477d66 100644--- a/apps/python-sdk/firecrawl/v2/methods/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py@@ -18,6 +18,30 @@ from ..utils.normalize import normalize_document_inputfrom ..types import CrawlErrorsResponse+def _parse_batch_scrape_documents(data_list: Optional[List[Any]]) -> List[Document]:+ documents: List[Document] = []+ for doc in data_list or []:+ if isinstance(doc, dict):+ normalized = normalize_document_input(doc)+ documents.append(Document(**normalized))+ return documents+++def _parse_batch_scrape_status_response(body: Dict[str, Any]) -> Dict[str, Any]:+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))++ return {+ "status": body.get("status"),+ "completed": body.get("completed", 0),+ "total": body.get("total", 0),+ "credits_used": body.get("creditsUsed"),+ "expires_at": body.get("expiresAt"),+ "next": body.get("next"),+ "data": _parse_batch_scrape_documents(body.get("data", []) or []),+ }++def start_batch_scrape(client: HttpClient,urls: List[str],@@ -104,37 +128,69 @@ def get_batch_scrape_status(# Parse responsebody = response.json()- if not body.get("success"):- raise Exception(body.get("error", "Unknown error occurred"))-- # Convert documents- documents: List[Document] = []- for doc in body.get("data", []) or []:- if isinstance(doc, dict):- normalized = normalize_document_input(doc)- documents.append(Document(**normalized))+ payload = _parse_batch_scrape_status_response(body)+ documents = payload["data"]# Handle pagination if requestedauto_paginate = pagination_config.auto_paginate if pagination_config else True- if auto_paginate and body.get("next"):+ if auto_paginate and payload["next"]:documents = _fetch_all_batch_pages(client,- body.get("next"),+ payload["next"],documents,pagination_config)return BatchScrapeJob(- status=body.get("status"),- completed=body.get("completed", 0),- total=body.get("total", 0),- credits_used=body.get("creditsUsed"),- expires_at=body.get("expiresAt"),- next=body.get("next") if not auto_paginate else None,+ status=payload["status"],+ completed=payload["completed"],+ total=payload["total"],+ credits_used=payload["credits_used"],+ expires_at=payload["expires_at"],+ next=payload["next"] if not auto_paginate else None,data=documents,)+def get_batch_scrape_status_page(+ client: HttpClient,+ next_url: str,+ *,+ request_timeout: Optional[float] = None,+) -> BatchScrapeJob:+ """+ Fetch a single page of batch scrape results using the provided next URL.++ Args:+ client: HTTP client instance+ next_url: Opaque next URL from a prior batch scrape status response+ request_timeout: Timeout (in seconds) for the HTTP request++ Returns:+ BatchScrapeJob with the page data and next URL (if any)++ Raises:+ Exception: If the request fails or returns an error response+ """+ response = client.get(next_url, timeout=request_timeout)++ if not response.ok:+ handle_response_error(response, "get batch scrape status page")++ body = response.json()+ payload = _parse_batch_scrape_status_response(body)++ return BatchScrapeJob(+ status=payload["status"],+ completed=payload["completed"],+ total=payload["total"],+ credits_used=payload["credits_used"],+ expires_at=payload["expires_at"],+ next=payload["next"],+ data=payload["data"],+ )++def _fetch_all_batch_pages(client: HttpClient,next_url: str,@@ -183,25 +239,24 @@ def _fetch_all_batch_pages(breakpage_data = response.json()-- if not page_data.get("success"):+ try:+ page_payload = _parse_batch_scrape_status_response(page_data)+ except Exception:break# Add documents from this page- for doc in page_data.get("data", []) or []:- if isinstance(doc, dict):- # Check max_results limit- if max_results is not None and len(documents) >= max_results:- break- normalized = normalize_document_input(doc)- documents.append(Document(**normalized))+ for document in page_payload["data"]:+ # Check max_results limit+ if max_results is not None and len(documents) >= max_results:+ break+ documents.append(document)# Check if we hit max_results limit after adding all docs from this pageif max_results is not None and len(documents) >= max_results:break# Get next URL- current_url = page_data.get("next")+ current_url = page_payload["next"]page_count += 1return documents@@ -496,4 +551,4 @@ def get_batch_scrape_errors(client: HttpClient, job_id: str) -> CrawlErrorsRespo"errors": payload.get("errors", []),"robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),}- return CrawlErrorsResponse(**normalized)\ No newline at end of file+ return CrawlErrorsResponse(**normalized)diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex 5c91b02d4..1730fc726 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -106,6 +106,29 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:return data+def _parse_crawl_documents(data_list: Optional[List[Any]]) -> List[Document]:+ documents: List[Document] = []+ for doc_data in data_list or []:+ if isinstance(doc_data, dict):+ documents.append(Document(**normalize_document_input(doc_data)))+ return documents+++def _parse_crawl_status_response(response_data: Dict[str, Any]) -> Dict[str, Any]:+ if not response_data.get("success"):+ raise Exception(response_data.get("error", "Unknown error occurred"))++ return {+ "status": response_data.get("status"),+ "completed": response_data.get("completed", 0),+ "total": response_data.get("total", 0),+ "credits_used": response_data.get("creditsUsed", 0),+ "expires_at": response_data.get("expiresAt"),+ "next": response_data.get("next"),+ "data": _parse_crawl_documents(response_data.get("data", [])),+ }++def start_crawl(client: HttpClient, request: CrawlRequest) -> CrawlResponse:"""Start a crawl job for a website.@@ -175,47 +198,74 @@ def get_crawl_status(# Parse responseresponse_data = response.json()- if response_data.get("success"):- # The API returns status fields at the top level, not in a data field-- # Convert documents- documents = []- data_list = response_data.get("data", [])- for doc_data in data_list:- if isinstance(doc_data, str):- # Handle case where API returns just URLs - this shouldn't happen for crawl- # but we'll handle it gracefully- continue- else:- documents.append(Document(**normalize_document_input(doc_data)))-- # Handle pagination if requested- auto_paginate = pagination_config.auto_paginate if pagination_config else True- if auto_paginate and response_data.get("next") and not (- pagination_config- and pagination_config.max_results is not None- and len(documents) >= pagination_config.max_results- ):- documents = _fetch_all_pages(- client,- response_data.get("next"),- documents,- pagination_config,- request_timeout=request_timeout,- )-- # Create CrawlJob with current status and data- return CrawlJob(- status=response_data.get("status"),- completed=response_data.get("completed", 0),- total=response_data.get("total", 0),- credits_used=response_data.get("creditsUsed", 0),- expires_at=response_data.get("expiresAt"),- next=response_data.get("next", None) if not auto_paginate else None,- data=documents+ payload = _parse_crawl_status_response(response_data)++ documents = payload["data"]++ # Handle pagination if requested+ auto_paginate = pagination_config.auto_paginate if pagination_config else True+ if auto_paginate and payload["next"] and not (+ pagination_config+ and pagination_config.max_results is not None+ and len(documents) >= pagination_config.max_results+ ):+ documents = _fetch_all_pages(+ client,+ payload["next"],+ documents,+ pagination_config,+ request_timeout=request_timeout,)- else:- raise Exception(response_data.get("error", "Unknown error occurred"))++ # Create CrawlJob with current status and data+ return CrawlJob(+ status=payload["status"],+ completed=payload["completed"],+ total=payload["total"],+ credits_used=payload["credits_used"],+ expires_at=payload["expires_at"],+ next=payload["next"] if not auto_paginate else None,+ data=documents,+ )+++def get_crawl_status_page(+ client: HttpClient,+ next_url: str,+ *,+ request_timeout: Optional[float] = None,+) -> CrawlJob:+ """+ Fetch a single page of crawl results using the provided next URL.++ Args:+ client: HTTP client instance+ next_url: Opaque next URL from a prior crawl status response+ request_timeout: Timeout (in seconds) for the HTTP request++ Returns:+ CrawlJob with the page data and next URL (if any)++ Raises:+ Exception: If the request fails or returns an error response+ """+ response = client.get(next_url, timeout=request_timeout)++ if not response.ok:+ handle_response_error(response, "get crawl status page")++ response_data = response.json()+ payload = _parse_crawl_status_response(response_data)++ return CrawlJob(+ status=payload["status"],+ completed=payload["completed"],+ total=payload["total"],+ credits_used=payload["credits_used"],+ expires_at=payload["expires_at"],+ next=payload["next"],+ data=payload["data"],+ )def _fetch_all_pages(@@ -270,26 +320,24 @@ def _fetch_all_pages(page_data = response.json()- if not page_data.get("success"):+ try:+ page_payload = _parse_crawl_status_response(page_data)+ except Exception:break# Add documents from this page- data_list = page_data.get("data", [])- for doc_data in data_list:- if isinstance(doc_data, str):- continue- else:- # Check max_results limit BEFORE adding each document- if max_results is not None and len(documents) >= max_results:- break- documents.append(Document(**normalize_document_input(doc_data)))+ for document in page_payload["data"]:+ # Check max_results limit BEFORE adding each document+ if max_results is not None and len(documents) >= max_results:+ break+ documents.append(document)# Check if we hit max_results limitif max_results is not None and len(documents) >= max_results:break# Get next URL- current_url = page_data.get("next")+ current_url = page_payload["next"]page_count += 1return documents