Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add manual pagination helpers for Python SDK (#2727)

firecrawl-spring[bot]5696a701d21f
diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.md
index 253553b68..50aa82ca2 100644
--- a/apps/python-sdk/README.md
+++ b/apps/python-sdk/README.md
@@ -87,6 +87,32 @@ crawl_status = firecrawl.get_crawl_status("<crawl_id>")
print(crawl_status)
```
+### Manual Pagination (v2)
+
+Crawl and batch scrape status responses may include a `next` URL when more data is available. The SDK auto-paginates by default; to page manually, disable auto-pagination and pass the opaque `next` URL back to the SDK.
+
+```python
+from firecrawl.v2.types import PaginationConfig
+
+# Crawl: fetch one page at a time
+crawl_job = firecrawl.start_crawl("https://firecrawl.dev", limit=100)
+status = firecrawl.get_crawl_status(
+ crawl_job.id,
+ pagination_config=PaginationConfig(auto_paginate=False),
+)
+if status.next:
+ page2 = firecrawl.get_crawl_status_page(status.next)
+
+# Batch scrape: fetch one page at a time
+batch_job = firecrawl.start_batch_scrape(["https://firecrawl.dev"])
+status = firecrawl.get_batch_scrape_status(
+ batch_job.id,
+ pagination_config=PaginationConfig(auto_paginate=False),
+)
+if status.next:
+ page2 = firecrawl.get_batch_scrape_status_page(status.next)
+```
+
### Cancelling a Crawl
To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status.
@@ -184,4 +210,4 @@ firecrawl = Firecrawl(api_key="YOUR_API_KEY")
doc_v1 = firecrawl.v1.scrape_url('https://firecrawl.dev', formats=['markdown', 'html'])
crawl_v1 = firecrawl.v1.crawl_url('https://firecrawl.dev', limit=100)
map_v1 = firecrawl.v1.map_url('https://firecrawl.dev')
-```
\ No newline at end of file
+```
diff --git a/apps/python-sdk/example_pagination.py b/apps/python-sdk/example_pagination.py
index 106d578ca..589ff9392 100644
--- a/apps/python-sdk/example_pagination.py
+++ b/apps/python-sdk/example_pagination.py
@@ -18,7 +18,8 @@ print(f"Total documents fetched: {len(crawl_result.data)}")
print(f"Next URL: {crawl_result.next}") # Should be None since auto-pagination is enabled
# Example 2: Manual crawl with pagination control
-# Use this when you need to control how many pages to fetch or want to process results incrementally
+# Use this when you need to control how many pages to fetch or want to process results incrementally.
+# The next URL is opaque; pass it back to the SDK to fetch the next page.
print("\n=== Example 2: Manual crawl with pagination control ===")
crawl_job = firecrawl.start_crawl("https://example.com", limit=100)
@@ -27,6 +28,9 @@ pagination_config = PaginationConfig(auto_paginate=False)
status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)
print(f"Documents from first page: {len(status.data)}")
print(f"Next URL: {status.next}") # Will show the next page URL
+if status.next:
+ next_page = firecrawl.get_crawl_status_page(status.next)
+ print(f"Documents from next page: {len(next_page.data)}")
# Example 3: Limited pagination - fetch only 3 pages
# Useful for controlling memory usage or processing time
@@ -68,11 +72,16 @@ batch_result = firecrawl.batch_scrape(urls)
print(f"Batch scrape documents: {len(batch_result.data)}")
# Example 8: Manual batch scrape with pagination control
-# Use this when you need to control how many pages to fetch or want to process results incrementally
+# Use this when you need to control how many pages to fetch or want to process results incrementally.
+# The next URL is opaque; pass it back to the SDK to fetch the next page.
print("\n=== Example 8: Manual batch scrape with pagination control ===")
batch_job = firecrawl.start_batch_scrape(urls)
-status = firecrawl.get_batch_scrape_status(batch_job.id)
+status = firecrawl.get_batch_scrape_status(batch_job.id, pagination_config=PaginationConfig(auto_paginate=False))
print(f"Batch scrape documents: {len(status.data)}")
+print(f"Next URL: {status.next}")
+if status.next:
+ next_page = firecrawl.get_batch_scrape_status_page(status.next)
+ print(f"Batch scrape next page documents: {len(next_page.data)}")
# Example 9: Async usage
print("\n=== Example 9: Async pagination ===")
@@ -94,6 +103,9 @@ async def async_example():
pagination_config=pagination_config
)
print(f"Async crawl with pagination: {len(status.data)}")
+ if status.next:
+ next_page = await async_client.get_crawl_status_page(status.next)
+ print(f"Async crawl next page documents: {len(next_page.data)}")
# Run async example
# asyncio.run(async_example())
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index fcda4c526..76deaa9be 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.13.2"
+__version__ = "4.13.3"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py
index f5fb8ecbe..a521d9b04 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_batch_scrape.py
@@ -1,8 +1,9 @@
import os
+import time
import pytest
from dotenv import load_dotenv
from firecrawl import Firecrawl
-from firecrawl.v2.types import ScrapeOptions
+from firecrawl.v2.types import ScrapeOptions, PaginationConfig
load_dotenv()
@@ -48,6 +49,39 @@ class TestBatchScrapeE2E:
assert job.status in ["scraping", "completed", "failed"]
assert job.total >= 0
+ def test_get_batch_scrape_status_page(self):
+ """Fetch a single batch scrape page using the next URL."""
+ urls = [f"https://docs.firecrawl.dev?batch={i}" for i in range(15)]
+
+ start_resp = self.client.start_batch_scrape(
+ urls,
+ formats=["markdown"],
+ ignore_invalid_urls=True,
+ )
+ assert start_resp.id is not None
+
+ pagination_config = PaginationConfig(auto_paginate=False)
+ deadline = time.time() + 120
+ status_job = None
+ while time.time() < deadline:
+ status_job = self.client.get_batch_scrape_status(
+ start_resp.id,
+ pagination_config=pagination_config,
+ )
+ if status_job.next:
+ break
+ if status_job.status in ["completed", "failed", "cancelled"]:
+ break
+ time.sleep(2)
+
+ assert status_job is not None
+ if not status_job.next:
+ pytest.skip("Batch scrape completed without pagination; skipping page fetch.")
+
+ next_page = self.client.get_batch_scrape_status_page(status_job.next)
+ assert isinstance(next_page.data, list)
+ assert next_page.status in ["scraping", "completed", "failed", "cancelled"]
+
def test_wait_batch_with_all_params(self):
"""Blocking waiter with JSON and changeTracking formats plus many options."""
urls = [
@@ -103,4 +137,3 @@ class TestBatchScrapeE2E:
cancelled = self.client.cancel_batch_scrape(start_resp.id)
assert cancelled is True
-
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py
index 2544445af..6fa88780c 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py
@@ -3,7 +3,7 @@ import time
import os
from dotenv import load_dotenv
from firecrawl import Firecrawl
-from firecrawl.v2.types import ScrapeOptions
+from firecrawl.v2.types import ScrapeOptions, PaginationConfig
load_dotenv()
@@ -66,6 +66,33 @@ class TestCrawlE2E:
assert status_job.next is None
assert isinstance(status_job.data, list)
+ def test_get_crawl_status_page(self):
+ """Fetch a single crawl page using the next URL."""
+ start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=25)
+ assert start_job.id is not None
+
+ pagination_config = PaginationConfig(auto_paginate=False)
+ deadline = time.time() + 120
+ status_job = None
+ while time.time() < deadline:
+ status_job = self.client.get_crawl_status(
+ start_job.id,
+ pagination_config=pagination_config,
+ )
+ if status_job.next:
+ break
+ if status_job.status in ["completed", "failed", "cancelled"]:
+ break
+ time.sleep(2)
+
+ assert status_job is not None
+ if not status_job.next:
+ pytest.skip("Crawl completed without pagination; skipping page fetch.")
+
+ next_page = self.client.get_crawl_status_page(status_job.next)
+ assert isinstance(next_page.data, list)
+ assert next_page.status in ["scraping", "completed", "failed", "cancelled"]
+
def test_cancel_crawl(self):
"""Test canceling a crawl."""
start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=3)
@@ -275,4 +302,4 @@ class TestCrawlE2E:
assert params_data is not None
assert params_data.limit is not None or params_data.include_paths is not None or params_data.max_discovery_depth is not None
assert 'blog/.*' in params_data.include_paths
- assert 'docs/.*' in params_data.include_paths
\ No newline at end of file
+ assert 'docs/.*' in params_data.include_paths
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py
index d05c562c7..a872db84c 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_pagination.py
@@ -14,10 +14,18 @@ from firecrawl.v2.types import (
Document,
DocumentMetadata
)
-from firecrawl.v2.methods.crawl import get_crawl_status, _fetch_all_pages
-from firecrawl.v2.methods.batch import get_batch_scrape_status, _fetch_all_batch_pages
-from firecrawl.v2.methods.aio.crawl import get_crawl_status as get_crawl_status_async, _fetch_all_pages_async
-from firecrawl.v2.methods.aio.batch import get_batch_scrape_status as get_batch_scrape_status_async, _fetch_all_batch_pages_async
+from firecrawl.v2.methods.crawl import get_crawl_status, get_crawl_status_page, _fetch_all_pages
+from firecrawl.v2.methods.batch import get_batch_scrape_status, get_batch_scrape_status_page, _fetch_all_batch_pages
+from firecrawl.v2.methods.aio.crawl import (
+ get_crawl_status as get_crawl_status_async,
+ get_crawl_status_page as get_crawl_status_page_async,
+ _fetch_all_pages_async,
+)
+from firecrawl.v2.methods.aio.batch import (
+ get_batch_scrape_status as get_batch_scrape_status_async,
+ get_batch_scrape_status_page as get_batch_scrape_status_page_async,
+ _fetch_all_batch_pages_async,
+)
class TestPaginationConfig:
@@ -123,6 +131,59 @@ class TestCrawlPagination:
self.mock_client.get.assert_called_with(
f"/v2/crawl/{self.job_id}", timeout=timeout_seconds
)
+
+ def test_get_crawl_status_page(self):
+ """Test get_crawl_status_page returns a single page."""
+ mock_response = Mock()
+ mock_response.ok = True
+ mock_response.json.return_value = {
+ "success": True,
+ "status": "completed",
+ "completed": 10,
+ "total": 20,
+ "creditsUsed": 5,
+ "expiresAt": "2024-01-01T00:00:00Z",
+ "next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=3",
+ "data": [self.sample_doc],
+ }
+
+ self.mock_client.get.return_value = mock_response
+ next_url = "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2"
+
+ result = get_crawl_status_page(self.mock_client, next_url)
+
+ assert result.status == "completed"
+ assert result.next == "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=3"
+ assert len(result.data) == 1
+ self.mock_client.get.assert_called_with(next_url, timeout=None)
+
+ def test_get_crawl_status_page_propagates_request_timeout(self):
+ """Ensure request_timeout is forwarded to crawl status page requests."""
+ mock_response = Mock()
+ mock_response.ok = True
+ mock_response.json.return_value = {
+ "success": True,
+ "status": "completed",
+ "completed": 1,
+ "total": 1,
+ "creditsUsed": 1,
+ "expiresAt": "2024-01-01T00:00:00Z",
+ "next": None,
+ "data": [self.sample_doc],
+ }
+
+ self.mock_client.get.return_value = mock_response
+
+ next_url = "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2"
+ timeout_seconds = 4.2
+ result = get_crawl_status_page(
+ self.mock_client,
+ next_url,
+ request_timeout=timeout_seconds,
+ )
+
+ assert result.status == "completed"
+ self.mock_client.get.assert_called_with(next_url, timeout=timeout_seconds)
def test_get_crawl_status_with_pagination(self):
"""Test get_crawl_status with auto_paginate=True."""
@@ -326,6 +387,59 @@ class TestBatchScrapePagination:
assert result.next == "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2"
assert len(result.data) == 1
assert isinstance(result.data[0], Document)
+
+ def test_get_batch_scrape_status_page(self):
+ """Test get_batch_scrape_status_page returns a single page."""
+ mock_response = Mock()
+ mock_response.ok = True
+ mock_response.json.return_value = {
+ "success": True,
+ "status": "completed",
+ "completed": 10,
+ "total": 20,
+ "creditsUsed": 5,
+ "expiresAt": "2024-01-01T00:00:00Z",
+ "next": "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=3",
+ "data": [self.sample_doc],
+ }
+
+ self.mock_client.get.return_value = mock_response
+ next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2"
+
+ result = get_batch_scrape_status_page(self.mock_client, next_url)
+
+ assert result.status == "completed"
+ assert result.next == "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=3"
+ assert len(result.data) == 1
+ self.mock_client.get.assert_called_with(next_url, timeout=None)
+
+ def test_get_batch_scrape_status_page_propagates_request_timeout(self):
+ """Ensure request_timeout is forwarded to batch status page requests."""
+ mock_response = Mock()
+ mock_response.ok = True
+ mock_response.json.return_value = {
+ "success": True,
+ "status": "completed",
+ "completed": 1,
+ "total": 1,
+ "creditsUsed": 1,
+ "expiresAt": "2024-01-01T00:00:00Z",
+ "next": None,
+ "data": [self.sample_doc],
+ }
+
+ self.mock_client.get.return_value = mock_response
+
+ next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2"
+ timeout_seconds = 2.7
+ result = get_batch_scrape_status_page(
+ self.mock_client,
+ next_url,
+ request_timeout=timeout_seconds,
+ )
+
+ assert result.status == "completed"
+ self.mock_client.get.assert_called_with(next_url, timeout=timeout_seconds)
def test_get_batch_scrape_status_with_pagination(self):
"""Test get_batch_scrape_status with auto_paginate=True."""
@@ -493,6 +607,61 @@ class TestAsyncPagination:
f"/v2/crawl/{self.job_id}", timeout=timeout_seconds
)
+ @pytest.mark.asyncio
+ async def test_get_crawl_status_page_async(self):
+ """Test async get_crawl_status_page returns a single page."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "success": True,
+ "status": "completed",
+ "completed": 10,
+ "total": 20,
+ "creditsUsed": 5,
+ "expiresAt": "2024-01-01T00:00:00Z",
+ "next": "https://api.firecrawl.dev/v2/crawl/test-async-123?page=3",
+ "data": [self.sample_doc],
+ }
+
+ self.mock_client.get.return_value = mock_response
+ next_url = "https://api.firecrawl.dev/v2/crawl/test-async-123?page=2"
+
+ result = await get_crawl_status_page_async(self.mock_client, next_url)
+
+ assert result.status == "completed"
+ assert result.next == "https://api.firecrawl.dev/v2/crawl/test-async-123?page=3"
+ assert len(result.data) == 1
+ self.mock_client.get.assert_awaited_with(next_url, timeout=None)
+
+ @pytest.mark.asyncio
+ async def test_get_crawl_status_page_async_propagates_request_timeout(self):
+ """Ensure async request_timeout is forwarded to crawl status page requests."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "success": True,
+ "status": "completed",
+ "completed": 1,
+ "total": 1,
+ "creditsUsed": 1,
+ "expiresAt": "2024-01-01T00:00:00Z",
+ "next": None,
+ "data": [self.sample_doc],
+ }
+
+ self.mock_client.get.return_value = mock_response
+
+ next_url = "https://api.firecrawl.dev/v2/crawl/test-async-123?page=2"
+ timeout_seconds = 6.1
+ result = await get_crawl_status_page_async(
+ self.mock_client,
+ next_url,
+ request_timeout=timeout_seconds,
+ )
+
+ assert result.status == "completed"
+ self.mock_client.get.assert_awaited_with(next_url, timeout=timeout_seconds)
+
@pytest.mark.asyncio
async def test_get_batch_scrape_status_async_with_pagination(self):
"""Test async get_batch_scrape_status with pagination."""
@@ -534,6 +703,61 @@ class TestAsyncPagination:
assert result.next is None
assert len(result.data) == 2
assert self.mock_client.get.call_count == 2
+
+ @pytest.mark.asyncio
+ async def test_get_batch_scrape_status_page_async(self):
+ """Test async get_batch_scrape_status_page returns a single page."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "success": True,
+ "status": "completed",
+ "completed": 10,
+ "total": 20,
+ "creditsUsed": 5,
+ "expiresAt": "2024-01-01T00:00:00Z",
+ "next": "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=3",
+ "data": [self.sample_doc],
+ }
+
+ self.mock_client.get.return_value = mock_response
+ next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=2"
+
+ result = await get_batch_scrape_status_page_async(self.mock_client, next_url)
+
+ assert result.status == "completed"
+ assert result.next == "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=3"
+ assert len(result.data) == 1
+ self.mock_client.get.assert_awaited_with(next_url, timeout=None)
+
+ @pytest.mark.asyncio
+ async def test_get_batch_scrape_status_page_async_propagates_request_timeout(self):
+ """Ensure async request_timeout is forwarded to batch status page requests."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "success": True,
+ "status": "completed",
+ "completed": 1,
+ "total": 1,
+ "creditsUsed": 1,
+ "expiresAt": "2024-01-01T00:00:00Z",
+ "next": None,
+ "data": [self.sample_doc],
+ }
+
+ self.mock_client.get.return_value = mock_response
+
+ next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=2"
+ timeout_seconds = 4.4
+ result = await get_batch_scrape_status_page_async(
+ self.mock_client,
+ next_url,
+ request_timeout=timeout_seconds,
+ )
+
+ assert result.status == "completed"
+ self.mock_client.get.assert_awaited_with(next_url, timeout=timeout_seconds)
@pytest.mark.asyncio
async def test_fetch_all_pages_async_limits(self):
diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.py
index db5c1c0a9..c15936fe0 100644
--- a/apps/python-sdk/firecrawl/client.py
+++ b/apps/python-sdk/firecrawl/client.py
@@ -61,6 +61,7 @@ class V2Proxy:
self.crawl = client_instance.crawl
self.start_crawl = client_instance.start_crawl
self.get_crawl_status = client_instance.get_crawl_status
+ self.get_crawl_status_page = client_instance.get_crawl_status_page
self.cancel_crawl = client_instance.cancel_crawl
self.get_crawl_errors = client_instance.get_crawl_errors
self.get_active_crawls = client_instance.get_active_crawls
@@ -78,6 +79,7 @@ class V2Proxy:
self.start_batch_scrape = client_instance.start_batch_scrape
self.get_batch_scrape_status = client_instance.get_batch_scrape_status
+ self.get_batch_scrape_status_page = client_instance.get_batch_scrape_status_page
self.cancel_batch_scrape = client_instance.cancel_batch_scrape
self.batch_scrape = client_instance.batch_scrape
self.get_batch_scrape_errors = client_instance.get_batch_scrape_errors
@@ -127,6 +129,7 @@ class AsyncV2Proxy:
self.start_crawl = client_instance.start_crawl
self.wait_crawl = client_instance.wait_crawl
self.get_crawl_status = client_instance.get_crawl_status
+ self.get_crawl_status_page = client_instance.get_crawl_status_page
self.cancel_crawl = client_instance.cancel_crawl
self.get_crawl_errors = client_instance.get_crawl_errors
self.get_active_crawls = client_instance.get_active_crawls
@@ -144,6 +147,7 @@ class AsyncV2Proxy:
self.start_batch_scrape = client_instance.start_batch_scrape
self.get_batch_scrape_status = client_instance.get_batch_scrape_status
+ self.get_batch_scrape_status_page = client_instance.get_batch_scrape_status_page
self.cancel_batch_scrape = client_instance.cancel_batch_scrape
self.wait_batch_scrape = client_instance.wait_batch_scrape
self.batch_scrape = client_instance.batch_scrape
@@ -198,6 +202,7 @@ class Firecrawl:
self.start_crawl = self._v2_client.start_crawl
self.crawl_params_preview = self._v2_client.crawl_params_preview
self.get_crawl_status = self._v2_client.get_crawl_status
+ self.get_crawl_status_page = self._v2_client.get_crawl_status_page
self.cancel_crawl = self._v2_client.cancel_crawl
self.get_crawl_errors = self._v2_client.get_crawl_errors
self.get_active_crawls = self._v2_client.get_active_crawls
@@ -205,6 +210,7 @@ class Firecrawl:
self.start_batch_scrape = self._v2_client.start_batch_scrape
self.get_batch_scrape_status = self._v2_client.get_batch_scrape_status
+ self.get_batch_scrape_status_page = self._v2_client.get_batch_scrape_status_page
self.cancel_batch_scrape = self._v2_client.cancel_batch_scrape
self.batch_scrape = self._v2_client.batch_scrape
self.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors
@@ -248,6 +254,7 @@ class AsyncFirecrawl:
self.start_crawl = self._v2_client.start_crawl
self.get_crawl_status = self._v2_client.get_crawl_status
+ self.get_crawl_status_page = self._v2_client.get_crawl_status_page
self.cancel_crawl = self._v2_client.cancel_crawl
self.crawl = self._v2_client.crawl
self.get_crawl_errors = self._v2_client.get_crawl_errors
@@ -256,6 +263,7 @@ class AsyncFirecrawl:
self.start_batch_scrape = self._v2_client.start_batch_scrape
self.get_batch_scrape_status = self._v2_client.get_batch_scrape_status
+ self.get_batch_scrape_status_page = self._v2_client.get_batch_scrape_status_page
self.cancel_batch_scrape = self._v2_client.cancel_batch_scrape
self.batch_scrape = self._v2_client.batch_scrape
self.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors
@@ -278,4 +286,4 @@ class AsyncFirecrawl:
# Export Firecrawl as an alias for FirecrawlApp
FirecrawlApp = Firecrawl
-AsyncFirecrawlApp = AsyncFirecrawl
\ No newline at end of file
+AsyncFirecrawlApp = AsyncFirecrawl
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index c5df6d9d8..8a476169e 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -423,6 +423,28 @@ class FirecrawlClient:
pagination_config=pagination_config,
request_timeout=request_timeout,
)
+
+ def get_crawl_status_page(
+ self,
+ next_url: str,
+ *,
+ request_timeout: Optional[float] = None,
+ ) -> CrawlJob:
+ """
+ Fetch a single page of crawl results using a next URL.
+
+ Args:
+ next_url: Opaque next URL from a prior crawl status response
+ request_timeout: Timeout (in seconds) for the HTTP request
+
+ Returns:
+ CrawlJob with the page data and next URL (if any)
+ """
+ return crawl_module.get_crawl_status_page(
+ self.http_client,
+ next_url,
+ request_timeout=request_timeout,
+ )
def get_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse:
"""
@@ -741,6 +763,27 @@ class FirecrawlClient:
pagination_config=pagination_config
)
+ def get_batch_scrape_status_page(
+ self,
+ next_url: str,
+ *,
+ request_timeout: Optional[float] = None,
+ ):
+ """Fetch a single page of batch scrape results using a next URL.
+
+ Args:
+ next_url: Opaque next URL from a prior batch scrape status response
+ request_timeout: Timeout (in seconds) for the HTTP request
+
+ Returns:
+ BatchScrapeJob with the page data and next URL (if any)
+ """
+ return batch_module.get_batch_scrape_status_page(
+ self.http_client,
+ next_url,
+ request_timeout=request_timeout,
+ )
+
def cancel_batch_scrape(self, job_id: str) -> bool:
"""Cancel a running batch scrape job.
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index c2493421f..5bd482977 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -178,6 +178,28 @@ class AsyncFirecrawlClient:
request_timeout=request_timeout,
)
+ async def get_crawl_status_page(
+ self,
+ next_url: str,
+ *,
+ request_timeout: Optional[float] = None,
+ ) -> CrawlJob:
+ """
+ Fetch a single page of crawl results using a next URL.
+
+ Args:
+ next_url: Opaque next URL from a prior crawl status response
+ request_timeout: Timeout (in seconds) for the HTTP request
+
+ Returns:
+ CrawlJob with the page data and next URL (if any)
+ """
+ return await async_crawl.get_crawl_status_page(
+ self.async_http_client,
+ next_url,
+ request_timeout=request_timeout,
+ )
+
async def cancel_crawl(self, job_id: str) -> bool:
return await async_crawl.cancel_crawl(self.async_http_client, job_id)
@@ -248,6 +270,18 @@ class AsyncFirecrawlClient:
pagination_config=pagination_config
)
+ async def get_batch_scrape_status_page(
+ self,
+ next_url: str,
+ *,
+ request_timeout: Optional[float] = None,
+ ):
+ return await async_batch.get_batch_scrape_status_page(
+ self.async_http_client,
+ next_url,
+ request_timeout=request_timeout,
+ )
+
async def cancel_batch_scrape(self, job_id: str) -> bool:
return await async_batch.cancel_batch_scrape(self.async_http_client, job_id)
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
index 958cd78db..4585f4443 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
@@ -7,6 +7,29 @@ from ...utils.normalize import normalize_document_input
from ...methods.batch import validate_batch_urls
import time
+def _parse_batch_scrape_documents(data_list: Optional[List[Any]]) -> List[Document]:
+ documents: List[Document] = []
+ for doc in data_list or []:
+ if isinstance(doc, dict):
+ normalized = normalize_document_input(doc)
+ documents.append(Document(**normalized))
+ return documents
+
+
+def _parse_batch_scrape_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+
+ return {
+ "status": body.get("status"),
+ "completed": body.get("completed", 0),
+ "total": body.get("total", 0),
+ "credits_used": body.get("creditsUsed"),
+ "expires_at": body.get("expiresAt"),
+ "next": body.get("next"),
+ "data": _parse_batch_scrape_documents(body.get("data", []) or []),
+ }
+
def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwargs) -> Dict[str, Any]:
if not urls:
raise ValueError("URLs list cannot be empty")
@@ -68,35 +91,66 @@ async def get_batch_scrape_status(
if response.status_code >= 400:
handle_response_error(response, "get batch scrape status")
body = response.json()
- if not body.get("success"):
- raise Exception(body.get("error", "Unknown error occurred"))
- docs: List[Document] = []
- for doc in body.get("data", []) or []:
- if isinstance(doc, dict):
- normalized = normalize_document_input(doc)
- docs.append(Document(**normalized))
+ payload = _parse_batch_scrape_status_response(body)
+ docs = payload["data"]
# Handle pagination if requested
auto_paginate = pagination_config.auto_paginate if pagination_config else True
- if auto_paginate and body.get("next"):
+ if auto_paginate and payload["next"]:
docs = await _fetch_all_batch_pages_async(
client,
- body.get("next"),
+ payload["next"],
docs,
pagination_config
)
return BatchScrapeJob(
- status=body.get("status"),
- completed=body.get("completed", 0),
- total=body.get("total", 0),
- credits_used=body.get("creditsUsed"),
- expires_at=body.get("expiresAt"),
- next=body.get("next") if not auto_paginate else None,
+ status=payload["status"],
+ completed=payload["completed"],
+ total=payload["total"],
+ credits_used=payload["credits_used"],
+ expires_at=payload["expires_at"],
+ next=payload["next"] if not auto_paginate else None,
data=docs,
)
+async def get_batch_scrape_status_page(
+ client: AsyncHttpClient,
+ next_url: str,
+ *,
+ request_timeout: Optional[float] = None,
+) -> BatchScrapeJob:
+ """
+ Fetch a single page of batch scrape results using the provided next URL.
+
+ Args:
+ client: Async HTTP client instance
+ next_url: Opaque next URL from a prior batch scrape status response
+ request_timeout: Timeout (in seconds) for the HTTP request
+
+ Returns:
+ BatchScrapeJob with the page data and next URL (if any)
+
+ Raises:
+ Exception: If the request fails or returns an error response
+ """
+ response = await client.get(next_url, timeout=request_timeout)
+ if response.status_code >= 400:
+ handle_response_error(response, "get batch scrape status page")
+ body = response.json()
+ payload = _parse_batch_scrape_status_response(body)
+ return BatchScrapeJob(
+ status=payload["status"],
+ completed=payload["completed"],
+ total=payload["total"],
+ credits_used=payload["credits_used"],
+ expires_at=payload["expires_at"],
+ next=payload["next"],
+ data=payload["data"],
+ )
+
+
async def _fetch_all_batch_pages_async(
client: AsyncHttpClient,
next_url: str,
@@ -145,25 +199,24 @@ async def _fetch_all_batch_pages_async(
break
page_data = response.json()
-
- if not page_data.get("success"):
+ try:
+ page_payload = _parse_batch_scrape_status_response(page_data)
+ except Exception:
break
# Add documents from this page
- for doc in page_data.get("data", []) or []:
- if isinstance(doc, dict):
- # Check max_results limit
- if (max_results is not None) and (len(documents) >= max_results):
- break
- normalized = normalize_document_input(doc)
- documents.append(Document(**normalized))
+ for document in page_payload["data"]:
+ # Check max_results limit
+ if (max_results is not None) and (len(documents) >= max_results):
+ break
+ documents.append(document)
# Check if we hit max_results limit
if (max_results is not None) and (len(documents) >= max_results):
break
# Get next URL
- current_url = page_data.get("next")
+ current_url = page_payload["next"]
page_count += 1
return documents
@@ -185,4 +238,3 @@ async def get_batch_scrape_errors(client: AsyncHttpClient, job_id: str) -> Dict[
if not body.get("success"):
raise Exception(body.get("error", "Unknown error occurred"))
return body
-
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
index c2a10d8d7..354735345 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
@@ -61,6 +61,30 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:
return data
+def _parse_crawl_documents(data_list: Optional[List[Any]]) -> List[Document]:
+ documents: List[Document] = []
+ for doc_data in data_list or []:
+ if isinstance(doc_data, dict):
+ normalized = normalize_document_input(doc_data)
+ documents.append(Document(**normalized))
+ return documents
+
+
+def _parse_crawl_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+
+ return {
+ "status": body.get("status"),
+ "completed": body.get("completed", 0),
+ "total": body.get("total", 0),
+ "credits_used": body.get("creditsUsed", 0),
+ "expires_at": body.get("expiresAt"),
+ "next": body.get("next"),
+ "data": _parse_crawl_documents(body.get("data", [])),
+ }
+
+
async def start_crawl(client: AsyncHttpClient, request: CrawlRequest) -> CrawlResponse:
"""
Start a crawl job for a website.
@@ -114,34 +138,66 @@ async def get_crawl_status(
if response.status_code >= 400:
handle_response_error(response, "get crawl status")
body = response.json()
- if body.get("success"):
- documents = []
- for doc_data in body.get("data", []):
- if isinstance(doc_data, dict):
- normalized = normalize_document_input(doc_data)
- documents.append(Document(**normalized))
-
- # Handle pagination if requested
- auto_paginate = pagination_config.auto_paginate if pagination_config else True
- if auto_paginate and body.get("next"):
- documents = await _fetch_all_pages_async(
- client,
- body.get("next"),
- documents,
- pagination_config,
- request_timeout=request_timeout,
- )
-
- return CrawlJob(
- status=body.get("status"),
- completed=body.get("completed", 0),
- total=body.get("total", 0),
- credits_used=body.get("creditsUsed", 0),
- expires_at=body.get("expiresAt"),
- next=body.get("next") if not auto_paginate else None,
- data=documents,
+ payload = _parse_crawl_status_response(body)
+
+ documents = payload["data"]
+
+ # Handle pagination if requested
+ auto_paginate = pagination_config.auto_paginate if pagination_config else True
+ if auto_paginate and payload["next"]:
+ documents = await _fetch_all_pages_async(
+ client,
+ payload["next"],
+ documents,
+ pagination_config,
+ request_timeout=request_timeout,
)
- raise Exception(body.get("error", "Unknown error occurred"))
+
+ return CrawlJob(
+ status=payload["status"],
+ completed=payload["completed"],
+ total=payload["total"],
+ credits_used=payload["credits_used"],
+ expires_at=payload["expires_at"],
+ next=payload["next"] if not auto_paginate else None,
+ data=documents,
+ )
+
+
+async def get_crawl_status_page(
+ client: AsyncHttpClient,
+ next_url: str,
+ *,
+ request_timeout: Optional[float] = None,
+) -> CrawlJob:
+ """
+ Fetch a single page of crawl results using the provided next URL.
+
+ Args:
+ client: Async HTTP client instance
+ next_url: Opaque next URL from a prior crawl status response
+ request_timeout: Timeout (in seconds) for the HTTP request
+
+ Returns:
+ CrawlJob with the page data and next URL (if any)
+
+ Raises:
+ Exception: If the request fails or returns an error response
+ """
+ response = await client.get(next_url, timeout=request_timeout)
+ if response.status_code >= 400:
+ handle_response_error(response, "get crawl status page")
+ body = response.json()
+ payload = _parse_crawl_status_response(body)
+ return CrawlJob(
+ status=payload["status"],
+ completed=payload["completed"],
+ total=payload["total"],
+ credits_used=payload["credits_used"],
+ expires_at=payload["expires_at"],
+ next=payload["next"],
+ data=payload["data"],
+ )
async def _fetch_all_pages_async(
@@ -195,25 +251,24 @@ async def _fetch_all_pages_async(
break
page_data = response.json()
-
- if not page_data.get("success"):
+ try:
+ page_payload = _parse_crawl_status_response(page_data)
+ except Exception:
break
# Add documents from this page
- for doc_data in page_data.get("data", []):
- if isinstance(doc_data, dict):
- # Check max_results limit
- if (max_results is not None) and (len(documents) >= max_results):
- break
- normalized = normalize_document_input(doc_data)
- documents.append(Document(**normalized))
+ for document in page_payload["data"]:
+ # Check max_results limit
+ if (max_results is not None) and (len(documents) >= max_results):
+ break
+ documents.append(document)
# Check if we hit max_results limit
if (max_results is not None) and (len(documents) >= max_results):
break
# Get next URL
- current_url = page_data.get("next")
+ current_url = page_payload["next"]
page_count += 1
return documents
diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.py
index 53d2ea44d..2b3477d66 100644
--- a/apps/python-sdk/firecrawl/v2/methods/batch.py
+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py
@@ -18,6 +18,30 @@ from ..utils.normalize import normalize_document_input
from ..types import CrawlErrorsResponse
+def _parse_batch_scrape_documents(data_list: Optional[List[Any]]) -> List[Document]:
+ documents: List[Document] = []
+ for doc in data_list or []:
+ if isinstance(doc, dict):
+ normalized = normalize_document_input(doc)
+ documents.append(Document(**normalized))
+ return documents
+
+
+def _parse_batch_scrape_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+
+ return {
+ "status": body.get("status"),
+ "completed": body.get("completed", 0),
+ "total": body.get("total", 0),
+ "credits_used": body.get("creditsUsed"),
+ "expires_at": body.get("expiresAt"),
+ "next": body.get("next"),
+ "data": _parse_batch_scrape_documents(body.get("data", []) or []),
+ }
+
+
def start_batch_scrape(
client: HttpClient,
urls: List[str],
@@ -104,37 +128,69 @@ def get_batch_scrape_status(
# Parse response
body = response.json()
- if not body.get("success"):
- raise Exception(body.get("error", "Unknown error occurred"))
-
- # Convert documents
- documents: List[Document] = []
- for doc in body.get("data", []) or []:
- if isinstance(doc, dict):
- normalized = normalize_document_input(doc)
- documents.append(Document(**normalized))
+ payload = _parse_batch_scrape_status_response(body)
+ documents = payload["data"]
# Handle pagination if requested
auto_paginate = pagination_config.auto_paginate if pagination_config else True
- if auto_paginate and body.get("next"):
+ if auto_paginate and payload["next"]:
documents = _fetch_all_batch_pages(
client,
- body.get("next"),
+ payload["next"],
documents,
pagination_config
)
return BatchScrapeJob(
- status=body.get("status"),
- completed=body.get("completed", 0),
- total=body.get("total", 0),
- credits_used=body.get("creditsUsed"),
- expires_at=body.get("expiresAt"),
- next=body.get("next") if not auto_paginate else None,
+ status=payload["status"],
+ completed=payload["completed"],
+ total=payload["total"],
+ credits_used=payload["credits_used"],
+ expires_at=payload["expires_at"],
+ next=payload["next"] if not auto_paginate else None,
data=documents,
)
+def get_batch_scrape_status_page(
+ client: HttpClient,
+ next_url: str,
+ *,
+ request_timeout: Optional[float] = None,
+) -> BatchScrapeJob:
+ """
+ Fetch a single page of batch scrape results using the provided next URL.
+
+ Args:
+ client: HTTP client instance
+ next_url: Opaque next URL from a prior batch scrape status response
+ request_timeout: Timeout (in seconds) for the HTTP request
+
+ Returns:
+ BatchScrapeJob with the page data and next URL (if any)
+
+ Raises:
+ Exception: If the request fails or returns an error response
+ """
+ response = client.get(next_url, timeout=request_timeout)
+
+ if not response.ok:
+ handle_response_error(response, "get batch scrape status page")
+
+ body = response.json()
+ payload = _parse_batch_scrape_status_response(body)
+
+ return BatchScrapeJob(
+ status=payload["status"],
+ completed=payload["completed"],
+ total=payload["total"],
+ credits_used=payload["credits_used"],
+ expires_at=payload["expires_at"],
+ next=payload["next"],
+ data=payload["data"],
+ )
+
+
def _fetch_all_batch_pages(
client: HttpClient,
next_url: str,
@@ -183,25 +239,24 @@ def _fetch_all_batch_pages(
break
page_data = response.json()
-
- if not page_data.get("success"):
+ try:
+ page_payload = _parse_batch_scrape_status_response(page_data)
+ except Exception:
break
# Add documents from this page
- for doc in page_data.get("data", []) or []:
- if isinstance(doc, dict):
- # Check max_results limit
- if max_results is not None and len(documents) >= max_results:
- break
- normalized = normalize_document_input(doc)
- documents.append(Document(**normalized))
+ for document in page_payload["data"]:
+ # Check max_results limit
+ if max_results is not None and len(documents) >= max_results:
+ break
+ documents.append(document)
# Check if we hit max_results limit after adding all docs from this page
if max_results is not None and len(documents) >= max_results:
break
# Get next URL
- current_url = page_data.get("next")
+ current_url = page_payload["next"]
page_count += 1
return documents
@@ -496,4 +551,4 @@ def get_batch_scrape_errors(client: HttpClient, job_id: str) -> CrawlErrorsRespo
"errors": payload.get("errors", []),
"robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
}
- return CrawlErrorsResponse(**normalized)
\ No newline at end of file
+ return CrawlErrorsResponse(**normalized)
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
index 5c91b02d4..1730fc726 100644
--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -106,6 +106,29 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:
return data
+def _parse_crawl_documents(data_list: Optional[List[Any]]) -> List[Document]:
+ documents: List[Document] = []
+ for doc_data in data_list or []:
+ if isinstance(doc_data, dict):
+ documents.append(Document(**normalize_document_input(doc_data)))
+ return documents
+
+
+def _parse_crawl_status_response(response_data: Dict[str, Any]) -> Dict[str, Any]:
+ if not response_data.get("success"):
+ raise Exception(response_data.get("error", "Unknown error occurred"))
+
+ return {
+ "status": response_data.get("status"),
+ "completed": response_data.get("completed", 0),
+ "total": response_data.get("total", 0),
+ "credits_used": response_data.get("creditsUsed", 0),
+ "expires_at": response_data.get("expiresAt"),
+ "next": response_data.get("next"),
+ "data": _parse_crawl_documents(response_data.get("data", [])),
+ }
+
+
def start_crawl(client: HttpClient, request: CrawlRequest) -> CrawlResponse:
"""
Start a crawl job for a website.
@@ -175,47 +198,74 @@ def get_crawl_status(
# Parse response
response_data = response.json()
- if response_data.get("success"):
- # The API returns status fields at the top level, not in a data field
-
- # Convert documents
- documents = []
- data_list = response_data.get("data", [])
- for doc_data in data_list:
- if isinstance(doc_data, str):
- # Handle case where API returns just URLs - this shouldn't happen for crawl
- # but we'll handle it gracefully
- continue
- else:
- documents.append(Document(**normalize_document_input(doc_data)))
-
- # Handle pagination if requested
- auto_paginate = pagination_config.auto_paginate if pagination_config else True
- if auto_paginate and response_data.get("next") and not (
- pagination_config
- and pagination_config.max_results is not None
- and len(documents) >= pagination_config.max_results
- ):
- documents = _fetch_all_pages(
- client,
- response_data.get("next"),
- documents,
- pagination_config,
- request_timeout=request_timeout,
- )
-
- # Create CrawlJob with current status and data
- return CrawlJob(
- status=response_data.get("status"),
- completed=response_data.get("completed", 0),
- total=response_data.get("total", 0),
- credits_used=response_data.get("creditsUsed", 0),
- expires_at=response_data.get("expiresAt"),
- next=response_data.get("next", None) if not auto_paginate else None,
- data=documents
+ payload = _parse_crawl_status_response(response_data)
+
+ documents = payload["data"]
+
+ # Handle pagination if requested
+ auto_paginate = pagination_config.auto_paginate if pagination_config else True
+ if auto_paginate and payload["next"] and not (
+ pagination_config
+ and pagination_config.max_results is not None
+ and len(documents) >= pagination_config.max_results
+ ):
+ documents = _fetch_all_pages(
+ client,
+ payload["next"],
+ documents,
+ pagination_config,
+ request_timeout=request_timeout,
)
- else:
- raise Exception(response_data.get("error", "Unknown error occurred"))
+
+ # Create CrawlJob with current status and data
+ return CrawlJob(
+ status=payload["status"],
+ completed=payload["completed"],
+ total=payload["total"],
+ credits_used=payload["credits_used"],
+ expires_at=payload["expires_at"],
+ next=payload["next"] if not auto_paginate else None,
+ data=documents,
+ )
+
+
+def get_crawl_status_page(
+ client: HttpClient,
+ next_url: str,
+ *,
+ request_timeout: Optional[float] = None,
+) -> CrawlJob:
+ """
+ Fetch a single page of crawl results using the provided next URL.
+
+ Args:
+ client: HTTP client instance
+ next_url: Opaque next URL from a prior crawl status response
+ request_timeout: Timeout (in seconds) for the HTTP request
+
+ Returns:
+ CrawlJob with the page data and next URL (if any)
+
+ Raises:
+ Exception: If the request fails or returns an error response
+ """
+ response = client.get(next_url, timeout=request_timeout)
+
+ if not response.ok:
+ handle_response_error(response, "get crawl status page")
+
+ response_data = response.json()
+ payload = _parse_crawl_status_response(response_data)
+
+ return CrawlJob(
+ status=payload["status"],
+ completed=payload["completed"],
+ total=payload["total"],
+ credits_used=payload["credits_used"],
+ expires_at=payload["expires_at"],
+ next=payload["next"],
+ data=payload["data"],
+ )
def _fetch_all_pages(
@@ -270,26 +320,24 @@ def _fetch_all_pages(
page_data = response.json()
- if not page_data.get("success"):
+ try:
+ page_payload = _parse_crawl_status_response(page_data)
+ except Exception:
break
# Add documents from this page
- data_list = page_data.get("data", [])
- for doc_data in data_list:
- if isinstance(doc_data, str):
- continue
- else:
- # Check max_results limit BEFORE adding each document
- if max_results is not None and len(documents) >= max_results:
- break
- documents.append(Document(**normalize_document_input(doc_data)))
+ for document in page_payload["data"]:
+ # Check max_results limit BEFORE adding each document
+ if max_results is not None and len(documents) >= max_results:
+ break
+ documents.append(document)
# Check if we hit max_results limit
if max_results is not None and len(documents) >= max_results:
break
# Get next URL
- current_url = page_data.get("next")
+ current_url = page_payload["next"]
page_count += 1
return documents