Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

better documentation

Abimael Martell69f7e588be55
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index bbe182346..005921fb7 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -260,8 +260,8 @@ class FirecrawlClient:
scrape_options: Page scraping configuration
zero_data_retention: Whether to delete data after 24 hours
poll_interval: Seconds between status checks
- timeout: Maximum seconds to wait (None for no timeout)
- request_timeout: Optional timeout (in seconds) for each status request
+ timeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout)
+ request_timeout: Timeout (in seconds) for each individual HTTP request, including pagination requests when fetching results. If there are multiple pages, each page request gets this timeout
Returns:
CrawlJob when job completes
@@ -383,7 +383,9 @@ class FirecrawlClient:
Args:
job_id: ID of the crawl job
pagination_config: Optional configuration for pagination behavior
- request_timeout: Optional timeout (in seconds) for each status request
+ request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination
+ is enabled (default) and there are multiple pages of results, this timeout applies to
+ each page request separately, not to the entire operation
Returns:
CrawlJob with current status and data
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index ab804cdaf..75d51b3d2 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -92,8 +92,8 @@ class AsyncFirecrawlClient:
Args:
job_id (str): The ID of the crawl job to poll.
poll_interval (int, optional): Number of seconds to wait between polling attempts. Defaults to 2.
- timeout (Optional[int], optional): Maximum number of seconds to wait before timing out. If None, waits indefinitely. Defaults to None.
- request_timeout (Optional[float], optional): Timeout in seconds for each individual status request. If None, no per-request timeout is set. Defaults to None.
+ timeout (Optional[int], optional): Maximum number of seconds to wait for the entire crawl job to complete before timing out. If None, waits indefinitely. Defaults to None.
+ request_timeout (Optional[float], optional): Timeout (in seconds) for each individual HTTP request, including pagination requests when fetching results. If there are multiple pages, each page request gets this timeout. If None, no per-request timeout is set. Defaults to None.
Returns:
CrawlJob: The final status of the crawl job when it reaches a terminal state.
@@ -142,6 +142,22 @@ class AsyncFirecrawlClient:
*,
request_timeout: Optional[float] = None,
) -> CrawlJob:
+ """
+ Get the status of a crawl job.
+
+ Args:
+ job_id: ID of the crawl job
+ pagination_config: Optional configuration for pagination behavior
+ request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination
+ is enabled (default) and there are multiple pages of results, this timeout applies to
+ each page request separately, not to the entire operation
+
+ Returns:
+ CrawlJob with current status and data
+
+ Raises:
+ Exception: If the status check fails
+ """
return await async_crawl.get_crawl_status(
self.async_http_client,
job_id,
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
index 89d6704e1..5d1d78830 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
@@ -100,7 +100,9 @@ async def get_crawl_status(
client: Async HTTP client instance
job_id: ID of the crawl job
pagination_config: Optional configuration for pagination limits
- request_timeout: Optional timeout (in seconds) for the underlying HTTP request
+ request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination
+ is enabled (default) and there are multiple pages of results, this timeout applies to
+ each page request separately, not to the entire operation
Returns:
CrawlJob with job information
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
index 928f6e148..5c91b02d4 100644
--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -155,7 +155,9 @@ def get_crawl_status(
client: HTTP client instance
job_id: ID of the crawl job
pagination_config: Optional configuration for pagination behavior
- request_timeout: Optional timeout (in seconds) for the underlying HTTP request
+ request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination
+ is enabled (default) and there are multiple pages of results, this timeout applies to
+ each page request separately, not to the entire operation
Returns:
CrawlJob with current status and data
@@ -377,8 +379,9 @@ def crawl(
client: HTTP client instance
request: CrawlRequest containing URL and options
poll_interval: Seconds between status checks
- timeout: Maximum seconds to wait (None for no timeout)
- request_timeout: Optional timeout (in seconds) for each status request
+ timeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout)
+ request_timeout: Timeout (in seconds) for each individual HTTP request, including pagination
+ requests when fetching results. If there are multiple pages, each page request gets this timeout
Returns:
CrawlJob when job completes