Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(python-sdk): wire timeout, max_retries, backoff_factor through to HTTP clients (#2872)

firecrawl-spring[bot]e1d600a8a7d0
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 071fdc456..907c9c134 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.15.1"
+__version__ = "4.16.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.py
index 11f5695b3..a1ca1ccb2 100644
--- a/apps/python-sdk/firecrawl/client.py
+++ b/apps/python-sdk/firecrawl/client.py
@@ -185,20 +185,36 @@ class Firecrawl:
Provides a single entrypoint that exposes the latest API directly while
keeping a feature-frozen v1 available for incremental migration.
"""
-
- def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):
+
+ def __init__(
+ self,
+ api_key: str = None,
+ api_url: str = "https://api.firecrawl.dev",
+ timeout: float = None,
+ max_retries: int = 3,
+ backoff_factor: float = 0.5,
+ ):
"""Initialize the unified client.
Args:
api_key: Firecrawl API key (or set ``FIRECRAWL_API_KEY``)
api_url: Base API URL (defaults to production)
+ timeout: Default request timeout in seconds for all HTTP requests
+ max_retries: Maximum number of retries for failed requests (default: 3)
+ backoff_factor: Exponential backoff factor for retries (default: 0.5)
"""
self.api_key = api_key
self.api_url = api_url
-
+
# Initialize version-specific clients
self._v1_client = V1FirecrawlApp(api_key=api_key, api_url=api_url) if V1FirecrawlApp else None
- self._v2_client = V2FirecrawlClient(api_key=api_key, api_url=api_url) if V2FirecrawlClient else None
+ self._v2_client = V2FirecrawlClient(
+ api_key=api_key,
+ api_url=api_url,
+ timeout=timeout,
+ max_retries=max_retries,
+ backoff_factor=backoff_factor,
+ ) if V2FirecrawlClient else None
# Create version-specific proxies
self.v1 = V1Proxy(self._v1_client) if self._v1_client else None
@@ -249,13 +265,26 @@ class Firecrawl:
class AsyncFirecrawl:
"""Async unified Firecrawl client (v2 by default, v1 under ``.v1``)."""
- def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):
+ def __init__(
+ self,
+ api_key: str = None,
+ api_url: str = "https://api.firecrawl.dev",
+ timeout: float = None,
+ max_retries: int = 3,
+ backoff_factor: float = 0.5,
+ ):
self.api_key = api_key
self.api_url = api_url
-
+
# Initialize version-specific clients
self._v1_client = AsyncV1FirecrawlApp(api_key=api_key, api_url=api_url) if AsyncV1FirecrawlApp else None
- self._v2_client = AsyncFirecrawlClient(api_key=api_key, api_url=api_url) if AsyncFirecrawlClient else None
+ self._v2_client = AsyncFirecrawlClient(
+ api_key=api_key,
+ api_url=api_url,
+ timeout=timeout,
+ max_retries=max_retries,
+ backoff_factor=backoff_factor,
+ ) if AsyncFirecrawlClient else None
# Create version-specific proxies
self.v1 = AsyncV1Proxy(self._v1_client) if self._v1_client else None
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index cf2e6db4e..561969b88 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -100,7 +100,13 @@ class FirecrawlClient:
backoff_factor=backoff_factor
)
- self.http_client = HttpClient(api_key, api_url)
+ self.http_client = HttpClient(
+ api_key,
+ api_url,
+ timeout=timeout,
+ max_retries=max_retries,
+ backoff_factor=backoff_factor,
+ )
def scrape(
self,
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index 3caec4efa..20fc9c5b2 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -55,13 +55,32 @@ class AsyncFirecrawlClient:
def _is_cloud_service(url: str) -> bool:
return "api.firecrawl.dev" in url.lower()
- def __init__(self, api_key: Optional[str] = None, api_url: str = "https://api.firecrawl.dev"):
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ api_url: str = "https://api.firecrawl.dev",
+ timeout: Optional[float] = None,
+ max_retries: int = 3,
+ backoff_factor: float = 0.5,
+ ):
if api_key is None:
api_key = os.getenv("FIRECRAWL_API_KEY")
if self._is_cloud_service(api_url) and not api_key:
raise ValueError("API key is required for the cloud API. Set FIRECRAWL_API_KEY or pass api_key.")
- self.http_client = HttpClient(api_key, api_url)
- self.async_http_client = AsyncHttpClient(api_key, api_url)
+ self.http_client = HttpClient(
+ api_key,
+ api_url,
+ timeout=timeout,
+ max_retries=max_retries,
+ backoff_factor=backoff_factor,
+ )
+ self.async_http_client = AsyncHttpClient(
+ api_key,
+ api_url,
+ timeout=timeout,
+ max_retries=max_retries,
+ backoff_factor=backoff_factor,
+ )
# Scrape
async def scrape(
diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.py
index 65b84d995..3bb134e11 100644
--- a/apps/python-sdk/firecrawl/v2/utils/http_client.py
+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py
@@ -13,9 +13,19 @@ version = get_version()
class HttpClient:
"""HTTP client with retry logic and error handling."""
- def __init__(self, api_key: Optional[str], api_url: str):
+ def __init__(
+ self,
+ api_key: Optional[str],
+ api_url: str,
+ timeout: Optional[float] = None,
+ max_retries: int = 3,
+ backoff_factor: float = 0.5,
+ ):
self.api_key = api_key
self.api_url = api_url
+ self.timeout = timeout
+ self.max_retries = max_retries
+ self.backoff_factor = backoff_factor
def _build_url(self, endpoint: str) -> str:
base = urlparse(self.api_url)
@@ -59,20 +69,27 @@ class HttpClient:
data: Dict[str, Any],
headers: Optional[Dict[str, str]] = None,
timeout: Optional[float] = None,
- retries: int = 3,
- backoff_factor: float = 0.5
+ retries: Optional[int] = None,
+ backoff_factor: Optional[float] = None,
) -> requests.Response:
"""Make a POST request with retry logic."""
if headers is None:
headers = self._prepare_headers()
+ if timeout is None:
+ timeout = self.timeout
+ if retries is None:
+ retries = self.max_retries
+ if backoff_factor is None:
+ backoff_factor = self.backoff_factor
data['origin'] = f'python-sdk@{version}'
-
+
url = self._build_url(endpoint)
-
+
last_exception = None
-
- for attempt in range(retries):
+ num_attempts = max(1, retries)
+
+ for attempt in range(num_attempts):
try:
response = requests.post(
url,
@@ -82,18 +99,18 @@ class HttpClient:
)
if response.status_code == 502:
- if attempt < retries - 1:
+ if attempt < num_attempts - 1:
time.sleep(backoff_factor * (2 ** attempt))
continue
-
+
return response
-
+
except requests.RequestException as e:
last_exception = e
- if attempt == retries - 1:
+ if attempt == num_attempts - 1:
raise e
time.sleep(backoff_factor * (2 ** attempt))
-
+
# This should never be reached due to the exception handling above
raise last_exception or Exception("Unexpected error in POST request")
@@ -102,38 +119,45 @@ class HttpClient:
endpoint: str,
headers: Optional[Dict[str, str]] = None,
timeout: Optional[float] = None,
- retries: int = 3,
- backoff_factor: float = 0.5
+ retries: Optional[int] = None,
+ backoff_factor: Optional[float] = None,
) -> requests.Response:
"""Make a GET request with retry logic."""
if headers is None:
headers = self._prepare_headers()
+ if timeout is None:
+ timeout = self.timeout
+ if retries is None:
+ retries = self.max_retries
+ if backoff_factor is None:
+ backoff_factor = self.backoff_factor
url = self._build_url(endpoint)
-
+
last_exception = None
-
- for attempt in range(retries):
+ num_attempts = max(1, retries)
+
+ for attempt in range(num_attempts):
try:
response = requests.get(
url,
headers=headers,
timeout=timeout
)
-
+
if response.status_code == 502:
- if attempt < retries - 1:
+ if attempt < num_attempts - 1:
time.sleep(backoff_factor * (2 ** attempt))
continue
-
+
return response
-
+
except requests.RequestException as e:
last_exception = e
- if attempt == retries - 1:
+ if attempt == num_attempts - 1:
raise e
time.sleep(backoff_factor * (2 ** attempt))
-
+
# This should never be reached due to the exception handling above
raise last_exception or Exception("Unexpected error in GET request")
@@ -142,37 +166,44 @@ class HttpClient:
endpoint: str,
headers: Optional[Dict[str, str]] = None,
timeout: Optional[float] = None,
- retries: int = 3,
- backoff_factor: float = 0.5
+ retries: Optional[int] = None,
+ backoff_factor: Optional[float] = None,
) -> requests.Response:
"""Make a DELETE request with retry logic."""
if headers is None:
headers = self._prepare_headers()
-
+ if timeout is None:
+ timeout = self.timeout
+ if retries is None:
+ retries = self.max_retries
+ if backoff_factor is None:
+ backoff_factor = self.backoff_factor
+
url = self._build_url(endpoint)
-
+
last_exception = None
-
- for attempt in range(retries):
+ num_attempts = max(1, retries)
+
+ for attempt in range(num_attempts):
try:
response = requests.delete(
url,
headers=headers,
timeout=timeout
)
-
+
if response.status_code == 502:
- if attempt < retries - 1:
+ if attempt < num_attempts - 1:
time.sleep(backoff_factor * (2 ** attempt))
continue
-
+
return response
-
+
except requests.RequestException as e:
last_exception = e
- if attempt == retries - 1:
+ if attempt == num_attempts - 1:
raise e
time.sleep(backoff_factor * (2 ** attempt))
-
+
# This should never be reached due to the exception handling above
raise last_exception or Exception("Unexpected error in DELETE request")
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
index 639b9891a..53df41fc7 100644
--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
@@ -1,3 +1,4 @@
+import asyncio
import httpx
from typing import Optional, Dict, Any
from .get_version import get_version
@@ -6,9 +7,20 @@ version = get_version()
class AsyncHttpClient:
- def __init__(self, api_key: Optional[str], api_url: str):
+ def __init__(
+ self,
+ api_key: Optional[str],
+ api_url: str,
+ timeout: Optional[float] = None,
+ max_retries: int = 3,
+ backoff_factor: float = 0.5,
+ ):
self.api_key = api_key
self.api_url = api_url
+ self.timeout = timeout
+ self.max_retries = max_retries
+ self.backoff_factor = backoff_factor
+
headers = {
"Content-Type": "application/json",
}
@@ -37,33 +49,115 @@ class AsyncHttpClient:
data: Dict[str, Any],
headers: Optional[Dict[str, str]] = None,
timeout: Optional[float] = None,
+ retries: Optional[int] = None,
+ backoff_factor: Optional[float] = None,
) -> httpx.Response:
+ if timeout is None:
+ timeout = self.timeout
+ if retries is None:
+ retries = self.max_retries
+ if backoff_factor is None:
+ backoff_factor = self.backoff_factor
+
payload = dict(data)
payload["origin"] = f"python-sdk@{version}"
- return await self._client.post(
- endpoint,
- json=payload,
- headers={**self._headers(), **(headers or {})},
- timeout=timeout,
- )
+
+ last_exception = None
+ num_attempts = max(1, retries)
+
+ for attempt in range(num_attempts):
+ try:
+ response = await self._client.post(
+ endpoint,
+ json=payload,
+ headers={**self._headers(), **(headers or {})},
+ timeout=timeout,
+ )
+ if response.status_code == 502:
+ if attempt < num_attempts - 1:
+ await asyncio.sleep(backoff_factor * (2 ** attempt))
+ continue
+ return response
+ except httpx.HTTPError as e:
+ last_exception = e
+ if attempt == num_attempts - 1:
+ raise e
+ await asyncio.sleep(backoff_factor * (2 ** attempt))
+
+ raise last_exception or Exception("Unexpected error in POST request")
async def get(
self,
endpoint: str,
headers: Optional[Dict[str, str]] = None,
timeout: Optional[float] = None,
+ retries: Optional[int] = None,
+ backoff_factor: Optional[float] = None,
) -> httpx.Response:
- return await self._client.get(
- endpoint, headers={**self._headers(), **(headers or {})}, timeout=timeout
- )
+ if timeout is None:
+ timeout = self.timeout
+ if retries is None:
+ retries = self.max_retries
+ if backoff_factor is None:
+ backoff_factor = self.backoff_factor
+
+ last_exception = None
+ num_attempts = max(1, retries)
+
+ for attempt in range(num_attempts):
+ try:
+ response = await self._client.get(
+ endpoint,
+ headers={**self._headers(), **(headers or {})},
+ timeout=timeout,
+ )
+ if response.status_code == 502:
+ if attempt < num_attempts - 1:
+ await asyncio.sleep(backoff_factor * (2 ** attempt))
+ continue
+ return response
+ except httpx.HTTPError as e:
+ last_exception = e
+ if attempt == num_attempts - 1:
+ raise e
+ await asyncio.sleep(backoff_factor * (2 ** attempt))
+
+ raise last_exception or Exception("Unexpected error in GET request")
async def delete(
self,
endpoint: str,
headers: Optional[Dict[str, str]] = None,
timeout: Optional[float] = None,
+ retries: Optional[int] = None,
+ backoff_factor: Optional[float] = None,
) -> httpx.Response:
- return await self._client.delete(
- endpoint, headers={**self._headers(), **(headers or {})}, timeout=timeout
- )
+ if timeout is None:
+ timeout = self.timeout
+ if retries is None:
+ retries = self.max_retries
+ if backoff_factor is None:
+ backoff_factor = self.backoff_factor
+
+ last_exception = None
+ num_attempts = max(1, retries)
+
+ for attempt in range(num_attempts):
+ try:
+ response = await self._client.delete(
+ endpoint,
+ headers={**self._headers(), **(headers or {})},
+ timeout=timeout,
+ )
+ if response.status_code == 502:
+ if attempt < num_attempts - 1:
+ await asyncio.sleep(backoff_factor * (2 ** attempt))
+ continue
+ return response
+ except httpx.HTTPError as e:
+ last_exception = e
+ if attempt == num_attempts - 1:
+ raise e
+ await asyncio.sleep(backoff_factor * (2 ** attempt))
+ raise last_exception or Exception("Unexpected error in DELETE request")