Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(python-sdk): wire timeout, max_retries, backoff_factor through to HTTP clients (#2872)
firecrawl-spring[bot]e1d600a8a7d0
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 071fdc456..907c9c134 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.15.1"+__version__ = "4.16.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex 11f5695b3..a1ca1ccb2 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -185,20 +185,36 @@ class Firecrawl:Provides a single entrypoint that exposes the latest API directly whilekeeping a feature-frozen v1 available for incremental migration."""-- def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):++ def __init__(+ self,+ api_key: str = None,+ api_url: str = "https://api.firecrawl.dev",+ timeout: float = None,+ max_retries: int = 3,+ backoff_factor: float = 0.5,+ ):"""Initialize the unified client.Args:api_key: Firecrawl API key (or set ``FIRECRAWL_API_KEY``)api_url: Base API URL (defaults to production)+ timeout: Default request timeout in seconds for all HTTP requests+ max_retries: Maximum number of retries for failed requests (default: 3)+ backoff_factor: Exponential backoff factor for retries (default: 0.5)"""self.api_key = api_keyself.api_url = api_url-+# Initialize version-specific clientsself._v1_client = V1FirecrawlApp(api_key=api_key, api_url=api_url) if V1FirecrawlApp else None- self._v2_client = V2FirecrawlClient(api_key=api_key, api_url=api_url) if V2FirecrawlClient else None+ self._v2_client = V2FirecrawlClient(+ api_key=api_key,+ api_url=api_url,+ timeout=timeout,+ max_retries=max_retries,+ backoff_factor=backoff_factor,+ ) if V2FirecrawlClient else None# Create version-specific proxiesself.v1 = V1Proxy(self._v1_client) if self._v1_client else None@@ -249,13 +265,26 @@ class Firecrawl:class AsyncFirecrawl:"""Async unified Firecrawl client (v2 by default, v1 under ``.v1``)."""- def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):+ def __init__(+ self,+ api_key: str = None,+ api_url: str = "https://api.firecrawl.dev",+ timeout: float = None,+ max_retries: int = 3,+ backoff_factor: float = 0.5,+ ):self.api_key = api_keyself.api_url = api_url-+# Initialize version-specific clientsself._v1_client = AsyncV1FirecrawlApp(api_key=api_key, api_url=api_url) if AsyncV1FirecrawlApp else None- self._v2_client = AsyncFirecrawlClient(api_key=api_key, api_url=api_url) if AsyncFirecrawlClient else None+ self._v2_client = AsyncFirecrawlClient(+ api_key=api_key,+ api_url=api_url,+ timeout=timeout,+ max_retries=max_retries,+ backoff_factor=backoff_factor,+ ) if AsyncFirecrawlClient else None# Create version-specific proxiesself.v1 = AsyncV1Proxy(self._v1_client) if self._v1_client else Nonediff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex cf2e6db4e..561969b88 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -100,7 +100,13 @@ class FirecrawlClient:backoff_factor=backoff_factor)- self.http_client = HttpClient(api_key, api_url)+ self.http_client = HttpClient(+ api_key,+ api_url,+ timeout=timeout,+ max_retries=max_retries,+ backoff_factor=backoff_factor,+ )def scrape(self,diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 3caec4efa..20fc9c5b2 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -55,13 +55,32 @@ class AsyncFirecrawlClient:def _is_cloud_service(url: str) -> bool:return "api.firecrawl.dev" in url.lower()- def __init__(self, api_key: Optional[str] = None, api_url: str = "https://api.firecrawl.dev"):+ def __init__(+ self,+ api_key: Optional[str] = None,+ api_url: str = "https://api.firecrawl.dev",+ timeout: Optional[float] = None,+ max_retries: int = 3,+ backoff_factor: float = 0.5,+ ):if api_key is None:api_key = os.getenv("FIRECRAWL_API_KEY")if self._is_cloud_service(api_url) and not api_key:raise ValueError("API key is required for the cloud API. Set FIRECRAWL_API_KEY or pass api_key.")- self.http_client = HttpClient(api_key, api_url)- self.async_http_client = AsyncHttpClient(api_key, api_url)+ self.http_client = HttpClient(+ api_key,+ api_url,+ timeout=timeout,+ max_retries=max_retries,+ backoff_factor=backoff_factor,+ )+ self.async_http_client = AsyncHttpClient(+ api_key,+ api_url,+ timeout=timeout,+ max_retries=max_retries,+ backoff_factor=backoff_factor,+ )# Scrapeasync def scrape(diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.pyindex 65b84d995..3bb134e11 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py@@ -13,9 +13,19 @@ version = get_version()class HttpClient:"""HTTP client with retry logic and error handling."""- def __init__(self, api_key: Optional[str], api_url: str):+ def __init__(+ self,+ api_key: Optional[str],+ api_url: str,+ timeout: Optional[float] = None,+ max_retries: int = 3,+ backoff_factor: float = 0.5,+ ):self.api_key = api_keyself.api_url = api_url+ self.timeout = timeout+ self.max_retries = max_retries+ self.backoff_factor = backoff_factordef _build_url(self, endpoint: str) -> str:base = urlparse(self.api_url)@@ -59,20 +69,27 @@ class HttpClient:data: Dict[str, Any],headers: Optional[Dict[str, str]] = None,timeout: Optional[float] = None,- retries: int = 3,- backoff_factor: float = 0.5+ retries: Optional[int] = None,+ backoff_factor: Optional[float] = None,) -> requests.Response:"""Make a POST request with retry logic."""if headers is None:headers = self._prepare_headers()+ if timeout is None:+ timeout = self.timeout+ if retries is None:+ retries = self.max_retries+ if backoff_factor is None:+ backoff_factor = self.backoff_factordata['origin'] = f'python-sdk@{version}'-+url = self._build_url(endpoint)-+last_exception = None-- for attempt in range(retries):+ num_attempts = max(1, retries)++ for attempt in range(num_attempts):try:response = requests.post(url,@@ -82,18 +99,18 @@ class HttpClient:)if response.status_code == 502:- if attempt < retries - 1:+ if attempt < num_attempts - 1:time.sleep(backoff_factor * (2 ** attempt))continue-+return response-+except requests.RequestException as e:last_exception = e- if attempt == retries - 1:+ if attempt == num_attempts - 1:raise etime.sleep(backoff_factor * (2 ** attempt))-+# This should never be reached due to the exception handling aboveraise last_exception or Exception("Unexpected error in POST request")@@ -102,38 +119,45 @@ class HttpClient:endpoint: str,headers: Optional[Dict[str, str]] = None,timeout: Optional[float] = None,- retries: int = 3,- backoff_factor: float = 0.5+ retries: Optional[int] = None,+ backoff_factor: Optional[float] = None,) -> requests.Response:"""Make a GET request with retry logic."""if headers is None:headers = self._prepare_headers()+ if timeout is None:+ timeout = self.timeout+ if retries is None:+ retries = self.max_retries+ if backoff_factor is None:+ backoff_factor = self.backoff_factorurl = self._build_url(endpoint)-+last_exception = None-- for attempt in range(retries):+ num_attempts = max(1, retries)++ for attempt in range(num_attempts):try:response = requests.get(url,headers=headers,timeout=timeout)-+if response.status_code == 502:- if attempt < retries - 1:+ if attempt < num_attempts - 1:time.sleep(backoff_factor * (2 ** attempt))continue-+return response-+except requests.RequestException as e:last_exception = e- if attempt == retries - 1:+ if attempt == num_attempts - 1:raise etime.sleep(backoff_factor * (2 ** attempt))-+# This should never be reached due to the exception handling aboveraise last_exception or Exception("Unexpected error in GET request")@@ -142,37 +166,44 @@ class HttpClient:endpoint: str,headers: Optional[Dict[str, str]] = None,timeout: Optional[float] = None,- retries: int = 3,- backoff_factor: float = 0.5+ retries: Optional[int] = None,+ backoff_factor: Optional[float] = None,) -> requests.Response:"""Make a DELETE request with retry logic."""if headers is None:headers = self._prepare_headers()-+ if timeout is None:+ timeout = self.timeout+ if retries is None:+ retries = self.max_retries+ if backoff_factor is None:+ backoff_factor = self.backoff_factor+url = self._build_url(endpoint)-+last_exception = None-- for attempt in range(retries):+ num_attempts = max(1, retries)++ for attempt in range(num_attempts):try:response = requests.delete(url,headers=headers,timeout=timeout)-+if response.status_code == 502:- if attempt < retries - 1:+ if attempt < num_attempts - 1:time.sleep(backoff_factor * (2 ** attempt))continue-+return response-+except requests.RequestException as e:last_exception = e- if attempt == retries - 1:+ if attempt == num_attempts - 1:raise etime.sleep(backoff_factor * (2 ** attempt))-+# This should never be reached due to the exception handling aboveraise last_exception or Exception("Unexpected error in DELETE request")\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.pyindex 639b9891a..53df41fc7 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py@@ -1,3 +1,4 @@+import asyncioimport httpxfrom typing import Optional, Dict, Anyfrom .get_version import get_version@@ -6,9 +7,20 @@ version = get_version()class AsyncHttpClient:- def __init__(self, api_key: Optional[str], api_url: str):+ def __init__(+ self,+ api_key: Optional[str],+ api_url: str,+ timeout: Optional[float] = None,+ max_retries: int = 3,+ backoff_factor: float = 0.5,+ ):self.api_key = api_keyself.api_url = api_url+ self.timeout = timeout+ self.max_retries = max_retries+ self.backoff_factor = backoff_factor+headers = {"Content-Type": "application/json",}@@ -37,33 +49,115 @@ class AsyncHttpClient:data: Dict[str, Any],headers: Optional[Dict[str, str]] = None,timeout: Optional[float] = None,+ retries: Optional[int] = None,+ backoff_factor: Optional[float] = None,) -> httpx.Response:+ if timeout is None:+ timeout = self.timeout+ if retries is None:+ retries = self.max_retries+ if backoff_factor is None:+ backoff_factor = self.backoff_factor+payload = dict(data)payload["origin"] = f"python-sdk@{version}"- return await self._client.post(- endpoint,- json=payload,- headers={**self._headers(), **(headers or {})},- timeout=timeout,- )++ last_exception = None+ num_attempts = max(1, retries)++ for attempt in range(num_attempts):+ try:+ response = await self._client.post(+ endpoint,+ json=payload,+ headers={**self._headers(), **(headers or {})},+ timeout=timeout,+ )+ if response.status_code == 502:+ if attempt < num_attempts - 1:+ await asyncio.sleep(backoff_factor * (2 ** attempt))+ continue+ return response+ except httpx.HTTPError as e:+ last_exception = e+ if attempt == num_attempts - 1:+ raise e+ await asyncio.sleep(backoff_factor * (2 ** attempt))++ raise last_exception or Exception("Unexpected error in POST request")async def get(self,endpoint: str,headers: Optional[Dict[str, str]] = None,timeout: Optional[float] = None,+ retries: Optional[int] = None,+ backoff_factor: Optional[float] = None,) -> httpx.Response:- return await self._client.get(- endpoint, headers={**self._headers(), **(headers or {})}, timeout=timeout- )+ if timeout is None:+ timeout = self.timeout+ if retries is None:+ retries = self.max_retries+ if backoff_factor is None:+ backoff_factor = self.backoff_factor++ last_exception = None+ num_attempts = max(1, retries)++ for attempt in range(num_attempts):+ try:+ response = await self._client.get(+ endpoint,+ headers={**self._headers(), **(headers or {})},+ timeout=timeout,+ )+ if response.status_code == 502:+ if attempt < num_attempts - 1:+ await asyncio.sleep(backoff_factor * (2 ** attempt))+ continue+ return response+ except httpx.HTTPError as e:+ last_exception = e+ if attempt == num_attempts - 1:+ raise e+ await asyncio.sleep(backoff_factor * (2 ** attempt))++ raise last_exception or Exception("Unexpected error in GET request")async def delete(self,endpoint: str,headers: Optional[Dict[str, str]] = None,timeout: Optional[float] = None,+ retries: Optional[int] = None,+ backoff_factor: Optional[float] = None,) -> httpx.Response:- return await self._client.delete(- endpoint, headers={**self._headers(), **(headers or {})}, timeout=timeout- )+ if timeout is None:+ timeout = self.timeout+ if retries is None:+ retries = self.max_retries+ if backoff_factor is None:+ backoff_factor = self.backoff_factor++ last_exception = None+ num_attempts = max(1, retries)++ for attempt in range(num_attempts):+ try:+ response = await self._client.delete(+ endpoint,+ headers={**self._headers(), **(headers or {})},+ timeout=timeout,+ )+ if response.status_code == 502:+ if attempt < num_attempts - 1:+ await asyncio.sleep(backoff_factor * (2 ** attempt))+ continue+ return response+ except httpx.HTTPError as e:+ last_exception = e+ if attempt == num_attempts - 1:+ raise e+ await asyncio.sleep(backoff_factor * (2 ** attempt))+ raise last_exception or Exception("Unexpected error in DELETE request")