Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
exponential retries improved
mertunsall84be050ed7f2
diff --git a/browser_use/llm/browser_use/chat.py b/browser_use/llm/browser_use/chat.pyindex b10aafdd4..cbd0afa5a 100644--- a/browser_use/llm/browser_use/chat.py+++ b/browser_use/llm/browser_use/chat.py@@ -5,8 +5,10 @@ This wraps the BaseChatModel protocol and sends requests to the browser-use cloufor optimized browser automation LLM inference."""+import asyncioimport loggingimport os+import randomfrom typing import TypeVar, overloadimport httpx@@ -21,6 +23,9 @@ T = TypeVar('T', bound=BaseModel)logger = logging.getLogger(__name__)+# HTTP status codes that should trigger a retry+RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}+class ChatBrowserUse(BaseChatModel):"""@@ -42,6 +47,9 @@ class ChatBrowserUse(BaseChatModel):api_key: str | None = None,base_url: str | None = None,timeout: float = 120.0,+ max_retries: int = 5,+ retry_base_delay: float = 1.0,+ retry_max_delay: float = 60.0,**kwargs,):"""@@ -52,6 +60,9 @@ class ChatBrowserUse(BaseChatModel):api_key: API key for browser-use cloud. Defaults to BROWSER_USE_API_KEY env var.base_url: Base URL for the API. Defaults to BROWSER_USE_LLM_URL env var or production URL.timeout: Request timeout in seconds.+ max_retries: Maximum number of retries for transient errors (default: 5).+ retry_base_delay: Base delay in seconds for exponential backoff (default: 1.0).+ retry_max_delay: Maximum delay in seconds between retries (default: 60.0)."""# Validate model namevalid_models = ['bu-latest', 'bu-1-0']@@ -63,6 +74,9 @@ class ChatBrowserUse(BaseChatModel):self.api_key = api_key or os.getenv('BROWSER_USE_API_KEY')self.base_url = base_url or os.getenv('BROWSER_USE_LLM_URL', 'https://llm.api.browser-use.com')self.timeout = timeout+ self.max_retries = max_retries+ self.retry_base_delay = retry_base_delay+ self.retry_max_delay = retry_max_delayif not self.api_key:raise ValueError(@@ -120,83 +134,127 @@ class ChatBrowserUse(BaseChatModel):if output_format is not None:payload['output_format'] = output_format.model_json_schema()- # Make API request- async with httpx.AsyncClient(timeout=self.timeout) as client:- try:- response = await client.post(- f'{self.base_url}/v1/chat/completions',- json=payload,- headers={- 'Authorization': f'Bearer {self.api_key}',- 'Content-Type': 'application/json',- },- )- response.raise_for_status()- result = response.json()+ last_error: Exception | None = None+ # Retry loop with exponential backoff+ for attempt in range(self.max_retries):+ try:+ result = await self._make_request(payload)+ breakexcept httpx.HTTPStatusError as e:- error_detail = ''- try:- error_data = e.response.json()- error_detail = error_data.get('detail', str(e))- except Exception:- error_detail = str(e)-- error_msg = ''- if e.response.status_code == 401:- error_msg = f'Invalid API key. {error_detail}'- elif e.response.status_code == 402:- error_msg = f'Insufficient credits. {error_detail}'- else:- error_msg = f'API request failed: {error_detail}'-- raise ValueError(error_msg)-- except httpx.TimeoutException:- error_msg = f'Request timed out after {self.timeout}s'- raise ValueError(error_msg)+ last_error = e+ status_code = e.response.status_code++ # Check if this is a retryable error+ if status_code in RETRYABLE_STATUS_CODES and attempt < self.max_retries - 1:+ delay = min(self.retry_base_delay * (2**attempt), self.retry_max_delay)+ jitter = random.uniform(0, delay * 0.1)+ total_delay = delay + jitter+ logger.warning(+ f'⚠️ Got {status_code} error, retrying in {total_delay:.1f}s... (attempt {attempt + 1}/{self.max_retries})'+ )+ await asyncio.sleep(total_delay)+ continue++ # Non-retryable HTTP error or exhausted retries+ self._raise_http_error(e)++ except (httpx.TimeoutException, httpx.ConnectError) as e:+ last_error = e+ # Network errors are retryable+ if attempt < self.max_retries - 1:+ delay = min(self.retry_base_delay * (2**attempt), self.retry_max_delay)+ jitter = random.uniform(0, delay * 0.1)+ total_delay = delay + jitter+ error_type = 'timeout' if isinstance(e, httpx.TimeoutException) else 'connection error'+ logger.warning(+ f'⚠️ Got {error_type}, retrying in {total_delay:.1f}s... (attempt {attempt + 1}/{self.max_retries})'+ )+ await asyncio.sleep(total_delay)+ continue++ # Exhausted retries+ if isinstance(e, httpx.TimeoutException):+ raise ValueError(f'Request timed out after {self.timeout}s (retried {self.max_retries} times)')+ raise ValueError(f'Failed to connect to browser-use API after {self.max_retries} attempts: {e}')except Exception as e:- error_msg = f'Failed to connect to browser-use API: {e}'- raise ValueError(error_msg)-- # Parse response - server returns structured data as dict- if output_format is not None:- # Server returns structured data as a dict, validate it- completion_data = result['completion']- logger.debug(- f'📥 Got structured data from service: {list(completion_data.keys()) if isinstance(completion_data, dict) else type(completion_data)}'- )+ raise ValueError(f'Failed to connect to browser-use API: {e}')+ else:+ # Loop completed without break (all retries exhausted)+ if last_error is not None:+ if isinstance(last_error, httpx.HTTPStatusError):+ self._raise_http_error(last_error)+ raise ValueError(f'Request failed after {self.max_retries} attempts: {last_error}')+ raise RuntimeError('Retry loop completed without return or exception')++ # Parse response - server returns structured data as dict+ if output_format is not None:+ # Server returns structured data as a dict, validate it+ completion_data = result['completion']+ logger.debug(+ f'📥 Got structured data from service: {list(completion_data.keys()) if isinstance(completion_data, dict) else type(completion_data)}'+ )- # Convert action dicts to ActionModel instances if needed- # llm-use returns dicts to avoid validation with empty ActionModel- if isinstance(completion_data, dict) and 'action' in completion_data:- actions = completion_data['action']- if actions and isinstance(actions[0], dict):- from typing import get_args+ # Convert action dicts to ActionModel instances if needed+ # llm-use returns dicts to avoid validation with empty ActionModel+ if isinstance(completion_data, dict) and 'action' in completion_data:+ actions = completion_data['action']+ if actions and isinstance(actions[0], dict):+ from typing import get_args- # Get ActionModel type from output_format- action_model_type = get_args(output_format.model_fields['action'].annotation)[0]+ # Get ActionModel type from output_format+ action_model_type = get_args(output_format.model_fields['action'].annotation)[0]- # Convert dicts to ActionModel instances- completion_data['action'] = [action_model_type.model_validate(action_dict) for action_dict in actions]+ # Convert dicts to ActionModel instances+ completion_data['action'] = [action_model_type.model_validate(action_dict) for action_dict in actions]- completion = output_format.model_validate(completion_data)- else:- completion = result['completion']+ completion = output_format.model_validate(completion_data)+ else:+ completion = result['completion']- # Parse usage info- usage = None- if 'usage' in result:- from browser_use.llm.views import ChatInvokeUsage+ # Parse usage info+ usage = None+ if 'usage' in result and result['usage'] is not None:+ from browser_use.llm.views import ChatInvokeUsage- usage = ChatInvokeUsage(**result['usage'])+ usage = ChatInvokeUsage(**result['usage'])return ChatInvokeCompletion(completion=completion,usage=usage,)+ async def _make_request(self, payload: dict) -> dict:+ """Make a single API request."""+ async with httpx.AsyncClient(timeout=self.timeout) as client:+ response = await client.post(+ f'{self.base_url}/v1/chat/completions',+ json=payload,+ headers={+ 'Authorization': f'Bearer {self.api_key}',+ 'Content-Type': 'application/json',+ },+ )+ response.raise_for_status()+ return response.json()++ def _raise_http_error(self, e: httpx.HTTPStatusError) -> None:+ """Raise a ValueError with appropriate error message for HTTP errors."""+ error_detail = ''+ try:+ error_data = e.response.json()+ error_detail = error_data.get('detail', str(e))+ except Exception:+ error_detail = str(e)++ if e.response.status_code == 401:+ raise ValueError(f'Invalid API key. {error_detail}')+ elif e.response.status_code == 402:+ raise ValueError(f'Insufficient credits. {error_detail}')+ else:+ raise ValueError(f'API request failed: {error_detail}')+def _serialize_message(self, message: BaseMessage) -> dict:"""Serialize a message to JSON format."""# Handle Union types by checking the actual message typediff --git a/browser_use/llm/google/chat.py b/browser_use/llm/google/chat.pyindex 16055191b..3fe3a17aa 100644--- a/browser_use/llm/google/chat.py+++ b/browser_use/llm/google/chat.py@@ -1,6 +1,7 @@import asyncioimport jsonimport logging+import randomimport timefrom dataclasses import dataclass, fieldfrom typing import Any, Literal, TypeVar, overload@@ -60,9 +61,10 @@ class ChatGoogle(BaseChatModel):http_options: HTTP options for the clientinclude_system_in_user: If True, system messages are included in the first user messagesupports_structured_output: If True, uses native JSON mode; if False, uses prompt-based fallback- max_retries: Number of retries for retryable errors (default: 3)- retryable_status_codes: List of HTTP status codes to retry on (default: [403, 503])- retry_delay: Delay in seconds between retries (default: 0.01)+ max_retries: Number of retries for retryable errors (default: 5)+ retryable_status_codes: List of HTTP status codes to retry on (default: [429, 500, 502, 503, 504])+ retry_base_delay: Base delay in seconds for exponential backoff (default: 1.0)+ retry_max_delay: Maximum delay in seconds between retries (default: 60.0)Example:from google.genai import types@@ -73,8 +75,9 @@ class ChatGoogle(BaseChatModel):'tools': [types.Tool(code_execution=types.ToolCodeExecution())]},max_retries=5,- retryable_status_codes=[403, 503],- retry_delay=0.02+ retryable_status_codes=[429, 500, 502, 503, 504],+ retry_base_delay=1.0,+ retry_max_delay=60.0,)"""@@ -88,9 +91,10 @@ class ChatGoogle(BaseChatModel):config: types.GenerateContentConfigDict | None = Noneinclude_system_in_user: bool = Falsesupports_structured_output: bool = True # New flag- max_retries: int = 3 # Number of retries for retryable errors- retryable_status_codes: list[int] = field(default_factory=lambda: [403, 503]) # Status codes to retry on- retry_delay: float = 0.01 # Delay in seconds between retries+ max_retries: int = 5 # Number of retries for retryable errors+ retryable_status_codes: list[int] = field(default_factory=lambda: [429, 500, 502, 503, 504]) # Status codes to retry on+ retry_base_delay: float = 1.0 # Base delay in seconds for exponential backoff+ retry_max_delay: float = 60.0 # Maximum delay in seconds between retries# Client initialization parametersapi_key: str | None = None@@ -411,7 +415,7 @@ class ChatGoogle(BaseChatModel):# Re-raise the exceptionraise- # Retry logic for certain errors+ # Retry logic for certain errors with exponential backoffassert self.max_retries >= 1, 'max_retries must be at least 1'for attempt in range(self.max_retries):@@ -420,8 +424,14 @@ class ChatGoogle(BaseChatModel):except ModelProviderError as e:# Retry if status code is in retryable list and we have attempts leftif e.status_code in self.retryable_status_codes and attempt < self.max_retries - 1:- self.logger.warning(f'⚠️ Got {e.status_code} error, retrying... (attempt {attempt + 1}/{self.max_retries})')- await asyncio.sleep(self.retry_delay)+ # Exponential backoff with jitter: base_delay * 2^attempt + random jitter+ delay = min(self.retry_base_delay * (2**attempt), self.retry_max_delay)+ jitter = random.uniform(0, delay * 0.1) # 10% jitter+ total_delay = delay + jitter+ self.logger.warning(+ f'⚠️ Got {e.status_code} error, retrying in {total_delay:.1f}s... (attempt {attempt + 1}/{self.max_retries})'+ )+ await asyncio.sleep(total_delay)continue# Otherwise raiseraisediff --git a/tests/ci/test_llm_retries.py b/tests/ci/test_llm_retries.pynew file mode 100644index 000000000..b474f5f38--- /dev/null+++ b/tests/ci/test_llm_retries.py@@ -0,0 +1,302 @@+"""+Test retry logic with exponential backoff for LLM clients.+"""++import time+from unittest.mock import AsyncMock, MagicMock, patch++import httpx+import pytest+++class TestChatBrowserUseRetries:+ """Test retry logic for ChatBrowserUse."""++ @pytest.fixture+ def mock_env(self, monkeypatch):+ """Set up environment for ChatBrowserUse."""+ monkeypatch.setenv('BROWSER_USE_API_KEY', 'test-api-key')++ @pytest.mark.asyncio+ async def test_retries_on_503_with_exponential_backoff(self, mock_env):+ """Test that 503 errors trigger retries with exponential backoff."""+ from browser_use.llm.browser_use.chat import ChatBrowserUse+ from browser_use.llm.messages import UserMessage++ # Track timing of each attempt+ attempt_times: list[float] = []+ attempt_count = 0++ async def mock_post(*args, **kwargs):+ nonlocal attempt_count+ attempt_times.append(time.monotonic())+ attempt_count += 1++ if attempt_count < 3:+ # First 2 attempts fail with 503+ response = MagicMock()+ response.status_code = 503+ response.json.return_value = {'detail': 'Service temporarily unavailable'}+ raise httpx.HTTPStatusError('503', request=MagicMock(), response=response)+ else:+ # Third attempt succeeds+ response = MagicMock()+ response.json.return_value = {+ 'completion': 'Success!',+ 'usage': {+ 'prompt_tokens': 10,+ 'completion_tokens': 5,+ 'total_tokens': 15,+ 'prompt_cached_tokens': None,+ 'prompt_cache_creation_tokens': None,+ 'prompt_image_tokens': None,+ },+ }+ response.raise_for_status = MagicMock()+ return response++ with patch('httpx.AsyncClient') as mock_client_class:+ mock_client = AsyncMock()+ mock_client.post = mock_post+ mock_client.__aenter__ = AsyncMock(return_value=mock_client)+ mock_client.__aexit__ = AsyncMock(return_value=None)+ mock_client_class.return_value = mock_client++ # Use short delays for testing+ client = ChatBrowserUse(retry_base_delay=0.1, retry_max_delay=1.0)+ result = await client.ainvoke([UserMessage(content='test')])++ # Should have made 3 attempts+ assert attempt_count == 3+ assert result.completion == 'Success!'++ # Verify exponential backoff timing (with some tolerance for test execution)+ # First retry: ~0.1s, Second retry: ~0.2s+ delay_1 = attempt_times[1] - attempt_times[0]+ delay_2 = attempt_times[2] - attempt_times[1]++ # Allow 50% tolerance for timing+ assert 0.05 <= delay_1 <= 0.2, f'First delay {delay_1:.3f}s not in expected range'+ assert 0.1 <= delay_2 <= 0.4, f'Second delay {delay_2:.3f}s not in expected range'+ # Second delay should be roughly 2x the first (exponential)+ assert delay_2 > delay_1, 'Second delay should be longer than first (exponential backoff)'++ @pytest.mark.asyncio+ async def test_no_retry_on_401(self, mock_env):+ """Test that 401 errors do NOT trigger retries."""+ from browser_use.llm.browser_use.chat import ChatBrowserUse+ from browser_use.llm.messages import UserMessage++ attempt_count = 0++ async def mock_post(*args, **kwargs):+ nonlocal attempt_count+ attempt_count += 1+ response = MagicMock()+ response.status_code = 401+ response.json.return_value = {'detail': 'Invalid API key'}+ raise httpx.HTTPStatusError('401', request=MagicMock(), response=response)++ with patch('httpx.AsyncClient') as mock_client_class:+ mock_client = AsyncMock()+ mock_client.post = mock_post+ mock_client.__aenter__ = AsyncMock(return_value=mock_client)+ mock_client.__aexit__ = AsyncMock(return_value=None)+ mock_client_class.return_value = mock_client++ client = ChatBrowserUse(retry_base_delay=0.01)++ with pytest.raises(ValueError, match='Invalid API key'):+ await client.ainvoke([UserMessage(content='test')])++ # Should only attempt once (no retries for 401)+ assert attempt_count == 1++ @pytest.mark.asyncio+ async def test_retries_on_timeout(self, mock_env):+ """Test that timeouts trigger retries."""+ from browser_use.llm.browser_use.chat import ChatBrowserUse+ from browser_use.llm.messages import UserMessage++ attempt_count = 0++ async def mock_post(*args, **kwargs):+ nonlocal attempt_count+ attempt_count += 1+ if attempt_count < 2:+ raise httpx.TimeoutException('Request timed out')+ # Second attempt succeeds (with no usage data to test None handling)+ response = MagicMock()+ response.json.return_value = {'completion': 'Success after timeout!', 'usage': None}+ response.raise_for_status = MagicMock()+ return response++ with patch('httpx.AsyncClient') as mock_client_class:+ mock_client = AsyncMock()+ mock_client.post = mock_post+ mock_client.__aenter__ = AsyncMock(return_value=mock_client)+ mock_client.__aexit__ = AsyncMock(return_value=None)+ mock_client_class.return_value = mock_client++ client = ChatBrowserUse(retry_base_delay=0.01)+ result = await client.ainvoke([UserMessage(content='test')])++ assert attempt_count == 2+ assert result.completion == 'Success after timeout!'++ @pytest.mark.asyncio+ async def test_max_retries_exhausted(self, mock_env):+ """Test that error is raised after max retries exhausted."""+ from browser_use.llm.browser_use.chat import ChatBrowserUse+ from browser_use.llm.messages import UserMessage++ attempt_count = 0++ async def mock_post(*args, **kwargs):+ nonlocal attempt_count+ attempt_count += 1+ response = MagicMock()+ response.status_code = 503+ response.json.return_value = {'detail': 'Service unavailable'}+ raise httpx.HTTPStatusError('503', request=MagicMock(), response=response)++ with patch('httpx.AsyncClient') as mock_client_class:+ mock_client = AsyncMock()+ mock_client.post = mock_post+ mock_client.__aenter__ = AsyncMock(return_value=mock_client)+ mock_client.__aexit__ = AsyncMock(return_value=None)+ mock_client_class.return_value = mock_client++ client = ChatBrowserUse(max_retries=3, retry_base_delay=0.01)++ with pytest.raises(ValueError, match='API request failed'):+ await client.ainvoke([UserMessage(content='test')])++ # Should have attempted max_retries times+ assert attempt_count == 3+++class TestChatGoogleRetries:+ """Test retry logic for ChatGoogle."""++ @pytest.fixture+ def mock_env(self, monkeypatch):+ """Set up environment for ChatGoogle."""+ monkeypatch.setenv('GOOGLE_API_KEY', 'test-api-key')++ @pytest.mark.asyncio+ async def test_retries_on_503_with_exponential_backoff(self, mock_env):+ """Test that 503 errors trigger retries with exponential backoff."""+ from browser_use.llm.exceptions import ModelProviderError+ from browser_use.llm.google.chat import ChatGoogle+ from browser_use.llm.messages import UserMessage++ attempt_times: list[float] = []+ attempt_count = 0++ # Mock the genai client+ with patch('browser_use.llm.google.chat.genai') as mock_genai:+ mock_client = MagicMock()+ mock_genai.Client.return_value = mock_client++ async def mock_generate(*args, **kwargs):+ nonlocal attempt_count+ attempt_times.append(time.monotonic())+ attempt_count += 1++ if attempt_count < 3:+ raise ModelProviderError(message='Service unavailable', status_code=503, model='gemini-2.0-flash')+ else:+ # Success on third attempt+ mock_response = MagicMock()+ mock_response.text = 'Success!'+ mock_response.usage_metadata = MagicMock(+ prompt_token_count=10, candidates_token_count=5, total_token_count=15, cached_content_token_count=0+ )+ mock_response.candidates = [MagicMock(content=MagicMock(parts=[MagicMock(text='Success!')]))]+ return mock_response++ # Mock the aio.models.generate_content path+ mock_client.aio.models.generate_content = mock_generate++ client = ChatGoogle(model='gemini-2.0-flash', api_key='test', retry_base_delay=0.1, retry_max_delay=1.0)+ result = await client.ainvoke([UserMessage(content='test')])++ assert attempt_count == 3+ assert result.completion == 'Success!'++ # Verify exponential backoff+ delay_1 = attempt_times[1] - attempt_times[0]+ delay_2 = attempt_times[2] - attempt_times[1]++ assert 0.05 <= delay_1 <= 0.2, f'First delay {delay_1:.3f}s not in expected range'+ assert 0.1 <= delay_2 <= 0.4, f'Second delay {delay_2:.3f}s not in expected range'+ assert delay_2 > delay_1, 'Second delay should be longer than first'++ @pytest.mark.asyncio+ async def test_no_retry_on_400(self, mock_env):+ """Test that 400 errors do NOT trigger retries."""+ from browser_use.llm.exceptions import ModelProviderError+ from browser_use.llm.google.chat import ChatGoogle+ from browser_use.llm.messages import UserMessage++ attempt_count = 0++ with patch('browser_use.llm.google.chat.genai') as mock_genai:+ mock_client = MagicMock()+ mock_genai.Client.return_value = mock_client++ async def mock_generate(*args, **kwargs):+ nonlocal attempt_count+ attempt_count += 1+ raise ModelProviderError(message='Bad request', status_code=400, model='gemini-2.0-flash')++ mock_client.aio.models.generate_content = mock_generate++ client = ChatGoogle(model='gemini-2.0-flash', api_key='test', retry_base_delay=0.01)++ with pytest.raises(ModelProviderError):+ await client.ainvoke([UserMessage(content='test')])++ # Should only attempt once (400 is not retryable)+ assert attempt_count == 1++ @pytest.mark.asyncio+ async def test_retries_on_429_rate_limit(self, mock_env):+ """Test that 429 rate limit errors trigger retries."""+ from browser_use.llm.exceptions import ModelProviderError+ from browser_use.llm.google.chat import ChatGoogle+ from browser_use.llm.messages import UserMessage++ attempt_count = 0++ with patch('browser_use.llm.google.chat.genai') as mock_genai:+ mock_client = MagicMock()+ mock_genai.Client.return_value = mock_client++ async def mock_generate(*args, **kwargs):+ nonlocal attempt_count+ attempt_count += 1++ if attempt_count < 2:+ raise ModelProviderError(message='Rate limit exceeded', status_code=429, model='gemini-2.0-flash')+ else:+ mock_response = MagicMock()+ mock_response.text = 'Success after rate limit!'+ mock_response.usage_metadata = MagicMock(+ prompt_token_count=10, candidates_token_count=5, total_token_count=15, cached_content_token_count=0+ )+ mock_response.candidates = [MagicMock(content=MagicMock(parts=[MagicMock(text='Success after rate limit!')]))]+ return mock_response++ mock_client.aio.models.generate_content = mock_generate++ client = ChatGoogle(model='gemini-2.0-flash', api_key='test', retry_base_delay=0.01)+ result = await client.ainvoke([UserMessage(content='test')])++ assert attempt_count == 2+ assert result.completion == 'Success after rate limit!'+++if __name__ == '__main__':+ pytest.main([__file__, '-v'])