Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Split model tests
Magnus Müller6e31a05acd77
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yamlindex e2bfcde3f..f915046f5 100644--- a/.github/workflows/test.yaml+++ b/.github/workflows/test.yaml@@ -174,13 +174,7 @@ jobs:timeout_minutes: 4max_attempts: 2retry_on: error- command: |- if [[ "${{ matrix.test_filename }}" == "models/test_llm_models" ]]; then- # Run model tests in parallel using pytest-xdist- pytest "tests/ci/${{ matrix.test_filename }}.py" -n auto- else- pytest "tests/ci/${{ matrix.test_filename }}.py"- fi+ command: pytest "tests/ci/${{ matrix.test_filename }}.py"evaluate-tasks:needs: setup-chromiumdiff --git a/tests/ci/models/model_test_helper.py b/tests/ci/models/model_test_helper.pynew file mode 100644index 000000000..c09cb242b--- /dev/null+++ b/tests/ci/models/model_test_helper.py@@ -0,0 +1,119 @@+"""Shared test helper for LLM model tests."""++import os++import pytest++from browser_use.agent.service import Agent+from browser_use.browser.profile import BrowserProfile+from browser_use.browser.session import BrowserSession+++async def run_model_button_click_test(+ model_class,+ model_name: str,+ api_key_env: str | None,+ extra_kwargs: dict,+ httpserver,+):+ """Test that an LLM model can click a button.++ This test verifies:+ 1. Model can be initialized with API key+ 2. Agent can navigate and click a button+ 3. Button click is verified by checking page state change+ 4. Completes within max 2 steps+ """+ # Check if running in GitHub Actions+ in_github_actions = os.getenv('GITHUB_ACTIONS') == 'true'++ # Handle API key validation+ if api_key_env is not None:+ api_key = os.getenv(api_key_env)+ if not api_key:+ if in_github_actions:+ pytest.fail(f'{api_key_env} not set - failing in GitHub Actions')+ else:+ pytest.skip(f'{api_key_env} not set')+ else:+ api_key = None++ # Handle Azure-specific endpoint validation+ from browser_use.llm.azure.chat import ChatAzureOpenAI++ if model_class is ChatAzureOpenAI:+ azure_endpoint = os.getenv('AZURE_OPENAI_ENDPOINT')+ if not azure_endpoint:+ if in_github_actions:+ pytest.fail('AZURE_OPENAI_ENDPOINT not set - failing in GitHub Actions')+ else:+ pytest.skip('AZURE_OPENAI_ENDPOINT not set')+ # Add the azure_endpoint to extra_kwargs+ extra_kwargs = {**extra_kwargs, 'azure_endpoint': azure_endpoint}++ # Create HTML page with a button that changes page content when clicked+ html = """+ <!DOCTYPE html>+ <html>+ <head><title>Button Test</title></head>+ <body>+ <h1>Button Click Test</h1>+ <button id="test-button" onclick="document.getElementById('result').innerText='SUCCESS'">+ Click Me+ </button>+ <div id="result">NOT_CLICKED</div>+ </body>+ </html>+ """+ httpserver.expect_request('/').respond_with_data(html, content_type='text/html')++ # Create LLM instance with extra kwargs if provided+ llm_kwargs = {'model': model_name}+ if api_key is not None:+ llm_kwargs['api_key'] = api_key+ llm_kwargs.update(extra_kwargs)+ llm = model_class(**llm_kwargs) # type: ignore[arg-type]++ # Create browser session+ browser = BrowserSession(+ browser_profile=BrowserProfile(+ headless=True,+ user_data_dir=None, # Use temporary directory+ )+ )++ try:+ # Start browser+ await browser.start()++ # Create agent with button click task (URL in task triggers auto-navigation)+ test_url = httpserver.url_for('/')+ agent = Agent(+ task=f'{test_url} - Click the button',+ llm=llm,+ browser_session=browser,+ max_steps=2, # Max 2 steps as per requirements+ )++ # Run the agent+ result = await agent.run()++ # Verify task completed+ assert result is not None+ assert len(result.history) > 0++ # Verify button was clicked by checking page state across any step+ button_clicked = False+ for step in result.history:+ # Check state_message which contains browser state with page text+ if step.state_message and 'SUCCESS' in step.state_message:+ button_clicked = True+ break++ # Check if SUCCESS appears in any step (indicating button was clicked)+ assert button_clicked, 'Button was not clicked - SUCCESS not found in any page state'++ finally:+ # Clean up browser session+ await browser.kill()+ await browser.event_bus.stop(clear=True, timeout=5)diff --git a/tests/ci/models/test_llm_anthropic.py b/tests/ci/models/test_llm_anthropic.pynew file mode 100644index 000000000..96e0ac82a--- /dev/null+++ b/tests/ci/models/test_llm_anthropic.py@@ -0,0 +1,15 @@+"""Test Anthropic model button click."""++from browser_use.llm.anthropic.chat import ChatAnthropic+from tests.ci.models.model_test_helper import run_model_button_click_test+++async def test_anthropic_claude_sonnet_4_0(httpserver):+ """Test Anthropic claude-sonnet-4-0 can click a button."""+ await run_model_button_click_test(+ model_class=ChatAnthropic,+ model_name='claude-sonnet-4-0',+ api_key_env='ANTHROPIC_API_KEY',+ extra_kwargs={},+ httpserver=httpserver,+ )diff --git a/tests/ci/models/test_llm_azure.py b/tests/ci/models/test_llm_azure.pynew file mode 100644index 000000000..a90ff89ba--- /dev/null+++ b/tests/ci/models/test_llm_azure.py@@ -0,0 +1,15 @@+"""Test Azure OpenAI model button click."""++from browser_use.llm.azure.chat import ChatAzureOpenAI+from tests.ci.models.model_test_helper import run_model_button_click_test+++async def test_azure_gpt_4_1_mini(httpserver):+ """Test Azure OpenAI gpt-4.1-mini can click a button."""+ await run_model_button_click_test(+ model_class=ChatAzureOpenAI,+ model_name='gpt-4.1-mini',+ api_key_env='AZURE_OPENAI_KEY',+ extra_kwargs={}, # Azure endpoint will be added by helper+ httpserver=httpserver,+ )diff --git a/tests/ci/models/test_llm_browseruse.py b/tests/ci/models/test_llm_browseruse.pynew file mode 100644index 000000000..4955eb043--- /dev/null+++ b/tests/ci/models/test_llm_browseruse.py@@ -0,0 +1,15 @@+"""Test Browser Use model button click."""++from browser_use.llm.browser_use.chat import ChatBrowserUse+from tests.ci.models.model_test_helper import run_model_button_click_test+++async def test_browseruse_bu_latest(httpserver):+ """Test Browser Use bu-latest can click a button."""+ await run_model_button_click_test(+ model_class=ChatBrowserUse,+ model_name='bu-latest',+ api_key_env='BROWSER_USE_API_KEY',+ extra_kwargs={},+ httpserver=httpserver,+ )diff --git a/tests/ci/models/test_llm_google.py b/tests/ci/models/test_llm_google.pynew file mode 100644index 000000000..9d8bc8632--- /dev/null+++ b/tests/ci/models/test_llm_google.py@@ -0,0 +1,15 @@+"""Test Google model button click."""++from browser_use.llm.google.chat import ChatGoogle+from tests.ci.models.model_test_helper import run_model_button_click_test+++async def test_google_gemini_flash_latest(httpserver):+ """Test Google gemini-flash-latest can click a button."""+ await run_model_button_click_test(+ model_class=ChatGoogle,+ model_name='gemini-flash-latest',+ api_key_env='GOOGLE_API_KEY',+ extra_kwargs={},+ httpserver=httpserver,+ )diff --git a/tests/ci/models/test_llm_models.py b/tests/ci/models/test_llm_models.pydeleted file mode 100644index c955c0887..000000000--- a/tests/ci/models/test_llm_models.py+++ /dev/null@@ -1,176 +0,0 @@-"""Tests for LLM model initialization and basic functionality.--This test verifies that all supported LLM models from examples/models/ can be initialized-and execute a simple button click task. Each model is tested in parallel.-"""--import os--import pytest--from browser_use.agent.service import Agent-from browser_use.browser.profile import BrowserProfile-from browser_use.browser.session import BrowserSession-from browser_use.llm.anthropic.chat import ChatAnthropic-from browser_use.llm.azure.chat import ChatAzureOpenAI-from browser_use.llm.browser_use.chat import ChatBrowserUse-from browser_use.llm.google.chat import ChatGoogle-from browser_use.llm.openai.chat import ChatOpenAI--# Define models to test based on the examples in examples/models/-# Each tuple is (model_class, model_name, required_api_key_env_var, extra_kwargs)-MODELS_TO_TEST = [- # OpenAI models (from gpt-4.1.py, gpt-5-mini.py)- pytest.param(ChatOpenAI, 'gpt-4.1-mini', 'OPENAI_API_KEY', {}, id='openai_gpt_4_1_mini'),- # Anthropic models (from claude-4-sonnet.py)- pytest.param(ChatAnthropic, 'claude-sonnet-4-0', 'ANTHROPIC_API_KEY', {}, id='anthropic_claude_sonnet_4_0'),- # Google models (from gemini.py)- pytest.param(ChatGoogle, 'gemini-flash-latest', 'GOOGLE_API_KEY', {}, id='google_gemini_flash_latest'),- # Azure OpenAI (from azure_openai.py) - needs both API key and endpoint- pytest.param(- ChatAzureOpenAI,- 'gpt-4.1-mini',- 'AZURE_OPENAI_KEY',- {'azure_endpoint': 'AZURE_OPENAI_ENDPOINT'}, # Special marker - will be resolved at runtime- id='azure_gpt_4_1_mini',- ),- # Browser Use LLM (from browser_use_llm.py)- pytest.param(ChatBrowserUse, 'bu-latest', 'BROWSER_USE_API_KEY', {}, id='browser_use_bu_latest'),- # OpenRouter (from openrouter.py)- pytest.param(- ChatOpenAI,- 'x-ai/grok-4-fast',- 'OPENROUTER_API_KEY',- {'base_url': 'https://openrouter.ai/api/v1'},- id='openrouter_grok_4_mini',- ),-]--[email protected]('model_class,model_name,api_key_env,extra_kwargs', MODELS_TO_TEST)-async def test_llm_model_button_click(model_class, model_name, api_key_env, extra_kwargs, httpserver):- """Test that each LLM model can click a button.-- This test verifies:- 1. Model can be initialized with API key- 2. Agent can navigate and click a button- 3. Button click is verified by checking page state change- 4. Completes within max 2 steps- """- # Check if running in GitHub Actions- in_github_actions = os.getenv('GITHUB_ACTIONS') == 'true'-- # Handle API key validation (except Ollama which is local)- if api_key_env is not None:- api_key = os.getenv(api_key_env)- if not api_key:- if in_github_actions:- pytest.fail(f'{api_key_env} not set - failing in GitHub Actions')- else:- pytest.skip(f'{api_key_env} not set')- else:- api_key = None-- # Handle Azure endpoint validation at runtime- if model_class is ChatAzureOpenAI:- azure_endpoint = os.getenv('AZURE_OPENAI_ENDPOINT')- if not azure_endpoint:- if in_github_actions:- pytest.fail('AZURE_OPENAI_ENDPOINT not set - failing in GitHub Actions')- else:- pytest.skip('AZURE_OPENAI_ENDPOINT not set')- # Replace the marker with actual endpoint value- extra_kwargs = {**extra_kwargs, 'azure_endpoint': azure_endpoint}-- # Create HTML page with a button that changes page content when clicked- html = """- <!DOCTYPE html>- <html>- <head><title>Button Test</title></head>- <body>- <h1>Button Click Test</h1>- <button id="test-button" onclick="document.getElementById('result').innerText='SUCCESS'">- Click Me- </button>- <div id="result">NOT_CLICKED</div>- </body>- </html>- """- httpserver.expect_request('/').respond_with_data(html, content_type='text/html')-- # Create LLM instance with extra kwargs if provided- llm_kwargs = {'model': model_name}- if api_key is not None:- llm_kwargs['api_key'] = api_key- llm_kwargs.update(extra_kwargs)- llm = model_class(**llm_kwargs)-- # Create browser session- browser = BrowserSession(- browser_profile=BrowserProfile(- headless=True,- user_data_dir=None, # Use temporary directory- )- )-- try:- # Start browser- await browser.start()-- # Create agent with button click task (URL in task triggers auto-navigation)- test_url = httpserver.url_for('/')- agent = Agent(- task=f'{test_url} - Click the button',- llm=llm,- browser_session=browser,- max_steps=2, # Max 2 steps as per requirements- )-- # Run the agent- result = await agent.run()-- # Verify task completed- assert result is not None- assert len(result.history) > 0-- # Verify button was clicked by checking page state across any step- # (Don't enforce strict step count since auto-navigation and optional done actions vary by model)- button_clicked = False- for step in result.history:- # Check state_message which contains browser state with page text- if step.state_message and 'SUCCESS' in step.state_message:- button_clicked = True- break-- # Check if SUCCESS appears in any step (indicating button was clicked)- assert button_clicked, 'Button was not clicked - SUCCESS not found in any page state'-- finally:- # Clean up browser session- await browser.kill()- await browser.event_bus.stop(clear=True, timeout=5)---async def test_all_models_discoverable():- """Test that all models defined in MODELS_TO_TEST are valid.-- This is a simple sanity check that doesn't require API keys.- """- assert len(MODELS_TO_TEST) > 0, 'No models defined in MODELS_TO_TEST'-- # Verify each model parameter is a tuple of the right length- for param in MODELS_TO_TEST:- assert len(param.values) == 4, f'Invalid model parameter: {param}'- model_class, model_name, api_key_env, extra_kwargs = param.values-- # Verify model_class is a class- assert callable(model_class), f'model_class {model_class} is not callable'-- # Verify model_name is a string- assert isinstance(model_name, str), f'model_name {model_name} is not a string'-- # Verify api_key_env is a string or None (for local models like Ollama)- assert api_key_env is None or isinstance(api_key_env, str), f'api_key_env {api_key_env} must be string or None'-- # Verify extra_kwargs is a dict- assert isinstance(extra_kwargs, dict), f'extra_kwargs {extra_kwargs} is not a dict'diff --git a/tests/ci/models/test_llm_openai.py b/tests/ci/models/test_llm_openai.pynew file mode 100644index 000000000..5fef442a1--- /dev/null+++ b/tests/ci/models/test_llm_openai.py@@ -0,0 +1,15 @@+"""Test OpenAI model button click."""++from browser_use.llm.openai.chat import ChatOpenAI+from tests.ci.models.model_test_helper import run_model_button_click_test+++async def test_openai_gpt_4_1_mini(httpserver):+ """Test OpenAI gpt-4.1-mini can click a button."""+ await run_model_button_click_test(+ model_class=ChatOpenAI,+ model_name='gpt-4.1-mini',+ api_key_env='OPENAI_API_KEY',+ extra_kwargs={},+ httpserver=httpserver,+ )diff --git a/tests/ci/models/test_llm_openrouter.py b/tests/ci/models/test_llm_openrouter.pynew file mode 100644index 000000000..62c734470--- /dev/null+++ b/tests/ci/models/test_llm_openrouter.py@@ -0,0 +1,15 @@+"""Test OpenRouter model button click."""++from browser_use.llm.openai.chat import ChatOpenAI+from tests.ci.models.model_test_helper import run_model_button_click_test+++async def test_openrouter_grok_4_mini(httpserver):+ """Test OpenRouter x-ai/grok-4-fast can click a button."""+ await run_model_button_click_test(+ model_class=ChatOpenAI,+ model_name='x-ai/grok-4-fast',+ api_key_env='OPENROUTER_API_KEY',+ extra_kwargs={'base_url': 'https://openrouter.ai/api/v1'},+ httpserver=httpserver,+ )