Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Split model tests

Magnus Müller6e31a05acd77
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index e2bfcde3f..f915046f5 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -174,13 +174,7 @@ jobs:
timeout_minutes: 4
max_attempts: 2
retry_on: error
- command: |
- if [[ "${{ matrix.test_filename }}" == "models/test_llm_models" ]]; then
- # Run model tests in parallel using pytest-xdist
- pytest "tests/ci/${{ matrix.test_filename }}.py" -n auto
- else
- pytest "tests/ci/${{ matrix.test_filename }}.py"
- fi
+ command: pytest "tests/ci/${{ matrix.test_filename }}.py"
evaluate-tasks:
needs: setup-chromium
diff --git a/tests/ci/models/model_test_helper.py b/tests/ci/models/model_test_helper.py
new file mode 100644
index 000000000..c09cb242b
--- /dev/null
+++ b/tests/ci/models/model_test_helper.py
@@ -0,0 +1,119 @@
+"""Shared test helper for LLM model tests."""
+
+import os
+
+import pytest
+
+from browser_use.agent.service import Agent
+from browser_use.browser.profile import BrowserProfile
+from browser_use.browser.session import BrowserSession
+
+
+async def run_model_button_click_test(
+ model_class,
+ model_name: str,
+ api_key_env: str | None,
+ extra_kwargs: dict,
+ httpserver,
+):
+ """Test that an LLM model can click a button.
+
+ This test verifies:
+ 1. Model can be initialized with API key
+ 2. Agent can navigate and click a button
+ 3. Button click is verified by checking page state change
+ 4. Completes within max 2 steps
+ """
+ # Check if running in GitHub Actions
+ in_github_actions = os.getenv('GITHUB_ACTIONS') == 'true'
+
+ # Handle API key validation
+ if api_key_env is not None:
+ api_key = os.getenv(api_key_env)
+ if not api_key:
+ if in_github_actions:
+ pytest.fail(f'{api_key_env} not set - failing in GitHub Actions')
+ else:
+ pytest.skip(f'{api_key_env} not set')
+ else:
+ api_key = None
+
+ # Handle Azure-specific endpoint validation
+ from browser_use.llm.azure.chat import ChatAzureOpenAI
+
+ if model_class is ChatAzureOpenAI:
+ azure_endpoint = os.getenv('AZURE_OPENAI_ENDPOINT')
+ if not azure_endpoint:
+ if in_github_actions:
+ pytest.fail('AZURE_OPENAI_ENDPOINT not set - failing in GitHub Actions')
+ else:
+ pytest.skip('AZURE_OPENAI_ENDPOINT not set')
+ # Add the azure_endpoint to extra_kwargs
+ extra_kwargs = {**extra_kwargs, 'azure_endpoint': azure_endpoint}
+
+ # Create HTML page with a button that changes page content when clicked
+ html = """
+ <!DOCTYPE html>
+ <html>
+ <head><title>Button Test</title></head>
+ <body>
+ <h1>Button Click Test</h1>
+ <button id="test-button" onclick="document.getElementById('result').innerText='SUCCESS'">
+ Click Me
+ </button>
+ <div id="result">NOT_CLICKED</div>
+ </body>
+ </html>
+ """
+ httpserver.expect_request('/').respond_with_data(html, content_type='text/html')
+
+ # Create LLM instance with extra kwargs if provided
+ llm_kwargs = {'model': model_name}
+ if api_key is not None:
+ llm_kwargs['api_key'] = api_key
+ llm_kwargs.update(extra_kwargs)
+ llm = model_class(**llm_kwargs) # type: ignore[arg-type]
+
+ # Create browser session
+ browser = BrowserSession(
+ browser_profile=BrowserProfile(
+ headless=True,
+ user_data_dir=None, # Use temporary directory
+ )
+ )
+
+ try:
+ # Start browser
+ await browser.start()
+
+ # Create agent with button click task (URL in task triggers auto-navigation)
+ test_url = httpserver.url_for('/')
+ agent = Agent(
+ task=f'{test_url} - Click the button',
+ llm=llm,
+ browser_session=browser,
+ max_steps=2, # Max 2 steps as per requirements
+ )
+
+ # Run the agent
+ result = await agent.run()
+
+ # Verify task completed
+ assert result is not None
+ assert len(result.history) > 0
+
+ # Verify button was clicked by checking page state across any step
+ button_clicked = False
+ for step in result.history:
+ # Check state_message which contains browser state with page text
+ if step.state_message and 'SUCCESS' in step.state_message:
+ button_clicked = True
+ break
+
+ # Check if SUCCESS appears in any step (indicating button was clicked)
+ assert button_clicked, 'Button was not clicked - SUCCESS not found in any page state'
+
+ finally:
+ # Clean up browser session
+ await browser.kill()
+ await browser.event_bus.stop(clear=True, timeout=5)
diff --git a/tests/ci/models/test_llm_anthropic.py b/tests/ci/models/test_llm_anthropic.py
new file mode 100644
index 000000000..96e0ac82a
--- /dev/null
+++ b/tests/ci/models/test_llm_anthropic.py
@@ -0,0 +1,15 @@
+"""Test Anthropic model button click."""
+
+from browser_use.llm.anthropic.chat import ChatAnthropic
+from tests.ci.models.model_test_helper import run_model_button_click_test
+
+
+async def test_anthropic_claude_sonnet_4_0(httpserver):
+ """Test Anthropic claude-sonnet-4-0 can click a button."""
+ await run_model_button_click_test(
+ model_class=ChatAnthropic,
+ model_name='claude-sonnet-4-0',
+ api_key_env='ANTHROPIC_API_KEY',
+ extra_kwargs={},
+ httpserver=httpserver,
+ )
diff --git a/tests/ci/models/test_llm_azure.py b/tests/ci/models/test_llm_azure.py
new file mode 100644
index 000000000..a90ff89ba
--- /dev/null
+++ b/tests/ci/models/test_llm_azure.py
@@ -0,0 +1,15 @@
+"""Test Azure OpenAI model button click."""
+
+from browser_use.llm.azure.chat import ChatAzureOpenAI
+from tests.ci.models.model_test_helper import run_model_button_click_test
+
+
+async def test_azure_gpt_4_1_mini(httpserver):
+ """Test Azure OpenAI gpt-4.1-mini can click a button."""
+ await run_model_button_click_test(
+ model_class=ChatAzureOpenAI,
+ model_name='gpt-4.1-mini',
+ api_key_env='AZURE_OPENAI_KEY',
+ extra_kwargs={}, # Azure endpoint will be added by helper
+ httpserver=httpserver,
+ )
diff --git a/tests/ci/models/test_llm_browseruse.py b/tests/ci/models/test_llm_browseruse.py
new file mode 100644
index 000000000..4955eb043
--- /dev/null
+++ b/tests/ci/models/test_llm_browseruse.py
@@ -0,0 +1,15 @@
+"""Test Browser Use model button click."""
+
+from browser_use.llm.browser_use.chat import ChatBrowserUse
+from tests.ci.models.model_test_helper import run_model_button_click_test
+
+
+async def test_browseruse_bu_latest(httpserver):
+ """Test Browser Use bu-latest can click a button."""
+ await run_model_button_click_test(
+ model_class=ChatBrowserUse,
+ model_name='bu-latest',
+ api_key_env='BROWSER_USE_API_KEY',
+ extra_kwargs={},
+ httpserver=httpserver,
+ )
diff --git a/tests/ci/models/test_llm_google.py b/tests/ci/models/test_llm_google.py
new file mode 100644
index 000000000..9d8bc8632
--- /dev/null
+++ b/tests/ci/models/test_llm_google.py
@@ -0,0 +1,15 @@
+"""Test Google model button click."""
+
+from browser_use.llm.google.chat import ChatGoogle
+from tests.ci.models.model_test_helper import run_model_button_click_test
+
+
+async def test_google_gemini_flash_latest(httpserver):
+ """Test Google gemini-flash-latest can click a button."""
+ await run_model_button_click_test(
+ model_class=ChatGoogle,
+ model_name='gemini-flash-latest',
+ api_key_env='GOOGLE_API_KEY',
+ extra_kwargs={},
+ httpserver=httpserver,
+ )
diff --git a/tests/ci/models/test_llm_models.py b/tests/ci/models/test_llm_models.py
deleted file mode 100644
index c955c0887..000000000
--- a/tests/ci/models/test_llm_models.py
+++ /dev/null
@@ -1,176 +0,0 @@
-"""Tests for LLM model initialization and basic functionality.
-
-This test verifies that all supported LLM models from examples/models/ can be initialized
-and execute a simple button click task. Each model is tested in parallel.
-"""
-
-import os
-
-import pytest
-
-from browser_use.agent.service import Agent
-from browser_use.browser.profile import BrowserProfile
-from browser_use.browser.session import BrowserSession
-from browser_use.llm.anthropic.chat import ChatAnthropic
-from browser_use.llm.azure.chat import ChatAzureOpenAI
-from browser_use.llm.browser_use.chat import ChatBrowserUse
-from browser_use.llm.google.chat import ChatGoogle
-from browser_use.llm.openai.chat import ChatOpenAI
-
-# Define models to test based on the examples in examples/models/
-# Each tuple is (model_class, model_name, required_api_key_env_var, extra_kwargs)
-MODELS_TO_TEST = [
- # OpenAI models (from gpt-4.1.py, gpt-5-mini.py)
- pytest.param(ChatOpenAI, 'gpt-4.1-mini', 'OPENAI_API_KEY', {}, id='openai_gpt_4_1_mini'),
- # Anthropic models (from claude-4-sonnet.py)
- pytest.param(ChatAnthropic, 'claude-sonnet-4-0', 'ANTHROPIC_API_KEY', {}, id='anthropic_claude_sonnet_4_0'),
- # Google models (from gemini.py)
- pytest.param(ChatGoogle, 'gemini-flash-latest', 'GOOGLE_API_KEY', {}, id='google_gemini_flash_latest'),
- # Azure OpenAI (from azure_openai.py) - needs both API key and endpoint
- pytest.param(
- ChatAzureOpenAI,
- 'gpt-4.1-mini',
- 'AZURE_OPENAI_KEY',
- {'azure_endpoint': 'AZURE_OPENAI_ENDPOINT'}, # Special marker - will be resolved at runtime
- id='azure_gpt_4_1_mini',
- ),
- # Browser Use LLM (from browser_use_llm.py)
- pytest.param(ChatBrowserUse, 'bu-latest', 'BROWSER_USE_API_KEY', {}, id='browser_use_bu_latest'),
- # OpenRouter (from openrouter.py)
- pytest.param(
- ChatOpenAI,
- 'x-ai/grok-4-fast',
- 'OPENROUTER_API_KEY',
- {'base_url': 'https://openrouter.ai/api/v1'},
- id='openrouter_grok_4_mini',
- ),
-]
-
-
[email protected]('model_class,model_name,api_key_env,extra_kwargs', MODELS_TO_TEST)
-async def test_llm_model_button_click(model_class, model_name, api_key_env, extra_kwargs, httpserver):
- """Test that each LLM model can click a button.
-
- This test verifies:
- 1. Model can be initialized with API key
- 2. Agent can navigate and click a button
- 3. Button click is verified by checking page state change
- 4. Completes within max 2 steps
- """
- # Check if running in GitHub Actions
- in_github_actions = os.getenv('GITHUB_ACTIONS') == 'true'
-
- # Handle API key validation (except Ollama which is local)
- if api_key_env is not None:
- api_key = os.getenv(api_key_env)
- if not api_key:
- if in_github_actions:
- pytest.fail(f'{api_key_env} not set - failing in GitHub Actions')
- else:
- pytest.skip(f'{api_key_env} not set')
- else:
- api_key = None
-
- # Handle Azure endpoint validation at runtime
- if model_class is ChatAzureOpenAI:
- azure_endpoint = os.getenv('AZURE_OPENAI_ENDPOINT')
- if not azure_endpoint:
- if in_github_actions:
- pytest.fail('AZURE_OPENAI_ENDPOINT not set - failing in GitHub Actions')
- else:
- pytest.skip('AZURE_OPENAI_ENDPOINT not set')
- # Replace the marker with actual endpoint value
- extra_kwargs = {**extra_kwargs, 'azure_endpoint': azure_endpoint}
-
- # Create HTML page with a button that changes page content when clicked
- html = """
- <!DOCTYPE html>
- <html>
- <head><title>Button Test</title></head>
- <body>
- <h1>Button Click Test</h1>
- <button id="test-button" onclick="document.getElementById('result').innerText='SUCCESS'">
- Click Me
- </button>
- <div id="result">NOT_CLICKED</div>
- </body>
- </html>
- """
- httpserver.expect_request('/').respond_with_data(html, content_type='text/html')
-
- # Create LLM instance with extra kwargs if provided
- llm_kwargs = {'model': model_name}
- if api_key is not None:
- llm_kwargs['api_key'] = api_key
- llm_kwargs.update(extra_kwargs)
- llm = model_class(**llm_kwargs)
-
- # Create browser session
- browser = BrowserSession(
- browser_profile=BrowserProfile(
- headless=True,
- user_data_dir=None, # Use temporary directory
- )
- )
-
- try:
- # Start browser
- await browser.start()
-
- # Create agent with button click task (URL in task triggers auto-navigation)
- test_url = httpserver.url_for('/')
- agent = Agent(
- task=f'{test_url} - Click the button',
- llm=llm,
- browser_session=browser,
- max_steps=2, # Max 2 steps as per requirements
- )
-
- # Run the agent
- result = await agent.run()
-
- # Verify task completed
- assert result is not None
- assert len(result.history) > 0
-
- # Verify button was clicked by checking page state across any step
- # (Don't enforce strict step count since auto-navigation and optional done actions vary by model)
- button_clicked = False
- for step in result.history:
- # Check state_message which contains browser state with page text
- if step.state_message and 'SUCCESS' in step.state_message:
- button_clicked = True
- break
-
- # Check if SUCCESS appears in any step (indicating button was clicked)
- assert button_clicked, 'Button was not clicked - SUCCESS not found in any page state'
-
- finally:
- # Clean up browser session
- await browser.kill()
- await browser.event_bus.stop(clear=True, timeout=5)
-
-
-async def test_all_models_discoverable():
- """Test that all models defined in MODELS_TO_TEST are valid.
-
- This is a simple sanity check that doesn't require API keys.
- """
- assert len(MODELS_TO_TEST) > 0, 'No models defined in MODELS_TO_TEST'
-
- # Verify each model parameter is a tuple of the right length
- for param in MODELS_TO_TEST:
- assert len(param.values) == 4, f'Invalid model parameter: {param}'
- model_class, model_name, api_key_env, extra_kwargs = param.values
-
- # Verify model_class is a class
- assert callable(model_class), f'model_class {model_class} is not callable'
-
- # Verify model_name is a string
- assert isinstance(model_name, str), f'model_name {model_name} is not a string'
-
- # Verify api_key_env is a string or None (for local models like Ollama)
- assert api_key_env is None or isinstance(api_key_env, str), f'api_key_env {api_key_env} must be string or None'
-
- # Verify extra_kwargs is a dict
- assert isinstance(extra_kwargs, dict), f'extra_kwargs {extra_kwargs} is not a dict'
diff --git a/tests/ci/models/test_llm_openai.py b/tests/ci/models/test_llm_openai.py
new file mode 100644
index 000000000..5fef442a1
--- /dev/null
+++ b/tests/ci/models/test_llm_openai.py
@@ -0,0 +1,15 @@
+"""Test OpenAI model button click."""
+
+from browser_use.llm.openai.chat import ChatOpenAI
+from tests.ci.models.model_test_helper import run_model_button_click_test
+
+
+async def test_openai_gpt_4_1_mini(httpserver):
+ """Test OpenAI gpt-4.1-mini can click a button."""
+ await run_model_button_click_test(
+ model_class=ChatOpenAI,
+ model_name='gpt-4.1-mini',
+ api_key_env='OPENAI_API_KEY',
+ extra_kwargs={},
+ httpserver=httpserver,
+ )
diff --git a/tests/ci/models/test_llm_openrouter.py b/tests/ci/models/test_llm_openrouter.py
new file mode 100644
index 000000000..62c734470
--- /dev/null
+++ b/tests/ci/models/test_llm_openrouter.py
@@ -0,0 +1,15 @@
+"""Test OpenRouter model button click."""
+
+from browser_use.llm.openai.chat import ChatOpenAI
+from tests.ci.models.model_test_helper import run_model_button_click_test
+
+
+async def test_openrouter_grok_4_mini(httpserver):
+ """Test OpenRouter x-ai/grok-4-fast can click a button."""
+ await run_model_button_click_test(
+ model_class=ChatOpenAI,
+ model_name='x-ai/grok-4-fast',
+ api_key_env='OPENROUTER_API_KEY',
+ extra_kwargs={'base_url': 'https://openrouter.ai/api/v1'},
+ httpserver=httpserver,
+ )