Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
fix radio button click occlusion when label overlaps input
Saurav Panda84babc6b75fb
diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.pyindex d2eb1c96e..f9c17147d 100644--- a/browser_use/browser/watchdogs/default_action_watchdog.py+++ b/browser_use/browser/watchdogs/default_action_watchdog.py@@ -612,10 +612,48 @@ class DefaultActionWatchdog(BaseWatchdog):// Simple containment-based clickability logic- const isClickable = this === elementAtPoint ||+ let isClickable = this === elementAtPoint ||this.contains(elementAtPoint) ||elementAtPoint.contains(this);+ // Check label-input associations when containment check fails+ if (!isClickable) {+ const target = this;+ const atPoint = elementAtPoint;++ // Case 1: target is <input>, atPoint is its associated <label> (or child of that label)+ if (target.tagName === 'INPUT' && target.id) {+ const escapedId = CSS.escape(target.id);+ const assocLabel = document.querySelector('label[for="' + escapedId + '"]');+ if (assocLabel && (assocLabel === atPoint || assocLabel.contains(atPoint))) {+ isClickable = true;+ }+ }++ // Case 2: target is <input>, atPoint is inside a <label> ancestor that wraps the target+ if (!isClickable && target.tagName === 'INPUT') {+ let ancestor = atPoint;+ for (let i = 0; i < 3 && ancestor; i++) {+ if (ancestor.tagName === 'LABEL' && ancestor.contains(target)) {+ isClickable = true;+ break;+ }+ ancestor = ancestor.parentElement;+ }+ }++ // Case 3: target is <label>, atPoint is the associated <input>+ if (!isClickable && target.tagName === 'LABEL') {+ if (target.htmlFor && atPoint.tagName === 'INPUT' && atPoint.id === target.htmlFor) {+ isClickable = true;+ }+ // Also check if atPoint is an input inside the label+ if (!isClickable && atPoint.tagName === 'INPUT' && target.contains(atPoint)) {+ isClickable = true;+ }+ }+ }+return {targetInfo: getElementInfo(this),elementAtPointInfo: getElementInfo(elementAtPoint),diff --git a/tests/ci/interactions/test_radio_buttons.html b/tests/ci/interactions/test_radio_buttons.htmldeleted file mode 100644index f2b5d7726..000000000--- a/tests/ci/interactions/test_radio_buttons.html+++ /dev/null@@ -1,106 +0,0 @@-<!DOCTYPE html>-<html>-<head>- <title>Radio Button Test</title>-</head>-<body>- <h1>Radio Button Test Page</h1>-- <form>- <fieldset>- <legend>Select your favorite color:</legend>-- <label>- <input type="radio" name="color" value="red" id="radio-red">- Red- </label>- <br>-- <label>- <input type="radio" name="color" value="blue" id="radio-blue">- Blue- </label>- <br>-- <label>- <input type="radio" name="color" value="green" id="radio-green">- Green- </label>- <br>- </fieldset>-- <fieldset>- <legend>Select your favorite animal:</legend>-- <label>- <input type="radio" name="animal" value="cat" id="radio-cat">- Cat- </label>- <br>-- <label>- <input type="radio" name="animal" value="dog" id="radio-dog">- Dog- </label>- <br>-- <label>- <input type="radio" name="animal" value="bird" id="radio-bird">- Bird- </label>- <br>- </fieldset>-- <div id="result-message" style="margin-top: 20px; padding: 10px; background-color: #f0f0f0; display: none;">- <p id="secret-text"></p>- </div>- </form>-- <script>- function checkSelection() {- const colorRadios = document.querySelectorAll('input[name="color"]');- const animalRadios = document.querySelectorAll('input[name="animal"]');-- let selectedColor = null;- let selectedAnimal = null;-- // Get selected color- for (const radio of colorRadios) {- if (radio.checked) {- selectedColor = radio.value;- break;- }- }-- // Get selected animal- for (const radio of animalRadios) {- if (radio.checked) {- selectedAnimal = radio.value;- break;- }- }-- const resultDiv = document.getElementById('result-message');- const secretText = document.getElementById('secret-text');-- // Show secret if both Blue and Dog are selected- if (selectedColor === 'blue' && selectedAnimal === 'dog') {- secretText.textContent = 'SECRET_SUCCESS_12345: Blue dog combination unlocked!';- resultDiv.style.display = 'block';- resultDiv.style.backgroundColor = '#d4edda';- } else if (selectedColor && selectedAnimal) {- secretText.textContent = `Selected: ${selectedColor} ${selectedAnimal}`;- resultDiv.style.display = 'block';- resultDiv.style.backgroundColor = '#f8d7da';- } else {- resultDiv.style.display = 'none';- }- }-- // Add event listeners to all radio buttons- document.querySelectorAll('input[type="radio"]').forEach(radio => {- radio.addEventListener('change', checkSelection);- });- </script>-</body>-</html>diff --git a/tests/ci/interactions/test_radio_buttons.py b/tests/ci/interactions/test_radio_buttons.pyindex 68f808dcb..7e4157cea 100644--- a/tests/ci/interactions/test_radio_buttons.py+++ b/tests/ci/interactions/test_radio_buttons.py@@ -1,46 +1,145 @@-# @file purpose: Test radio button interactions and serialization in browser-use-"""-Test file for verifying radio button clicking functionality and DOM serialization.--This test creates a simple HTML page with radio buttons, sends an agent to click them,-and logs the final agent message to show how radio buttons are represented in the serializer.--The serialization shows radio buttons as:-[index]<input type=radio name=groupname value=optionvalue checked=true/false />+"""Test radio button click interactions with various label association patterns.-Usage:- uv run pytest tests/ci/test_radio_buttons.py -v -s--Note: This test requires a real LLM API key and is skipped in CI environments.+Verifies that the occlusion check correctly identifies label-input associations+so that CDP click dispatch works for radio buttons whose labels visually overlap them."""-import os-from pathlib import Path-import pytestfrom pytest_httpserver import HTTPServer-from browser_use.agent.service import Agentfrom browser_use.browser import BrowserSessionfrom browser_use.browser.profile import BrowserProfile+from browser_use.tools.service import Tools++# -- HTML fixtures --++RADIO_SIBLING_HTML = """+<!DOCTYPE html>+<html>+<head><title>Radio Sibling Label Test</title>+<style>+ .radio-group { display: flex; flex-direction: column; gap: 8px; }+ .radio-group input[type="radio"] { width: 16px; height: 16px; }+ .radio-group label { cursor: pointer; padding: 4px 8px; }+</style>+</head>+<body>+ <div class="radio-group">+ <div>+ <input type="radio" name="color" value="red" id="radio-red">+ <label for="radio-red">Red</label>+ </div>+ <div>+ <input type="radio" name="color" value="blue" id="radio-blue">+ <label for="radio-blue">Blue</label>+ </div>+ <div>+ <input type="radio" name="color" value="green" id="radio-green">+ <label for="radio-green">Green</label>+ </div>+ </div>+ <div id="result"></div>+ <script>+ document.querySelectorAll('input[name="color"]').forEach(r => {+ r.addEventListener('change', () => {+ document.getElementById('result').textContent = 'selected:' + r.value;+ });+ });+ </script>+</body>+</html>+"""++RADIO_WRAPPED_HTML = """+<!DOCTYPE html>+<html>+<head><title>Radio Wrapped Label Test</title>+<style>+ .radio-group label { display: block; cursor: pointer; padding: 8px; }+</style>+</head>+<body>+ <div class="radio-group">+ <label><input type="radio" name="fruit" value="apple" id="radio-apple"> Apple</label>+ <label><input type="radio" name="fruit" value="banana" id="radio-banana"> Banana</label>+ <label><input type="radio" name="fruit" value="cherry" id="radio-cherry"> Cherry</label>+ </div>+ <div id="result"></div>+ <script>+ document.querySelectorAll('input[name="fruit"]').forEach(r => {+ r.addEventListener('change', () => {+ document.getElementById('result').textContent = 'selected:' + r.value;+ });+ });+ </script>+</body>+</html>+"""++RADIO_CUSTOM_HTML = """+<!DOCTYPE html>+<html>+<head><title>Radio Custom Styled Test</title>+<style>+ .radio-group { display: flex; flex-direction: column; gap: 8px; }+ .radio-group input[type="radio"] {+ appearance: none;+ -webkit-appearance: none;+ width: 20px;+ height: 20px;+ border: 2px solid #999;+ border-radius: 50%;+ position: relative;+ }+ .radio-group input[type="radio"]:checked {+ border-color: #007bff;+ background: #007bff;+ }+ .radio-group label {+ cursor: pointer;+ padding: 4px 8px;+ display: inline-flex;+ align-items: center;+ gap: 6px;+ }+</style>+</head>+<body>+ <div class="radio-group">+ <div>+ <input type="radio" name="size" value="small" id="radio-small">+ <label for="radio-small">Small</label>+ </div>+ <div>+ <input type="radio" name="size" value="medium" id="radio-medium">+ <label for="radio-medium">Medium</label>+ </div>+ <div>+ <input type="radio" name="size" value="large" id="radio-large">+ <label for="radio-large">Large</label>+ </div>+ </div>+ <div id="result"></div>+ <script>+ document.querySelectorAll('input[name="size"]').forEach(r => {+ r.addEventListener('change', () => {+ document.getElementById('result').textContent = 'selected:' + r.value;+ });+ });+ </script>+</body>+</html>+"""@pytest.fixture(scope='session')def http_server():- """Create and provide a test HTTP server that serves static content."""server = HTTPServer()server.start()- # Read the HTML file content- html_file = Path(__file__).parent / 'test_radio_buttons.html'- with open(html_file) as f:- html_content = f.read()-- # Add route for radio buttons test page- server.expect_request('/radio-test').respond_with_data(- html_content,- content_type='text/html',- )+ server.expect_request('/radio-sibling').respond_with_data(RADIO_SIBLING_HTML, content_type='text/html')+ server.expect_request('/radio-wrapped').respond_with_data(RADIO_WRAPPED_HTML, content_type='text/html')+ server.expect_request('/radio-custom').respond_with_data(RADIO_CUSTOM_HTML, content_type='text/html')yield serverserver.stop()@@ -48,18 +147,17 @@ def http_server():@pytest.fixture(scope='session')def base_url(http_server):- """Return the base URL for the test HTTP server."""return f'http://{http_server.host}:{http_server.port}'@pytest.fixture(scope='module')async def browser_session():- """Create and provide a Browser instance with security disabled."""browser_session = BrowserSession(browser_profile=BrowserProfile(headless=True,user_data_dir=None,keep_alive=True,+ chromium_sandbox=False,))await browser_session.start()@@ -67,39 +165,110 @@ async def browser_session():await browser_session.kill()- os.getenv('CI') == 'true' or os.getenv('GITHUB_ACTIONS') == 'true',- reason='Skipped in CI: requires real LLM API key which blocks other tests',-)[email protected](scope='function')+def tools():+ return Tools()+++async def _get_checked_and_result(browser_session: BrowserSession, input_id: str) -> tuple[bool, str]:+ """Helper: returns (is_checked, result_div_text) via CDP."""+ cdp_session = await browser_session.get_or_create_cdp_session()+ sid = cdp_session.session_id++ checked_result = await cdp_session.cdp_client.send.Runtime.evaluate(+ params={+ 'expression': f"document.getElementById('{input_id}').checked",+ 'returnByValue': True,+ },+ session_id=sid,+ )+ is_checked = checked_result.get('result', {}).get('value', False)++ text_result = await cdp_session.cdp_client.send.Runtime.evaluate(+ params={+ 'expression': "document.getElementById('result').textContent",+ 'returnByValue': True,+ },+ session_id=sid,+ )+ result_text = text_result.get('result', {}).get('value', '')++ return is_checked, result_text++class TestRadioButtons:- """Test cases for radio button interactions."""+ """Test radio button clicks across label association patterns."""- async def test_radio_button_clicking(self, browser_session, base_url):- """Test that agent can click radio buttons by checking for secret message."""+ async def test_sibling_label_radio_click(self, tools: Tools, browser_session: BrowserSession, base_url: str):+ """Click a radio whose sibling <label for=...> may occlude it."""+ await tools.navigate(url=f'{base_url}/radio-sibling', new_tab=False, browser_session=browser_session)+ await browser_session.get_browser_state_summary()- task = f"Go to {base_url}/radio-test and click on the 'Blue' radio button and the 'Dog' radio button. After clicking both buttons, look for any text message that appears on the page and report exactly what you see."+ idx = await browser_session.get_index_by_id('radio-blue')+ assert idx is not None, 'Could not find radio-blue in selector map'- agent = Agent(- task=task,- browser_session=browser_session,- max_actions_per_step=5,- flash_mode=True,- )+ result = await tools.click(index=idx, browser_session=browser_session)+ assert result.error is None, f'Click failed: {result.error}'- # Run the agent- history = await agent.run(max_steps=8)+ is_checked, result_text = await _get_checked_and_result(browser_session, 'radio-blue')+ assert is_checked, 'radio-blue should be checked after click'+ assert 'selected:blue' in result_text, f'Change event not fired, result: {result_text}'- # Check if the secret message appears in the final response- secret_found = False- final_response = history.final_result()+ async def test_wrapped_label_radio_click(self, tools: Tools, browser_session: BrowserSession, base_url: str):+ """Click a radio wrapped inside its <label>."""+ await tools.navigate(url=f'{base_url}/radio-wrapped', new_tab=False, browser_session=browser_session)+ await browser_session.get_browser_state_summary()- if final_response and 'SECRET_SUCCESS_12345' in final_response:- secret_found = True- print('\n✅ SUCCESS: Secret message found! Radio buttons were clicked correctly.')+ idx = await browser_session.get_index_by_id('radio-banana')+ assert idx is not None, 'Could not find radio-banana in selector map'- assert secret_found, (- "Secret message 'SECRET_SUCCESS_12345' should be present, indicating both Blue and Dog radio buttons were clicked. Actual response: "- + str(final_response)- )+ result = await tools.click(index=idx, browser_session=browser_session)+ assert result.error is None, f'Click failed: {result.error}'++ is_checked, result_text = await _get_checked_and_result(browser_session, 'radio-banana')+ assert is_checked, 'radio-banana should be checked after click'+ assert 'selected:banana' in result_text, f'Change event not fired, result: {result_text}'++ async def test_custom_styled_radio_click(self, tools: Tools, browser_session: BrowserSession, base_url: str):+ """Click a CSS-customized radio (appearance:none) with sibling label."""+ await tools.navigate(url=f'{base_url}/radio-custom', new_tab=False, browser_session=browser_session)+ await browser_session.get_browser_state_summary()++ idx = await browser_session.get_index_by_id('radio-large')+ assert idx is not None, 'Could not find radio-large in selector map'++ result = await tools.click(index=idx, browser_session=browser_session)+ assert result.error is None, f'Click failed: {result.error}'++ is_checked, result_text = await _get_checked_and_result(browser_session, 'radio-large')+ assert is_checked, 'radio-large should be checked after click'+ assert 'selected:large' in result_text, f'Change event not fired, result: {result_text}'++ async def test_radio_group_switching(self, tools: Tools, browser_session: BrowserSession, base_url: str):+ """Click one radio then another in the same group; first should uncheck."""+ await tools.navigate(url=f'{base_url}/radio-sibling', new_tab=False, browser_session=browser_session)+ await browser_session.get_browser_state_summary()++ # Click red first+ red_idx = await browser_session.get_index_by_id('radio-red')+ assert red_idx is not None+ result = await tools.click(index=red_idx, browser_session=browser_session)+ assert result.error is None, f'Click red failed: {result.error}'++ is_red_checked, _ = await _get_checked_and_result(browser_session, 'radio-red')+ assert is_red_checked, 'radio-red should be checked'++ # Re-fetch state so indices are current, then click green+ await browser_session.get_browser_state_summary()+ green_idx = await browser_session.get_index_by_id('radio-green')+ assert green_idx is not None+ result = await tools.click(index=green_idx, browser_session=browser_session)+ assert result.error is None, f'Click green failed: {result.error}'++ is_green_checked, result_text = await _get_checked_and_result(browser_session, 'radio-green')+ assert is_green_checked, 'radio-green should be checked'+ assert 'selected:green' in result_text- print(f'\n🎉 Test completed successfully! Agent completed {len(history)} steps and found the secret message.')+ # Verify red is now unchecked+ is_red_checked, _ = await _get_checked_and_result(browser_session, 'radio-red')+ assert not is_red_checked, 'radio-red should be unchecked after selecting green'