Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
fix: detect label/span wrappers for AntD-style controls
ditingdapeng882ac6471a74
diff --git a/browser_use/dom/serializer/clickable_elements.py b/browser_use/dom/serializer/clickable_elements.pyindex f274985a8..2084b5f53 100644--- a/browser_use/dom/serializer/clickable_elements.py+++ b/browser_use/dom/serializer/clickable_elements.py@@ -6,6 +6,24 @@ class ClickableElementDetector:def is_interactive(node: EnhancedDOMTreeNode) -> bool:"""Check if this node is clickable/interactive using enhanced scoring."""+ def has_form_control_descendant(element: EnhancedDOMTreeNode, max_depth: int = 2) -> bool:+ """Detect nested form controls within limited depth (handles label/span wrappers)."""+ if max_depth <= 0:+ return False++ for child in element.children_and_shadow_roots:+ if child.node_type != NodeType.ELEMENT_NODE:+ continue++ tag_name = child.tag_name+ if tag_name in {'input', 'select', 'textarea'}:+ return True++ if has_form_control_descendant(child, max_depth=max_depth - 1):+ return True++ return False+# Skip non-element nodesif node.node_type != NodeType.ELEMENT_NODE:return False@@ -32,6 +50,23 @@ class ClickableElementDetector:# Note: Size 0 elements can still be interactive (e.g., invisible clickable overlays)# Visibility is determined separately by CSS styles, not just bounding box size+ # Specialized handling for labels used as component wrappers (e.g., Ant Design radio/checkbox)+ if node.tag_name == 'label':+ # Skip labels that proxy via "for" to avoid double-activating external inputs+ if node.attributes and node.attributes.get('for'):+ return False++ # Detect labels that wrap form controls up to two levels deep (label > span > input)+ if has_form_control_descendant(node, max_depth=2):+ return True+ # Fall through to pointer/role/attribute heuristics for other label cases++ # Span wrappers for UI components (detect clear interactive signals only)+ if node.tag_name == 'span':+ if has_form_control_descendant(node, max_depth=2):+ return True+ # Allow other heuristics (aria roles, event handlers, pointer) to decide+# SEARCH ELEMENT DETECTION: Check for search-related classes and attributesif node.attributes:search_indicators = {diff --git a/tests/ci/browser/test_dom_serializer.py b/tests/ci/browser/test_dom_serializer.pyindex e91c9a33e..7d773e6e0 100644--- a/tests/ci/browser/test_dom_serializer.py+++ b/tests/ci/browser/test_dom_serializer.py@@ -84,9 +84,10 @@ class TestDOMSerializer:- Regular DOM: 3 elements (button, input, link on main page)- Shadow DOM: 3 elements (2 buttons, 1 input inside shadow root)- Same-origin iframe: 2 elements (button, input inside iframe)+ - AntD-like controls: 2 elements (labels wrapping radio inputs)- Cross-origin iframe placeholder: about:blank (no interactive elements)- Iframe tags: 2 elements (the iframe elements themselves)- Total: ~10 interactive elements+ Total: ~14 interactive elements"""from browser_use.tools.service import Tools@@ -155,6 +156,7 @@ class TestDOMSerializer:shadow_elements = []iframe_content_elements = []iframe_tags = []+ antd_elements = []# Categorize elements by their IDs (more stable than hardcoded indices)# Check element attributes to identify their location@@ -171,6 +173,8 @@ class TestDOMSerializer:# Iframe content elements have IDs starting with "iframe-"elif elem_id.startswith('iframe-'):iframe_content_elements.append((idx, element))+ elif elem_id.startswith('antd-'):+ antd_elements.append((idx, element))# Everything else is regular DOMelse:regular_elements.append((idx, element))@@ -182,8 +186,9 @@ class TestDOMSerializer:# - Regular DOM: 3-4 elements (button, input, link on main page + possible cross-origin content)# - Shadow DOM: 3 elements (2 buttons, 1 input inside shadow root)# - Iframe content: 2 elements (button, input from same-origin iframe)+ # - AntD-like controls: 2+ elements (labels + inputs)# - Iframe tags: 2 elements (the iframe elements themselves)- # Total: ~10-11 interactive elements depending on cross-origin iframe extraction+ # Total: ~14 interactive elements depending on cross-origin iframe extractionprint('\n✅ DOM Serializer Test Summary:')print(f' • Regular DOM: {len(regular_elements)} elements {"✓" if len(regular_elements) >= 3 else "✗"}')@@ -191,6 +196,7 @@ class TestDOMSerializer:print(f' • Same-origin iframe content: {len(iframe_content_elements)} elements {"✓" if len(iframe_content_elements) >= 2 else "✗"}')+ print(f' • AntD-like controls: {len(antd_elements)} elements {"✓" if len(antd_elements) >= 2 else "✗"}')print(f' • Iframe tags: {len(iframe_tags)} elements {"✓" if len(iframe_tags) >= 2 else "✗"}')print(f' • Total elements: {len(selector_map)}')@@ -201,6 +207,7 @@ class TestDOMSerializer:assert len(iframe_content_elements) >= 1, (f'Should find at least 1 iframe content element, found {len(iframe_content_elements)}')+ assert len(antd_elements) >= 2, f'Should find at least 2 AntD-like label elements, found {len(antd_elements)}'# Now test clicking elements from each category using tools.click(index)print('\n🖱️ Testing Click Functionality:')@@ -236,6 +243,11 @@ class TestDOMSerializer:if iframe_button_idx:await click(iframe_button_idx, 'Same-origin iframe button', browser_session)+ # Test clicking an AntD-like label wrapper+ if antd_elements:+ antd_label_idx = next((idx for idx, el in antd_elements if el.tag_name == 'label'), None) or antd_elements[0][0]+ await click(antd_label_idx, 'AntD-like label', browser_session)+# Validate click counter - verify all 3 clicks actually executed JavaScriptprint('\n✅ Validating click counter...')@@ -257,7 +269,7 @@ class TestDOMSerializer:click_count = result.get('result', {}).get('value', 0)print(f' Click counter value: {click_count}')- assert click_count == 3, (+ assert click_count == 4, (f'Expected 3 clicks (Regular DOM + Shadow DOM + Iframe), but counter shows {click_count}. 'f'This means some clicks did not execute JavaScript properly.')diff --git a/tests/ci/browser/test_page_template.html b/tests/ci/browser/test_page_template.htmlindex cde8e161f..5081dabb1 100644--- a/tests/ci/browser/test_page_template.html+++ b/tests/ci/browser/test_page_template.html@@ -59,6 +59,23 @@<iframe id="same-origin-iframe" src="/iframe-same-origin" style="width:100%; height:200px; border:1px solid #999;"></iframe></div>+ <!-- Ant Design-like components (labels wrapping inputs with spans) -->+ <div class="section" id="antd-section">+ <h2>Ant Design-like Controls</h2>+ <label class="ant-radio-button-wrapper" id="antd-radio-1">+ <span class="ant-radio-button">+ <input type="radio" class="ant-radio-button-input" name="demoRadio" id="antd-radio-input-1" />+ </span>+ <span class="ant-radio-label">AntD Option A</span>+ </label>+ <label class="ant-radio-button-wrapper" id="antd-radio-2">+ <span class="ant-radio-button">+ <input type="radio" class="ant-radio-button-input" name="demoRadio" id="antd-radio-input-2" />+ </span>+ <span class="ant-radio-label">AntD Option B</span>+ </label>+ </div>+<!-- Cross-origin iframe placeholder (external domain removed for test isolation) --><div class="section"><h2>Cross-Origin Iframe (Placeholder)</h2>@@ -113,6 +130,11 @@shadowRoot.getElementById('shadow-btn-1').addEventListener('click', function() {incrementCounter('Shadow DOM');});++ // Add click handlers for Ant Design-like labels+ document.querySelectorAll('.ant-radio-button-wrapper').forEach((label) => {+ label.addEventListener('click', () => incrementCounter('AntD'));+ });</script></body></html>