Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fix: detect label/span wrappers for AntD-style controls

ditingdapeng882ac6471a74
diff --git a/browser_use/dom/serializer/clickable_elements.py b/browser_use/dom/serializer/clickable_elements.py
index f274985a8..2084b5f53 100644
--- a/browser_use/dom/serializer/clickable_elements.py
+++ b/browser_use/dom/serializer/clickable_elements.py
@@ -6,6 +6,24 @@ class ClickableElementDetector:
def is_interactive(node: EnhancedDOMTreeNode) -> bool:
"""Check if this node is clickable/interactive using enhanced scoring."""
+ def has_form_control_descendant(element: EnhancedDOMTreeNode, max_depth: int = 2) -> bool:
+ """Detect nested form controls within limited depth (handles label/span wrappers)."""
+ if max_depth <= 0:
+ return False
+
+ for child in element.children_and_shadow_roots:
+ if child.node_type != NodeType.ELEMENT_NODE:
+ continue
+
+ tag_name = child.tag_name
+ if tag_name in {'input', 'select', 'textarea'}:
+ return True
+
+ if has_form_control_descendant(child, max_depth=max_depth - 1):
+ return True
+
+ return False
+
# Skip non-element nodes
if node.node_type != NodeType.ELEMENT_NODE:
return False
@@ -32,6 +50,23 @@ class ClickableElementDetector:
# Note: Size 0 elements can still be interactive (e.g., invisible clickable overlays)
# Visibility is determined separately by CSS styles, not just bounding box size
+ # Specialized handling for labels used as component wrappers (e.g., Ant Design radio/checkbox)
+ if node.tag_name == 'label':
+ # Skip labels that proxy via "for" to avoid double-activating external inputs
+ if node.attributes and node.attributes.get('for'):
+ return False
+
+ # Detect labels that wrap form controls up to two levels deep (label > span > input)
+ if has_form_control_descendant(node, max_depth=2):
+ return True
+ # Fall through to pointer/role/attribute heuristics for other label cases
+
+ # Span wrappers for UI components (detect clear interactive signals only)
+ if node.tag_name == 'span':
+ if has_form_control_descendant(node, max_depth=2):
+ return True
+ # Allow other heuristics (aria roles, event handlers, pointer) to decide
+
# SEARCH ELEMENT DETECTION: Check for search-related classes and attributes
if node.attributes:
search_indicators = {
diff --git a/tests/ci/browser/test_dom_serializer.py b/tests/ci/browser/test_dom_serializer.py
index e91c9a33e..7d773e6e0 100644
--- a/tests/ci/browser/test_dom_serializer.py
+++ b/tests/ci/browser/test_dom_serializer.py
@@ -84,9 +84,10 @@ class TestDOMSerializer:
- Regular DOM: 3 elements (button, input, link on main page)
- Shadow DOM: 3 elements (2 buttons, 1 input inside shadow root)
- Same-origin iframe: 2 elements (button, input inside iframe)
+ - AntD-like controls: 2 elements (labels wrapping radio inputs)
- Cross-origin iframe placeholder: about:blank (no interactive elements)
- Iframe tags: 2 elements (the iframe elements themselves)
- Total: ~10 interactive elements
+ Total: ~14 interactive elements
"""
from browser_use.tools.service import Tools
@@ -155,6 +156,7 @@ class TestDOMSerializer:
shadow_elements = []
iframe_content_elements = []
iframe_tags = []
+ antd_elements = []
# Categorize elements by their IDs (more stable than hardcoded indices)
# Check element attributes to identify their location
@@ -171,6 +173,8 @@ class TestDOMSerializer:
# Iframe content elements have IDs starting with "iframe-"
elif elem_id.startswith('iframe-'):
iframe_content_elements.append((idx, element))
+ elif elem_id.startswith('antd-'):
+ antd_elements.append((idx, element))
# Everything else is regular DOM
else:
regular_elements.append((idx, element))
@@ -182,8 +186,9 @@ class TestDOMSerializer:
# - Regular DOM: 3-4 elements (button, input, link on main page + possible cross-origin content)
# - Shadow DOM: 3 elements (2 buttons, 1 input inside shadow root)
# - Iframe content: 2 elements (button, input from same-origin iframe)
+ # - AntD-like controls: 2+ elements (labels + inputs)
# - Iframe tags: 2 elements (the iframe elements themselves)
- # Total: ~10-11 interactive elements depending on cross-origin iframe extraction
+ # Total: ~14 interactive elements depending on cross-origin iframe extraction
print('\n✅ DOM Serializer Test Summary:')
print(f' • Regular DOM: {len(regular_elements)} elements {"✓" if len(regular_elements) >= 3 else "✗"}')
@@ -191,6 +196,7 @@ class TestDOMSerializer:
print(
f' • Same-origin iframe content: {len(iframe_content_elements)} elements {"✓" if len(iframe_content_elements) >= 2 else "✗"}'
)
+ print(f' • AntD-like controls: {len(antd_elements)} elements {"✓" if len(antd_elements) >= 2 else "✗"}')
print(f' • Iframe tags: {len(iframe_tags)} elements {"✓" if len(iframe_tags) >= 2 else "✗"}')
print(f' • Total elements: {len(selector_map)}')
@@ -201,6 +207,7 @@ class TestDOMSerializer:
assert len(iframe_content_elements) >= 1, (
f'Should find at least 1 iframe content element, found {len(iframe_content_elements)}'
)
+ assert len(antd_elements) >= 2, f'Should find at least 2 AntD-like label elements, found {len(antd_elements)}'
# Now test clicking elements from each category using tools.click(index)
print('\n🖱️ Testing Click Functionality:')
@@ -236,6 +243,11 @@ class TestDOMSerializer:
if iframe_button_idx:
await click(iframe_button_idx, 'Same-origin iframe button', browser_session)
+ # Test clicking an AntD-like label wrapper
+ if antd_elements:
+ antd_label_idx = next((idx for idx, el in antd_elements if el.tag_name == 'label'), None) or antd_elements[0][0]
+ await click(antd_label_idx, 'AntD-like label', browser_session)
+
# Validate click counter - verify all 3 clicks actually executed JavaScript
print('\n✅ Validating click counter...')
@@ -257,7 +269,7 @@ class TestDOMSerializer:
click_count = result.get('result', {}).get('value', 0)
print(f' Click counter value: {click_count}')
- assert click_count == 3, (
+ assert click_count == 4, (
f'Expected 3 clicks (Regular DOM + Shadow DOM + Iframe), but counter shows {click_count}. '
f'This means some clicks did not execute JavaScript properly.'
)
diff --git a/tests/ci/browser/test_page_template.html b/tests/ci/browser/test_page_template.html
index cde8e161f..5081dabb1 100644
--- a/tests/ci/browser/test_page_template.html
+++ b/tests/ci/browser/test_page_template.html
@@ -59,6 +59,23 @@
<iframe id="same-origin-iframe" src="/iframe-same-origin" style="width:100%; height:200px; border:1px solid #999;"></iframe>
</div>
+ <!-- Ant Design-like components (labels wrapping inputs with spans) -->
+ <div class="section" id="antd-section">
+ <h2>Ant Design-like Controls</h2>
+ <label class="ant-radio-button-wrapper" id="antd-radio-1">
+ <span class="ant-radio-button">
+ <input type="radio" class="ant-radio-button-input" name="demoRadio" id="antd-radio-input-1" />
+ </span>
+ <span class="ant-radio-label">AntD Option A</span>
+ </label>
+ <label class="ant-radio-button-wrapper" id="antd-radio-2">
+ <span class="ant-radio-button">
+ <input type="radio" class="ant-radio-button-input" name="demoRadio" id="antd-radio-input-2" />
+ </span>
+ <span class="ant-radio-label">AntD Option B</span>
+ </label>
+ </div>
+
<!-- Cross-origin iframe placeholder (external domain removed for test isolation) -->
<div class="section">
<h2>Cross-Origin Iframe (Placeholder)</h2>
@@ -113,6 +130,11 @@
shadowRoot.getElementById('shadow-btn-1').addEventListener('click', function() {
incrementCounter('Shadow DOM');
});
+
+ // Add click handlers for Ant Design-like labels
+ document.querySelectorAll('.ant-radio-button-wrapper').forEach((label) => {
+ label.addEventListener('click', () => incrementCounter('AntD'));
+ });
</script>
</body>
</html>