Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Works

Magnus Müller90e27e07d172
diff --git a/browser_use/dom/playground/extraction.py b/browser_use/dom/playground/extraction.py
index a62d2a8e0..b526dc749 100644
--- a/browser_use/dom/playground/extraction.py
+++ b/browser_use/dom/playground/extraction.py
@@ -33,6 +33,7 @@ async def test_focus_vs_all_elements():
# 10 Sample websites with various interactive elements
sample_websites = [
+ 'https://browser-use.github.io/stress-tests/challenges/iframe-inception-level2.html',
'https://www.google.com/travel/flights',
'https://v0-simple-ui-test-site.vercel.app',
'https://browser-use.github.io/stress-tests/challenges/iframe-inception-level1.html',
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.py
index 1b199965d..2a113984e 100644
--- a/browser_use/dom/serializer/serializer.py
+++ b/browser_use/dom/serializer/serializer.py
@@ -107,6 +107,48 @@ class DOMTreeSerializer:
return SerializedDOMState(_root=filtered_tree, selector_map=self._selector_map), self.timing_info
+
+ def _add_compound_components(self, simplified: SimplifiedNode, node: EnhancedDOMTreeNode) -> None:
+ """Enhance compound controls with information from their child components."""
+ # Only process form controls that might have compound components
+ if (
+ node.tag_name not in ['input', 'select']
+ or not node.attributes
+ or node.attributes.get('type') not in ['date', 'time', 'datetime-local', 'month', 'week']
+ ):
+ return
+
+ # Check if this node's AX info has child_ids (indicating compound components)
+ if not node.ax_node or not node.ax_node.child_ids:
+ return
+
+ # For now, add basic compound component information based on the input type
+ input_type = node.attributes.get('type', '')
+
+ if input_type == 'date':
+ node._compound_children.extend([
+ {'role': 'spinbutton', 'name': 'Day', 'valuemin': 1, 'valuemax': 31, 'valuenow': None},
+ {'role': 'spinbutton', 'name': 'Month', 'valuemin': 1, 'valuemax': 12, 'valuenow': None},
+ {'role': 'spinbutton', 'name': 'Year', 'valuemin': 1, 'valuemax': 275760, 'valuenow': None}
+ ])
+ simplified.is_compound_component = True
+ elif input_type == 'time':
+ node._compound_children.extend([
+ {'role': 'spinbutton', 'name': 'Hour', 'valuemin': 0, 'valuemax': 23, 'valuenow': None},
+ {'role': 'spinbutton', 'name': 'Minute', 'valuemin': 0, 'valuemax': 59, 'valuenow': None}
+ ])
+ simplified.is_compound_component = True
+ elif input_type in ['datetime-local', 'month', 'week']:
+ # For other compound input types, add basic info
+ node._compound_children.append({
+ 'role': 'compound',
+ 'name': f'{input_type} components',
+ 'valuemin': None,
+ 'valuemax': None,
+ 'valuenow': None
+ })
+ simplified.is_compound_component = True
+
def _is_interactive_cached(self, node: EnhancedDOMTreeNode) -> bool:
"""Cached version of clickable element detection to avoid redundant calls."""
if node.node_id not in self._clickable_cache:
@@ -169,7 +211,13 @@ class DOMTreeSerializer:
# ENHANCED SHADOW DOM DETECTION: Include shadow hosts even if not visible
is_shadow_host = any(child.node_type == NodeType.DOCUMENT_FRAGMENT_NODE for child in node.children_and_shadow_roots)
- # Include if interactive (regardless of visibility), scrollable, has children, or is shadow host
+ # Override visibility for elements with validation attributes
+ if not is_visible and node.attributes:
+ has_validation_attrs = any(attr.startswith(('aria-', 'pseudo')) for attr in node.attributes.keys())
+ if has_validation_attrs:
+ is_visible = True # Force visibility for validation elements
+
+ # Include if visible, interactive, scrollable, has children, or is shadow host
if is_visible or is_scrollable or has_shadow_content or is_shadow_host:
simplified = SimplifiedNode(original_node=node, children=[], is_shadow_host=is_shadow_host)
@@ -179,6 +227,9 @@ class DOMTreeSerializer:
if simplified_child:
simplified.children.append(simplified_child)
+ # COMPOUND CONTROL PROCESSING: Add virtual components for compound controls
+ self._add_compound_components(simplified, node)
+
# SHADOW DOM SPECIAL CASE: Always include shadow hosts even if not visible
# Many SPA frameworks (React, Vue) render content in shadow DOM
if is_shadow_host and simplified.children:
@@ -211,7 +262,6 @@ class DOMTreeSerializer:
node.children = optimized_children
# Keep meaningful nodes
- is_interactive_opt = self._is_interactive_cached(node.original_node)
is_visible = node.original_node.snapshot_node and node.original_node.is_visible
if (
@@ -243,7 +293,7 @@ class DOMTreeSerializer:
# Skip assigning index to excluded nodes, or ignored by paint order
if not node.excluded_by_parent and not node.ignored_by_paint_order:
- # Assign index to clickable elements that are also visible
+ # Regular interactive element assignment (including enhanced compound controls)
is_interactive_assign = self._is_interactive_cached(node.original_node)
is_visible = node.original_node.snapshot_node and node.original_node.is_visible
@@ -254,8 +304,11 @@ class DOMTreeSerializer:
self._selector_map[self._interactive_counter] = node.original_node
self._interactive_counter += 1
- # Check if node is new
- if self._previous_cached_selector_map:
+ # Mark compound components as new for visibility
+ if node.is_compound_component:
+ node.is_new = True
+ elif self._previous_cached_selector_map:
+ # Check if node is new for regular elements
previous_backend_node_ids = {node.backend_node_id for node in self._previous_cached_selector_map.values()}
if node.original_node.backend_node_id not in previous_backend_node_ids:
node.is_new = True
@@ -456,8 +509,35 @@ class DOMTreeSerializer:
):
next_depth += 1
- # Build attributes string
- attributes_html_str = DOMTreeSerializer._build_attributes_string(node.original_node, include_attributes, '')
+ # Build attributes string with compound component info
+ text_content = ''
+ attributes_html_str = DOMTreeSerializer._build_attributes_string(node.original_node, include_attributes, text_content)
+
+ # Add compound component information to attributes if present
+ if node.original_node._compound_children:
+ compound_info = []
+ for child_info in node.original_node._compound_children:
+ parts = []
+ if child_info['name']:
+ parts.append(f"name={child_info['name']}")
+ if child_info['role']:
+ parts.append(f"role={child_info['role']}")
+ if child_info['valuemin'] is not None:
+ parts.append(f"min={child_info['valuemin']}")
+ if child_info['valuemax'] is not None:
+ parts.append(f"max={child_info['valuemax']}")
+ if child_info['valuenow'] is not None:
+ parts.append(f"current={child_info['valuenow']}")
+
+ if parts:
+ compound_info.append(f"({','.join(parts)})")
+
+ if compound_info:
+ compound_attr = f"compound_components={','.join(compound_info)}"
+ if attributes_html_str:
+ attributes_html_str += f' {compound_attr}'
+ else:
+ attributes_html_str = compound_attr
# Build the line with shadow host indicator
shadow_prefix = ''
diff --git a/browser_use/dom/service.py b/browser_use/dom/service.py
index 13b368356..dfa8245d0 100644
--- a/browser_use/dom/service.py
+++ b/browser_use/dom/service.py
@@ -130,6 +130,7 @@ class DomService:
name=ax_node.get('name', {}).get('value', None),
description=ax_node.get('description', {}).get('value', None),
properties=properties,
+ child_ids=ax_node.get('childIds', []) if ax_node.get('childIds') else None,
)
return enhanced_ax_node
diff --git a/browser_use/dom/views.py b/browser_use/dom/views.py
index 061b070a8..72ee8e429 100644
--- a/browser_use/dom/views.py
+++ b/browser_use/dom/views.py
@@ -31,13 +31,29 @@ DEFAULT_INCLUDE_ATTRIBUTES = [
'aria-expanded',
'data-state',
'aria-checked',
+ # ARIA value attributes for datetime/range inputs
+ 'aria-valuemin',
+ 'aria-valuemax',
+ 'aria-valuenow',
+ 'aria-placeholder',
+ # Validation attributes - help agents avoid brute force attempts
+ 'pattern',
+ 'min',
+ 'max',
+ 'minlength',
+ 'maxlength',
+ 'step',
+ # Webkit shadow DOM identifiers
+ 'pseudo',
# Accessibility properties from ax_node (ordered by importance for automation)
'checked',
'selected',
'expanded',
'pressed',
'disabled',
- # 'invalid',
+ 'invalid', # Current validation state from AX node
+ 'valuemin', # Min value from AX node (for datetime/range)
+ 'valuemax', # Max value from AX node (for datetime/range)
'valuenow',
'keyshortcuts',
'haspopup',
@@ -96,6 +112,12 @@ STATIC_ATTRIBUTES = {
'itemscope',
'itemtype',
'itemprop',
+ # Webkit shadow DOM attributes
+ 'pseudo',
+ 'aria-valuemin',
+ 'aria-valuemax',
+ 'aria-valuenow',
+ 'aria-placeholder',
}
@@ -141,6 +163,7 @@ class SimplifiedNode:
ignored_by_paint_order: bool = False # More info in dom/serializer/paint_order.py
excluded_by_parent: bool = False # New field for bbox filtering
is_shadow_host: bool = False # New field for shadow DOM hosts
+ is_compound_component: bool = False # True for virtual components of compound controls
def _clean_original_node_json(self, node_json: dict) -> dict:
"""Recursively remove children_nodes and shadow_roots from original_node JSON."""
@@ -229,6 +252,7 @@ class EnhancedAXNode:
description: str | None
properties: list[EnhancedAXProperty] | None
+ child_ids: list[str] | None
@dataclass(slots=True)
@@ -354,6 +378,9 @@ class EnhancedDOMTreeNode:
# Interactive element index
element_index: int | None = None
+ # Compound control child components information
+ _compound_children: list[dict[str, Any]] = field(default_factory=list)
+
uuid: str = field(default_factory=uuid7str)
@property