Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

some optimizations to extraction

mertunsallcd71662ab8fd
diff --git a/browser_use/dom/serializer/clickable_elements.py b/browser_use/dom/serializer/clickable_elements.py
index 146651600..c40a6b005 100644
--- a/browser_use/dom/serializer/clickable_elements.py
+++ b/browser_use/dom/serializer/clickable_elements.py
@@ -84,7 +84,7 @@ class ClickableElementDetector:
# Skip properties we can't process
continue
- # ENHANCED TAG CHECK: Include SVG and other common interactive elements
+ # ENHANCED TAG CHECK: Include truly interactive elements
interactive_tags = {
'button',
'input',
@@ -96,21 +96,29 @@ class ClickableElementDetector:
'summary',
'option',
'optgroup',
- 'svg',
- 'path',
- 'circle',
- 'rect',
- 'polygon',
- 'ellipse',
}
if node.tag_name in interactive_tags:
return True
-
- # NAVIGATION ELEMENT CHECK: Include common navigation containers
- if node.tag_name in {'nav', 'header'} and node.attributes:
- # Navigation elements are often interactive if they have classes or roles
- if 'class' in node.attributes or 'role' in node.attributes:
- return True
+
+ # SVG elements need special handling - only interactive if they have explicit handlers
+ # svg_tags = {'svg', 'path', 'circle', 'rect', 'polygon', 'ellipse', 'line', 'polyline', 'g'}
+ # if node.tag_name in svg_tags:
+ # # Only consider SVG elements interactive if they have:
+ # # 1. Explicit event handlers
+ # # 2. Interactive role attributes
+ # # 3. Cursor pointer style
+ # if node.attributes:
+ # # Check for event handlers
+ # if any(attr.startswith('on') for attr in node.attributes):
+ # return True
+ # # Check for interactive roles
+ # if node.attributes.get('role') in {'button', 'link', 'menuitem'}:
+ # return True
+ # # Check for cursor pointer (indicating clickability)
+ # if node.attributes.get('style') and 'cursor: pointer' in node.attributes.get('style', ''):
+ # return True
+ # # Otherwise, SVG elements are decorative
+ # return False
# Tertiary check: elements with interactive attributes
if node.attributes:
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.py
index b3a749668..328a2b98a 100644
--- a/browser_use/dom/serializer/serializer.py
+++ b/browser_use/dom/serializer/serializer.py
@@ -3,7 +3,15 @@
from browser_use.dom.serializer.clickable_elements import ClickableElementDetector
from browser_use.dom.utils import cap_text_length
-from browser_use.dom.views import DOMSelectorMap, EnhancedDOMTreeNode, NodeType, SerializedDOMState, SimplifiedNode
+from browser_use.dom.views import (
+ DOMRect,
+ DOMSelectorMap,
+ EnhancedDOMTreeNode,
+ NodeType,
+ PropagatingBounds,
+ SerializedDOMState,
+ SimplifiedNode,
+)
DISABLED_ELEMENTS = {'style', 'script', 'head', 'meta', 'link', 'title'}
@@ -11,7 +19,28 @@ DISABLED_ELEMENTS = {'style', 'script', 'head', 'meta', 'link', 'title'}
class DOMTreeSerializer:
"""Serializes enhanced DOM trees to string format."""
- def __init__(self, root_node: EnhancedDOMTreeNode, previous_cached_state: SerializedDOMState | None = None):
+ # Configuration - elements that propagate bounds to their children
+ PROPAGATING_ELEMENTS = [
+ {'tag': 'a', 'role': None}, # Any <a> tag
+ {'tag': 'button', 'role': None}, # Any <button> tag
+ {'tag': 'div', 'role': 'button'}, # <div role="button">
+ {'tag': 'div', 'role': 'combobox'}, # <div role="combobox"> - dropdowns/selects
+ {'tag': 'span', 'role': 'button'}, # <span role="button">
+ {'tag': 'span', 'role': 'combobox'},# <span role="combobox">
+ {'tag': 'input', 'role': 'combobox'},# <input role="combobox"> - autocomplete inputs
+ {'tag': 'input', 'role': 'combobox'}, # <input type="text"> - text inputs with suggestions
+ #{'tag': 'div', 'role': 'link'}, # <div role="link">
+ #{'tag': 'span', 'role': 'link'}, # <span role="link">
+ ]
+ DEFAULT_CONTAINMENT_THRESHOLD = 0.99 # 99% containment by default
+
+ def __init__(
+ self,
+ root_node: EnhancedDOMTreeNode,
+ previous_cached_state: SerializedDOMState | None = None,
+ enable_bbox_filtering: bool = True,
+ containment_threshold: float | None = None,
+ ):
self.root_node = root_node
self._interactive_counter = 1
self._selector_map: DOMSelectorMap = {}
@@ -20,6 +49,9 @@ class DOMTreeSerializer:
self.timing_info: dict[str, float] = {}
# Cache for clickable element detection to avoid redundant calls
self._clickable_cache: dict[int, bool] = {}
+ # Bounding box filtering configuration
+ self.enable_bbox_filtering = enable_bbox_filtering
+ self.containment_threshold = containment_threshold or self.DEFAULT_CONTAINMENT_THRESHOLD
def serialize_accessible_elements(self) -> tuple[SerializedDOMState, dict[str, float]]:
import time
@@ -44,20 +76,29 @@ class DOMTreeSerializer:
end_step2 = time.time()
self.timing_info['optimize_tree'] = end_step2 - start_step2
- # # Step 3: Detect and group semantic elements
- # if optimized_tree:
- # self._detect_semantic_groups(optimized_tree)
+ # # Step 3: Detect and group semantic elements
+ # if optimized_tree:
+ # self._detect_semantic_groups(optimized_tree)
+
+ # Step 3: Apply bounding box filtering (NEW)
+ if self.enable_bbox_filtering and optimized_tree:
+ start_step3 = time.time()
+ filtered_tree = self._apply_bounding_box_filtering(optimized_tree)
+ end_step3 = time.time()
+ self.timing_info['bbox_filtering'] = end_step3 - start_step3
+ else:
+ filtered_tree = optimized_tree
# Step 4: Assign interactive indices to clickable elements
start_step4 = time.time()
- self._assign_interactive_indices_and_mark_new_nodes(optimized_tree)
+ self._assign_interactive_indices_and_mark_new_nodes(filtered_tree)
end_step4 = time.time()
self.timing_info['assign_interactive_indices'] = end_step4 - start_step4
end_total = time.time()
self.timing_info['serialize_accessible_elements_total'] = end_total - start_total
- return SerializedDOMState(_root=optimized_tree, selector_map=self._selector_map), self.timing_info
+ return SerializedDOMState(_root=filtered_tree, selector_map=self._selector_map), self.timing_info
def _is_interactive_cached(self, node: EnhancedDOMTreeNode) -> bool:
"""Cached version of clickable element detection to avoid redundant calls."""
@@ -187,33 +228,198 @@ class DOMTreeSerializer:
if not node:
return
- # Assign index to clickable elements that are also visible
- is_interactive_assign = self._is_interactive_cached(node.original_node)
- is_visible = node.original_node.snapshot_node and node.original_node.is_visible
+ # Skip assigning index to excluded nodes
+ if not (hasattr(node, 'excluded_by_parent') and node.excluded_by_parent):
+ # Assign index to clickable elements that are also visible
+ is_interactive_assign = self._is_interactive_cached(node.original_node)
+ is_visible = node.original_node.snapshot_node and node.original_node.is_visible
- # Only add to selector map if element is both interactive AND visible
- if is_interactive_assign and is_visible:
- node.interactive_index = self._interactive_counter
- node.original_node.element_index = self._interactive_counter
- self._selector_map[self._interactive_counter] = node.original_node
- self._interactive_counter += 1
+ # Only add to selector map if element is both interactive AND visible
+ if is_interactive_assign and is_visible:
+ node.interactive_index = self._interactive_counter
+ node.original_node.element_index = self._interactive_counter
+ self._selector_map[self._interactive_counter] = node.original_node
+ self._interactive_counter += 1
- # Check if node is new
- if self._previous_cached_selector_map:
- previous_backend_node_ids = {node.backend_node_id for node in self._previous_cached_selector_map.values()}
- if node.original_node.backend_node_id not in previous_backend_node_ids:
- node.is_new = True
+ # Check if node is new
+ if self._previous_cached_selector_map:
+ previous_backend_node_ids = {node.backend_node_id for node in self._previous_cached_selector_map.values()}
+ if node.original_node.backend_node_id not in previous_backend_node_ids:
+ node.is_new = True
# Process children
for child in node.children:
self._assign_interactive_indices_and_mark_new_nodes(child)
+ def _apply_bounding_box_filtering(self, node: SimplifiedNode | None) -> SimplifiedNode | None:
+ """Filter children contained within propagating parent bounds."""
+ if not node:
+ return None
+
+ # Start with no active bounds
+ self._filter_tree_recursive(node, active_bounds=None, depth=0)
+
+ # Log statistics
+ excluded_count = self._count_excluded_nodes(node)
+ if excluded_count > 0:
+ import logging
+ logging.debug(f"BBox filtering excluded {excluded_count} nodes")
+
+ return node
+
+ def _filter_tree_recursive(
+ self, node: SimplifiedNode, active_bounds: PropagatingBounds | None = None, depth: int = 0
+ ):
+ """
+ Recursively filter tree with bounding box propagation.
+ Bounds propagate to ALL descendants until overridden.
+ """
+
+ # Check if this node should be excluded by active bounds
+ if active_bounds and self._should_exclude_child(node, active_bounds):
+ node.excluded_by_parent = True
+ # Important: Still check if this node starts NEW propagation
+
+ # Check if this node starts new propagation (even if excluded!)
+ new_bounds = None
+ tag = node.original_node.tag_name.lower()
+ role = node.original_node.attributes.get('role') if node.original_node.attributes else None
+ attributes = {
+ 'tag': tag,
+ 'role': role,
+ }
+ # Check if this element matches any propagating element pattern
+ if self._is_propagating_element(attributes):
+ # This node propagates bounds to ALL its descendants
+ if node.original_node.snapshot_node and node.original_node.snapshot_node.bounds:
+ new_bounds = PropagatingBounds(
+ tag=tag,
+ bounds=node.original_node.snapshot_node.bounds,
+ node_id=node.original_node.node_id,
+ depth=depth,
+ )
+
+ # Propagate to ALL children
+ # Use new_bounds if this node starts propagation, otherwise continue with active_bounds
+ propagate_bounds = new_bounds if new_bounds else active_bounds
+
+ for child in node.children:
+ self._filter_tree_recursive(child, propagate_bounds, depth + 1)
+
+ def _should_exclude_child(self, node: SimplifiedNode, active_bounds: PropagatingBounds) -> bool:
+ """
+ Determine if child should be excluded based on propagating bounds.
+ """
+
+ # Never exclude text nodes - we always want to preserve text content
+ if node.original_node.node_type == NodeType.TEXT_NODE:
+ return False
+
+ # Get child bounds
+ if not node.original_node.snapshot_node or not node.original_node.snapshot_node.bounds:
+ return False # No bounds = can't determine containment
+
+ child_bounds = node.original_node.snapshot_node.bounds
+
+ # Check containment with configured threshold
+ if not self._is_contained(child_bounds, active_bounds.bounds, self.containment_threshold):
+ return False # Not sufficiently contained
+
+ # EXCEPTION RULES - Keep these even if contained:
+
+ child_tag = node.original_node.tag_name.lower()
+ child_role = node.original_node.attributes.get('role') if node.original_node.attributes else None
+ child_attributes = {
+ 'tag': child_tag,
+ 'role': child_role,
+ }
+
+ # 1. Never exclude form elements (they need individual interaction)
+ if child_tag in ['input', 'select', 'textarea', 'label']:
+ return False
+
+ # 2. Keep if child is also a propagating element
+ # (might have stopPropagation, e.g., button in button)
+ if self._is_propagating_element(child_attributes):
+ return False
+
+ # 3. Keep if has explicit onclick handler
+ if node.original_node.attributes and 'onclick' in node.original_node.attributes:
+ return False
+
+ # 4. Keep if has aria-label suggesting it's independently interactive
+ if node.original_node.attributes:
+ aria_label = node.original_node.attributes.get('aria-label')
+ if aria_label and aria_label.strip():
+ # Has meaningful aria-label, likely interactive
+ return False
+
+ # 5. Keep if has role suggesting interactivity
+ if node.original_node.attributes:
+ role = node.original_node.attributes.get('role')
+ if role in ['button', 'link', 'checkbox', 'radio', 'tab', 'menuitem']:
+ return False
+
+ # Default: exclude this child
+ return True
+
+ def _is_contained(self, child: DOMRect, parent: DOMRect, threshold: float) -> bool:
+ """
+ Check if child is contained within parent bounds.
+
+ Args:
+ threshold: Percentage (0.0-1.0) of child that must be within parent
+ """
+ # Calculate intersection
+ x_overlap = max(0, min(child.x + child.width, parent.x + parent.width) - max(child.x, parent.x))
+ y_overlap = max(0, min(child.y + child.height, parent.y + parent.height) - max(child.y, parent.y))
+
+ intersection_area = x_overlap * y_overlap
+ child_area = child.width * child.height
+
+ if child_area == 0:
+ return False # Zero-area element
+
+ containment_ratio = intersection_area / child_area
+ return containment_ratio >= threshold
+
+ def _count_excluded_nodes(self, node: SimplifiedNode, count: int = 0) -> int:
+ """Count how many nodes were excluded (for debugging)."""
+ if hasattr(node, 'excluded_by_parent') and node.excluded_by_parent:
+ count += 1
+ for child in node.children:
+ count = self._count_excluded_nodes(child, count)
+ return count
+
+ def _is_propagating_element(self, attributes: dict[str, str | None]) -> bool:
+ """
+ Check if an element should propagate bounds based on attributes.
+ If the element satisfies one of the patterns, it propagates bounds to all its children.
+ """
+ keys_to_check = ['tag', 'role']
+ for pattern in self.PROPAGATING_ELEMENTS:
+ # Check if the element satisfies the pattern
+ check = [pattern.get(key) is None or pattern.get(key) == attributes.get(key) for key in keys_to_check]
+ if all(check):
+ return True
+
+ return False
+
@staticmethod
def serialize_tree(node: SimplifiedNode | None, include_attributes: list[str], depth: int = 0) -> str:
"""Serialize the optimized tree to string format."""
if not node:
return ''
+ # Skip rendering excluded nodes, but process their children
+ if hasattr(node, 'excluded_by_parent') and node.excluded_by_parent:
+ formatted_text = []
+ for child in node.children:
+ child_text = DOMTreeSerializer.serialize_tree(child, include_attributes, depth)
+ if child_text:
+ formatted_text.append(child_text)
+ return '\n'.join(formatted_text)
+
formatted_text = []
depth_str = depth * '\t'
next_depth = depth
diff --git a/browser_use/dom/views.py b/browser_use/dom/views.py
index 2e6e778ba..0992e8552 100644
--- a/browser_use/dom/views.py
+++ b/browser_use/dom/views.py
@@ -49,6 +49,15 @@ class TargetAllTrees:
cdp_timing: dict[str, float]
+@dataclass(slots=True)
+class PropagatingBounds:
+ """Track bounds that propagate from parent elements to filter children."""
+ tag: str # The tag that started propagation ('a' or 'button')
+ bounds: 'DOMRect' # The bounding box
+ node_id: int # Node ID for debugging
+ depth: int # How deep in tree this started (for debugging)
+
+
@dataclass(slots=True)
class SimplifiedNode:
"""Simplified tree node for optimization."""
@@ -59,6 +68,7 @@ class SimplifiedNode:
interactive_index: int | None = None
is_new: bool = False
+ excluded_by_parent: bool = False # New field for bbox filtering
def __json__(self) -> dict:
original_node_json = self.original_node.__json__()