Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
some optimizations to extraction
mertunsallcd71662ab8fd
diff --git a/browser_use/dom/serializer/clickable_elements.py b/browser_use/dom/serializer/clickable_elements.pyindex 146651600..c40a6b005 100644--- a/browser_use/dom/serializer/clickable_elements.py+++ b/browser_use/dom/serializer/clickable_elements.py@@ -84,7 +84,7 @@ class ClickableElementDetector:# Skip properties we can't processcontinue- # ENHANCED TAG CHECK: Include SVG and other common interactive elements+ # ENHANCED TAG CHECK: Include truly interactive elementsinteractive_tags = {'button','input',@@ -96,21 +96,29 @@ class ClickableElementDetector:'summary','option','optgroup',- 'svg',- 'path',- 'circle',- 'rect',- 'polygon',- 'ellipse',}if node.tag_name in interactive_tags:return True-- # NAVIGATION ELEMENT CHECK: Include common navigation containers- if node.tag_name in {'nav', 'header'} and node.attributes:- # Navigation elements are often interactive if they have classes or roles- if 'class' in node.attributes or 'role' in node.attributes:- return True++ # SVG elements need special handling - only interactive if they have explicit handlers+ # svg_tags = {'svg', 'path', 'circle', 'rect', 'polygon', 'ellipse', 'line', 'polyline', 'g'}+ # if node.tag_name in svg_tags:+ # # Only consider SVG elements interactive if they have:+ # # 1. Explicit event handlers+ # # 2. Interactive role attributes+ # # 3. Cursor pointer style+ # if node.attributes:+ # # Check for event handlers+ # if any(attr.startswith('on') for attr in node.attributes):+ # return True+ # # Check for interactive roles+ # if node.attributes.get('role') in {'button', 'link', 'menuitem'}:+ # return True+ # # Check for cursor pointer (indicating clickability)+ # if node.attributes.get('style') and 'cursor: pointer' in node.attributes.get('style', ''):+ # return True+ # # Otherwise, SVG elements are decorative+ # return False# Tertiary check: elements with interactive attributesif node.attributes:diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.pyindex b3a749668..328a2b98a 100644--- a/browser_use/dom/serializer/serializer.py+++ b/browser_use/dom/serializer/serializer.py@@ -3,7 +3,15 @@from browser_use.dom.serializer.clickable_elements import ClickableElementDetectorfrom browser_use.dom.utils import cap_text_length-from browser_use.dom.views import DOMSelectorMap, EnhancedDOMTreeNode, NodeType, SerializedDOMState, SimplifiedNode+from browser_use.dom.views import (+ DOMRect,+ DOMSelectorMap,+ EnhancedDOMTreeNode,+ NodeType,+ PropagatingBounds,+ SerializedDOMState,+ SimplifiedNode,+)DISABLED_ELEMENTS = {'style', 'script', 'head', 'meta', 'link', 'title'}@@ -11,7 +19,28 @@ DISABLED_ELEMENTS = {'style', 'script', 'head', 'meta', 'link', 'title'}class DOMTreeSerializer:"""Serializes enhanced DOM trees to string format."""- def __init__(self, root_node: EnhancedDOMTreeNode, previous_cached_state: SerializedDOMState | None = None):+ # Configuration - elements that propagate bounds to their children+ PROPAGATING_ELEMENTS = [+ {'tag': 'a', 'role': None}, # Any <a> tag+ {'tag': 'button', 'role': None}, # Any <button> tag+ {'tag': 'div', 'role': 'button'}, # <div role="button">+ {'tag': 'div', 'role': 'combobox'}, # <div role="combobox"> - dropdowns/selects+ {'tag': 'span', 'role': 'button'}, # <span role="button">+ {'tag': 'span', 'role': 'combobox'},# <span role="combobox">+ {'tag': 'input', 'role': 'combobox'},# <input role="combobox"> - autocomplete inputs+ {'tag': 'input', 'role': 'combobox'}, # <input type="text"> - text inputs with suggestions+ #{'tag': 'div', 'role': 'link'}, # <div role="link">+ #{'tag': 'span', 'role': 'link'}, # <span role="link">+ ]+ DEFAULT_CONTAINMENT_THRESHOLD = 0.99 # 99% containment by default++ def __init__(+ self,+ root_node: EnhancedDOMTreeNode,+ previous_cached_state: SerializedDOMState | None = None,+ enable_bbox_filtering: bool = True,+ containment_threshold: float | None = None,+ ):self.root_node = root_nodeself._interactive_counter = 1self._selector_map: DOMSelectorMap = {}@@ -20,6 +49,9 @@ class DOMTreeSerializer:self.timing_info: dict[str, float] = {}# Cache for clickable element detection to avoid redundant callsself._clickable_cache: dict[int, bool] = {}+ # Bounding box filtering configuration+ self.enable_bbox_filtering = enable_bbox_filtering+ self.containment_threshold = containment_threshold or self.DEFAULT_CONTAINMENT_THRESHOLDdef serialize_accessible_elements(self) -> tuple[SerializedDOMState, dict[str, float]]:import time@@ -44,20 +76,29 @@ class DOMTreeSerializer:end_step2 = time.time()self.timing_info['optimize_tree'] = end_step2 - start_step2- # # Step 3: Detect and group semantic elements- # if optimized_tree:- # self._detect_semantic_groups(optimized_tree)+ # # Step 3: Detect and group semantic elements+ # if optimized_tree:+ # self._detect_semantic_groups(optimized_tree)++ # Step 3: Apply bounding box filtering (NEW)+ if self.enable_bbox_filtering and optimized_tree:+ start_step3 = time.time()+ filtered_tree = self._apply_bounding_box_filtering(optimized_tree)+ end_step3 = time.time()+ self.timing_info['bbox_filtering'] = end_step3 - start_step3+ else:+ filtered_tree = optimized_tree# Step 4: Assign interactive indices to clickable elementsstart_step4 = time.time()- self._assign_interactive_indices_and_mark_new_nodes(optimized_tree)+ self._assign_interactive_indices_and_mark_new_nodes(filtered_tree)end_step4 = time.time()self.timing_info['assign_interactive_indices'] = end_step4 - start_step4end_total = time.time()self.timing_info['serialize_accessible_elements_total'] = end_total - start_total- return SerializedDOMState(_root=optimized_tree, selector_map=self._selector_map), self.timing_info+ return SerializedDOMState(_root=filtered_tree, selector_map=self._selector_map), self.timing_infodef _is_interactive_cached(self, node: EnhancedDOMTreeNode) -> bool:"""Cached version of clickable element detection to avoid redundant calls."""@@ -187,33 +228,198 @@ class DOMTreeSerializer:if not node:return- # Assign index to clickable elements that are also visible- is_interactive_assign = self._is_interactive_cached(node.original_node)- is_visible = node.original_node.snapshot_node and node.original_node.is_visible+ # Skip assigning index to excluded nodes+ if not (hasattr(node, 'excluded_by_parent') and node.excluded_by_parent):+ # Assign index to clickable elements that are also visible+ is_interactive_assign = self._is_interactive_cached(node.original_node)+ is_visible = node.original_node.snapshot_node and node.original_node.is_visible- # Only add to selector map if element is both interactive AND visible- if is_interactive_assign and is_visible:- node.interactive_index = self._interactive_counter- node.original_node.element_index = self._interactive_counter- self._selector_map[self._interactive_counter] = node.original_node- self._interactive_counter += 1+ # Only add to selector map if element is both interactive AND visible+ if is_interactive_assign and is_visible:+ node.interactive_index = self._interactive_counter+ node.original_node.element_index = self._interactive_counter+ self._selector_map[self._interactive_counter] = node.original_node+ self._interactive_counter += 1- # Check if node is new- if self._previous_cached_selector_map:- previous_backend_node_ids = {node.backend_node_id for node in self._previous_cached_selector_map.values()}- if node.original_node.backend_node_id not in previous_backend_node_ids:- node.is_new = True+ # Check if node is new+ if self._previous_cached_selector_map:+ previous_backend_node_ids = {node.backend_node_id for node in self._previous_cached_selector_map.values()}+ if node.original_node.backend_node_id not in previous_backend_node_ids:+ node.is_new = True# Process childrenfor child in node.children:self._assign_interactive_indices_and_mark_new_nodes(child)+ def _apply_bounding_box_filtering(self, node: SimplifiedNode | None) -> SimplifiedNode | None:+ """Filter children contained within propagating parent bounds."""+ if not node:+ return None++ # Start with no active bounds+ self._filter_tree_recursive(node, active_bounds=None, depth=0)++ # Log statistics+ excluded_count = self._count_excluded_nodes(node)+ if excluded_count > 0:+ import logging+ logging.debug(f"BBox filtering excluded {excluded_count} nodes")++ return node++ def _filter_tree_recursive(+ self, node: SimplifiedNode, active_bounds: PropagatingBounds | None = None, depth: int = 0+ ):+ """+ Recursively filter tree with bounding box propagation.+ Bounds propagate to ALL descendants until overridden.+ """++ # Check if this node should be excluded by active bounds+ if active_bounds and self._should_exclude_child(node, active_bounds):+ node.excluded_by_parent = True+ # Important: Still check if this node starts NEW propagation++ # Check if this node starts new propagation (even if excluded!)+ new_bounds = None+ tag = node.original_node.tag_name.lower()+ role = node.original_node.attributes.get('role') if node.original_node.attributes else None+ attributes = {+ 'tag': tag,+ 'role': role,+ }+ # Check if this element matches any propagating element pattern+ if self._is_propagating_element(attributes):+ # This node propagates bounds to ALL its descendants+ if node.original_node.snapshot_node and node.original_node.snapshot_node.bounds:+ new_bounds = PropagatingBounds(+ tag=tag,+ bounds=node.original_node.snapshot_node.bounds,+ node_id=node.original_node.node_id,+ depth=depth,+ )++ # Propagate to ALL children+ # Use new_bounds if this node starts propagation, otherwise continue with active_bounds+ propagate_bounds = new_bounds if new_bounds else active_bounds++ for child in node.children:+ self._filter_tree_recursive(child, propagate_bounds, depth + 1)++ def _should_exclude_child(self, node: SimplifiedNode, active_bounds: PropagatingBounds) -> bool:+ """+ Determine if child should be excluded based on propagating bounds.+ """++ # Never exclude text nodes - we always want to preserve text content+ if node.original_node.node_type == NodeType.TEXT_NODE:+ return False++ # Get child bounds+ if not node.original_node.snapshot_node or not node.original_node.snapshot_node.bounds:+ return False # No bounds = can't determine containment++ child_bounds = node.original_node.snapshot_node.bounds++ # Check containment with configured threshold+ if not self._is_contained(child_bounds, active_bounds.bounds, self.containment_threshold):+ return False # Not sufficiently contained++ # EXCEPTION RULES - Keep these even if contained:++ child_tag = node.original_node.tag_name.lower()+ child_role = node.original_node.attributes.get('role') if node.original_node.attributes else None+ child_attributes = {+ 'tag': child_tag,+ 'role': child_role,+ }++ # 1. Never exclude form elements (they need individual interaction)+ if child_tag in ['input', 'select', 'textarea', 'label']:+ return False++ # 2. Keep if child is also a propagating element+ # (might have stopPropagation, e.g., button in button)+ if self._is_propagating_element(child_attributes):+ return False++ # 3. Keep if has explicit onclick handler+ if node.original_node.attributes and 'onclick' in node.original_node.attributes:+ return False++ # 4. Keep if has aria-label suggesting it's independently interactive+ if node.original_node.attributes:+ aria_label = node.original_node.attributes.get('aria-label')+ if aria_label and aria_label.strip():+ # Has meaningful aria-label, likely interactive+ return False++ # 5. Keep if has role suggesting interactivity+ if node.original_node.attributes:+ role = node.original_node.attributes.get('role')+ if role in ['button', 'link', 'checkbox', 'radio', 'tab', 'menuitem']:+ return False++ # Default: exclude this child+ return True++ def _is_contained(self, child: DOMRect, parent: DOMRect, threshold: float) -> bool:+ """+ Check if child is contained within parent bounds.++ Args:+ threshold: Percentage (0.0-1.0) of child that must be within parent+ """+ # Calculate intersection+ x_overlap = max(0, min(child.x + child.width, parent.x + parent.width) - max(child.x, parent.x))+ y_overlap = max(0, min(child.y + child.height, parent.y + parent.height) - max(child.y, parent.y))++ intersection_area = x_overlap * y_overlap+ child_area = child.width * child.height++ if child_area == 0:+ return False # Zero-area element++ containment_ratio = intersection_area / child_area+ return containment_ratio >= threshold++ def _count_excluded_nodes(self, node: SimplifiedNode, count: int = 0) -> int:+ """Count how many nodes were excluded (for debugging)."""+ if hasattr(node, 'excluded_by_parent') and node.excluded_by_parent:+ count += 1+ for child in node.children:+ count = self._count_excluded_nodes(child, count)+ return count++ def _is_propagating_element(self, attributes: dict[str, str | None]) -> bool:+ """+ Check if an element should propagate bounds based on attributes.+ If the element satisfies one of the patterns, it propagates bounds to all its children.+ """+ keys_to_check = ['tag', 'role']+ for pattern in self.PROPAGATING_ELEMENTS:+ # Check if the element satisfies the pattern+ check = [pattern.get(key) is None or pattern.get(key) == attributes.get(key) for key in keys_to_check]+ if all(check):+ return True++ return False+@staticmethoddef serialize_tree(node: SimplifiedNode | None, include_attributes: list[str], depth: int = 0) -> str:"""Serialize the optimized tree to string format."""if not node:return ''+ # Skip rendering excluded nodes, but process their children+ if hasattr(node, 'excluded_by_parent') and node.excluded_by_parent:+ formatted_text = []+ for child in node.children:+ child_text = DOMTreeSerializer.serialize_tree(child, include_attributes, depth)+ if child_text:+ formatted_text.append(child_text)+ return '\n'.join(formatted_text)+formatted_text = []depth_str = depth * '\t'next_depth = depthdiff --git a/browser_use/dom/views.py b/browser_use/dom/views.pyindex 2e6e778ba..0992e8552 100644--- a/browser_use/dom/views.py+++ b/browser_use/dom/views.py@@ -49,6 +49,15 @@ class TargetAllTrees:cdp_timing: dict[str, float]+@dataclass(slots=True)+class PropagatingBounds:+ """Track bounds that propagate from parent elements to filter children."""+ tag: str # The tag that started propagation ('a' or 'button')+ bounds: 'DOMRect' # The bounding box+ node_id: int # Node ID for debugging+ depth: int # How deep in tree this started (for debugging)++@dataclass(slots=True)class SimplifiedNode:"""Simplified tree node for optimization."""@@ -59,6 +68,7 @@ class SimplifiedNode:interactive_index: int | None = Noneis_new: bool = False+ excluded_by_parent: bool = False # New field for bbox filteringdef __json__(self) -> dict:original_node_json = self.original_node.__json__()