Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
paint order filtering
Gregor Žuničc2eede8e20b3
diff --git a/browser_use/browser/profile.py b/browser_use/browser/profile.pyindex 7507d7203..04be517fa 100644--- a/browser_use/browser/profile.py+++ b/browser_use/browser/profile.py@@ -604,6 +604,7 @@ class BrowserProfile(BrowserConnectArgs, BrowserLaunchPersistentContextArgs, Brofilter_highlight_ids: bool = Field(default=True, description='Only show element IDs in highlights if llm_representation is less than 10 characters.')+ paint_order_filtering: bool = Field(default=True, description='Enable paint order filtering. Experimental for now.')# --- Downloads ---auto_download_pdfs: bool = Field(default=True, description='Automatically download PDFs when navigating to PDF viewer pages.')diff --git a/browser_use/browser/session.py b/browser_use/browser/session.pyindex 70c7235e7..205edbb5c 100644--- a/browser_use/browser/session.py+++ b/browser_use/browser/session.py@@ -259,15 +259,17 @@ class BrowserSession(BaseModel):enable_default_extensions: bool | None = None,window_size: dict | None = None,window_position: dict | None = None,- cross_origin_iframes: bool | None = None,minimum_wait_page_load_time: float | None = None,wait_for_network_idle_page_load_time: float | None = None,wait_between_actions: float | None = None,- highlight_elements: bool | None = None,filter_highlight_ids: bool | None = None,auto_download_pdfs: bool | None = None,profile_directory: str | None = None,cookie_whitelist_domains: list[str] | None = None,+ # DOM extraction layer configuration+ cross_origin_iframes: bool | None = None,+ highlight_elements: bool | None = None,+ paint_order_filtering: bool | None = None,):# Following the same pattern as AgentSettings in service.py# Only pass non-None values to avoid validation errorsdiff --git a/browser_use/browser/watchdogs/dom_watchdog.py b/browser_use/browser/watchdogs/dom_watchdog.pyindex 153c861f4..3c14f13cc 100644--- a/browser_use/browser/watchdogs/dom_watchdog.py+++ b/browser_use/browser/watchdogs/dom_watchdog.py@@ -360,6 +360,7 @@ class DOMWatchdog(BaseWatchdog):browser_session=self.browser_session,logger=self.logger,cross_origin_iframes=self.browser_session.browser_profile.cross_origin_iframes,+ paint_order_filtering=self.browser_session.browser_profile.paint_order_filtering,)# Get serialized DOM tree using the servicediff --git a/browser_use/dom/enhanced_snapshot.py b/browser_use/dom/enhanced_snapshot.pyindex 6c08c637a..6fe0e814a 100644--- a/browser_use/dom/enhanced_snapshot.py+++ b/browser_use/dom/enhanced_snapshot.py@@ -26,6 +26,7 @@ REQUIRED_COMPUTED_STYLES = ['cursor', # Used in enhanced_snapshot.py cursor extraction'pointer-events', # Used for clickability logic'position', # Used for visibility logic+ 'background-color', # Used for visibility logic]diff --git a/browser_use/dom/playground/extraction.py b/browser_use/dom/playground/extraction.pyindex 459e81771..a62d2a8e0 100644--- a/browser_use/dom/playground/extraction.py+++ b/browser_use/dom/playground/extraction.py@@ -27,11 +27,14 @@ async def test_focus_vs_all_elements():wait_for_network_idle_page_load_time=1,headless=False,args=['--incognito'],+ paint_order_filtering=True,),)# 10 Sample websites with various interactive elementssample_websites = [+ 'https://www.google.com/travel/flights',+ 'https://v0-simple-ui-test-site.vercel.app','https://browser-use.github.io/stress-tests/challenges/iframe-inception-level1.html','https://browser-use.github.io/stress-tests/challenges/angular-form.html','https://www.google.com/travel/flights',@@ -194,7 +197,7 @@ async def test_focus_vs_all_elements():# Calculate percentagestotal_time = all_timing.get('get_state_summary_total', 0)- if total_time > 0:+ if total_time > 0 and total_elements > 0:timing_text += '\n📈 PERCENTAGE BREAKDOWN:\n'timing_text += f'{"─" * 30}\n'for key, value in all_timing.items():@@ -205,7 +208,7 @@ async def test_focus_vs_all_elements():timing_text += '\n🎯 CLICKABLE DETECTION ANALYSIS:\n'timing_text += f'{"─" * 35}\n'clickable_time = all_timing.get('clickable_detection_time', 0)- if clickable_time > 0:+ if clickable_time > 0 and total_elements > 0:avg_per_element = (clickable_time / total_elements) * 1000000 # microsecondstiming_text += f'Total clickable detection time: {clickable_time * 1000:.2f} ms\n'timing_text += f'Average per element: {avg_per_element:.2f} μs\n'diff --git a/browser_use/dom/serializer/paint_order.py b/browser_use/dom/serializer/paint_order.pynew file mode 100644index 000000000..d82d14401--- /dev/null+++ b/browser_use/dom/serializer/paint_order.py@@ -0,0 +1,197 @@+from collections import defaultdict+from dataclasses import dataclass++from browser_use.dom.views import SimplifiedNode++"""+Helper class for maintaining a union of rectangles (used for order of elements calculation)+"""+++@dataclass(frozen=True, slots=True)+class Rect:+ """Closed axis-aligned rectangle with (x1,y1) bottom-left, (x2,y2) top-right."""++ x1: float+ y1: float+ x2: float+ y2: float++ def __post_init__(self):+ if not (self.x1 <= self.x2 and self.y1 <= self.y2):+ return False++ # --- fast relations ----------------------------------------------------+ def area(self) -> float:+ return (self.x2 - self.x1) * (self.y2 - self.y1)++ def intersects(self, other: 'Rect') -> bool:+ return not (self.x2 <= other.x1 or other.x2 <= self.x1 or self.y2 <= other.y1 or other.y2 <= self.y1)++ def contains(self, other: 'Rect') -> bool:+ return self.x1 <= other.x1 and self.y1 <= other.y1 and self.x2 >= other.x2 and self.y2 >= other.y2+++class RectUnionPure:+ """+ Maintains a *disjoint* set of rectangles.+ No external dependencies - fine for a few thousand rectangles.+ """++ __slots__ = ('_rects',)++ def __init__(self):+ self._rects: list[Rect] = []++ # -----------------------------------------------------------------+ def _split_diff(self, a: Rect, b: Rect) -> list[Rect]:+ r"""+ Return list of up to 4 rectangles = a \ b.+ Assumes a intersects b.+ """+ parts = []++ # Bottom slice+ if a.y1 < b.y1:+ parts.append(Rect(a.x1, a.y1, a.x2, b.y1))+ # Top slice+ if b.y2 < a.y2:+ parts.append(Rect(a.x1, b.y2, a.x2, a.y2))++ # Middle (vertical) strip: y overlap is [max(a.y1,b.y1), min(a.y2,b.y2)]+ y_lo = max(a.y1, b.y1)+ y_hi = min(a.y2, b.y2)++ # Left slice+ if a.x1 < b.x1:+ parts.append(Rect(a.x1, y_lo, b.x1, y_hi))+ # Right slice+ if b.x2 < a.x2:+ parts.append(Rect(b.x2, y_lo, a.x2, y_hi))++ return parts++ # -----------------------------------------------------------------+ def contains(self, r: Rect) -> bool:+ """+ True iff r is fully covered by the current union.+ """+ if not self._rects:+ return False++ stack = [r]+ for s in self._rects:+ new_stack = []+ for piece in stack:+ if s.contains(piece):+ # piece completely gone+ continue+ if piece.intersects(s):+ new_stack.extend(self._split_diff(piece, s))+ else:+ new_stack.append(piece)+ if not new_stack: # everything eaten – covered+ return True+ stack = new_stack+ return False # something survived++ # -----------------------------------------------------------------+ def add(self, r: Rect) -> bool:+ """+ Insert r unless it is already covered.+ Returns True if the union grew.+ """+ if self.contains(r):+ return False++ pending = [r]+ i = 0+ while i < len(self._rects):+ s = self._rects[i]+ new_pending = []+ changed = False+ for piece in pending:+ if piece.intersects(s):+ new_pending.extend(self._split_diff(piece, s))+ changed = True+ else:+ new_pending.append(piece)+ pending = new_pending+ if changed:+ # s unchanged; proceed with next existing rectangle+ i += 1+ else:+ i += 1++ # Any left‑over pieces are new, non‑overlapping areas+ self._rects.extend(pending)+ return True+++class PaintOrderRemover:+ """+ Calculates which elements should be removed based on the paint order parameter.+ """++ def __init__(self, root: SimplifiedNode):+ self.root = root++ def calculate_paint_order(self) -> None:+ all_simplified_nodes_with_paint_order: list[SimplifiedNode] = []++ def collect_paint_order(node: SimplifiedNode) -> None:+ if (+ node.original_node.snapshot_node+ and node.original_node.snapshot_node.paint_order is not None+ and node.original_node.snapshot_node.bounds is not None+ ):+ all_simplified_nodes_with_paint_order.append(node)++ for child in node.children:+ collect_paint_order(child)++ collect_paint_order(self.root)++ grouped_by_paint_order: defaultdict[int, list[SimplifiedNode]] = defaultdict(list)++ for node in all_simplified_nodes_with_paint_order:+ if node.original_node.snapshot_node and node.original_node.snapshot_node.paint_order is not None:+ grouped_by_paint_order[node.original_node.snapshot_node.paint_order].append(node)++ rect_union = RectUnionPure()++ for paint_order, nodes in sorted(grouped_by_paint_order.items(), key=lambda x: -x[0]):+ rects_to_add = []++ for node in nodes:+ if not node.original_node.snapshot_node or not node.original_node.snapshot_node.bounds:+ continue # shouldn't happen by how we filter them out in the first place++ rect = Rect(+ x1=node.original_node.snapshot_node.bounds.x,+ y1=node.original_node.snapshot_node.bounds.y,+ x2=node.original_node.snapshot_node.bounds.x + node.original_node.snapshot_node.bounds.width,+ y2=node.original_node.snapshot_node.bounds.y + node.original_node.snapshot_node.bounds.height,+ )++ if rect_union.contains(rect):+ node.ignored_by_paint_order = True++ # don't add to the nodes if opacity is less then 0.95 or background-color is transparent+ if (+ node.original_node.snapshot_node.computed_styles+ and node.original_node.snapshot_node.computed_styles.get('background-color', 'rgba(0, 0, 0, 0)')+ == 'rgba(0, 0, 0, 0)'+ ) or (+ node.original_node.snapshot_node.computed_styles+ and float(node.original_node.snapshot_node.computed_styles.get('opacity', '1'))+ < 0.8 # this is highly vibes based number+ ):+ continue++ rects_to_add.append(rect)++ for rect in rects_to_add:+ rect_union.add(rect)++ return Nonediff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.pyindex ec74e0702..8b0084c3b 100644--- a/browser_use/dom/serializer/serializer.py+++ b/browser_use/dom/serializer/serializer.py@@ -2,6 +2,7 @@from browser_use.dom.serializer.clickable_elements import ClickableElementDetector+from browser_use.dom.serializer.paint_order import PaintOrderRemoverfrom browser_use.dom.utils import cap_text_lengthfrom browser_use.dom.views import (DOMRect,@@ -40,6 +41,7 @@ class DOMTreeSerializer:previous_cached_state: SerializedDOMState | None = None,enable_bbox_filtering: bool = True,containment_threshold: float | None = None,+ paint_order_filtering: bool = True,):self.root_node = root_nodeself._interactive_counter = 1@@ -52,6 +54,8 @@ class DOMTreeSerializer:# Bounding box filtering configurationself.enable_bbox_filtering = enable_bbox_filteringself.containment_threshold = containment_threshold or self.DEFAULT_CONTAINMENT_THRESHOLD+ # Paint order filtering configuration+ self.paint_order_filtering = paint_order_filteringdef serialize_accessible_elements(self) -> tuple[SerializedDOMState, dict[str, float]]:import time@@ -70,16 +74,19 @@ class DOMTreeSerializer:end_step1 = time.time()self.timing_info['create_simplified_tree'] = end_step1 - start_step1+ # Step 3: Remove elements based on paint order+ start_step3 = time.time()+ if self.paint_order_filtering and simplified_tree:+ PaintOrderRemover(simplified_tree).calculate_paint_order()+ end_step3 = time.time()+ self.timing_info['calculate_paint_order'] = end_step3 - start_step3+# Step 2: Optimize tree (remove unnecessary parents)start_step2 = time.time()optimized_tree = self._optimize_tree(simplified_tree)end_step2 = time.time()self.timing_info['optimize_tree'] = end_step2 - start_step2- # # Step 3: Detect and group semantic elements- # if optimized_tree:- # self._detect_semantic_groups(optimized_tree)-# Step 3: Apply bounding box filtering (NEW)if self.enable_bbox_filtering and optimized_tree:start_step3 = time.time()@@ -148,20 +155,16 @@ class DOMTreeSerializer:simplified = SimplifiedNode(original_node=node, children=[])for child in node.content_document.children_nodes or []:simplified_child = self._create_simplified_tree(child)- if simplified_child:+ if simplified_child is not None:simplified.children.append(simplified_child)return simplified- # Use enhanced scoring for inclusion decision- is_interactive = self._is_interactive_cached(node)-- is_visible = node.snapshot_node and node.is_visible+ is_visible = node.is_visibleis_scrollable = node.is_actually_scrollable# Include if interactive (regardless of visibility), or scrollable, or has children to process- should_include = (is_interactive and is_visible) or is_scrollable or bool(node.children_and_shadow_roots)- if should_include:+ if is_visible or is_scrollable or bool(node.children_and_shadow_roots):simplified = SimplifiedNode(original_node=node, children=[])# simplified._analysis = analysis # Store analysis for grouping@@ -172,7 +175,7 @@ class DOMTreeSerializer:simplified.children.append(simplified_child)# Return if meaningful or has meaningful children- if (is_interactive and is_visible) or is_scrollable or simplified.children:+ if is_visible or is_scrollable or simplified.children:return simplifiedelif node.node_type == NodeType.TEXT_NODE:@@ -202,7 +205,7 @@ class DOMTreeSerializer:is_visible = node.original_node.snapshot_node and node.original_node.is_visibleif (- (is_interactive_opt and is_visible) # Only keep interactive nodes that are visible+ is_visible # Keep all visible nodesor node.original_node.is_actually_scrollableor node.original_node.node_type == NodeType.TEXT_NODEor node.children@@ -228,8 +231,8 @@ class DOMTreeSerializer:if not node:return- # Skip assigning index to excluded nodes- if not (hasattr(node, 'excluded_by_parent') and node.excluded_by_parent):+ # Skip assigning index to excluded nodes, or ignored by paint order+ if not node.excluded_by_parent and not node.ignored_by_paint_order:# Assign index to clickable elements that are also visibleis_interactive_assign = self._is_interactive_cached(node.original_node)is_visible = node.original_node.snapshot_node and node.original_node.is_visiblediff --git a/browser_use/dom/service.py b/browser_use/dom/service.pyindex 155c2d498..2f20735a2 100644--- a/browser_use/dom/service.py+++ b/browser_use/dom/service.py@@ -40,11 +40,16 @@ class DomService:logger: logging.Loggerdef __init__(- self, browser_session: 'BrowserSession', logger: logging.Logger | None = None, cross_origin_iframes: bool = False+ self,+ browser_session: 'BrowserSession',+ logger: logging.Logger | None = None,+ cross_origin_iframes: bool = False,+ paint_order_filtering: bool = True,):self.browser_session = browser_sessionself.logger = logger or browser_session.loggerself.cross_origin_iframes = cross_origin_iframes+ self.paint_order_filtering = paint_order_filteringasync def __aenter__(self):return self@@ -670,7 +675,7 @@ class DomService:start = time.time()serialized_dom_state, serializer_timing = DOMTreeSerializer(- enhanced_dom_tree, previous_cached_state+ enhanced_dom_tree, previous_cached_state, paint_order_filtering=self.paint_order_filtering).serialize_accessible_elements()end = time.time()diff --git a/browser_use/dom/views.py b/browser_use/dom/views.pyindex 62b9222c3..06b1e8b32 100644--- a/browser_use/dom/views.py+++ b/browser_use/dom/views.py@@ -90,6 +90,8 @@ class SimplifiedNode:interactive_index: int | None = Noneis_new: bool = False++ ignored_by_paint_order: bool = False # More info in dom/serializer/paint_order.pyexcluded_by_parent: bool = False # New field for bbox filteringdef _clean_original_node_json(self, node_json: dict) -> dict:@@ -113,6 +115,8 @@ class SimplifiedNode:return {'should_display': self.should_display,'interactive_index': self.interactive_index,+ 'ignored_by_paint_order': self.ignored_by_paint_order,+ 'excluded_by_parent': self.excluded_by_parent,'original_node': cleaned_original_node_json,'children': [c.__json__() for c in self.children],}@@ -372,6 +376,7 @@ class EnhancedDOMTreeNode:'node_type': self.node_type.name,'node_name': self.node_name,'node_value': self.node_value,+ 'is_visible': self.is_visible,'attributes': self.attributes,'is_scrollable': self.is_scrollable,'session_id': self.session_id,