Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

paint order filtering

Gregor Žuničc2eede8e20b3
diff --git a/browser_use/browser/profile.py b/browser_use/browser/profile.py
index 7507d7203..04be517fa 100644
--- a/browser_use/browser/profile.py
+++ b/browser_use/browser/profile.py
@@ -604,6 +604,7 @@ class BrowserProfile(BrowserConnectArgs, BrowserLaunchPersistentContextArgs, Bro
filter_highlight_ids: bool = Field(
default=True, description='Only show element IDs in highlights if llm_representation is less than 10 characters.'
)
+ paint_order_filtering: bool = Field(default=True, description='Enable paint order filtering. Experimental for now.')
# --- Downloads ---
auto_download_pdfs: bool = Field(default=True, description='Automatically download PDFs when navigating to PDF viewer pages.')
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 70c7235e7..205edbb5c 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -259,15 +259,17 @@ class BrowserSession(BaseModel):
enable_default_extensions: bool | None = None,
window_size: dict | None = None,
window_position: dict | None = None,
- cross_origin_iframes: bool | None = None,
minimum_wait_page_load_time: float | None = None,
wait_for_network_idle_page_load_time: float | None = None,
wait_between_actions: float | None = None,
- highlight_elements: bool | None = None,
filter_highlight_ids: bool | None = None,
auto_download_pdfs: bool | None = None,
profile_directory: str | None = None,
cookie_whitelist_domains: list[str] | None = None,
+ # DOM extraction layer configuration
+ cross_origin_iframes: bool | None = None,
+ highlight_elements: bool | None = None,
+ paint_order_filtering: bool | None = None,
):
# Following the same pattern as AgentSettings in service.py
# Only pass non-None values to avoid validation errors
diff --git a/browser_use/browser/watchdogs/dom_watchdog.py b/browser_use/browser/watchdogs/dom_watchdog.py
index 153c861f4..3c14f13cc 100644
--- a/browser_use/browser/watchdogs/dom_watchdog.py
+++ b/browser_use/browser/watchdogs/dom_watchdog.py
@@ -360,6 +360,7 @@ class DOMWatchdog(BaseWatchdog):
browser_session=self.browser_session,
logger=self.logger,
cross_origin_iframes=self.browser_session.browser_profile.cross_origin_iframes,
+ paint_order_filtering=self.browser_session.browser_profile.paint_order_filtering,
)
# Get serialized DOM tree using the service
diff --git a/browser_use/dom/enhanced_snapshot.py b/browser_use/dom/enhanced_snapshot.py
index 6c08c637a..6fe0e814a 100644
--- a/browser_use/dom/enhanced_snapshot.py
+++ b/browser_use/dom/enhanced_snapshot.py
@@ -26,6 +26,7 @@ REQUIRED_COMPUTED_STYLES = [
'cursor', # Used in enhanced_snapshot.py cursor extraction
'pointer-events', # Used for clickability logic
'position', # Used for visibility logic
+ 'background-color', # Used for visibility logic
]
diff --git a/browser_use/dom/playground/extraction.py b/browser_use/dom/playground/extraction.py
index 459e81771..a62d2a8e0 100644
--- a/browser_use/dom/playground/extraction.py
+++ b/browser_use/dom/playground/extraction.py
@@ -27,11 +27,14 @@ async def test_focus_vs_all_elements():
wait_for_network_idle_page_load_time=1,
headless=False,
args=['--incognito'],
+ paint_order_filtering=True,
),
)
# 10 Sample websites with various interactive elements
sample_websites = [
+ 'https://www.google.com/travel/flights',
+ 'https://v0-simple-ui-test-site.vercel.app',
'https://browser-use.github.io/stress-tests/challenges/iframe-inception-level1.html',
'https://browser-use.github.io/stress-tests/challenges/angular-form.html',
'https://www.google.com/travel/flights',
@@ -194,7 +197,7 @@ async def test_focus_vs_all_elements():
# Calculate percentages
total_time = all_timing.get('get_state_summary_total', 0)
- if total_time > 0:
+ if total_time > 0 and total_elements > 0:
timing_text += '\n📈 PERCENTAGE BREAKDOWN:\n'
timing_text += f'{"─" * 30}\n'
for key, value in all_timing.items():
@@ -205,7 +208,7 @@ async def test_focus_vs_all_elements():
timing_text += '\n🎯 CLICKABLE DETECTION ANALYSIS:\n'
timing_text += f'{"─" * 35}\n'
clickable_time = all_timing.get('clickable_detection_time', 0)
- if clickable_time > 0:
+ if clickable_time > 0 and total_elements > 0:
avg_per_element = (clickable_time / total_elements) * 1000000 # microseconds
timing_text += f'Total clickable detection time: {clickable_time * 1000:.2f} ms\n'
timing_text += f'Average per element: {avg_per_element:.2f} μs\n'
diff --git a/browser_use/dom/serializer/paint_order.py b/browser_use/dom/serializer/paint_order.py
new file mode 100644
index 000000000..d82d14401
--- /dev/null
+++ b/browser_use/dom/serializer/paint_order.py
@@ -0,0 +1,197 @@
+from collections import defaultdict
+from dataclasses import dataclass
+
+from browser_use.dom.views import SimplifiedNode
+
+"""
+Helper class for maintaining a union of rectangles (used for order of elements calculation)
+"""
+
+
+@dataclass(frozen=True, slots=True)
+class Rect:
+ """Closed axis-aligned rectangle with (x1,y1) bottom-left, (x2,y2) top-right."""
+
+ x1: float
+ y1: float
+ x2: float
+ y2: float
+
+ def __post_init__(self):
+ if not (self.x1 <= self.x2 and self.y1 <= self.y2):
+ return False
+
+ # --- fast relations ----------------------------------------------------
+ def area(self) -> float:
+ return (self.x2 - self.x1) * (self.y2 - self.y1)
+
+ def intersects(self, other: 'Rect') -> bool:
+ return not (self.x2 <= other.x1 or other.x2 <= self.x1 or self.y2 <= other.y1 or other.y2 <= self.y1)
+
+ def contains(self, other: 'Rect') -> bool:
+ return self.x1 <= other.x1 and self.y1 <= other.y1 and self.x2 >= other.x2 and self.y2 >= other.y2
+
+
+class RectUnionPure:
+ """
+ Maintains a *disjoint* set of rectangles.
+ No external dependencies - fine for a few thousand rectangles.
+ """
+
+ __slots__ = ('_rects',)
+
+ def __init__(self):
+ self._rects: list[Rect] = []
+
+ # -----------------------------------------------------------------
+ def _split_diff(self, a: Rect, b: Rect) -> list[Rect]:
+ r"""
+ Return list of up to 4 rectangles = a \ b.
+ Assumes a intersects b.
+ """
+ parts = []
+
+ # Bottom slice
+ if a.y1 < b.y1:
+ parts.append(Rect(a.x1, a.y1, a.x2, b.y1))
+ # Top slice
+ if b.y2 < a.y2:
+ parts.append(Rect(a.x1, b.y2, a.x2, a.y2))
+
+ # Middle (vertical) strip: y overlap is [max(a.y1,b.y1), min(a.y2,b.y2)]
+ y_lo = max(a.y1, b.y1)
+ y_hi = min(a.y2, b.y2)
+
+ # Left slice
+ if a.x1 < b.x1:
+ parts.append(Rect(a.x1, y_lo, b.x1, y_hi))
+ # Right slice
+ if b.x2 < a.x2:
+ parts.append(Rect(b.x2, y_lo, a.x2, y_hi))
+
+ return parts
+
+ # -----------------------------------------------------------------
+ def contains(self, r: Rect) -> bool:
+ """
+ True iff r is fully covered by the current union.
+ """
+ if not self._rects:
+ return False
+
+ stack = [r]
+ for s in self._rects:
+ new_stack = []
+ for piece in stack:
+ if s.contains(piece):
+ # piece completely gone
+ continue
+ if piece.intersects(s):
+ new_stack.extend(self._split_diff(piece, s))
+ else:
+ new_stack.append(piece)
+ if not new_stack: # everything eaten – covered
+ return True
+ stack = new_stack
+ return False # something survived
+
+ # -----------------------------------------------------------------
+ def add(self, r: Rect) -> bool:
+ """
+ Insert r unless it is already covered.
+ Returns True if the union grew.
+ """
+ if self.contains(r):
+ return False
+
+ pending = [r]
+ i = 0
+ while i < len(self._rects):
+ s = self._rects[i]
+ new_pending = []
+ changed = False
+ for piece in pending:
+ if piece.intersects(s):
+ new_pending.extend(self._split_diff(piece, s))
+ changed = True
+ else:
+ new_pending.append(piece)
+ pending = new_pending
+ if changed:
+ # s unchanged; proceed with next existing rectangle
+ i += 1
+ else:
+ i += 1
+
+ # Any left‑over pieces are new, non‑overlapping areas
+ self._rects.extend(pending)
+ return True
+
+
+class PaintOrderRemover:
+ """
+ Calculates which elements should be removed based on the paint order parameter.
+ """
+
+ def __init__(self, root: SimplifiedNode):
+ self.root = root
+
+ def calculate_paint_order(self) -> None:
+ all_simplified_nodes_with_paint_order: list[SimplifiedNode] = []
+
+ def collect_paint_order(node: SimplifiedNode) -> None:
+ if (
+ node.original_node.snapshot_node
+ and node.original_node.snapshot_node.paint_order is not None
+ and node.original_node.snapshot_node.bounds is not None
+ ):
+ all_simplified_nodes_with_paint_order.append(node)
+
+ for child in node.children:
+ collect_paint_order(child)
+
+ collect_paint_order(self.root)
+
+ grouped_by_paint_order: defaultdict[int, list[SimplifiedNode]] = defaultdict(list)
+
+ for node in all_simplified_nodes_with_paint_order:
+ if node.original_node.snapshot_node and node.original_node.snapshot_node.paint_order is not None:
+ grouped_by_paint_order[node.original_node.snapshot_node.paint_order].append(node)
+
+ rect_union = RectUnionPure()
+
+ for paint_order, nodes in sorted(grouped_by_paint_order.items(), key=lambda x: -x[0]):
+ rects_to_add = []
+
+ for node in nodes:
+ if not node.original_node.snapshot_node or not node.original_node.snapshot_node.bounds:
+ continue # shouldn't happen by how we filter them out in the first place
+
+ rect = Rect(
+ x1=node.original_node.snapshot_node.bounds.x,
+ y1=node.original_node.snapshot_node.bounds.y,
+ x2=node.original_node.snapshot_node.bounds.x + node.original_node.snapshot_node.bounds.width,
+ y2=node.original_node.snapshot_node.bounds.y + node.original_node.snapshot_node.bounds.height,
+ )
+
+ if rect_union.contains(rect):
+ node.ignored_by_paint_order = True
+
+ # don't add to the nodes if opacity is less then 0.95 or background-color is transparent
+ if (
+ node.original_node.snapshot_node.computed_styles
+ and node.original_node.snapshot_node.computed_styles.get('background-color', 'rgba(0, 0, 0, 0)')
+ == 'rgba(0, 0, 0, 0)'
+ ) or (
+ node.original_node.snapshot_node.computed_styles
+ and float(node.original_node.snapshot_node.computed_styles.get('opacity', '1'))
+ < 0.8 # this is highly vibes based number
+ ):
+ continue
+
+ rects_to_add.append(rect)
+
+ for rect in rects_to_add:
+ rect_union.add(rect)
+
+ return None
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.py
index ec74e0702..8b0084c3b 100644
--- a/browser_use/dom/serializer/serializer.py
+++ b/browser_use/dom/serializer/serializer.py
@@ -2,6 +2,7 @@
from browser_use.dom.serializer.clickable_elements import ClickableElementDetector
+from browser_use.dom.serializer.paint_order import PaintOrderRemover
from browser_use.dom.utils import cap_text_length
from browser_use.dom.views import (
DOMRect,
@@ -40,6 +41,7 @@ class DOMTreeSerializer:
previous_cached_state: SerializedDOMState | None = None,
enable_bbox_filtering: bool = True,
containment_threshold: float | None = None,
+ paint_order_filtering: bool = True,
):
self.root_node = root_node
self._interactive_counter = 1
@@ -52,6 +54,8 @@ class DOMTreeSerializer:
# Bounding box filtering configuration
self.enable_bbox_filtering = enable_bbox_filtering
self.containment_threshold = containment_threshold or self.DEFAULT_CONTAINMENT_THRESHOLD
+ # Paint order filtering configuration
+ self.paint_order_filtering = paint_order_filtering
def serialize_accessible_elements(self) -> tuple[SerializedDOMState, dict[str, float]]:
import time
@@ -70,16 +74,19 @@ class DOMTreeSerializer:
end_step1 = time.time()
self.timing_info['create_simplified_tree'] = end_step1 - start_step1
+ # Step 3: Remove elements based on paint order
+ start_step3 = time.time()
+ if self.paint_order_filtering and simplified_tree:
+ PaintOrderRemover(simplified_tree).calculate_paint_order()
+ end_step3 = time.time()
+ self.timing_info['calculate_paint_order'] = end_step3 - start_step3
+
# Step 2: Optimize tree (remove unnecessary parents)
start_step2 = time.time()
optimized_tree = self._optimize_tree(simplified_tree)
end_step2 = time.time()
self.timing_info['optimize_tree'] = end_step2 - start_step2
- # # Step 3: Detect and group semantic elements
- # if optimized_tree:
- # self._detect_semantic_groups(optimized_tree)
-
# Step 3: Apply bounding box filtering (NEW)
if self.enable_bbox_filtering and optimized_tree:
start_step3 = time.time()
@@ -148,20 +155,16 @@ class DOMTreeSerializer:
simplified = SimplifiedNode(original_node=node, children=[])
for child in node.content_document.children_nodes or []:
simplified_child = self._create_simplified_tree(child)
- if simplified_child:
+ if simplified_child is not None:
simplified.children.append(simplified_child)
return simplified
- # Use enhanced scoring for inclusion decision
- is_interactive = self._is_interactive_cached(node)
-
- is_visible = node.snapshot_node and node.is_visible
+ is_visible = node.is_visible
is_scrollable = node.is_actually_scrollable
# Include if interactive (regardless of visibility), or scrollable, or has children to process
- should_include = (is_interactive and is_visible) or is_scrollable or bool(node.children_and_shadow_roots)
- if should_include:
+ if is_visible or is_scrollable or bool(node.children_and_shadow_roots):
simplified = SimplifiedNode(original_node=node, children=[])
# simplified._analysis = analysis # Store analysis for grouping
@@ -172,7 +175,7 @@ class DOMTreeSerializer:
simplified.children.append(simplified_child)
# Return if meaningful or has meaningful children
- if (is_interactive and is_visible) or is_scrollable or simplified.children:
+ if is_visible or is_scrollable or simplified.children:
return simplified
elif node.node_type == NodeType.TEXT_NODE:
@@ -202,7 +205,7 @@ class DOMTreeSerializer:
is_visible = node.original_node.snapshot_node and node.original_node.is_visible
if (
- (is_interactive_opt and is_visible) # Only keep interactive nodes that are visible
+ is_visible # Keep all visible nodes
or node.original_node.is_actually_scrollable
or node.original_node.node_type == NodeType.TEXT_NODE
or node.children
@@ -228,8 +231,8 @@ class DOMTreeSerializer:
if not node:
return
- # Skip assigning index to excluded nodes
- if not (hasattr(node, 'excluded_by_parent') and node.excluded_by_parent):
+ # Skip assigning index to excluded nodes, or ignored by paint order
+ if not node.excluded_by_parent and not node.ignored_by_paint_order:
# Assign index to clickable elements that are also visible
is_interactive_assign = self._is_interactive_cached(node.original_node)
is_visible = node.original_node.snapshot_node and node.original_node.is_visible
diff --git a/browser_use/dom/service.py b/browser_use/dom/service.py
index 155c2d498..2f20735a2 100644
--- a/browser_use/dom/service.py
+++ b/browser_use/dom/service.py
@@ -40,11 +40,16 @@ class DomService:
logger: logging.Logger
def __init__(
- self, browser_session: 'BrowserSession', logger: logging.Logger | None = None, cross_origin_iframes: bool = False
+ self,
+ browser_session: 'BrowserSession',
+ logger: logging.Logger | None = None,
+ cross_origin_iframes: bool = False,
+ paint_order_filtering: bool = True,
):
self.browser_session = browser_session
self.logger = logger or browser_session.logger
self.cross_origin_iframes = cross_origin_iframes
+ self.paint_order_filtering = paint_order_filtering
async def __aenter__(self):
return self
@@ -670,7 +675,7 @@ class DomService:
start = time.time()
serialized_dom_state, serializer_timing = DOMTreeSerializer(
- enhanced_dom_tree, previous_cached_state
+ enhanced_dom_tree, previous_cached_state, paint_order_filtering=self.paint_order_filtering
).serialize_accessible_elements()
end = time.time()
diff --git a/browser_use/dom/views.py b/browser_use/dom/views.py
index 62b9222c3..06b1e8b32 100644
--- a/browser_use/dom/views.py
+++ b/browser_use/dom/views.py
@@ -90,6 +90,8 @@ class SimplifiedNode:
interactive_index: int | None = None
is_new: bool = False
+
+ ignored_by_paint_order: bool = False # More info in dom/serializer/paint_order.py
excluded_by_parent: bool = False # New field for bbox filtering
def _clean_original_node_json(self, node_json: dict) -> dict:
@@ -113,6 +115,8 @@ class SimplifiedNode:
return {
'should_display': self.should_display,
'interactive_index': self.interactive_index,
+ 'ignored_by_paint_order': self.ignored_by_paint_order,
+ 'excluded_by_parent': self.excluded_by_parent,
'original_node': cleaned_original_node_json,
'children': [c.__json__() for c in self.children],
}
@@ -372,6 +376,7 @@ class EnhancedDOMTreeNode:
'node_type': self.node_type.name,
'node_name': self.node_name,
'node_value': self.node_value,
+ 'is_visible': self.is_visible,
'attributes': self.attributes,
'is_scrollable': self.is_scrollable,
'session_id': self.session_id,