Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Scope paint-order filtering by iframe document

MagMueller211091828923
diff --git a/browser_use/dom/serializer/paint_order.py b/browser_use/dom/serializer/paint_order.py
index 61032e209..aa3056ec0 100644
--- a/browser_use/dom/serializer/paint_order.py
+++ b/browser_use/dom/serializer/paint_order.py
@@ -151,6 +151,17 @@ class PaintOrderRemover:
def __init__(self, root: SimplifiedNode):
self.root = root
+ @staticmethod
+ def _document_context(node: SimplifiedNode) -> tuple[str | None, str | None]:
+ """Identify the CDP session and iframe document owning a node's snapshot."""
+ original_node = node.original_node
+ parent = original_node.parent_node
+ while parent is not None:
+ if parent.tag_name in {'iframe', 'frame'}:
+ return str(original_node.session_id), parent.frame_id
+ parent = parent.parent_node
+ return str(original_node.session_id), None
+
def calculate_paint_order(self) -> None:
all_simplified_nodes_with_paint_order: list[SimplifiedNode] = []
@@ -173,10 +184,10 @@ class PaintOrderRemover:
if node.original_node.snapshot_node and node.original_node.snapshot_node.paint_order is not None:
grouped_by_paint_order[node.original_node.snapshot_node.paint_order].append(node)
- rect_union = RectUnionPure()
+ rect_unions: defaultdict[tuple[str | None, str | None], RectUnionPure] = defaultdict(RectUnionPure)
for paint_order, nodes in sorted(grouped_by_paint_order.items(), key=lambda x: -x[0]):
- rects_to_add = []
+ rects_to_add: defaultdict[tuple[str | None, str | None], list[Rect]] = defaultdict(list)
for node in nodes:
if not node.original_node.snapshot_node or not node.original_node.snapshot_node.bounds:
@@ -188,8 +199,9 @@ class PaintOrderRemover:
x2=node.original_node.snapshot_node.bounds.x + node.original_node.snapshot_node.bounds.width,
y2=node.original_node.snapshot_node.bounds.y + node.original_node.snapshot_node.bounds.height,
)
+ context = self._document_context(node)
- if rect_union.contains(rect):
+ if rect_unions[context].contains(rect):
node.ignored_by_paint_order = True
# don't add to the nodes if opacity is less then 0.95 or background-color is transparent
@@ -204,9 +216,10 @@ class PaintOrderRemover:
):
continue
- rects_to_add.append(rect)
+ rects_to_add[context].append(rect)
- for rect in rects_to_add:
- rect_union.add(rect)
+ for context, rects in rects_to_add.items():
+ for rect in rects:
+ rect_unions[context].add(rect)
return None
diff --git a/tests/ci/browser/test_dom_serializer_session_identity.py b/tests/ci/browser/test_dom_serializer_session_identity.py
index 3d23decfb..4ccb81c54 100644
--- a/tests/ci/browser/test_dom_serializer_session_identity.py
+++ b/tests/ci/browser/test_dom_serializer_session_identity.py
@@ -1,10 +1,19 @@
"""Regression coverage for CDP node identity across iframe sessions."""
+from browser_use.dom.serializer.paint_order import PaintOrderRemover
from browser_use.dom.serializer.serializer import DOMTreeSerializer
-from browser_use.dom.views import DOMRect, EnhancedDOMTreeNode, EnhancedSnapshotNode, NodeType
+from browser_use.dom.views import DOMRect, EnhancedDOMTreeNode, EnhancedSnapshotNode, NodeType, SimplifiedNode
-def _node(tag_name: str, *, node_id: int, backend_node_id: int, session_id: str) -> EnhancedDOMTreeNode:
+def _node(
+ tag_name: str,
+ *,
+ node_id: int,
+ backend_node_id: int,
+ session_id: str,
+ paint_order: int | None = None,
+ background_color: str = 'rgba(0, 0, 0, 0)',
+) -> EnhancedDOMTreeNode:
return EnhancedDOMTreeNode(
node_id=node_id,
backend_node_id=backend_node_id,
@@ -34,9 +43,9 @@ def _node(tag_name: str, *, node_id: int, backend_node_id: int, session_id: str)
'display': 'block',
'visibility': 'visible',
'opacity': '1',
- 'background-color': 'rgba(0, 0, 0, 0)',
+ 'background-color': background_color,
},
- paint_order=None,
+ paint_order=paint_order,
stacking_contexts=None,
),
)
@@ -59,3 +68,73 @@ def test_clickability_cache_scopes_node_ids_to_cdp_session():
assert list(serialized_state.selector_map.values()) == [iframe_input]
assert '[21]<input' in serialized_state.llm_representation()
+
+
+def test_paint_order_filtering_is_isolated_between_iframe_documents():
+ """Sibling iframe-local rectangles cannot occlude each other."""
+ card_frame = _node('iframe', node_id=10, backend_node_id=10, session_id='wrapper')
+ card_frame.frame_id = 'card-frame'
+ cvv_frame = _node('iframe', node_id=11, backend_node_id=11, session_id='wrapper')
+ cvv_frame.frame_id = 'cvv-frame'
+ card_input = _node(
+ 'input',
+ node_id=1,
+ backend_node_id=1,
+ session_id='wrapper',
+ paint_order=1,
+ background_color='rgb(255, 255, 255)',
+ )
+ cvv_input = _node(
+ 'input',
+ node_id=2,
+ backend_node_id=2,
+ session_id='wrapper',
+ paint_order=2,
+ background_color='rgb(255, 255, 255)',
+ )
+ card_input.parent_node = card_frame
+ cvv_input.parent_node = cvv_frame
+ root = SimplifiedNode(
+ original_node=_node('html', node_id=100, backend_node_id=100, session_id='main'),
+ children=[
+ SimplifiedNode(original_node=card_input, children=[]),
+ SimplifiedNode(original_node=cvv_input, children=[]),
+ ],
+ )
+
+ PaintOrderRemover(root).calculate_paint_order()
+
+ assert root.children[0].ignored_by_paint_order is False
+ assert root.children[1].ignored_by_paint_order is False
+
+
+def test_paint_order_filtering_still_applies_within_one_document():
+ """A higher opaque sibling still occludes the same rectangle in its document."""
+ lower_input = _node(
+ 'input',
+ node_id=1,
+ backend_node_id=1,
+ session_id='main',
+ paint_order=1,
+ background_color='rgb(255, 255, 255)',
+ )
+ upper_input = _node(
+ 'input',
+ node_id=2,
+ backend_node_id=2,
+ session_id='main',
+ paint_order=2,
+ background_color='rgb(255, 255, 255)',
+ )
+ root = SimplifiedNode(
+ original_node=_node('html', node_id=100, backend_node_id=100, session_id='main'),
+ children=[
+ SimplifiedNode(original_node=lower_input, children=[]),
+ SimplifiedNode(original_node=upper_input, children=[]),
+ ],
+ )
+
+ PaintOrderRemover(root).calculate_paint_order()
+
+ assert root.children[0].ignored_by_paint_order is True
+ assert root.children[1].ignored_by_paint_order is False