Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Scope paint-order filtering by iframe document
MagMueller211091828923
diff --git a/browser_use/dom/serializer/paint_order.py b/browser_use/dom/serializer/paint_order.pyindex 61032e209..aa3056ec0 100644--- a/browser_use/dom/serializer/paint_order.py+++ b/browser_use/dom/serializer/paint_order.py@@ -151,6 +151,17 @@ class PaintOrderRemover:def __init__(self, root: SimplifiedNode):self.root = root+ @staticmethod+ def _document_context(node: SimplifiedNode) -> tuple[str | None, str | None]:+ """Identify the CDP session and iframe document owning a node's snapshot."""+ original_node = node.original_node+ parent = original_node.parent_node+ while parent is not None:+ if parent.tag_name in {'iframe', 'frame'}:+ return str(original_node.session_id), parent.frame_id+ parent = parent.parent_node+ return str(original_node.session_id), None+def calculate_paint_order(self) -> None:all_simplified_nodes_with_paint_order: list[SimplifiedNode] = []@@ -173,10 +184,10 @@ class PaintOrderRemover:if node.original_node.snapshot_node and node.original_node.snapshot_node.paint_order is not None:grouped_by_paint_order[node.original_node.snapshot_node.paint_order].append(node)- rect_union = RectUnionPure()+ rect_unions: defaultdict[tuple[str | None, str | None], RectUnionPure] = defaultdict(RectUnionPure)for paint_order, nodes in sorted(grouped_by_paint_order.items(), key=lambda x: -x[0]):- rects_to_add = []+ rects_to_add: defaultdict[tuple[str | None, str | None], list[Rect]] = defaultdict(list)for node in nodes:if not node.original_node.snapshot_node or not node.original_node.snapshot_node.bounds:@@ -188,8 +199,9 @@ class PaintOrderRemover:x2=node.original_node.snapshot_node.bounds.x + node.original_node.snapshot_node.bounds.width,y2=node.original_node.snapshot_node.bounds.y + node.original_node.snapshot_node.bounds.height,)+ context = self._document_context(node)- if rect_union.contains(rect):+ if rect_unions[context].contains(rect):node.ignored_by_paint_order = True# don't add to the nodes if opacity is less then 0.95 or background-color is transparent@@ -204,9 +216,10 @@ class PaintOrderRemover:):continue- rects_to_add.append(rect)+ rects_to_add[context].append(rect)- for rect in rects_to_add:- rect_union.add(rect)+ for context, rects in rects_to_add.items():+ for rect in rects:+ rect_unions[context].add(rect)return Nonediff --git a/tests/ci/browser/test_dom_serializer_session_identity.py b/tests/ci/browser/test_dom_serializer_session_identity.pyindex 3d23decfb..4ccb81c54 100644--- a/tests/ci/browser/test_dom_serializer_session_identity.py+++ b/tests/ci/browser/test_dom_serializer_session_identity.py@@ -1,10 +1,19 @@"""Regression coverage for CDP node identity across iframe sessions."""+from browser_use.dom.serializer.paint_order import PaintOrderRemoverfrom browser_use.dom.serializer.serializer import DOMTreeSerializer-from browser_use.dom.views import DOMRect, EnhancedDOMTreeNode, EnhancedSnapshotNode, NodeType+from browser_use.dom.views import DOMRect, EnhancedDOMTreeNode, EnhancedSnapshotNode, NodeType, SimplifiedNode-def _node(tag_name: str, *, node_id: int, backend_node_id: int, session_id: str) -> EnhancedDOMTreeNode:+def _node(+ tag_name: str,+ *,+ node_id: int,+ backend_node_id: int,+ session_id: str,+ paint_order: int | None = None,+ background_color: str = 'rgba(0, 0, 0, 0)',+) -> EnhancedDOMTreeNode:return EnhancedDOMTreeNode(node_id=node_id,backend_node_id=backend_node_id,@@ -34,9 +43,9 @@ def _node(tag_name: str, *, node_id: int, backend_node_id: int, session_id: str)'display': 'block','visibility': 'visible','opacity': '1',- 'background-color': 'rgba(0, 0, 0, 0)',+ 'background-color': background_color,},- paint_order=None,+ paint_order=paint_order,stacking_contexts=None,),)@@ -59,3 +68,73 @@ def test_clickability_cache_scopes_node_ids_to_cdp_session():assert list(serialized_state.selector_map.values()) == [iframe_input]assert '[21]<input' in serialized_state.llm_representation()+++def test_paint_order_filtering_is_isolated_between_iframe_documents():+ """Sibling iframe-local rectangles cannot occlude each other."""+ card_frame = _node('iframe', node_id=10, backend_node_id=10, session_id='wrapper')+ card_frame.frame_id = 'card-frame'+ cvv_frame = _node('iframe', node_id=11, backend_node_id=11, session_id='wrapper')+ cvv_frame.frame_id = 'cvv-frame'+ card_input = _node(+ 'input',+ node_id=1,+ backend_node_id=1,+ session_id='wrapper',+ paint_order=1,+ background_color='rgb(255, 255, 255)',+ )+ cvv_input = _node(+ 'input',+ node_id=2,+ backend_node_id=2,+ session_id='wrapper',+ paint_order=2,+ background_color='rgb(255, 255, 255)',+ )+ card_input.parent_node = card_frame+ cvv_input.parent_node = cvv_frame+ root = SimplifiedNode(+ original_node=_node('html', node_id=100, backend_node_id=100, session_id='main'),+ children=[+ SimplifiedNode(original_node=card_input, children=[]),+ SimplifiedNode(original_node=cvv_input, children=[]),+ ],+ )++ PaintOrderRemover(root).calculate_paint_order()++ assert root.children[0].ignored_by_paint_order is False+ assert root.children[1].ignored_by_paint_order is False+++def test_paint_order_filtering_still_applies_within_one_document():+ """A higher opaque sibling still occludes the same rectangle in its document."""+ lower_input = _node(+ 'input',+ node_id=1,+ backend_node_id=1,+ session_id='main',+ paint_order=1,+ background_color='rgb(255, 255, 255)',+ )+ upper_input = _node(+ 'input',+ node_id=2,+ backend_node_id=2,+ session_id='main',+ paint_order=2,+ background_color='rgb(255, 255, 255)',+ )+ root = SimplifiedNode(+ original_node=_node('html', node_id=100, backend_node_id=100, session_id='main'),+ children=[+ SimplifiedNode(original_node=lower_input, children=[]),+ SimplifiedNode(original_node=upper_input, children=[]),+ ],+ )++ PaintOrderRemover(root).calculate_paint_order()++ assert root.children[0].ignored_by_paint_order is True+ assert root.children[1].ignored_by_paint_order is False