Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
fix(dom): expose image context for clickable elements
UniversePeak96117ef8f559
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.pyindex 9f9017874..9fcee00c6 100644--- a/browser_use/dom/serializer/serializer.py+++ b/browser_use/dom/serializer/serializer.py@@ -918,6 +918,52 @@ class DOMTreeSerializer:return False+ @staticmethod+ def _get_child_image_context(node: SimplifiedNode) -> str:+ """Extract compact context from image descendants of an interactive element."""++ image_context: list[str] = []++ def normalize_src(src: str) -> str:+ clean_src = src.strip()+ if clean_src.lower().startswith('data:'):+ return ''+ path_without_query = clean_src.split('?', 1)[0].split('#', 1)[0].rstrip('/')+ return path_without_query.rsplit('/', 1)[-1] or clean_src++ def collect(current: SimplifiedNode) -> None:+ if len(image_context) >= 3:+ return++ original_node = current.original_node+ if original_node.node_type == NodeType.ELEMENT_NODE and original_node.tag_name == 'img':+ attributes = original_node.attributes or {}+ parts = []++ for attr_name, output_name in (+ ('alt', 'image_alt'),+ ('title', 'image_title'),+ ('aria-label', 'image_label'),+ ):+ attr_value = attributes.get(attr_name, '').strip()+ if attr_value:+ parts.append(f'{output_name}={cap_text_length(attr_value, 100)}')++ src = normalize_src(attributes.get('src', ''))+ if src:+ parts.append(f'image_src={cap_text_length(src, 100)}')++ if parts:+ image_context.append(' '.join(parts))++ for child in current.children:+ collect(child)++ for child in node.children:+ collect(child)++ return ' '.join(image_context)+@staticmethoddef serialize_tree(node: SimplifiedNode | None, include_attributes: list[str], depth: int = 0) -> str:"""Serialize the optimized tree to string format."""@@ -989,6 +1035,13 @@ class DOMTreeSerializer:attributes_html_str = DOMTreeSerializer._build_attributes_string(node.original_node, include_attributes, text_content)+ if node.is_interactive:+ image_context = DOMTreeSerializer._get_child_image_context(node)+ if image_context:+ if attributes_html_str:+ attributes_html_str += f' {image_context}'+ else:+ attributes_html_str = image_context# Add compound component information to attributes if presentif node.original_node._compound_children:diff --git a/tests/ci/test_image_only_dom_representation.py b/tests/ci/test_image_only_dom_representation.pynew file mode 100644index 000000000..7bb83fcc0--- /dev/null+++ b/tests/ci/test_image_only_dom_representation.py@@ -0,0 +1,81 @@+from browser_use.dom.serializer.serializer import DOMTreeSerializer+from browser_use.dom.views import (+ DEFAULT_INCLUDE_ATTRIBUTES,+ DOMRect,+ EnhancedDOMTreeNode,+ EnhancedSnapshotNode,+ NodeType,+ SimplifiedNode,+)+++def _make_element_node(+ backend_node_id: int,+ tag_name: str,+ attributes: dict[str, str],+ x: float,+ y: float,+ width: float = 64,+ height: float = 64,+ parent: EnhancedDOMTreeNode | None = None,+) -> EnhancedDOMTreeNode:+ bounds = DOMRect(x=x, y=y, width=width, height=height)+ return EnhancedDOMTreeNode(+ node_id=backend_node_id,+ backend_node_id=backend_node_id,+ node_type=NodeType.ELEMENT_NODE,+ node_name=tag_name.upper(),+ node_value='',+ attributes=attributes,+ is_scrollable=None,+ is_visible=True,+ absolute_position=bounds,+ target_id='target-1',+ frame_id=None,+ session_id=None,+ content_document=None,+ shadow_root_type=None,+ shadow_roots=None,+ parent_node=parent,+ children_nodes=None,+ ax_node=None,+ snapshot_node=EnhancedSnapshotNode(+ is_clickable=tag_name in {'a', 'button'},+ cursor_style='pointer' if tag_name in {'a', 'button'} else None,+ bounds=bounds,+ clientRects=bounds,+ scrollRects=None,+ computed_styles=None,+ paint_order=None,+ stacking_contexts=None,+ ),+ )+++def test_image_only_interactive_parent_includes_child_image_context_in_llm_dom():+ """Image-only clickable cards should expose child image context."""+ link = _make_element_node(201, 'a', {'href': '/select-payment-method'}, x=10, y=10, width=80, height=80)+ image = _make_element_node(+ 202,+ 'img',+ {'src': 'https://cdn.example.test/logos/acme-bank-primary-card.png'},+ x=18,+ y=18,+ width=64,+ height=64,+ parent=link,+ )+ link.children_nodes = [image]++ llm_dom = DOMTreeSerializer.serialize_tree(+ SimplifiedNode(+ original_node=link,+ children=[SimplifiedNode(original_node=image, children=[])],+ is_interactive=True,+ selector_index=201,+ ),+ DEFAULT_INCLUDE_ATTRIBUTES,+ )++ assert '[201]<a' in llm_dom+ assert 'acme-bank-primary-card.png' in llm_dom