Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fix(dom): expose image context for clickable elements

UniversePeak96117ef8f559
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.py
index 9f9017874..9fcee00c6 100644
--- a/browser_use/dom/serializer/serializer.py
+++ b/browser_use/dom/serializer/serializer.py
@@ -918,6 +918,52 @@ class DOMTreeSerializer:
return False
+ @staticmethod
+ def _get_child_image_context(node: SimplifiedNode) -> str:
+ """Extract compact context from image descendants of an interactive element."""
+
+ image_context: list[str] = []
+
+ def normalize_src(src: str) -> str:
+ clean_src = src.strip()
+ if clean_src.lower().startswith('data:'):
+ return ''
+ path_without_query = clean_src.split('?', 1)[0].split('#', 1)[0].rstrip('/')
+ return path_without_query.rsplit('/', 1)[-1] or clean_src
+
+ def collect(current: SimplifiedNode) -> None:
+ if len(image_context) >= 3:
+ return
+
+ original_node = current.original_node
+ if original_node.node_type == NodeType.ELEMENT_NODE and original_node.tag_name == 'img':
+ attributes = original_node.attributes or {}
+ parts = []
+
+ for attr_name, output_name in (
+ ('alt', 'image_alt'),
+ ('title', 'image_title'),
+ ('aria-label', 'image_label'),
+ ):
+ attr_value = attributes.get(attr_name, '').strip()
+ if attr_value:
+ parts.append(f'{output_name}={cap_text_length(attr_value, 100)}')
+
+ src = normalize_src(attributes.get('src', ''))
+ if src:
+ parts.append(f'image_src={cap_text_length(src, 100)}')
+
+ if parts:
+ image_context.append(' '.join(parts))
+
+ for child in current.children:
+ collect(child)
+
+ for child in node.children:
+ collect(child)
+
+ return ' '.join(image_context)
+
@staticmethod
def serialize_tree(node: SimplifiedNode | None, include_attributes: list[str], depth: int = 0) -> str:
"""Serialize the optimized tree to string format."""
@@ -989,6 +1035,13 @@ class DOMTreeSerializer:
attributes_html_str = DOMTreeSerializer._build_attributes_string(
node.original_node, include_attributes, text_content
)
+ if node.is_interactive:
+ image_context = DOMTreeSerializer._get_child_image_context(node)
+ if image_context:
+ if attributes_html_str:
+ attributes_html_str += f' {image_context}'
+ else:
+ attributes_html_str = image_context
# Add compound component information to attributes if present
if node.original_node._compound_children:
diff --git a/tests/ci/test_image_only_dom_representation.py b/tests/ci/test_image_only_dom_representation.py
new file mode 100644
index 000000000..7bb83fcc0
--- /dev/null
+++ b/tests/ci/test_image_only_dom_representation.py
@@ -0,0 +1,81 @@
+from browser_use.dom.serializer.serializer import DOMTreeSerializer
+from browser_use.dom.views import (
+ DEFAULT_INCLUDE_ATTRIBUTES,
+ DOMRect,
+ EnhancedDOMTreeNode,
+ EnhancedSnapshotNode,
+ NodeType,
+ SimplifiedNode,
+)
+
+
+def _make_element_node(
+ backend_node_id: int,
+ tag_name: str,
+ attributes: dict[str, str],
+ x: float,
+ y: float,
+ width: float = 64,
+ height: float = 64,
+ parent: EnhancedDOMTreeNode | None = None,
+) -> EnhancedDOMTreeNode:
+ bounds = DOMRect(x=x, y=y, width=width, height=height)
+ return EnhancedDOMTreeNode(
+ node_id=backend_node_id,
+ backend_node_id=backend_node_id,
+ node_type=NodeType.ELEMENT_NODE,
+ node_name=tag_name.upper(),
+ node_value='',
+ attributes=attributes,
+ is_scrollable=None,
+ is_visible=True,
+ absolute_position=bounds,
+ target_id='target-1',
+ frame_id=None,
+ session_id=None,
+ content_document=None,
+ shadow_root_type=None,
+ shadow_roots=None,
+ parent_node=parent,
+ children_nodes=None,
+ ax_node=None,
+ snapshot_node=EnhancedSnapshotNode(
+ is_clickable=tag_name in {'a', 'button'},
+ cursor_style='pointer' if tag_name in {'a', 'button'} else None,
+ bounds=bounds,
+ clientRects=bounds,
+ scrollRects=None,
+ computed_styles=None,
+ paint_order=None,
+ stacking_contexts=None,
+ ),
+ )
+
+
+def test_image_only_interactive_parent_includes_child_image_context_in_llm_dom():
+ """Image-only clickable cards should expose child image context."""
+ link = _make_element_node(201, 'a', {'href': '/select-payment-method'}, x=10, y=10, width=80, height=80)
+ image = _make_element_node(
+ 202,
+ 'img',
+ {'src': 'https://cdn.example.test/logos/acme-bank-primary-card.png'},
+ x=18,
+ y=18,
+ width=64,
+ height=64,
+ parent=link,
+ )
+ link.children_nodes = [image]
+
+ llm_dom = DOMTreeSerializer.serialize_tree(
+ SimplifiedNode(
+ original_node=link,
+ children=[SimplifiedNode(original_node=image, children=[])],
+ is_interactive=True,
+ selector_index=201,
+ ),
+ DEFAULT_INCLUDE_ATTRIBUTES,
+ )
+
+ assert '[201]<a' in llm_dom
+ assert 'acme-bank-primary-card.png' in llm_dom