Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fix(dom): expose image context for directly clickable images

r266-tech81a472a3e2c6
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.py
index 8b86a9c75..1c2f9df32 100644
--- a/browser_use/dom/serializer/serializer.py
+++ b/browser_use/dom/serializer/serializer.py
@@ -16,6 +16,7 @@ from browser_use.dom.views import (
)
DISABLED_ELEMENTS = {'style', 'script', 'head', 'meta', 'link', 'title'}
+URL_C0_CONTROL_OR_SPACE = ''.join(chr(codepoint) for codepoint in range(0x21))
# SVG child elements to skip (decorative only, no interaction value)
SVG_ELEMENTS = {
@@ -57,6 +58,7 @@ class DOMTreeSerializer:
DEFAULT_CONTAINMENT_THRESHOLD = 0.99 # 99% containment by default
MAX_CHILD_IMAGE_CONTEXTS = 3
MAX_CHILD_IMAGE_DESCENDANTS = 100
+ MAX_IMAGE_CONTEXT_ATTRIBUTE_LENGTH = 4096
def __init__(
self,
@@ -922,17 +924,47 @@ class DOMTreeSerializer:
@staticmethod
def _get_child_image_context(node: SimplifiedNode) -> str:
- """Extract compact context from image descendants of an interactive element."""
+ """Extract compact context from an interactive image and its descendants."""
image_context: list[str] = []
def normalize_src(src: str) -> str:
- clean_src = src.strip()
+ if len(src) > DOMTreeSerializer.MAX_IMAGE_CONTEXT_ATTRIBUTE_LENGTH:
+ return ''
+ clean_src = src.strip(URL_C0_CONTROL_OR_SPACE).replace('\t', '').replace('\n', '').replace('\r', '')
if clean_src.lower().startswith('data:'):
return ''
path_without_query = clean_src.split('?', 1)[0].split('#', 1)[0].rstrip('/')
return path_without_query.rsplit('/', 1)[-1]
+ def add_image_context(original_node: EnhancedDOMTreeNode) -> None:
+ if original_node.node_type != NodeType.ELEMENT_NODE or original_node.tag_name != 'img':
+ return
+
+ attributes = original_node.attributes or {}
+ parts = []
+
+ for attr_name, output_name in (
+ ('alt', 'image_alt'),
+ ('title', 'image_title'),
+ ('aria-label', 'image_label'),
+ ):
+ raw_attr_value = str(attributes.get(attr_name) or '')
+ if len(raw_attr_value) > DOMTreeSerializer.MAX_IMAGE_CONTEXT_ATTRIBUTE_LENGTH:
+ continue
+ attr_value = raw_attr_value.strip()
+ if attr_value:
+ parts.append(f'{output_name}={cap_text_length(attr_value, 100)}')
+
+ src = normalize_src(str(attributes.get('src') or ''))
+ if src:
+ parts.append(f'image_src={cap_text_length(src, 100)}')
+
+ if parts:
+ image_context.append(' '.join(parts))
+
+ add_image_context(node.original_node)
+
child_iterators = [iter(node.children)]
visited_descendants = 0
while (
@@ -946,26 +978,7 @@ class DOMTreeSerializer:
child_iterators.pop()
continue
visited_descendants += 1
- original_node = current.original_node
- if original_node.node_type == NodeType.ELEMENT_NODE and original_node.tag_name == 'img':
- attributes = original_node.attributes or {}
- parts = []
-
- for attr_name, output_name in (
- ('alt', 'image_alt'),
- ('title', 'image_title'),
- ('aria-label', 'image_label'),
- ):
- attr_value = str(attributes.get(attr_name) or '').strip()
- if attr_value:
- parts.append(f'{output_name}={cap_text_length(attr_value, 100)}')
-
- src = normalize_src(str(attributes.get('src') or ''))
- if src:
- parts.append(f'image_src={cap_text_length(src, 100)}')
-
- if parts:
- image_context.append(' '.join(parts))
+ add_image_context(current.original_node)
if current.children:
child_iterators.append(iter(current.children))
diff --git a/tests/ci/test_image_only_dom_representation.py b/tests/ci/test_image_only_dom_representation.py
index 1ff1d3fe9..3aa48030b 100644
--- a/tests/ci/test_image_only_dom_representation.py
+++ b/tests/ci/test_image_only_dom_representation.py
@@ -81,6 +81,79 @@ def test_image_only_interactive_parent_includes_child_image_context_in_llm_dom()
assert 'acme-bank-primary-card.png' in llm_dom
+def test_direct_interactive_image_includes_own_context_in_llm_dom():
+ """A directly clickable image should expose its own sanitized source context."""
+ image = _make_element_node(
+ 221,
+ 'img',
+ {'src': 'https://cdn.example.test/logos/acme-card.png?token=must-not-leak#preview'},
+ x=18,
+ y=18,
+ )
+
+ llm_dom = DOMTreeSerializer.serialize_tree(
+ SimplifiedNode(
+ original_node=image,
+ children=[],
+ is_interactive=True,
+ selector_index=221,
+ ),
+ DEFAULT_INCLUDE_ATTRIBUTES,
+ )
+
+ assert '[221]<img' in llm_dom
+ assert 'image_src=acme-card.png' in llm_dom
+ assert 'must-not-leak' not in llm_dom
+
+
+def test_direct_interactive_image_rejects_browser_normalized_data_src():
+ """Data URLs remain hidden when URL parsing ignores control characters in the scheme."""
+ image = _make_element_node(
+ 222,
+ 'img',
+ {'src': '\x00Da\nTa:image/png;base64,must-not-leak'},
+ x=18,
+ y=18,
+ )
+
+ llm_dom = DOMTreeSerializer.serialize_tree(
+ SimplifiedNode(
+ original_node=image,
+ children=[],
+ is_interactive=True,
+ selector_index=222,
+ ),
+ DEFAULT_INCLUDE_ATTRIBUTES,
+ )
+
+ assert llm_dom == '[222]<img />'
+ assert 'must-not-leak' not in llm_dom
+
+
+def test_direct_interactive_image_omits_oversized_src_context():
+ """Image context does not scan or serialize oversized raw source attributes."""
+ image = _make_element_node(
+ 223,
+ 'img',
+ {'src': f'/{"a" * DOMTreeSerializer.MAX_IMAGE_CONTEXT_ATTRIBUTE_LENGTH}/must-not-leak.png'},
+ x=18,
+ y=18,
+ )
+
+ llm_dom = DOMTreeSerializer.serialize_tree(
+ SimplifiedNode(
+ original_node=image,
+ children=[],
+ is_interactive=True,
+ selector_index=223,
+ ),
+ DEFAULT_INCLUDE_ATTRIBUTES,
+ )
+
+ assert llm_dom == '[223]<img />'
+ assert 'must-not-leak' not in llm_dom
+
+
def _simplified_image(backend_node_id: int, attributes: dict[str, str]) -> SimplifiedNode:
image = _make_element_node(backend_node_id, 'img', attributes, x=18, y=18)
return SimplifiedNode(original_node=image, children=[])