Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
fix(dom): expose image context for directly clickable images
r266-tech81a472a3e2c6
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.pyindex 8b86a9c75..1c2f9df32 100644--- a/browser_use/dom/serializer/serializer.py+++ b/browser_use/dom/serializer/serializer.py@@ -16,6 +16,7 @@ from browser_use.dom.views import ()DISABLED_ELEMENTS = {'style', 'script', 'head', 'meta', 'link', 'title'}+URL_C0_CONTROL_OR_SPACE = ''.join(chr(codepoint) for codepoint in range(0x21))# SVG child elements to skip (decorative only, no interaction value)SVG_ELEMENTS = {@@ -57,6 +58,7 @@ class DOMTreeSerializer:DEFAULT_CONTAINMENT_THRESHOLD = 0.99 # 99% containment by defaultMAX_CHILD_IMAGE_CONTEXTS = 3MAX_CHILD_IMAGE_DESCENDANTS = 100+ MAX_IMAGE_CONTEXT_ATTRIBUTE_LENGTH = 4096def __init__(self,@@ -922,17 +924,47 @@ class DOMTreeSerializer:@staticmethoddef _get_child_image_context(node: SimplifiedNode) -> str:- """Extract compact context from image descendants of an interactive element."""+ """Extract compact context from an interactive image and its descendants."""image_context: list[str] = []def normalize_src(src: str) -> str:- clean_src = src.strip()+ if len(src) > DOMTreeSerializer.MAX_IMAGE_CONTEXT_ATTRIBUTE_LENGTH:+ return ''+ clean_src = src.strip(URL_C0_CONTROL_OR_SPACE).replace('\t', '').replace('\n', '').replace('\r', '')if clean_src.lower().startswith('data:'):return ''path_without_query = clean_src.split('?', 1)[0].split('#', 1)[0].rstrip('/')return path_without_query.rsplit('/', 1)[-1]+ def add_image_context(original_node: EnhancedDOMTreeNode) -> None:+ if original_node.node_type != NodeType.ELEMENT_NODE or original_node.tag_name != 'img':+ return++ attributes = original_node.attributes or {}+ parts = []++ for attr_name, output_name in (+ ('alt', 'image_alt'),+ ('title', 'image_title'),+ ('aria-label', 'image_label'),+ ):+ raw_attr_value = str(attributes.get(attr_name) or '')+ if len(raw_attr_value) > DOMTreeSerializer.MAX_IMAGE_CONTEXT_ATTRIBUTE_LENGTH:+ continue+ attr_value = raw_attr_value.strip()+ if attr_value:+ parts.append(f'{output_name}={cap_text_length(attr_value, 100)}')++ src = normalize_src(str(attributes.get('src') or ''))+ if src:+ parts.append(f'image_src={cap_text_length(src, 100)}')++ if parts:+ image_context.append(' '.join(parts))++ add_image_context(node.original_node)+child_iterators = [iter(node.children)]visited_descendants = 0while (@@ -946,26 +978,7 @@ class DOMTreeSerializer:child_iterators.pop()continuevisited_descendants += 1- original_node = current.original_node- if original_node.node_type == NodeType.ELEMENT_NODE and original_node.tag_name == 'img':- attributes = original_node.attributes or {}- parts = []-- for attr_name, output_name in (- ('alt', 'image_alt'),- ('title', 'image_title'),- ('aria-label', 'image_label'),- ):- attr_value = str(attributes.get(attr_name) or '').strip()- if attr_value:- parts.append(f'{output_name}={cap_text_length(attr_value, 100)}')-- src = normalize_src(str(attributes.get('src') or ''))- if src:- parts.append(f'image_src={cap_text_length(src, 100)}')-- if parts:- image_context.append(' '.join(parts))+ add_image_context(current.original_node)if current.children:child_iterators.append(iter(current.children))diff --git a/tests/ci/test_image_only_dom_representation.py b/tests/ci/test_image_only_dom_representation.pyindex 1ff1d3fe9..3aa48030b 100644--- a/tests/ci/test_image_only_dom_representation.py+++ b/tests/ci/test_image_only_dom_representation.py@@ -81,6 +81,79 @@ def test_image_only_interactive_parent_includes_child_image_context_in_llm_dom()assert 'acme-bank-primary-card.png' in llm_dom+def test_direct_interactive_image_includes_own_context_in_llm_dom():+ """A directly clickable image should expose its own sanitized source context."""+ image = _make_element_node(+ 221,+ 'img',+ {'src': 'https://cdn.example.test/logos/acme-card.png?token=must-not-leak#preview'},+ x=18,+ y=18,+ )++ llm_dom = DOMTreeSerializer.serialize_tree(+ SimplifiedNode(+ original_node=image,+ children=[],+ is_interactive=True,+ selector_index=221,+ ),+ DEFAULT_INCLUDE_ATTRIBUTES,+ )++ assert '[221]<img' in llm_dom+ assert 'image_src=acme-card.png' in llm_dom+ assert 'must-not-leak' not in llm_dom+++def test_direct_interactive_image_rejects_browser_normalized_data_src():+ """Data URLs remain hidden when URL parsing ignores control characters in the scheme."""+ image = _make_element_node(+ 222,+ 'img',+ {'src': '\x00Da\nTa:image/png;base64,must-not-leak'},+ x=18,+ y=18,+ )++ llm_dom = DOMTreeSerializer.serialize_tree(+ SimplifiedNode(+ original_node=image,+ children=[],+ is_interactive=True,+ selector_index=222,+ ),+ DEFAULT_INCLUDE_ATTRIBUTES,+ )++ assert llm_dom == '[222]<img />'+ assert 'must-not-leak' not in llm_dom+++def test_direct_interactive_image_omits_oversized_src_context():+ """Image context does not scan or serialize oversized raw source attributes."""+ image = _make_element_node(+ 223,+ 'img',+ {'src': f'/{"a" * DOMTreeSerializer.MAX_IMAGE_CONTEXT_ATTRIBUTE_LENGTH}/must-not-leak.png'},+ x=18,+ y=18,+ )++ llm_dom = DOMTreeSerializer.serialize_tree(+ SimplifiedNode(+ original_node=image,+ children=[],+ is_interactive=True,+ selector_index=223,+ ),+ DEFAULT_INCLUDE_ATTRIBUTES,+ )++ assert llm_dom == '[223]<img />'+ assert 'must-not-leak' not in llm_dom++def _simplified_image(backend_node_id: int, attributes: dict[str, str]) -> SimplifiedNode:image = _make_element_node(backend_node_id, 'img', attributes, x=18, y=18)return SimplifiedNode(original_node=image, children=[])