Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fix(dom): address image context review feedback

UniversePeak469ed940e093
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.py
index 9fcee00c6..827a9a35e 100644
--- a/browser_use/dom/serializer/serializer.py
+++ b/browser_use/dom/serializer/serializer.py
@@ -55,6 +55,8 @@ class DOMTreeSerializer:
# {'tag': 'span', 'role': 'link'}, # <span role="link">
]
DEFAULT_CONTAINMENT_THRESHOLD = 0.99 # 99% containment by default
+ MAX_CHILD_IMAGE_CONTEXTS = 3
+ MAX_CHILD_IMAGE_DESCENDANTS = 100
def __init__(
self,
@@ -929,12 +931,17 @@ class DOMTreeSerializer:
if clean_src.lower().startswith('data:'):
return ''
path_without_query = clean_src.split('?', 1)[0].split('#', 1)[0].rstrip('/')
- return path_without_query.rsplit('/', 1)[-1] or clean_src
-
- def collect(current: SimplifiedNode) -> None:
- if len(image_context) >= 3:
- return
-
+ return path_without_query.rsplit('/', 1)[-1]
+
+ pending = list(reversed(node.children))
+ visited_descendants = 0
+ while (
+ pending
+ and visited_descendants < DOMTreeSerializer.MAX_CHILD_IMAGE_DESCENDANTS
+ and len(image_context) < DOMTreeSerializer.MAX_CHILD_IMAGE_CONTEXTS
+ ):
+ current = pending.pop()
+ visited_descendants += 1
original_node = current.original_node
if original_node.node_type == NodeType.ELEMENT_NODE and original_node.tag_name == 'img':
attributes = original_node.attributes or {}
@@ -945,22 +952,18 @@ class DOMTreeSerializer:
('title', 'image_title'),
('aria-label', 'image_label'),
):
- attr_value = attributes.get(attr_name, '').strip()
+ attr_value = str(attributes.get(attr_name) or '').strip()
if attr_value:
parts.append(f'{output_name}={cap_text_length(attr_value, 100)}')
- src = normalize_src(attributes.get('src', ''))
+ src = normalize_src(str(attributes.get('src') or ''))
if src:
parts.append(f'image_src={cap_text_length(src, 100)}')
if parts:
image_context.append(' '.join(parts))
- for child in current.children:
- collect(child)
-
- for child in node.children:
- collect(child)
+ pending.extend(reversed(current.children))
return ' '.join(image_context)
diff --git a/tests/ci/test_image_only_dom_representation.py b/tests/ci/test_image_only_dom_representation.py
index 7bb83fcc0..65a6c92f5 100644
--- a/tests/ci/test_image_only_dom_representation.py
+++ b/tests/ci/test_image_only_dom_representation.py
@@ -79,3 +79,80 @@ def test_image_only_interactive_parent_includes_child_image_context_in_llm_dom()
assert '[201]<a' in llm_dom
assert 'acme-bank-primary-card.png' in llm_dom
+
+
+def _simplified_image(backend_node_id: int, attributes: dict[str, str]) -> SimplifiedNode:
+ image = _make_element_node(backend_node_id, 'img', attributes, x=18, y=18)
+ return SimplifiedNode(original_node=image, children=[])
+
+
+def test_child_image_context_strips_query_and_fragment_without_leaking_query_only_sources():
+ parent = _make_element_node(301, 'a', {'href': '/cards'}, x=10, y=10)
+ node = SimplifiedNode(
+ original_node=parent,
+ children=[
+ _simplified_image(302, {'src': 'https://cdn.test/card.png?token=secret#preview'}),
+ _simplified_image(303, {'src': '?token=must-not-leak'}),
+ ],
+ )
+
+ context = DOMTreeSerializer._get_child_image_context(node)
+
+ assert context == 'image_src=card.png'
+ assert 'secret' not in context
+ assert 'must-not-leak' not in context
+
+
+def test_child_image_context_ignores_data_src_but_keeps_accessible_attributes():
+ parent = _make_element_node(401, 'button', {}, x=10, y=10)
+ node = SimplifiedNode(
+ original_node=parent,
+ children=[
+ _simplified_image(
+ 402,
+ {
+ 'src': 'data:image/png;base64,private-payload',
+ 'alt': 'Payment card',
+ 'title': 'Choose card',
+ 'aria-label': 'Primary payment method',
+ },
+ ),
+ ],
+ )
+
+ context = DOMTreeSerializer._get_child_image_context(node)
+
+ assert context == 'image_alt=Payment card image_title=Choose card image_label=Primary payment method'
+ assert 'data:' not in context
+ assert 'private-payload' not in context
+
+
+def test_child_image_context_limits_returned_images():
+ parent = _make_element_node(501, 'a', {'href': '/gallery'}, x=10, y=10)
+ node = SimplifiedNode(
+ original_node=parent,
+ children=[_simplified_image(502 + index, {'src': f'/image-{index}.png'}) for index in range(4)],
+ )
+
+ context = DOMTreeSerializer._get_child_image_context(node)
+
+ assert 'image-0.png' in context
+ assert 'image-1.png' in context
+ assert 'image-2.png' in context
+ assert 'image-3.png' not in context
+
+
+def test_child_image_context_caps_traversal_even_when_images_have_no_context():
+ parent = _make_element_node(601, 'a', {'href': '/gallery'}, x=10, y=10)
+ ignored_images = [
+ _simplified_image(602 + index, {'src': 'data:image/png;base64,ignored'})
+ for index in range(DOMTreeSerializer.MAX_CHILD_IMAGE_DESCENDANTS)
+ ]
+ node = SimplifiedNode(
+ original_node=parent,
+ children=[*ignored_images, _simplified_image(999, {'src': '/too-deep.png'})],
+ )
+
+ context = DOMTreeSerializer._get_child_image_context(node)
+
+ assert context == ''