Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Include information about dom metadata in prompt

Magnus Müller6244c97876ba
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.py
index 928ed05ba..bd9436df0 100644
--- a/browser_use/agent/prompts.py
+++ b/browser_use/agent/prompts.py
@@ -2,6 +2,7 @@ import importlib.resources
from datetime import datetime
from typing import TYPE_CHECKING, Literal, Optional
+from browser_use.dom.views import NodeType, SimplifiedNode
from browser_use.llm.messages import ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessage
from browser_use.observability import observe_debug
from browser_use.utils import is_new_tab_page
@@ -112,8 +113,93 @@ class AgentMessagePrompt:
self.sample_images = sample_images or []
assert self.browser_state
+ def _extract_page_statistics(self) -> dict[str, int]:
+ """Extract high-level page statistics from DOM tree for LLM context"""
+ stats = {
+ 'links': 0,
+ 'iframes': 0,
+ 'shadow_open': 0,
+ 'shadow_closed': 0,
+ 'scroll_containers': 0,
+ 'images': 0,
+ 'interactive_elements': 0,
+ 'total_elements': 0,
+ }
+
+ if not self.browser_state.dom_state or not self.browser_state.dom_state._root:
+ return stats
+
+ def traverse_node(node: SimplifiedNode) -> None:
+ """Recursively traverse simplified DOM tree to count elements"""
+ if not node or not node.original_node:
+ return
+
+ original = node.original_node
+ stats['total_elements'] += 1
+
+ # Count by node type and tag
+ if original.node_type == NodeType.ELEMENT_NODE:
+ tag = original.tag_name.lower() if original.tag_name else ''
+
+ if tag == 'a':
+ stats['links'] += 1
+ elif tag in ('iframe', 'frame'):
+ stats['iframes'] += 1
+ elif tag == 'img':
+ stats['images'] += 1
+
+ # Check if scrollable
+ if original.is_actually_scrollable:
+ stats['scroll_containers'] += 1
+
+ # Check if interactive
+ if node.interactive_index is not None:
+ stats['interactive_elements'] += 1
+
+ # Check if this element hosts shadow DOM
+ if node.is_shadow_host:
+ # Check if any shadow children are closed
+ has_closed_shadow = any(
+ child.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE
+ and child.original_node.shadow_root_type
+ and child.original_node.shadow_root_type.lower() == 'closed'
+ for child in node.children
+ )
+ if has_closed_shadow:
+ stats['shadow_closed'] += 1
+ else:
+ stats['shadow_open'] += 1
+
+ elif original.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
+ # Shadow DOM fragment - these are the actual shadow roots
+ # But don't double-count since we count them at the host level above
+ pass
+
+ # Traverse children
+ for child in node.children:
+ traverse_node(child)
+
+ traverse_node(self.browser_state.dom_state._root)
+ return stats
+
@observe_debug(ignore_input=True, ignore_output=True, name='_get_browser_state_description')
def _get_browser_state_description(self) -> str:
+ # Extract page statistics first
+ page_stats = self._extract_page_statistics()
+
+ # Format statistics for LLM
+ stats_text = '<page_stats>'
+ if page_stats['total_elements'] < 10:
+ stats_text += 'Page appears empty (SPA not loaded?) - '
+ stats_text += f'{page_stats["links"]} links, {page_stats["interactive_elements"]} interactive, '
+ stats_text += f'{page_stats["iframes"]} iframes, {page_stats["scroll_containers"]} scroll containers'
+ if page_stats['shadow_open'] > 0 or page_stats['shadow_closed'] > 0:
+ stats_text += f', {page_stats["shadow_open"]} shadow(open), {page_stats["shadow_closed"]} shadow(closed)'
+ if page_stats['images'] > 0:
+ stats_text += f', {page_stats["images"]} images'
+ stats_text += f', {page_stats["total_elements"]} total elements'
+ stats_text += '</page_stats>\n\n'
+
elements_text = self.browser_state.dom_state.llm_representation(include_attributes=self.include_attributes)
if len(elements_text) > self.max_clickable_elements_length:
@@ -189,7 +275,7 @@ class AgentMessagePrompt:
if self.include_recent_events and self.browser_state.recent_events:
recent_events_text = f'Recent browser events: {self.browser_state.recent_events}\n'
- browser_state = f"""{current_tab_text}
+ browser_state = f"""{stats_text}{current_tab_text}
Available tabs:
{tabs_text}
{page_info_text}
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.py
index 436faf20e..1b199965d 100644
--- a/browser_use/dom/serializer/serializer.py
+++ b/browser_use/dom/serializer/serializer.py
@@ -137,13 +137,16 @@ class DOMTreeSerializer:
return None
if node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
- # Super simple pass-through for shadow DOM elements
+ # ENHANCED shadow DOM processing - always include shadow content
simplified = SimplifiedNode(original_node=node, children=[])
for child in node.children_and_shadow_roots:
simplified_child = self._create_simplified_tree(child, depth + 1)
if simplified_child:
simplified.children.append(simplified_child)
- return simplified
+
+ # Always return shadow DOM fragments, even if children seem empty
+ # Shadow DOM often contains the actual interactive content in SPAs
+ return simplified if simplified.children else SimplifiedNode(original_node=node, children=[])
elif node.node_type == NodeType.ELEMENT_NODE:
# Skip non-content elements
@@ -161,19 +164,26 @@ class DOMTreeSerializer:
is_visible = node.is_visible
is_scrollable = node.is_actually_scrollable
+ has_shadow_content = bool(node.children_and_shadow_roots)
- # Include if interactive (regardless of visibility), or scrollable, or has children to process
+ # ENHANCED SHADOW DOM DETECTION: Include shadow hosts even if not visible
+ is_shadow_host = any(child.node_type == NodeType.DOCUMENT_FRAGMENT_NODE for child in node.children_and_shadow_roots)
- if is_visible or is_scrollable or bool(node.children_and_shadow_roots):
- simplified = SimplifiedNode(original_node=node, children=[])
- # simplified._analysis = analysis # Store analysis for grouping
+ # Include if interactive (regardless of visibility), scrollable, has children, or is shadow host
+ if is_visible or is_scrollable or has_shadow_content or is_shadow_host:
+ simplified = SimplifiedNode(original_node=node, children=[], is_shadow_host=is_shadow_host)
- # Process children
+ # Process ALL children including shadow roots with enhanced logging
for child in node.children_and_shadow_roots:
simplified_child = self._create_simplified_tree(child, depth + 1)
if simplified_child:
simplified.children.append(simplified_child)
+ # SHADOW DOM SPECIAL CASE: Always include shadow hosts even if not visible
+ # Many SPA frameworks (React, Vue) render content in shadow DOM
+ if is_shadow_host and simplified.children:
+ return simplified
+
# Return if meaningful or has meaningful children
if is_visible or is_scrollable or simplified.children:
return simplified
@@ -449,23 +459,34 @@ class DOMTreeSerializer:
# Build attributes string
attributes_html_str = DOMTreeSerializer._build_attributes_string(node.original_node, include_attributes, '')
- # Build the line
+ # Build the line with shadow host indicator
+ shadow_prefix = ''
+ if node.is_shadow_host:
+ # Check if any shadow children are closed
+ has_closed_shadow = any(
+ child.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE
+ and child.original_node.shadow_root_type
+ and child.original_node.shadow_root_type.lower() == 'closed'
+ for child in node.children
+ )
+ shadow_prefix = '|SHADOW(closed)|' if has_closed_shadow else '|SHADOW(open)|'
+
if should_show_scroll and node.interactive_index is None:
# Scrollable container but not clickable
- line = f'{depth_str}|SCROLL|<{node.original_node.tag_name}'
+ line = f'{depth_str}{shadow_prefix}|SCROLL|<{node.original_node.tag_name}'
elif node.interactive_index is not None:
# Clickable (and possibly scrollable)
new_prefix = '*' if node.is_new else ''
scroll_prefix = '|SCROLL+' if should_show_scroll else '['
- line = f'{depth_str}{new_prefix}{scroll_prefix}{node.interactive_index}]<{node.original_node.tag_name}'
+ line = f'{depth_str}{shadow_prefix}{new_prefix}{scroll_prefix}{node.interactive_index}]<{node.original_node.tag_name}'
elif node.original_node.tag_name.upper() == 'IFRAME':
# Iframe element (not interactive)
- line = f'{depth_str}|IFRAME|<{node.original_node.tag_name}'
+ line = f'{depth_str}{shadow_prefix}|IFRAME|<{node.original_node.tag_name}'
elif node.original_node.tag_name.upper() == 'FRAME':
# Frame element (not interactive)
- line = f'{depth_str}|FRAME|<{node.original_node.tag_name}'
+ line = f'{depth_str}{shadow_prefix}|FRAME|<{node.original_node.tag_name}'
else:
- line = f'{depth_str}<{node.original_node.tag_name}'
+ line = f'{depth_str}{shadow_prefix}<{node.original_node.tag_name}'
if attributes_html_str:
line += f' {attributes_html_str}'
@@ -480,6 +501,25 @@ class DOMTreeSerializer:
formatted_text.append(line)
+ elif node.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
+ # Shadow DOM representation - show clearly to LLM
+ if node.original_node.shadow_root_type and node.original_node.shadow_root_type.lower() == 'closed':
+ formatted_text.append(f'{depth_str}▼ Shadow Content (Closed)')
+ else:
+ formatted_text.append(f'{depth_str}▼ Shadow Content (Open)')
+
+ next_depth += 1
+
+ # Process shadow DOM children
+ for child in node.children:
+ child_text = DOMTreeSerializer.serialize_tree(child, include_attributes, next_depth)
+ if child_text:
+ formatted_text.append(child_text)
+
+ # Close shadow DOM indicator
+ if node.children: # Only show close if we had content
+ formatted_text.append(f'{depth_str}▲ Shadow Content End')
+
elif node.original_node.node_type == NodeType.TEXT_NODE:
# Include visible text
is_visible = node.original_node.snapshot_node and node.original_node.is_visible
@@ -492,11 +532,12 @@ class DOMTreeSerializer:
clean_text = node.original_node.node_value.strip()
formatted_text.append(f'{depth_str}{clean_text}')
- # Process children
- for child in node.children:
- child_text = DOMTreeSerializer.serialize_tree(child, include_attributes, next_depth)
- if child_text:
- formatted_text.append(child_text)
+ # Process children (for non-shadow elements)
+ if node.original_node.node_type != NodeType.DOCUMENT_FRAGMENT_NODE:
+ for child in node.children:
+ child_text = DOMTreeSerializer.serialize_tree(child, include_attributes, next_depth)
+ if child_text:
+ formatted_text.append(child_text)
return '\n'.join(formatted_text)
diff --git a/browser_use/dom/views.py b/browser_use/dom/views.py
index 854565879..061b070a8 100644
--- a/browser_use/dom/views.py
+++ b/browser_use/dom/views.py
@@ -19,6 +19,8 @@ DEFAULT_INCLUDE_ATTRIBUTES = [
'title',
'type',
'checked',
+ # 'class',
+ 'id',
'name',
'role',
'value',
@@ -138,6 +140,7 @@ class SimplifiedNode:
ignored_by_paint_order: bool = False # More info in dom/serializer/paint_order.py
excluded_by_parent: bool = False # New field for bbox filtering
+ is_shadow_host: bool = False # New field for shadow DOM hosts
def _clean_original_node_json(self, node_json: dict) -> dict:
"""Recursively remove children_nodes and shadow_roots from original_node JSON."""