Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Include information about dom metadata in prompt
Magnus Müller6244c97876ba
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.pyindex 928ed05ba..bd9436df0 100644--- a/browser_use/agent/prompts.py+++ b/browser_use/agent/prompts.py@@ -2,6 +2,7 @@ import importlib.resourcesfrom datetime import datetimefrom typing import TYPE_CHECKING, Literal, Optional+from browser_use.dom.views import NodeType, SimplifiedNodefrom browser_use.llm.messages import ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessagefrom browser_use.observability import observe_debugfrom browser_use.utils import is_new_tab_page@@ -112,8 +113,93 @@ class AgentMessagePrompt:self.sample_images = sample_images or []assert self.browser_state+ def _extract_page_statistics(self) -> dict[str, int]:+ """Extract high-level page statistics from DOM tree for LLM context"""+ stats = {+ 'links': 0,+ 'iframes': 0,+ 'shadow_open': 0,+ 'shadow_closed': 0,+ 'scroll_containers': 0,+ 'images': 0,+ 'interactive_elements': 0,+ 'total_elements': 0,+ }++ if not self.browser_state.dom_state or not self.browser_state.dom_state._root:+ return stats++ def traverse_node(node: SimplifiedNode) -> None:+ """Recursively traverse simplified DOM tree to count elements"""+ if not node or not node.original_node:+ return++ original = node.original_node+ stats['total_elements'] += 1++ # Count by node type and tag+ if original.node_type == NodeType.ELEMENT_NODE:+ tag = original.tag_name.lower() if original.tag_name else ''++ if tag == 'a':+ stats['links'] += 1+ elif tag in ('iframe', 'frame'):+ stats['iframes'] += 1+ elif tag == 'img':+ stats['images'] += 1++ # Check if scrollable+ if original.is_actually_scrollable:+ stats['scroll_containers'] += 1++ # Check if interactive+ if node.interactive_index is not None:+ stats['interactive_elements'] += 1++ # Check if this element hosts shadow DOM+ if node.is_shadow_host:+ # Check if any shadow children are closed+ has_closed_shadow = any(+ child.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE+ and child.original_node.shadow_root_type+ and child.original_node.shadow_root_type.lower() == 'closed'+ for child in node.children+ )+ if has_closed_shadow:+ stats['shadow_closed'] += 1+ else:+ stats['shadow_open'] += 1++ elif original.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:+ # Shadow DOM fragment - these are the actual shadow roots+ # But don't double-count since we count them at the host level above+ pass++ # Traverse children+ for child in node.children:+ traverse_node(child)++ traverse_node(self.browser_state.dom_state._root)+ return stats+@observe_debug(ignore_input=True, ignore_output=True, name='_get_browser_state_description')def _get_browser_state_description(self) -> str:+ # Extract page statistics first+ page_stats = self._extract_page_statistics()++ # Format statistics for LLM+ stats_text = '<page_stats>'+ if page_stats['total_elements'] < 10:+ stats_text += 'Page appears empty (SPA not loaded?) - '+ stats_text += f'{page_stats["links"]} links, {page_stats["interactive_elements"]} interactive, '+ stats_text += f'{page_stats["iframes"]} iframes, {page_stats["scroll_containers"]} scroll containers'+ if page_stats['shadow_open'] > 0 or page_stats['shadow_closed'] > 0:+ stats_text += f', {page_stats["shadow_open"]} shadow(open), {page_stats["shadow_closed"]} shadow(closed)'+ if page_stats['images'] > 0:+ stats_text += f', {page_stats["images"]} images'+ stats_text += f', {page_stats["total_elements"]} total elements'+ stats_text += '</page_stats>\n\n'+elements_text = self.browser_state.dom_state.llm_representation(include_attributes=self.include_attributes)if len(elements_text) > self.max_clickable_elements_length:@@ -189,7 +275,7 @@ class AgentMessagePrompt:if self.include_recent_events and self.browser_state.recent_events:recent_events_text = f'Recent browser events: {self.browser_state.recent_events}\n'- browser_state = f"""{current_tab_text}+ browser_state = f"""{stats_text}{current_tab_text}Available tabs:{tabs_text}{page_info_text}diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.pyindex 436faf20e..1b199965d 100644--- a/browser_use/dom/serializer/serializer.py+++ b/browser_use/dom/serializer/serializer.py@@ -137,13 +137,16 @@ class DOMTreeSerializer:return Noneif node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:- # Super simple pass-through for shadow DOM elements+ # ENHANCED shadow DOM processing - always include shadow contentsimplified = SimplifiedNode(original_node=node, children=[])for child in node.children_and_shadow_roots:simplified_child = self._create_simplified_tree(child, depth + 1)if simplified_child:simplified.children.append(simplified_child)- return simplified++ # Always return shadow DOM fragments, even if children seem empty+ # Shadow DOM often contains the actual interactive content in SPAs+ return simplified if simplified.children else SimplifiedNode(original_node=node, children=[])elif node.node_type == NodeType.ELEMENT_NODE:# Skip non-content elements@@ -161,19 +164,26 @@ class DOMTreeSerializer:is_visible = node.is_visibleis_scrollable = node.is_actually_scrollable+ has_shadow_content = bool(node.children_and_shadow_roots)- # Include if interactive (regardless of visibility), or scrollable, or has children to process+ # ENHANCED SHADOW DOM DETECTION: Include shadow hosts even if not visible+ is_shadow_host = any(child.node_type == NodeType.DOCUMENT_FRAGMENT_NODE for child in node.children_and_shadow_roots)- if is_visible or is_scrollable or bool(node.children_and_shadow_roots):- simplified = SimplifiedNode(original_node=node, children=[])- # simplified._analysis = analysis # Store analysis for grouping+ # Include if interactive (regardless of visibility), scrollable, has children, or is shadow host+ if is_visible or is_scrollable or has_shadow_content or is_shadow_host:+ simplified = SimplifiedNode(original_node=node, children=[], is_shadow_host=is_shadow_host)- # Process children+ # Process ALL children including shadow roots with enhanced loggingfor child in node.children_and_shadow_roots:simplified_child = self._create_simplified_tree(child, depth + 1)if simplified_child:simplified.children.append(simplified_child)+ # SHADOW DOM SPECIAL CASE: Always include shadow hosts even if not visible+ # Many SPA frameworks (React, Vue) render content in shadow DOM+ if is_shadow_host and simplified.children:+ return simplified+# Return if meaningful or has meaningful childrenif is_visible or is_scrollable or simplified.children:return simplified@@ -449,23 +459,34 @@ class DOMTreeSerializer:# Build attributes stringattributes_html_str = DOMTreeSerializer._build_attributes_string(node.original_node, include_attributes, '')- # Build the line+ # Build the line with shadow host indicator+ shadow_prefix = ''+ if node.is_shadow_host:+ # Check if any shadow children are closed+ has_closed_shadow = any(+ child.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE+ and child.original_node.shadow_root_type+ and child.original_node.shadow_root_type.lower() == 'closed'+ for child in node.children+ )+ shadow_prefix = '|SHADOW(closed)|' if has_closed_shadow else '|SHADOW(open)|'+if should_show_scroll and node.interactive_index is None:# Scrollable container but not clickable- line = f'{depth_str}|SCROLL|<{node.original_node.tag_name}'+ line = f'{depth_str}{shadow_prefix}|SCROLL|<{node.original_node.tag_name}'elif node.interactive_index is not None:# Clickable (and possibly scrollable)new_prefix = '*' if node.is_new else ''scroll_prefix = '|SCROLL+' if should_show_scroll else '['- line = f'{depth_str}{new_prefix}{scroll_prefix}{node.interactive_index}]<{node.original_node.tag_name}'+ line = f'{depth_str}{shadow_prefix}{new_prefix}{scroll_prefix}{node.interactive_index}]<{node.original_node.tag_name}'elif node.original_node.tag_name.upper() == 'IFRAME':# Iframe element (not interactive)- line = f'{depth_str}|IFRAME|<{node.original_node.tag_name}'+ line = f'{depth_str}{shadow_prefix}|IFRAME|<{node.original_node.tag_name}'elif node.original_node.tag_name.upper() == 'FRAME':# Frame element (not interactive)- line = f'{depth_str}|FRAME|<{node.original_node.tag_name}'+ line = f'{depth_str}{shadow_prefix}|FRAME|<{node.original_node.tag_name}'else:- line = f'{depth_str}<{node.original_node.tag_name}'+ line = f'{depth_str}{shadow_prefix}<{node.original_node.tag_name}'if attributes_html_str:line += f' {attributes_html_str}'@@ -480,6 +501,25 @@ class DOMTreeSerializer:formatted_text.append(line)+ elif node.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:+ # Shadow DOM representation - show clearly to LLM+ if node.original_node.shadow_root_type and node.original_node.shadow_root_type.lower() == 'closed':+ formatted_text.append(f'{depth_str}▼ Shadow Content (Closed)')+ else:+ formatted_text.append(f'{depth_str}▼ Shadow Content (Open)')++ next_depth += 1++ # Process shadow DOM children+ for child in node.children:+ child_text = DOMTreeSerializer.serialize_tree(child, include_attributes, next_depth)+ if child_text:+ formatted_text.append(child_text)++ # Close shadow DOM indicator+ if node.children: # Only show close if we had content+ formatted_text.append(f'{depth_str}▲ Shadow Content End')+elif node.original_node.node_type == NodeType.TEXT_NODE:# Include visible textis_visible = node.original_node.snapshot_node and node.original_node.is_visible@@ -492,11 +532,12 @@ class DOMTreeSerializer:clean_text = node.original_node.node_value.strip()formatted_text.append(f'{depth_str}{clean_text}')- # Process children- for child in node.children:- child_text = DOMTreeSerializer.serialize_tree(child, include_attributes, next_depth)- if child_text:- formatted_text.append(child_text)+ # Process children (for non-shadow elements)+ if node.original_node.node_type != NodeType.DOCUMENT_FRAGMENT_NODE:+ for child in node.children:+ child_text = DOMTreeSerializer.serialize_tree(child, include_attributes, next_depth)+ if child_text:+ formatted_text.append(child_text)return '\n'.join(formatted_text)diff --git a/browser_use/dom/views.py b/browser_use/dom/views.pyindex 854565879..061b070a8 100644--- a/browser_use/dom/views.py+++ b/browser_use/dom/views.py@@ -19,6 +19,8 @@ DEFAULT_INCLUDE_ATTRIBUTES = ['title','type','checked',+ # 'class',+ 'id','name','role','value',@@ -138,6 +140,7 @@ class SimplifiedNode:ignored_by_paint_order: bool = False # More info in dom/serializer/paint_order.pyexcluded_by_parent: bool = False # New field for bbox filtering+ is_shadow_host: bool = False # New field for shadow DOM hostsdef _clean_original_node_json(self, node_json: dict) -> dict:"""Recursively remove children_nodes and shadow_roots from original_node JSON."""