Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Include more dropdown info
Magnus Müller209e69f8499f
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.pyindex fe14c7085..ea6e76d9b 100644--- a/browser_use/dom/serializer/serializer.py+++ b/browser_use/dom/serializer/serializer.py@@ -1,5 +1,6 @@# @file purpose: Serializes enhanced DOM trees to string format for LLM consumption+from typing import Anyfrom browser_use.dom.serializer.clickable_elements import ClickableElementDetectorfrom browser_use.dom.serializer.paint_order import PaintOrderRemover@@ -107,7 +108,6 @@ class DOMTreeSerializer:return SerializedDOMState(_root=filtered_tree, selector_map=self._selector_map), self.timing_info-def _add_compound_components(self, simplified: SimplifiedNode, node: EnhancedDOMTreeNode) -> None:"""Enhance compound controls with information from their child components."""# Only process elements that might have compound components@@ -116,8 +116,17 @@ class DOMTreeSerializer:# For input elements, check for compound input typesif node.tag_name == 'input':- if (not node.attributes or- node.attributes.get('type') not in ['date', 'time', 'datetime-local', 'month', 'week', 'range', 'number', 'color', 'file']):+ if not node.attributes or node.attributes.get('type') not in [+ 'date',+ 'time',+ 'datetime-local',+ 'month',+ 'week',+ 'range',+ 'number',+ 'color',+ 'file',+ ]:return# For other elements, check if they have AX child indicatorselif not node.ax_node or not node.ax_node.child_ids:@@ -129,117 +138,254 @@ class DOMTreeSerializer:if element_type == 'input':if input_type == 'date':- node._compound_children.extend([- {'role': 'spinbutton', 'name': 'Day', 'valuemin': 1, 'valuemax': 31, 'valuenow': None},- {'role': 'spinbutton', 'name': 'Month', 'valuemin': 1, 'valuemax': 12, 'valuenow': None},- {'role': 'spinbutton', 'name': 'Year', 'valuemin': 1, 'valuemax': 275760, 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'spinbutton', 'name': 'Day', 'valuemin': 1, 'valuemax': 31, 'valuenow': None},+ {'role': 'spinbutton', 'name': 'Month', 'valuemin': 1, 'valuemax': 12, 'valuenow': None},+ {'role': 'spinbutton', 'name': 'Year', 'valuemin': 1, 'valuemax': 275760, 'valuenow': None},+ ]+ )simplified.is_compound_component = Trueelif input_type == 'time':- node._compound_children.extend([- {'role': 'spinbutton', 'name': 'Hour', 'valuemin': 0, 'valuemax': 23, 'valuenow': None},- {'role': 'spinbutton', 'name': 'Minute', 'valuemin': 0, 'valuemax': 59, 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'spinbutton', 'name': 'Hour', 'valuemin': 0, 'valuemax': 23, 'valuenow': None},+ {'role': 'spinbutton', 'name': 'Minute', 'valuemin': 0, 'valuemax': 59, 'valuenow': None},+ ]+ )simplified.is_compound_component = Trueelif input_type == 'datetime-local':- node._compound_children.extend([- {'role': 'spinbutton', 'name': 'Day', 'valuemin': 1, 'valuemax': 31, 'valuenow': None},- {'role': 'spinbutton', 'name': 'Month', 'valuemin': 1, 'valuemax': 12, 'valuenow': None},- {'role': 'spinbutton', 'name': 'Year', 'valuemin': 1, 'valuemax': 275760, 'valuenow': None},- {'role': 'spinbutton', 'name': 'Hour', 'valuemin': 0, 'valuemax': 23, 'valuenow': None},- {'role': 'spinbutton', 'name': 'Minute', 'valuemin': 0, 'valuemax': 59, 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'spinbutton', 'name': 'Day', 'valuemin': 1, 'valuemax': 31, 'valuenow': None},+ {'role': 'spinbutton', 'name': 'Month', 'valuemin': 1, 'valuemax': 12, 'valuenow': None},+ {'role': 'spinbutton', 'name': 'Year', 'valuemin': 1, 'valuemax': 275760, 'valuenow': None},+ {'role': 'spinbutton', 'name': 'Hour', 'valuemin': 0, 'valuemax': 23, 'valuenow': None},+ {'role': 'spinbutton', 'name': 'Minute', 'valuemin': 0, 'valuemax': 59, 'valuenow': None},+ ]+ )simplified.is_compound_component = Trueelif input_type == 'month':- node._compound_children.extend([- {'role': 'spinbutton', 'name': 'Month', 'valuemin': 1, 'valuemax': 12, 'valuenow': None},- {'role': 'spinbutton', 'name': 'Year', 'valuemin': 1, 'valuemax': 275760, 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'spinbutton', 'name': 'Month', 'valuemin': 1, 'valuemax': 12, 'valuenow': None},+ {'role': 'spinbutton', 'name': 'Year', 'valuemin': 1, 'valuemax': 275760, 'valuenow': None},+ ]+ )simplified.is_compound_component = Trueelif input_type == 'week':- node._compound_children.extend([- {'role': 'spinbutton', 'name': 'Week', 'valuemin': 1, 'valuemax': 53, 'valuenow': None},- {'role': 'spinbutton', 'name': 'Year', 'valuemin': 1, 'valuemax': 275760, 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'spinbutton', 'name': 'Week', 'valuemin': 1, 'valuemax': 53, 'valuenow': None},+ {'role': 'spinbutton', 'name': 'Year', 'valuemin': 1, 'valuemax': 275760, 'valuenow': None},+ ]+ )simplified.is_compound_component = Trueelif input_type == 'range':# Range slider with value indicatormin_val = node.attributes.get('min', '0') if node.attributes else '0'max_val = node.attributes.get('max', '100') if node.attributes else '100'- node._compound_children.append({- 'role': 'slider',- 'name': 'Value',- 'valuemin': int(min_val) if min_val.isdigit() else 0,- 'valuemax': int(max_val) if max_val.isdigit() else 100,- 'valuenow': None- })+ node._compound_children.append(+ {+ 'role': 'slider',+ 'name': 'Value',+ 'valuemin': int(min_val) if min_val.isdigit() else 0,+ 'valuemax': int(max_val) if max_val.isdigit() else 100,+ 'valuenow': None,+ }+ )simplified.is_compound_component = Trueelif input_type == 'number':# Number input with increment/decrement buttonsmin_val = node.attributes.get('min') if node.attributes else Nonemax_val = node.attributes.get('max') if node.attributes else None- node._compound_children.extend([- {'role': 'button', 'name': 'Increment', 'valuemin': None, 'valuemax': None, 'valuenow': None},- {'role': 'button', 'name': 'Decrement', 'valuemin': None, 'valuemax': None, 'valuenow': None},- {'role': 'textbox', 'name': 'Value',- 'valuemin': int(min_val) if min_val and min_val.lstrip('-').isdigit() else None,- 'valuemax': int(max_val) if max_val and max_val.lstrip('-').isdigit() else None,- 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'button', 'name': 'Increment', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ {'role': 'button', 'name': 'Decrement', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ {+ 'role': 'textbox',+ 'name': 'Value',+ 'valuemin': int(min_val) if min_val and min_val.lstrip('-').isdigit() else None,+ 'valuemax': int(max_val) if max_val and max_val.lstrip('-').isdigit() else None,+ 'valuenow': None,+ },+ ]+ )simplified.is_compound_component = Trueelif input_type == 'color':# Color picker with components- node._compound_children.extend([- {'role': 'textbox', 'name': 'Hex Value', 'valuemin': None, 'valuemax': None, 'valuenow': None},- {'role': 'button', 'name': 'Color Picker', 'valuemin': None, 'valuemax': None, 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'textbox', 'name': 'Hex Value', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ {'role': 'button', 'name': 'Color Picker', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ ]+ )simplified.is_compound_component = Trueelif input_type == 'file':# File input with browse buttonmultiple = 'multiple' in node.attributes if node.attributes else False- node._compound_children.extend([- {'role': 'button', 'name': 'Browse Files', 'valuemin': None, 'valuemax': None, 'valuenow': None},- {'role': 'textbox', 'name': f'{"Files" if multiple else "File"} Selected', 'valuemin': None, 'valuemax': None, 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'button', 'name': 'Browse Files', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ {+ 'role': 'textbox',+ 'name': f'{"Files" if multiple else "File"} Selected',+ 'valuemin': None,+ 'valuemax': None,+ 'valuenow': None,+ },+ ]+ )simplified.is_compound_component = Trueelif element_type == 'select':- # Select dropdown with option list- node._compound_children.extend([- {'role': 'button', 'name': 'Dropdown Toggle', 'valuemin': None, 'valuemax': None, 'valuenow': None},- {'role': 'listbox', 'name': 'Options', 'valuemin': None, 'valuemax': None, 'valuenow': None}- ])+ # Select dropdown with option list and detailed option information+ base_components = [+ {'role': 'button', 'name': 'Dropdown Toggle', 'valuemin': None, 'valuemax': None, 'valuenow': None}+ ]++ # Extract option information from child nodes+ options_info = self._extract_select_options(node)+ if options_info:+ options_component = {+ 'role': 'listbox',+ 'name': 'Options',+ 'valuemin': None,+ 'valuemax': None,+ 'valuenow': None,+ 'options_count': options_info['count'],+ 'first_options': options_info['first_options'],+ }+ if options_info['format_hint']:+ options_component['format_hint'] = options_info['format_hint']+ base_components.append(options_component)+ else:+ base_components.append(+ {'role': 'listbox', 'name': 'Options', 'valuemin': None, 'valuemax': None, 'valuenow': None}+ )++ node._compound_children.extend(base_components)simplified.is_compound_component = Trueelif element_type == 'details':# Details/summary disclosure widget- node._compound_children.extend([- {'role': 'button', 'name': 'Toggle Disclosure', 'valuemin': None, 'valuemax': None, 'valuenow': None},- {'role': 'region', 'name': 'Content Area', 'valuemin': None, 'valuemax': None, 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'button', 'name': 'Toggle Disclosure', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ {'role': 'region', 'name': 'Content Area', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ ]+ )simplified.is_compound_component = Trueelif element_type == 'audio':# Audio player controls- node._compound_children.extend([- {'role': 'button', 'name': 'Play/Pause', 'valuemin': None, 'valuemax': None, 'valuenow': None},- {'role': 'slider', 'name': 'Progress', 'valuemin': 0, 'valuemax': 100, 'valuenow': None},- {'role': 'button', 'name': 'Mute', 'valuemin': None, 'valuemax': None, 'valuenow': None},- {'role': 'slider', 'name': 'Volume', 'valuemin': 0, 'valuemax': 100, 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'button', 'name': 'Play/Pause', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ {'role': 'slider', 'name': 'Progress', 'valuemin': 0, 'valuemax': 100, 'valuenow': None},+ {'role': 'button', 'name': 'Mute', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ {'role': 'slider', 'name': 'Volume', 'valuemin': 0, 'valuemax': 100, 'valuenow': None},+ ]+ )simplified.is_compound_component = Trueelif element_type == 'video':# Video player controls- node._compound_children.extend([- {'role': 'button', 'name': 'Play/Pause', 'valuemin': None, 'valuemax': None, 'valuenow': None},- {'role': 'slider', 'name': 'Progress', 'valuemin': 0, 'valuemax': 100, 'valuenow': None},- {'role': 'button', 'name': 'Mute', 'valuemin': None, 'valuemax': None, 'valuenow': None},- {'role': 'slider', 'name': 'Volume', 'valuemin': 0, 'valuemax': 100, 'valuenow': None},- {'role': 'button', 'name': 'Fullscreen', 'valuemin': None, 'valuemax': None, 'valuenow': None}- ])+ node._compound_children.extend(+ [+ {'role': 'button', 'name': 'Play/Pause', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ {'role': 'slider', 'name': 'Progress', 'valuemin': 0, 'valuemax': 100, 'valuenow': None},+ {'role': 'button', 'name': 'Mute', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ {'role': 'slider', 'name': 'Volume', 'valuemin': 0, 'valuemax': 100, 'valuenow': None},+ {'role': 'button', 'name': 'Fullscreen', 'valuemin': None, 'valuemax': None, 'valuenow': None},+ ]+ )simplified.is_compound_component = True+ def _extract_select_options(self, select_node: EnhancedDOMTreeNode) -> dict[str, Any] | None:+ """Extract option information from a select element."""+ if not select_node.children:+ return None++ options = []+ option_values = []++ def extract_options_recursive(node: EnhancedDOMTreeNode) -> None:+ """Recursively extract option elements, including from optgroups."""+ if node.tag_name.lower() == 'option':+ # Extract option text and value+ option_text = ''+ option_value = ''++ # Get value attribute if present+ if node.attributes and 'value' in node.attributes:+ option_value = str(node.attributes['value']).strip()++ # Get text content from direct child text nodes only to avoid duplication+ def get_direct_text_content(n: EnhancedDOMTreeNode) -> str:+ text = ''+ for child in n.children:+ if child.node_type == NodeType.TEXT_NODE and child.node_value:+ text += child.node_value.strip() + ' '+ return text.strip()++ option_text = get_direct_text_content(node)++ # Use text as value if no explicit value+ if not option_value and option_text:+ option_value = option_text++ if option_text or option_value:+ options.append({'text': option_text, 'value': option_value})+ option_values.append(option_value)++ elif node.tag_name.lower() == 'optgroup':+ # Process optgroup children+ for child in node.children:+ extract_options_recursive(child)+ else:+ # Process other children that might contain options+ for child in node.children:+ extract_options_recursive(child)++ # Extract all options from select children+ for child in select_node.children:+ extract_options_recursive(child)++ if not options:+ return None++ # Prepare first 4 options for display+ first_options = []+ for option in options[:4]:+ if option['text'] and option['value'] and option['text'] != option['value']:+ # Limit individual option text to avoid overly long attributes+ text = option['text'][:20] + ('...' if len(option['text']) > 20 else '')+ value = option['value'][:10] + ('...' if len(option['value']) > 10 else '')+ first_options.append(f'{text} ({value})')+ elif option['text']:+ text = option['text'][:25] + ('...' if len(option['text']) > 25 else '')+ first_options.append(text)+ elif option['value']:+ value = option['value'][:25] + ('...' if len(option['value']) > 25 else '')+ first_options.append(value)++ # Try to infer format hint from option values+ format_hint = None+ if len(option_values) >= 2:+ # Check for common patterns+ if all(val.isdigit() for val in option_values[:5] if val):+ format_hint = 'numeric'+ elif all(len(val) == 2 and val.isupper() for val in option_values[:5] if val):+ format_hint = 'country/state codes'+ elif all('/' in val or '-' in val for val in option_values[:5] if val):+ format_hint = 'date/path format'+ elif any('@' in val for val in option_values[:5] if val):+ format_hint = 'email addresses'++ return {'count': len(options), 'first_options': first_options, 'format_hint': format_hint}+def _is_interactive_cached(self, node: EnhancedDOMTreeNode) -> bool:"""Cached version of clickable element detection to avoid redundant calls."""if node.node_id not in self._clickable_cache:@@ -602,7 +748,9 @@ class DOMTreeSerializer:# Build attributes string with compound component infotext_content = ''- attributes_html_str = DOMTreeSerializer._build_attributes_string(node.original_node, include_attributes, text_content)+ attributes_html_str = DOMTreeSerializer._build_attributes_string(+ node.original_node, include_attributes, text_content+ )# Add compound component information to attributes if presentif node.original_node._compound_children:@@ -610,21 +758,30 @@ class DOMTreeSerializer:for child_info in node.original_node._compound_children:parts = []if child_info['name']:- parts.append(f"name={child_info['name']}")+ parts.append(f'name={child_info["name"]}')if child_info['role']:- parts.append(f"role={child_info['role']}")+ parts.append(f'role={child_info["role"]}')if child_info['valuemin'] is not None:- parts.append(f"min={child_info['valuemin']}")+ parts.append(f'min={child_info["valuemin"]}')if child_info['valuemax'] is not None:- parts.append(f"max={child_info['valuemax']}")+ parts.append(f'max={child_info["valuemax"]}')if child_info['valuenow'] is not None:- parts.append(f"current={child_info['valuenow']}")+ parts.append(f'current={child_info["valuenow"]}')++ # Add select-specific information+ if 'options_count' in child_info and child_info['options_count'] is not None:+ parts.append(f'count={child_info["options_count"]}')+ if 'first_options' in child_info and child_info['first_options']:+ options_str = '|'.join(child_info['first_options'][:4]) # Limit to 4 options+ parts.append(f'options={options_str}')+ if 'format_hint' in child_info and child_info['format_hint']:+ parts.append(f'format={child_info["format_hint"]}')if parts:- compound_info.append(f"({','.join(parts)})")+ compound_info.append(f'({",".join(parts)})')if compound_info:- compound_attr = f"compound_components={','.join(compound_info)}"+ compound_attr = f'compound_components={",".join(compound_info)}'if attributes_html_str:attributes_html_str += f' {compound_attr}'else: