Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Merge code7 branch changes

Magnus Müller9caa248316eb
diff --git a/.gitignore b/.gitignore
index d764c0f7f..64cd29727 100644
--- a/.gitignore
+++ b/.gitignore
@@ -72,3 +72,9 @@ TOOL_CHANGES_SUMMARY.md
claude-code-todo
result_judge.md
result.md
+Brainstorm.md
+example.ipynb
+*SUMMARY.md
+todo.md
+product_extraction.ipynb
+product_extraction.py
diff --git a/browser_use/__init__.py b/browser_use/__init__.py
index 7882c7ffe..59a70cfd2 100644
--- a/browser_use/__init__.py
+++ b/browser_use/__init__.py
@@ -47,9 +47,12 @@ base_subprocess.BaseSubprocessTransport.__del__ = _patched_del
if TYPE_CHECKING:
from browser_use.agent.prompts import SystemPrompt
from browser_use.agent.service import Agent
+
+ # from browser_use.agent.service import Agent
from browser_use.agent.views import ActionModel, ActionResult, AgentHistoryList
from browser_use.browser import BrowserProfile, BrowserSession
from browser_use.browser import BrowserSession as Browser
+ from browser_use.code_use.service import CodeAgent
from browser_use.dom.service import DomService
from browser_use.llm import models
from browser_use.llm.anthropic.chat import ChatAnthropic
@@ -66,6 +69,9 @@ if TYPE_CHECKING:
# Lazy imports mapping - only import when actually accessed
_LAZY_IMPORTS = {
# Agent service (heavy due to dependencies)
+ # 'Agent': ('browser_use.agent.service', 'Agent'),
+ # Code-use agent (Jupiter notebook-like execution)
+ 'CodeAgent': ('browser_use.code_use.service', 'CodeAgent'),
'Agent': ('browser_use.agent.service', 'Agent'),
# System prompt (moderate weight due to agent.views imports)
'SystemPrompt': ('browser_use.agent.prompts', 'SystemPrompt'),
@@ -119,6 +125,8 @@ def __getattr__(name: str):
__all__ = [
'Agent',
+ 'CodeAgent',
+ # 'CodeAgent',
'BrowserSession',
'Browser', # Alias for BrowserSession
'BrowserProfile',
diff --git a/browser_use/actor/README.md b/browser_use/actor/README.md
index e9c3cf1d5..24363ac13 100644
--- a/browser_use/actor/README.md
+++ b/browser_use/actor/README.md
@@ -76,6 +76,11 @@ value = await element.get_attribute("value")
box = await element.get_bounding_box() # Returns BoundingBox or None
info = await element.get_basic_info() # Comprehensive element info
screenshot_b64 = await element.screenshot(format='jpeg')
+
+# Execute JavaScript on element (this context is the element)
+text = await element.evaluate("() => this.textContent")
+await element.evaluate("(color) => this.style.backgroundColor = color", "yellow")
+classes = await element.evaluate("() => Array.from(this.classList)")
```
## Mouse Operations
@@ -173,6 +178,7 @@ products = await page.extract_content(
- `check()` - Toggle checkbox/radio button (clicks to change state)
- `select_option(values: str | list[str])` - Select dropdown options
- `drag_to(target_element: Element | Position, source_position=None, target_position=None)` - Drag to target element
+- `evaluate(page_function: str, *args)` → `str` - Execute JavaScript on element (this = element)
- `get_attribute(name: str)` → `str | None` - Get attribute value
- `get_bounding_box()` → `BoundingBox | None` - Get element position/size
- `screenshot(format='jpeg', quality=None)` → `str` - Take element screenshot, return base64
@@ -221,10 +227,11 @@ class ElementInfo(TypedDict):
**This is browser-use actor, NOT Playwright or Selenium.** Only use the methods documented above.
### Critical JavaScript Rules
-- `page.evaluate()` MUST use `(...args) => {}` arrow function format
+- `page.evaluate()` and `element.evaluate()` MUST use `(...args) => {}` arrow function format
- Always returns string (objects are JSON-stringified automatically)
- Use single quotes around the function: `page.evaluate('() => document.title')`
- For complex selectors in JS: `'() => document.querySelector("input[name=\\"email\\"]")'`
+- `element.evaluate()`: `this` context is bound to the element automatically
### Method Restrictions
- `get_elements_by_css_selector()` returns immediately (no automatic waiting)
diff --git a/browser_use/actor/element.py b/browser_use/actor/element.py
index ad7da6b6d..755995047 100644
--- a/browser_use/actor/element.py
+++ b/browser_use/actor/element.py
@@ -22,6 +22,7 @@ if TYPE_CHECKING:
from cdp_use.cdp.input.types import MouseButton
from cdp_use.cdp.page.commands import CaptureScreenshotParameters
from cdp_use.cdp.page.types import Viewport
+ from cdp_use.cdp.runtime.commands import CallFunctionOnParameters
from browser_use.browser.session import BrowserSession
@@ -707,6 +708,126 @@ class Element:
return result['data']
+ async def evaluate(self, page_function: str, *args) -> str:
+ """Execute JavaScript code in the context of this element.
+
+ The JavaScript code executes with 'this' bound to the element, allowing direct
+ access to element properties and methods.
+
+ Args:
+ page_function: JavaScript code that MUST start with (...args) => format
+ *args: Arguments to pass to the function
+
+ Returns:
+ String representation of the JavaScript execution result.
+ Objects and arrays are JSON-stringified.
+
+ Example:
+ # Get element's text content
+ text = await element.evaluate("() => this.textContent")
+
+ # Set style with argument
+ await element.evaluate("(color) => this.style.color = color", "red")
+
+ # Get computed style
+ color = await element.evaluate("() => getComputedStyle(this).color")
+
+ # Async operations
+ result = await element.evaluate("async () => { await new Promise(r => setTimeout(r, 100)); return this.id; }")
+ """
+ # Get remote object ID for this element
+ object_id = await self._get_remote_object_id()
+ if not object_id:
+ raise RuntimeError('Element has no remote object ID (element may be detached from DOM)')
+
+ # Validate arrow function format (allow async prefix)
+ page_function = page_function.strip()
+ # Check for arrow function with optional async prefix
+ if not ('=>' in page_function and (page_function.startswith('(') or page_function.startswith('async'))):
+ raise ValueError(
+ f'JavaScript code must start with (...args) => or async (...args) => format. Got: {page_function[:50]}...'
+ )
+
+ # Convert arrow function to function declaration for CallFunctionOn
+ # CallFunctionOn expects 'function(...args) { ... }' format, not arrow functions
+ # We need to convert: '() => expression' to 'function() { return expression; }'
+ # or: '(x, y) => { statements }' to 'function(x, y) { statements }'
+
+ # Extract parameters and body from arrow function
+ import re
+
+ # Check if it's an async arrow function
+ is_async = page_function.strip().startswith('async')
+ async_prefix = 'async ' if is_async else ''
+
+ # Match: (params) => body or async (params) => body
+ # Strip 'async' prefix if present for parsing
+ func_to_parse = page_function.strip()
+ if is_async:
+ func_to_parse = func_to_parse[5:].strip() # Remove 'async' prefix
+
+ arrow_match = re.match(r'\s*\(([^)]*)\)\s*=>\s*(.+)', func_to_parse, re.DOTALL)
+ if not arrow_match:
+ raise ValueError(f'Could not parse arrow function: {page_function[:50]}...')
+
+ params_str = arrow_match.group(1).strip() # e.g., '', 'x', 'x, y'
+ body = arrow_match.group(2).strip()
+
+ # If body doesn't start with {, it's an expression that needs implicit return
+ if not body.startswith('{'):
+ function_declaration = f'{async_prefix}function({params_str}) {{ return {body}; }}'
+ else:
+ # Body already has braces, use as-is
+ function_declaration = f'{async_prefix}function({params_str}) {body}'
+
+ # Build CallArgument list for args if provided
+ call_arguments = []
+ if args:
+ from cdp_use.cdp.runtime.types import CallArgument
+
+ for arg in args:
+ # Convert Python values to CallArgument format
+ call_arguments.append(CallArgument(value=arg))
+
+ # Prepare CallFunctionOn parameters
+
+ params: 'CallFunctionOnParameters' = {
+ 'functionDeclaration': function_declaration,
+ 'objectId': object_id,
+ 'returnByValue': True,
+ 'awaitPromise': True,
+ }
+
+ if call_arguments:
+ params['arguments'] = call_arguments
+
+ # Execute the function on the element
+ result = await self._client.send.Runtime.callFunctionOn(
+ params,
+ session_id=self._session_id,
+ )
+
+ # Handle exceptions
+ if 'exceptionDetails' in result:
+ raise RuntimeError(f'JavaScript evaluation failed: {result["exceptionDetails"]}')
+
+ # Extract and return value
+ value = result.get('result', {}).get('value')
+
+ # Return string representation (matching Page.evaluate behavior)
+ if value is None:
+ return ''
+ elif isinstance(value, str):
+ return value
+ else:
+ # Convert objects, numbers, booleans to string
+ import json
+
+ try:
+ return json.dumps(value) if isinstance(value, (dict, list)) else str(value)
+ except (TypeError, ValueError):
+ return str(value)
+
# Helpers for modifiers etc
def _get_char_modifiers_and_vk(self, char: str) -> tuple[int, int, str]:
"""Get modifiers, virtual key code, and base key for a character.
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index c56a2e746..df61bf137 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -1329,6 +1329,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
cdp_url=urlparse(self.browser_session.cdp_url).hostname
if self.browser_session and self.browser_session.cdp_url
else None,
+ agent_type=None, # Regular Agent (not code-use)
action_errors=self.history.errors(),
action_history=action_history_data,
urls_visited=self.history.urls(),
@@ -1409,6 +1410,54 @@ class Agent(Generic[Context, AgentStructuredOutput]):
return None
+ async def _execute_step(
+ self,
+ step: int,
+ max_steps: int,
+ step_info: AgentStepInfo,
+ on_step_start: AgentHookFunc | None = None,
+ on_step_end: AgentHookFunc | None = None,
+ ) -> bool:
+ """
+ Execute a single step with timeout.
+
+ Returns:
+ bool: True if task is done, False otherwise
+ """
+ if on_step_start is not None:
+ await on_step_start(self)
+
+ self.logger.debug(f'🚶 Starting step {step + 1}/{max_steps}...')
+
+ try:
+ await asyncio.wait_for(
+ self.step(step_info),
+ timeout=180, # 3 minute timeout
+ )
+ self.logger.debug(f'✅ Completed step {step + 1}/{max_steps}')
+ except TimeoutError:
+ # Handle step timeout gracefully
+ error_msg = f'Step {step + 1} timed out after 180 seconds'
+ self.logger.error(f'⏰ {error_msg}')
+ self.state.consecutive_failures += 1
+ self.state.last_result = [ActionResult(error=error_msg)]
+
+ if on_step_end is not None:
+ await on_step_end(self)
+
+ if self.history.is_done():
+ await self.log_completion()
+
+ if self.register_done_callback:
+ if inspect.iscoroutinefunction(self.register_done_callback):
+ await self.register_done_callback(self.history)
+ else:
+ self.register_done_callback(self.history)
+
+ return True
+
+ return False
+
@observe(name='agent.run', metadata={'task': '{{task}}', 'debug': '{{debug}}'})
@time_execution_async('--run')
async def run(
@@ -1501,37 +1550,10 @@ class Agent(Generic[Context, AgentStructuredOutput]):
agent_run_error = 'Agent stopped programmatically'
break
- if on_step_start is not None:
- await on_step_start(self)
-
- self.logger.debug(f'🚶 Starting step {step + 1}/{max_steps}...')
step_info = AgentStepInfo(step_number=step, max_steps=max_steps)
+ is_done = await self._execute_step(step, max_steps, step_info, on_step_start, on_step_end)
- try:
- await asyncio.wait_for(
- self.step(step_info),
- timeout=self.settings.step_timeout,
- )
- self.logger.debug(f'✅ Completed step {step + 1}/{max_steps}')
- except TimeoutError:
- # Handle step timeout gracefully
- error_msg = f'Step {step + 1} timed out after {self.settings.step_timeout} seconds'
- self.logger.error(f'⏰ {error_msg}')
- self.state.consecutive_failures += 1
- self.state.last_result = [ActionResult(error=error_msg)]
-
- if on_step_end is not None:
- await on_step_end(self)
-
- if self.history.is_done():
- await self.log_completion()
-
- if self.register_done_callback:
- if inspect.iscoroutinefunction(self.register_done_callback):
- await self.register_done_callback(self.history)
- else:
- self.register_done_callback(self.history)
-
+ if is_done:
# Task completed
break
else:
diff --git a/browser_use/browser/views.py b/browser_use/browser/views.py
index f9d5b4529..bf4fe7fb4 100644
--- a/browser_use/browser/views.py
+++ b/browser_use/browser/views.py
@@ -65,6 +65,27 @@ class PageInfo(BaseModel):
# Page statistics are now computed dynamically instead of stored
+@dataclass
+class NetworkRequest:
+ """Information about a pending network request"""
+
+ url: str
+ method: str = 'GET'
+ loading_duration_ms: float = 0.0 # How long this request has been loading (ms since request started, max 10s)
+ resource_type: str | None = None # e.g., 'Document', 'Stylesheet', 'Image', 'Script', 'XHR', 'Fetch'
+
+
+@dataclass
+class PaginationButton:
+ """Information about a pagination button detected on the page"""
+
+ button_type: str # 'next', 'prev', 'first', 'last', 'page_number'
+ element_index: int # Interactive element index for clicking
+ text: str # Button text/label
+ selector: str # XPath or other selector to locate the element
+ is_disabled: bool = False # Whether the button appears disabled
+
+
@dataclass
class BrowserStateSummary:
"""The summary of the browser's current state designed for an LLM to process"""
@@ -84,6 +105,8 @@ class BrowserStateSummary:
browser_errors: list[str] = field(default_factory=list)
is_pdf_viewer: bool = False # Whether the current page is a PDF viewer
recent_events: str | None = None # Text summary of recent browser events
+ pending_network_requests: list[NetworkRequest] = field(default_factory=list) # Currently loading network requests
+ pagination_buttons: list[PaginationButton] = field(default_factory=list) # Detected pagination buttons
@dataclass
diff --git a/browser_use/browser/watchdogs/default_action_watchdog.py b/browser_use/browser/watchdogs/default_action_watchdog.py
index 6a0eb2cbb..f2e7b3b7d 100644
--- a/browser_use/browser/watchdogs/default_action_watchdog.py
+++ b/browser_use/browser/watchdogs/default_action_watchdog.py
@@ -563,9 +563,17 @@ class DefaultActionWatchdog(BaseWatchdog):
if element_node.element_index:
element_info += f' index={element_node.element_index}'
element_info += '>'
+
+ # Create helpful error message based on context
+ error_detail = f'Failed to click element {element_info}. The element may not be interactable or visible.'
+
+ # Add hint if element has index (common in code-use mode)
+ if element_node.element_index:
+ error_detail += f' If the page changed after navigation/interaction, the index [{element_node.element_index}] may be stale. Get fresh browser state before retrying.'
+
raise BrowserError(
message=f'Failed to click element: {e}',
- long_term_memory=f'Failed to click element {element_info}. The element may not be interactable or visible.',
+ long_term_memory=error_detail,
)
async def _type_to_page(self, text: str):
diff --git a/browser_use/browser/watchdogs/dom_watchdog.py b/browser_use/browser/watchdogs/dom_watchdog.py
index 4c6bbca1c..370e184ea 100644
--- a/browser_use/browser/watchdogs/dom_watchdog.py
+++ b/browser_use/browser/watchdogs/dom_watchdog.py
@@ -20,7 +20,7 @@ from browser_use.observability import observe_debug
from browser_use.utils import time_execution_async
if TYPE_CHECKING:
- from browser_use.browser.views import BrowserStateSummary, PageInfo
+ from browser_use.browser.views import BrowserStateSummary, NetworkRequest, PageInfo, PaginationButton
class DOMWatchdog(BaseWatchdog):
@@ -42,6 +42,9 @@ class DOMWatchdog(BaseWatchdog):
# Internal DOM service
_dom_service: DomService | None = None
+ # Network tracking - maps request_id to (url, start_time, method, resource_type)
+ _pending_requests: dict[str, tuple[str, float, str, str | None]] = {}
+
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
# self.logger.debug('Setting up init scripts in browser')
return None
@@ -85,6 +88,158 @@ class DOMWatchdog(BaseWatchdog):
return json.dumps([]) # Return empty JSON array on error
+ async def _get_pending_network_requests(self) -> list['NetworkRequest']:
+ """Get list of currently pending network requests.
+
+ Uses document.readyState and performance API to detect pending requests.
+ Filters out ads, tracking, and other noise.
+
+ Returns:
+ List of NetworkRequest objects representing currently loading resources
+ """
+ from browser_use.browser.views import NetworkRequest
+
+ try:
+ if not self.browser_session.agent_focus:
+ return []
+
+ cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True)
+
+ # Use performance API to get pending requests
+ js_code = """
+(function() {
+ const now = performance.now();
+ const resources = performance.getEntriesByType('resource');
+ const pending = [];
+
+ // Check document readyState
+ const docLoading = document.readyState !== 'complete';
+
+ // Common ad/tracking domains and patterns to filter out
+ const adDomains = [
+ // Standard ad/tracking networks
+ 'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',
+ 'facebook.net', 'analytics', 'ads', 'tracking', 'pixel',
+ 'hotjar.com', 'clarity.ms', 'mixpanel.com', 'segment.com',
+ // Analytics platforms
+ 'demdex.net', 'omtrdc.net', 'adobedtm.com', 'ensighten.com',
+ 'newrelic.com', 'nr-data.net', 'google-analytics.com',
+ // Social media trackers
+ 'connect.facebook.net', 'platform.twitter.com', 'platform.linkedin.com',
+ // CDN/image hosts (usually not critical for functionality)
+ '.cloudfront.net/image/', '.akamaized.net/image/',
+ // Common tracking paths
+ '/tracker/', '/collector/', '/beacon/', '/telemetry/', '/log/',
+ '/events/', '/eventBatch', '/track.', '/metrics/'
+ ];
+
+ // Get resources that are still loading (responseEnd is 0)
+ let totalResourcesChecked = 0;
+ let filteredByResponseEnd = 0;
+ const allDomains = new Set();
+
+ for (const entry of resources) {
+ totalResourcesChecked++;
+
+ // Track all domains from recent resources (for logging)
+ try {
+ const hostname = new URL(entry.name).hostname;
+ if (hostname) allDomains.add(hostname);
+ } catch (e) {}
+
+ if (entry.responseEnd === 0) {
+ filteredByResponseEnd++;
+ const url = entry.name;
+
+ // Filter out ads and tracking
+ const isAd = adDomains.some(domain => url.includes(domain));
+ if (isAd) continue;
+
+ // Filter out data: URLs and very long URLs (often inline resources)
+ if (url.startsWith('data:') || url.length > 500) continue;
+
+ const loadingDuration = now - entry.startTime;
+
+ // Skip requests that have been loading for >10 seconds (likely stuck/polling)
+ if (loadingDuration > 100000) continue;
+
+ const resourceType = entry.initiatorType || 'unknown';
+
+ // Filter out non-critical resources (images, fonts, icons) if loading >3 seconds
+ const nonCriticalTypes = ['img', 'image', 'icon', 'font'];
+ if (nonCriticalTypes.includes(resourceType) && loadingDuration > 3000) continue;
+
+ // Filter out image URLs even if type is unknown
+ const isImageUrl = /\\.(jpg|jpeg|png|gif|webp|svg|ico)(\\?|$)/i.test(url);
+ if (isImageUrl && loadingDuration > 3000) continue;
+
+ pending.push({
+ url: url,
+ method: 'GET',
+ loading_duration_ms: Math.round(loadingDuration),
+ resource_type: resourceType
+ });
+ }
+ }
+
+ return {
+ pending_requests: pending,
+ document_loading: docLoading,
+ document_ready_state: document.readyState,
+ debug: {
+ total_resources: totalResourcesChecked,
+ with_response_end_zero: filteredByResponseEnd,
+ after_all_filters: pending.length,
+ all_domains: Array.from(allDomains)
+ }
+ };
+})()
+"""
+
+ result = await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={'expression': js_code, 'returnByValue': True}, session_id=cdp_session.session_id
+ )
+
+ if result.get('result', {}).get('type') == 'object':
+ data = result['result'].get('value', {})
+ pending = data.get('pending_requests', [])
+ doc_state = data.get('document_ready_state', 'unknown')
+ doc_loading = data.get('document_loading', False)
+ debug_info = data.get('debug', {})
+
+ # Get all domains that had recent activity (from JS)
+ all_domains = debug_info.get('all_domains', [])
+ all_domains_str = ', '.join(sorted(all_domains)[:5]) if all_domains else 'none'
+ if len(all_domains) > 5:
+ all_domains_str += f' +{len(all_domains) - 5} more'
+
+ # Debug logging
+ self.logger.debug(
+ f'🔍 Network check: document.readyState={doc_state}, loading={doc_loading}, '
+ f'total_resources={debug_info.get("total_resources", 0)}, '
+ f'responseEnd=0: {debug_info.get("with_response_end_zero", 0)}, '
+ f'after_filters={len(pending)}, domains=[{all_domains_str}]'
+ )
+
+ # Convert to NetworkRequest objects
+ network_requests = []
+ for req in pending[:20]: # Limit to 20 to avoid overwhelming the context
+ network_requests.append(
+ NetworkRequest(
+ url=req['url'],
+ method=req.get('method', 'GET'),
+ loading_duration_ms=req.get('loading_duration_ms', 0.0),
+ resource_type=req.get('resource_type'),
+ )
+ )
+
+ return network_requests
+
+ except Exception as e:
+ self.logger.debug(f'Failed to get pending network requests: {e}')
+
+ return []
+
@observe_debug(ignore_input=True, ignore_output=True, name='browser_state_request_event')
async def on_BrowserStateRequestEvent(self, event: BrowserStateRequestEvent) -> 'BrowserStateSummary':
"""Handle browser state request by coordinating DOM building and screenshot capture.
@@ -112,11 +267,22 @@ class DOMWatchdog(BaseWatchdog):
# check if we should skip DOM tree build for pointless pages
not_a_meaningful_website = page_url.lower().split(':', 1)[0] not in ('http', 'https')
+ # Check for pending network requests BEFORE waiting (so we can see what's loading)
+ pending_requests_before_wait = []
+ if not not_a_meaningful_website:
+ try:
+ pending_requests_before_wait = await self._get_pending_network_requests()
+ if pending_requests_before_wait:
+ self.logger.debug(f'🔍 Found {len(pending_requests_before_wait)} pending requests before stability wait')
+ except Exception as e:
+ self.logger.debug(f'Failed to get pending requests before wait: {e}')
+ pending_requests = pending_requests_before_wait
# Wait for page stability using browser profile settings (main branch pattern)
if not not_a_meaningful_website:
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ⏳ Waiting for page stability...')
try:
- await self._wait_for_stable_network()
+ if pending_requests_before_wait:
+ await asyncio.sleep(1)
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Page stability complete')
except Exception as e:
self.logger.warning(
@@ -181,6 +347,8 @@ class DOMWatchdog(BaseWatchdog):
browser_errors=[],
is_pdf_viewer=False,
recent_events=self._get_recent_events_str() if event.include_recent_events else None,
+ pending_network_requests=[], # Empty page has no pending requests
+ pagination_buttons=[], # Empty page has no pagination
)
# Execute DOM building and screenshot capture in parallel
@@ -227,7 +395,14 @@ class DOMWatchdog(BaseWatchdog):
screenshot_b64 = None
# Apply Python-based highlighting if both DOM and screenshot are available
- if screenshot_b64 and content and content.selector_map and self.browser_session.browser_profile.highlight_elements:
+ # COMMENTED OUT: Removes highlight numbers from screenshots for code-use mode
+ if (
+ False
+ and screenshot_b64
+ and content
+ and content.selector_map
+ and self.browser_session.browser_profile.highlight_elements
+ ):
try:
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🎨 Applying Python-based highlighting...')
from browser_use.browser.python_highlights import create_highlighted_screenshot_async
@@ -300,6 +475,11 @@ class DOMWatchdog(BaseWatchdog):
# Check for PDF viewer
is_pdf_viewer = page_url.endswith('.pdf') or '/pdf/' in page_url
+ # Detect pagination buttons from the DOM
+ pagination_buttons_data = []
+ if content and content.selector_map:
+ pagination_buttons_data = self._detect_pagination_buttons(content.selector_map)
+
# Build and cache the browser state summary
if screenshot_b64:
self.logger.debug(
@@ -322,6 +502,8 @@ class DOMWatchdog(BaseWatchdog):
browser_errors=[],
is_pdf_viewer=is_pdf_viewer,
recent_events=self._get_recent_events_str() if event.include_recent_events else None,
+ pending_network_requests=pending_requests,
+ pagination_buttons=pagination_buttons_data,
)
# Cache the state
@@ -357,6 +539,8 @@ class DOMWatchdog(BaseWatchdog):
browser_errors=[str(e)],
is_pdf_viewer=False,
recent_events=None,
+ pending_network_requests=[], # Error state has no pending requests
+ pagination_buttons=[], # Error state has no pagination
)
@time_execution_async('build_dom_tree_without_highlights')
@@ -470,6 +654,41 @@ class DOMWatchdog(BaseWatchdog):
elapsed = time.time() - start_time
self.logger.debug(f'✅ Page stability wait completed in {elapsed:.2f}s')
+ def _detect_pagination_buttons(self, selector_map: dict[int, EnhancedDOMTreeNode]) -> list['PaginationButton']:
+ """Detect pagination buttons from the DOM selector map.
+
+ Args:
+ selector_map: Dictionary mapping element indices to DOM tree nodes
+
+ Returns:
+ List of PaginationButton instances found in the DOM
+ """
+ from browser_use.browser.views import PaginationButton
+
+ pagination_buttons_data = []
+ try:
+ self.logger.debug('🔍 DOMWatchdog._detect_pagination_buttons: Detecting pagination buttons...')
+ pagination_buttons_raw = DomService.detect_pagination_buttons(selector_map)
+ # Convert to PaginationButton instances
+ pagination_buttons_data = [
+ PaginationButton(
+ button_type=btn['button_type'], # type: ignore
+ element_index=btn['element_index'], # type: ignore
+ text=btn['text'], # type: ignore
+ selector=btn['selector'], # type: ignore
+ is_disabled=btn['is_disabled'], # type: ignore
+ )
+ for btn in pagination_buttons_raw
+ ]
+ if pagination_buttons_data:
+ self.logger.debug(
+ f'🔍 DOMWatchdog._detect_pagination_buttons: Found {len(pagination_buttons_data)} pagination buttons'
+ )
+ except Exception as e:
+ self.logger.warning(f'🔍 DOMWatchdog._detect_pagination_buttons: Pagination detection failed: {e}')
+
+ return pagination_buttons_data
+
async def _get_page_info(self) -> 'PageInfo':
"""Get comprehensive page information using a single CDP call.
diff --git a/browser_use/code_use/README.md b/browser_use/code_use/README.md
new file mode 100644
index 000000000..862c3fbe8
--- /dev/null
+++ b/browser_use/code_use/README.md
@@ -0,0 +1,84 @@
+# Code-Use Mode
+
+Code-Use Mode is a Notebook-like code execution system for browser automation. Instead of the agent choosing from a predefined set of actions, the LLM writes Python code that gets executed in a persistent namespace with all browser control functions available.
+
+## Problem Solved
+
+**Code-Use Mode solves this** by giving the agent a Python execution environment where it can:
+- Store extracted data in variables
+- Loop through pages programmatically
+- Combine results from multiple extractions
+- Process and filter data before saving
+- Use conditional logic to decide what to do next
+- Output more tokens than the LLM writes
+
+### Namespace
+The namespace is initialized with:
+
+**Browser Control Functions:**
+- `navigate(url)` - Navigate to a URL
+- `click(index)` - Click an element
+- `input(index, text)` - Type text
+- `scroll(down, pages)` - Scroll the page
+- `upload_file(path)` - Upload a file
+- `evaluate(code, variables={})` - Execute JavaScript
+- `done(text, success, files_to_display=[])` - Mark task complete
+
+**Custom evaluate() Function:**
+```python
+# Returns values directly, not wrapped in ActionResult
+result = await evaluate('''
+(function(){
+ return Array.from(document.querySelectorAll('.product')).map(p => ({
+ name: p.querySelector('.name').textContent,
+ price: p.querySelector('.price').textContent
+ }))
+})()
+''')
+# result is now a list of dicts, ready to use!
+```
+
+**Utilities:**
+The agent can just utilize packages like `requests`, `pandas`, `numpy`, `matplotlib`, `BeautifulSoup`, `tabulate`, `csv`, ...
+
+The agent will write code like:
+
+### Step 1: Navigate
+```python
+# Navigate to first page
+await navigate(url='https://example.com/products?page=1')
+```
+### Step 2 analyse our DOM state and write code to extract the data we need.
+
+```js extract_products
+(function(){
+ return Array.from(document.querySelectorAll('.product')).map(p => ({
+ name: p.querySelector('.name')?.textContent || '',
+ price: p.querySelector('.price')?.textContent || '',
+ rating: p.querySelector('.rating')?.textContent || ''
+ }))
+})()
+```
+
+```python
+# Extract products using JavaScript
+all_products = []
+for page in range(1, 6):
+ if page > 1:
+ await navigate(url=f'https://example.com/products?page={page}')
+
+ products = await evaluate(extract_products)
+ all_products.extend(products)
+ print(f'Page {page}: Found {len(products)} products')
+```
+
+### Step 3: Analyse output & save the data to a file
+```python
+# Save to file
+import json
+with open('products.json', 'w') as f:
+ json.dump(all_products, f, indent=2)
+
+print(f'Total: {len(all_products)} products saved to products.json')
+await done(text='Extracted all products', success=True, files_to_display=['products.json'])
+```
diff --git a/browser_use/code_use/__init__.py b/browser_use/code_use/__init__.py
new file mode 100644
index 000000000..9fd858dec
--- /dev/null
+++ b/browser_use/code_use/__init__.py
@@ -0,0 +1,16 @@
+"""Code-use mode - Jupiter notebook-like code execution for browser automation."""
+
+from browser_use.code_use.namespace import create_namespace
+from browser_use.code_use.notebook_export import export_to_ipynb, session_to_python_script
+from browser_use.code_use.service import CodeAgent
+from browser_use.code_use.views import CodeCell, ExecutionStatus, NotebookSession
+
+__all__ = [
+ 'CodeAgent',
+ 'create_namespace',
+ 'export_to_ipynb',
+ 'session_to_python_script',
+ 'CodeCell',
+ 'ExecutionStatus',
+ 'NotebookSession',
+]
diff --git a/browser_use/code_use/formatting.py b/browser_use/code_use/formatting.py
new file mode 100644
index 000000000..8b59ca721
--- /dev/null
+++ b/browser_use/code_use/formatting.py
@@ -0,0 +1,192 @@
+"""Browser state formatting helpers for code-use agent."""
+
+import logging
+from typing import Any
+
+from browser_use.browser.session import BrowserSession
+from browser_use.browser.views import BrowserStateSummary
+
+logger = logging.getLogger(__name__)
+
+
+async def format_browser_state_for_llm(
+ state: BrowserStateSummary,
+ namespace: dict[str, Any],
+ browser_session: BrowserSession,
+) -> str:
+ """
+ Format browser state summary for LLM consumption in code-use mode.
+
+ Args:
+ state: Browser state summary from browser_session.get_browser_state_summary()
+ namespace: The code execution namespace (for showing available variables)
+ browser_session: Browser session for additional checks (jQuery, etc.)
+
+ Returns:
+ Formatted browser state text for LLM
+ """
+ assert state.dom_state is not None
+ dom_state = state.dom_state
+
+ # Use eval_representation (compact serializer for code agents)
+ dom_html = dom_state.eval_representation()
+ if dom_html == '':
+ dom_html = 'Empty DOM tree (you might have to wait for the page to load)'
+
+ # Format with URL and title header
+ lines = ['## Browser State']
+ lines.append(f'**URL:** {state.url}')
+ lines.append(f'**Title:** {state.title}')
+ lines.append('')
+
+ # Add tabs info if multiple tabs exist
+ if len(state.tabs) > 1:
+ lines.append('**Tabs:**')
+ current_target_candidates = []
+ # Find tabs that match current URL and title
+ for tab in state.tabs:
+ if tab.url == state.url and tab.title == state.title:
+ current_target_candidates.append(tab.target_id)
+ current_target_id = current_target_candidates[0] if len(current_target_candidates) == 1 else None
+
+ for tab in state.tabs:
+ is_current = ' (current)' if tab.target_id == current_target_id else ''
+ lines.append(f' - Tab {tab.target_id[-4:]}: {tab.url} - {tab.title[:30]}{is_current}')
+ lines.append('')
+
+ # Add page scroll info if available
+ if state.page_info:
+ pi = state.page_info
+ pages_above = pi.pixels_above / pi.viewport_height if pi.viewport_height > 0 else 0
+ pages_below = pi.pixels_below / pi.viewport_height if pi.viewport_height > 0 else 0
+ total_pages = pi.page_height / pi.viewport_height if pi.viewport_height > 0 else 0
+
+ scroll_info = f'**Page:** {pages_above:.1f} pages above, {pages_below:.1f} pages below'
+ if total_pages > 1.2: # Only mention total if significantly > 1 page
+ scroll_info += f', {total_pages:.1f} total pages'
+ lines.append(scroll_info)
+ lines.append('')
+
+ # Add network loading info if there are pending requests
+ if state.pending_network_requests:
+ # Remove duplicates by URL (keep first occurrence with earliest duration)
+ seen_urls = set()
+ unique_requests = []
+ for req in state.pending_network_requests:
+ if req.url not in seen_urls:
+ seen_urls.add(req.url)
+ unique_requests.append(req)
+
+ lines.append(f'**⏳ Loading:** {len(unique_requests)} network requests still loading')
+ # Show up to 20 unique requests with truncated URLs (30 chars max)
+ for req in unique_requests[:20]:
+ duration_sec = req.loading_duration_ms / 1000
+ url_display = req.url if len(req.url) <= 30 else req.url[:27] + '...'
+ logger.info(f' - [{duration_sec:.1f}s] {url_display}')
+ lines.append(f' - [{duration_sec:.1f}s] {url_display}')
+ if len(unique_requests) > 20:
+ lines.append(f' - ... and {len(unique_requests) - 20} more')
+ lines.append('**Tip:** Content may still be loading. Consider waiting with `await asyncio.sleep(1)` if data is missing.')
+ lines.append('')
+
+ # Add available variables and functions BEFORE DOM structure
+ # Show useful utilities (json, asyncio, etc.) and user-defined vars, but hide system objects
+ skip_vars = {
+ 'browser',
+ 'file_system', # System objects
+ 'np',
+ 'pd',
+ 'plt',
+ 'numpy',
+ 'pandas',
+ 'matplotlib',
+ 'requests',
+ 'BeautifulSoup',
+ 'bs4',
+ 'pypdf',
+ 'PdfReader',
+ 'wait',
+ }
+
+ # Highlight code block variables separately from regular variables
+ code_block_vars = []
+ regular_vars = []
+ tracked_code_blocks = namespace.get('_code_block_vars', set())
+ for name in namespace.keys():
+ # Skip private vars and system objects/actions
+ if not name.startswith('_') and name not in skip_vars:
+ if name in tracked_code_blocks:
+ code_block_vars.append(name)
+ else:
+ regular_vars.append(name)
+
+ # Sort for consistent display
+ available_vars_sorted = sorted(regular_vars)
+ code_block_vars_sorted = sorted(code_block_vars)
+
+ # Build available line with code blocks and variables
+ parts = []
+ if code_block_vars_sorted:
+ # Show detailed info for code block variables
+ code_block_details = []
+ for var_name in code_block_vars_sorted:
+ value = namespace.get(var_name)
+ if value is not None:
+ type_name = type(value).__name__
+ value_str = str(value) if not isinstance(value, str) else value
+
+ # Check if it's a function (starts with "(function" or "(async function")
+ is_function = value_str.strip().startswith('(function') or value_str.strip().startswith('(async function')
+
+ if is_function:
+ # For functions, only show name and type
+ detail = f'{var_name}({type_name})'
+ else:
+ # For non-functions, show first and last 20 chars
+ first_20 = value_str[:20].replace('\n', '\\n').replace('\t', '\\t')
+ last_20 = value_str[-20:].replace('\n', '\\n').replace('\t', '\\t') if len(value_str) > 20 else ''
+
+ if last_20 and first_20 != last_20:
+ detail = f'{var_name}({type_name}): "{first_20}...{last_20}"'
+ else:
+ detail = f'{var_name}({type_name}): "{first_20}"'
+ code_block_details.append(detail)
+
+ parts.append(f'**Code block variables:** {" | ".join(code_block_details)}')
+ if available_vars_sorted:
+ parts.append(f'**Variables:** {", ".join(available_vars_sorted)}')
+
+ lines.append(f'**Available:** {" | ".join(parts)}')
+ lines.append('')
+
+ # Add DOM structure
+ lines.append('**DOM Structure:**')
+
+ # Add scroll position hints for DOM
+ if state.page_info:
+ pi = state.page_info
+ pages_above = pi.pixels_above / pi.viewport_height if pi.viewport_height > 0 else 0
+ pages_below = pi.pixels_below / pi.viewport_height if pi.viewport_height > 0 else 0
+
+ if pages_above > 0:
+ dom_html = f'... {pages_above:.1f} pages above \n{dom_html}'
+ else:
+ dom_html = '[Start of page]\n' + dom_html
+
+ if pages_below > 0:
+ dom_html += f'\n... {pages_below:.1f} pages below '
+ else:
+ dom_html += '\n[End of page]'
+
+ # Truncate DOM if too long and notify LLM
+ max_dom_length = 60000
+ if len(dom_html) > max_dom_length:
+ lines.append(dom_html[:max_dom_length])
+ lines.append(
+ f'\n[DOM truncated after {max_dom_length} characters. Full page contains {len(dom_html)} characters total. Use evaluate to explore more.]'
+ )
+ else:
+ lines.append(dom_html)
+
+ browser_state_text = '\n'.join(lines)
+ return browser_state_text
diff --git a/browser_use/code_use/namespace.py b/browser_use/code_use/namespace.py
new file mode 100644
index 000000000..c054fa327
--- /dev/null
+++ b/browser_use/code_use/namespace.py
@@ -0,0 +1,648 @@
+"""Namespace initialization for code-use mode.
+
+This module creates a namespace with all browser tools available as functions,
+similar to a Jupiter notebook environment.
+"""
+
+import asyncio
+import csv
+import datetime
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+import requests
+
+from browser_use.browser import BrowserSession
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.llm.base import BaseChatModel
+from browser_use.tools.service import CodeAgentTools, Tools
+
+logger = logging.getLogger(__name__)
+
+# Try to import optional data science libraries
+try:
+ import numpy as np
+
+ NUMPY_AVAILABLE = True
+except ImportError:
+ NUMPY_AVAILABLE = False
+
+try:
+ import pandas as pd
+
+ PANDAS_AVAILABLE = True
+except ImportError:
+ PANDAS_AVAILABLE = False
+
+try:
+ import matplotlib.pyplot as plt
+
+ MATPLOTLIB_AVAILABLE = True
+except ImportError:
+ MATPLOTLIB_AVAILABLE = False
+
+try:
+ from bs4 import BeautifulSoup
+
+ BS4_AVAILABLE = True
+except ImportError:
+ BS4_AVAILABLE = False
+
+try:
+ from pypdf import PdfReader
+
+ PYPDF_AVAILABLE = True
+except ImportError:
+ PYPDF_AVAILABLE = False
+
+try:
+ from tabulate import tabulate
+
+ TABULATE_AVAILABLE = True
+except ImportError:
+ TABULATE_AVAILABLE = False
+
+
+def _strip_js_comments(js_code: str) -> str:
+ """
+ Remove JavaScript comments before CDP evaluation.
+ CDP's Runtime.evaluate doesn't handle comments in all contexts.
+
+ Args:
+ js_code: JavaScript code potentially containing comments
+
+ Returns:
+ JavaScript code with comments stripped
+ """
+ # Remove multi-line comments (/* ... */)
+ js_code = re.sub(r'/\*.*?\*/', '', js_code, flags=re.DOTALL)
+
+ # Remove single-line comments - only lines that START with // (after whitespace)
+ # This avoids breaking XPath strings, URLs, regex patterns, etc.
+ js_code = re.sub(r'^\s*//.*$', '', js_code, flags=re.MULTILINE)
+
+ return js_code
+
+
+class EvaluateError(Exception):
+ """Special exception raised by evaluate() to stop Python execution immediately."""
+
+ pass
+
+
+async def validate_task_completion(
+ task: str,
+ output: str | None,
+ llm: BaseChatModel,
+) -> tuple[bool, str]:
+ """
+ Validate if task is truly complete by asking LLM without system prompt or history.
+
+ Args:
+ task: The original task description
+ output: The output from the done() call
+ llm: The LLM to use for validation
+
+ Returns:
+ Tuple of (is_complete, reasoning)
+ """
+ from browser_use.llm.messages import UserMessage
+
+ # Build validation prompt
+ validation_prompt = f"""You are a task completion validator. Analyze if the agent has truly completed the user's task.
+
+**Original Task:**
+{task}
+
+**Agent's Output:**
+{output[:100000] if output else '(No output provided)'}
+
+**Your Task:**
+Determine if the agent has successfully completed the user's task. Consider:
+1. Has the agent delivered what the user requested?
+2. If data extraction was requested, is there actual data?
+3. If the task is impossible (e.g., localhost website, login required but no credentials), is it truly impossible?
+4. Could the agent continue and make meaningful progress?
+
+**Response Format:**
+Reasoning: [Your analysis of whether the task is complete]
+Verdict: [YES or NO]
+
+YES = Task is complete OR truly impossible to complete
+NO = Agent should continue working"""
+
+ try:
+ # Call LLM with just the validation prompt (no system prompt, no history)
+ response = await llm.ainvoke([UserMessage(content=validation_prompt)])
+ response_text = response.completion
+
+ # Parse the response
+ reasoning = ''
+ verdict = 'NO'
+
+ # Extract reasoning and verdict
+ lines = response_text.split('\n')
+ for line in lines:
+ if line.strip().lower().startswith('reasoning:'):
+ reasoning = line.split(':', 1)[1].strip()
+ elif line.strip().lower().startswith('verdict:'):
+ verdict_text = line.split(':', 1)[1].strip().upper()
+ if 'YES' in verdict_text:
+ verdict = 'YES'
+ elif 'NO' in verdict_text:
+ verdict = 'NO'
+
+ # If we couldn't parse, try to find YES/NO in the response
+ if not reasoning:
+ reasoning = response_text
+
+ is_complete = verdict == 'YES'
+
+ logger.info(f'Task validation: {verdict}')
+ logger.debug(f'Validation reasoning: {reasoning}')
+
+ return is_complete, reasoning
+
+ except Exception as e:
+ logger.warning(f'Failed to validate task completion: {e}')
+ # On error, assume the agent knows what they're doing
+ return True, f'Validation failed: {e}'
+
+
+async def evaluate(code: str, browser_session: BrowserSession) -> Any:
+ """
+ Execute JavaScript code in the browser and return the result.
+
+ Args:
+ code: JavaScript code to execute (must be wrapped in IIFE)
+
+ Returns:
+ The result of the JavaScript execution
+
+ Raises:
+ EvaluateError: If JavaScript execution fails. This stops Python execution immediately.
+
+ Example:
+ result = await evaluate('''
+ (function(){
+ return Array.from(document.querySelectorAll('.product')).map(p => ({
+ name: p.querySelector('.name').textContent,
+ price: p.querySelector('.price').textContent
+ }))
+ })()
+ ''')
+ """
+ # Strip JavaScript comments before CDP evaluation (CDP doesn't support them in all contexts)
+ code = _strip_js_comments(code)
+
+ cdp_session = await browser_session.get_or_create_cdp_session()
+
+ try:
+ # Execute JavaScript with proper error handling
+ result = await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={'expression': code, 'returnByValue': True, 'awaitPromise': True},
+ session_id=cdp_session.session_id,
+ )
+
+ # Check for JavaScript execution errors
+ if result.get('exceptionDetails'):
+ exception = result['exceptionDetails']
+ error_text = exception.get('text', 'Unknown error')
+
+ # Try to get more details from the exception
+ error_details = []
+ if 'exception' in exception:
+ exc_obj = exception['exception']
+ if 'description' in exc_obj:
+ error_details.append(exc_obj['description'])
+ elif 'value' in exc_obj:
+ error_details.append(str(exc_obj['value']))
+
+ # Build comprehensive error message with full CDP context
+ error_msg = f'JavaScript execution error: {error_text}'
+ if error_details:
+ error_msg += f'\nDetails: {" | ".join(error_details)}'
+
+ # Raise special exception that will stop Python execution immediately
+ raise EvaluateError(error_msg)
+
+ # Get the result data
+ result_data = result.get('result', {})
+
+ # Get the actual value
+ value = result_data.get('value')
+
+ # Return the value directly
+ if value is None:
+ return None if 'value' in result_data else 'undefined'
+ elif isinstance(value, (dict, list)):
+ # Complex objects - already deserialized by returnByValue
+ return value
+ else:
+ # Primitive values
+ return value
+
+ except EvaluateError:
+ # Re-raise EvaluateError as-is to stop Python execution
+ raise
+ except Exception as e:
+ # Wrap other exceptions in EvaluateError
+ raise EvaluateError(f'Failed to execute JavaScript: {type(e).__name__}: {e}') from e
+
+
+def create_namespace(
+ browser_session: BrowserSession,
+ tools: Tools | None = None,
+ page_extraction_llm: BaseChatModel | None = None,
+ file_system: FileSystem | None = None,
+ available_file_paths: list[str] | None = None,
+ sensitive_data: dict[str, str | dict[str, str]] | None = None,
+) -> dict[str, Any]:
+ """
+ Create a namespace with all browser tools available as functions.
+
+ This function creates a dictionary of functions that can be used to interact
+ with the browser, similar to a Jupiter notebook environment.
+
+ Args:
+ browser_session: The browser session to use
+ tools: Optional Tools instance (will create default if not provided)
+ page_extraction_llm: Optional LLM for page extraction
+ file_system: Optional file system for file operations
+ available_file_paths: Optional list of available file paths
+ sensitive_data: Optional sensitive data dictionary
+
+ Returns:
+ Dictionary containing all available functions and objects
+
+ Example:
+ namespace = create_namespace(browser_session)
+ await namespace['navigate'](url='https://google.com')
+ result = await namespace['evaluate']('document.title')
+ """
+ if tools is None:
+ # Use CodeAgentTools with default exclusions optimized for code-use mode
+ # For code-use, we keep: navigate, evaluate, wait, done
+ # and exclude: most browser interaction, file system actions (use Python instead)
+ tools = CodeAgentTools()
+
+ if available_file_paths is None:
+ available_file_paths = []
+
+ namespace: dict[str, Any] = {
+ # Core objects
+ 'browser': browser_session,
+ 'file_system': file_system,
+ # Standard library modules (always available)
+ 'json': json,
+ 'asyncio': asyncio,
+ 'Path': Path,
+ 'csv': csv,
+ 're': re,
+ 'datetime': datetime,
+ 'requests': requests,
+ }
+
+ # Add optional data science libraries if available
+ if NUMPY_AVAILABLE:
+ namespace['np'] = np
+ namespace['numpy'] = np
+ if PANDAS_AVAILABLE:
+ namespace['pd'] = pd
+ namespace['pandas'] = pd
+ if MATPLOTLIB_AVAILABLE:
+ namespace['plt'] = plt
+ namespace['matplotlib'] = plt
+ if BS4_AVAILABLE:
+ namespace['BeautifulSoup'] = BeautifulSoup
+ namespace['bs4'] = BeautifulSoup
+ if PYPDF_AVAILABLE:
+ namespace['PdfReader'] = PdfReader
+ namespace['pypdf'] = PdfReader
+ if TABULATE_AVAILABLE:
+ namespace['tabulate'] = tabulate
+
+ # Track failed evaluate() calls to detect repeated failed approaches
+ if '_evaluate_failures' not in namespace:
+ namespace['_evaluate_failures'] = []
+
+ # Add custom evaluate function that returns values directly
+ async def evaluate_wrapper(
+ code: str | None = None, variables: dict[str, Any] | None = None, *_args: Any, **kwargs: Any
+ ) -> Any:
+ # Handle both positional and keyword argument styles
+ if code is None:
+ # Check if code was passed as keyword arg
+ code = kwargs.get('code', kwargs.get('js_code', kwargs.get('expression', '')))
+ # Extract variables if passed as kwarg
+ if variables is None:
+ variables = kwargs.get('variables')
+
+ if not code:
+ raise ValueError('No JavaScript code provided to evaluate()')
+
+ # Inject variables if provided
+ if variables:
+ vars_json = json.dumps(variables)
+ stripped = code.strip()
+
+ # Check if code is already a function expression expecting params
+ # Pattern: (function(params) { ... }) or (async function(params) { ... })
+ if re.match(r'\((?:async\s+)?function\s*\(\s*\w+\s*\)', stripped):
+ # Already expects params, wrap to call it with our variables
+ code = f'(function(){{ const params = {vars_json}; return {stripped}(params); }})()'
+ else:
+ # Not a parameterized function, inject params in scope
+ # Check if already wrapped in IIFE
+ is_wrapped = (stripped.startswith('(function()') and '})()' in stripped[-10:]) or (
+ stripped.startswith('(async function()') and '})()' in stripped[-10:]
+ )
+ if is_wrapped:
+ # Already wrapped, inject params at the start
+ # Find the opening brace of the function body
+ match = re.match(r'(\((?:async\s+)?function\s*\(\s*\)\s*\{)', stripped)
+ if match:
+ prefix = match.group(1)
+ rest = stripped[len(prefix) :]
+ code = f'{prefix} const params = {vars_json}; {rest}'
+ else:
+ # Fallback: wrap in outer function
+ code = f'(function(){{ const params = {vars_json}; return {stripped}; }})()'
+ else:
+ # Not wrapped, wrap with params
+ code = f'(function(){{ const params = {vars_json}; {code} }})()'
+ # Skip auto-wrap below
+ return await evaluate(code, browser_session)
+
+ # Auto-wrap in IIFE if not already wrapped (and no variables were injected)
+ if not variables:
+ stripped = code.strip()
+ is_wrapped = (stripped.startswith('(function()') and '})()' in stripped[-10:]) or (
+ stripped.startswith('(async function()') and '})()' in stripped[-10:]
+ )
+ if not is_wrapped:
+ code = f'(function(){{{code}}})()'
+
+ # Execute and track failures
+ try:
+ result = await evaluate(code, browser_session)
+
+ # Print result structure for debugging
+ if isinstance(result, list) and result and isinstance(result[0], dict):
+ result_preview = f'list of dicts - len={len(result)}, example 1:\n'
+ sample_result = result[0]
+ for key, value in list(sample_result.items())[:10]:
+ value_str = str(value)[:10] if not isinstance(value, (int, float, bool, type(None))) else str(value)
+ result_preview += f' {key}: {value_str}...\n'
+ if len(sample_result) > 10:
+ result_preview += f' ... {len(sample_result) - 10} more keys'
+ print(result_preview)
+
+ elif isinstance(result, list):
+ if len(result) == 0:
+ print('type=list, len=0')
+ else:
+ result_preview = str(result)[:100]
+ print(f'type=list, len={len(result)}, preview={result_preview}...')
+ elif isinstance(result, dict):
+ result_preview = f'type=dict, len={len(result)}, sample keys:\n'
+ for key, value in list(result.items())[:10]:
+ value_str = str(value)[:10] if not isinstance(value, (int, float, bool, type(None))) else str(value)
+ result_preview += f' {key}: {value_str}...\n'
+ if len(result) > 10:
+ result_preview += f' ... {len(result) - 10} more keys'
+ print(result_preview)
+
+ else:
+ print(f'type={type(result).__name__}, value={repr(result)[:50]}')
+
+ return result
+ except Exception as e:
+ # Track errors for pattern detection
+ namespace['_evaluate_failures'].append({'error': str(e), 'type': 'exception'})
+ raise
+
+ namespace['evaluate'] = evaluate_wrapper
+
+ # Add get_selector_from_index helper for code_use mode
+ async def get_selector_from_index_wrapper(index: int) -> str:
+ """
+ Get the CSS selector for an element by its interactive index.
+
+ This allows you to use the element's index from the browser state to get
+ its CSS selector for use in JavaScript evaluate() calls.
+
+ Args:
+ index: The interactive index from the browser state (e.g., [123])
+
+ Returns:
+ str: CSS selector that can be used in JavaScript
+
+ Example:
+ selector = await get_selector_from_index(123)
+ await evaluate(f'''
+ (function(){{
+ const el = document.querySelector({json.dumps(selector)});
+ if (el) el.click();
+ }})()
+ ''')
+ """
+ from browser_use.dom.utils import generate_css_selector_for_element
+
+ # Get element by index from browser session
+ node = await browser_session.get_element_by_index(index)
+ if node is None:
+ msg = f'Element index {index} not available - page may have changed. Try refreshing browser state.'
+ logger.warning(f'⚠️ {msg}')
+ raise RuntimeError(msg)
+
+ # Check if element is in shadow DOM
+ shadow_hosts = []
+ current = node.parent_node
+ while current:
+ if current.shadow_root_type is not None:
+ # This is a shadow host
+ host_tag = current.tag_name.lower()
+ host_id = current.attributes.get('id', '') if current.attributes else ''
+ host_desc = f'{host_tag}#{host_id}' if host_id else host_tag
+ shadow_hosts.insert(0, host_desc)
+ current = current.parent_node
+
+ # Check if in iframe
+ in_iframe = False
+ current = node.parent_node
+ while current:
+ if current.tag_name.lower() == 'iframe':
+ in_iframe = True
+ break
+ current = current.parent_node
+
+ # Use the robust selector generation function (now handles special chars in IDs)
+ selector = generate_css_selector_for_element(node)
+
+ # Log shadow DOM/iframe info if detected
+ if shadow_hosts:
+ shadow_path = ' > '.join(shadow_hosts)
+ logger.info(f'Element [{index}] is inside Shadow DOM. Path: {shadow_path}')
+ logger.info(f' Selector: {selector}')
+ logger.info(
+ f' To access: document.querySelector("{shadow_hosts[0].split("#")[0]}").shadowRoot.querySelector("{selector}")'
+ )
+ if in_iframe:
+ logger.info(f"Element [{index}] is inside an iframe. Regular querySelector won't work.")
+
+ if selector:
+ return selector
+
+ # Fallback: just use tag name if available
+ if node.tag_name:
+ return node.tag_name.lower()
+
+ raise ValueError(f'Could not generate selector for element index {index}')
+
+ namespace['get_selector_from_index'] = get_selector_from_index_wrapper
+
+ # Inject all tools as functions into the namespace
+ # Skip 'evaluate' since we have a custom implementation above
+ for action_name, action in tools.registry.registry.actions.items():
+ if action_name == 'evaluate':
+ continue # Skip - use custom evaluate that returns Python objects directly
+ param_model = action.param_model
+ action_function = action.function
+
+ # Create a closure to capture the current action_name, param_model, and action_function
+ def make_action_wrapper(act_name, par_model, act_func):
+ async def action_wrapper(*args, **kwargs):
+ # Convert positional args to kwargs based on param model fields
+ if args:
+ # Get the field names from the pydantic model
+ field_names = list(par_model.model_fields.keys())
+ for i, arg in enumerate(args):
+ if i < len(field_names):
+ kwargs[field_names[i]] = arg
+
+ # Create params from kwargs
+ try:
+ params = par_model(**kwargs)
+ except Exception as e:
+ raise ValueError(f'Invalid parameters for {act_name}: {e}') from e
+
+ # Special validation for done() - enforce minimal code cell
+ if act_name == 'done':
+ consecutive_failures = namespace.get('_consecutive_errors')
+ if consecutive_failures and consecutive_failures > 3:
+ pass
+
+ else:
+ # Check if there are multiple Python blocks in this response
+ all_blocks = namespace.get('_all_code_blocks', {})
+ python_blocks = [k for k in sorted(all_blocks.keys()) if k.startswith('python_')]
+
+ if len(python_blocks) > 1:
+ msg = (
+ 'done() should be the ONLY code block in the response.\n'
+ 'You have multiple Python blocks in this response. Consider calling done() in a separate response '
+ 'Now verify the last output and if it satisfies the task, call done(), else continue working.'
+ )
+ print(msg)
+
+ # Get the current cell code from namespace (injected by service.py before execution)
+ current_code = namespace.get('_current_cell_code')
+ if current_code and isinstance(current_code, str):
+ # Count non-empty, non-comment lines
+ lines = [line.strip() for line in current_code.strip().split('\n')]
+ code_lines = [line for line in lines if line and not line.startswith('#')]
+
+ # Check if the line above await done() contains an if block
+ done_line_index = -1
+ for i, line in enumerate(reversed(code_lines)):
+ if 'await done()' in line or 'await done(' in line:
+ done_line_index = len(code_lines) - 1 - i
+ break
+
+ has_if_above = False
+ has_else_above = False
+ has_elif_above = False
+ if done_line_index > 0:
+ line_above = code_lines[done_line_index - 1]
+ has_if_above = line_above.strip().startswith('if ') and line_above.strip().endswith(':')
+ has_else_above = line_above.strip().startswith('else:')
+ has_elif_above = line_above.strip().startswith('elif ')
+ if has_if_above or has_else_above or has_elif_above:
+ msg = (
+ 'done() should be called individually after verifying the result from any logic.\n'
+ 'Consider validating your output first, THEN call done() in a final step without if/else/elif blocks only if the task is truly complete.'
+ )
+ logger.warning(msg)
+ print(msg)
+
+ # Build special context
+ special_context = {
+ 'browser_session': browser_session,
+ 'page_extraction_llm': page_extraction_llm,
+ 'available_file_paths': available_file_paths,
+ 'has_sensitive_data': False, # Can be handled separately if needed
+ 'file_system': file_system,
+ }
+
+ # Execute the action
+ result = await act_func(params=params, **special_context)
+
+ # For code-use mode, we want to return the result directly
+ # not wrapped in ActionResult
+ if hasattr(result, 'extracted_content'):
+ # Special handling for done action - mark task as complete
+ if act_name == 'done' and hasattr(result, 'is_done') and result.is_done:
+ namespace['_task_done'] = True
+ # Store the extracted content as the final result
+ if result.extracted_content:
+ namespace['_task_result'] = result.extracted_content
+ # Store the self-reported success status
+ if hasattr(result, 'success'):
+ namespace['_task_success'] = result.success
+
+ # If there's extracted content, return it
+ if result.extracted_content:
+ return result.extracted_content
+ # If there's an error, raise it
+ if result.error:
+ raise RuntimeError(result.error)
+ # Otherwise return None
+ return None
+ return result
+
+ return action_wrapper
+
+ # Rename 'input' to 'input_text' to avoid shadowing Python's built-in input()
+ namespace_action_name = 'input_text' if action_name == 'input' else action_name
+
+ # Add the wrapper to the namespace
+ namespace[namespace_action_name] = make_action_wrapper(action_name, param_model, action_function)
+
+ return namespace
+
+
+def get_namespace_documentation(namespace: dict[str, Any]) -> str:
+ """
+ Generate documentation for all available functions in the namespace.
+
+ Args:
+ namespace: The namespace dictionary
+
+ Returns:
+ Markdown-formatted documentation string
+ """
+ docs = ['# Available Functions\n']
+
+ # Document each function
+ for name, obj in sorted(namespace.items()):
+ if callable(obj) and not name.startswith('_'):
+ # Get function signature and docstring
+ if hasattr(obj, '__doc__') and obj.__doc__:
+ docs.append(f'## {name}\n')
+ docs.append(f'{obj.__doc__}\n')
+
+ return '\n'.join(docs)
diff --git a/browser_use/code_use/notebook_export.py b/browser_use/code_use/notebook_export.py
new file mode 100644
index 000000000..31dec8b65
--- /dev/null
+++ b/browser_use/code_use/notebook_export.py
@@ -0,0 +1,195 @@
+"""Export code-use session to Jupyter notebook format."""
+
+import json
+from pathlib import Path
+
+from .views import CellType, NotebookExport, NotebookSession
+
+
+def export_to_ipynb(session: NotebookSession, output_path: str | Path) -> Path:
+ """
+ Export a NotebookSession to a Jupyter notebook (.ipynb) file.
+
+ Args:
+ session: The NotebookSession to export
+ output_path: Path where to save the notebook file
+
+ Returns:
+ Path to the saved notebook file
+
+ Example:
+ ```python
+ session = await agent.run()
+ notebook_path = export_to_ipynb(session, 'my_automation.ipynb')
+ print(f'Notebook saved to {notebook_path}')
+ ```
+ """
+ output_path = Path(output_path)
+
+ # Create notebook structure
+ notebook = NotebookExport(
+ metadata={
+ 'kernelspec': {'display_name': 'Python 3', 'language': 'python', 'name': 'python3'},
+ 'language_info': {
+ 'name': 'python',
+ 'version': '3.11.0',
+ 'mimetype': 'text/x-python',
+ 'codemirror_mode': {'name': 'ipython', 'version': 3},
+ 'pygments_lexer': 'ipython3',
+ 'nbconvert_exporter': 'python',
+ 'file_extension': '.py',
+ },
+ }
+ )
+
+ # Add setup cell at the beginning with proper type hints
+ setup_code = """import asyncio
+from typing import Any
+from browser_use import BrowserSession
+from browser_use.code_use import create_namespace
+
+# Initialize browser and namespace
+browser = BrowserSession()
+await browser.start()
+
+# Create namespace with all browser control functions
+namespace: dict[str, Any] = create_namespace(browser)
+
+# Import all functions into the current namespace
+globals().update(namespace)
+
+# Type hints for better IDE support (these are now available globally)
+# navigate, click, input, evaluate, search, extract, scroll, done, etc.
+
+print("Browser-use environment initialized!")
+print("Available functions: navigate, click, input, evaluate, search, extract, done, etc.")"""
+
+ setup_cell = {
+ 'cell_type': 'code',
+ 'metadata': {},
+ 'source': setup_code.split('\n'),
+ 'execution_count': None,
+ 'outputs': [],
+ }
+ notebook.cells.append(setup_cell)
+
+ # Convert cells
+ for cell in session.cells:
+ notebook_cell: dict = {
+ 'cell_type': cell.cell_type.value,
+ 'metadata': {},
+ 'source': cell.source.split('\n'),
+ }
+
+ if cell.cell_type == CellType.CODE:
+ notebook_cell['execution_count'] = cell.execution_count
+ notebook_cell['outputs'] = []
+
+ # Add output if available
+ if cell.output:
+ notebook_cell['outputs'].append(
+ {
+ 'output_type': 'stream',
+ 'name': 'stdout',
+ 'text': cell.output.split('\n'),
+ }
+ )
+
+ # Add error if available
+ if cell.error:
+ notebook_cell['outputs'].append(
+ {
+ 'output_type': 'error',
+ 'ename': 'Error',
+ 'evalue': cell.error.split('\n')[0] if cell.error else '',
+ 'traceback': cell.error.split('\n') if cell.error else [],
+ }
+ )
+
+ # Add browser state as a separate output
+ if cell.browser_state:
+ notebook_cell['outputs'].append(
+ {
+ 'output_type': 'stream',
+ 'name': 'stdout',
+ 'text': [f'Browser State:\n{cell.browser_state}'],
+ }
+ )
+
+ notebook.cells.append(notebook_cell)
+
+ # Write to file
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ with open(output_path, 'w', encoding='utf-8') as f:
+ json.dump(notebook.model_dump(), f, indent=2, ensure_ascii=False)
+
+ return output_path
+
+
+def session_to_python_script(session: NotebookSession) -> str:
+ """
+ Convert a NotebookSession to a Python script.
+
+ Args:
+ session: The NotebookSession to convert
+
+ Returns:
+ Python script as a string
+
+ Example:
+ ```python
+ session = await agent.run()
+ script = session_to_python_script(session)
+ print(script)
+ ```
+ """
+ lines = []
+
+ lines.append('# Generated from browser-use code-use session\n')
+ lines.append('import asyncio\n')
+ lines.append('from browser_use import BrowserSession\n')
+ lines.append('from browser_use.code_use import create_namespace\n\n')
+
+ lines.append('async def main():\n')
+ lines.append('\t# Initialize browser and namespace\n')
+ lines.append('\tbrowser = BrowserSession()\n')
+ lines.append('\tawait browser.start()\n\n')
+ lines.append('\t# Create namespace with all browser control functions\n')
+ lines.append('\tnamespace = create_namespace(browser)\n\n')
+ lines.append('\t# Extract functions from namespace for direct access\n')
+ lines.append('\tnavigate = namespace["navigate"]\n')
+ lines.append('\tclick = namespace["click"]\n')
+ lines.append('\tinput_text = namespace["input"]\n')
+ lines.append('\tevaluate = namespace["evaluate"]\n')
+ lines.append('\tsearch = namespace["search"]\n')
+ lines.append('\textract = namespace["extract"]\n')
+ lines.append('\tscroll = namespace["scroll"]\n')
+ lines.append('\tdone = namespace["done"]\n')
+ lines.append('\tgo_back = namespace["go_back"]\n')
+ lines.append('\twait = namespace["wait"]\n')
+ lines.append('\tscreenshot = namespace["screenshot"]\n')
+ lines.append('\tfind_text = namespace["find_text"]\n')
+ lines.append('\tswitch_tab = namespace["switch"]\n')
+ lines.append('\tclose_tab = namespace["close"]\n')
+ lines.append('\tdropdown_options = namespace["dropdown_options"]\n')
+ lines.append('\tselect_dropdown = namespace["select_dropdown"]\n')
+ lines.append('\tupload_file = namespace["upload_file"]\n')
+ lines.append('\tsend_keys = namespace["send_keys"]\n\n')
+
+ for i, cell in enumerate(session.cells):
+ if cell.cell_type == CellType.CODE:
+ lines.append(f'\t# Cell {i + 1}\n')
+
+ # Indent each line of source
+ source_lines = cell.source.split('\n')
+ for line in source_lines:
+ if line.strip(): # Only add non-empty lines
+ lines.append(f'\t{line}\n')
+
+ lines.append('\n')
+
+ lines.append('\tawait browser.stop()\n\n')
+ lines.append("if __name__ == '__main__':\n")
+ lines.append('\tasyncio.run(main())\n')
+
+ return ''.join(lines)
diff --git a/browser_use/code_use/service.py b/browser_use/code_use/service.py
new file mode 100644
index 000000000..b1518e4e1
--- /dev/null
+++ b/browser_use/code_use/service.py
@@ -0,0 +1,1332 @@
+"""Code-use agent service - Jupiter notebook-like code execution for browser automation."""
+
+import asyncio
+import datetime
+import logging
+import re
+import traceback
+from pathlib import Path
+from typing import Any
+
+from uuid_extensions import uuid7str
+
+from browser_use.browser import BrowserSession
+from browser_use.browser.profile import BrowserProfile
+from browser_use.dom.service import DomService
+from browser_use.filesystem.file_system import FileSystem
+from browser_use.llm.base import BaseChatModel
+from browser_use.llm.messages import (
+ AssistantMessage,
+ BaseMessage,
+ ContentPartImageParam,
+ ContentPartTextParam,
+ ImageURL,
+ UserMessage,
+)
+from browser_use.screenshots.service import ScreenshotService
+from browser_use.telemetry.service import ProductTelemetry
+from browser_use.telemetry.views import AgentTelemetryEvent
+from browser_use.tokens.service import TokenCost
+from browser_use.tokens.views import UsageSummary
+from browser_use.tools.service import Tools
+from browser_use.utils import get_browser_use_version
+
+from .formatting import format_browser_state_for_llm
+from .namespace import EvaluateError, create_namespace
+from .utils import detect_token_limit_issue, extract_code_blocks, extract_url_from_task, truncate_message_content
+from .views import (
+ CodeAgentHistory,
+ CodeAgentModelOutput,
+ CodeAgentResult,
+ CodeAgentState,
+ CodeAgentStepMetadata,
+ ExecutionStatus,
+ NotebookSession,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class CodeAgent:
+ """
+ Agent that executes Python code in a notebook-like environment for browser automation.
+
+ This agent provides a Jupiter notebook-like interface where the LLM writes Python code
+ that gets executed in a persistent namespace with browser control functions available.
+ """
+
+ def __init__(
+ self,
+ task: str,
+ # Optional parameters
+ browser_session: BrowserSession | None = None,
+ browser: BrowserSession | None = None, # Alias for browser_session
+ tools: Tools | None = None,
+ controller: Tools | None = None, # Alias for tools
+ # Agent settings
+ page_extraction_llm: BaseChatModel | None = None,
+ file_system: FileSystem | None = None,
+ available_file_paths: list[str] | None = None,
+ sensitive_data: dict[str, str | dict[str, str]] | None = None,
+ max_steps: int = 100,
+ max_failures: int = 3,
+ max_validations: int = 0,
+ use_vision: bool = True,
+ calculate_cost: bool = False,
+ **kwargs,
+ ):
+ """
+ Initialize the code-use agent.
+
+ Args:
+ task: The task description for the agent
+ browser_session: Optional browser session (will be created if not provided) [DEPRECATED: use browser]
+ browser: Optional browser session (cleaner API)
+ tools: Optional Tools instance (will create default if not provided) [DEPRECATED: use controller]
+ controller: Optional Tools instance (cleaner API)
+ page_extraction_llm: Optional LLM for page extraction
+ file_system: Optional file system for file operations
+ available_file_paths: Optional list of available file paths
+ sensitive_data: Optional sensitive data dictionary
+ max_steps: Maximum number of execution steps
+ max_failures: Maximum consecutive errors before termination (default: 3)
+ max_validations: Maximum number of times to run the validator agent (default: 0)
+ use_vision: Whether to include screenshots in LLM messages (default: True)
+ calculate_cost: Whether to calculate token costs (default: False)
+ **kwargs: Additional keyword arguments for compatibility (ignored)
+ """
+ # Log and ignore unknown kwargs for compatibility
+ if kwargs:
+ logger.debug(f'Ignoring additional kwargs for CodeAgent compatibility: {list(kwargs.keys())}')
+
+ # Create ChatBrowserUse LLM (used internally by CodeAgent)
+ try:
+ from browser_use import ChatBrowserUse
+
+ llm = ChatBrowserUse()
+ logger.debug('CodeAgent using ChatBrowserUse')
+ except Exception as e:
+ raise RuntimeError(f'Failed to initialize CodeAgent LLM: {e}')
+
+ # Handle browser vs browser_session parameter (browser takes precedence)
+ if browser and browser_session:
+ raise ValueError('Cannot specify both "browser" and "browser_session" parameters. Use "browser" for the cleaner API.')
+ browser_session = browser or browser_session
+
+ # Handle controller vs tools parameter (controller takes precedence)
+ if controller and tools:
+ raise ValueError('Cannot specify both "controller" and "tools" parameters. Use "controller" for the cleaner API.')
+ tools = controller or tools
+
+ # Store browser_profile for creating browser session if needed
+ self._browser_profile_for_init = BrowserProfile() if browser_session is None else None
+
+ self.task = task
+ self.llm = llm
+ self.browser_session = browser_session
+ self.tools = tools or Tools()
+ self.page_extraction_llm = page_extraction_llm
+ self.file_system = file_system if file_system is not None else FileSystem(base_dir='./')
+ self.available_file_paths = available_file_paths or []
+ self.sensitive_data = sensitive_data
+ self.max_steps = max_steps
+ self.max_failures = max_failures
+ self.max_validations = max_validations
+ self.use_vision = use_vision
+
+ self.session = NotebookSession()
+ self.namespace: dict[str, Any] = {}
+ self._llm_messages: list[BaseMessage] = [] # Internal LLM conversation history
+ self.complete_history: list[CodeAgentHistory] = [] # Type-safe history with model_output and result
+ self.dom_service: DomService | None = None
+ self._last_browser_state_text: str | None = None # Track last browser state text
+ self._last_screenshot: str | None = None # Track last screenshot (base64)
+ self._consecutive_errors = 0 # Track consecutive errors for auto-termination
+ self._validation_count = 0 # Track number of validator runs
+ self._last_llm_usage: Any | None = None # Track last LLM call usage stats
+ self._step_start_time = 0.0 # Track step start time for duration calculation
+ self.usage_summary: UsageSummary | None = None # Track usage summary across run for history property
+
+ # Initialize screenshot service for eval tracking
+ self.id = uuid7str()
+ timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+ base_tmp = Path('/tmp')
+ self.agent_directory = base_tmp / f'browser_use_code_agent_{self.id}_{timestamp}'
+ self.screenshot_service = ScreenshotService(agent_directory=self.agent_directory)
+
+ # Initialize token cost service for usage tracking
+ self.token_cost_service = TokenCost(include_cost=calculate_cost)
+ self.token_cost_service.register_llm(llm)
+ if page_extraction_llm:
+ self.token_cost_service.register_llm(page_extraction_llm)
+
+ # Set version and source for telemetry
+ self.version = get_browser_use_version()
+ try:
+ package_root = Path(__file__).parent.parent.parent
+ repo_files = ['.git', 'README.md', 'docs', 'examples']
+ if all(Path(package_root / file).exists() for file in repo_files):
+ self.source = 'git'
+ else:
+ self.source = 'pip'
+ except Exception:
+ self.source = 'unknown'
+
+ # Telemetry
+ self.telemetry = ProductTelemetry()
+
+ async def run(self, max_steps: int | None = None) -> NotebookSession:
+ """
+ Run the agent to complete the task.
+
+ Args:
+ max_steps: Optional override for maximum number of steps (uses __init__ value if not provided)
+
+ Returns:
+ The notebook session with all executed cells
+ """
+ # Use override if provided, otherwise use value from __init__
+ steps_to_run = max_steps if max_steps is not None else self.max_steps
+ self.max_steps = steps_to_run
+ # Start browser if not provided
+ if self.browser_session is None:
+ assert self._browser_profile_for_init is not None
+ self.browser_session = BrowserSession(browser_profile=self._browser_profile_for_init)
+ await self.browser_session.start()
+
+ # Initialize DOM service with cross-origin iframe support enabled
+ self.dom_service = DomService(
+ browser_session=self.browser_session,
+ cross_origin_iframes=True, # Enable for code-use agent to access forms in iframes
+ )
+
+ # Create namespace with all tools
+ self.namespace = create_namespace(
+ browser_session=self.browser_session,
+ tools=self.tools,
+ page_extraction_llm=self.page_extraction_llm,
+ file_system=self.file_system,
+ available_file_paths=self.available_file_paths,
+ sensitive_data=self.sensitive_data,
+ )
+
+ # Initialize conversation with task
+ self._llm_messages.append(UserMessage(content=f'Task: {self.task}'))
+
+ # Track agent run error for telemetry
+ agent_run_error: str | None = None
+
+ # Extract URL from task and navigate if found
+ initial_url = extract_url_from_task(self.task)
+ if initial_url:
+ try:
+ logger.info(f'Extracted URL from task, navigating to: {initial_url}')
+ # Use the navigate action from namespace
+ await self.namespace['navigate'](initial_url)
+ # Wait for page load
+ await asyncio.sleep(2)
+
+ # Record this navigation as a cell in the notebook
+ nav_code = f"await navigate('{initial_url}')"
+ cell = self.session.add_cell(source=nav_code)
+ cell.status = ExecutionStatus.SUCCESS
+ cell.execution_count = self.session.increment_execution_count()
+ cell.output = f'Navigated to {initial_url}'
+
+ # Get browser state after navigation for the cell
+ if self.dom_service:
+ try:
+ browser_state_text, _ = await self._get_browser_state()
+ cell.browser_state = browser_state_text
+ except Exception as state_error:
+ logger.debug(f'Failed to capture browser state for initial navigation cell: {state_error}')
+
+ except Exception as e:
+ logger.warning(f'Failed to navigate to extracted URL {initial_url}: {e}')
+ # Record failed navigation as error cell
+ nav_code = f"await navigate('{initial_url}')"
+ cell = self.session.add_cell(source=nav_code)
+ cell.status = ExecutionStatus.ERROR
+ cell.execution_count = self.session.increment_execution_count()
+ cell.error = str(e)
+
+ # Get initial browser state before first LLM call
+ if self.browser_session and self.dom_service:
+ try:
+ browser_state_text, screenshot = await self._get_browser_state()
+ self._last_browser_state_text = browser_state_text
+ self._last_screenshot = screenshot
+ except Exception as e:
+ logger.warning(f'Failed to get initial browser state: {e}')
+
+ # Main execution loop
+ for step in range(self.max_steps):
+ logger.info(f'\n\n\n\n\n\n\nStep {step + 1}/{self.max_steps}')
+
+ # Start timing this step
+ self._step_start_time = datetime.datetime.now().timestamp()
+
+ # Check if we're approaching the step limit or error limit and inject warning
+ steps_remaining = self.max_steps - step - 1
+ errors_remaining = self.max_failures - self._consecutive_errors
+
+ should_warn = (
+ steps_remaining <= 1 # Last step or next to last
+ or errors_remaining <= 1 # One more error will terminate
+ or (steps_remaining <= 2 and self._consecutive_errors >= 2) # Close to both limits
+ )
+
+ if should_warn:
+ warning_message = (
+ f'\n\n⚠️ CRITICAL WARNING: You are approaching execution limits!\n'
+ f'- Steps remaining: {steps_remaining + 1}\n'
+ f'- Consecutive errors: {self._consecutive_errors}/{self.max_failures}\n\n'
+ f'YOU MUST call done() in your NEXT response, even if the task is incomplete:\n'
+ f"- Set success=False if you couldn't complete the task\n"
+ f'- Return EVERYTHING you found so far (partial data is better than nothing)\n'
+ f"- Include any variables you've stored (products, all_data, etc.)\n"
+ f"- Explain what worked and what didn't\n\n"
+ f'Without done(), the user will receive NOTHING.'
+ )
+ self._llm_messages.append(UserMessage(content=warning_message))
+
+ try:
+ # Fetch fresh browser state right before LLM call (only if not already set)
+ if not self._last_browser_state_text and self.browser_session and self.dom_service:
+ try:
+ logger.debug('🔍 Fetching browser state before LLM call...')
+ browser_state_text, screenshot = await self._get_browser_state()
+ self._last_browser_state_text = browser_state_text
+ self._last_screenshot = screenshot
+
+ # # Log browser state
+ # if len(browser_state_text) > 2000:
+ # logger.info(
+ # f'Browser state (before LLM):\n{browser_state_text[:2000]}...\n[Truncated, full state {len(browser_state_text)} chars sent to LLM]'
+ # )
+ # else:
+ # logger.info(f'Browser state (before LLM):\n{browser_state_text}')
+ except Exception as e:
+ logger.warning(f'Failed to get browser state before LLM call: {e}')
+
+ # Get code from LLM (this also adds to self._llm_messages)
+ try:
+ code, full_llm_response = await self._get_code_from_llm()
+ except Exception as llm_error:
+ # LLM call failed - count as consecutive error and retry
+ self._consecutive_errors += 1
+ logger.warning(
+ f'LLM call failed (consecutive errors: {self._consecutive_errors}/{self.max_failures}), retrying: {llm_error}'
+ )
+
+ # Check if we've hit the consecutive error limit
+ if self._consecutive_errors >= self.max_failures:
+ logger.error(f'Terminating: {self.max_failures} consecutive LLM failures')
+ break
+
+ await asyncio.sleep(1) # Brief pause before retry
+ continue
+
+ if not code or code.strip() == '':
+ # If task is already done, empty code is fine (LLM explaining completion)
+ if self._is_task_done():
+ logger.info('Task already marked as done, LLM provided explanation without code')
+ # Add the text response to history as a non-code step
+ await self._add_step_to_complete_history(
+ model_output_code='',
+ full_llm_response=full_llm_response,
+ output=full_llm_response, # Treat the explanation as output
+ error=None,
+ screenshot_path=await self._capture_screenshot(step + 1),
+ )
+ break # Exit the loop since task is done
+
+ logger.warning('LLM returned empty code')
+ self._consecutive_errors += 1
+
+ # new state
+ if self.browser_session and self.dom_service:
+ try:
+ browser_state_text, screenshot = await self._get_browser_state()
+ self._last_browser_state_text = browser_state_text
+ self._last_screenshot = screenshot
+ except Exception as e:
+ logger.warning(f'Failed to get new browser state: {e}')
+ continue
+
+ # Execute code blocks sequentially if multiple python blocks exist
+ # This allows JS/bash blocks to be injected into namespace before Python code uses them
+ all_blocks = self.namespace.get('_all_code_blocks', {})
+ python_blocks = [k for k in sorted(all_blocks.keys()) if k.startswith('python_')]
+
+ if len(python_blocks) > 1:
+ # Multiple Python blocks - execute each sequentially
+ output = None
+ error = None
+
+ for i, block_key in enumerate(python_blocks):
+ logger.info(f'Executing Python block {i + 1}/{len(python_blocks)}')
+ block_code = all_blocks[block_key]
+ block_output, block_error, _ = await self._execute_code(block_code)
+
+ # Accumulate outputs
+ if block_output:
+ output = (output or '') + block_output
+ if block_error:
+ error = block_error
+ # Stop on first error
+ break
+ else:
+ # Single Python block - execute normally
+ output, error, _ = await self._execute_code(code)
+
+ # Track consecutive errors
+ if error:
+ self._consecutive_errors += 1
+ logger.warning(f'Consecutive errors: {self._consecutive_errors}/{self.max_failures}')
+
+ # Check if we've hit the consecutive error limit
+ if self._consecutive_errors >= self.max_failures:
+ logger.error(
+ f'Terminating: {self.max_failures} consecutive errors reached. The agent is unable to make progress.'
+ )
+ # Add termination message to complete history before breaking
+ await self._add_step_to_complete_history(
+ model_output_code=code,
+ full_llm_response=f'[Terminated after {self.max_failures} consecutive errors]',
+ output=None,
+ error=f'Auto-terminated: {self.max_failures} consecutive errors without progress',
+ screenshot_path=None,
+ )
+ break
+ else:
+ # Reset consecutive error counter on success
+ self._consecutive_errors = 0
+
+ # Check if task is done - validate completion first if not at limits
+ if self._is_task_done():
+ # Get the final result from namespace (from done() call)
+ final_result: str | None = self.namespace.get('_task_result') # type: ignore[assignment]
+
+ # Check if we should validate (not at step/error limits and under max validations)
+ steps_remaining = self.max_steps - step - 1
+ should_validate = (
+ self._validation_count < self.max_validations # Haven't exceeded max validations
+ and steps_remaining >= 4 # At least 4 steps away from limit
+ and self._consecutive_errors < 3 # Not close to error limit (8 consecutive)
+ )
+
+ if should_validate:
+ self._validation_count += 1
+ logger.info('Validating task completion with LLM...')
+ from .namespace import validate_task_completion
+
+ is_complete, reasoning = await validate_task_completion(
+ task=self.task,
+ output=final_result,
+ llm=self.llm,
+ )
+
+ if not is_complete:
+ # Task not truly complete - inject feedback and continue
+ logger.warning('Validator: Task not complete, continuing...')
+ validation_feedback = (
+ f'\n\n⚠️ VALIDATOR FEEDBACK:\n'
+ f'Your done() call was rejected. The task is NOT complete yet.\n\n'
+ f'Validation reasoning:\n{reasoning}\n\n'
+ f'You must continue working on the task. Analyze what is missing and complete it.\n'
+ f'Do NOT call done() again until the task is truly finished.'
+ )
+
+ # Clear the done flag so execution continues
+ self.namespace['_task_done'] = False
+ self.namespace.pop('_task_result', None)
+ self.namespace.pop('_task_success', None)
+
+ # Add validation feedback to LLM messages
+ self._llm_messages.append(UserMessage(content=validation_feedback))
+
+ # Don't override output - let execution continue normally
+ else:
+ logger.info('Validator: Task complete')
+ # Override output with done message for final step
+ if final_result:
+ output = final_result
+ else:
+ # At limits - skip validation and accept done()
+ if self._validation_count >= self.max_validations:
+ logger.info(
+ f'Reached max validations ({self.max_validations}) - skipping validation and accepting done()'
+ )
+ else:
+ logger.info('At step/error limits - skipping validation')
+ if final_result:
+ output = final_result
+
+ if output:
+ # Check if this is the final done() output
+ if self._is_task_done():
+ # Show done() output more prominently
+ logger.info(
+ f'✓ Task completed - Final output from done():\n{output[:300] if len(output) > 300 else output}'
+ )
+ # Also show files_to_display if they exist in namespace
+ attachments: list[str] | None = self.namespace.get('_task_attachments') # type: ignore[assignment]
+ if attachments:
+ logger.info(f'Files displayed: {", ".join(attachments)}')
+ else:
+ logger.info(f'Code output:\n{output}')
+
+ # Browser state is now only logged when fetched before LLM call (not after execution)
+
+ # Take screenshot for eval tracking
+ screenshot_path = await self._capture_screenshot(step + 1)
+
+ # Add step to complete_history for eval system
+ await self._add_step_to_complete_history(
+ model_output_code=code,
+ full_llm_response=full_llm_response,
+ output=output,
+ error=error,
+ screenshot_path=screenshot_path,
+ )
+
+ # Check if task is done (after validation)
+ if self._is_task_done():
+ # Get the final result from namespace
+ final_result: str | None = self.namespace.get('_task_result', output) # type: ignore[assignment]
+ logger.info('Task completed successfully')
+ if final_result:
+ logger.info(f'Final result: {final_result}')
+ break
+ # If validation rejected done(), continue to next iteration
+ # The feedback message has already been added to _llm_messages
+
+ # Add result to LLM messages for next iteration (without browser state)
+ result_message = self._format_execution_result(code, output, error, current_step=step + 1)
+ truncated_result = truncate_message_content(result_message)
+ self._llm_messages.append(UserMessage(content=truncated_result))
+
+ except Exception as e:
+ logger.error(f'Error in step {step + 1}: {e}')
+ traceback.print_exc()
+ break
+ else:
+ # Loop completed without break - max_steps reached
+ logger.warning(f'Maximum steps ({self.max_steps}) reached without task completion')
+
+ # If task is not done, capture the last step's output as partial result
+ if not self._is_task_done() and self.complete_history:
+ # Get the last step's output/error and use it as final extracted_content
+ last_step = self.complete_history[-1]
+ last_result = last_step.result[0] if last_step.result else None
+ last_output = last_result.extracted_content if last_result else None
+ last_error = last_result.error if last_result else None
+
+ # Build a partial result message from the last step
+ partial_result_parts = []
+ partial_result_parts.append(f'Task incomplete - reached step limit ({self.max_steps} steps).')
+ partial_result_parts.append('Last step output:')
+
+ if last_output:
+ partial_result_parts.append(f'\nOutput: {last_output}')
+ if last_error:
+ partial_result_parts.append(f'\nError: {last_error}')
+
+ # Add any accumulated variables that might contain useful data
+ data_vars = []
+ for var_name in sorted(self.namespace.keys()):
+ if not var_name.startswith('_') and var_name not in {'json', 'asyncio', 'csv', 're', 'datetime', 'Path'}:
+ var_value = self.namespace[var_name]
+ # Check if it's a list or dict that might contain collected data
+ if isinstance(var_value, (list, dict)) and var_value:
+ data_vars.append(f' - {var_name}: {type(var_value).__name__} with {len(var_value)} items')
+
+ if data_vars:
+ partial_result_parts.append('\nVariables in namespace that may contain partial data:')
+ partial_result_parts.extend(data_vars)
+
+ partial_result = '\n'.join(partial_result_parts)
+
+ # Update the last step's extracted_content with this partial result
+ if last_result:
+ last_result.extracted_content = partial_result
+ last_result.is_done = False
+ last_result.success = False
+
+ logger.info(f'\nPartial result captured from last step:\n{partial_result}')
+
+ # Log final summary if task was completed
+ if self._is_task_done():
+ logger.info('\n' + '=' * 60)
+ logger.info('TASK COMPLETED SUCCESSFULLY')
+ logger.info('=' * 60)
+ final_result: str | None = self.namespace.get('_task_result') # type: ignore[assignment]
+ if final_result:
+ logger.info(f'\nFinal Output:\n{final_result}')
+
+ attachments: list[str] | None = self.namespace.get('_task_attachments') # type: ignore[assignment]
+ if attachments:
+ logger.info(f'\nFiles Attached:\n{chr(10).join(attachments)}')
+ logger.info('=' * 60 + '\n')
+
+ # Auto-close browser if keep_alive is False
+ await self.close()
+
+ # Store usage summary for history property
+ self.usage_summary = await self.token_cost_service.get_usage_summary()
+
+ # Log token usage summary
+ await self.token_cost_service.log_usage_summary()
+
+ # Log telemetry event
+ try:
+ self._log_agent_event(max_steps=self.max_steps, agent_run_error=agent_run_error)
+ except Exception as log_e:
+ logger.error(f'Failed to log telemetry event: {log_e}', exc_info=True)
+
+ return self.session
+
+ async def _get_code_from_llm(self) -> tuple[str, str]:
+ """Get Python code from the LLM.
+
+ Returns:
+ Tuple of (extracted_code, full_llm_response)
+ """
+ # Prepare messages for this request
+ # Include browser state as separate message if available (not accumulated in history)
+ messages_to_send = self._llm_messages.copy()
+
+ if self._last_browser_state_text:
+ # Create message with optional screenshot
+ if self.use_vision and self._last_screenshot:
+ # Build content with text + screenshot
+ content_parts: list[ContentPartTextParam | ContentPartImageParam] = [
+ ContentPartTextParam(text=self._last_browser_state_text)
+ ]
+
+ # Add screenshot
+ content_parts.append(
+ ContentPartImageParam(
+ image_url=ImageURL(
+ url=f'data:image/jpeg;base64,{self._last_screenshot}',
+ media_type='image/jpeg',
+ detail='auto',
+ ),
+ )
+ )
+
+ messages_to_send.append(UserMessage(content=content_parts))
+ else:
+ # Text only
+ messages_to_send.append(UserMessage(content=self._last_browser_state_text))
+
+ # Clear browser state after including it so it's only in this request
+ self._last_browser_state_text = None
+ self._last_screenshot = None
+
+ # Call LLM with message history (including temporary browser state message)
+ response = await self.llm.ainvoke(messages_to_send)
+
+ # Store usage stats from this LLM call
+ self._last_llm_usage = response.usage
+
+ # Log the LLM's raw output for debugging
+ logger.info(f'LLM Response:\n{response.completion}')
+
+ # Check for token limit or repetition issues
+ max_tokens = getattr(self.llm, 'max_tokens', None)
+ completion_tokens = response.usage.completion_tokens if response.usage else None
+ is_problematic, issue_message = detect_token_limit_issue(
+ completion=response.completion,
+ completion_tokens=completion_tokens,
+ max_tokens=max_tokens,
+ stop_reason=response.stop_reason,
+ )
+
+ if is_problematic:
+ logger.warning(f'Token limit issue detected: {issue_message}')
+ # Don't add the bad response to history
+ # Instead, inject a system message prompting recovery
+ recovery_prompt = (
+ f'Your previous response hit a token limit or became repetitive: {issue_message}\n\n'
+ 'Please write a SHORT plan (2 sentences) for what to do next, then execute ONE simple action.'
+ )
+ self._llm_messages.append(UserMessage(content=recovery_prompt))
+ # Return a controlled error message instead of corrupted code
+ return '', f'[Token limit error: {issue_message}]'
+
+ # Store the full response
+ full_response = response.completion
+
+ # Extract code blocks from response
+ # Support multiple code block types: python, js, bash, markdown
+ code_blocks = extract_code_blocks(response.completion)
+
+ # Inject non-python blocks into namespace as variables
+ # Track which variables are code blocks for browser state display
+ if '_code_block_vars' not in self.namespace:
+ self.namespace['_code_block_vars'] = set()
+
+ for block_type, block_content in code_blocks.items():
+ if not block_type.startswith('python'):
+ # Store js, bash, markdown blocks (and named variants) as variables in namespace
+ self.namespace[block_type] = block_content
+ self.namespace['_code_block_vars'].add(block_type)
+ print(f'→ Code block variable: {block_type} (str, {len(block_content)} chars)')
+ logger.debug(f'Injected {block_type} block into namespace ({len(block_content)} chars)')
+
+ # Store all code blocks for sequential execution
+ self.namespace['_all_code_blocks'] = code_blocks
+
+ # Get Python code if it exists
+ # If no python block exists and no other code blocks exist, return empty string to skip execution
+ # This prevents treating plain text explanations as code
+ code = code_blocks.get('python', response.completion)
+
+ # Add to LLM messages (truncate for history to save context)
+ truncated_completion = truncate_message_content(response.completion)
+ self._llm_messages.append(AssistantMessage(content=truncated_completion))
+
+ return code, full_response
+
+ def _print_variable_info(self, var_name: str, value: Any) -> None:
+ """Print compact info about a variable assignment."""
+ # Skip built-in modules and known imports
+ skip_names = {
+ 'json',
+ 'asyncio',
+ 'csv',
+ 're',
+ 'datetime',
+ 'Path',
+ 'pd',
+ 'np',
+ 'plt',
+ 'requests',
+ 'BeautifulSoup',
+ 'PdfReader',
+ 'browser',
+ 'file_system',
+ }
+ if var_name in skip_names:
+ return
+
+ # Skip code block variables (already printed)
+ if '_code_block_vars' in self.namespace and var_name in self.namespace.get('_code_block_vars', set()):
+ return
+
+ # Print compact variable info
+ if isinstance(value, (list, dict)):
+ preview = str(value)[:100]
+ print(f'→ Variable: {var_name} ({type(value).__name__}, len={len(value)}, preview={preview}...)')
+ elif isinstance(value, str) and len(value) > 50:
+ print(f'→ Variable: {var_name} (str, {len(value)} chars, preview={value[:50]}...)')
+ elif callable(value):
+ print(f'→ Variable: {var_name} (function)')
+ else:
+ print(f'→ Variable: {var_name} ({type(value).__name__}, value={repr(value)[:50]})')
+
+ async def _execute_code(self, code: str) -> tuple[str | None, str | None, str | None]:
+ """
+ Execute Python code in the namespace.
+
+ Args:
+ code: The Python code to execute
+
+ Returns:
+ Tuple of (output, error, browser_state)
+ """
+ # Create new cell
+ cell = self.session.add_cell(source=code)
+ cell.status = ExecutionStatus.RUNNING
+ cell.execution_count = self.session.increment_execution_count()
+
+ output = None
+ error = None
+ browser_state = None
+
+ try:
+ # Capture output
+ import ast
+ import io
+ import sys
+
+ old_stdout = sys.stdout
+ sys.stdout = io.StringIO()
+
+ try:
+ # Add asyncio to namespace if not already there
+ if 'asyncio' not in self.namespace:
+ self.namespace['asyncio'] = asyncio
+
+ # Store the current code in namespace for done() validation
+ self.namespace['_current_cell_code'] = code
+ # Store consecutive errors count for done() validation
+ self.namespace['_consecutive_errors'] = self._consecutive_errors
+
+ # Check if code contains await expressions - if so, wrap in async function
+ # This mimics how Jupyter/IPython handles top-level await
+ try:
+ tree = ast.parse(code, mode='exec')
+ has_await = any(isinstance(node, (ast.Await, ast.AsyncWith, ast.AsyncFor)) for node in ast.walk(tree))
+ except SyntaxError:
+ # If parse fails, let exec handle the error
+ has_await = False
+
+ if has_await:
+ # When code has await, we must wrap in async function
+ # To make variables persist naturally (like Jupyter without needing 'global'):
+ # 1. Extract all assigned variable names from the code
+ # 2. Inject 'global' declarations for variables that already exist in namespace
+ # 3. Extract user's explicit global declarations and pre-define those vars
+ # 4. Return locals() so we can update namespace with new variables
+
+ # Find all variable names being assigned + user's explicit globals
+ try:
+ assigned_names = set()
+ user_global_names = set()
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Assign):
+ for target in node.targets:
+ if isinstance(target, ast.Name):
+ assigned_names.add(target.id)
+ elif isinstance(node, ast.AugAssign) and isinstance(node.target, ast.Name):
+ assigned_names.add(node.target.id)
+ elif isinstance(node, (ast.AnnAssign, ast.NamedExpr)):
+ if hasattr(node, 'target') and isinstance(node.target, ast.Name):
+ assigned_names.add(node.target.id)
+ elif isinstance(node, ast.Global):
+ # Track user's explicit global declarations
+ user_global_names.update(node.names)
+
+ # Pre-define any user-declared globals that don't exist yet
+ # This prevents NameError when user writes "global foo" before "foo = ..."
+ for name in user_global_names:
+ if name not in self.namespace:
+ self.namespace[name] = None
+
+ # Filter to only existing namespace vars (like Jupyter does)
+ # Include both: assigned vars that exist + user's explicit globals
+ existing_vars = {name for name in (assigned_names | user_global_names) if name in self.namespace}
+ except Exception as e:
+ existing_vars = set()
+
+ # Build global declaration if needed
+ global_decl = ''
+ has_global_decl = False
+ if existing_vars:
+ vars_str = ', '.join(sorted(existing_vars))
+ global_decl = f' global {vars_str}\n'
+ has_global_decl = True
+
+ indented_code = '\n'.join(' ' + line if line.strip() else line for line in code.split('\n'))
+ wrapped_code = f"""async def __code_exec__():
+{global_decl}{indented_code}
+ # Return locals so we can update the namespace
+ return locals()
+
+__code_exec_coro__ = __code_exec__()
+"""
+ # Store whether we added a global declaration (needed for error line mapping)
+ self.namespace['_has_global_decl'] = has_global_decl
+
+ # Compile and execute wrapper at module level
+ compiled_code = compile(wrapped_code, '<code>', 'exec')
+ exec(compiled_code, self.namespace, self.namespace)
+
+ # Get and await the coroutine, then update namespace with new/modified variables
+ coro = self.namespace.get('__code_exec_coro__')
+ if coro:
+ result_locals = await coro
+ # Update namespace with all variables from the function's locals
+ # This makes variable assignments persist across cells
+ if result_locals:
+ for key, value in result_locals.items():
+ if not key.startswith('_'):
+ self.namespace[key] = value
+ # Variable info is tracked in "Available" section, no need for verbose inline output
+
+ # Clean up temporary variables
+ self.namespace.pop('__code_exec_coro__', None)
+ self.namespace.pop('__code_exec__', None)
+ else:
+ # No await - execute directly at module level for natural variable scoping
+ # This means x = x + 10 will work without needing 'global x'
+
+ # Track variables before execution
+ vars_before = set(self.namespace.keys())
+
+ compiled_code = compile(code, '<code>', 'exec')
+ exec(compiled_code, self.namespace, self.namespace)
+
+ # Track newly created/modified variables (info shown in "Available" section)
+ vars_after = set(self.namespace.keys())
+ new_vars = vars_after - vars_before
+
+ # Get output
+ output_value = sys.stdout.getvalue()
+ if output_value:
+ output = output_value
+
+ finally:
+ sys.stdout = old_stdout
+
+ # Wait 2 seconds for page to stabilize after code execution
+ await asyncio.sleep(0.5)
+
+ # Note: Browser state is now fetched right before LLM call instead of after each execution
+ # This reduces unnecessary state fetches for operations that don't affect the browser
+
+ cell.status = ExecutionStatus.SUCCESS
+ cell.output = output
+ cell.browser_state = None # Will be captured in next iteration before LLM call
+
+ except Exception as e:
+ # Handle EvaluateError specially - JavaScript execution failed
+ if isinstance(e, EvaluateError):
+ error = str(e)
+ cell.status = ExecutionStatus.ERROR
+ cell.error = error
+ logger.error(f'Code execution error: {error}')
+
+ await asyncio.sleep(1)
+
+ # Browser state will be fetched before next LLM call
+ # Return immediately - do not continue executing code
+ return output, error, None
+
+ # Handle NameError specially - check for code block variable confusion
+ if isinstance(e, NameError):
+ error_msg = str(e)
+ cell.status = ExecutionStatus.ERROR
+ cell.error = error
+
+ # Browser state will be fetched before next LLM call
+ await asyncio.sleep(0.5)
+ return output, error, None
+
+ # For syntax errors and common parsing errors, show just the error message
+ # without the full traceback to keep output clean
+ if isinstance(e, SyntaxError):
+ error_msg = e.msg if e.msg else str(e)
+ error = f'{type(e).__name__}: {error_msg}'
+
+ # Detect common f-string issues with JSON/JavaScript code
+ if 'unterminated' in error_msg.lower() and 'string' in error_msg.lower() and code:
+ # Check if code contains f-strings with potential JSON/JS content
+ has_fstring = bool(re.search(r'\bf["\']', code))
+ has_json_pattern = bool(re.search(r'json\.dumps|"[^"]*\{[^"]*\}[^"]*"|\'[^\']*\{[^\']*\}[^\']*\'', code))
+ has_js_pattern = bool(re.search(r'evaluate\(|await evaluate', code))
+
+ if has_fstring and (has_json_pattern or has_js_pattern):
+ error += (
+ '\n\n💡 TIP: Detected f-string with JSON/JavaScript code containing {}.\n'
+ ' Use separate ```js or ```markdown blocks instead of f-strings to avoid escaping issues.\n'
+ ' If your code block needs ``` inside it, wrap with 4+ backticks: ````markdown code`\n'
+ )
+
+ # Detect and provide helpful hints for common string literal errors
+ if 'unterminated' in error_msg.lower() and 'string' in error_msg.lower():
+ # Detect what type of string literal is unterminated
+ is_triple = 'triple-quoted' in error_msg.lower()
+ msg_lower = error_msg.lower()
+
+ # Detect prefix type from error message
+ if 'f-string' in msg_lower and 'raw' in msg_lower:
+ prefix = 'rf or fr'
+ desc = 'raw f-string'
+ elif 'f-string' in msg_lower:
+ prefix = 'f'
+ desc = 'f-string'
+ elif 'raw' in msg_lower and 'bytes' in msg_lower:
+ prefix = 'rb or br'
+ desc = 'raw bytes'
+ elif 'raw' in msg_lower:
+ prefix = 'r'
+ desc = 'raw string'
+ elif 'bytes' in msg_lower:
+ prefix = 'b'
+ desc = 'bytes'
+ else:
+ prefix = ''
+ desc = 'string'
+
+ # Build hint based on triple-quoted vs single/double quoted
+ if is_triple:
+ if prefix:
+ hint = f"Hint: Unterminated {prefix}'''...''' or {prefix}\"\"\"...\"\" ({desc}). Check for missing closing quotes or unescaped quotes inside."
+ else:
+ hint = "Hint: Unterminated '''...''' or \"\"\"...\"\" detected. Check for missing closing quotes or unescaped quotes inside."
+ hint += '\n If you need ``` inside your string, use a ````markdown varname` code block with 4+ backticks instead.'
+ else:
+ if prefix:
+ hint = f'Hint: Unterminated {prefix}\'...\' or {prefix}"..." ({desc}). Check for missing closing quote or unescaped quotes inside.'
+ else:
+ hint = 'Hint: Unterminated \'...\' or "..." detected. Check for missing closing quote or unescaped quotes inside the string.'
+ error += f'\n{hint}'
+
+ # Show the problematic line from the code
+ if e.text:
+ error += f'\n{e.text}'
+ elif e.lineno and code:
+ # If e.text is empty, extract the line from the code
+ lines = code.split('\n')
+ if 0 < e.lineno <= len(lines):
+ error += f'\n{lines[e.lineno - 1]}'
+
+ else:
+ # For other errors, try to extract useful information
+ error_str = str(e)
+ error = f'{type(e).__name__}: {error_str}' if error_str else f'{type(e).__name__} occurred'
+
+ # For RuntimeError or other exceptions, try to extract traceback info
+ # to show which line in the user's code actually failed
+ if hasattr(e, '__traceback__'):
+ # Walk the traceback to find the frame with '<code>' filename
+ tb = e.__traceback__
+ user_code_lineno = None
+ while tb is not None:
+ frame = tb.tb_frame
+ if frame.f_code.co_filename == '<code>':
+ # Found the frame executing user code
+ # Get the line number from the traceback
+ user_code_lineno = tb.tb_lineno
+ break
+ tb = tb.tb_next
+
+ cell.status = ExecutionStatus.ERROR
+ cell.error = error
+ logger.error(f'Code execution error: {error}')
+
+ await asyncio.sleep(1)
+
+ # Browser state will be fetched before next LLM call
+
+ return output, error, None
+
+ async def _get_browser_state(self) -> tuple[str, str | None]:
+ """Get the current browser state as text with ultra-minimal DOM structure for code agents.
+
+ Returns:
+ Tuple of (browser_state_text, screenshot_base64)
+ """
+ if not self.browser_session or not self.dom_service:
+ return 'Browser state not available', None
+
+ try:
+ # Get full browser state including screenshot if use_vision is enabled
+ include_screenshot = True
+ state = await self.browser_session.get_browser_state_summary(include_screenshot=include_screenshot)
+
+ # Format browser state with namespace context
+ browser_state_text = await format_browser_state_for_llm(
+ state=state, namespace=self.namespace, browser_session=self.browser_session
+ )
+
+ screenshot = state.screenshot if include_screenshot else None
+ return browser_state_text, screenshot
+
+ except Exception as e:
+ logger.error(f'Failed to get browser state: {e}')
+ return f'Error getting browser state: {e}', None
+
+ def _format_execution_result(self, code: str, output: str | None, error: str | None, current_step: int | None = None) -> str:
+ """Format the execution result for the LLM (without browser state)."""
+ result = []
+
+ # Add step progress header if step number provided
+ if current_step is not None:
+ progress_header = f'Step {current_step}/{self.max_steps} executed'
+ # Add consecutive failure tracking if there are errors
+ if error and self._consecutive_errors > 0:
+ progress_header += f' | Consecutive failures: {self._consecutive_errors}/{self.max_failures}'
+ result.append(progress_header)
+
+ if error:
+ result.append(f'Error: {error}')
+
+ if output:
+ # Truncate output if too long
+ if len(output) > 10000:
+ output = output[:9950] + '\n[Truncated after 10000 characters]'
+ result.append(f'Output: {output}')
+ if len(result) == 0:
+ result.append('Executed')
+ return '\n'.join(result)
+
+ def _is_task_done(self) -> bool:
+ """Check if the task is marked as done in the namespace."""
+ # Check if 'done' was called by looking for a special marker in namespace
+ return self.namespace.get('_task_done', False)
+
+ async def _capture_screenshot(self, step_number: int) -> str | None:
+ """Capture and store screenshot for eval tracking."""
+ if not self.browser_session:
+ return None
+
+ try:
+ # Get browser state summary which includes screenshot
+ state = await self.browser_session.get_browser_state_summary(include_screenshot=True)
+ if state and state.screenshot:
+ # Store screenshot using screenshot service
+ screenshot_path = await self.screenshot_service.store_screenshot(state.screenshot, step_number)
+ return str(screenshot_path) if screenshot_path else None
+ except Exception as e:
+ logger.warning(f'Failed to capture screenshot for step {step_number}: {e}')
+ return None
+
+ async def _add_step_to_complete_history(
+ self,
+ model_output_code: str,
+ full_llm_response: str,
+ output: str | None,
+ error: str | None,
+ screenshot_path: str | None,
+ ) -> None:
+ """Add a step to complete_history using type-safe models."""
+ # Get current browser URL and title for state
+ url: str | None = None
+ title: str | None = None
+ if self.browser_session:
+ try:
+ url = await self.browser_session.get_current_page_url()
+ # Get title from browser
+ cdp_session = await self.browser_session.get_or_create_cdp_session()
+ result = await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={'expression': 'document.title', 'returnByValue': True},
+ session_id=cdp_session.session_id,
+ )
+ title = result.get('result', {}).get('value')
+ except Exception as e:
+ logger.debug(f'Failed to get browser URL/title for history: {e}')
+
+ # Check if this is a done result
+ is_done = self._is_task_done()
+
+ # Get self-reported success from done() call if task is done
+ self_reported_success: bool | None = None
+ if is_done:
+ task_success = self.namespace.get('_task_success')
+ self_reported_success = task_success if isinstance(task_success, bool) else None
+
+ # Create result entry using typed model
+ result_entry = CodeAgentResult(
+ extracted_content=output if output else None,
+ error=error if error else None,
+ is_done=is_done,
+ success=self_reported_success,
+ )
+
+ # Create state entry using typed model
+ state_entry = CodeAgentState(url=url, title=title, screenshot_path=screenshot_path)
+
+ # Create metadata entry using typed model
+ step_end_time = datetime.datetime.now().timestamp()
+ metadata_entry = CodeAgentStepMetadata(
+ input_tokens=self._last_llm_usage.prompt_tokens if self._last_llm_usage else None,
+ output_tokens=self._last_llm_usage.completion_tokens if self._last_llm_usage else None,
+ step_start_time=self._step_start_time,
+ step_end_time=step_end_time,
+ )
+
+ # Create model output entry using typed model (if there's code to track)
+ model_output_entry: CodeAgentModelOutput | None = None
+ if model_output_code or full_llm_response:
+ model_output_entry = CodeAgentModelOutput(
+ model_output=model_output_code if model_output_code else '',
+ full_response=full_llm_response if full_llm_response else '',
+ )
+
+ # Create history entry using typed model
+ history_entry = CodeAgentHistory(
+ model_output=model_output_entry,
+ result=[result_entry],
+ state=state_entry,
+ metadata=metadata_entry,
+ screenshot_path=screenshot_path, # Keep for backward compatibility
+ )
+
+ self.complete_history.append(history_entry)
+
+ def _log_agent_event(self, max_steps: int, agent_run_error: str | None = None) -> None:
+ """Send the agent event for this run to telemetry."""
+ from urllib.parse import urlparse
+
+ token_summary = self.token_cost_service.get_usage_tokens_for_model(self.llm.model)
+
+ # For CodeAgent, we don't have action history like Agent does
+ # Instead we track the code execution cells
+ action_history_data: list[list[dict[str, Any]] | None] = []
+ for step in self.complete_history:
+ # Extract code from model_output if available (type-safe access)
+ if step.model_output and step.model_output.model_output:
+ code = step.model_output.model_output
+ # Represent each code cell as a simple action entry
+ action_history_data.append([{'code_cell': True, 'code_length': len(code)}])
+ else:
+ action_history_data.append(None)
+
+ # Get final result from the last step or namespace (type-safe)
+ final_result: Any = self.namespace.get('_task_result')
+ final_result_str: str | None = final_result if isinstance(final_result, str) else None
+
+ # Get URLs visited from complete_history (type-safe access)
+ urls_visited: list[str] = []
+ for step in self.complete_history:
+ if step.state.url and step.state.url not in urls_visited:
+ urls_visited.append(step.state.url)
+
+ # Get errors from complete_history (type-safe access)
+ errors: list[str] = []
+ for step in self.complete_history:
+ for result in step.result:
+ if result.error:
+ errors.append(result.error)
+
+ # Determine success from task completion status (type-safe)
+ is_done = self._is_task_done()
+ task_success: Any = self.namespace.get('_task_success')
+ self_reported_success: bool | None = task_success if isinstance(task_success, bool) else (False if is_done else None)
+
+ self.telemetry.capture(
+ AgentTelemetryEvent(
+ task=self.task,
+ model=self.llm.model,
+ model_provider=self.llm.provider,
+ max_steps=max_steps,
+ max_actions_per_step=1, # CodeAgent executes one code cell per step
+ use_vision=self.use_vision,
+ version=self.version,
+ source=self.source,
+ cdp_url=urlparse(self.browser_session.cdp_url).hostname
+ if self.browser_session and self.browser_session.cdp_url
+ else None,
+ agent_type='code', # CodeAgent identifier
+ action_errors=errors,
+ action_history=action_history_data,
+ urls_visited=urls_visited,
+ steps=len(self.complete_history),
+ total_input_tokens=token_summary.prompt_tokens,
+ total_duration_seconds=sum(step.metadata.duration_seconds for step in self.complete_history if step.metadata),
+ success=self_reported_success,
+ final_result_response=final_result_str,
+ error_message=agent_run_error,
+ )
+ )
+
+ def screenshot_paths(self, n_last: int | None = None) -> list[str | None]:
+ """
+ Get screenshot paths from complete_history for eval system.
+
+ Args:
+ n_last: Optional number of last screenshots to return
+
+ Returns:
+ List of screenshot file paths (or None for missing screenshots)
+ """
+ paths = [step.screenshot_path for step in self.complete_history]
+
+ if n_last is not None:
+ return paths[-n_last:] if len(paths) > n_last else paths
+
+ return paths
+
+ @property
+ def message_manager(self) -> Any:
+ """
+ Compatibility property for eval system.
+ Returns a mock object with last_input_messages attribute.
+ """
+
+ class MockMessageManager:
+ def __init__(self, llm_messages: list[BaseMessage]) -> None:
+ # Convert code-use LLM messages to format expected by eval system
+ self.last_input_messages = llm_messages
+
+ return MockMessageManager(self._llm_messages)
+
+ @property
+ def history(self) -> Any:
+ """
+ Compatibility property for eval system.
+ Returns a mock AgentHistoryList object with history attribute containing complete_history.
+ This is what the eval system expects when it does: agent_history = agent.history
+ """
+
+ class DictToObject:
+ """Convert dict to object with attribute access for eval compatibility."""
+
+ def __init__(self, data: dict[str, Any]) -> None:
+ for key, value in data.items():
+ if isinstance(value, dict):
+ setattr(self, key, DictToObject(value))
+ elif isinstance(value, list):
+ setattr(self, key, [DictToObject(item) if isinstance(item, dict) else item for item in value])
+ else:
+ setattr(self, key, value)
+
+ def __getattr__(self, name: str) -> None:
+ """Provide safe attribute access with defaults for missing attributes."""
+ # Return None for missing attributes instead of raising AttributeError
+ # This handles cases where eval system checks attributes that CodeAgent doesn't set
+ return None
+
+ def model_dump(self) -> dict[str, Any]:
+ """Support model_dump() calls from eval system."""
+ result = {}
+ for key, value in self.__dict__.items():
+ if isinstance(value, DictToObject):
+ result[key] = value.model_dump()
+ elif isinstance(value, list):
+ result[key] = [item.model_dump() if isinstance(item, DictToObject) else item for item in value]
+ else:
+ result[key] = value
+ return result
+
+ def get_screenshot(self) -> str | None:
+ """Support get_screenshot() calls for state objects."""
+ # Load screenshot from disk and return as base64 string (matching BrowserStateHistory implementation)
+ if not hasattr(self, 'screenshot_path') or not self.screenshot_path:
+ return None
+
+ import base64
+ from pathlib import Path
+
+ path_obj = Path(self.screenshot_path)
+ if not path_obj.exists():
+ return None
+
+ try:
+ with open(path_obj, 'rb') as f:
+ screenshot_data = f.read()
+ return base64.b64encode(screenshot_data).decode('utf-8')
+ except Exception:
+ return None
+
+ class MockAgentHistoryList:
+ def __init__(self, complete_history: list[CodeAgentHistory], usage_summary: UsageSummary | None) -> None:
+ # Convert each CodeAgentHistory to dict, then to object with attribute access
+ self.history = [DictToObject(item.model_dump()) for item in complete_history]
+ # Use the provided usage summary
+ self.usage = usage_summary
+
+ return MockAgentHistoryList(self.complete_history, self.usage_summary)
+
+ async def close(self) -> None:
+ """Close the browser session."""
+ if self.browser_session:
+ # Check if we should close the browser based on keep_alive setting
+ if not self.browser_session.browser_profile.keep_alive:
+ await self.browser_session.kill()
+ else:
+ logger.debug('Browser keep_alive is True, not closing browser session')
+
+ async def __aenter__(self) -> 'CodeAgent':
+ """Async context manager entry."""
+ return self
+
+ async def __aexit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: Any) -> None:
+ """Async context manager exit."""
+ await self.close()
diff --git a/browser_use/code_use/system_prompt.md b/browser_use/code_use/system_prompt.md
new file mode 100644
index 000000000..a54351ccb
--- /dev/null
+++ b/browser_use/code_use/system_prompt.md
@@ -0,0 +1,574 @@
+# Coding Browser Agent - System Prompt
+
+You are created by browser-use for complex automated browser tasks.
+
+## Core Concept
+You execute Python code in a notebook like environment to control a browser and complete tasks.
+
+**Mental Model**: Write one code cell per step → Gets automatically executed → **you receive the new output + * in the next response you write the next code cell → Repeat.
+
+
+---
+
+## INPUT: What You See
+
+### Browser State Format
+- **URL & DOM**: Compressed DOM tree with interactive elements marked as `[i_123]`
+- **Loading Status**: Network requests currently pending (automatically filtered for ads/tracking)
+ - Shows URL, loading duration, and resource type for each pending request
+
+- **Element Markers**:
+ - `[i_123]` - Interactive elements (buttons, inputs, links)
+ - `|SHADOW(open/closed)|` - Shadow DOM boundaries (content auto-included)
+ - `|IFRAME|` or `|FRAME|` - Iframe boundaries (content auto-included)
+ - `|SCROLL|` - Scrollable containers
+
+### Execution Environment
+- **Variables persist** across steps (like Jupyter) - NEVER use `global` keyword - thats not needed we do the injection for you.
+- **Multiple code blocks in ONE response are COMBINED** - earlier blocks' variables available in later blocks
+- **8 consecutive errors = auto-termination**
+
+### Multi-Block Code Support
+Non-Python blocks are saved as string variables:
+- ````js extract_products` → saved to `extract_products` variable (named blocks)
+- ````markdown result_summary` → saved to `result_summary` variable
+- ````bash bash_code` → saved to `bash_code` variable
+
+Variable name matches exactly what you write after language name!
+
+**Nested Code Blocks**: If your code contains ``` inside it (e.g., markdown with code blocks), use 4+ backticks:
+- `````markdown fix_code` with ``` inside → use 4 backticks to wrap
+- ``````python complex_code` with ```` inside → use 5+ backticks to wrap
+
+---
+
+## OUTPUT: How You Respond
+
+### Response Format - Cell-by-Cell Execution
+
+**This is a Jupyter-like notebook environment**: Execute ONE code cell → See output + browser state → Execute next cell.
+
+[1 short sentence about previous step code result and new DOM]
+[1 short sentence about next step]
+
+```python
+# 1 cell of code here that will be executed
+print(results)
+```
+Stop generating and inspect the output before continuing.
+
+
+
+
+## TOOLS: Available Functions
+
+### 1. Navigation
+```python
+await navigate('https://example.com')
+await asyncio.sleep(1)
+```
+- **Auto-wait**: System automatically waits 1s if network requests are pending before showing you the state
+- Loaded fully? Check URL/DOM and **⏳ Loading** status in next browser state
+- If you see pending network requests in the state, consider waiting longer: `await asyncio.sleep(2)`
+- In your next browser state after navigation analyse the screenshot: Is data still loading? Do you expect more data? → Wait longer with.
+- All previous indices [i_index] become invalid after navigation
+
+**After navigate(), dismiss overlays**:
+```js dismiss_overlays
+(function(){
+ const dismissed = [];
+ ['button[id*="accept"]', '[class*="cookie"] button'].forEach(sel => {
+ document.querySelectorAll(sel).forEach(btn => {
+ if (btn.offsetParent !== null) {
+ btn.click();
+ dismissed.push('cookie');
+ }
+ });
+ });
+ document.dispatchEvent(new KeyboardEvent('keydown', {key: 'Escape', keyCode: 27}));
+ return dismissed.length > 0 ? dismissed : null;
+})()
+```
+
+```python
+dismissed = await evaluate(dismiss_overlays)
+if dismissed:
+ print(f"OK Dismissed: {dismissed}")
+```
+
+For web search use duckduckgo.com by default to avoid CAPTCHAS.
+If direct navigation is blocked by CAPTCHA or challenge that cannot be solved after one try, pivot to alternative methods: try alternative URLs for the same content, third-party aggregators (user intent has highest priority).
+
+### 2. Interactive Elements
+The index is the label inside your browser state [i_index] inside the element you want to interact with. Only use indices from the current state. After page changes these become invalid.
+```python
+await click(index=456) # accepts only index integer from browser state
+await input_text(index=456, text="hello", clear=True) # Clear False to append text
+await upload_file(index=789, path="/path/to/file.pdf")
+await dropdown_options(index=123)
+await select_dropdown(index=123, text="CA") # Text can be the element text or value.
+await scroll(down=True, pages=1.0, index=None) # Down=False to scroll up. Pages=10.0 to scroll 10 pages. Use Index to scroll in the container of this element.
+await send_keys(keys="Enter") # Use e.g. for Escape, Arrow keys, Page Up, Page Down, Home, End, etc.
+await switch(tab_id="a1b2") # Switch to a 4 character tab by id from the browser state.
+await close(tab_id="a1b2") # Close a tab by id from the browser state.
+await go_back() # Navigate back in the browser history.
+```
+
+Indices Work Only once. After page changes (click, navigation, DOM update), ALL indices `[i_*]` become invalid and must be re-queried.
+
+Do not do:
+```python
+link_indices = [456, 457, 458]
+for idx in link_indices:
+ await click(index=idx) # FAILS - indices stale after first click
+```
+
+RIGHT - Option 1 (Extract URLs first):
+```python
+links = await evaluate('(function(){ return Array.from(document.querySelectorAll("a.product")).map(a => a.href); })()')
+for url in links:
+ await navigate(url)
+ # extract data
+ await go_back()
+```
+
+
+### 3. get_selector_from_index(index: int) → str
+Get stable CSS selector for element with index `[i_456]`:
+
+```python
+import json
+selector = await get_selector_from_index(index=456)
+print(f"OK Selector: {selector}") # Always print for debugging!
+el_text = await evaluate(f'(function(){{ return document.querySelector({json.dumps(selector)}).textContent; }})()')
+```
+
+**When to use**:
+- Clicking same element type repeatedly (e.g., "Next" button in pagination)
+- Loops where DOM changes between iterations
+
+### 4. evaluate(js: str, variables: dict = None) → Python data
+Execute JavaScript, returns dict/list/str/number/bool/None.
+
+**ALWAYS use ```js blocks for anything beyond one-liners**:
+
+```js extract_products
+(function(){
+ return Array.from(document.querySelectorAll('.product')).map(p => ({
+ name: p.querySelector('.name')?.textContent,
+ price: p.querySelector('.price')?.textContent
+ }));
+})()
+```
+
+```python
+products = await evaluate(extract_products)
+print(f"Found {len(products)} products")
+```
+
+**Passing Python variables to JavaScript**:
+```js extract_data
+(function(params) {
+ const maxItems = params.max_items || 100;
+ return Array.from(document.querySelectorAll('.item'))
+ .slice(0, maxItems)
+ .map(item => ({name: item.textContent}));
+})
+```
+
+```python
+result = await evaluate(extract_data, variables={'max_items': 50})
+```
+
+**Key rules**:
+- Wrap in IIFE: `(function(){ ... })()`
+- For variables: use `(function(params){ ... })` without final `()`
+- NO JavaScript comments (`//` or `/* */`)
+- NO backticks (\`) inside code blocks
+- Use standard JS (NO jQuery)
+- Do optional checks - and print the results to help you debug.
+- Avoid complex queries where possible. Do all data processing in python.
+- Avoid syntax errors. For more complex data use json.dumps(data).
+
+### 5. done() - MANDATORY FINAL STEP
+Final Output with done(text:str, success:bool, files_to_display:list[str] = [])
+
+```python
+summary = "Successfully extracted 600 items on 40 pages and saved them to the results.json file."
+await done(
+ text=summary,
+ success=True,
+ files_to_display=['results.json', 'data.csv']
+)
+```
+
+**Rules**:
+1. `done()` must be the ONLY statement in this cell/response. In the steps before you must verify the final result.
+3. For structured data/code: write to files, use `files_to_display`
+4. For short tasks (<5 lines output): print directly in `done(text=...)`, skip file creation
+5. NEVER embed JSON/code blocks in markdown templates (breaks `.format()`). Instead use json.dumps(data) or + to concatenate strings.
+6. Set `success=False` if task impossible after many many different attempts
+
+
+---
+
+## HINTS: Common Patterns & Pitfalls
+
+### JavaScript Search > Scrolling
+Before scrolling 2+ times, use JS to search entire document:
+
+```js search_document
+(function(){
+ const fullText = document.body.innerText;
+ return {
+ found: fullText.includes('Balance Sheet'),
+ sampleText: fullText.substring(0, 200)
+ };
+})()
+```
+
+### Verify Search Results Loaded
+After search submission, ALWAYS verify results exist:
+
+```js verify_search_results
+(function(){
+ return document.querySelectorAll("[class*=\\"result\\"]").length;
+})()
+```
+
+```python
+await input_text(index=SEARCH_INPUT, text="query", clear=True)
+await send_keys(keys="Enter")
+await asyncio.sleep(1)
+
+result_count = await evaluate(verify_search_results)
+if result_count == 0:
+ print("Search failed, trying alternative")
+ await navigate(f"https://site.com/search?q={query.replace(' ', '+')}")
+else:
+ print(f"Search returned {result_count} results")
+```
+
+### Handle Dynamic/Obfuscated Classes
+Modern sites use hashed classes (`_30jeq3`). After 2 failures, switch strategy:
+In the exploration phase you can combine multiple in parallel with error handling to find the best approach quickly..
+
+**Strategy 1**: Extract by structure/position
+```js extract_products_by_structure
+(function(){
+ return Array.from(document.querySelectorAll('.product')).map(p => {
+ const link = p.querySelector('a[href*="/product/"]');
+ const priceContainer = p.querySelector('div:nth-child(3)');
+ return {
+ name: link?.textContent,
+ priceText: priceContainer?.textContent
+ };
+ });
+})()
+```
+
+**Strategy 2**: Extract all text, parse in Python with regex
+```python
+items = await evaluate(extract_products_by_structure)
+import re
+for item in items:
+ prices = re.findall(r'[$₹€][\d,]+', item['priceText'])
+ item['price'] = prices[0] if prices else None
+```
+
+**Strategy 3**: Debug by printing structure
+```js print_structure
+(function(){
+ const el = document.querySelector('.product');
+ return {
+ html: el?.outerHTML.substring(0, 500),
+ classes: Array.from(el?.querySelectorAll('*') || [])
+ .map(e => e.className)
+ .filter(c => c.includes('price'))
+ };
+})()
+```
+
+### Pagination: Try URL First
+**Priority order**:
+1. **Try URL parameters** (1 attempt): `?page=2`, `?p=2`, `?offset=20`, `/page/2/`
+2. **If URL fails, search & click the next page button**
+
+### Pre-Extraction Checklist
+First verify page is loaded and you set the filters/settings correctly:
+
+```js product_count
+(function(){
+ return document.querySelectorAll(".product").length;
+})()
+```
+
+```python
+print("=== Applying filters ===")
+await select_dropdown(index=789, text="Under $100")
+await click(index=567) # Apply button
+print("OK Filters applied")
+
+filtered_count = await evaluate(product_count)
+print(f"OK Page loaded with {filtered_count} products")
+```
+---
+
+## STRATEGY: Execution Flow
+
+### Phase 1: Exploration
+- Navigate to target URL
+- Dismiss overlays (cookies, modals)
+- Apply all filters/settings BEFORE extraction
+- Use JavaScript to search entire document for target content
+- Explore DOM structure with various small test extractions in parallel with error handling
+- Use try/except and null checks
+- Print sub-information to validate approach
+
+### Phase 2: Validation (Execute Cell-by-Cell!)
+- Write general extraction function
+- Test on small subset (1-5 items) with error handling
+- Verify data structure in Python
+- Check for missing/null fields
+- Print sample data
+- If extraction fails 2x, switch strategy
+
+### Phase 3: Batch Processing
+- Once strategy validated, increase batch size
+- Loop with explicit counters
+- Save incrementally to avoid data loss
+- Handle pagination (URL first, then buttons)
+- Track progress: `print(f"Page {i}: {len(items)} items. Total: {len(all_data)}")`
+- Check if it works and then increase the batch size.
+
+### Phase 4: Cleanup & Verification
+- Verify all required data collected
+- Filter duplicates
+- Missing fields / Data? -> change strategy and keep going.
+- Format/clean data in Python (NOT JavaScript)
+- Write to files (JSON/CSV)
+- Print final stats, but not all the data to avoid overwhelming the context.
+- Inspect the output and reason if this is exactly the user intent or if the user wants more.
+
+### Phase 5: Done
+- Verify task completion
+- Call `done()` with summary + `files_to_display`
+
+---
+
+## EXAMPLE: Complete Flow
+
+**Task**: Extract products from paginated e-commerce site, save to JSON
+
+### Step 1: Navigate + Dismiss Overlays
+
+```js page_loaded
+(function(){
+ return document.readyState === 'complete';
+})()
+```
+
+```python
+await navigate('https://example.com/products')
+await asyncio.sleep(2)
+loaded = await evaluate(page_loaded)
+if not loaded:
+ print("Page not loaded, trying again")
+ await asyncio.sleep(1)
+
+```
+### Receive current browser state after cell execution - analyse it.
+
+### Step 2: Dismiss Modals
+```js dismiss_overlays
+(function(){
+ document.querySelectorAll('button[id*="accept"]').forEach(b => b.click());
+ document.dispatchEvent(new KeyboardEvent('keydown', {key: 'Escape'}));
+ return 'dismissed';
+})()
+```
+
+```python
+await evaluate(dismiss_overlays)
+```
+
+### Step 3: Apply Filters
+```python
+await select_dropdown(index=123, text="Under $50")
+await click(index=456) # Apply filters button
+```
+
+### Step 4: Explore - Test Single Element
+```js test_single_element
+(function(){
+ const first = document.querySelector('.product');
+ return {
+ html: first?.outerHTML.substring(0, 300),
+ name: first?.querySelector('.name')?.textContent,
+ price: first?.querySelector('.price')?.textContent
+ };
+})()
+```
+
+```js find_heading_by_text
+(function(){
+ const headings = Array.from(document.querySelectorAll('h2, h3'));
+ const target = headings.find(h => h.textContent.includes('Full Year 2024'));
+ return target ? target.textContent : null;
+})()
+```
+
+```js find_element_by_text_content
+(function(){
+ const elements = Array.from(document.querySelectorAll('dt'));
+ const locationLabel = elements.find(el => el.textContent.includes('Location'));
+ const nextSibling = locationLabel?.nextElementSibling;
+ return nextSibling ? nextSibling.textContent : null;
+})()
+```
+
+```js get_product_urls
+(function(){
+ return Array.from(document.querySelectorAll('a[href*="product"]').slice(0, 10)).map(a => a.href);
+})()
+```
+
+```python
+# load more
+scroll(down=True, pages=3.0)
+await asyncio.sleep(0.5)
+scroll(down=False, pages=2.5)
+try:
+ list_of_urls = await evaluate(get_product_urls)
+ print(f"found {len(list_of_urls)} product urls, sample {list_of_urls[0] if list_of_urls else 'no urls found'}")
+except Exception as e:
+ # different strategies
+ print("Error: No elements found")
+try:
+ test = await evaluate(test_single_element)
+ print(f"Sample product: {test}")
+except Exception as e:
+ # different strategies
+ print(f"Error: {e}")
+```
+
+### Step 5: Write General Extraction Function
+```js extract_products
+(function(){
+ return Array.from(document.querySelectorAll('.product')).map(p => ({
+ name: p.querySelector('.name')?.textContent?.trim(),
+ price: p.querySelector('.price')?.textContent?.trim(),
+ url: p.querySelector('a')?.href
+ })).filter(p => p.name && p.price);
+})()
+```
+
+```python
+products_page1 = await evaluate(extract_products)
+print(f"Extracted {len(products_page1)} products from page 1: {products_page1[0] if products_page1 else 'no products found'}")
+```
+
+### Step 6: Test Pagination with URL
+```python
+await navigate('https://example.com/products?page=2')
+await asyncio.sleep(2)
+products_page2 = await evaluate(extract_products)
+if len(products_page2) > 0:
+ print("OK URL pagination works!")
+```
+
+### Step 7: Loop and Collect All Pages
+```python
+all_products = []
+page_num = 1
+
+while page_num <= 50:
+ url = f"https://example.com/products?page={page_num}"
+ await navigate(url)
+ await asyncio.sleep(3)
+
+ items = await evaluate(extract_products)
+ if len(items) == 0:
+ print(f"Page {page_num} empty - reached end")
+ break
+
+ all_products.extend(items)
+ print(f"Page {page_num}: {len(items)} items. Total: {len(all_products)}")
+ page_num += 1
+ # if you have to click in the loop use selector and not the interactive index, because they invalidate after navigation.
+```
+
+### Step 8: Clean Data & Deduplicate
+```python
+import re
+
+for product in all_products:
+ price_str = product['price']
+ price_clean = re.sub(r'[^0-9.]', '', price_str)
+ product['price_numeric'] = float(price_clean) if price_clean else None
+
+# deduplicate
+all_products = list(set(all_products))
+# number of prices
+valid_products = [p for p in all_products if p.get('price_numeric')]
+
+print(f"OK {len(valid_products)} valid products with prices")
+print(f"OK Cleaned {len(all_products)} products")
+print(f"Sample cleaned: {json.dumps(valid_products[0], indent=2) if valid_products else 'no products found'}")
+```
+
+### Step 9: Prepare output, write File & verify result
+
+
+```markdown summary
+# Product Extraction Complete
+
+Successfully extracted 100 products from 20 pages.
+
+Full data saved to: products.json.
+
+```
+```python
+
+with open('products.json', 'w', encoding='utf-8') as f:
+ json.dump(valid_products, f, indent=2, ensure_ascii=False)
+
+print(f"OK Wrote products.json ({len(valid_products)} products)")
+sample = json.dumps(valid_products[0], indent=2)
+
+# Be careful with escaping and always print before using done.
+final_summary = summary + "\nSample:\n" + sample
+print(summary)
+```
+
+### Stop and inspect the output before continuing.
+### If data is missing go back and change the strategy until all data is collected or you reach max steps.
+
+### Step 10: Done in single response (After verifying the previous output)
+
+
+```python
+await done(text=final_summary, success=True, files_to_display=['products.json'])
+```
+
+---
+
+## CRITICAL RULES
+
+1. **NO `global` keyword** - Variables persist automatically
+2. **No comments** in Python or JavaScript code, write concise code.
+3. **Verify results after search** - Check result count > 0
+4. **Call done(text, success) in separate step** - After verifying results - else continue
+5. **Write structured data to files** - Never embed in markdown
+6. Do not use jQuery.
+7. Reason about the browser state and what you need to keep in mind on this page. E.g. popups, dynamic content, closed shadow DOM, iframes, scroll to load more...
+8. If selectors fail, simply try different once. Print many and then try different strategies.
+---
+
+## Available Libraries
+**Pre-imported**: `json`, `asyncio`, `csv`, `re`, `datetime`, `Path`, `requests`
+
+
+## User Task
+Analyze user intent and complete the task successfully. Do not stop until completed.
+Respond in the format the user requested.
diff --git a/browser_use/code_use/utils.py b/browser_use/code_use/utils.py
new file mode 100644
index 000000000..8c00193fd
--- /dev/null
+++ b/browser_use/code_use/utils.py
@@ -0,0 +1,150 @@
+"""Utility functions for code-use agent."""
+
+import re
+
+
+def truncate_message_content(content: str, max_length: int = 10000) -> str:
+ """Truncate message content to max_length characters for history."""
+ if len(content) <= max_length:
+ return content
+ # Truncate and add marker
+ return content[:max_length] + f'\n\n[... truncated {len(content) - max_length} characters for history]'
+
+
+def detect_token_limit_issue(
+ completion: str,
+ completion_tokens: int | None,
+ max_tokens: int | None,
+ stop_reason: str | None,
+) -> tuple[bool, str | None]:
+ """
+ Detect if the LLM response hit token limits or is repetitive garbage.
+
+ Returns: (is_problematic, error_message)
+ """
+ # Check 1: Stop reason indicates max_tokens
+ if stop_reason == 'max_tokens':
+ return True, f'Response terminated due to max_tokens limit (stop_reason: {stop_reason})'
+
+ # Check 2: Used 90%+ of max_tokens (if we have both values)
+ if completion_tokens is not None and max_tokens is not None and max_tokens > 0:
+ usage_ratio = completion_tokens / max_tokens
+ if usage_ratio >= 0.9:
+ return True, f'Response used {usage_ratio:.1%} of max_tokens ({completion_tokens}/{max_tokens})'
+
+ # Check 3: Last 6 characters repeat 40+ times (repetitive garbage)
+ if len(completion) >= 6:
+ last_6 = completion[-6:]
+ repetition_count = completion.count(last_6)
+ if repetition_count >= 40:
+ return True, f'Repetitive output detected: last 6 chars "{last_6}" appears {repetition_count} times'
+
+ return False, None
+
+
+def extract_url_from_task(task: str) -> str | None:
+ """Extract URL from task string using naive pattern matching."""
+ # Remove email addresses from task before looking for URLs
+ task_without_emails = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '', task)
+
+ # Look for common URL patterns
+ patterns = [
+ r'https?://[^\s<>"\']+', # Full URLs with http/https
+ r'(?:www\.)?[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*\.[a-zA-Z]{2,}(?:/[^\s<>"\']*)?', # Domain names with subdomains and optional paths
+ ]
+
+ found_urls = []
+ for pattern in patterns:
+ matches = re.finditer(pattern, task_without_emails)
+ for match in matches:
+ url = match.group(0)
+
+ # Remove trailing punctuation that's not part of URLs
+ url = re.sub(r'[.,;:!?()\[\]]+$', '', url)
+ # Add https:// if missing
+ if not url.startswith(('http://', 'https://')):
+ url = 'https://' + url
+ found_urls.append(url)
+
+ unique_urls = list(set(found_urls))
+ # If multiple URLs found, skip auto-navigation to avoid ambiguity
+ if len(unique_urls) > 1:
+ return None
+
+ # If exactly one URL found, return it
+ if len(unique_urls) == 1:
+ return unique_urls[0]
+
+ return None
+
+
+def extract_code_blocks(text: str) -> dict[str, str]:
+ """Extract all code blocks from markdown response.
+
+ Supports:
+ - ```python, ```js, ```javascript, ```bash, ```markdown, ```md
+ - Named blocks: ```js variable_name → saved as 'variable_name' in namespace
+ - Nested blocks: Use 4+ backticks for outer block when inner content has 3 backticks
+
+ Returns dict mapping block_name -> content
+
+ Note: Python blocks are NO LONGER COMBINED. Each python block executes separately
+ to allow sequential execution with JS/bash blocks in between.
+ """
+ # Pattern to match code blocks with language identifier and optional variable name
+ # Matches: ```lang\n or ```lang varname\n or ````+lang\n (4+ backticks for nested blocks)
+ # Uses non-greedy matching and backreferences to match opening/closing backticks
+ pattern = r'(`{3,})(\w+)(?:\s+(\w+))?\n(.*?)\1(?:\n|$)'
+ matches = re.findall(pattern, text, re.DOTALL)
+
+ blocks: dict[str, str] = {}
+ python_block_counter = 0
+
+ for backticks, lang, var_name, content in matches:
+ lang = lang.lower()
+
+ # Normalize language names
+ if lang in ('javascript', 'js'):
+ lang_normalized = 'js'
+ elif lang in ('markdown', 'md'):
+ lang_normalized = 'markdown'
+ elif lang in ('sh', 'shell'):
+ lang_normalized = 'bash'
+ elif lang == 'python':
+ lang_normalized = 'python'
+ else:
+ # Unknown language, skip
+ continue
+
+ # Only process supported types
+ if lang_normalized in ('python', 'js', 'bash', 'markdown'):
+ content = content.rstrip() # Only strip trailing whitespace, preserve leading for indentation
+ if content:
+ # Determine the key to use
+ if var_name:
+ # Named block - use the variable name
+ block_key = var_name
+ blocks[block_key] = content
+ elif lang_normalized == 'python':
+ # Unnamed Python blocks - give each a unique key to preserve order
+ block_key = f'python_{python_block_counter}'
+ blocks[block_key] = content
+ python_block_counter += 1
+ else:
+ # Other unnamed blocks (js, bash, markdown) - keep last one only
+ blocks[lang_normalized] = content
+
+ # If we have multiple python blocks, mark the first one as 'python' for backward compat
+ if python_block_counter > 0:
+ blocks['python'] = blocks['python_0']
+
+ # Fallback: if no python block but there's generic ``` block, treat as python
+ if python_block_counter == 0 and 'python' not in blocks:
+ generic_pattern = r'```\n(.*?)```'
+ generic_matches = re.findall(generic_pattern, text, re.DOTALL)
+ if generic_matches:
+ combined = '\n\n'.join(m.strip() for m in generic_matches if m.strip())
+ if combined:
+ blocks['python'] = combined
+
+ return blocks
diff --git a/browser_use/code_use/views.py b/browser_use/code_use/views.py
new file mode 100644
index 000000000..04da0708a
--- /dev/null
+++ b/browser_use/code_use/views.py
@@ -0,0 +1,171 @@
+"""Data models for code-use mode."""
+
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+from uuid_extensions import uuid7str
+
+
+class CellType(str, Enum):
+ """Type of notebook cell."""
+
+ CODE = 'code'
+ MARKDOWN = 'markdown'
+
+
+class ExecutionStatus(str, Enum):
+ """Execution status of a cell."""
+
+ PENDING = 'pending'
+ RUNNING = 'running'
+ SUCCESS = 'success'
+ ERROR = 'error'
+
+
+class CodeCell(BaseModel):
+ """Represents a code cell in the notebook-like execution."""
+
+ model_config = ConfigDict(extra='forbid')
+
+ id: str = Field(default_factory=uuid7str)
+ cell_type: CellType = CellType.CODE
+ source: str = Field(description='The code to execute')
+ output: str | None = Field(default=None, description='The output of the code execution')
+ execution_count: int | None = Field(default=None, description='The execution count')
+ status: ExecutionStatus = Field(default=ExecutionStatus.PENDING)
+ error: str | None = Field(default=None, description='Error message if execution failed')
+ browser_state: str | None = Field(default=None, description='Browser state after execution')
+
+
+class NotebookSession(BaseModel):
+ """Represents a notebook-like session."""
+
+ model_config = ConfigDict(extra='forbid')
+
+ id: str = Field(default_factory=uuid7str)
+ cells: list[CodeCell] = Field(default_factory=list)
+ current_execution_count: int = Field(default=0)
+ namespace: dict[str, Any] = Field(default_factory=dict, description='Current namespace state')
+
+ def add_cell(self, source: str) -> CodeCell:
+ """Add a new code cell to the session."""
+ cell = CodeCell(source=source)
+ self.cells.append(cell)
+ return cell
+
+ def get_cell(self, cell_id: str) -> CodeCell | None:
+ """Get a cell by ID."""
+ for cell in self.cells:
+ if cell.id == cell_id:
+ return cell
+ return None
+
+ def get_latest_cell(self) -> CodeCell | None:
+ """Get the most recently added cell."""
+ if self.cells:
+ return self.cells[-1]
+ return None
+
+ def increment_execution_count(self) -> int:
+ """Increment and return the execution count."""
+ self.current_execution_count += 1
+ return self.current_execution_count
+
+
+class NotebookExport(BaseModel):
+ """Export format for Jupyter notebook."""
+
+ model_config = ConfigDict(extra='forbid')
+
+ nbformat: int = Field(default=4)
+ nbformat_minor: int = Field(default=5)
+ metadata: dict[str, Any] = Field(default_factory=dict)
+ cells: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class CodeAgentModelOutput(BaseModel):
+ """Model output for CodeAgent - contains the code and full LLM response."""
+
+ model_config = ConfigDict(extra='forbid')
+
+ model_output: str = Field(description='The extracted code from the LLM response')
+ full_response: str = Field(description='The complete LLM response including any text/reasoning')
+
+
+class CodeAgentResult(BaseModel):
+ """Result of executing a code cell in CodeAgent."""
+
+ model_config = ConfigDict(extra='forbid')
+
+ extracted_content: str | None = Field(default=None, description='Output from code execution')
+ error: str | None = Field(default=None, description='Error message if execution failed')
+ is_done: bool = Field(default=False, description='Whether task is marked as done')
+ success: bool | None = Field(default=None, description='Self-reported success from done() call')
+
+
+class CodeAgentState(BaseModel):
+ """State information for a CodeAgent step."""
+
+ model_config = ConfigDict(extra='forbid', arbitrary_types_allowed=True)
+
+ url: str | None = Field(default=None, description='Current page URL')
+ title: str | None = Field(default=None, description='Current page title')
+ screenshot_path: str | None = Field(default=None, description='Path to screenshot file')
+
+ def get_screenshot(self) -> str | None:
+ """Load screenshot from disk and return as base64 string."""
+ if not self.screenshot_path:
+ return None
+
+ import base64
+ from pathlib import Path
+
+ path_obj = Path(self.screenshot_path)
+ if not path_obj.exists():
+ return None
+
+ try:
+ with open(path_obj, 'rb') as f:
+ screenshot_data = f.read()
+ return base64.b64encode(screenshot_data).decode('utf-8')
+ except Exception:
+ return None
+
+
+class CodeAgentStepMetadata(BaseModel):
+ """Metadata for a single CodeAgent step including timing and token information."""
+
+ model_config = ConfigDict(extra='forbid')
+
+ input_tokens: int | None = Field(default=None, description='Number of input tokens used')
+ output_tokens: int | None = Field(default=None, description='Number of output tokens used')
+ step_start_time: float = Field(description='Step start timestamp (Unix time)')
+ step_end_time: float = Field(description='Step end timestamp (Unix time)')
+
+ @property
+ def duration_seconds(self) -> float:
+ """Calculate step duration in seconds."""
+ return self.step_end_time - self.step_start_time
+
+
+class CodeAgentHistory(BaseModel):
+ """History item for CodeAgent actions."""
+
+ model_config = ConfigDict(extra='forbid', arbitrary_types_allowed=True)
+
+ model_output: CodeAgentModelOutput | None = Field(default=None, description='LLM output for this step')
+ result: list[CodeAgentResult] = Field(default_factory=list, description='Results from code execution')
+ state: CodeAgentState = Field(description='Browser state at this step')
+ metadata: CodeAgentStepMetadata | None = Field(default=None, description='Step timing and token metadata')
+ screenshot_path: str | None = Field(default=None, description='Legacy field for screenshot path')
+
+ def model_dump(self, **kwargs) -> dict[str, Any]:
+ """Custom serialization for CodeAgentHistory."""
+ return {
+ 'model_output': self.model_output.model_dump() if self.model_output else None,
+ 'result': [r.model_dump() for r in self.result],
+ 'state': self.state.model_dump(),
+ 'metadata': self.metadata.model_dump() if self.metadata else None,
+ 'screenshot_path': self.screenshot_path,
+ }
diff --git a/browser_use/dom/serializer/code_use_serializer.py b/browser_use/dom/serializer/code_use_serializer.py
new file mode 100644
index 000000000..b127b576b
--- /dev/null
+++ b/browser_use/dom/serializer/code_use_serializer.py
@@ -0,0 +1,287 @@
+# @file purpose: Ultra-compact serializer optimized for code-use agents
+# Focuses on minimal token usage while preserving essential interactive context
+
+from browser_use.dom.utils import cap_text_length
+from browser_use.dom.views import (
+ EnhancedDOMTreeNode,
+ NodeType,
+ SimplifiedNode,
+)
+
+# Minimal but sufficient attribute list for code agents
+CODE_USE_KEY_ATTRIBUTES = [
+ 'id', # Essential for element selection
+ 'name', # For form inputs
+ 'type', # For input types
+ 'placeholder', # For empty inputs
+ 'aria-label', # For buttons without text
+ 'value', # Current values
+ 'alt', # For images
+ 'class', # Keep top 2 classes for common selectors
+]
+
+# Interactive elements agent can use
+INTERACTIVE_ELEMENTS = {
+ 'a',
+ 'button',
+ 'input',
+ 'textarea',
+ 'select',
+ 'form',
+}
+
+# Semantic structure elements - expanded to include more content containers
+SEMANTIC_STRUCTURE = {
+ 'h1',
+ 'h2',
+ 'h3',
+ 'h4',
+ 'h5',
+ 'h6',
+ 'nav',
+ 'main',
+ 'header',
+ 'footer',
+ 'article',
+ 'section',
+ 'p', # Paragraphs often contain prices and product info
+ 'span', # Spans often contain prices and labels
+ 'div', # Divs with useful attributes (id/class) should be shown
+ 'ul',
+ 'ol',
+ 'li',
+ 'label',
+ 'img',
+}
+
+
+class DOMCodeAgentSerializer:
+ """Optimized DOM serializer for code-use agents - balances token efficiency with context."""
+
+ @staticmethod
+ def serialize_tree(node: SimplifiedNode | None, include_attributes: list[str], depth: int = 0) -> str:
+ """
+ Serialize DOM tree with smart token optimization.
+
+ Strategy:
+ - Keep top 2 CSS classes for querySelector compatibility
+ - Show div/span/p elements with useful attributes or text
+ - Show all interactive + semantic elements
+ - Inline text up to 80 chars for better context
+ """
+ if not node:
+ return ''
+
+ # Skip excluded/hidden nodes
+ if hasattr(node, 'excluded_by_parent') and node.excluded_by_parent:
+ return DOMCodeAgentSerializer._serialize_children(node, include_attributes, depth)
+
+ if not node.should_display:
+ return DOMCodeAgentSerializer._serialize_children(node, include_attributes, depth)
+
+ formatted_text = []
+ depth_str = ' ' * depth # Use 2 spaces instead of tabs for compactness
+
+ if node.original_node.node_type == NodeType.ELEMENT_NODE:
+ tag = node.original_node.tag_name.lower()
+ is_visible = node.original_node.snapshot_node and node.original_node.is_visible
+
+ # Skip invisible (except iframes)
+ if not is_visible and tag not in ['iframe', 'frame']:
+ return DOMCodeAgentSerializer._serialize_children(node, include_attributes, depth)
+
+ # Special handling for iframes
+ if tag in ['iframe', 'frame']:
+ return DOMCodeAgentSerializer._serialize_iframe(node, include_attributes, depth)
+
+ # Build minimal attributes
+ attributes_str = DOMCodeAgentSerializer._build_minimal_attributes(node.original_node)
+
+ # Decide if element should be shown
+ is_interactive = tag in INTERACTIVE_ELEMENTS
+ is_semantic = tag in SEMANTIC_STRUCTURE
+ has_useful_attrs = bool(attributes_str)
+ has_text = DOMCodeAgentSerializer._has_direct_text(node)
+
+ # Skip non-semantic, non-interactive containers without attributes
+ if not is_interactive and not is_semantic and not has_useful_attrs and not has_text:
+ return DOMCodeAgentSerializer._serialize_children(node, include_attributes, depth)
+
+ # Collapse pointless wrappers
+ if tag in {'div', 'span'} and not has_useful_attrs and not has_text and len(node.children) == 1:
+ return DOMCodeAgentSerializer._serialize_children(node, include_attributes, depth)
+
+ # Build element
+ line = f'{depth_str}<{tag}'
+
+ if attributes_str:
+ line += f' {attributes_str}'
+
+ # Inline text
+ inline_text = DOMCodeAgentSerializer._get_inline_text(node)
+ if inline_text:
+ line += f'>{inline_text}'
+ else:
+ line += '>'
+
+ formatted_text.append(line)
+
+ # Children (only if no inline text)
+ if node.children and not inline_text:
+ children_text = DOMCodeAgentSerializer._serialize_children(node, include_attributes, depth + 1)
+ if children_text:
+ formatted_text.append(children_text)
+
+ elif node.original_node.node_type == NodeType.TEXT_NODE:
+ # Handled inline with parent
+ pass
+
+ elif node.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
+ # Shadow DOM - minimal marker
+ if node.children:
+ formatted_text.append(f'{depth_str}#shadow')
+ children_text = DOMCodeAgentSerializer._serialize_children(node, include_attributes, depth + 1)
+ if children_text:
+ formatted_text.append(children_text)
+
+ return '\n'.join(formatted_text)
+
+ @staticmethod
+ def _serialize_children(node: SimplifiedNode, include_attributes: list[str], depth: int) -> str:
+ """Serialize children."""
+ children_output = []
+ for child in node.children:
+ child_text = DOMCodeAgentSerializer.serialize_tree(child, include_attributes, depth)
+ if child_text:
+ children_output.append(child_text)
+ return '\n'.join(children_output)
+
+ @staticmethod
+ def _build_minimal_attributes(node: EnhancedDOMTreeNode) -> str:
+ """Build minimal but useful attributes - keep top 2 classes for selectors."""
+ attrs = []
+
+ if node.attributes:
+ for attr in CODE_USE_KEY_ATTRIBUTES:
+ if attr in node.attributes:
+ value = str(node.attributes[attr]).strip()
+ if value:
+ # Special handling for class - keep only first 2 classes
+ if attr == 'class':
+ classes = value.split()[:2]
+ value = ' '.join(classes)
+ # Cap at 25 chars
+ value = cap_text_length(value, 25)
+ attrs.append(f'{attr}="{value}"')
+
+ return ' '.join(attrs)
+
+ @staticmethod
+ def _has_direct_text(node: SimplifiedNode) -> bool:
+ """Check if node has direct text children."""
+ for child in node.children:
+ if child.original_node.node_type == NodeType.TEXT_NODE:
+ text = child.original_node.node_value.strip() if child.original_node.node_value else ''
+ if len(text) > 1:
+ return True
+ return False
+
+ @staticmethod
+ def _get_inline_text(node: SimplifiedNode) -> str:
+ """Get inline text (max 80 chars for better context)."""
+ text_parts = []
+ for child in node.children:
+ if child.original_node.node_type == NodeType.TEXT_NODE:
+ text = child.original_node.node_value.strip() if child.original_node.node_value else ''
+ if text and len(text) > 1:
+ text_parts.append(text)
+
+ if not text_parts:
+ return ''
+
+ combined = ' '.join(text_parts)
+ return cap_text_length(combined, 40)
+
+ @staticmethod
+ def _serialize_iframe(node: SimplifiedNode, include_attributes: list[str], depth: int) -> str:
+ """Handle iframe minimally."""
+ formatted_text = []
+ depth_str = ' ' * depth
+ tag = node.original_node.tag_name.lower()
+
+ # Minimal iframe marker
+ attributes_str = DOMCodeAgentSerializer._build_minimal_attributes(node.original_node)
+ line = f'{depth_str}<{tag}'
+ if attributes_str:
+ line += f' {attributes_str}'
+ line += '>'
+ formatted_text.append(line)
+
+ # Iframe content
+ if node.original_node.content_document:
+ formatted_text.append(f'{depth_str} #iframe-content')
+
+ # Find and serialize body content only
+ for child_node in node.original_node.content_document.children_nodes or []:
+ if child_node.tag_name.lower() == 'html':
+ for html_child in child_node.children:
+ if html_child.tag_name.lower() == 'body':
+ for body_child in html_child.children:
+ DOMCodeAgentSerializer._serialize_document_node(
+ body_child, formatted_text, include_attributes, depth + 2
+ )
+ break
+
+ return '\n'.join(formatted_text)
+
+ @staticmethod
+ def _serialize_document_node(
+ dom_node: EnhancedDOMTreeNode, output: list[str], include_attributes: list[str], depth: int
+ ) -> None:
+ """Serialize document node without SimplifiedNode wrapper."""
+ depth_str = ' ' * depth
+
+ if dom_node.node_type == NodeType.ELEMENT_NODE:
+ tag = dom_node.tag_name.lower()
+
+ # Skip invisible
+ is_visible = dom_node.snapshot_node and dom_node.is_visible
+ if not is_visible:
+ return
+
+ # Check if worth showing
+ is_interactive = tag in INTERACTIVE_ELEMENTS
+ is_semantic = tag in SEMANTIC_STRUCTURE
+ attributes_str = DOMCodeAgentSerializer._build_minimal_attributes(dom_node)
+
+ if not is_interactive and not is_semantic and not attributes_str:
+ # Skip but process children
+ for child in dom_node.children:
+ DOMCodeAgentSerializer._serialize_document_node(child, output, include_attributes, depth)
+ return
+
+ # Build element
+ line = f'{depth_str}<{tag}'
+ if attributes_str:
+ line += f' {attributes_str}'
+
+ # Get text
+ text_parts = []
+ for child in dom_node.children:
+ if child.node_type == NodeType.TEXT_NODE and child.node_value:
+ text = child.node_value.strip()
+ if text and len(text) > 1:
+ text_parts.append(text)
+
+ if text_parts:
+ combined = ' '.join(text_parts)
+ line += f'>{cap_text_length(combined, 25)}'
+ else:
+ line += '>'
+
+ output.append(line)
+
+ # Process non-text children
+ for child in dom_node.children:
+ if child.node_type != NodeType.TEXT_NODE:
+ DOMCodeAgentSerializer._serialize_document_node(child, output, include_attributes, depth + 1)
diff --git a/browser_use/dom/serializer/eval_serializer.py b/browser_use/dom/serializer/eval_serializer.py
new file mode 100644
index 000000000..4f349699f
--- /dev/null
+++ b/browser_use/dom/serializer/eval_serializer.py
@@ -0,0 +1,478 @@
+# @file purpose: Concise evaluation serializer for DOM trees - optimized for LLM query writing
+
+
+from browser_use.dom.utils import cap_text_length
+from browser_use.dom.views import (
+ EnhancedDOMTreeNode,
+ NodeType,
+ SimplifiedNode,
+)
+
+# Critical attributes for query writing and form interaction
+# NOTE: Removed 'id' and 'class' to force more robust structural selectors
+EVAL_KEY_ATTRIBUTES = [
+ 'id', # Removed - can have special chars, forces structural selectors
+ 'class', # Removed - can have special chars like +, forces structural selectors
+ 'name',
+ 'type',
+ 'placeholder',
+ 'aria-label',
+ 'role',
+ 'value',
+ # 'href',
+ 'data-testid',
+ 'alt', # for images
+ 'title', # useful for tooltips/link context
+ # State attributes (critical for form interaction)
+ 'checked',
+ 'selected',
+ 'disabled',
+ 'required',
+ 'readonly',
+ # ARIA states
+ 'aria-expanded',
+ 'aria-pressed',
+ 'aria-checked',
+ 'aria-selected',
+ 'aria-invalid',
+ # Validation attributes (help agents avoid brute force)
+ 'pattern',
+ 'min',
+ 'max',
+ 'minlength',
+ 'maxlength',
+ 'step',
+ 'aria-valuemin',
+ 'aria-valuemax',
+ 'aria-valuenow',
+]
+
+# Semantic elements that should always be shown
+SEMANTIC_ELEMENTS = {
+ 'html', # Always show document root
+ 'body', # Always show body
+ 'h1',
+ 'h2',
+ 'h3',
+ 'h4',
+ 'h5',
+ 'h6',
+ 'a',
+ 'button',
+ 'input',
+ 'textarea',
+ 'select',
+ 'form',
+ 'label',
+ 'nav',
+ 'header',
+ 'footer',
+ 'main',
+ 'article',
+ 'section',
+ 'table',
+ 'thead',
+ 'tbody',
+ 'tr',
+ 'th',
+ 'td',
+ 'ul',
+ 'ol',
+ 'li',
+ 'img',
+ 'iframe',
+ 'video',
+ 'audio',
+}
+
+# Container elements that can be collapsed if they only wrap one child
+COLLAPSIBLE_CONTAINERS = {'div', 'span', 'section', 'article'}
+
+# SVG child elements to skip (decorative only, no interaction value)
+SVG_ELEMENTS = {
+ 'path',
+ 'rect',
+ 'g',
+ 'circle',
+ 'ellipse',
+ 'line',
+ 'polyline',
+ 'polygon',
+ 'use',
+ 'defs',
+ 'clipPath',
+ 'mask',
+ 'pattern',
+ 'image',
+ 'text',
+ 'tspan',
+}
+
+
+class DOMEvalSerializer:
+ """Ultra-concise DOM serializer for quick LLM query writing."""
+
+ @staticmethod
+ def serialize_tree(node: SimplifiedNode | None, include_attributes: list[str], depth: int = 0) -> str:
+ """
+ Serialize complete DOM tree structure for LLM understanding.
+
+ Strategy:
+ - Show ALL elements to preserve DOM structure
+ - Non-interactive elements show just tag name
+ - Interactive elements show full attributes + [index]
+ - Self-closing tags only (no closing tags)
+ """
+ if not node:
+ return ''
+
+ # Skip excluded nodes but process children
+ if hasattr(node, 'excluded_by_parent') and node.excluded_by_parent:
+ return DOMEvalSerializer._serialize_children(node, include_attributes, depth)
+
+ # Skip nodes marked as should_display=False
+ if not node.should_display:
+ return DOMEvalSerializer._serialize_children(node, include_attributes, depth)
+
+ formatted_text = []
+ depth_str = depth * '\t'
+
+ if node.original_node.node_type == NodeType.ELEMENT_NODE:
+ tag = node.original_node.tag_name.lower()
+ is_visible = node.original_node.snapshot_node and node.original_node.is_visible
+
+ # Container elements that should be shown even if invisible (might have visible children)
+ container_tags = {'html', 'body', 'div', 'main', 'section', 'article', 'aside', 'header', 'footer', 'nav'}
+
+ # Skip invisible elements UNLESS they're containers or iframes (which might have visible children)
+ if not is_visible and tag not in container_tags and tag not in ['iframe', 'frame']:
+ return DOMEvalSerializer._serialize_children(node, include_attributes, depth)
+
+ # Special handling for iframes - show them with their content
+ if tag in ['iframe', 'frame']:
+ return DOMEvalSerializer._serialize_iframe(node, include_attributes, depth)
+
+ # Skip SVG elements entirely - they're just decorative graphics with no interaction value
+ # Show the <svg> tag itself to indicate graphics, but don't recurse into children
+ if tag == 'svg':
+ line = f'{depth_str}'
+ # Add [i_X] for interactive SVG elements only
+ if node.interactive_index is not None:
+ line += f'[i_{node.original_node.backend_node_id}] '
+ line += '<svg'
+ attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)
+ if attributes_str:
+ line += f' {attributes_str}'
+ line += ' /> <!-- SVG content collapsed -->'
+ return line
+
+ # Skip SVG child elements entirely (path, rect, g, circle, etc.)
+ if tag in SVG_ELEMENTS:
+ return ''
+
+ # Build compact attributes string
+ attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)
+
+ # Decide if this element should be shown
+ is_semantic = tag in SEMANTIC_ELEMENTS
+ has_useful_attrs = bool(attributes_str)
+ has_text_content = DOMEvalSerializer._has_direct_text(node)
+ has_children = len(node.children) > 0
+
+ # Build compact element representation
+ line = f'{depth_str}'
+ # Add backend node ID notation - [i_X] for interactive elements only
+ if node.interactive_index is not None:
+ line += f'[i_{node.original_node.backend_node_id}] '
+ # Non-interactive elements don't get an index notation
+ line += f'<{tag}'
+
+ if attributes_str:
+ line += f' {attributes_str}'
+
+ # Add scroll info if element is scrollable
+ if node.original_node.should_show_scroll_info:
+ scroll_text = node.original_node.get_scroll_info_text()
+ if scroll_text:
+ line += f' scroll="{scroll_text}"'
+
+ # Add inline text if present (keep it on same line for compactness)
+ inline_text = DOMEvalSerializer._get_inline_text(node)
+
+ # For containers (html, body, div, etc.), always show children even if there's inline text
+ # For other elements, inline text replaces children (more compact)
+ is_container = tag in container_tags
+
+ if inline_text and not is_container:
+ line += f'>{inline_text}'
+ else:
+ line += ' />'
+
+ formatted_text.append(line)
+
+ # Process children (always for containers, only if no inline_text for others)
+ if has_children and (is_container or not inline_text):
+ children_text = DOMEvalSerializer._serialize_children(node, include_attributes, depth + 1)
+ if children_text:
+ formatted_text.append(children_text)
+
+ elif node.original_node.node_type == NodeType.TEXT_NODE:
+ # Text nodes are handled inline with their parent
+ pass
+
+ elif node.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
+ # Shadow DOM - just show children directly with minimal marker
+ if node.children:
+ formatted_text.append(f'{depth_str}#shadow')
+ children_text = DOMEvalSerializer._serialize_children(node, include_attributes, depth + 1)
+ if children_text:
+ formatted_text.append(children_text)
+
+ return '\n'.join(formatted_text)
+
+ @staticmethod
+ def _serialize_children(node: SimplifiedNode, include_attributes: list[str], depth: int) -> str:
+ """Helper to serialize all children of a node."""
+ children_output = []
+
+ # Check if parent is a list container (ul, ol)
+ is_list_container = node.original_node.node_type == NodeType.ELEMENT_NODE and node.original_node.tag_name.lower() in [
+ 'ul',
+ 'ol',
+ ]
+
+ # Track list items and consecutive links
+ li_count = 0
+ max_list_items = 50
+ consecutive_link_count = 0
+ max_consecutive_links = 50
+ total_links_skipped = 0
+
+ for child in node.children:
+ # Get tag name for this child
+ current_tag = None
+ if child.original_node.node_type == NodeType.ELEMENT_NODE:
+ current_tag = child.original_node.tag_name.lower()
+
+ # If we're in a list container and this child is an li element
+ if is_list_container and current_tag == 'li':
+ li_count += 1
+ # Skip li elements after the 5th one
+ if li_count > max_list_items:
+ continue
+
+ # Track consecutive anchor tags (links)
+ if current_tag == 'a':
+ consecutive_link_count += 1
+ # Skip links after the 5th consecutive one
+ if consecutive_link_count > max_consecutive_links:
+ total_links_skipped += 1
+ continue
+ else:
+ # Reset counter when we hit a non-link element
+ # But first add truncation message if we skipped links
+ if total_links_skipped > 0:
+ depth_str = depth * '\t'
+ children_output.append(f'{depth_str}... ({total_links_skipped} more links in this list)')
+ total_links_skipped = 0
+ consecutive_link_count = 0
+
+ child_text = DOMEvalSerializer.serialize_tree(child, include_attributes, depth)
+ if child_text:
+ children_output.append(child_text)
+
+ # Add truncation message if we skipped items at the end
+ if is_list_container and li_count > max_list_items:
+ depth_str = depth * '\t'
+ children_output.append(
+ f'{depth_str}... ({li_count - max_list_items} more items in this list (truncated) use evaluate to get more.'
+ )
+
+ # Add truncation message for links if we skipped any at the end
+ if total_links_skipped > 0:
+ depth_str = depth * '\t'
+ children_output.append(
+ f'{depth_str}... ({total_links_skipped} more links in this list) (truncated) use evaluate to get more.'
+ )
+
+ return '\n'.join(children_output)
+
+ @staticmethod
+ def _build_compact_attributes(node: EnhancedDOMTreeNode) -> str:
+ """Build ultra-compact attributes string with only key attributes."""
+ attrs = []
+
+ # Prioritize attributes that help with query writing
+ if node.attributes:
+ for attr in EVAL_KEY_ATTRIBUTES:
+ if attr in node.attributes:
+ value = str(node.attributes[attr]).strip()
+ if not value:
+ continue
+
+ # Special handling for different attributes
+ if attr == 'class':
+ # For class, limit to first 2 classes to save space
+ classes = value.split()[:3]
+ value = ' '.join(classes)
+ elif attr == 'href':
+ # For href, cap at 20 chars to save space
+ value = cap_text_length(value, 80)
+ else:
+ # Cap at 25 chars for other attributes
+ value = cap_text_length(value, 80)
+
+ attrs.append(f'{attr}="{value}"')
+
+ # Note: We intentionally don't add role from ax_node here because:
+ # 1. If role is explicitly set in HTML, it's already captured above via EVAL_KEY_ATTRIBUTES
+ # 2. Inferred roles from AX tree (like link, listitem, LineBreak) are redundant with the tag name
+ # 3. This reduces noise - <a href="..." role="link"> is redundant, we already know <a> is a link
+
+ return ' '.join(attrs)
+
+ @staticmethod
+ def _has_direct_text(node: SimplifiedNode) -> bool:
+ """Check if node has direct text children (not nested in other elements)."""
+ for child in node.children:
+ if child.original_node.node_type == NodeType.TEXT_NODE:
+ text = child.original_node.node_value.strip() if child.original_node.node_value else ''
+ if len(text) > 1:
+ return True
+ return False
+
+ @staticmethod
+ def _get_inline_text(node: SimplifiedNode) -> str:
+ """Get text content to display inline (max 40 chars)."""
+ text_parts = []
+ for child in node.children:
+ if child.original_node.node_type == NodeType.TEXT_NODE:
+ text = child.original_node.node_value.strip() if child.original_node.node_value else ''
+ if text and len(text) > 1:
+ text_parts.append(text)
+
+ if not text_parts:
+ return ''
+
+ combined = ' '.join(text_parts)
+ return cap_text_length(combined, 80)
+
+ @staticmethod
+ def _serialize_iframe(node: SimplifiedNode, include_attributes: list[str], depth: int) -> str:
+ """Handle iframe serialization with content document."""
+ formatted_text = []
+ depth_str = depth * '\t'
+ tag = node.original_node.tag_name.lower()
+
+ # Build minimal iframe marker with key attributes
+ attributes_str = DOMEvalSerializer._build_compact_attributes(node.original_node)
+ line = f'{depth_str}<{tag}'
+ if attributes_str:
+ line += f' {attributes_str}'
+
+ # Add scroll info for iframe content
+ if node.original_node.should_show_scroll_info:
+ scroll_text = node.original_node.get_scroll_info_text()
+ if scroll_text:
+ line += f' scroll="{scroll_text}"'
+
+ line += ' />'
+ formatted_text.append(line)
+
+ # If iframe has content document, serialize its content
+ if node.original_node.content_document:
+ # Add marker for iframe content
+ formatted_text.append(f'{depth_str}\t#iframe-content')
+
+ # Process content document children
+ for child_node in node.original_node.content_document.children_nodes or []:
+ # Process html documents
+ if child_node.tag_name.lower() == 'html':
+ # Find and serialize body content only (skip head)
+ for html_child in child_node.children:
+ if html_child.tag_name.lower() == 'body':
+ for body_child in html_child.children:
+ # Recursively process body children (iframe content)
+ DOMEvalSerializer._serialize_document_node(
+ body_child, formatted_text, include_attributes, depth + 2, is_iframe_content=True
+ )
+ break # Stop after processing body
+ else:
+ # Not an html element - serialize directly
+ DOMEvalSerializer._serialize_document_node(
+ child_node, formatted_text, include_attributes, depth + 1, is_iframe_content=True
+ )
+
+ return '\n'.join(formatted_text)
+
+ @staticmethod
+ def _serialize_document_node(
+ dom_node: EnhancedDOMTreeNode,
+ output: list[str],
+ include_attributes: list[str],
+ depth: int,
+ is_iframe_content: bool = True,
+ ) -> None:
+ """Helper to serialize a document node without SimplifiedNode wrapper.
+
+ Args:
+ is_iframe_content: If True, be more permissive with visibility checks since
+ iframe content might not have snapshot data from parent page.
+ """
+ depth_str = depth * '\t'
+
+ if dom_node.node_type == NodeType.ELEMENT_NODE:
+ tag = dom_node.tag_name.lower()
+
+ # For iframe content, be permissive - show all semantic elements even without snapshot data
+ # For regular content, skip invisible elements
+ if is_iframe_content:
+ # Only skip if we have snapshot data AND it's explicitly invisible
+ # If no snapshot data, assume visible (cross-origin iframe content)
+ is_visible = (not dom_node.snapshot_node) or dom_node.is_visible
+ else:
+ # Regular strict visibility check
+ is_visible = dom_node.snapshot_node and dom_node.is_visible
+
+ if not is_visible:
+ return
+
+ # Check if semantic or has useful attributes
+ is_semantic = tag in SEMANTIC_ELEMENTS
+ attributes_str = DOMEvalSerializer._build_compact_attributes(dom_node)
+
+ if not is_semantic and not attributes_str:
+ # Skip but process children
+ for child in dom_node.children:
+ DOMEvalSerializer._serialize_document_node(
+ child, output, include_attributes, depth, is_iframe_content=is_iframe_content
+ )
+ return
+
+ # Build element line
+ line = f'{depth_str}<{tag}'
+ if attributes_str:
+ line += f' {attributes_str}'
+
+ # Get direct text content
+ text_parts = []
+ for child in dom_node.children:
+ if child.node_type == NodeType.TEXT_NODE and child.node_value:
+ text = child.node_value.strip()
+ if text and len(text) > 1:
+ text_parts.append(text)
+
+ if text_parts:
+ combined = ' '.join(text_parts)
+ line += f'>{cap_text_length(combined, 100)}'
+ else:
+ line += ' />'
+
+ output.append(line)
+
+ # Process non-text children
+ for child in dom_node.children:
+ if child.node_type != NodeType.TEXT_NODE:
+ DOMEvalSerializer._serialize_document_node(
+ child, output, include_attributes, depth + 1, is_iframe_content=is_iframe_content
+ )
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.py
index c21229f67..a69f1a6cf 100644
--- a/browser_use/dom/serializer/serializer.py
+++ b/browser_use/dom/serializer/serializer.py
@@ -530,33 +530,35 @@ class DOMTreeSerializer:
return None
def _collect_interactive_elements(self, node: SimplifiedNode, elements: list[SimplifiedNode]) -> None:
- """Recursively collect interactive elements that are also visible."""
+ """Recursively collect all interactive elements (scroll-to-view on interaction)."""
is_interactive = self._is_interactive_cached(node.original_node)
- is_visible = node.original_node.snapshot_node and node.original_node.is_visible
- # Only collect elements that are both interactive AND visible
- if is_interactive and is_visible:
+ # Collect all interactive elements regardless of viewport visibility
+ # Actions will scroll elements into view as needed
+ if is_interactive:
elements.append(node)
for child in node.children:
self._collect_interactive_elements(child, elements)
def _assign_interactive_indices_and_mark_new_nodes(self, node: SimplifiedNode | None) -> None:
- """Assign interactive indices to clickable elements that are also visible."""
+ """Assign interactive indices to all clickable elements (scroll-to-view on click)."""
if not node:
return
# Skip assigning index to excluded nodes, or ignored by paint order
if not node.excluded_by_parent and not node.ignored_by_paint_order:
- # Regular interactive element assignment (including enhanced compound controls)
- is_interactive_assign = self._is_interactive_cached(node.original_node)
is_visible = node.original_node.snapshot_node and node.original_node.is_visible
- # Only add to selector map if element is both interactive AND visible
- if is_interactive_assign and is_visible:
+ # Add ALL visible elements to selector map (keyed by backend_node_id)
+ self._selector_map[node.original_node.backend_node_id] = node.original_node
+
+ # Assign interactive index to ALL interactive elements (visible or not)
+ # Since click actions scroll elements into view, we should expose all interactive elements
+ is_interactive_assign = self._is_interactive_cached(node.original_node)
+ if is_interactive_assign:
node.interactive_index = self._interactive_counter
node.original_node.element_index = self._interactive_counter
- self._selector_map[self._interactive_counter] = node.original_node
self._interactive_counter += 1
# Mark compound components as new for visibility
@@ -823,8 +825,8 @@ class DOMTreeSerializer:
elif node.interactive_index is not None:
# Clickable (and possibly scrollable)
new_prefix = '*' if node.is_new else ''
- scroll_prefix = '|SCROLL[' if should_show_scroll else '['
- line = f'{depth_str}{shadow_prefix}{new_prefix}{scroll_prefix}{node.interactive_index}]<{node.original_node.tag_name}'
+ scroll_prefix = '|SCROLL|' if should_show_scroll else ''
+ line = f'{depth_str}{shadow_prefix}{new_prefix}{scroll_prefix}<{node.original_node.tag_name}'
elif node.original_node.tag_name.upper() == 'IFRAME':
# Iframe element (not interactive)
line = f'{depth_str}{shadow_prefix}|IFRAME|<{node.original_node.tag_name}'
@@ -837,6 +839,11 @@ class DOMTreeSerializer:
if attributes_html_str:
line += f' {attributes_html_str}'
+ # Add backend node ID notation - [i_X] for interactive elements only
+ if node.interactive_index is not None:
+ line += f' [i_{node.original_node.backend_node_id}]'
+ # Non-interactive elements don't get an index notation
+
line += ' />'
# Add scroll information only when we should show it
@@ -961,6 +968,6 @@ class DOMTreeSerializer:
del attributes_to_include[attr]
if attributes_to_include:
- return ' '.join(f'{key}={cap_text_length(value, 100)}' for key, value in attributes_to_include.items())
+ return ' '.join(f'{key}={cap_text_length(value, 30)}' for key, value in attributes_to_include.items())
return ''
diff --git a/browser_use/dom/service.py b/browser_use/dom/service.py
index 72c4312b8..836b1054d 100644
--- a/browser_use/dom/service.py
+++ b/browser_use/dom/service.py
@@ -660,7 +660,7 @@ class DomService:
f'Skipping iframe at depth {iframe_depth} to prevent infinite recursion (max depth: {self.max_iframe_depth})'
)
else:
- # Check if iframe is visible and large enough (>= 200px in both dimensions)
+ # Check if iframe is visible and large enough (>= 50px in both dimensions)
should_process_iframe = False
# First check if the iframe element itself is visible
@@ -671,13 +671,13 @@ class DomService:
width = bounds.width
height = bounds.height
- # Only process if iframe is at least 200px in both dimensions
- if width >= 200 and height >= 200:
+ # Only process if iframe is at least 50px in both dimensions
+ if width >= 50 and height >= 50:
should_process_iframe = True
self.logger.debug(f'Processing cross-origin iframe: visible=True, width={width}, height={height}')
else:
self.logger.debug(
- f'Skipping small cross-origin iframe: width={width}, height={height} (needs >= 200px)'
+ f'Skipping small cross-origin iframe: width={width}, height={height} (needs >= 50px)'
)
else:
self.logger.debug('Skipping cross-origin iframe: no bounds available')
@@ -748,3 +748,79 @@ class DomService:
all_timing = {**serializer_timing, **serialize_total_timing}
return serialized_dom_state, enhanced_dom_tree, all_timing
+
+ @staticmethod
+ def detect_pagination_buttons(selector_map: dict[int, EnhancedDOMTreeNode]) -> list[dict[str, str | int | bool]]:
+ """Detect pagination buttons from the selector map.
+
+ Args:
+ selector_map: Map of element indices to EnhancedDOMTreeNode
+
+ Returns:
+ List of pagination button information dicts with:
+ - button_type: 'next', 'prev', 'first', 'last', 'page_number'
+ - element_index: Interactive element index for clicking
+ - text: Button text/label
+ - selector: XPath selector
+ - is_disabled: Whether the button appears disabled
+ """
+ pagination_buttons: list[dict[str, str | int | bool]] = []
+
+ # Common pagination patterns to look for
+ next_patterns = ['next', '>', '»', '→', 'siguiente', 'suivant', 'weiter', 'volgende']
+ prev_patterns = ['prev', 'previous', '<', '«', '←', 'anterior', 'précédent', 'zurück', 'vorige']
+ first_patterns = ['first', '⇤', '«', 'primera', 'première', 'erste', 'eerste']
+ last_patterns = ['last', '⇥', '»', 'última', 'dernier', 'letzte', 'laatste']
+
+ for index, node in selector_map.items():
+ # Skip non-clickable elements
+ if not node.snapshot_node or not node.snapshot_node.is_clickable:
+ continue
+
+ # Get element text and attributes
+ text = node.get_all_children_text().lower().strip()
+ aria_label = node.attributes.get('aria-label', '').lower()
+ title = node.attributes.get('title', '').lower()
+ class_name = node.attributes.get('class', '').lower()
+ role = node.attributes.get('role', '').lower()
+
+ # Combine all text sources for pattern matching
+ all_text = f'{text} {aria_label} {title} {class_name}'.strip()
+
+ # Check if it's disabled
+ is_disabled = (
+ node.attributes.get('disabled') == 'true'
+ or node.attributes.get('aria-disabled') == 'true'
+ or 'disabled' in class_name
+ )
+
+ button_type: str | None = None
+
+ # Check for next button
+ if any(pattern in all_text for pattern in next_patterns):
+ button_type = 'next'
+ # Check for previous button
+ elif any(pattern in all_text for pattern in prev_patterns):
+ button_type = 'prev'
+ # Check for first button
+ elif any(pattern in all_text for pattern in first_patterns):
+ button_type = 'first'
+ # Check for last button
+ elif any(pattern in all_text for pattern in last_patterns):
+ button_type = 'last'
+ # Check for numeric page buttons (single or double digit)
+ elif text.isdigit() and len(text) <= 2 and role in ['button', 'link', '']:
+ button_type = 'page_number'
+
+ if button_type:
+ pagination_buttons.append(
+ {
+ 'button_type': button_type,
+ 'element_index': index,
+ 'text': node.get_all_children_text().strip() or aria_label or title,
+ 'selector': node.xpath,
+ 'is_disabled': is_disabled,
+ }
+ )
+
+ return pagination_buttons
diff --git a/browser_use/dom/utils.py b/browser_use/dom/utils.py
index ce45ec732..a9aada27b 100644
--- a/browser_use/dom/utils.py
+++ b/browser_use/dom/utils.py
@@ -3,3 +3,127 @@ def cap_text_length(text: str, max_length: int) -> str:
if len(text) <= max_length:
return text
return text[:max_length] + '...'
+
+
+def generate_css_selector_for_element(enhanced_node) -> str | None:
+ """Generate a CSS selector using node properties from version 0.5.0 approach."""
+ import re
+
+ if not enhanced_node or not hasattr(enhanced_node, 'tag_name') or not enhanced_node.tag_name:
+ return None
+
+ # Get base selector from tag name (simplified since we don't have xpath in EnhancedDOMTreeNode)
+ tag_name = enhanced_node.tag_name.lower().strip()
+ if not tag_name or not re.match(r'^[a-zA-Z][a-zA-Z0-9]*$', tag_name):
+ return None
+
+ css_selector = tag_name
+
+ # Add ID if available (most specific)
+ if enhanced_node.attributes and 'id' in enhanced_node.attributes:
+ element_id = enhanced_node.attributes['id']
+ if element_id and element_id.strip():
+ element_id = element_id.strip()
+ # Validate ID contains only valid characters for # selector
+ if re.match(r'^[a-zA-Z][a-zA-Z0-9_-]*$', element_id):
+ return f'#{element_id}'
+ else:
+ # For IDs with special characters ($, ., :, etc.), use attribute selector
+ # Escape quotes in the ID value
+ escaped_id = element_id.replace('"', '\\"')
+ return f'{tag_name}[id="{escaped_id}"]'
+
+ # Handle class attributes (from version 0.5.0 approach)
+ if enhanced_node.attributes and 'class' in enhanced_node.attributes and enhanced_node.attributes['class']:
+ # Define a regex pattern for valid class names in CSS
+ valid_class_name_pattern = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_-]*$')
+
+ # Iterate through the class attribute values
+ classes = enhanced_node.attributes['class'].split()
+ for class_name in classes:
+ # Skip empty class names
+ if not class_name.strip():
+ continue
+
+ # Check if the class name is valid
+ if valid_class_name_pattern.match(class_name):
+ # Append the valid class name to the CSS selector
+ css_selector += f'.{class_name}'
+
+ # Expanded set of safe attributes that are stable and useful for selection (from v0.5.0)
+ SAFE_ATTRIBUTES = {
+ # Data attributes (if they're stable in your application)
+ 'id',
+ # Standard HTML attributes
+ 'name',
+ 'type',
+ 'placeholder',
+ # Accessibility attributes
+ 'aria-label',
+ 'aria-labelledby',
+ 'aria-describedby',
+ 'role',
+ # Common form attributes
+ 'for',
+ 'autocomplete',
+ 'required',
+ 'readonly',
+ # Media attributes
+ 'alt',
+ 'title',
+ 'src',
+ # Custom stable attributes (add any application-specific ones)
+ 'href',
+ 'target',
+ }
+
+ # Always include dynamic attributes (include_dynamic_attributes=True equivalent)
+ include_dynamic_attributes = True
+ if include_dynamic_attributes:
+ dynamic_attributes = {
+ 'data-id',
+ 'data-qa',
+ 'data-cy',
+ 'data-testid',
+ }
+ SAFE_ATTRIBUTES.update(dynamic_attributes)
+
+ # Handle other attributes (from version 0.5.0 approach)
+ if enhanced_node.attributes:
+ for attribute, value in enhanced_node.attributes.items():
+ if attribute == 'class':
+ continue
+
+ # Skip invalid attribute names
+ if not attribute.strip():
+ continue
+
+ if attribute not in SAFE_ATTRIBUTES:
+ continue
+
+ # Escape special characters in attribute names
+ safe_attribute = attribute.replace(':', r'\:')
+
+ # Handle different value cases
+ if value == '':
+ css_selector += f'[{safe_attribute}]'
+ elif any(char in value for char in '"\'<>`\n\r\t'):
+ # Use contains for values with special characters
+ # For newline-containing text, only use the part before the newline
+ if '\n' in value:
+ value = value.split('\n')[0]
+ # Regex-substitute *any* whitespace with a single space, then strip.
+ collapsed_value = re.sub(r'\s+', ' ', value).strip()
+ # Escape embedded double-quotes.
+ safe_value = collapsed_value.replace('"', '\\"')
+ css_selector += f'[{safe_attribute}*="{safe_value}"]'
+ else:
+ css_selector += f'[{safe_attribute}="{value}"]'
+
+ # Final validation: ensure the selector is safe and doesn't contain problematic characters
+ # Note: quotes are allowed in attribute selectors like [name="value"]
+ if css_selector and not any(char in css_selector for char in ['\n', '\r', '\t']):
+ return css_selector
+
+ # If we get here, the selector was problematic, return just the tag name as fallback
+ return tag_name
diff --git a/browser_use/dom/views.py b/browser_use/dom/views.py
index 952db8532..22475376a 100644
--- a/browser_use/dom/views.py
+++ b/browser_use/dom/views.py
@@ -819,6 +819,29 @@ class SerializedDOMState:
return DOMTreeSerializer.serialize_tree(self._root, include_attributes)
+ @observe_debug(ignore_input=True, ignore_output=True, name='eval_representation')
+ def eval_representation(
+ self,
+ include_attributes: list[str] | None = None,
+ ) -> str:
+ """
+ Evaluation-focused DOM representation without interactive indexes.
+
+ This serializer is designed for evaluation/judge contexts where:
+ - No interactive indexes are needed (we're not clicking)
+ - Full HTML structure should be preserved for context
+ - More attribute information is helpful
+ - Text content is important for understanding page structure
+ """
+ from browser_use.dom.serializer.eval_serializer import DOMEvalSerializer
+
+ if not self._root:
+ return 'Empty DOM tree (you might have to wait for the page to load)'
+
+ include_attributes = include_attributes or DEFAULT_INCLUDE_ATTRIBUTES
+
+ return DOMEvalSerializer.serialize_tree(self._root, include_attributes)
+
@dataclass
class DOMInteractedElement:
diff --git a/browser_use/llm/anthropic/chat.py b/browser_use/llm/anthropic/chat.py
index e5667d7f4..0c2e55c2b 100644
--- a/browser_use/llm/anthropic/chat.py
+++ b/browser_use/llm/anthropic/chat.py
@@ -166,6 +166,7 @@ class ChatAnthropic(BaseChatModel):
return ChatInvokeCompletion(
completion=response_text,
usage=usage,
+ stop_reason=response.stop_reason,
)
else:
@@ -212,7 +213,11 @@ class ChatAnthropic(BaseChatModel):
if hasattr(content_block, 'type') and content_block.type == 'tool_use':
# Parse the tool input as the structured output
try:
- return ChatInvokeCompletion(completion=output_format.model_validate(content_block.input), usage=usage)
+ return ChatInvokeCompletion(
+ completion=output_format.model_validate(content_block.input),
+ usage=usage,
+ stop_reason=response.stop_reason,
+ )
except Exception as e:
# If validation fails, try to parse it as JSON first
if isinstance(content_block.input, str):
@@ -220,6 +225,7 @@ class ChatAnthropic(BaseChatModel):
return ChatInvokeCompletion(
completion=output_format.model_validate(data),
usage=usage,
+ stop_reason=response.stop_reason,
)
raise e
diff --git a/browser_use/llm/google/chat.py b/browser_use/llm/google/chat.py
index 757c9e895..f7ac30f1f 100644
--- a/browser_use/llm/google/chat.py
+++ b/browser_use/llm/google/chat.py
@@ -74,11 +74,11 @@ class ChatGoogle(BaseChatModel):
# Model configuration
model: VerifiedGeminiModels | str
- temperature: float | None = 0.2
+ temperature: float | None = 0.5
top_p: float | None = None
seed: int | None = None
thinking_budget: int | None = None # for gemini-2.5 flash and flash-lite models, default will be set to 0
- max_output_tokens: int | None = 4096
+ max_output_tokens: int | None = 8096
config: types.GenerateContentConfigDict | None = None
include_system_in_user: bool = False
supports_structured_output: bool = True # New flag
@@ -139,6 +139,12 @@ class ChatGoogle(BaseChatModel):
def name(self) -> str:
return str(self.model)
+ def _get_stop_reason(self, response: types.GenerateContentResponse) -> str | None:
+ """Extract stop_reason from Google response."""
+ if hasattr(response, 'candidates') and response.candidates:
+ return str(response.candidates[0].finish_reason) if hasattr(response.candidates[0], 'finish_reason') else None
+ return None
+
def _get_usage(self, response: types.GenerateContentResponse) -> ChatInvokeUsage | None:
usage: ChatInvokeUsage | None = None
@@ -246,6 +252,7 @@ class ChatGoogle(BaseChatModel):
return ChatInvokeCompletion(
completion=text,
usage=usage,
+ stop_reason=self._get_stop_reason(response),
)
else:
@@ -291,6 +298,7 @@ class ChatGoogle(BaseChatModel):
return ChatInvokeCompletion(
completion=output_format.model_validate(parsed_data),
usage=usage,
+ stop_reason=self._get_stop_reason(response),
)
except (json.JSONDecodeError, ValueError) as e:
self.logger.error(f'❌ Failed to parse JSON response: {str(e)}')
@@ -313,12 +321,14 @@ class ChatGoogle(BaseChatModel):
return ChatInvokeCompletion(
completion=response.parsed,
usage=usage,
+ stop_reason=self._get_stop_reason(response),
)
else:
# If it's not the expected type, try to validate it
return ChatInvokeCompletion(
completion=output_format.model_validate(response.parsed),
usage=usage,
+ stop_reason=self._get_stop_reason(response),
)
else:
# Fallback: Request JSON in the prompt for models without native JSON mode
@@ -369,6 +379,7 @@ class ChatGoogle(BaseChatModel):
return ChatInvokeCompletion(
completion=output_format.model_validate(parsed_data),
usage=usage,
+ stop_reason=self._get_stop_reason(response),
)
except (json.JSONDecodeError, ValueError) as e:
self.logger.error(f'❌ Failed to parse fallback JSON: {str(e)}')
diff --git a/browser_use/llm/openai/chat.py b/browser_use/llm/openai/chat.py
index d33d0d1bd..e3e6d48b3 100644
--- a/browser_use/llm/openai/chat.py
+++ b/browser_use/llm/openai/chat.py
@@ -198,6 +198,7 @@ class ChatOpenAI(BaseChatModel):
return ChatInvokeCompletion(
completion=response.choices[0].message.content or '',
usage=usage,
+ stop_reason=response.choices[0].finish_reason if response.choices else None,
)
else:
@@ -239,6 +240,7 @@ class ChatOpenAI(BaseChatModel):
return ChatInvokeCompletion(
completion=parsed,
usage=usage,
+ stop_reason=response.choices[0].finish_reason if response.choices else None,
)
except RateLimitError as e:
diff --git a/browser_use/llm/views.py b/browser_use/llm/views.py
index 4707db104..830f51471 100644
--- a/browser_use/llm/views.py
+++ b/browser_use/llm/views.py
@@ -43,3 +43,6 @@ class ChatInvokeCompletion(BaseModel, Generic[T]):
usage: ChatInvokeUsage | None
"""The usage of the response."""
+
+ stop_reason: str | None = None
+ """The reason the model stopped generating. Common values: 'end_turn', 'max_tokens', 'stop_sequence'."""
diff --git a/browser_use/telemetry/views.py b/browser_use/telemetry/views.py
index 486842f8e..10089773c 100644
--- a/browser_use/telemetry/views.py
+++ b/browser_use/telemetry/views.py
@@ -33,6 +33,7 @@ class AgentTelemetryEvent(BaseTelemetryEvent):
version: str
source: str
cdp_url: str | None
+ agent_type: str | None # 'code' for CodeAgent, None for regular Agent
# step details
action_errors: Sequence[str | None]
action_history: Sequence[list[dict] | None]
diff --git a/browser_use/tokens/service.py b/browser_use/tokens/service.py
index f0a90ffbf..8783c5d11 100644
--- a/browser_use/tokens/service.py
+++ b/browser_use/tokens/service.py
@@ -341,12 +341,13 @@ class TokenCost:
result = await original_ainvoke(messages, output_format, **kwargs)
# Track usage if available (no await needed since add_usage is now sync)
+ # Use llm.model instead of llm.name for consistency with get_usage_tokens_for_model()
if result.usage:
- usage = token_cost_service.add_usage(llm.name, result.usage)
+ usage = token_cost_service.add_usage(llm.model, result.usage)
logger.debug(f'Token cost service: {usage}')
- asyncio.create_task(token_cost_service._log_usage(llm.name, usage))
+ asyncio.create_task(token_cost_service._log_usage(llm.model, usage))
# else:
# await token_cost_service._log_non_usage_llm(llm)
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index 20a1c320d..9476dfbd1 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -247,7 +247,9 @@ class Tools(Generic[Context]):
# Look up the node from the selector map
node = await browser_session.get_element_by_index(params.index)
if node is None:
- raise ValueError(f'Element index {params.index} not found in browser state')
+ msg = f'Element index {params.index} not available - page may have changed. Try refreshing browser state.'
+ logger.warning(f'⚠️ {msg}')
+ return ActionResult(extracted_content=msg)
# Highlight the element being clicked (truly non-blocking)
asyncio.create_task(browser_session.highlight_interaction_element(node))
@@ -296,7 +298,9 @@ class Tools(Generic[Context]):
# Look up the node from the selector map
node = await browser_session.get_element_by_index(params.index)
if node is None:
- raise ValueError(f'Element index {params.index} not found in browser state')
+ msg = f'Element index {params.index} not available - page may have changed. Try refreshing browser state.'
+ logger.warning(f'⚠️ {msg}')
+ return ActionResult(extracted_content=msg)
# Highlight the element being typed into (truly non-blocking)
asyncio.create_task(browser_session.highlight_interaction_element(node))
@@ -353,38 +357,47 @@ class Tools(Generic[Context]):
param_model=UploadFileAction,
)
async def upload_file(
- params: UploadFileAction, browser_session: BrowserSession, available_file_paths: list[str], file_system: FileSystem
+ params: UploadFileAction,
+ browser_session: BrowserSession,
+ available_file_paths: list[str],
+ file_system: FileSystem,
):
- # Check if file is in available_file_paths (user-provided or downloaded files)
- # For remote browsers (is_local=False), we allow absolute remote paths even if not tracked locally
- if params.path not in available_file_paths:
- # Also check if it's a recently downloaded file that might not be in available_file_paths yet
- downloaded_files = browser_session.downloaded_files
- if params.path not in downloaded_files:
- # Finally, check if it's a file in the FileSystem service
- if file_system and file_system.get_dir():
- # Check if the file is actually managed by the FileSystem service
- # The path should be just the filename for FileSystem files
- file_obj = file_system.get_file(params.path)
- if file_obj:
- # File is managed by FileSystem, construct the full path
- file_system_path = str(file_system.get_dir() / params.path)
- params = UploadFileAction(index=params.index, path=file_system_path)
+ # Path validation logic:
+ # 1. If available_file_paths provided (security mode), enforce it as a whitelist
+ # 2. If no whitelist, for local browsers just check file exists
+ # 3. For remote browsers, allow any path (assume it exists remotely)
+
+ # If whitelist provided, validate path is in it
+ if available_file_paths:
+ if params.path not in available_file_paths:
+ # Also check if it's a recently downloaded file
+ downloaded_files = browser_session.downloaded_files
+ if params.path not in downloaded_files:
+ # Finally, check if it's a file in the FileSystem service (if provided)
+ if file_system is not None and file_system.get_dir():
+ # Check if the file is actually managed by the FileSystem service
+ # The path should be just the filename for FileSystem files
+ file_obj = file_system.get_file(params.path)
+ if file_obj:
+ # File is managed by FileSystem, construct the full path
+ file_system_path = str(file_system.get_dir() / params.path)
+ params = UploadFileAction(index=params.index, path=file_system_path)
+ else:
+ # If browser is remote, allow passing a remote-accessible absolute path
+ if not browser_session.is_local:
+ pass
+ else:
+ msg = f'File path {params.path} is not available. Upload files must be in available_file_paths, downloaded_files, or a file managed by file_system.'
+ logger.error(f'❌ {msg}')
+ return ActionResult(error=msg)
else:
# If browser is remote, allow passing a remote-accessible absolute path
if not browser_session.is_local:
pass
else:
- msg = f'File path {params.path} is not available. Upload files must be in available_file_paths, downloaded_files, or a file managed by file_system.'
+ msg = f'File path {params.path} is not available. Upload files must be in available_file_paths or downloaded_files.'
logger.error(f'❌ {msg}')
return ActionResult(error=msg)
- else:
- # If browser is remote, allow passing a remote-accessible absolute path
- if not browser_session.is_local:
- pass
- else:
- msg = f'File path {params.path} is not available. Upload files must be in available_file_paths, downloaded_files, or a file managed by file_system.'
- raise BrowserError(message=msg, long_term_memory=msg)
# For local browsers, ensure the file exists on the local filesystem
if browser_session.is_local:
@@ -506,7 +519,10 @@ class Tools(Generic[Context]):
# Tab Management Actions
- @self.registry.action('', param_model=SwitchTabAction)
+ @self.registry.action(
+ 'Switch to another open tab by tab_id. Tab IDs are shown in browser state tabs list (last 4 chars of target_id). Use when you need to work with content in a different tab.',
+ param_model=SwitchTabAction,
+ )
async def switch(params: SwitchTabAction, browser_session: BrowserSession):
# Simple switch tab logic
try:
@@ -528,7 +544,10 @@ class Tools(Generic[Context]):
memory = f'Attempted to switch to tab #{params.tab_id}'
return ActionResult(extracted_content=memory, long_term_memory=memory)
- @self.registry.action('', param_model=CloseTabAction)
+ @self.registry.action(
+ 'Close a tab by tab_id. Tab IDs are shown in browser state tabs list (last 4 chars of target_id). Use to clean up tabs you no longer need.',
+ param_model=CloseTabAction,
+ )
async def close(params: CloseTabAction, browser_session: BrowserSession):
# Simple close tab logic
try:
@@ -638,7 +657,7 @@ You will be given a query and the markdown of a webpage that has been filtered t
<instructions>
- You are tasked to extract information from the webpage that is relevant to the query.
-- You should ONLY use the information available in the webpage to answer the query. Do not make up information or provide guess from your own knowledge.
+- You should ONLY use the information available in the webpage to answer the query. Do not make up information or provide guess from your own knowledge.
- If the information relevant to the query is not available in the page, your response should mention that.
- If the query asks for all items, products, etc., make sure to directly list all of them.
- If the content was truncated and you need more information, note that the user can use start_from_char parameter to continue from where truncation occurred.
@@ -854,7 +873,9 @@ You will be given a query and the markdown of a webpage that has been filtered t
# Look up the node from the selector map
node = await browser_session.get_element_by_index(params.index)
if node is None:
- raise ValueError(f'Element index {params.index} not found in browser state')
+ msg = f'Element index {params.index} not available - page may have changed. Try refreshing browser state.'
+ logger.warning(f'⚠️ {msg}')
+ return ActionResult(extracted_content=msg)
# Dispatch GetDropdownOptionsEvent to the event handler
@@ -880,7 +901,9 @@ You will be given a query and the markdown of a webpage that has been filtered t
# Look up the node from the selector map
node = await browser_session.get_element_by_index(params.index)
if node is None:
- raise ValueError(f'Element index {params.index} not found in browser state')
+ msg = f'Element index {params.index} not available - page may have changed. Try refreshing browser state.'
+ logger.warning(f'⚠️ {msg}')
+ return ActionResult(extracted_content=msg)
# Dispatch SelectDropdownOptionEvent to the event handler
from browser_use.browser.events import SelectDropdownOptionEvent
@@ -915,6 +938,7 @@ You will be given a query and the markdown of a webpage that has been filtered t
return ActionResult(error=error_msg)
# File System Actions
+
@self.registry.action('')
async def write_file(
file_name: str,
@@ -971,7 +995,7 @@ You will be given a query and the markdown of a webpage that has been filtered t
)
@self.registry.action(
- """Execute browser JavaScript. MUST: wrap in IIFE (function(){...})(). Use ONLY browser APIs (document, window, DOM). NO Node.js APIs (fs, require, process). Add try-catch. Example: (function(){try{const el=document.querySelector('#id');return el?el.value:'not found'}catch(e){return 'Error: '+e.message}})() Never use comments. You are not allowed to use // in code. No human reads this. Use e.g. for hover, drag, zoom, custom selectors, extract/filter links, shadow DOM or to analyse page structure. Limit output.""",
+ """Execute browser JavaScript. Best practice: wrap in IIFE (function(){...})() with try-catch for safety. Use ONLY browser APIs (document, window, DOM). NO Node.js APIs (fs, require, process). Example: (function(){try{const el=document.querySelector('#id');return el?el.value:'not found'}catch(e){return 'Error: '+e.message}})() Avoid comments. Use for hover, drag, zoom, custom selectors, extract/filter links, shadow DOM, or analysing page structure. Limit output size.""",
)
async def evaluate(code: str, browser_session: BrowserSession):
# Execute JavaScript with proper error handling and promise support
@@ -992,8 +1016,6 @@ You will be given a query and the markdown of a webpage that has been filtered t
if result.get('exceptionDetails'):
exception = result['exceptionDetails']
error_msg = f'JavaScript execution error: {exception.get("text", "Unknown error")}'
- if 'lineNumber' in exception:
- error_msg += f' at line {exception["lineNumber"]}'
# Enhanced error message with debugging info
enhanced_msg = f"""JavaScript Execution Failed:
@@ -1003,7 +1025,7 @@ Validated Code (after quote fixing):
{validated_code[:500]}{'...' if len(validated_code) > 500 else ''}
"""
- logger.info(enhanced_msg)
+ logger.debug(enhanced_msg)
return ActionResult(error=enhanced_msg)
# Get the result data
@@ -1011,8 +1033,8 @@ Validated Code (after quote fixing):
# Check for wasThrown flag (backup error detection)
if result_data.get('wasThrown'):
- msg = f'Code: {code}\n\nError: JavaScript execution failed (wasThrown=true)'
- logger.info(msg)
+ msg = 'JavaScript execution failed (wasThrown=true)'
+ logger.debug(f'Code: {code[:200]}...')
return ActionResult(error=msg)
# Get the actual value
@@ -1034,16 +1056,17 @@ Validated Code (after quote fixing):
result_text = str(value)
# Apply length limit with better truncation
- if len(result_text) > 20000:
- result_text = result_text[:19950] + '\n... [Truncated after 20000 characters]'
- msg = f'Code: {code}\n\nResult: {result_text}'
- logger.info(msg)
- return ActionResult(extracted_content=f'Code: {code}\n\nResult: {result_text}')
+ if len(result_text) > 50000:
+ result_text = result_text[:49950] + '\n... [Truncated after 50000 characters]'
+ # Don't log the code - it's already visible in the user's cell
+ logger.debug(f'JavaScript executed successfully, result length: {len(result_text)}')
+ # Return only the result, not the code (code is already in user's cell)
+ return ActionResult(extracted_content=result_text)
except Exception as e:
# CDP communication or other system errors
- error_msg = f'Code: {code}\n\nError: Failed to execute JavaScript: {type(e).__name__}: {e}'
- logger.info(error_msg)
+ error_msg = f'Failed to execute JavaScript: {type(e).__name__}: {e}'
+ logger.debug(f'Code that failed: {code[:200]}...')
return ActionResult(error=error_msg)
def _validate_and_fix_javascript(self, code: str) -> str:
@@ -1263,3 +1286,131 @@ Validated Code (after quote fixing):
# Alias for backwards compatibility
Controller = Tools
+
+
+class CodeAgentTools(Tools[Context]):
+ """Specialized Tools for CodeAgent agent optimized for Python-based browser automation.
+
+ Includes:
+ - All browser interaction tools (click, input, scroll, navigate, etc.)
+ - JavaScript evaluation
+ - Tab management (switch, close)
+ - Navigation actions (go_back)
+ - Upload file support
+ - Dropdown interactions
+
+ Excludes (optimized for code-use mode):
+ - extract: Use Python + evaluate() instead
+ - find_text: Use Python string operations
+ - screenshot: Not needed in code-use mode
+ - search: Use navigate() directly
+ - File system actions (write_file, read_file, replace_file): Use Python file operations instead
+ """
+
+ def __init__(
+ self,
+ exclude_actions: list[str] | None = None,
+ output_model: type[T] | None = None,
+ display_files_in_done_text: bool = True,
+ ):
+ # Default exclusions for CodeAgent agent
+ if exclude_actions is None:
+ exclude_actions = [
+ # 'scroll', # Keep for code-use
+ 'extract', # Exclude - use Python + evaluate()
+ 'find_text', # Exclude - use Python string ops
+ # 'select_dropdown', # Keep for code-use
+ # 'dropdown_options', # Keep for code-use
+ 'screenshot', # Exclude - not needed
+ 'search', # Exclude - use navigate() directly
+ # 'click', # Keep for code-use
+ # 'input', # Keep for code-use
+ # 'switch', # Keep for code-use
+ # 'send_keys', # Keep for code-use
+ # 'close', # Keep for code-use
+ # 'go_back', # Keep for code-use
+ # 'upload_file', # Keep for code-use
+ # Exclude file system actions - CodeAgent should use Python file operations
+ 'write_file',
+ 'read_file',
+ 'replace_file',
+ ]
+
+ super().__init__(
+ exclude_actions=exclude_actions,
+ output_model=output_model,
+ display_files_in_done_text=display_files_in_done_text,
+ )
+
+ # Override done action for CodeAgent with enhanced file handling
+ self._register_code_use_done_action(output_model, display_files_in_done_text)
+
+ def _register_code_use_done_action(self, output_model: type[T] | None, display_files_in_done_text: bool = True):
+ """Register enhanced done action for CodeAgent that can read files from disk."""
+ if output_model is not None:
+ # Structured output done - use parent's implementation
+ return
+
+ # Override the done action with enhanced version
+ @self.registry.action(
+ 'Complete task.',
+ param_model=DoneAction,
+ )
+ async def done(params: DoneAction, file_system: FileSystem):
+ user_message = params.text
+
+ len_text = len(params.text)
+ len_max_memory = 100
+ memory = f'Task completed: {params.success} - {params.text[:len_max_memory]}'
+ if len_text > len_max_memory:
+ memory += f' - {len_text - len_max_memory} more characters'
+
+ attachments = []
+ if params.files_to_display:
+ if self.display_files_in_done_text:
+ file_msg = ''
+ for file_name in params.files_to_display:
+ file_content = file_system.display_file(file_name)
+ if file_content:
+ file_msg += f'\n\n{file_name}:\n{file_content}'
+ attachments.append(file_name)
+ elif os.path.exists(file_name):
+ # File exists on disk but not in FileSystem - just add to attachments
+ attachments.append(file_name)
+ if file_msg:
+ user_message += '\n\nAttachments:'
+ user_message += file_msg
+ else:
+ logger.warning('Agent wanted to display files but none were found')
+ else:
+ for file_name in params.files_to_display:
+ file_content = file_system.display_file(file_name)
+ if file_content:
+ attachments.append(file_name)
+ elif os.path.exists(file_name):
+ attachments.append(file_name)
+
+ # Convert relative paths to absolute paths - handle both FileSystem-managed and regular files
+ resolved_attachments = []
+ for file_name in attachments:
+ if os.path.isabs(file_name):
+ # Already absolute
+ resolved_attachments.append(file_name)
+ elif file_system.get_file(file_name):
+ # Managed by FileSystem
+ resolved_attachments.append(str(file_system.get_dir() / file_name))
+ elif os.path.exists(file_name):
+ # Regular file in current directory
+ resolved_attachments.append(os.path.abspath(file_name))
+ else:
+ # File doesn't exist, but include the path anyway for error visibility
+ resolved_attachments.append(str(file_system.get_dir() / file_name))
+ attachments = resolved_attachments
+
+ return ActionResult(
+ is_done=True,
+ success=params.success,
+ extracted_content=user_message,
+ long_term_memory=memory,
+ attachments=attachments,
+ )
diff --git a/browser_use/tools/views.py b/browser_use/tools/views.py
index 0df2ead22..dddcd99f3 100644
--- a/browser_use/tools/views.py
+++ b/browser_use/tools/views.py
@@ -38,7 +38,7 @@ class InputTextAction(BaseModel):
class DoneAction(BaseModel):
text: str = Field(description='summary for user')
- success: bool = Field(description='True if user_request completed successfully')
+ success: bool = Field(default=True, description='True if user_request completed successfully')
files_to_display: list[str] | None = Field(default=[])
diff --git a/examples/code_use_examples/shopping.py b/examples/code_use_examples/shopping.py
new file mode 100644
index 000000000..e9e236745
--- /dev/null
+++ b/examples/code_use_examples/shopping.py
@@ -0,0 +1,90 @@
+"""
+Example: Simple product extraction with CodeAgent.
+
+This is a simplified version demonstrating code-use mode with a smaller task:
+extracting ~20 products from 2 categories on an e-commerce site.
+
+The agent will:
+- Navigate to product categories
+- Extract product data using JavaScript
+- Combine results from multiple categories
+- Save to a JSON file
+"""
+
+import asyncio
+from pathlib import Path
+
+from lmnr import Laminar
+
+from browser_use.code_use import CodeAgent, export_to_ipynb, session_to_python_script
+
+Laminar.initialize()
+
+
+async def main():
+ """
+ Task: Extract ~20 products from 2 categories on an e-commerce site.
+ """
+
+ task = """
+Go to https://www.amazon.com and extract approximately 20 products total from these 2 categories:
+
+1. Electronics (laptops or tablets) - 10 products
+2. Home & Kitchen (small appliances) - 10 products
+
+For each product, collect:
+- Product name
+- Price
+- Rating (if available)
+- Product URL
+
+Save the results to a JSON file called 'simple_products.json'.
+"""
+
+ # Create code-use agent (uses ChatBrowserUse automatically)
+ agent = CodeAgent(
+ task=task,
+ max_steps=20,
+ )
+
+ try:
+ # Run the agent
+ print('Running code-use agent for simple product extraction...')
+ session = await agent.run()
+
+ # Print summary
+ print(f'\n{"=" * 60}')
+ print('Session Summary')
+ print(f'{"=" * 60}')
+ print(f'Total cells executed: {len(session.cells)}')
+ print(f'Total execution count: {session.current_execution_count}')
+
+ # Print each cell
+ for i, cell in enumerate(session.cells):
+ print(f'\n{"-" * 60}')
+ print(f'Cell {i + 1} (Status: {cell.status.value})')
+ print(f'{"-" * 60}')
+ print('Code:')
+ print(cell.source)
+ if cell.output:
+ print('\nOutput:')
+ print(cell.output)
+ if cell.error:
+ print('\nError:')
+ print(cell.error)
+
+ # Export to Jupyter notebook
+ notebook_path = export_to_ipynb(session, 'simple_shopping.ipynb')
+ print(f'\n✓ Exported session to Jupyter notebook: {notebook_path}')
+
+ # Export to Python script
+ script = session_to_python_script(session)
+ await asyncio.to_thread(Path('simple_shopping.py').write_text, script)
+ print('✓ Exported session to Python script: simple_shopping.py')
+
+ finally:
+ await agent.close()
+
+
+if __name__ == '__main__':
+ asyncio.run(main())
diff --git a/examples/code_use_extract_products.py b/examples/code_use_extract_products.py
new file mode 100644
index 000000000..84af1fb22
--- /dev/null
+++ b/examples/code_use_extract_products.py
@@ -0,0 +1,93 @@
+"""
+Example: Using code-use mode to extract products from multiple pages.
+
+This example demonstrates the new code-use mode, which works like a Jupiter notebook
+where the LLM writes Python code that gets executed in a persistent namespace.
+
+The agent can:
+- Navigate to pages
+- Extract data using JavaScript
+- Combine results from multiple pages
+- Save data to files
+- Export the session as a Jupyter notebook
+
+This solves the problem from the brainstorm where extraction of multiple items
+was difficult with the extract tool alone.
+"""
+
+import asyncio
+from pathlib import Path
+
+from lmnr import Laminar
+
+from browser_use.code_use import CodeAgent, export_to_ipynb, session_to_python_script
+
+Laminar.initialize()
+
+
+async def main():
+ """
+ Task: Extract product listings from an e-commerce site.
+
+ The agent will:
+ 1. Navigate to the products page
+ 2. Extract products using JavaScript
+ 3. Scroll to load more products
+ 4. Combine all results
+ 5. Save to a JSON file
+ """
+
+ task = """
+
+Go to https://www.flipkart.com. Continue collecting products from Flipkart in the following categories. I need approximately 50 products from:\n\n1. Books & Media (books, stationery) - 15 products\n2. Sports & Fitness (equipment, clothing, accessories) - 15 products \n3. Beauty & Personal Care (cosmetics, skincare, grooming) - 10 products\nAnd 2 other categories you find interesting.\nNavigate to these categories and collect products with:\n- Product URL (working link)\n- Product name/description\n- Actual price (MRP)\n- Deal price (current selling price) \n- Discount percentage\n\nFocus on products with good discounts and clear pricing. Target around 40 products total from these three categories.
+
+ """
+ task = "Usec requests to get GitHub's API for trending Python repos, then visit the top 3 repos in the browser and extract their README"
+
+ # Create code-use agent (uses ChatBrowserUse automatically)
+ agent = CodeAgent(
+ task=task,
+ max_steps=30,
+ )
+
+ try:
+ # Run the agent
+ print('Running code-use agent...')
+ session = await agent.run()
+
+ # Print summary
+ print(f'\n{"=" * 60}')
+ print('Session Summary')
+ print(f'{"=" * 60}')
+ print(f'Total cells executed: {len(session.cells)}')
+ print(f'Total execution count: {session.current_execution_count}')
+
+ # Print each cell
+ for i, cell in enumerate(session.cells):
+ print(f'\n{"-" * 60}')
+ print(f'Cell {i + 1} (Status: {cell.status.value})')
+ print(f'{"-" * 60}')
+ print('Code:')
+ print(cell.source)
+ if cell.output:
+ print('\nOutput:')
+ print(cell.output)
+ if cell.error:
+ print('\nError:')
+ print(cell.error)
+
+ # Export to Jupyter notebook
+ notebook_path = export_to_ipynb(session, 'product_extraction.ipynb')
+ print(f'\n✓ Exported session to Jupyter notebook: {notebook_path}')
+
+ # Export to Python script
+ script = session_to_python_script(session)
+ await asyncio.to_thread(Path('product_extraction.py').write_text, script)
+ print('✓ Exported session to Python script: product_extraction.py')
+
+ finally:
+ await agent.close()
+
+
+if __name__ == '__main__':
+ asyncio.run(main())
diff --git a/pyproject.toml b/pyproject.toml
index 6ca3c9dd9..2085f7e89 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,6 +42,8 @@ dependencies = [
"pyotp>=2.9.0",
"pillow>=11.2.1",
"markdownify>=1.2.0",
+
+
]
# google-api-core: only used for Google LLM APIs
# pyperclip: only used for examples that use copy/paste
@@ -59,6 +61,13 @@ cli = [
"click>=8.1.8",
"textual>=3.2.0",
]
+code = [
+ "matplotlib>=3.9.0",
+ "numpy>=2.3.2",
+ "pandas>=2.2.0",
+ "tabulate>=0.9.0",
+
+]
aws = [
"boto3>=1.38.45"
]
@@ -133,7 +142,7 @@ skip-magic-trailing-comma = false
[tool.pyright]
typeCheckingMode = "basic"
-exclude = [".venv/", ".git/", "__pycache__/", "./test_*.py", "./debug_*.py", "private_example/", "debug/*", "tests/scripts/*", "tests/old/*", "browser_use/dom/playground/*", "examples/use-cases/onepassword.py", "browser_use/llm/oci_raw/*"]
+exclude = [".venv/", ".git/", "__pycache__/", "./test_*.py", "./debug_*.py", "private_example/", "debug/*", "tests/scripts/*", "tests/old/*", "browser_use/dom/playground/*", "examples/use-cases/onepassword.py", "browser_use/llm/oci_raw/*", "browser_use/llm/tests/test_chat_models.py", "browser_use/llm/tests/test_single_step.py"]
venvPath = "."
venv = ".venv"
@@ -146,6 +155,7 @@ include = [
"browser_use/agent/system_prompt.md",
"browser_use/agent/system_prompt_no_thinking.md",
"browser_use/agent/system_prompt_flash.md",
+ "browser_use/code_use/system_prompt.md",
"browser_use/py.typed",
"browser_use/dom/**/*.js",
"!tests/**/*.py",
diff --git a/tests/ci/test_registry_action_parameter_injection.py b/tests/ci/test_registry_action_parameter_injection.py
index 9407397c3..3202dc4dd 100644
--- a/tests/ci/test_registry_action_parameter_injection.py
+++ b/tests/ci/test_registry_action_parameter_injection.py
@@ -116,11 +116,7 @@ class TestBrowserContext:
# urlparse will return an empty netloc for some malformed URLs.
assert watchdog2._is_url_allowed('notaurl') is False
- def test_convert_simple_xpath_to_css_selector(self):
- """
- Test removed: _convert_simple_xpath_to_css_selector method no longer exists.
- """
- pass # Method was removed from BrowserSession
+ # Method was removed from BrowserSession
def test_enhanced_css_selector_for_element(self):
"""