Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

feature flags for iframes and recent_events, split security and nav actions

Nick Sweeting20b4b2a6848d
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index 71bb18f32..f2469dae3 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -120,7 +120,6 @@ AgentHookFunc = Callable[['Agent'], Awaitable[None]]
class Agent(Generic[Context, AgentStructuredOutput]):
browser_session: BrowserSession | None = None
_logger: logging.Logger | None = None
- _owns_browser_session: bool = PrivateAttr(default=False)
@time_execution_sync('--init')
def __init__(
@@ -185,6 +184,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
llm_timeout: int = 60,
step_timeout: int = 180,
preload: bool = False,
+ recent_events: bool = False,
**kwargs,
):
if not isinstance(llm, BaseChatModel):
@@ -233,6 +233,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
self.task = task
self.llm = llm
self.preload = preload
+ self.recent_events = recent_events
self.controller = (
controller if controller is not None else Controller(display_files_in_done_text=display_files_in_done_text)
)
@@ -367,8 +368,6 @@ class Agent(Generic[Context, AgentStructuredOutput]):
# Use the browser session directly without copying
# This ensures the CDP client and watchdogs are properly accessible
self.browser_session = browser_session
- # Browser session was provided externally, don't own it
- self._owns_browser_session = False
else:
if browser is not None:
assert isinstance(browser, Browser), 'Browser is not set up'
@@ -376,8 +375,6 @@ class Agent(Generic[Context, AgentStructuredOutput]):
browser_profile=browser_profile,
id=uuid7str()[:-4] + self.id[-4:], # re-use the same 4-char suffix so they show up together in logs
)
- # We created the browser session, so we own it
- self._owns_browser_session = True
if self.sensitive_data:
# Check if sensitive_data has domain-specific credentials
@@ -701,11 +698,13 @@ class Agent(Generic[Context, AgentStructuredOutput]):
self.logger.debug(f'๐ŸŒ Step {self.state.n_steps}: Getting browser state...')
# Always take screenshots for all steps
- self.logger.debug('๐Ÿ“ธ Requesting browser state with include_screenshot=True, cached=True')
+ # Use caching based on preload setting - if preload is False, don't use cached state
+ use_cache = self.preload
+ self.logger.debug(f'๐Ÿ“ธ Requesting browser state with include_screenshot=True, cached={use_cache}')
browser_state_summary = await self.browser_session.get_browser_state_summary(
cache_clickable_elements_hashes=True,
include_screenshot=True,
- cached=True,
+ cached=use_cache,
)
if browser_state_summary.screenshot:
self.logger.debug(f'๐Ÿ“ธ Got browser state WITH screenshot, length: {len(browser_state_summary.screenshot)}')
@@ -1779,12 +1778,12 @@ class Agent(Generic[Context, AgentStructuredOutput]):
async def close(self):
"""Close all resources"""
try:
- # Only close browser resources if we created them
- if self._owns_browser_session:
- assert self.browser_session is not None, 'BrowserSession is not set up'
- # Kill the browser session - this dispatches BrowserStopEvent,
- # stops the EventBus with clear=True, and recreates a fresh EventBus
- await self.browser_session.kill()
+ # Only close browser if keep_alive is False (or not set)
+ if self.browser_session is not None:
+ if not self.browser_session.browser_profile.keep_alive:
+ # Kill the browser session - this dispatches BrowserStopEvent,
+ # stops the EventBus with clear=True, and recreates a fresh EventBus
+ await self.browser_session.kill()
# Force garbage collection
gc.collect()
diff --git a/browser_use/browser/default_action_watchdog.py b/browser_use/browser/default_action_watchdog.py
index 913f814a3..0a2e63eb7 100644
--- a/browser_use/browser/default_action_watchdog.py
+++ b/browser_use/browser/default_action_watchdog.py
@@ -189,7 +189,7 @@ class DefaultActionWatchdog(BaseWatchdog):
async def _click_element_node_impl(self, element_node, new_tab: bool = False) -> str | None:
"""
- Click an element using pure CDP.
+ Click an element using pure CDP with multiple fallback methods for getting element geometry.
Args:
element_node: The DOM element to click
@@ -207,17 +207,137 @@ class DefaultActionWatchdog(BaseWatchdog):
# Get element bounds
backend_node_id = element_node.backend_node_id
- # Get bounds from CDP
- box_model = await cdp_session.cdp_client.send.DOM.getBoxModel(
- params={'backendNodeId': backend_node_id}, session_id=session_id
- )
- content_quad = box_model['model']['content']
- if len(content_quad) < 8:
- raise Exception('Invalid content quad')
-
- # Calculate center point from quad
- center_x = (content_quad[0] + content_quad[2] + content_quad[4] + content_quad[6]) / 4
- center_y = (content_quad[1] + content_quad[3] + content_quad[5] + content_quad[7]) / 4
+ # Get viewport dimensions for visibility checks
+ layout_metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=session_id)
+ viewport_width = layout_metrics['layoutViewport']['clientWidth']
+ viewport_height = layout_metrics['layoutViewport']['clientHeight']
+
+ # Try multiple methods to get element geometry
+ quads = []
+
+ # Method 1: Try DOM.getContentQuads first (best for inline elements and complex layouts)
+ try:
+ content_quads_result = await cdp_session.cdp_client.send.DOM.getContentQuads(
+ params={'backendNodeId': backend_node_id}, session_id=session_id
+ )
+ if 'quads' in content_quads_result and content_quads_result['quads']:
+ quads = content_quads_result['quads']
+ self.logger.debug(f'Got {len(quads)} quads from DOM.getContentQuads')
+ except Exception as e:
+ self.logger.debug(f'DOM.getContentQuads failed: {e}')
+
+ # Method 2: Fall back to DOM.getBoxModel
+ if not quads:
+ try:
+ box_model = await cdp_session.cdp_client.send.DOM.getBoxModel(
+ params={'backendNodeId': backend_node_id}, session_id=session_id
+ )
+ if 'model' in box_model and 'content' in box_model['model']:
+ content_quad = box_model['model']['content']
+ if len(content_quad) >= 8:
+ # Convert box model format to quad format
+ quads = [[
+ content_quad[0], content_quad[1], # x1, y1
+ content_quad[2], content_quad[3], # x2, y2
+ content_quad[4], content_quad[5], # x3, y3
+ content_quad[6], content_quad[7] # x4, y4
+ ]]
+ self.logger.debug('Got quad from DOM.getBoxModel')
+ except Exception as e:
+ self.logger.debug(f'DOM.getBoxModel failed: {e}')
+
+ # Method 3: Fall back to JavaScript getBoundingClientRect
+ if not quads:
+ try:
+ result = await cdp_session.cdp_client.send.DOM.resolveNode(
+ params={'backendNodeId': backend_node_id},
+ session_id=session_id,
+ )
+ if 'object' in result and 'objectId' in result['object']:
+ object_id = result['object']['objectId']
+
+ # Get bounding rect via JavaScript
+ bounds_result = await cdp_session.cdp_client.send.Runtime.callFunctionOn(
+ params={
+ 'functionDeclaration': '''
+ function() {
+ const rect = this.getBoundingClientRect();
+ return {
+ x: rect.left,
+ y: rect.top,
+ width: rect.width,
+ height: rect.height
+ };
+ }
+ ''',
+ 'objectId': object_id,
+ 'returnByValue': True
+ },
+ session_id=session_id,
+ )
+
+ if 'result' in bounds_result and 'value' in bounds_result['result']:
+ rect = bounds_result['result']['value']
+ # Convert rect to quad format
+ x, y, w, h = rect['x'], rect['y'], rect['width'], rect['height']
+ quads = [[
+ x, y, # top-left
+ x + w, y, # top-right
+ x + w, y + h, # bottom-right
+ x, y + h # bottom-left
+ ]]
+ self.logger.debug('Got quad from getBoundingClientRect')
+ except Exception as e:
+ self.logger.debug(f'JavaScript getBoundingClientRect failed: {e}')
+
+ # If we still don't have quads, fail
+ if not quads:
+ raise Exception('Could not get element geometry from any method')
+
+ # Find the largest visible quad within the viewport
+ best_quad = None
+ best_area = 0
+
+ for quad in quads:
+ if len(quad) < 8:
+ continue
+
+ # Calculate quad bounds
+ xs = [quad[i] for i in range(0, 8, 2)]
+ ys = [quad[i] for i in range(1, 8, 2)]
+ min_x, max_x = min(xs), max(xs)
+ min_y, max_y = min(ys), max(ys)
+
+ # Check if quad intersects with viewport
+ if max_x < 0 or max_y < 0 or min_x > viewport_width or min_y > viewport_height:
+ continue # Quad is completely outside viewport
+
+ # Calculate visible area (intersection with viewport)
+ visible_min_x = max(0, min_x)
+ visible_max_x = min(viewport_width, max_x)
+ visible_min_y = max(0, min_y)
+ visible_max_y = min(viewport_height, max_y)
+
+ visible_width = visible_max_x - visible_min_x
+ visible_height = visible_max_y - visible_min_y
+ visible_area = visible_width * visible_height
+
+ if visible_area > best_area:
+ best_area = visible_area
+ best_quad = quad
+
+ if not best_quad:
+ # No visible quad found, use the first quad anyway
+ best_quad = quads[0]
+ self.logger.warning('No visible quad found, using first quad')
+
+ # Calculate center point of the best quad
+ center_x = sum(best_quad[i] for i in range(0, 8, 2)) / 4
+ center_y = sum(best_quad[i] for i in range(1, 8, 2)) / 4
+
+ # Ensure click point is within viewport bounds
+ center_x = max(0, min(viewport_width - 1, center_x))
+ center_y = max(0, min(viewport_height - 1, center_y))
# Scroll element into view
try:
diff --git a/browser_use/browser/navigation_watchdog.py b/browser_use/browser/navigation_watchdog.py
deleted file mode 100644
index 13e970905..000000000
--- a/browser_use/browser/navigation_watchdog.py
+++ /dev/null
@@ -1,691 +0,0 @@
-"""Navigation watchdog for monitoring tab lifecycle and agent focus tracking."""
-
-import asyncio
-from typing import TYPE_CHECKING, Any, ClassVar
-
-from bubus import BaseEvent
-from pydantic import PrivateAttr
-
-from browser_use.browser.events import (
- AgentFocusChangedEvent,
- BrowserConnectedEvent,
- BrowserErrorEvent,
- BrowserStoppedEvent,
- NavigateToUrlEvent,
- NavigationCompleteEvent,
- NavigationStartedEvent,
- SwitchTabEvent,
- TabClosedEvent,
- TabCreatedEvent,
-)
-from browser_use.browser.watchdog_base import BaseWatchdog
-
-if TYPE_CHECKING:
- pass
-
-
-class NavigationWatchdog(BaseWatchdog):
- """Monitors tab creation, lifecycle events, and tracks agent focus."""
-
- # Event contracts
- LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
- BrowserConnectedEvent,
- BrowserStoppedEvent,
- TabCreatedEvent,
- TabClosedEvent,
- SwitchTabEvent,
- NavigateToUrlEvent,
- NavigationCompleteEvent,
- ]
- EMITS: ClassVar[list[type[BaseEvent]]] = [
- AgentFocusChangedEvent,
- BrowserErrorEvent,
- TabClosedEvent,
- NavigationStartedEvent,
- NavigationCompleteEvent,
- ]
-
- # Track target close handlers
- _target_close_handlers: dict[str, Any] = PrivateAttr(default_factory=dict)
-
- async def attach_to_target(self, target_id: str) -> None:
- """Set up monitoring for a target - tracks target lifecycle."""
- # CDP doesn't have direct target close events, we'll handle via Target.targetDestroyed
- self._target_close_handlers[target_id] = True
-
- # ========== Browser Lifecycle Events ==========
-
- async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
- """Initialize agent focus when browser is connected."""
- self.logger.debug('[NavigationWatchdog] Browser connected, initializing agent focus')
- await self._initialize_agent_focus()
-
- async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None:
- """Clear agent focus when browser stops."""
- # Clear target handlers
- self._target_close_handlers.clear()
- if self.browser_session.agent_focus:
- self.browser_session.agent_focus.target_id = '' # Clear target_id
-
- # ========== Tab Lifecycle Events ==========
-
- async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
- """Handle new tab creation - track target and possibly move agent focus."""
- # self.logger.debug(f'[NavigationWatchdog] Tab created: {event.url}')
-
- # Get all targets and track the new one
- try:
- targets = await self.browser_session._cdp_get_all_pages()
- if event.tab_index < len(targets):
- target_id = targets[event.tab_index]['targetId']
- self._track_target(target_id)
- except Exception as e:
- self.logger.error(f'[NavigationWatchdog] Error tracking new target: {e}')
-
- # Only update focus if we're not already focused on this tab
- # (NavigateToUrlEvent with new_tab=True already handles focus change)
- current_tab_index = await self._get_current_tab_index()
- if current_tab_index != event.tab_index:
- # When a new tab is created by other means, agent typically focuses on it immediately
- await self._update_agent_focus_to_latest_tab()
-
- async def on_TabClosedEvent(self, event: TabClosedEvent) -> None:
- """Handle tab closure."""
- self.logger.debug(f'[NavigationWatchdog] Tab closed at index {event.tab_index}')
-
- # If the closed tab was the agent's current target, find a new one
- current_tab_index = await self._get_current_tab_index()
- if current_tab_index == event.tab_index:
- await self._find_new_agent_target()
- # No need to adjust tab_index since it's computed from target position
-
- async def on_SwitchTabEvent(self, event: SwitchTabEvent) -> None:
- """Handle explicit tab switches by the agent."""
- self.logger.debug(f'[NavigationWatchdog] Agent switching to tab {event.tab_index}')
-
- # Clear cached browser state when switching tabs since DOM will be different
- self.logger.debug('๐Ÿ”„ Tab switch, clearing cached browser state and DOM')
- self.browser_session._cached_browser_state_summary = None
- self.browser_session._cached_selector_map.clear()
-
- # Clear DOM watchdog cache if available
- if self.browser_session._dom_watchdog:
- self.browser_session._dom_watchdog.clear_cache()
-
- await self._switch_agent_focus_to_tab(event.tab_index)
-
- async def on_NavigateToUrlEvent(self, event: NavigateToUrlEvent) -> None:
- """Handle all navigation requests with security enforcement and complete logic."""
-
- # Check if browser session is still available
- if not self.browser_session:
- self.logger.debug('[NavigationWatchdog] No browser session available, ignoring navigation')
- return
-
- # SECURITY CHECK: Block disallowed URLs before navigation starts
- if not self._is_url_allowed(event.url):
- self.logger.warning(f'โ›”๏ธ Blocking navigation to disallowed URL: {event.url}')
- self.event_bus.dispatch(
- BrowserErrorEvent(
- error_type='NavigationBlocked',
- message=f'Navigation blocked to disallowed URL: {event.url}',
- details={'url': event.url, 'reason': 'not_in_allowed_domains'},
- )
- )
- return
-
- try:
- # Handle new tab navigation
- if event.new_tab:
- # Check if we can reuse an existing about:blank tab
- target_id = None
- tab_index = None
- targets = await self.browser_session._cdp_get_all_pages()
-
- # Look for a reusable about:blank tab (not the current one)
- current_target_id = self.browser_session.agent_focus.target_id if self.browser_session.agent_focus else None
- for idx, target in enumerate(targets):
- target_url = target.get('url', '')
- # Reuse if it's about:blank and not the current tab
- if target_url == 'about:blank' and target['targetId'] != current_target_id:
- target_id = target['targetId']
- tab_index = idx
- self.logger.info(f'[NavigationWatchdog] Reusing existing about:blank tab at index {tab_index}')
- break
-
- # Create new tab if no reusable one found
- if target_id is None:
- target_id = await self.browser_session._cdp_create_new_page(event.url or 'about:blank')
- self._track_target(target_id)
- # Get updated targets list and tab index
- targets = await self.browser_session._cdp_get_all_pages()
- tab_index = len(targets) - 1
- self.logger.info(f'[NavigationWatchdog] Created new tab at index {tab_index}')
-
- # Update agent focus to the new/reused tab FIRST before dispatching any events
- # This ensures the agent is looking at the right tab when events are processed
- self.logger.info(f'[NavigationWatchdog] Attaching CDP session to tab target: {target_id}')
- await self.browser_session.get_or_create_cdp_session(target_id)
- await self._dispatch_focus_changed()
-
- # Navigate to URL if provided
- # Note: For new tabs, navigation already happened in _cdp_create_new_page if URL was provided
- # For reused tabs, we need to navigate them now
- if event.url:
- # Dispatch navigation started event
- from browser_use.browser.events import NavigationStartedEvent
-
- self.event_bus.dispatch(
- NavigationStartedEvent(
- tab_index=tab_index,
- url=event.url,
- )
- )
-
- try:
- # For reused tabs, navigate to the URL
- # For new tabs, navigation already happened in _cdp_create_new_page
- current_url = await self._get_target_url(target_id)
- if current_url == 'about:blank':
- # This is a reused tab, need to navigate it
- self.logger.info(f'[NavigationWatchdog] Navigating reused tab to {event.url}')
- await self.browser_session._cdp_navigate(event.url, target_id)
-
- # Wait a bit to ensure the page has loaded
- await asyncio.sleep(0.5)
-
- # Just dispatch completion
- response_status = None # CDP doesn't return response status directly
-
- # Dispatch completion event
- self.event_bus.dispatch(
- NavigationCompleteEvent(
- tab_index=tab_index,
- url=event.url,
- status=response_status,
- )
- )
-
- # Network monitoring is handled by CrashWatchdog
-
- except TimeoutError:
- # Handle navigation timeout for new tab
- error_message = 'Navigation timed out after 30 seconds'
- loading_status = 'Navigation timeout: 30 seconds'
- self.logger.warning(f'[NavigationWatchdog] Navigation to {event.url} timed out in new tab')
-
- self.event_bus.dispatch(
- NavigationCompleteEvent(
- tab_index=tab_index,
- url=event.url,
- status=None,
- error_message=error_message,
- loading_status=loading_status,
- )
- )
- except Exception as e:
- # Handle other navigation errors for new tab
- error_message = str(e)
- loading_status = None
- if 'timeout' in error_message.lower() or 'timed out' in error_message.lower():
- loading_status = f'Navigation timeout: {error_message}'
-
- self.event_bus.dispatch(
- NavigationCompleteEvent(
- tab_index=tab_index,
- url=event.url,
- status=None,
- error_message=error_message,
- loading_status=loading_status,
- )
- )
-
- # Get the actual URL from the target (may still be navigating)
- target_url = await self._get_target_url(target_id)
- self.logger.info(f'[NavigationWatchdog] New tab URL: {target_url or event.url}')
-
- # ONLY dispatch TabCreatedEvent for NEW tabs AFTER focus has been updated
- from browser_use.browser.events import TabCreatedEvent
- self.event_bus.dispatch(
- TabCreatedEvent(
- tab_index=tab_index,
- url=target_url or event.url,
- )
- )
- return
-
- # Handle timeout navigation
- if event.timeout_ms is not None:
- # Create a new navigation event without timeout for the actual navigation
- nav_event = self.event_bus.dispatch(NavigateToUrlEvent(url=event.url, wait_until=event.wait_until))
- try:
- await asyncio.wait_for(nav_event, timeout=event.timeout_ms / 1000.0)
- except TimeoutError:
- self.logger.warning(f'Navigation to {event.url} timed out after {event.timeout_ms}ms')
- # Dispatch NavigationCompleteEvent for timeout error
- self.event_bus.dispatch(
- NavigationCompleteEvent(
- tab_index=0, # Default tab index since we don't have access to the actual target
- url=event.url,
- status=None,
- error_message=f'Navigation timed out after {event.timeout_ms}ms',
- loading_status=f'Navigation timeout after {event.timeout_ms}ms',
- )
- )
- return
-
- # Handle standard navigation
- # Get the target to navigate - use current agent target or create one
- target_id = None
- try:
- target_id = await self.get_or_create_target()
- except Exception as e:
- self.logger.warning(f'[NavigationWatchdog] get_or_create_target failed: {e}')
- # Fallback to first target
- try:
- targets = await self.browser_session._cdp_get_all_pages()
- if targets:
- target_id = targets[0]['targetId']
- except Exception as e2:
- self.logger.error(f'[NavigationWatchdog] Failed to get any targets: {e2}')
- raise
-
- if not target_id:
- raise ValueError('Failed to get or create a target for navigation')
-
- # Get tab index
- tab_index = await self._get_tab_index_for_target(target_id)
-
- # Dispatch navigation started event
- from browser_use.browser.events import NavigationStartedEvent
-
- self.event_bus.dispatch(
- NavigationStartedEvent(
- tab_index=tab_index,
- url=event.url,
- )
- )
-
- # Perform navigation with timeout protection using CDP
- await asyncio.wait_for(
- self.browser_session._cdp_navigate(event.url, target_id),
- timeout=30.0, # 30 second timeout to prevent hanging
- )
- response_status = None # CDP doesn't return response status directly
-
- # Dispatch completion event
- self.event_bus.dispatch(
- NavigationCompleteEvent(
- tab_index=tab_index,
- url=event.url,
- status=response_status,
- )
- )
-
- # Network monitoring is handled by CrashWatchdog
-
- except TimeoutError:
- # Handle navigation timeout for standard navigation
- error_message = 'Navigation timed out after 30 seconds'
- loading_status = 'Navigation timeout: 30 seconds'
- self.logger.warning(f'[NavigationWatchdog] Navigation to {event.url} timed out in standard navigation')
-
- # Get tab index for timeout error reporting
- tab_index = await self._get_tab_index_for_target(target_id) if target_id else 0
-
- # Dispatch NavigationCompleteEvent with timeout error details
- self.event_bus.dispatch(
- NavigationCompleteEvent(
- tab_index=tab_index,
- url=event.url,
- status=None,
- error_message=error_message,
- loading_status=loading_status,
- )
- )
- except Exception as e:
- # Handle other navigation errors
- error_message = str(e)
- loading_status = None
-
- # Check for timeout errors
- if 'timeout' in error_message.lower() or 'timed out' in error_message.lower():
- loading_status = f'Navigation timeout: {error_message}'
-
- # Get tab index for error reporting
- tab_index = 0 # Default to first tab for errors
-
- # Dispatch NavigationCompleteEvent with error details
- self.event_bus.dispatch(
- NavigationCompleteEvent(
- tab_index=tab_index,
- url=event.url,
- status=None,
- error_message=error_message,
- loading_status=loading_status,
- )
- )
-
- async def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None:
- """Update agent focus when navigation completes and enforce security."""
- # Clear cached browser state and DOM after navigation
- self.logger.debug('๐Ÿ”„ Navigation complete, clearing cached browser state and DOM')
- self.browser_session._cached_browser_state_summary = None
- self.browser_session._cached_selector_map.clear()
-
- # Clear DOM watchdog cache if available
- if self.browser_session._dom_watchdog:
- self.browser_session._dom_watchdog.clear_cache()
-
- # Check if the navigated URL is allowed
- if not self._is_url_allowed(event.url):
- self.logger.warning(f'โ›”๏ธ Navigation to non-allowed URL detected: {event.url}')
- # Dispatch browser error
- self.event_bus.dispatch(
- BrowserErrorEvent(
- error_type='NavigationBlocked',
- message=f'Navigation to non-allowed URL: {event.url}',
- details={'url': event.url, 'tab_index': event.tab_index},
- )
- )
- # Close the target that navigated to the disallowed URL
- try:
- targets = await self.browser_session._cdp_get_all_pages()
- if 0 <= event.tab_index < len(targets):
- target_id = targets[event.tab_index]['targetId']
- await self.browser_session._cdp_close_page(target_id)
- self.logger.info(f'โ›”๏ธ Closed target with non-allowed URL: {event.url}')
- except Exception as e:
- self.logger.error(f'โ›”๏ธ Failed to close target with non-allowed URL: {str(e)}')
- return
-
- # Don't switch focus - the tab that navigated should already have focus
- # This prevents incorrectly switching back to the wrong tab after new tab creation
- self.logger.debug(f'[NavigationWatchdog] Navigation complete for tab {event.tab_index}, keeping current focus')
-
- # ========== Tab Tracking Methods ==========
-
- def _track_target(self, target_id: str) -> None:
- """Track a target for lifecycle events."""
- # self.logger.debug(f'[NavigationWatchdog] Started tracking target: {target_id}')
- self._target_close_handlers[target_id] = True
-
- def _handle_target_close(self, target_id: str) -> None:
- """Handle target close event."""
- try:
- # Try to get tab index before target is fully closed
- tab_index = asyncio.run(self._get_tab_index_for_target(target_id))
- if tab_index == -1:
- tab_index = 0 # Fallback if target already removed
-
- # Emit TabClosedEvent
- self.event_bus.dispatch(TabClosedEvent(tab_index=tab_index))
- self.logger.info(f'[NavigationWatchdog] Target closed, emitted TabClosedEvent for tab {tab_index}')
-
- # Clean up the handler from our tracking dict
- self._target_close_handlers.pop(target_id, None)
-
- except Exception as e:
- self.logger.error(f'[NavigationWatchdog] Error handling target close: {e}')
-
- # ========== Security Methods ==========
-
- def _is_url_allowed(self, url: str) -> bool:
- """Check if a URL is allowed based on the allowed_domains configuration.
-
- Args:
- url: The URL to check
-
- Returns:
- True if the URL is allowed, False otherwise
- """
- # If no allowed_domains specified, allow all URLs
- if not self.browser_session.browser_profile.allowed_domains:
- return True
-
- # Always allow internal browser targets
- if url in ['about:blank', 'chrome://new-tab-page/', 'chrome://new-tab-page', 'chrome://newtab/']:
- return True
-
- # Parse the URL to extract components
- from urllib.parse import urlparse
-
- try:
- parsed = urlparse(url)
- except Exception:
- # Invalid URL
- return False
-
- # Get the actual host (domain)
- host = parsed.hostname
- if not host:
- return False
-
- # Full URL for matching (scheme + host)
- full_url_pattern = f'{parsed.scheme}://{host}'
-
- # Check each allowed domain pattern
- for pattern in self.browser_session.browser_profile.allowed_domains:
- # Handle glob patterns
- if '*' in pattern:
- import fnmatch
-
- # Check if pattern matches the host
- if pattern.startswith('*.'):
- # Pattern like *.example.com should match subdomains and main domain
- # But only for http/https URLs unless scheme is specified
- domain_part = pattern[2:] # Remove *.
- if host == domain_part or host.endswith('.' + domain_part):
- # Only match http/https URLs for domain-only patterns
- if parsed.scheme in ['http', 'https']:
- return True
- elif pattern.endswith('/*'):
- # Pattern like brave://* should match any brave:// URL
- prefix = pattern[:-1] # Remove the * at the end
- if url.startswith(prefix):
- return True
- else:
- # Use fnmatch for other glob patterns
- if fnmatch.fnmatch(host, pattern):
- return True
- else:
- # Exact match
- if pattern.startswith(('http://', 'https://', 'chrome://', 'brave://', 'file://')):
- # Full URL pattern
- if url.startswith(pattern):
- return True
- else:
- # Domain-only pattern
- if host == pattern:
- return True
-
- return False
-
- # ========== Agent Focus Tracking Methods ==========
-
- @property
- def current_agent_target(self) -> str | None:
- """Get the current target ID the agent is focused on."""
- return self.browser_session.agent_focus.target_id if self.browser_session.agent_focus else None
-
- @property
- def current_agent_tab_index(self) -> int:
- """Get the current tab index the agent is focused on."""
- return asyncio.run(self._get_current_tab_index())
-
- @property
- def tab_index(self) -> int:
- """Get the current tab index derived from target position."""
- return asyncio.run(self._get_current_tab_index())
-
- async def _get_current_tab_index(self) -> int:
- """Get the current tab index from target position."""
- if self.browser_session.agent_focus is None or self.browser_session.agent_focus.target_id is None:
- return 0
- return await self._get_tab_index_for_target(self.browser_session.agent_focus.target_id)
-
- async def _get_tab_index_for_target(self, target_id: str) -> int:
- """Get tab index for a specific target ID."""
- try:
- targets = await self.browser_session._cdp_get_all_pages()
- for i, target in enumerate(targets):
- if target['targetId'] == target_id:
- return i
- except Exception:
- pass
- return -1
-
- async def _get_target_url(self, target_id: str) -> str | None:
- """Get the URL for a specific target ID."""
- try:
- targets = await self.browser_session._cdp_get_all_pages()
- for target in targets:
- if target['targetId'] == target_id:
- return target.get('url', '')
- except Exception:
- pass
- return None
-
- async def _initialize_agent_focus(self) -> None:
- """Initialize agent focus to first available target and dispatch TabCreatedEvent for initial pages."""
- # Get all targets and filter for valid page/tab targets
- all_targets = await self.browser_session.cdp_client.send.Target.getTargets()
- targets = [
- t
- for t in all_targets.get('targetInfos', [])
- if self.browser_session._is_valid_target(t) and t.get('type') in ('page', 'tab')
- ]
-
- if targets:
- # Set initial focus to first target BEFORE dispatching events
- await self.browser_session.get_or_create_cdp_session(targets[0]['targetId'])
-
- # Dispatch TabCreatedEvent for all initial pages so watchdogs can process them
- from browser_use.browser.events import TabCreatedEvent
-
- for i, target in enumerate(targets):
- target_url = target.get('url', '')
- self.logger.debug(f'[NavigationWatchdog] Dispatching TabCreatedEvent for initial tab {i}: {target_url}')
- self.event_bus.dispatch(TabCreatedEvent(tab_index=i, url=target_url))
-
- # Dispatch focus changed event
- await self._dispatch_focus_changed()
- target_url = self.browser_session.agent_focus.url if self.browser_session.agent_focus else ''
- self.logger.info(f'[NavigationWatchdog] Initial agent focus set to tab 0: {target_url}')
-
- async def _update_agent_focus_to_latest_tab(self) -> None:
- """Update agent focus to the latest (most recently created) tab."""
- targets = await self.browser_session._cdp_get_all_pages()
- if targets:
- # Focus on the last target (most recently created)
- # Properly attach CDP session to the new target
- await self.browser_session.get_or_create_cdp_session(targets[-1]['targetId'])
- await self._dispatch_focus_changed()
- target_url = None
- if self.browser_session.agent_focus and self.browser_session.agent_focus.target_id:
- target_url = await self._get_target_url(self.browser_session.agent_focus.target_id)
- tab_index = await self._get_current_tab_index()
- self.logger.info(f'[NavigationWatchdog] ๐Ÿ‘€ Agent focus moved to new tab {tab_index}: {target_url}')
-
- async def _switch_agent_focus_to_tab(self, tab_index: int) -> None:
- """Switch agent focus to a specific tab index."""
- targets = await self.browser_session._cdp_get_all_pages()
- if 0 <= tab_index < len(targets):
- target_id = targets[tab_index]['targetId']
- # Properly attach CDP session to the new target
- await self.browser_session.get_or_create_cdp_session(target_id)
- await self._dispatch_focus_changed()
- target_url = await self._get_target_url(target_id)
- self.logger.info(f'[NavigationWatchdog] Agent focus switched to tab {tab_index}: {target_url}')
-
- async def _find_new_agent_target(self) -> None:
- """Find a new target for agent focus when current one is closed."""
- targets = await self.browser_session._cdp_get_all_pages()
- if targets:
- # Try to stay at the same index, or go to the previous one
- current_index = await self._get_current_tab_index()
- new_index = min(current_index, len(targets) - 1)
- # Properly attach CDP session to the new target
- await self.browser_session.get_or_create_cdp_session(targets[new_index]['targetId'])
- await self._dispatch_focus_changed()
- target_url = None
- if self.browser_session.agent_focus and self.browser_session.agent_focus.target_id:
- target_url = await self._get_target_url(self.browser_session.agent_focus.target_id)
- self.logger.info(f'[NavigationWatchdog] Agent focus moved to tab {new_index}: {target_url}')
- else:
- if self.browser_session and self.browser_session.agent_focus:
- self.browser_session.agent_focus.target_id = ''
-
- async def _dispatch_focus_changed(self) -> None:
- """Dispatch event when agent focus changes."""
- # Clear cached browser state when focus changes since we're on a different tab now
- self.logger.debug('๐Ÿ”„ Agent focus changed, clearing cached browser state and DOM')
- self.browser_session._cached_browser_state_summary = None
- self.browser_session._cached_selector_map.clear()
-
- # Clear DOM watchdog cache if available
- if self.browser_session._dom_watchdog:
- self.browser_session._dom_watchdog.clear_cache()
-
- if self.browser_session.agent_focus and self.browser_session.agent_focus.target_id:
- try:
- # Get URL asynchronously
- target_url = await self._get_target_url(self.browser_session.agent_focus.target_id)
- tab_index = await self._get_current_tab_index()
- self.event_bus.dispatch(
- AgentFocusChangedEvent(
- tab_index=tab_index,
- url=target_url or '',
- )
- )
- except Exception as e:
- self.logger.error(f'[NavigationWatchdog] Error dispatching focus change: {e}')
-
- async def get_or_create_target(self) -> str:
- """Get current agent target or create a new one if none exists."""
- if self.browser_session.agent_focus and self.browser_session.agent_focus.target_id:
- # Check if current target URL is still allowed
- target_url = await self._get_target_url(self.browser_session.agent_focus.target_id)
- if target_url and not self._is_url_allowed(target_url):
- self.logger.warning(f'โ›”๏ธ Current target URL no longer allowed: {target_url}')
- # Close the current target and find/create a new one
- try:
- await self.browser_session._cdp_close_page(self.browser_session.agent_focus.target_id)
- except Exception:
- pass
- self.browser_session.agent_focus.target_id = ''
- else:
- return self.browser_session.agent_focus.target_id
-
- try:
- targets = await self.browser_session._cdp_get_all_pages()
- for target in targets:
- target_url = target.get('url', '')
- if self._is_url_allowed(target_url):
- # Properly attach CDP session to the target
- await self.browser_session.get_or_create_cdp_session(target['targetId'])
- await self._dispatch_focus_changed()
- if self.browser_session.agent_focus:
- return self.browser_session.agent_focus.target_id
- except Exception as e:
- self.logger.warning(f'[NavigationWatchdog] Failed to get targets: {e}')
- # Don't pass TimeoutError, re-raise other exceptions
- if not isinstance(e, TimeoutError):
- raise
-
- # No tabs available or no allowed tabs, create a new one with about:blank
- self.logger.info('[NavigationWatchdog] No active target, creating new tab')
-
- nav_event = self.event_bus.dispatch(NavigateToUrlEvent(url='about:blank', new_tab=True))
- await nav_event # Wait for navigation to complete
-
- # Try to get the new target
- targets = await self.browser_session._cdp_get_all_pages()
- if targets:
- # Properly attach CDP session to the new target
- await self.browser_session.get_or_create_cdp_session(targets[-1]['targetId'])
- await self._dispatch_focus_changed()
- if self.browser_session.agent_focus:
- return self.browser_session.agent_focus.target_id
-
- raise ValueError('Failed to create or find an active target')
diff --git a/browser_use/browser/profile.py b/browser_use/browser/profile.py
index b5b254aef..ded78c8e0 100644
--- a/browser_use/browser/profile.py
+++ b/browser_use/browser/profile.py
@@ -586,6 +586,10 @@ class BrowserProfile(BrowserConnectArgs, BrowserLaunchPersistentContextArgs, Bro
default_factory=lambda: {'width': 0, 'height': 0},
description='Window position to use for the browser x,y from the top left when headless=False.',
)
+ cross_origin_iframes: bool = Field(
+ default=False,
+ description='Enable cross-origin iframe support (OOPIF/Out-of-Process iframes). When False (default), only same-origin frames are processed to avoid complexity and hanging.',
+ )
# --- Page load/wait timings ---
default_navigation_timeout: float | None = Field(default=None, description='Default page navigation timeout.')
diff --git a/browser_use/browser/security_watchdog.py b/browser_use/browser/security_watchdog.py
new file mode 100644
index 000000000..cac780f61
--- /dev/null
+++ b/browser_use/browser/security_watchdog.py
@@ -0,0 +1,166 @@
+"""Security watchdog for enforcing URL access policies."""
+
+import asyncio
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from bubus import BaseEvent
+from pydantic import PrivateAttr
+
+from browser_use.browser.events import (
+ BrowserErrorEvent,
+ NavigateToUrlEvent,
+ NavigationCompleteEvent,
+ TabCreatedEvent,
+)
+from browser_use.browser.watchdog_base import BaseWatchdog
+
+if TYPE_CHECKING:
+ pass
+
+
+class SecurityWatchdog(BaseWatchdog):
+ """Monitors and enforces security policies for URL access."""
+
+ # Event contracts
+ LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
+ NavigateToUrlEvent,
+ NavigationCompleteEvent,
+ TabCreatedEvent,
+ ]
+ EMITS: ClassVar[list[type[BaseEvent]]] = [
+ BrowserErrorEvent,
+ ]
+
+ async def on_NavigateToUrlEvent(self, event: NavigateToUrlEvent) -> None:
+ """Check if navigation URL is allowed before navigation starts."""
+ # Security check BEFORE navigation
+ if not self._is_url_allowed(event.url):
+ self.logger.warning(f'โ›”๏ธ Blocking navigation to disallowed URL: {event.url}')
+ self.event_bus.dispatch(
+ BrowserErrorEvent(
+ error_type='NavigationBlocked',
+ message=f'Navigation blocked to disallowed URL: {event.url}',
+ details={'url': event.url, 'reason': 'not_in_allowed_domains'},
+ )
+ )
+ # Stop event propagation by raising exception
+ raise ValueError(f'Navigation to {event.url} blocked by security policy')
+
+ async def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None:
+ """Check if navigated URL is allowed and close tab if not."""
+ # Check if the navigated URL is allowed (in case of redirects)
+ if not self._is_url_allowed(event.url):
+ self.logger.warning(f'โ›”๏ธ Navigation to non-allowed URL detected: {event.url}')
+
+ # Dispatch browser error
+ self.event_bus.dispatch(
+ BrowserErrorEvent(
+ error_type='NavigationBlocked',
+ message=f'Navigation to non-allowed URL: {event.url}',
+ details={'url': event.url, 'tab_index': event.tab_index},
+ )
+ )
+
+ # Close the target that navigated to the disallowed URL
+ try:
+ targets = await self.browser_session._cdp_get_all_pages()
+ if 0 <= event.tab_index < len(targets):
+ target_id = targets[event.tab_index]['targetId']
+ await self.browser_session._cdp_close_page(target_id)
+ self.logger.info(f'โ›”๏ธ Closed target with non-allowed URL: {event.url}')
+ except Exception as e:
+ self.logger.error(f'โ›”๏ธ Failed to close target with non-allowed URL: {str(e)}')
+
+ async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
+ """Check if new tab URL is allowed."""
+ if not self._is_url_allowed(event.url):
+ self.logger.warning(f'โ›”๏ธ New tab created with disallowed URL: {event.url}')
+
+ # Dispatch error and try to close the tab
+ self.event_bus.dispatch(
+ BrowserErrorEvent(
+ error_type='TabCreationBlocked',
+ message=f'Tab created with non-allowed URL: {event.url}',
+ details={'url': event.url, 'tab_index': event.tab_index},
+ )
+ )
+
+ # Try to close the offending tab
+ try:
+ targets = await self.browser_session._cdp_get_all_pages()
+ if 0 <= event.tab_index < len(targets):
+ target_id = targets[event.tab_index]['targetId']
+ await self.browser_session._cdp_close_page(target_id)
+ self.logger.info(f'โ›”๏ธ Closed new tab with non-allowed URL: {event.url}')
+ except Exception as e:
+ self.logger.error(f'โ›”๏ธ Failed to close new tab with non-allowed URL: {str(e)}')
+
+ def _is_url_allowed(self, url: str) -> bool:
+ """Check if a URL is allowed based on the allowed_domains configuration.
+
+ Args:
+ url: The URL to check
+
+ Returns:
+ True if the URL is allowed, False otherwise
+ """
+ # If no allowed_domains specified, allow all URLs
+ if not self.browser_session.browser_profile.allowed_domains:
+ return True
+
+ # Always allow internal browser targets
+ if url in ['about:blank', 'chrome://new-tab-page/', 'chrome://new-tab-page', 'chrome://newtab/']:
+ return True
+
+ # Parse the URL to extract components
+ from urllib.parse import urlparse
+
+ try:
+ parsed = urlparse(url)
+ except Exception:
+ # Invalid URL
+ return False
+
+ # Get the actual host (domain)
+ host = parsed.hostname
+ if not host:
+ return False
+
+ # Full URL for matching (scheme + host)
+ full_url_pattern = f'{parsed.scheme}://{host}'
+
+ # Check each allowed domain pattern
+ for pattern in self.browser_session.browser_profile.allowed_domains:
+ # Handle glob patterns
+ if '*' in pattern:
+ import fnmatch
+
+ # Check if pattern matches the host
+ if pattern.startswith('*.'):
+ # Pattern like *.example.com should match subdomains and main domain
+ domain_part = pattern[2:] # Remove *.
+ if host == domain_part or host.endswith('.' + domain_part):
+ # Only match http/https URLs for domain-only patterns
+ if parsed.scheme in ['http', 'https']:
+ return True
+ elif pattern.endswith('/*'):
+ # Pattern like brave://* should match any brave:// URL
+ prefix = pattern[:-1] # Remove the * at the end
+ if url.startswith(prefix):
+ return True
+ else:
+ # Use fnmatch for other glob patterns
+ if fnmatch.fnmatch(host, pattern):
+ return True
+ else:
+ # Exact match
+ if pattern.startswith(('http://', 'https://', 'chrome://', 'brave://', 'file://')):
+ # Full URL pattern
+ if url.startswith(pattern):
+ return True
+ else:
+ # Domain-only pattern
+ if host == pattern:
+ return True
+
+ return False
\ No newline at end of file
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index e64289a52..2f4d2a0b7 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -24,6 +24,7 @@ for logger_name in ['cdp_use', 'cdp_use.client', 'cdp_use.cdp', 'cdp_use.cdp.reg
cdp_logger.propagate = False # Don't propagate to root logger
from browser_use.browser.events import (
+ AgentFocusChangedEvent,
BrowserConnectedEvent,
BrowserErrorEvent,
BrowserLaunchEvent,
@@ -31,6 +32,12 @@ from browser_use.browser.events import (
BrowserStateRequestEvent,
BrowserStopEvent,
BrowserStoppedEvent,
+ NavigateToUrlEvent,
+ NavigationCompleteEvent,
+ NavigationStartedEvent,
+ SwitchTabEvent,
+ TabClosedEvent,
+ TabCreatedEvent,
)
from browser_use.browser.profile import BrowserProfile
from browser_use.browser.views import BrowserStateSummary, TabInfo
@@ -58,7 +65,10 @@ def _log_glob_warning(domain: str, glob: str, logger: logging.Logger):
class CDPSession(BaseModel):
- """Info about a single CDP session bound to a specific target."""
+ """Info about a single CDP session bound to a specific target.
+
+ Can optionally use its own WebSocket connection for better isolation.
+ """
model_config = ConfigDict(arbitrary_types_allowed=True, revalidate_instances='never')
@@ -68,14 +78,45 @@ class CDPSession(BaseModel):
session_id: str
title: str = 'Unknown title'
url: str = 'about:blank'
+
+ # Track if this session owns its CDP client (for cleanup)
+ owns_cdp_client: bool = False
@classmethod
- async def for_target(cls, cdp_client: CDPClient, target_id: str):
- cdp_session = cls(
- cdp_client=cdp_client,
- target_id=target_id,
- session_id='connecting',
- )
+ async def for_target(cls, cdp_client: CDPClient, target_id: str, create_own_connection: bool = False, cdp_url: str | None = None):
+ """Create a CDP session for a target.
+
+ Args:
+ cdp_client: Existing CDP client to use (or just for reference if creating own)
+ target_id: Target ID to attach to
+ create_own_connection: If True, create a dedicated WebSocket connection for this target
+ cdp_url: CDP URL (required if create_own_connection is True)
+ """
+ if create_own_connection:
+ if not cdp_url:
+ raise ValueError('cdp_url required when create_own_connection=True')
+ # Create a new CDP client with its own WebSocket connection
+ import logging
+ logger = logging.getLogger(f'browser_use.CDPSession.{target_id[-4:]}')
+ logger.info(f'๐Ÿ”Œ Creating dedicated WebSocket connection for target {target_id}')
+
+ target_cdp_client = CDPClient(cdp_url)
+ await target_cdp_client.start()
+
+ cdp_session = cls(
+ cdp_client=target_cdp_client,
+ target_id=target_id,
+ session_id='connecting',
+ owns_cdp_client=True,
+ )
+ else:
+ # Use shared CDP client
+ cdp_session = cls(
+ cdp_client=cdp_client,
+ target_id=target_id,
+ session_id='connecting',
+ owns_cdp_client=False,
+ )
return await cdp_session.attach()
async def attach(self) -> Self:
@@ -107,6 +148,14 @@ class CDPSession(BaseModel):
self.title = target_info['title']
self.url = target_info['url']
return self
+
+ async def disconnect(self) -> None:
+ """Disconnect and cleanup if this session owns its CDP client."""
+ if self.owns_cdp_client and self.cdp_client:
+ try:
+ await self.cdp_client.stop()
+ except Exception:
+ pass # Ignore errors during cleanup
async def get_tab_info(self) -> TabInfo:
target_info = await self.get_target_info()
@@ -164,7 +213,7 @@ class BrowserSession(BaseModel):
_crash_watchdog: Any | None = PrivateAttr(default=None)
_downloads_watchdog: Any | None = PrivateAttr(default=None)
_aboutblank_watchdog: Any | None = PrivateAttr(default=None)
- _navigation_watchdog: Any | None = PrivateAttr(default=None)
+ _security_watchdog: Any | None = PrivateAttr(default=None)
_storage_state_watchdog: Any | None = PrivateAttr(default=None)
_local_browser_watchdog: Any | None = PrivateAttr(default=None)
_default_action_watchdog: Any | None = PrivateAttr(default=None)
@@ -208,7 +257,7 @@ class BrowserSession(BaseModel):
self._crash_watchdog = None
self._downloads_watchdog = None
self._aboutblank_watchdog = None
- self._navigation_watchdog = None
+ self._security_watchdog = None
self._storage_state_watchdog = None
self._local_browser_watchdog = None
self._default_action_watchdog = None
@@ -231,6 +280,10 @@ class BrowserSession(BaseModel):
# Register BrowserSession's event handlers
self.event_bus.on(BrowserStartEvent, self.on_BrowserStartEvent)
self.event_bus.on(BrowserStopEvent, self.on_BrowserStopEvent)
+ # Core navigation events that shouldn't be in optional watchdogs
+ self.event_bus.on(NavigateToUrlEvent, self.on_NavigateToUrlEvent)
+ self.event_bus.on(SwitchTabEvent, self.on_SwitchTabEvent)
+ self.event_bus.on(TabClosedEvent, self.on_TabClosedEvent)
async def start(self) -> None:
"""Start the browser session."""
@@ -294,6 +347,156 @@ class BrowserSession(BaseModel):
)
)
raise
+
+ async def on_NavigateToUrlEvent(self, event: NavigateToUrlEvent) -> None:
+ """Handle navigation requests - core browser functionality."""
+ if not self.agent_focus:
+ self.logger.warning('Cannot navigate - browser not connected')
+ return
+
+ try:
+ # Find or create target for navigation
+ target_id = None
+ tab_index = 0
+
+ if event.new_tab:
+ # Look for existing about:blank tab that's not the current one
+ targets = await self._cdp_get_all_pages()
+ current_target_id = self.agent_focus.target_id if self.agent_focus else None
+
+ for idx, target in enumerate(targets):
+ if target.get('url') == 'about:blank' and target['targetId'] != current_target_id:
+ target_id = target['targetId']
+ tab_index = idx
+ self.logger.info(f'Reusing existing about:blank tab at index {tab_index}')
+ break
+
+ # Create new tab if no reusable one found
+ if not target_id:
+ target_id = await self._cdp_create_new_page('about:blank')
+ targets = await self._cdp_get_all_pages()
+ tab_index = len(targets) - 1
+ self.logger.info(f'Created new tab at index {tab_index}')
+ # Dispatch TabCreatedEvent for new tab
+ self.event_bus.dispatch(TabCreatedEvent(tab_index=tab_index, url='about:blank'))
+ else:
+ # Use current tab
+ target_id = self.agent_focus.target_id
+ tab_index = await self.get_tab_index(target_id)
+
+ # Activate target (bring to foreground)
+ await self.cdp_client.send.Target.activateTarget(params={'targetId': target_id})
+
+ # Update agent focus to the target
+ self.agent_focus = await self.get_or_create_cdp_session(target_id)
+
+ # Clear any cached state since we're navigating
+ if self._dom_watchdog:
+ self._dom_watchdog.clear_cache()
+
+ # Dispatch focus changed event
+ self.event_bus.dispatch(AgentFocusChangedEvent(
+ tab_index=tab_index,
+ url=event.url,
+ ))
+
+ # Dispatch navigation started
+ self.event_bus.dispatch(NavigationStartedEvent(
+ tab_index=tab_index,
+ url=event.url,
+ ))
+
+ # Navigate to URL
+ await self.agent_focus.cdp_client.send.Page.navigate(
+ params={'url': event.url},
+ session_id=self.agent_focus.session_id
+ )
+
+ # Wait a bit to ensure page starts loading
+ await asyncio.sleep(0.5)
+
+ # Dispatch navigation complete
+ self.event_bus.dispatch(NavigationCompleteEvent(
+ tab_index=tab_index,
+ url=event.url,
+ status=None, # CDP doesn't provide status directly
+ ))
+
+ # Note: These should be handled by dedicated watchdogs:
+ # - Security checks (security_watchdog)
+ # - Page health checks (crash_watchdog)
+ # - Dialog handling (dialog_watchdog)
+ # - Download handling (downloads_watchdog)
+ # - DOM rebuilding (dom_watchdog)
+
+ except Exception as e:
+ self.logger.error(f'Navigation failed: {e}')
+ self.event_bus.dispatch(NavigationCompleteEvent(
+ tab_index=tab_index if 'tab_index' in locals() else 0,
+ url=event.url,
+ status=None,
+ error_message=str(e),
+ ))
+
+ async def on_SwitchTabEvent(self, event: SwitchTabEvent) -> None:
+ """Handle tab switching - core browser functionality."""
+ if not self.agent_focus:
+ self.logger.warning('Cannot switch tabs - browser not connected')
+ return
+
+ targets = await self._cdp_get_all_pages()
+ if 0 <= event.tab_index < len(targets):
+ target_id = targets[event.tab_index]['targetId']
+
+ # Activate target
+ await self.cdp_client.send.Target.activateTarget(params={'targetId': target_id})
+
+ # Update agent focus
+ self.agent_focus = await self.get_or_create_cdp_session(target_id)
+
+ # Clear cached state when switching tabs
+ if self._dom_watchdog:
+ self._dom_watchdog.clear_cache()
+
+ # Dispatch focus changed event
+ target_url = targets[event.tab_index].get('url', '')
+ self.event_bus.dispatch(AgentFocusChangedEvent(
+ tab_index=event.tab_index,
+ url=target_url,
+ ))
+
+ self.logger.info(f'Switched to tab {event.tab_index}: {target_url}')
+ else:
+ self.logger.warning(f'Invalid tab index: {event.tab_index}')
+
+ async def on_TabClosedEvent(self, event: TabClosedEvent) -> None:
+ """Handle tab closure - update focus if needed."""
+ if not self.agent_focus:
+ return
+
+ # Get current tab index
+ current_tab_index = await self.get_tab_index(self.agent_focus.target_id)
+
+ # If the closed tab was the current one, find a new target
+ if current_tab_index == event.tab_index:
+ targets = await self._cdp_get_all_pages()
+ if targets:
+ # Try to stay at same index or go to previous
+ new_index = min(current_tab_index, len(targets) - 1)
+ if new_index >= 0:
+ new_target_id = targets[new_index]['targetId']
+ self.agent_focus = await self.get_or_create_cdp_session(new_target_id)
+
+ # Dispatch focus changed
+ self.event_bus.dispatch(AgentFocusChangedEvent(
+ tab_index=new_index,
+ url=targets[new_index].get('url', ''),
+ ))
+ self.logger.info(f'Moved focus to tab {new_index} after tab closure')
+ else:
+ self.agent_focus = None
+ else:
+ self.agent_focus = None
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
"""Handle browser stop request."""
@@ -440,7 +643,7 @@ class BrowserSession(BaseModel):
from browser_use.browser.dom_watchdog import DOMWatchdog
from browser_use.browser.downloads_watchdog import DownloadsWatchdog
from browser_use.browser.local_browser_watchdog import LocalBrowserWatchdog
- from browser_use.browser.navigation_watchdog import NavigationWatchdog
+ from browser_use.browser.security_watchdog import SecurityWatchdog
from browser_use.browser.screenshot_watchdog import ScreenshotWatchdog
from browser_use.browser.storage_state_watchdog import StorageStateWatchdog
@@ -478,17 +681,12 @@ class BrowserSession(BaseModel):
# self.event_bus.on(BrowserStopEvent, self._local_browser_watchdog.on_BrowserStopEvent)
await self._local_browser_watchdog.attach_to_session()
- # Initialize NavigationWatchdog
- NavigationWatchdog.model_rebuild()
- self._navigation_watchdog = NavigationWatchdog(event_bus=self.event_bus, browser_session=self)
- # self.event_bus.on(BrowserConnectedEvent, self._navigation_watchdog.on_BrowserConnectedEvent)
- # self.event_bus.on(BrowserStoppedEvent, self._navigation_watchdog.on_BrowserStoppedEvent)
- # self.event_bus.on(TabCreatedEvent, self._navigation_watchdog.on_TabCreatedEvent)
- # self.event_bus.on(TabClosedEvent, self._navigation_watchdog.on_TabClosedEvent)
- # self.event_bus.on(SwitchTabEvent, self._navigation_watchdog.on_SwitchTabEvent)
- # self.event_bus.on(NavigateToUrlEvent, self._navigation_watchdog.on_NavigateToUrlEvent)
- # self.event_bus.on(NavigationCompleteEvent, self._navigation_watchdog.on_NavigationCompleteEvent)
- await self._navigation_watchdog.attach_to_session()
+ # Initialize SecurityWatchdog (replaces NavigationWatchdog for security checks only)
+ SecurityWatchdog.model_rebuild()
+ self._security_watchdog = SecurityWatchdog(event_bus=self.event_bus, browser_session=self)
+ # Core navigation is now handled in BrowserSession directly
+ # SecurityWatchdog only handles security policy enforcement
+ await self._security_watchdog.attach_to_session()
# Initialize AboutBlankWatchdog
AboutBlankWatchdog.model_rebuild()
@@ -634,6 +832,19 @@ class BrowserSession(BaseModel):
except Exception as e:
self.logger.warning(f'Failed to create CDP session: {e}')
raise
+
+ # Dispatch TabCreatedEvent for all initial tabs (so watchdogs can initialize)
+ # This replaces the duplicated logic from navigation_watchdog's _initialize_agent_focus
+ for idx, target in enumerate(page_targets):
+ target_url = target.get('url', '')
+ self.logger.debug(f'Dispatching TabCreatedEvent for initial tab {idx}: {target_url}')
+ self.event_bus.dispatch(TabCreatedEvent(tab_index=idx, url=target_url))
+
+ # Dispatch initial focus event
+ if page_targets:
+ initial_url = page_targets[0].get('url', '')
+ self.event_bus.dispatch(AgentFocusChangedEvent(tab_index=0, url=initial_url))
+ self.logger.info(f'Initial agent focus set to tab 0: {initial_url}')
except Exception as e:
# Fatal error - browser is not usable without CDP connection
@@ -650,6 +861,10 @@ class BrowserSession(BaseModel):
async def get_tabs(self) -> list[TabInfo]:
"""Get information about all open tabs using CDP Target.getTargetInfo for speed."""
tabs = []
+
+ # Safety check - return empty list if browser not connected yet
+ if not self._cdp_client_root:
+ return tabs
# Get all page targets using CDP
pages = await self._cdp_get_all_pages()
@@ -883,6 +1098,9 @@ class BrowserSession(BaseModel):
include_chrome_error: bool = False,
) -> list[TargetInfo]:
"""Get all browser pages/tabs using CDP Target.getTargets."""
+ # Safety check - return empty list if browser not connected yet
+ if not self._cdp_client_root:
+ return []
targets = await self.cdp_client.send.Target.getTargets()
# Filter for valid page/tab targets only
return [
@@ -1071,26 +1289,43 @@ class BrowserSession(BaseModel):
all_frames = {} # frame_id -> FrameInfo dict
target_sessions = {} # target_id -> session_id (keep sessions alive during collection)
- # Get all targets
+ # Check if cross-origin iframe support is enabled
+ include_cross_origin = self.browser_profile.cross_origin_iframes
+
+ # Get all targets - only include iframes if cross-origin support is enabled
targets = await self._cdp_get_all_pages(
include_http=True,
include_about=True,
include_pages=True,
- include_iframes=True,
+ include_iframes=include_cross_origin, # Only include iframe targets if flag is set
include_workers=False,
include_chrome=False,
include_chrome_extensions=False,
- include_chrome_error=True,
+ include_chrome_error=include_cross_origin, # Only include error pages if cross-origin is enabled
)
all_targets = targets
# First pass: collect frame trees from ALL targets
for target in all_targets:
target_id = target['targetId']
-
- # Get cached session for this target (don't change focus - iterating frames)
- cdp_session = await self.get_or_create_cdp_session(target_id, focus=False)
- target_sessions[target_id] = cdp_session.session_id
+
+ # Skip iframe targets if cross-origin support is disabled
+ if not include_cross_origin and target.get('type') == 'iframe':
+ continue
+
+ # When cross-origin support is disabled, only process the current target
+ if not include_cross_origin:
+ # Only process the current focus target
+ if self.agent_focus and target_id != self.agent_focus.target_id:
+ continue
+ # Use the existing agent_focus session
+ cdp_session = self.agent_focus
+ else:
+ # Get cached session for this target (don't change focus - iterating frames)
+ cdp_session = await self.get_or_create_cdp_session(target_id, focus=False)
+
+ if cdp_session:
+ target_sessions[target_id] = cdp_session.session_id
try:
# Try to get frame tree (not all target types support this)
@@ -1136,6 +1371,10 @@ class BrowserSession(BaseModel):
# For iframe targets, the frame itself is likely cross-origin
if target.get('type') == 'iframe':
frame_info['isCrossOrigin'] = True
+
+ # Skip cross-origin frames if support is disabled
+ if not include_cross_origin and frame_info.get('isCrossOrigin'):
+ return # Skip this frame and its children
# Add child frame IDs (note: OOPIFs won't appear here)
child_frames = node.get('childFrames', [])
@@ -1156,9 +1395,10 @@ class BrowserSession(BaseModel):
else:
all_frames[current_frame_id] = frame_info
- # Process child frames recursively
- for child in child_frames:
- process_frame_tree(child, current_frame_id)
+ # Process child frames recursively (only if we're not skipping this frame)
+ if include_cross_origin or not frame_info.get('isCrossOrigin'):
+ for child in child_frames:
+ process_frame_tree(child, current_frame_id)
# Process the entire frame tree
process_frame_tree(frame_tree_result.get('frameTree', {}))
@@ -1172,7 +1412,9 @@ class BrowserSession(BaseModel):
pass
# Second pass: populate backend node IDs and parent target IDs
- await self._populate_frame_metadata(all_frames, target_sessions)
+ # Only do this if cross-origin support is enabled
+ if include_cross_origin:
+ await self._populate_frame_metadata(all_frames, target_sessions)
return all_frames, target_sessions
@@ -1247,6 +1489,10 @@ class BrowserSession(BaseModel):
Raises:
ValueError: If the frame is not found in any target
"""
+ # If cross-origin iframes are disabled, just use the main session
+ if not self.browser_profile.cross_origin_iframes:
+ return await self.get_or_create_cdp_session()
+
# Get complete frame hierarchy
all_frames, target_sessions = await self.get_all_frames()
@@ -1269,8 +1515,20 @@ class BrowserSession(BaseModel):
async def cdp_client_for_node(self, node: EnhancedDOMTreeNode) -> CDPSession:
"""Get CDP client for a specific DOM node based on its frame."""
if node.frame_id:
- return await self.cdp_client_for_frame(node.frame_id)
- return await self.get_or_create_cdp_session()
+ # If cross-origin iframes are disabled, always use the main session
+ if not self.browser_profile.cross_origin_iframes:
+ assert self.agent_focus is not None, 'No active CDP session'
+ return self.agent_focus
+ # Otherwise, try to get the frame-specific session
+ try:
+ return await self.cdp_client_for_frame(node.frame_id)
+ except (ValueError, Exception) as e:
+ # Fall back to main session if frame not found
+ self.logger.debug(f'Failed to get CDP client for frame {node.frame_id}: {e}, using main session')
+ assert self.agent_focus is not None, 'No active CDP session'
+ return self.agent_focus
+ assert self.agent_focus is not None, 'No active CDP session'
+ return self.agent_focus
# Fix Pydantic circular dependency for all watchdogs
@@ -1280,7 +1538,7 @@ _watchdog_modules = [
'browser_use.browser.downloads_watchdog.DownloadsWatchdog',
'browser_use.browser.local_browser_watchdog.LocalBrowserWatchdog',
'browser_use.browser.storage_state_watchdog.StorageStateWatchdog',
- 'browser_use.browser.navigation_watchdog.NavigationWatchdog',
+ 'browser_use.browser.security_watchdog.SecurityWatchdog',
'browser_use.browser.aboutblank_watchdog.AboutBlankWatchdog',
'browser_use.browser.default_action_watchdog.DefaultActionWatchdog',
'browser_use.browser.dom_watchdog.DOMWatchdog',
diff --git a/browser_use/cli.py b/browser_use/cli.py
index 454d9f230..99d4a3a98 100644
--- a/browser_use/cli.py
+++ b/browser_use/cli.py
@@ -512,13 +512,15 @@ class BrowserUseApp(App):
logger.error(f'Error focusing input field: {str(e)}', exc_info=True)
# Non-critical, continue
- # Step 5: Setup CDP logger early (event bus listener will be set up when browser session is available)
- logger.debug('Setting up CDP logging...')
+ # Step 5: Setup CDP logger and event bus listener if browser session is available
+ logger.debug('Setting up CDP logging and event bus listener...')
try:
self.setup_cdp_logger()
- logger.debug('CDP logging setup complete')
+ if self.browser_session:
+ self.setup_event_bus_listener()
+ logger.debug('CDP logging and event bus setup complete')
except Exception as e:
- logger.error(f'Error setting up CDP logging: {str(e)}', exc_info=True)
+ logger.error(f'Error setting up CDP logging/event bus: {str(e)}', exc_info=True)
# Non-critical, continue
# Capture telemetry for CLI start
@@ -645,9 +647,17 @@ class BrowserUseApp(App):
"""Setup listener for browser session event bus."""
if not self.browser_session or not self.browser_session.event_bus:
return
-
- # Get the events log widget
- events_log = self.query_one('#events-log', RichLog)
+
+ # Check if we already have a handler registered
+ if self._event_bus_handler_id is not None:
+ return
+
+ try:
+ # Get the events log widget
+ events_log = self.query_one('#events-log', RichLog)
+ except Exception:
+ # Widget not ready yet
+ return
# Create handler to log all events
def log_event(event):
@@ -1127,6 +1137,10 @@ async def textual_interface(config: dict[str, Any]):
app.browser_session = browser_session
app.controller = controller
app.llm = llm
+
+ # Set up event bus listener now that browser session is available
+ # Note: This needs to be called before run_async() but after browser_session is set
+ # We'll defer this to on_mount() since it needs the widgets to be available
# Configure logging for Textual UI before going fullscreen
setup_textual_logging()
diff --git a/examples/custom-functions/action_filters.py b/examples/custom-functions/action_filters.py
index 37d0aa54f..8d56d46a1 100644
--- a/examples/custom-functions/action_filters.py
+++ b/examples/custom-functions/action_filters.py
@@ -1,12 +1,12 @@
"""
-Action filters (domains and page_filter) let you limit actions available to the Agent on a step-by-step/page-by-page basis.
+Action filters (domains) let you limit actions available to the Agent on a step-by-step/page-by-page basis.
[email protected](..., domains=['*'], page_filter=lambda page: return True)
[email protected](..., domains=['*'])
async def some_action(browser_session: BrowserSession):
...
This helps prevent the LLM from deciding to use an action that is not compatible with the current page.
-It helps limit decision fatique by scoping actions only to pages where they make sense.
+It helps limit decision fatigue by scoping actions only to pages where they make sense.
It also helps prevent mis-triggering stateful actions or actions that could break other programs or leak secrets.
For example:
@@ -38,29 +38,58 @@ registry = controller.registry
# Action will only be available to Agent on Google domains because of the domain filter
@registry.action(description='Trigger disco mode', domains=['google.com', '*.google.com'])
-async def disco_mode(page: Page):
- await page.evaluate("""() => {
- // define the wiggle animation
- document.styleSheets[0].insertRule('@keyframes wiggle { 0% { transform: rotate(0deg); } 50% { transform: rotate(10deg); } 100% { transform: rotate(0deg); } }');
-
- document.querySelectorAll("*").forEach(element => {
- element.style.animation = "wiggle 0.5s infinite";
- });
- }""")
-
-
-# you can create a custom page filter function that determines if the action should be available for a given page
-def is_login_page(page: Page) -> bool:
- return 'login' in page.url.lower() or 'signin' in page.url.lower()
-
+async def disco_mode(browser_session: BrowserSession):
+ # Execute JavaScript using CDP
+ cdp_session = await browser_session.get_or_create_cdp_session()
+ await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={
+ 'expression': """(() => {
+ // define the wiggle animation
+ document.styleSheets[0].insertRule('@keyframes wiggle { 0% { transform: rotate(0deg); } 50% { transform: rotate(10deg); } 100% { transform: rotate(0deg); } }');
+
+ document.querySelectorAll("*").forEach(element => {
+ element.style.animation = "wiggle 0.5s infinite";
+ });
+ })()"""
+ },
+ session_id=cdp_session.session_id
+ )
-# then use it in the action decorator to limit the action to only be available on pages where the filter returns True
[email protected](description='Use the force, luke', page_filter=is_login_page)
-async def use_the_force(page: Page):
- # this will only ever run on pages that matched the filter
- assert is_login_page(page)
- await page.evaluate("""() => { document.querySelector('body').innerHTML = 'These are not the droids you are looking for';}""")
+# Custom filter function that checks URL
+async def is_login_page(browser_session: BrowserSession) -> bool:
+ """Check if current page is a login page."""
+ try:
+ # Get current URL using CDP
+ cdp_session = await browser_session.get_or_create_cdp_session()
+ result = await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={'expression': 'window.location.href', 'returnByValue': True},
+ session_id=cdp_session.session_id
+ )
+ url = result.get('result', {}).get('value', '')
+ return 'login' in url.lower() or 'signin' in url.lower()
+ except:
+ return False
+
+
+# Note: page_filter is not directly supported anymore, so we'll just use domains
+# and check the condition inside the function
[email protected](description='Use the force, luke', domains=['*'])
+async def use_the_force(browser_session: BrowserSession):
+ # Check if it's a login page
+ if not await is_login_page(browser_session):
+ return # Skip if not a login page
+
+ # Execute JavaScript using CDP
+ cdp_session = await browser_session.get_or_create_cdp_session()
+ await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={
+ 'expression': """(() => {
+ document.querySelector('body').innerHTML = 'These are not the droids you are looking for';
+ })()"""
+ },
+ session_id=cdp_session.session_id
+ )
async def main():
@@ -89,4 +118,4 @@ async def main():
if __name__ == '__main__':
- asyncio.run(main())
+ asyncio.run(main())
\ No newline at end of file
diff --git a/examples/custom-functions/cua.py b/examples/custom-functions/cua.py
index 09ab2a023..1719917ac 100644
--- a/examples/custom-functions/cua.py
+++ b/examples/custom-functions/cua.py
@@ -130,14 +130,15 @@ async def openai_cua_fallback(params: OpenAICUAAction, browser_session: BrowserS
print(f'๐ŸŽฏ CUA Action Starting - Goal: {params.description}')
try:
- page = await browser_session.get_current_page()
- page_info = browser_session.browser_state_summary.page_info
+ # Get browser state summary
+ state = await browser_session.get_browser_state_summary()
+ page_info = state.page_info
if not page_info:
raise Exception('Page info not found - cannot execute CUA action')
print(f'๐Ÿ“ Viewport size: {page_info.viewport_width}x{page_info.viewport_height}')
- screenshot_b64 = browser_session.browser_state_summary.screenshot
+ screenshot_b64 = state.screenshot
if not screenshot_b64:
raise Exception('Screenshot not found - cannot execute CUA action')
diff --git a/examples/custom-functions/file_upload.py b/examples/custom-functions/file_upload.py
index 5af95cb34..cdbb84a22 100644
--- a/examples/custom-functions/file_upload.py
+++ b/examples/custom-functions/file_upload.py
@@ -1,8 +1,13 @@
+"""
+Example of implementing file upload functionality.
+
+This shows how to upload files to file input elements on web pages.
+"""
+
import asyncio
import logging
import os
import sys
-from pathlib import Path
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
@@ -10,20 +15,15 @@ from dotenv import load_dotenv
load_dotenv()
-from lmnr import Laminar
-
-try:
- Laminar.initialize(project_api_key=os.getenv('LMNR_PROJECT_API_KEY'))
-except Exception:
- pass
-
-from browser_use import Agent, Controller
+from browser_use.agent.service import Agent, Controller
from browser_use.agent.views import ActionResult
from browser_use.browser import BrowserSession
+from browser_use.browser.events import UploadFileEvent
from browser_use.llm import ChatOpenAI
logger = logging.getLogger(__name__)
+# Initialize controller
controller = Controller()
@@ -35,56 +35,82 @@ async def upload_file(index: int, path: str, browser_session: BrowserSession, av
if not os.path.exists(path):
return ActionResult(error=f'File {path} does not exist')
- file_upload_dom_el = await browser_session.find_file_upload_element_by_index(index)
-
- if file_upload_dom_el is None:
- msg = f'No file upload element found at index {index}'
- logger.info(msg)
- return ActionResult(error=msg)
-
- file_upload_el = await browser_session.get_locate_element(file_upload_dom_el)
-
- if file_upload_el is None:
- msg = f'No file upload element found at index {index}'
- logger.info(msg)
- return ActionResult(error=msg)
-
try:
- await file_upload_el.set_input_files(path)
+ # Get the DOM element by index
+ dom_element = await browser_session.get_dom_element_by_index(index)
+
+ if dom_element is None:
+ msg = f'No element found at index {index}'
+ logger.info(msg)
+ return ActionResult(error=msg)
+
+ # Check if it's a file input element
+ if dom_element.tag_name.lower() != 'input' or dom_element.attributes.get('type') != 'file':
+ msg = f'Element at index {index} is not a file input element'
+ logger.info(msg)
+ return ActionResult(error=msg)
+
+ # Dispatch the upload file event
+ event = browser_session.event_bus.dispatch(
+ UploadFileEvent(element_node=dom_element, file_path=path)
+ )
+ await event
+
msg = f'Successfully uploaded file to index {index}'
logger.info(msg)
return ActionResult(extracted_content=msg, include_in_memory=True)
+
except Exception as e:
msg = f'Failed to upload file to index {index}: {str(e)}'
logger.info(msg)
return ActionResult(error=msg)
-def create_file(file_type: str = 'txt'):
- with open(f'tmp.{file_type}', 'w') as f:
- f.write('test')
- file_path = Path.cwd() / f'tmp.{file_type}'
- logger.info(f'Created file: {file_path}')
- return str(file_path)
-
-
async def main():
- task = 'Go to https://kzmpmkh2zfk1ojnpxfn1.lite.vusercontent.net/ and - read the file content and upload them to fields'
- task = 'Go to https://www.freepdfconvert.com/, upload the file tmp.pdf into the field choose a file - dont click the fileupload button'
- available_file_paths = [create_file('txt'), create_file('pdf'), create_file('csv')]
-
- model = ChatOpenAI(model='gpt-4.1-mini')
+ """Main function to run the example"""
+ browser_session = BrowserSession()
+ await browser_session.start()
+ llm = ChatOpenAI(model='gpt-4.1')
+
+ # List of file paths the agent is allowed to upload
+ # In a real scenario, you'd want to be very careful about what files
+ # the agent can access and upload
+ available_file_paths = [
+ '/tmp/test_document.pdf',
+ '/tmp/test_image.jpg',
+ ]
+
+ # Create test files if they don't exist
+ for file_path in available_file_paths:
+ if not os.path.exists(file_path):
+ with open(file_path, 'w') as f:
+ f.write('Test file content for upload example')
+
+ # Create the agent with file upload capability
agent = Agent(
- task=task,
- llm=model,
+ task="""
+ Go to a website with a file upload form and upload a test document.
+ For example, go to https://www.w3schools.com/howto/howto_html_file_upload_button.asp
+ and try to upload one of the available test files.
+ """,
+ llm=llm,
+ browser_session=browser_session,
controller=controller,
- available_file_paths=available_file_paths,
+ # Pass the available file paths to the controller context
+ custom_context={'available_file_paths': available_file_paths},
)
- await agent.run()
+ # Run the agent
+ await agent.run(max_steps=10)
+
+ # Cleanup
+ await browser_session.kill()
- input('Press Enter to close...')
+ # Clean up test files
+ for file_path in available_file_paths:
+ if os.path.exists(file_path):
+ os.remove(file_path)
if __name__ == '__main__':
- asyncio.run(main())
+ asyncio.run(main())
\ No newline at end of file
diff --git a/examples/custom-functions/hover_element.py b/examples/custom-functions/hover_element.py
index 588b3fefa..2c9aad689 100644
--- a/examples/custom-functions/hover_element.py
+++ b/examples/custom-functions/hover_element.py
@@ -1,31 +1,35 @@
+"""
+Example of implementing hover functionality for elements.
+
+This shows how to hover over elements to trigger hover states and tooltips.
+"""
+
import asyncio
import os
import sys
+from typing import Optional
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
from dotenv import load_dotenv
+from pydantic import BaseModel
load_dotenv()
-from pydantic import BaseModel
-
-from browser_use import Agent, Controller
+from browser_use.agent.service import Agent, Controller
from browser_use.agent.views import ActionResult
-from browser_use.browser import BrowserProfile, BrowserSession
+from browser_use.browser import BrowserSession
from browser_use.llm import ChatOpenAI
-
-class HoverAction(BaseModel):
- index: int | None = None
- xpath: str | None = None
- selector: str | None = None
+# Initialize controller
+controller = Controller()
-browser_profile = BrowserProfile(
- headless=False,
-)
-controller = Controller()
+class HoverAction(BaseModel):
+ """Parameters for hover action"""
+ index: Optional[int] = None
+ xpath: Optional[str] = None
+ selector: Optional[str] = None
@controller.registry.action(
@@ -36,59 +40,128 @@ async def hover_element(params: HoverAction, browser_session: BrowserSession):
"""
Hovers over the element specified by its index from the cached selector map or by XPath.
"""
- if params.xpath:
- # Use XPath to locate the element
- element_handle = await browser_session.get_locate_element_by_xpath(params.xpath)
- if element_handle is None:
- raise Exception(f'Failed to locate element with XPath {params.xpath}')
- elif params.selector:
- # Use CSS selector to locate the element
- element_handle = await browser_session.get_locate_element_by_css_selector(params.selector)
- if element_handle is None:
- raise Exception(f'Failed to locate element with CSS Selector {params.selector}')
- elif params.index is not None:
- # Use index to locate the element
- selector_map = await browser_session.get_selector_map()
- if params.index not in selector_map:
- raise Exception(f'Element index {params.index} does not exist - retry or use alternative actions')
- element_node = selector_map[params.index]
- element_handle = await browser_session.get_locate_element(element_node)
- if element_handle is None:
- raise Exception(f'Failed to locate element with index {params.index}')
- else:
- raise Exception('Either index or xpath must be provided')
-
try:
- await element_handle.hover()
+ element_node = None
+
+ if params.xpath:
+ # Find element by XPath using CDP
+ cdp_session = await browser_session.get_or_create_cdp_session()
+ result = await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={
+ 'expression': f"""
+ (() => {{
+ const element = document.evaluate('{params.xpath}', document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+ if (element) {{
+ const rect = element.getBoundingClientRect();
+ return {{found: true, x: rect.x + rect.width/2, y: rect.y + rect.height/2}};
+ }}
+ return {{found: false}};
+ }})()
+ """,
+ 'returnByValue': True
+ },
+ session_id=cdp_session.session_id
+ )
+ element_info = result.get('result', {}).get('value', {})
+ if not element_info.get('found'):
+ raise Exception(f'Failed to locate element with XPath {params.xpath}')
+ x, y = element_info['x'], element_info['y']
+
+ elif params.selector:
+ # Find element by CSS selector using CDP
+ cdp_session = await browser_session.get_or_create_cdp_session()
+ result = await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={
+ 'expression': f"""
+ (() => {{
+ const element = document.querySelector('{params.selector}');
+ if (element) {{
+ const rect = element.getBoundingClientRect();
+ return {{found: true, x: rect.x + rect.width/2, y: rect.y + rect.height/2}};
+ }}
+ return {{found: false}};
+ }})()
+ """,
+ 'returnByValue': True
+ },
+ session_id=cdp_session.session_id
+ )
+ element_info = result.get('result', {}).get('value', {})
+ if not element_info.get('found'):
+ raise Exception(f'Failed to locate element with CSS Selector {params.selector}')
+ x, y = element_info['x'], element_info['y']
+
+ elif params.index is not None:
+ # Use index to locate the element
+ selector_map = await browser_session.get_selector_map()
+ if params.index not in selector_map:
+ raise Exception(f'Element index {params.index} does not exist - retry or use alternative actions')
+ element_node = selector_map[params.index]
+
+ # Get element position
+ if not element_node.absolute_position:
+ raise Exception(f'Element at index {params.index} has no position information')
+
+ x = element_node.absolute_position.x + element_node.absolute_position.width / 2
+ y = element_node.absolute_position.y + element_node.absolute_position.height / 2
+
+ else:
+ raise Exception('Either index, xpath, or selector must be provided')
+
+ # Perform hover using CDP mouse events
+ cdp_session = await browser_session.get_or_create_cdp_session()
+
+ # Move mouse to the element position
+ await cdp_session.cdp_client.send.Input.dispatchMouseEvent(
+ params={
+ 'type': 'mouseMoved',
+ 'x': x,
+ 'y': y,
+ },
+ session_id=cdp_session.session_id
+ )
+
+ # Wait a bit for hover state to trigger
+ await asyncio.sleep(0.1)
+
msg = (
f'๐Ÿ–ฑ๏ธ Hovered over element at index {params.index}'
if params.index is not None
else f'๐Ÿ–ฑ๏ธ Hovered over element with XPath {params.xpath}'
+ if params.xpath
+ else f'๐Ÿ–ฑ๏ธ Hovered over element with selector {params.selector}'
)
return ActionResult(extracted_content=msg, include_in_memory=True)
+
except Exception as e:
- err_msg = f'โŒ Failed to hover over element: {str(e)}'
- raise Exception(err_msg)
+ error_msg = f'โŒ Failed to hover over element: {str(e)}'
+ return ActionResult(error=error_msg)
async def main():
- task = 'Open https://testpages.eviltester.com/styled/csspseudo/css-hover.html and hover the element with the css selector #hoverdivpara, then click on "Can you click me?"'
- # task = 'Open https://testpages.eviltester.com/styled/csspseudo/css-hover.html and hover the element with the xpath //*[@id="hoverdivpara"], then click on "Can you click me?"'
- model = ChatOpenAI(model='gpt-4.1')
- browser_session = BrowserSession(browser_profile=browser_profile)
+ """Main function to run the example"""
+ browser_session = BrowserSession()
await browser_session.start()
+ llm = ChatOpenAI(model='gpt-4.1')
+
+ # Create the agent with hover capability
agent = Agent(
- task=task,
- llm=model,
- controller=controller,
+ task="""
+ Go to a website with hover interactions, like https://www.w3schools.com/howto/howto_css_dropdown.asp
+ Try hovering over the dropdown menu to see the dropdown items appear.
+ Then describe what happens when you hover.
+ """,
+ llm=llm,
browser_session=browser_session,
+ controller=controller,
)
- await agent.run()
- await browser_session.kill()
+ # Run the agent
+ await agent.run(max_steps=10)
- input('Press Enter to close...')
+ # Cleanup
+ await browser_session.kill()
if __name__ == '__main__':
- asyncio.run(main())
+ asyncio.run(main())
\ No newline at end of file
diff --git a/tests/ci/test_ClickElementEvent.py b/tests/ci/test_ClickElementEvent.py
index bcfed75c9..855de8041 100644
--- a/tests/ci/test_ClickElementEvent.py
+++ b/tests/ci/test_ClickElementEvent.py
@@ -140,7 +140,7 @@ class TestClickElementEvent:
await asyncio.sleep(0.5) # Give page time to load
# Initialize the DOM state to populate the selector map
- await browser_session.get_state_summary(cache_clickable_elements_hashes=True)
+ await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=True)
# Get the selector map
selector_map = await browser_session.get_selector_map()
@@ -153,7 +153,7 @@ class TestClickElementEvent:
# Look for the first div with class "clickable"
if element.tag_name.lower() == 'div' and 'clickable' in str(element.attributes.get('class', '')):
button_index = idx
- button_text = element.get_all_text_till_next_clickable_element(max_depth=2).strip()
+ button_text = element.get_all_children_text(max_depth=2).strip()
break
# Verify we found a clickable element
@@ -185,7 +185,7 @@ class TestClickElementEvent:
# Core logic validation: Verify click was successful
assert result.extracted_content is not None
- assert f'Clicked button with index {button_index}' in result.extracted_content, (
+ assert f'Clicked element with index {button_index}' in result.extracted_content, (
f'Expected click confirmation in result content, got: {result.extracted_content}'
)
if button_text:
@@ -240,7 +240,7 @@ class TestClickElementEvent:
await asyncio.sleep(0.5) # Give page time to load
# Get the page state which includes clickable elements
- state = await browser_session.get_state_summary(cache_clickable_elements_hashes=False)
+ state = await browser_session.get_browser_state_summary(cache_clickable_elements_hashes=False)
# Find the custom element index
custom_element_index = None
diff --git a/tests/ci/test_ScrollEvent.py b/tests/ci/test_ScrollEvent.py
new file mode 100644
index 000000000..82fe66814
--- /dev/null
+++ b/tests/ci/test_ScrollEvent.py
@@ -0,0 +1,190 @@
+import asyncio
+
+import pytest
+from pytest_httpserver import HTTPServer
+
+from browser_use.agent.views import ActionResult
+from browser_use.browser import BrowserSession
+from browser_use.browser.profile import BrowserProfile
+from browser_use.controller.service import Controller
+from browser_use.controller.views import (
+ ActionModel,
+ GoToUrlAction,
+ ScrollAction,
+)
+
+
[email protected](scope='session')
+def http_server():
+ """Create and provide a test HTTP server that serves static content."""
+ server = HTTPServer()
+ server.start()
+
+ # Add routes for common test pages
+ server.expect_request('/').respond_with_data(
+ '<html><head><title>Test Home Page</title></head><body><h1>Test Home Page</h1><p>Welcome to the test site</p></body></html>',
+ content_type='text/html',
+ )
+
+ server.expect_request('/scrollable').respond_with_data(
+ """
+ <!DOCTYPE html>
+ <html>
+ <head>
+ <title>Scrollable Page</title>
+ <style>
+ body { margin: 0; padding: 20px; }
+ .content { height: 3000px; background: linear-gradient(to bottom, #f0f0f0, #333); }
+ .marker { padding: 20px; background: #007bff; color: white; margin: 500px 0; }
+ </style>
+ </head>
+ <body>
+ <h1>Scrollable Test Page</h1>
+ <div class="content">
+ <div class="marker" id="marker1">Marker 1</div>
+ <div class="marker" id="marker2">Marker 2</div>
+ <div class="marker" id="marker3">Marker 3</div>
+ </div>
+ </body>
+ </html>
+ """,
+ content_type='text/html',
+ )
+
+ yield server
+ server.stop()
+
+
[email protected](scope='session')
+def base_url(http_server):
+ """Return the base URL for the test HTTP server."""
+ return f'http://{http_server.host}:{http_server.port}'
+
+
[email protected](scope='module')
+async def browser_session():
+ """Create and provide a Browser instance with security disabled."""
+ profile = BrowserProfile(headless=True, disable_security=True, cross_origin_iframes=False)
+ session = BrowserSession(browser_profile=profile)
+ await session.start()
+ yield session
+ await session.kill()
+
+
+def controller():
+ """Create and provide a Controller instance."""
+ return Controller()
+
+
+class TestScrollActions:
+ """Test scroll-related actions and events."""
+
+ async def test_scroll_actions(self, controller, browser_session, base_url, http_server):
+ """Test basic scroll action functionality."""
+
+ # Navigate to scrollable page
+ goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/scrollable', new_tab=False)}
+
+ class GoToUrlActionModel(ActionModel):
+ go_to_url: GoToUrlAction | None = None
+
+ await controller.act(GoToUrlActionModel(**goto_action), browser_session)
+
+ # Test 1: Basic page scroll down
+ scroll_action = {'scroll': ScrollAction(down=True, num_pages=1.0)}
+
+ class ScrollActionModel(ActionModel):
+ scroll: ScrollAction | None = None
+
+ result = await controller.act(ScrollActionModel(**scroll_action), browser_session)
+
+ # Verify scroll down succeeded
+ assert isinstance(result, ActionResult)
+ assert result.error is None, f'Scroll down failed: {result.error}'
+ assert result.extracted_content is not None
+ assert 'Scrolled down' in result.extracted_content
+ assert 'the page' in result.extracted_content
+ assert result.include_in_memory is True
+
+ # Test 2: Basic page scroll up
+ scroll_up_action = {'scroll': ScrollAction(down=False, num_pages=0.5)}
+ result = await controller.act(ScrollActionModel(**scroll_up_action), browser_session)
+
+ assert isinstance(result, ActionResult)
+ assert result.error is None, f'Scroll up failed: {result.error}'
+ assert result.extracted_content is not None
+ assert 'Scrolled up' in result.extracted_content
+ assert '0.5 pages' in result.extracted_content
+
+ # Test 3: Test with invalid element index (should error)
+ invalid_scroll_action = {'scroll': ScrollAction(down=True, num_pages=1.0, index=999)}
+ result = await controller.act(ScrollActionModel(**invalid_scroll_action), browser_session)
+
+ # This should fail with error about element not found
+ assert isinstance(result, ActionResult)
+ assert result.error is not None, 'Expected error for invalid element index'
+ assert 'Element index 999 not found' in result.error or 'Failed to scroll' in result.error
+
+ # Test 4: Model parameter validation
+ scroll_with_index = ScrollAction(down=True, num_pages=1.0, index=5)
+ assert scroll_with_index.down is True
+ assert scroll_with_index.num_pages == 1.0
+ assert scroll_with_index.index == 5
+
+ scroll_without_index = ScrollAction(down=False, num_pages=0.25)
+ assert scroll_without_index.down is False
+ assert scroll_without_index.num_pages == 0.25
+ assert scroll_without_index.index is None
+
+ async def test_scroll_with_cross_origin_disabled(self, browser_session, base_url):
+ """Test that scroll works when cross_origin_iframes is disabled."""
+ from browser_use.browser.events import ScrollEvent
+
+ # Navigate to a page
+ await browser_session._cdp_navigate(f'{base_url}/scrollable')
+ await asyncio.sleep(0.5)
+
+ # Test simple scroll - should not hang
+ event = browser_session.event_bus.dispatch(ScrollEvent(direction='down', amount=500))
+ result = await asyncio.wait_for(event, timeout=3.0)
+ assert result is not None
+
+ # Test scroll up
+ event = browser_session.event_bus.dispatch(ScrollEvent(direction='up', amount=200))
+ result = await asyncio.wait_for(event, timeout=3.0)
+ assert result is not None
+
+ async def test_scroll_event_directly(self, browser_session):
+ """Test ScrollEvent directly through the event bus."""
+ from browser_use.browser.events import ScrollEvent
+
+ # Test scroll on about:blank (should work)
+ event = browser_session.event_bus.dispatch(ScrollEvent(direction='down', amount=100))
+ result = await asyncio.wait_for(event, timeout=2.0)
+ event_result = await result.event_result()
+ assert event_result is not None
+ assert event_result.get('success') is True
+
+ async def test_scroll_with_element_node(self, browser_session, base_url):
+ """Test scrolling a specific element via node."""
+ from browser_use.browser.events import ScrollEvent
+ from browser_use.dom.service import EnhancedDOMTreeNode
+
+ # Navigate to scrollable page
+ await browser_session._cdp_navigate(f'{base_url}/scrollable')
+ await asyncio.sleep(0.5)
+
+ # Create a mock node with frame_id to test cross-origin handling
+ class MockNode:
+ def __init__(self):
+ self.frame_id = None # No frame_id for simple element
+ self.backend_node_id = 1
+
+ mock_node = MockNode()
+
+ # Test element scroll - should use main session when cross_origin_iframes=False
+ event = browser_session.event_bus.dispatch(ScrollEvent(direction='down', amount=300, node=mock_node))
+ result = await asyncio.wait_for(event, timeout=3.0)
+ event_result = await result.event_result()
+ assert event_result is not None
\ No newline at end of file
diff --git a/tests/ci/test_controller.py b/tests/ci/test_controller.py
index 9dfb1be98..c918f9838 100644
--- a/tests/ci/test_controller.py
+++ b/tests/ci/test_controller.py
@@ -113,88 +113,6 @@ class TestControllerIntegration:
assert result.extracted_content is not None
assert f'Navigated to {base_url}' in result.extracted_content
- async def test_scroll_actions(self, controller, browser_session, base_url, http_server):
- """Test basic scroll action functionality."""
-
- # Add a page with scrollable content
- http_server.expect_request('/scrollable').respond_with_data(
- """
- <!DOCTYPE html>
- <html>
- <head>
- <title>Scrollable Page</title>
- <style>
- body { margin: 0; padding: 20px; }
- .content { height: 3000px; background: linear-gradient(to bottom, #f0f0f0, #333); }
- .marker { padding: 20px; background: #007bff; color: white; margin: 500px 0; }
- </style>
- </head>
- <body>
- <h1>Scrollable Test Page</h1>
- <div class="content">
- <div class="marker" id="marker1">Marker 1</div>
- <div class="marker" id="marker2">Marker 2</div>
- <div class="marker" id="marker3">Marker 3</div>
- </div>
- </body>
- </html>
- """,
- content_type='text/html',
- )
-
- # Navigate to scrollable page
- goto_action = {'go_to_url': GoToUrlAction(url=f'{base_url}/scrollable', new_tab=False)}
-
- class GoToUrlActionModel(ActionModel):
- go_to_url: GoToUrlAction | None = None
-
- await controller.act(GoToUrlActionModel(**goto_action), browser_session)
-
- # Test 1: Basic page scroll down
- scroll_action = {'scroll': ScrollAction(down=True, num_pages=1.0)}
-
- class ScrollActionModel(ActionModel):
- scroll: ScrollAction | None = None
-
- result = await controller.act(ScrollActionModel(**scroll_action), browser_session)
-
- # Verify scroll down succeeded
- assert isinstance(result, ActionResult)
- assert result.error is None, f'Scroll down failed: {result.error}'
- assert result.extracted_content is not None
- assert 'Scrolled down' in result.extracted_content
- assert 'the page' in result.extracted_content
- assert result.include_in_memory is True
-
- # Test 2: Basic page scroll up
- scroll_up_action = {'scroll': ScrollAction(down=False, num_pages=0.5)}
- result = await controller.act(ScrollActionModel(**scroll_up_action), browser_session)
-
- assert isinstance(result, ActionResult)
- assert result.error is None, f'Scroll up failed: {result.error}'
- assert result.extracted_content is not None
- assert 'Scrolled up' in result.extracted_content
- assert '0.5 pages' in result.extracted_content
-
- # Test 3: Test with invalid element index (should error)
- invalid_scroll_action = {'scroll': ScrollAction(down=True, num_pages=1.0, index=999)}
- result = await controller.act(ScrollActionModel(**invalid_scroll_action), browser_session)
-
- # This should fail with error about element not found
- assert isinstance(result, ActionResult)
- assert result.error is not None, 'Expected error for invalid element index'
- assert 'Element index 999 not found' in result.error or 'Failed to scroll' in result.error
-
- # Test 4: Model parameter validation
- scroll_with_index = ScrollAction(down=True, num_pages=1.0, index=5)
- assert scroll_with_index.down is True
- assert scroll_with_index.num_pages == 1.0
- assert scroll_with_index.index == 5
-
- scroll_without_index = ScrollAction(down=False, num_pages=0.25)
- assert scroll_without_index.down is False
- assert scroll_without_index.num_pages == 0.25
- assert scroll_without_index.index is None
async def test_registry_actions(self, controller, browser_session):
"""Test that the registry contains the expected default actions."""