Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

BBox on entire page works

Magnus Müller1375013df791
diff --git a/browser_use/browser/dom_watchdog.py b/browser_use/browser/dom_watchdog.py
index 82d5144a0..a57e46ed6 100644
--- a/browser_use/browser/dom_watchdog.py
+++ b/browser_use/browser/dom_watchdog.py
@@ -232,69 +232,68 @@ class DOMWatchdog(BaseWatchdog):
recent_events=self._get_recent_events_str() if event.include_recent_events else None,
)
- # Execute DOM building and screenshot capture in parallel
- dom_task = None
- screenshot_task = None
-
- # Start DOM building task if requested
- if event.include_dom:
- self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🌳 Starting DOM tree build task...')
-
- previous_state = (
- self.browser_session._cached_browser_state_summary.dom_state
- if self.browser_session._cached_browser_state_summary
- else None
- )
-
- dom_task = asyncio.create_task(self._build_dom_tree_without_highlights(previous_state))
-
- # Start clean screenshot task if requested (without JS highlights)
- if event.include_screenshot:
- self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Starting clean screenshot task...')
- screenshot_task = asyncio.create_task(self._capture_clean_screenshot())
-
- # Wait for both tasks to complete
- content = None
- screenshot_b64 = None
-
- if dom_task:
- try:
- content = await dom_task
- self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ DOM tree build completed')
- except Exception as e:
- self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: DOM build failed: {e}, using minimal state')
- content = SerializedDOMState(_root=None, selector_map={})
- else:
- content = SerializedDOMState(_root=None, selector_map={})
-
- if screenshot_task:
- try:
- screenshot_b64 = await screenshot_task
- self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Clean screenshot captured')
- except Exception as e:
- self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Clean screenshot failed: {e}')
- screenshot_b64 = None
-
- # Apply Python-based highlighting if both DOM and screenshot are available
- if (screenshot_b64 and content and content.selector_map and
- self.browser_session.browser_profile.highlight_elements):
- try:
- self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🎨 Applying Python-based highlighting...')
- from browser_use.browser.python_highlights import create_highlighted_screenshot_async
-
- # Get CDP session for viewport info
- cdp_session = await self.browser_session.get_or_create_cdp_session()
-
- screenshot_b64 = await create_highlighted_screenshot_async(
- screenshot_b64, content.selector_map, cdp_session
+ # Execute DOM building and screenshot capture in parallel
+ dom_task = None
+ screenshot_task = None
+
+ # Start DOM building task if requested
+ if event.include_dom:
+ self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🌳 Starting DOM tree build task...')
+
+ previous_state = (
+ self.browser_session._cached_browser_state_summary.dom_state
+ if self.browser_session._cached_browser_state_summary
+ else None
)
- self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Applied highlights to {len(content.selector_map)} elements')
- except Exception as e:
- self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Python highlighting failed: {e}')
-
- # Ensure we have valid content
- if not content:
- content = SerializedDOMState(_root=None, selector_map={})
+
+ dom_task = asyncio.create_task(self._build_dom_tree_without_highlights(previous_state))
+
+ # Start clean screenshot task if requested (without JS highlights)
+ if event.include_screenshot:
+ self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Starting clean screenshot task...')
+ screenshot_task = asyncio.create_task(self._capture_clean_screenshot())
+
+ # Wait for both tasks to complete
+ content = None
+ screenshot_b64 = None
+
+ if dom_task:
+ try:
+ content = await dom_task
+ self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ DOM tree build completed')
+ except Exception as e:
+ self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: DOM build failed: {e}, using minimal state')
+ content = SerializedDOMState(_root=None, selector_map={})
+ else:
+ content = SerializedDOMState(_root=None, selector_map={})
+
+ if screenshot_task:
+ try:
+ screenshot_b64 = await screenshot_task
+ self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Clean screenshot captured')
+ except Exception as e:
+ self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Clean screenshot failed: {e}')
+ screenshot_b64 = None
+
+ # Apply Python-based highlighting if both DOM and screenshot are available
+ if screenshot_b64 and content and content.selector_map and self.browser_session.browser_profile.highlight_elements:
+ try:
+ self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🎨 Applying Python-based highlighting...')
+ from browser_use.browser.python_highlights import create_highlighted_screenshot_async
+
+ # Get CDP session for viewport info
+ cdp_session = await self.browser_session.get_or_create_cdp_session()
+
+ screenshot_b64 = await create_highlighted_screenshot_async(screenshot_b64, content.selector_map, cdp_session)
+ self.logger.debug(
+ f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Applied highlights to {len(content.selector_map)} elements'
+ )
+ except Exception as e:
+ self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Python highlighting failed: {e}')
+
+ # Ensure we have valid content
+ if not content:
+ content = SerializedDOMState(_root=None, selector_map={})
# Tabs info already fetched at the beginning
@@ -466,6 +465,87 @@ class DOMWatchdog(BaseWatchdog):
)
raise
+ async def _build_dom_tree_without_highlights(self, previous_state: SerializedDOMState | None = None) -> SerializedDOMState:
+ """Build DOM tree without injecting JavaScript highlights (for parallel execution)."""
+ try:
+ self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: STARTING DOM tree build')
+
+ # Create or reuse DOM service
+ if self._dom_service is None:
+ self._dom_service = DomService(browser_session=self.browser_session, logger=self.logger)
+
+ # Get serialized DOM tree using the service
+ self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Calling DomService.get_serialized_dom_tree...')
+ start = time.time()
+ self.current_dom_state, self.enhanced_dom_tree, timing_info = await self._dom_service.get_serialized_dom_tree(
+ previous_cached_state=previous_state,
+ )
+ end = time.time()
+ self.logger.debug(
+ '🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ DomService.get_serialized_dom_tree completed'
+ )
+
+ self.logger.debug(f'Time taken to get DOM tree: {end - start} seconds')
+ self.logger.debug(f'Timing breakdown: {timing_info}')
+
+ # Update selector map for other watchdogs
+ self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Updating selector maps...')
+ self.selector_map = self.current_dom_state.selector_map
+ # Update BrowserSession's cached selector map
+ if self.browser_session:
+ self.browser_session.update_cached_selector_map(self.selector_map)
+ self.logger.debug(
+ f'🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ Selector maps updated, {len(self.selector_map)} elements'
+ )
+
+ # Skip JavaScript highlighting injection - Python highlighting will be applied later
+ self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ COMPLETED DOM tree build (no JS highlights)')
+ return self.current_dom_state
+
+ except Exception as e:
+ self.logger.error(f'Failed to build DOM tree without highlights: {e}')
+ self.event_bus.dispatch(
+ BrowserErrorEvent(
+ error_type='DOMBuildFailed',
+ message=str(e),
+ )
+ )
+ raise
+
+ async def _capture_clean_screenshot(self) -> str:
+ """Capture a clean screenshot without JavaScript highlights."""
+ try:
+ self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: Capturing clean screenshot...')
+
+ # Ensure we have a focused CDP session
+ assert self.browser_session.agent_focus is not None, 'No current target ID'
+ await self.browser_session.get_or_create_cdp_session(target_id=self.browser_session.agent_focus.target_id, focus=True)
+
+ # Check if handler is registered
+ handlers = self.event_bus.handlers.get('ScreenshotEvent', [])
+ handler_names = [getattr(h, '__name__', str(h)) for h in handlers]
+ self.logger.debug(f'📸 ScreenshotEvent handlers registered: {len(handlers)} - {handler_names}')
+
+ screenshot_event = self.event_bus.dispatch(ScreenshotEvent(full_page=False))
+ self.logger.debug('📸 Dispatched ScreenshotEvent, waiting for event to complete...')
+
+ # Wait for the event itself to complete (this waits for all handlers)
+ await screenshot_event
+
+ # Get the single handler result
+ screenshot_b64 = await screenshot_event.event_result(raise_if_any=True, raise_if_none=True)
+ if screenshot_b64 is None:
+ raise RuntimeError('Screenshot handler returned None')
+ self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: ✅ Clean screenshot captured successfully')
+ return str(screenshot_b64)
+
+ except TimeoutError:
+ self.logger.warning('📸 Clean screenshot timed out after 6 seconds - no handler registered or slow page?')
+ raise
+ except Exception as e:
+ self.logger.warning(f'📸 Clean screenshot failed: {type(e).__name__}: {e}')
+ raise
+
async def _wait_for_stable_network(self):
"""Wait for page stability - simplified for CDP-only branch."""
start_time = time.time()
diff --git a/browser_use/browser/python_highlights.py b/browser_use/browser/python_highlights.py
index 357532c66..a07f59aaa 100644
--- a/browser_use/browser/python_highlights.py
+++ b/browser_use/browser/python_highlights.py
@@ -6,240 +6,261 @@ to draw bounding boxes around interactive elements directly on screenshots.
import base64
import io
-from typing import Dict, Any, Optional, Tuple
-from PIL import Image, ImageDraw, ImageFont
import logging
+from typing import Optional, Tuple
+
+from PIL import Image, ImageDraw, ImageFont
-from browser_use.dom.serializer.types import DOMSelectorMap
+from browser_use.dom.views import DOMSelectorMap
+from browser_use.observability import observe_debug
logger = logging.getLogger(__name__)
# Color scheme for different element types
ELEMENT_COLORS = {
- 'button': '#FF6B6B', # Red for buttons
- 'input': '#4ECDC4', # Teal for inputs
- 'select': '#45B7D1', # Blue for dropdowns
- 'a': '#96CEB4', # Green for links
- 'textarea': '#FFEAA7', # Yellow for text areas
- 'default': '#DDA0DD', # Light purple for other interactive elements
+ 'button': '#FF6B6B', # Red for buttons
+ 'input': '#4ECDC4', # Teal for inputs
+ 'select': '#45B7D1', # Blue for dropdowns
+ 'a': '#96CEB4', # Green for links
+ 'textarea': '#FFEAA7', # Yellow for text areas
+ 'default': '#DDA0DD', # Light purple for other interactive elements
}
# Element type mappings
ELEMENT_TYPE_MAP = {
- 'button': 'button',
- 'input': 'input',
- 'select': 'select',
- 'a': 'a',
- 'textarea': 'textarea',
+ 'button': 'button',
+ 'input': 'input',
+ 'select': 'select',
+ 'a': 'a',
+ 'textarea': 'textarea',
}
+
def get_element_color(tag_name: str, element_type: Optional[str] = None) -> str:
- """Get color for element based on tag name and type."""
- # Check input type first
- if tag_name == 'input' and element_type:
- if element_type in ['button', 'submit']:
- return ELEMENT_COLORS['button']
-
- # Use tag-based color
- return ELEMENT_COLORS.get(tag_name.lower(), ELEMENT_COLORS['default'])
+ """Get color for element based on tag name and type."""
+ # Check input type first
+ if tag_name == 'input' and element_type:
+ if element_type in ['button', 'submit']:
+ return ELEMENT_COLORS['button']
+
+ # Use tag-based color
+ return ELEMENT_COLORS.get(tag_name.lower(), ELEMENT_COLORS['default'])
+
def should_show_text_overlay(text: Optional[str]) -> bool:
- """Determine if text overlay should be shown based on length."""
- if not text:
- return False
- return len(text.strip()) <= 10
+ """Determine if text overlay should be shown based on length."""
+ if not text:
+ return False
+ return len(text.strip()) <= 10
+
def draw_bounding_box_with_text(
- draw: ImageDraw.Draw,
- bbox: Tuple[int, int, int, int],
- color: str,
- text: Optional[str] = None,
- font: Optional[ImageFont.FreeTypeFont] = None
+ draw, # ImageDraw.Draw - avoiding type annotation due to PIL typing issues
+ bbox: Tuple[int, int, int, int],
+ color: str,
+ text: Optional[str] = None,
+ font: Optional[ImageFont.FreeTypeFont] = None,
) -> None:
- """Draw a bounding box with optional text overlay."""
- x1, y1, x2, y2 = bbox
-
- # Draw bounding box with 2px width
- for i in range(2):
- draw.rectangle([x1 + i, y1 + i, x2 - i, y2 - i], outline=color, fill=None)
-
- # Draw text overlay if provided and short enough
- if text and should_show_text_overlay(text):
- try:
- # Get text size
- if font:
- bbox_text = draw.textbbox((0, 0), text, font=font)
- text_width = bbox_text[2] - bbox_text[0]
- text_height = bbox_text[3] - bbox_text[1]
- else:
- # Fallback for default font
- bbox_text = draw.textbbox((0, 0), text)
- text_width = bbox_text[2] - bbox_text[0]
- text_height = bbox_text[3] - bbox_text[1]
-
- # Position text at top-left of bounding box
- text_x = max(0, x1)
- text_y = max(0, y1 - text_height - 2) # Above the box
-
- # Draw background rectangle for text
- draw.rectangle(
- [text_x - 2, text_y - 2, text_x + text_width + 2, text_y + text_height + 2],
- fill=color,
- outline=None
- )
-
- # Draw text
- draw.text((text_x, text_y), text, fill='white', font=font)
-
- except Exception as e:
- logger.debug(f"Failed to draw text overlay: {e}")
+ """Draw a bounding box with optional text overlay."""
+ x1, y1, x2, y2 = bbox
+
+ # Draw bounding box with 2px width
+ for i in range(2):
+ draw.rectangle([x1 + i, y1 + i, x2 - i, y2 - i], outline=color, fill=None)
+
+ # Draw text overlay if provided and short enough
+ if text and should_show_text_overlay(text):
+ try:
+ # Get text size
+ if font:
+ bbox_text = draw.textbbox((0, 0), text, font=font)
+ text_width = bbox_text[2] - bbox_text[0]
+ text_height = bbox_text[3] - bbox_text[1]
+ else:
+ # Fallback for default font
+ bbox_text = draw.textbbox((0, 0), text)
+ text_width = bbox_text[2] - bbox_text[0]
+ text_height = bbox_text[3] - bbox_text[1]
+
+ # Position text at top-left of bounding box
+ text_x = max(0, x1)
+ text_y = max(0, y1 - text_height - 2) # Above the box
+
+ # Draw background rectangle for text
+ draw.rectangle([text_x - 2, text_y - 2, text_x + text_width + 2, text_y + text_height + 2], fill=color, outline=None)
+
+ # Draw text
+ draw.text((text_x, text_y), text, fill='white', font=font)
+ except Exception as e:
+ logger.debug(f'Failed to draw text overlay: {e}')
+
+
+@observe_debug(ignore_input=True, ignore_output=True, name='create_highlighted_screenshot')
def create_highlighted_screenshot(
- screenshot_b64: str,
- selector_map: DOMSelectorMap,
- device_pixel_ratio: float = 1.0,
- viewport_offset_x: int = 0,
- viewport_offset_y: int = 0
-) -> str:
- """Create a highlighted screenshot with bounding boxes around interactive elements.
-
- Args:
- screenshot_b64: Base64 encoded screenshot
- selector_map: Map of interactive elements with their positions
- device_pixel_ratio: Device pixel ratio for scaling coordinates
- viewport_offset_x: X offset for viewport positioning
- viewport_offset_y: Y offset for viewport positioning
-
- Returns:
- Base64 encoded highlighted screenshot
- """
- try:
- # Decode screenshot
- screenshot_data = base64.b64decode(screenshot_b64)
- image = Image.open(io.BytesIO(screenshot_data)).convert('RGBA')
-
- # Create drawing context
- draw = ImageDraw.Draw(image)
-
- # Try to load a font, fall back to default if not available
- font = None
- try:
- font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 12)
- except:
- try:
- font = ImageFont.truetype("arial.ttf", 12)
- except:
- font = None # Use default font
-
- # Process each interactive element
- for element_id, element in selector_map.items():
- try:
- # Get element bounds
- bounds = element.bounds
- if not bounds:
- continue
-
- # Scale coordinates by device pixel ratio and apply viewport offset
- x1 = int((bounds.x + viewport_offset_x) * device_pixel_ratio)
- y1 = int((bounds.y + viewport_offset_y) * device_pixel_ratio)
- x2 = int((bounds.x + bounds.width + viewport_offset_x) * device_pixel_ratio)
- y2 = int((bounds.y + bounds.height + viewport_offset_y) * device_pixel_ratio)
-
- # Ensure coordinates are within image bounds
- img_width, img_height = image.size
- x1 = max(0, min(x1, img_width))
- y1 = max(0, min(y1, img_height))
- x2 = max(x1, min(x2, img_width))
- y2 = max(y1, min(y2, img_height))
-
- # Skip if bounding box is too small or invalid
- if x2 - x1 < 2 or y2 - y1 < 2:
- continue
-
- # Get element color based on type
- tag_name = element.tag_name if hasattr(element, 'tag_name') else 'div'
- element_type = None
- if hasattr(element, 'attributes') and element.attributes:
- element_type = element.attributes.get('type')
-
- color = get_element_color(tag_name, element_type)
-
- # Get text for overlay (if short enough)
- text = None
- if hasattr(element, 'text') and element.text:
- text = element.text.strip()
- elif hasattr(element, 'attributes') and element.attributes:
- # Try to get meaningful text from attributes
- text = (element.attributes.get('aria-label') or
- element.attributes.get('title') or
- element.attributes.get('placeholder') or
- element.attributes.get('value', ''))
-
- # Draw bounding box with optional text
- draw_bounding_box_with_text(
- draw, (x1, y1, x2, y2), color, text, font
- )
-
- except Exception as e:
- logger.debug(f"Failed to draw highlight for element {element_id}: {e}")
- continue
-
- # Convert back to base64
- output_buffer = io.BytesIO()
- image.save(output_buffer, format='PNG')
- output_buffer.seek(0)
-
- highlighted_b64 = base64.b64encode(output_buffer.getvalue()).decode('utf-8')
-
- logger.debug(f"Successfully created highlighted screenshot with {len(selector_map)} elements")
- return highlighted_b64
-
- except Exception as e:
- logger.error(f"Failed to create highlighted screenshot: {e}")
- # Return original screenshot on error
- return screenshot_b64
-
-def get_viewport_info_from_cdp(cdp_session) -> Tuple[float, int, int]:
- """Get viewport information from CDP session.
-
- Returns:
- Tuple of (device_pixel_ratio, viewport_offset_x, viewport_offset_y)
- """
- # This is a placeholder - in real implementation, you'd get this from CDP
- # For now, return sensible defaults
- return 1.0, 0, 0
-
-async def create_highlighted_screenshot_async(
- screenshot_b64: str,
- selector_map: DOMSelectorMap,
- cdp_session = None
+ screenshot_b64: str,
+ selector_map: DOMSelectorMap,
+ device_pixel_ratio: float = 1.0,
+ viewport_offset_x: int = 0,
+ viewport_offset_y: int = 0,
) -> str:
- """Async wrapper for creating highlighted screenshots.
-
- Args:
- screenshot_b64: Base64 encoded screenshot
- selector_map: Map of interactive elements
- cdp_session: CDP session for getting viewport info
-
- Returns:
- Base64 encoded highlighted screenshot
- """
- # Get viewport information if CDP session is available
- device_pixel_ratio = 1.0
- viewport_offset_x = 0
- viewport_offset_y = 0
-
- if cdp_session:
- try:
- device_pixel_ratio, viewport_offset_x, viewport_offset_y = get_viewport_info_from_cdp(cdp_session)
- except Exception as e:
- logger.debug(f"Failed to get viewport info from CDP: {e}")
-
- # Create highlighted screenshot (run in thread pool if needed for performance)
- return create_highlighted_screenshot(
- screenshot_b64,
- selector_map,
- device_pixel_ratio,
- viewport_offset_x,
- viewport_offset_y
- )
\ No newline at end of file
+ """Create a highlighted screenshot with bounding boxes around interactive elements.
+
+ Args:
+ screenshot_b64: Base64 encoded screenshot
+ selector_map: Map of interactive elements with their positions
+ device_pixel_ratio: Device pixel ratio for scaling coordinates
+ viewport_offset_x: X offset for viewport positioning
+ viewport_offset_y: Y offset for viewport positioning
+
+ Returns:
+ Base64 encoded highlighted screenshot
+ """
+ try:
+ # Decode screenshot
+ screenshot_data = base64.b64decode(screenshot_b64)
+ image = Image.open(io.BytesIO(screenshot_data)).convert('RGBA')
+
+ # Create drawing context
+ draw = ImageDraw.Draw(image)
+
+ # Try to load a font, fall back to default if not available
+ font = None
+ try:
+ font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf', 12)
+ except:
+ try:
+ font = ImageFont.truetype('arial.ttf', 12)
+ except:
+ font = None # Use default font
+
+ # Process each interactive element
+ for element_id, element in selector_map.items():
+ try:
+ # Use snapshot bounds (document coordinates) if available, otherwise absolute_position
+ bounds = None
+ if element.snapshot_node and element.snapshot_node.bounds:
+ bounds = element.snapshot_node.bounds
+ elif element.absolute_position:
+ bounds = element.absolute_position
+
+ if not bounds:
+ continue
+
+ # Convert from CSS pixels to device pixels for screenshot coordinates
+ # Note: bounds are already in CSS pixels, screenshot is in device pixels
+ x1 = int((bounds.x - viewport_offset_x) * device_pixel_ratio)
+ y1 = int((bounds.y - viewport_offset_y) * device_pixel_ratio)
+ x2 = int((bounds.x + bounds.width - viewport_offset_x) * device_pixel_ratio)
+ y2 = int((bounds.y + bounds.height - viewport_offset_y) * device_pixel_ratio)
+
+ # Ensure coordinates are within image bounds
+ img_width, img_height = image.size
+ x1 = max(0, min(x1, img_width))
+ y1 = max(0, min(y1, img_height))
+ x2 = max(x1, min(x2, img_width))
+ y2 = max(y1, min(y2, img_height))
+
+ # Skip if bounding box is too small or invalid
+ if x2 - x1 < 2 or y2 - y1 < 2:
+ continue
+
+ # Get element color based on type
+ tag_name = element.tag_name if hasattr(element, 'tag_name') else 'div'
+ element_type = None
+ if hasattr(element, 'attributes') and element.attributes:
+ element_type = element.attributes.get('type')
+
+ color = get_element_color(tag_name, element_type)
+
+ # Get text for overlay (if short enough)
+ text = None
+ if hasattr(element, 'node_value') and element.node_value:
+ text = element.node_value.strip()
+ elif hasattr(element, 'attributes') and element.attributes:
+ # Try to get meaningful text from attributes
+ text = (
+ element.attributes.get('aria-label')
+ or element.attributes.get('title')
+ or element.attributes.get('placeholder')
+ or element.attributes.get('value', '')
+ )
+
+ # Draw bounding box with optional text
+ draw_bounding_box_with_text(draw, (x1, y1, x2, y2), color, text, font)
+
+ except Exception as e:
+ logger.debug(f'Failed to draw highlight for element {element_id}: {e}')
+ continue
+
+ # Convert back to base64
+ output_buffer = io.BytesIO()
+ image.save(output_buffer, format='PNG')
+ output_buffer.seek(0)
+
+ highlighted_b64 = base64.b64encode(output_buffer.getvalue()).decode('utf-8')
+
+ logger.debug(f'Successfully created highlighted screenshot with {len(selector_map)} elements')
+ return highlighted_b64
+
+ except Exception as e:
+ logger.error(f'Failed to create highlighted screenshot: {e}')
+ # Return original screenshot on error
+ return screenshot_b64
+
+
+async def get_viewport_info_from_cdp(cdp_session) -> Tuple[float, int, int]:
+ """Get viewport information from CDP session.
+
+ Returns:
+ Tuple of (device_pixel_ratio, scroll_x, scroll_y)
+ """
+ try:
+ # Get layout metrics which includes viewport info and device pixel ratio
+ metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)
+
+ # Extract viewport information
+ visual_viewport = metrics.get('visualViewport', {})
+ css_visual_viewport = metrics.get('cssVisualViewport', {})
+ css_layout_viewport = metrics.get('cssLayoutViewport', {})
+
+ # Calculate device pixel ratio
+ css_width = css_visual_viewport.get('clientWidth', css_layout_viewport.get('clientWidth', 1280.0))
+ device_width = visual_viewport.get('clientWidth', css_width)
+ device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0
+
+ # Get scroll position in CSS pixels
+ scroll_x = int(css_visual_viewport.get('pageX', 0))
+ scroll_y = int(css_visual_viewport.get('pageY', 0))
+
+ return float(device_pixel_ratio), scroll_x, scroll_y
+
+ except Exception as e:
+ logger.debug(f'Failed to get viewport info from CDP: {e}')
+ return 1.0, 0, 0
+
+
+async def create_highlighted_screenshot_async(screenshot_b64: str, selector_map: DOMSelectorMap, cdp_session=None) -> str:
+ """Async wrapper for creating highlighted screenshots.
+
+ Args:
+ screenshot_b64: Base64 encoded screenshot
+ selector_map: Map of interactive elements
+ cdp_session: CDP session for getting viewport info
+
+ Returns:
+ Base64 encoded highlighted screenshot
+ """
+ # Get viewport information if CDP session is available
+ device_pixel_ratio = 1.0
+ viewport_offset_x = 0
+ viewport_offset_y = 0
+
+ if cdp_session:
+ try:
+ device_pixel_ratio, viewport_offset_x, viewport_offset_y = await get_viewport_info_from_cdp(cdp_session)
+ except Exception as e:
+ logger.debug(f'Failed to get viewport info from CDP: {e}')
+
+ # Create highlighted screenshot (run in thread pool if needed for performance)
+ return create_highlighted_screenshot(screenshot_b64, selector_map, device_pixel_ratio, viewport_offset_x, viewport_offset_y)
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 78747141f..2e28f17a4 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -35,6 +35,7 @@ from browser_use.browser.events import (
from browser_use.browser.profile import BrowserProfile
from browser_use.browser.views import BrowserStateSummary, TabInfo
from browser_use.dom.views import EnhancedDOMTreeNode, TargetInfo
+from browser_use.observability import observe_debug
from browser_use.utils import _log_pretty_url, is_new_tab_page
DEFAULT_BROWSER_PROFILE = BrowserProfile()
@@ -737,7 +738,7 @@ class BrowserSession(BaseModel):
return self.agent_focus.session_id if self.agent_focus else None
# ========== Helper Methods ==========
-
+ @observe_debug(ignore_input=True, ignore_output=True, name='get_browser_state_summary')
async def get_browser_state_summary(
self,
cache_clickable_elements_hashes: bool = True,