Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Add Python-based screenshot highlighting with element bounding boxes

Cursor Agent6ad7e708d8e6
diff --git a/browser_use/browser/dom_watchdog.py b/browser_use/browser/dom_watchdog.py
index dc0992715..82d5144a0 100644
--- a/browser_use/browser/dom_watchdog.py
+++ b/browser_use/browser/dom_watchdog.py
@@ -232,65 +232,69 @@ class DOMWatchdog(BaseWatchdog):
recent_events=self._get_recent_events_str() if event.include_recent_events else None,
)
- # Normal path: Build DOM tree if requested
- if event.include_dom:
- self.logger.debug('πŸ” DOMWatchdog.on_BrowserStateRequestEvent: 🌳 Building DOM tree...')
-
- # Build the DOM directly using the internal method
- previous_state = (
- self.browser_session._cached_browser_state_summary.dom_state
- if self.browser_session._cached_browser_state_summary
- else None
- )
-
- try:
- # Call the DOM building method directly
- self.logger.debug('πŸ” DOMWatchdog.on_BrowserStateRequestEvent: Starting _build_dom_tree...')
- content = await self._build_dom_tree(previous_state)
- self.logger.debug('πŸ” DOMWatchdog.on_BrowserStateRequestEvent: βœ… _build_dom_tree completed')
- except Exception as e:
- self.logger.warning(f'πŸ” DOMWatchdog.on_BrowserStateRequestEvent: DOM build failed: {e}, using minimal state')
- content = SerializedDOMState(_root=None, selector_map={})
-
- if not content:
- # Fallback to minimal DOM state
- self.logger.warning('DOM build returned no content, using minimal state')
- content = SerializedDOMState(_root=None, selector_map={})
- else:
- # Skip DOM building if not requested
+ # Execute DOM building and screenshot capture in parallel
+ dom_task = None
+ screenshot_task = None
+
+ # Start DOM building task if requested
+ if event.include_dom:
+ self.logger.debug('πŸ” DOMWatchdog.on_BrowserStateRequestEvent: 🌳 Starting DOM tree build task...')
+
+ previous_state = (
+ self.browser_session._cached_browser_state_summary.dom_state
+ if self.browser_session._cached_browser_state_summary
+ else None
+ )
+
+ dom_task = asyncio.create_task(self._build_dom_tree_without_highlights(previous_state))
+
+ # Start clean screenshot task if requested (without JS highlights)
+ if event.include_screenshot:
+ self.logger.debug('πŸ” DOMWatchdog.on_BrowserStateRequestEvent: πŸ“Έ Starting clean screenshot task...')
+ screenshot_task = asyncio.create_task(self._capture_clean_screenshot())
+
+ # Wait for both tasks to complete
+ content = None
+ screenshot_b64 = None
+
+ if dom_task:
+ try:
+ content = await dom_task
+ self.logger.debug('πŸ” DOMWatchdog.on_BrowserStateRequestEvent: βœ… DOM tree build completed')
+ except Exception as e:
+ self.logger.warning(f'πŸ” DOMWatchdog.on_BrowserStateRequestEvent: DOM build failed: {e}, using minimal state')
content = SerializedDOMState(_root=None, selector_map={})
-
- # re-focus top-level page session context
- assert self.browser_session.agent_focus is not None, 'No current target ID'
- await self.browser_session.get_or_create_cdp_session(target_id=self.browser_session.agent_focus.target_id, focus=True)
-
- # Get screenshot if requested
- screenshot_b64 = None
- if event.include_screenshot:
- self.logger.debug(
- f'πŸ” DOMWatchdog.on_BrowserStateRequestEvent: πŸ“Έ DOM watchdog requesting screenshot, include_screenshot={event.include_screenshot}'
+ else:
+ content = SerializedDOMState(_root=None, selector_map={})
+
+ if screenshot_task:
+ try:
+ screenshot_b64 = await screenshot_task
+ self.logger.debug('πŸ” DOMWatchdog.on_BrowserStateRequestEvent: βœ… Clean screenshot captured')
+ except Exception as e:
+ self.logger.warning(f'πŸ” DOMWatchdog.on_BrowserStateRequestEvent: Clean screenshot failed: {e}')
+ screenshot_b64 = None
+
+ # Apply Python-based highlighting if both DOM and screenshot are available
+ if (screenshot_b64 and content and content.selector_map and
+ self.browser_session.browser_profile.highlight_elements):
+ try:
+ self.logger.debug('πŸ” DOMWatchdog.on_BrowserStateRequestEvent: 🎨 Applying Python-based highlighting...')
+ from browser_use.browser.python_highlights import create_highlighted_screenshot_async
+
+ # Get CDP session for viewport info
+ cdp_session = await self.browser_session.get_or_create_cdp_session()
+
+ screenshot_b64 = await create_highlighted_screenshot_async(
+ screenshot_b64, content.selector_map, cdp_session
)
- try:
- # Check if handler is registered
- handlers = self.event_bus.handlers.get('ScreenshotEvent', [])
- handler_names = [getattr(h, '__name__', str(h)) for h in handlers]
- self.logger.debug(f'πŸ“Έ ScreenshotEvent handlers registered: {len(handlers)} - {handler_names}')
-
- screenshot_event = self.event_bus.dispatch(ScreenshotEvent(full_page=False))
- self.logger.debug('πŸ“Έ Dispatched ScreenshotEvent, waiting for event to complete...')
-
- # Wait for the event itself to complete (this waits for all handlers)
- await screenshot_event
-
- # Get the single handler result
- screenshot_b64 = await screenshot_event.event_result(raise_if_any=True, raise_if_none=True)
- except TimeoutError:
- self.logger.warning('πŸ“Έ Screenshot timed out after 6 seconds - no handler registered or slow page?')
-
- except Exception as e:
- self.logger.warning(f'πŸ“Έ Screenshot failed: {type(e).__name__}: {e}')
- else:
- self.logger.debug(f'πŸ“Έ Skipping screenshot, include_screenshot={event.include_screenshot}')
+ self.logger.debug(f'πŸ” DOMWatchdog.on_BrowserStateRequestEvent: βœ… Applied highlights to {len(content.selector_map)} elements')
+ except Exception as e:
+ self.logger.warning(f'πŸ” DOMWatchdog.on_BrowserStateRequestEvent: Python highlighting failed: {e}')
+
+ # Ensure we have valid content
+ if not content:
+ content = SerializedDOMState(_root=None, selector_map={})
# Tabs info already fetched at the beginning
diff --git a/browser_use/browser/python_highlights.py b/browser_use/browser/python_highlights.py
new file mode 100644
index 000000000..357532c66
--- /dev/null
+++ b/browser_use/browser/python_highlights.py
@@ -0,0 +1,245 @@
+"""Python-based highlighting system for drawing bounding boxes on screenshots.
+
+This module replaces JavaScript-based highlighting with fast Python image processing
+to draw bounding boxes around interactive elements directly on screenshots.
+"""
+
+import base64
+import io
+from typing import Dict, Any, Optional, Tuple
+from PIL import Image, ImageDraw, ImageFont
+import logging
+
+from browser_use.dom.serializer.types import DOMSelectorMap
+
+logger = logging.getLogger(__name__)
+
+# Color scheme for different element types
+ELEMENT_COLORS = {
+ 'button': '#FF6B6B', # Red for buttons
+ 'input': '#4ECDC4', # Teal for inputs
+ 'select': '#45B7D1', # Blue for dropdowns
+ 'a': '#96CEB4', # Green for links
+ 'textarea': '#FFEAA7', # Yellow for text areas
+ 'default': '#DDA0DD', # Light purple for other interactive elements
+}
+
+# Element type mappings
+ELEMENT_TYPE_MAP = {
+ 'button': 'button',
+ 'input': 'input',
+ 'select': 'select',
+ 'a': 'a',
+ 'textarea': 'textarea',
+}
+
+def get_element_color(tag_name: str, element_type: Optional[str] = None) -> str:
+ """Get color for element based on tag name and type."""
+ # Check input type first
+ if tag_name == 'input' and element_type:
+ if element_type in ['button', 'submit']:
+ return ELEMENT_COLORS['button']
+
+ # Use tag-based color
+ return ELEMENT_COLORS.get(tag_name.lower(), ELEMENT_COLORS['default'])
+
+def should_show_text_overlay(text: Optional[str]) -> bool:
+ """Determine if text overlay should be shown based on length."""
+ if not text:
+ return False
+ return len(text.strip()) <= 10
+
+def draw_bounding_box_with_text(
+ draw: ImageDraw.Draw,
+ bbox: Tuple[int, int, int, int],
+ color: str,
+ text: Optional[str] = None,
+ font: Optional[ImageFont.FreeTypeFont] = None
+) -> None:
+ """Draw a bounding box with optional text overlay."""
+ x1, y1, x2, y2 = bbox
+
+ # Draw bounding box with 2px width
+ for i in range(2):
+ draw.rectangle([x1 + i, y1 + i, x2 - i, y2 - i], outline=color, fill=None)
+
+ # Draw text overlay if provided and short enough
+ if text and should_show_text_overlay(text):
+ try:
+ # Get text size
+ if font:
+ bbox_text = draw.textbbox((0, 0), text, font=font)
+ text_width = bbox_text[2] - bbox_text[0]
+ text_height = bbox_text[3] - bbox_text[1]
+ else:
+ # Fallback for default font
+ bbox_text = draw.textbbox((0, 0), text)
+ text_width = bbox_text[2] - bbox_text[0]
+ text_height = bbox_text[3] - bbox_text[1]
+
+ # Position text at top-left of bounding box
+ text_x = max(0, x1)
+ text_y = max(0, y1 - text_height - 2) # Above the box
+
+ # Draw background rectangle for text
+ draw.rectangle(
+ [text_x - 2, text_y - 2, text_x + text_width + 2, text_y + text_height + 2],
+ fill=color,
+ outline=None
+ )
+
+ # Draw text
+ draw.text((text_x, text_y), text, fill='white', font=font)
+
+ except Exception as e:
+ logger.debug(f"Failed to draw text overlay: {e}")
+
+def create_highlighted_screenshot(
+ screenshot_b64: str,
+ selector_map: DOMSelectorMap,
+ device_pixel_ratio: float = 1.0,
+ viewport_offset_x: int = 0,
+ viewport_offset_y: int = 0
+) -> str:
+ """Create a highlighted screenshot with bounding boxes around interactive elements.
+
+ Args:
+ screenshot_b64: Base64 encoded screenshot
+ selector_map: Map of interactive elements with their positions
+ device_pixel_ratio: Device pixel ratio for scaling coordinates
+ viewport_offset_x: X offset for viewport positioning
+ viewport_offset_y: Y offset for viewport positioning
+
+ Returns:
+ Base64 encoded highlighted screenshot
+ """
+ try:
+ # Decode screenshot
+ screenshot_data = base64.b64decode(screenshot_b64)
+ image = Image.open(io.BytesIO(screenshot_data)).convert('RGBA')
+
+ # Create drawing context
+ draw = ImageDraw.Draw(image)
+
+ # Try to load a font, fall back to default if not available
+ font = None
+ try:
+ font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 12)
+ except:
+ try:
+ font = ImageFont.truetype("arial.ttf", 12)
+ except:
+ font = None # Use default font
+
+ # Process each interactive element
+ for element_id, element in selector_map.items():
+ try:
+ # Get element bounds
+ bounds = element.bounds
+ if not bounds:
+ continue
+
+ # Scale coordinates by device pixel ratio and apply viewport offset
+ x1 = int((bounds.x + viewport_offset_x) * device_pixel_ratio)
+ y1 = int((bounds.y + viewport_offset_y) * device_pixel_ratio)
+ x2 = int((bounds.x + bounds.width + viewport_offset_x) * device_pixel_ratio)
+ y2 = int((bounds.y + bounds.height + viewport_offset_y) * device_pixel_ratio)
+
+ # Ensure coordinates are within image bounds
+ img_width, img_height = image.size
+ x1 = max(0, min(x1, img_width))
+ y1 = max(0, min(y1, img_height))
+ x2 = max(x1, min(x2, img_width))
+ y2 = max(y1, min(y2, img_height))
+
+ # Skip if bounding box is too small or invalid
+ if x2 - x1 < 2 or y2 - y1 < 2:
+ continue
+
+ # Get element color based on type
+ tag_name = element.tag_name if hasattr(element, 'tag_name') else 'div'
+ element_type = None
+ if hasattr(element, 'attributes') and element.attributes:
+ element_type = element.attributes.get('type')
+
+ color = get_element_color(tag_name, element_type)
+
+ # Get text for overlay (if short enough)
+ text = None
+ if hasattr(element, 'text') and element.text:
+ text = element.text.strip()
+ elif hasattr(element, 'attributes') and element.attributes:
+ # Try to get meaningful text from attributes
+ text = (element.attributes.get('aria-label') or
+ element.attributes.get('title') or
+ element.attributes.get('placeholder') or
+ element.attributes.get('value', ''))
+
+ # Draw bounding box with optional text
+ draw_bounding_box_with_text(
+ draw, (x1, y1, x2, y2), color, text, font
+ )
+
+ except Exception as e:
+ logger.debug(f"Failed to draw highlight for element {element_id}: {e}")
+ continue
+
+ # Convert back to base64
+ output_buffer = io.BytesIO()
+ image.save(output_buffer, format='PNG')
+ output_buffer.seek(0)
+
+ highlighted_b64 = base64.b64encode(output_buffer.getvalue()).decode('utf-8')
+
+ logger.debug(f"Successfully created highlighted screenshot with {len(selector_map)} elements")
+ return highlighted_b64
+
+ except Exception as e:
+ logger.error(f"Failed to create highlighted screenshot: {e}")
+ # Return original screenshot on error
+ return screenshot_b64
+
+def get_viewport_info_from_cdp(cdp_session) -> Tuple[float, int, int]:
+ """Get viewport information from CDP session.
+
+ Returns:
+ Tuple of (device_pixel_ratio, viewport_offset_x, viewport_offset_y)
+ """
+ # This is a placeholder - in real implementation, you'd get this from CDP
+ # For now, return sensible defaults
+ return 1.0, 0, 0
+
+async def create_highlighted_screenshot_async(
+ screenshot_b64: str,
+ selector_map: DOMSelectorMap,
+ cdp_session = None
+) -> str:
+ """Async wrapper for creating highlighted screenshots.
+
+ Args:
+ screenshot_b64: Base64 encoded screenshot
+ selector_map: Map of interactive elements
+ cdp_session: CDP session for getting viewport info
+
+ Returns:
+ Base64 encoded highlighted screenshot
+ """
+ # Get viewport information if CDP session is available
+ device_pixel_ratio = 1.0
+ viewport_offset_x = 0
+ viewport_offset_y = 0
+
+ if cdp_session:
+ try:
+ device_pixel_ratio, viewport_offset_x, viewport_offset_y = get_viewport_info_from_cdp(cdp_session)
+ except Exception as e:
+ logger.debug(f"Failed to get viewport info from CDP: {e}")
+
+ # Create highlighted screenshot (run in thread pool if needed for performance)
+ return create_highlighted_screenshot(
+ screenshot_b64,
+ selector_map,
+ device_pixel_ratio,
+ viewport_offset_x,
+ viewport_offset_y
+ )
\ No newline at end of file