Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Add display highlights on screen feature and enhance bounding box drawing

Magnus Müller634dd20e3707
diff --git a/browser_use/browser/profile.py b/browser_use/browser/profile.py
index e29d910b0..c9170b8b6 100644
--- a/browser_use/browser/profile.py
+++ b/browser_use/browser/profile.py
@@ -583,6 +583,9 @@ class BrowserProfile(BrowserConnectArgs, BrowserLaunchPersistentContextArgs, Bro
# --- UI/viewport/DOM ---
highlight_elements: bool = Field(default=True, description='Highlight interactive elements on the page.')
+ display_highlights_on_screen: bool = Field(
+ default=True, description='Display highlights directly on the browser screen in addition to screenshots.'
+ )
# --- Downloads ---
auto_download_pdfs: bool = Field(
diff --git a/browser_use/browser/python_highlights.py b/browser_use/browser/python_highlights.py
index a1d34f35b..688d2def8 100644
--- a/browser_use/browser/python_highlights.py
+++ b/browser_use/browser/python_highlights.py
@@ -52,6 +52,114 @@ def should_show_index_overlay(element_index: Optional[int]) -> bool:
return element_index is not None
+def draw_enhanced_bounding_box_with_text(
+ draw, # ImageDraw.Draw - avoiding type annotation due to PIL typing issues
+ bbox: Tuple[int, int, int, int],
+ color: str,
+ text: Optional[str] = None,
+ font: Optional[ImageFont.FreeTypeFont] = None,
+ element_type: str = 'div',
+) -> None:
+ """Draw an enhanced bounding box with bigger index and solid borders for better visibility."""
+ x1, y1, x2, y2 = bbox
+
+ # Draw solid bounding box (not dashed) with thicker lines for better visibility
+ line_width = 3
+
+ # Draw the main bounding box
+ draw.rectangle([x1, y1, x2, y2], outline=color, width=line_width)
+
+ # Add a subtle inner highlight for better visibility
+ if x2 - x1 > 6 and y2 - y1 > 6:
+ draw.rectangle([x1 + 1, y1 + 1, x2 - 1, y2 - 1], outline=color, width=1)
+
+ # Draw bigger index overlay if we have index text
+ if text:
+ try:
+ # Use bigger font size for index
+ big_font = None
+ try:
+ big_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf', 16)
+ except (OSError, IOError):
+ try:
+ big_font = ImageFont.truetype('arial.ttf', 16)
+ except (OSError, IOError):
+ big_font = font # Fallback to original font
+
+ # Get text size with bigger font
+ if big_font:
+ bbox_text = draw.textbbox((0, 0), text, font=big_font)
+ text_width = bbox_text[2] - bbox_text[0]
+ text_height = bbox_text[3] - bbox_text[1]
+ else:
+ # Fallback for default font
+ bbox_text = draw.textbbox((0, 0), text)
+ text_width = bbox_text[2] - bbox_text[0]
+ text_height = bbox_text[3] - bbox_text[1]
+
+ # Bigger padding for more prominent index
+ padding = 8
+ element_width = x2 - x1
+ element_height = y2 - y1
+
+ # Always try to place inside the element first, then outside if too small
+ if element_width >= text_width + padding * 2 and element_height >= text_height + padding * 2:
+ # Place in top-left corner inside the element
+ text_x = x1 + padding
+ text_y = y1 + padding
+ else:
+ # Place outside above the element
+ text_x = x1
+ text_y = max(0, y1 - text_height - padding)
+
+ # Ensure text stays within image bounds
+ img_width = 1200 # Default assumption, could be passed as parameter
+ img_height = 800
+ text_x = max(0, min(text_x, img_width - text_width - padding))
+ text_y = max(0, min(text_y, img_height - text_height - padding))
+
+ # Draw bigger background rectangle with element-type-specific styling
+ bg_x1 = text_x - padding
+ bg_y1 = text_y - padding
+ bg_x2 = text_x + text_width + padding
+ bg_y2 = text_y + text_height + padding
+
+ # Use element color as background with white text for high contrast
+ draw.rectangle([bg_x1, bg_y1, bg_x2, bg_y2], fill=color, outline='white', width=2)
+
+ # Draw white text on colored background for maximum visibility
+ draw.text((text_x, text_y), text, fill='white', font=big_font or font)
+
+ # Add element type indicator if space allows
+ if element_width >= 60 and element_height >= 40:
+ type_text = element_type.upper()[:3] # Show first 3 chars of element type
+ try:
+ small_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 10)
+ except (OSError, IOError):
+ small_font = font
+
+ if small_font:
+ type_bbox = draw.textbbox((0, 0), type_text, font=small_font)
+ type_width = type_bbox[2] - type_bbox[0]
+ type_height = type_bbox[3] - type_bbox[1]
+
+ # Place type text in bottom-right corner
+ type_x = x2 - type_width - 4
+ type_y = y2 - type_height - 4
+
+ # Small background for type text
+ draw.rectangle(
+ [type_x - 2, type_y - 1, type_x + type_width + 2, type_y + type_height + 1],
+ fill='rgba(0,0,0,128)',
+ outline=color,
+ width=1,
+ )
+ draw.text((type_x, type_y), type_text, fill='white', font=small_font)
+
+ except Exception as e:
+ logger.debug(f'Failed to draw enhanced text overlay: {e}')
+
+
def draw_bounding_box_with_text(
draw, # ImageDraw.Draw - avoiding type annotation due to PIL typing issues
bbox: Tuple[int, int, int, int],
@@ -199,22 +307,18 @@ def create_highlighted_screenshot(
# Process each interactive element
for element_id, element in selector_map.items():
try:
- # Use snapshot bounds (document coordinates) if available, otherwise absolute_position
- bounds = None
- if element.snapshot_node and element.snapshot_node.bounds:
- bounds = element.snapshot_node.bounds
- elif element.absolute_position:
- bounds = element.absolute_position
-
- if not bounds:
+ # Use absolute_position coordinates directly
+ if not element.absolute_position:
continue
- # Convert from CSS pixels to device pixels for screenshot coordinates
- # Note: bounds are already in CSS pixels, screenshot is in device pixels
- x1 = int((bounds.x - viewport_offset_x) * device_pixel_ratio)
- y1 = int((bounds.y - viewport_offset_y) * device_pixel_ratio)
- x2 = int((bounds.x + bounds.width - viewport_offset_x) * device_pixel_ratio)
- y2 = int((bounds.y + bounds.height - viewport_offset_y) * device_pixel_ratio)
+ bounds = element.absolute_position
+
+ # Scale coordinates from CSS pixels to device pixels for screenshot
+ # The screenshot is captured at device pixel resolution, but coordinates are in CSS pixels
+ x1 = int(bounds.x * device_pixel_ratio)
+ y1 = int(bounds.y * device_pixel_ratio)
+ x2 = int((bounds.x + bounds.width) * device_pixel_ratio)
+ y2 = int((bounds.y + bounds.height) * device_pixel_ratio)
# Ensure coordinates are within image bounds
img_width, img_height = image.size
@@ -239,8 +343,8 @@ def create_highlighted_screenshot(
element_index = getattr(element, 'element_index', None)
index_text = str(element_index) if element_index is not None else None
- # Draw bounding box with index
- draw_bounding_box_with_text(draw, (x1, y1, x2, y2), color, index_text, font)
+ # Draw enhanced bounding box with bigger index
+ draw_enhanced_bounding_box_with_text(draw, (x1, y1, x2, y2), color, index_text, font, tag_name)
except Exception as e:
logger.debug(f'Failed to draw highlight for element {element_id}: {e}')
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index b4b01f62c..910324cf9 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -265,6 +265,7 @@ class BrowserSession(BaseModel):
wait_for_network_idle_page_load_time: float | None = None,
wait_between_actions: float | None = None,
highlight_elements: bool | None = None,
+ display_highlights_on_screen: bool | None = None,
auto_download_pdfs: bool | None = None,
profile_directory: str | None = None,
):
diff --git a/browser_use/browser/watchdogs/dom_watchdog.py b/browser_use/browser/watchdogs/dom_watchdog.py
index 0db217d8b..72dee72cc 100644
--- a/browser_use/browser/watchdogs/dom_watchdog.py
+++ b/browser_use/browser/watchdogs/dom_watchdog.py
@@ -239,6 +239,94 @@ class DOMWatchdog(BaseWatchdog):
except Exception as e:
self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Python highlighting failed: {e}')
+ # Display highlighted screenshot overlay on screen if requested
+ if (
+ screenshot_b64
+ and content
+ and content.selector_map
+ and self.browser_session.browser_profile.display_highlights_on_screen
+ ):
+ try:
+ self.logger.debug(
+ '🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🖥️ Displaying highlighted screenshot overlay...'
+ )
+
+ # Use the existing screenshot (which has correct scaling) and add highlights to it
+ # Get CDP session for viewport info
+ cdp_session = await self.browser_session.get_or_create_cdp_session()
+
+ # Create highlighted version of the existing screenshot
+ from browser_use.browser.python_highlights import create_highlighted_screenshot_async
+
+ highlighted_screenshot_b64 = await create_highlighted_screenshot_async(
+ screenshot_b64, content.selector_map, cdp_session
+ )
+
+ # Get viewport dimensions and device pixel ratio for proper scaling
+ viewport_info = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)
+ visual_viewport = viewport_info.get('visualViewport', {})
+ css_visual_viewport = viewport_info.get('cssVisualViewport', {})
+
+ # Get CSS pixel dimensions (what the browser shows)
+ css_width = css_visual_viewport.get('clientWidth', 1280)
+ css_height = css_visual_viewport.get('clientHeight', 720)
+
+ # Get device pixel dimensions (what the screenshot is)
+ device_width = visual_viewport.get('clientWidth', css_width)
+ device_height = visual_viewport.get('clientHeight', css_height)
+
+ # Calculate device pixel ratio
+ device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0
+
+ # Inject the highlighted screenshot as an overlay with correct DPR scaling
+ overlay_script = f"""
+ (function() {{
+ // Remove any existing browser-use overlay
+ const existingOverlay = document.getElementById('browser-use-highlight-overlay');
+ if (existingOverlay) {{
+ existingOverlay.remove();
+ }}
+
+ // Get actual viewport dimensions in CSS pixels
+ const cssWidth = window.innerWidth;
+ const cssHeight = window.innerHeight;
+ const devicePixelRatio = window.devicePixelRatio || 1;
+
+ // Create overlay container
+ const overlay = document.createElement('div');
+ overlay.id = 'browser-use-highlight-overlay';
+ overlay.style.cssText = `
+ position: fixed;
+ top: 0;
+ left: 0;
+ width: ${{cssWidth}}px;
+ height: ${{cssHeight}}px;
+ pointer-events: none;
+ z-index: 2147483647;
+ opacity: 0.4;
+ background-image: url(data:image/png;base64,{highlighted_screenshot_b64});
+ background-size: ${{cssWidth}}px ${{cssHeight}}px;
+ background-position: top left;
+ background-repeat: no-repeat;
+ `;
+
+ document.body.appendChild(overlay);
+ console.log('Browser-use highlighted screenshot overlay injected with', '{len(content.selector_map)}', 'elements');
+ console.log('CSS viewport:', cssWidth, 'x', cssHeight, 'DPR:', devicePixelRatio);
+ }})();
+ """
+
+ # Execute the overlay script
+ await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={'expression': overlay_script, 'returnByValue': True}, session_id=cdp_session.session_id
+ )
+
+ self.logger.debug(
+ f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Displayed highlighted screenshot overlay with {len(content.selector_map)} elements'
+ )
+ except Exception as e:
+ self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Screenshot overlay failed: {e}')
+
# Ensure we have valid content
if not content:
content = SerializedDOMState(_root=None, selector_map={})