Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Add display highlights on screen feature and enhance bounding box drawing
Magnus Müller634dd20e3707
diff --git a/browser_use/browser/profile.py b/browser_use/browser/profile.pyindex e29d910b0..c9170b8b6 100644--- a/browser_use/browser/profile.py+++ b/browser_use/browser/profile.py@@ -583,6 +583,9 @@ class BrowserProfile(BrowserConnectArgs, BrowserLaunchPersistentContextArgs, Bro# --- UI/viewport/DOM ---highlight_elements: bool = Field(default=True, description='Highlight interactive elements on the page.')+ display_highlights_on_screen: bool = Field(+ default=True, description='Display highlights directly on the browser screen in addition to screenshots.'+ )# --- Downloads ---auto_download_pdfs: bool = Field(diff --git a/browser_use/browser/python_highlights.py b/browser_use/browser/python_highlights.pyindex a1d34f35b..688d2def8 100644--- a/browser_use/browser/python_highlights.py+++ b/browser_use/browser/python_highlights.py@@ -52,6 +52,114 @@ def should_show_index_overlay(element_index: Optional[int]) -> bool:return element_index is not None+def draw_enhanced_bounding_box_with_text(+ draw, # ImageDraw.Draw - avoiding type annotation due to PIL typing issues+ bbox: Tuple[int, int, int, int],+ color: str,+ text: Optional[str] = None,+ font: Optional[ImageFont.FreeTypeFont] = None,+ element_type: str = 'div',+) -> None:+ """Draw an enhanced bounding box with bigger index and solid borders for better visibility."""+ x1, y1, x2, y2 = bbox++ # Draw solid bounding box (not dashed) with thicker lines for better visibility+ line_width = 3++ # Draw the main bounding box+ draw.rectangle([x1, y1, x2, y2], outline=color, width=line_width)++ # Add a subtle inner highlight for better visibility+ if x2 - x1 > 6 and y2 - y1 > 6:+ draw.rectangle([x1 + 1, y1 + 1, x2 - 1, y2 - 1], outline=color, width=1)++ # Draw bigger index overlay if we have index text+ if text:+ try:+ # Use bigger font size for index+ big_font = None+ try:+ big_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf', 16)+ except (OSError, IOError):+ try:+ big_font = ImageFont.truetype('arial.ttf', 16)+ except (OSError, IOError):+ big_font = font # Fallback to original font++ # Get text size with bigger font+ if big_font:+ bbox_text = draw.textbbox((0, 0), text, font=big_font)+ text_width = bbox_text[2] - bbox_text[0]+ text_height = bbox_text[3] - bbox_text[1]+ else:+ # Fallback for default font+ bbox_text = draw.textbbox((0, 0), text)+ text_width = bbox_text[2] - bbox_text[0]+ text_height = bbox_text[3] - bbox_text[1]++ # Bigger padding for more prominent index+ padding = 8+ element_width = x2 - x1+ element_height = y2 - y1++ # Always try to place inside the element first, then outside if too small+ if element_width >= text_width + padding * 2 and element_height >= text_height + padding * 2:+ # Place in top-left corner inside the element+ text_x = x1 + padding+ text_y = y1 + padding+ else:+ # Place outside above the element+ text_x = x1+ text_y = max(0, y1 - text_height - padding)++ # Ensure text stays within image bounds+ img_width = 1200 # Default assumption, could be passed as parameter+ img_height = 800+ text_x = max(0, min(text_x, img_width - text_width - padding))+ text_y = max(0, min(text_y, img_height - text_height - padding))++ # Draw bigger background rectangle with element-type-specific styling+ bg_x1 = text_x - padding+ bg_y1 = text_y - padding+ bg_x2 = text_x + text_width + padding+ bg_y2 = text_y + text_height + padding++ # Use element color as background with white text for high contrast+ draw.rectangle([bg_x1, bg_y1, bg_x2, bg_y2], fill=color, outline='white', width=2)++ # Draw white text on colored background for maximum visibility+ draw.text((text_x, text_y), text, fill='white', font=big_font or font)++ # Add element type indicator if space allows+ if element_width >= 60 and element_height >= 40:+ type_text = element_type.upper()[:3] # Show first 3 chars of element type+ try:+ small_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 10)+ except (OSError, IOError):+ small_font = font++ if small_font:+ type_bbox = draw.textbbox((0, 0), type_text, font=small_font)+ type_width = type_bbox[2] - type_bbox[0]+ type_height = type_bbox[3] - type_bbox[1]++ # Place type text in bottom-right corner+ type_x = x2 - type_width - 4+ type_y = y2 - type_height - 4++ # Small background for type text+ draw.rectangle(+ [type_x - 2, type_y - 1, type_x + type_width + 2, type_y + type_height + 1],+ fill='rgba(0,0,0,128)',+ outline=color,+ width=1,+ )+ draw.text((type_x, type_y), type_text, fill='white', font=small_font)++ except Exception as e:+ logger.debug(f'Failed to draw enhanced text overlay: {e}')++def draw_bounding_box_with_text(draw, # ImageDraw.Draw - avoiding type annotation due to PIL typing issuesbbox: Tuple[int, int, int, int],@@ -199,22 +307,18 @@ def create_highlighted_screenshot(# Process each interactive elementfor element_id, element in selector_map.items():try:- # Use snapshot bounds (document coordinates) if available, otherwise absolute_position- bounds = None- if element.snapshot_node and element.snapshot_node.bounds:- bounds = element.snapshot_node.bounds- elif element.absolute_position:- bounds = element.absolute_position-- if not bounds:+ # Use absolute_position coordinates directly+ if not element.absolute_position:continue- # Convert from CSS pixels to device pixels for screenshot coordinates- # Note: bounds are already in CSS pixels, screenshot is in device pixels- x1 = int((bounds.x - viewport_offset_x) * device_pixel_ratio)- y1 = int((bounds.y - viewport_offset_y) * device_pixel_ratio)- x2 = int((bounds.x + bounds.width - viewport_offset_x) * device_pixel_ratio)- y2 = int((bounds.y + bounds.height - viewport_offset_y) * device_pixel_ratio)+ bounds = element.absolute_position++ # Scale coordinates from CSS pixels to device pixels for screenshot+ # The screenshot is captured at device pixel resolution, but coordinates are in CSS pixels+ x1 = int(bounds.x * device_pixel_ratio)+ y1 = int(bounds.y * device_pixel_ratio)+ x2 = int((bounds.x + bounds.width) * device_pixel_ratio)+ y2 = int((bounds.y + bounds.height) * device_pixel_ratio)# Ensure coordinates are within image boundsimg_width, img_height = image.size@@ -239,8 +343,8 @@ def create_highlighted_screenshot(element_index = getattr(element, 'element_index', None)index_text = str(element_index) if element_index is not None else None- # Draw bounding box with index- draw_bounding_box_with_text(draw, (x1, y1, x2, y2), color, index_text, font)+ # Draw enhanced bounding box with bigger index+ draw_enhanced_bounding_box_with_text(draw, (x1, y1, x2, y2), color, index_text, font, tag_name)except Exception as e:logger.debug(f'Failed to draw highlight for element {element_id}: {e}')diff --git a/browser_use/browser/session.py b/browser_use/browser/session.pyindex b4b01f62c..910324cf9 100644--- a/browser_use/browser/session.py+++ b/browser_use/browser/session.py@@ -265,6 +265,7 @@ class BrowserSession(BaseModel):wait_for_network_idle_page_load_time: float | None = None,wait_between_actions: float | None = None,highlight_elements: bool | None = None,+ display_highlights_on_screen: bool | None = None,auto_download_pdfs: bool | None = None,profile_directory: str | None = None,):diff --git a/browser_use/browser/watchdogs/dom_watchdog.py b/browser_use/browser/watchdogs/dom_watchdog.pyindex 0db217d8b..72dee72cc 100644--- a/browser_use/browser/watchdogs/dom_watchdog.py+++ b/browser_use/browser/watchdogs/dom_watchdog.py@@ -239,6 +239,94 @@ class DOMWatchdog(BaseWatchdog):except Exception as e:self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Python highlighting failed: {e}')+ # Display highlighted screenshot overlay on screen if requested+ if (+ screenshot_b64+ and content+ and content.selector_map+ and self.browser_session.browser_profile.display_highlights_on_screen+ ):+ try:+ self.logger.debug(+ '🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🖥️ Displaying highlighted screenshot overlay...'+ )++ # Use the existing screenshot (which has correct scaling) and add highlights to it+ # Get CDP session for viewport info+ cdp_session = await self.browser_session.get_or_create_cdp_session()++ # Create highlighted version of the existing screenshot+ from browser_use.browser.python_highlights import create_highlighted_screenshot_async++ highlighted_screenshot_b64 = await create_highlighted_screenshot_async(+ screenshot_b64, content.selector_map, cdp_session+ )++ # Get viewport dimensions and device pixel ratio for proper scaling+ viewport_info = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)+ visual_viewport = viewport_info.get('visualViewport', {})+ css_visual_viewport = viewport_info.get('cssVisualViewport', {})++ # Get CSS pixel dimensions (what the browser shows)+ css_width = css_visual_viewport.get('clientWidth', 1280)+ css_height = css_visual_viewport.get('clientHeight', 720)++ # Get device pixel dimensions (what the screenshot is)+ device_width = visual_viewport.get('clientWidth', css_width)+ device_height = visual_viewport.get('clientHeight', css_height)++ # Calculate device pixel ratio+ device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0++ # Inject the highlighted screenshot as an overlay with correct DPR scaling+ overlay_script = f"""+ (function() {{+ // Remove any existing browser-use overlay+ const existingOverlay = document.getElementById('browser-use-highlight-overlay');+ if (existingOverlay) {{+ existingOverlay.remove();+ }}++ // Get actual viewport dimensions in CSS pixels+ const cssWidth = window.innerWidth;+ const cssHeight = window.innerHeight;+ const devicePixelRatio = window.devicePixelRatio || 1;++ // Create overlay container+ const overlay = document.createElement('div');+ overlay.id = 'browser-use-highlight-overlay';+ overlay.style.cssText = `+ position: fixed;+ top: 0;+ left: 0;+ width: ${{cssWidth}}px;+ height: ${{cssHeight}}px;+ pointer-events: none;+ z-index: 2147483647;+ opacity: 0.4;+ background-image: url(data:image/png;base64,{highlighted_screenshot_b64});+ background-size: ${{cssWidth}}px ${{cssHeight}}px;+ background-position: top left;+ background-repeat: no-repeat;+ `;++ document.body.appendChild(overlay);+ console.log('Browser-use highlighted screenshot overlay injected with', '{len(content.selector_map)}', 'elements');+ console.log('CSS viewport:', cssWidth, 'x', cssHeight, 'DPR:', devicePixelRatio);+ }})();+ """++ # Execute the overlay script+ await cdp_session.cdp_client.send.Runtime.evaluate(+ params={'expression': overlay_script, 'returnByValue': True}, session_id=cdp_session.session_id+ )++ self.logger.debug(+ f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Displayed highlighted screenshot overlay with {len(content.selector_map)} elements'+ )+ except Exception as e:+ self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Screenshot overlay failed: {e}')+# Ensure we have valid contentif not content:content = SerializedDOMState(_root=None, selector_map={})