Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
BBox on entire page works
Magnus Müller1375013df791
diff --git a/browser_use/browser/dom_watchdog.py b/browser_use/browser/dom_watchdog.pyindex 82d5144a0..a57e46ed6 100644--- a/browser_use/browser/dom_watchdog.py+++ b/browser_use/browser/dom_watchdog.py@@ -232,69 +232,68 @@ class DOMWatchdog(BaseWatchdog):recent_events=self._get_recent_events_str() if event.include_recent_events else None,)- # Execute DOM building and screenshot capture in parallel- dom_task = None- screenshot_task = None-- # Start DOM building task if requested- if event.include_dom:- self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🌳 Starting DOM tree build task...')-- previous_state = (- self.browser_session._cached_browser_state_summary.dom_state- if self.browser_session._cached_browser_state_summary- else None- )-- dom_task = asyncio.create_task(self._build_dom_tree_without_highlights(previous_state))-- # Start clean screenshot task if requested (without JS highlights)- if event.include_screenshot:- self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Starting clean screenshot task...')- screenshot_task = asyncio.create_task(self._capture_clean_screenshot())-- # Wait for both tasks to complete- content = None- screenshot_b64 = None-- if dom_task:- try:- content = await dom_task- self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ DOM tree build completed')- except Exception as e:- self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: DOM build failed: {e}, using minimal state')- content = SerializedDOMState(_root=None, selector_map={})- else:- content = SerializedDOMState(_root=None, selector_map={})-- if screenshot_task:- try:- screenshot_b64 = await screenshot_task- self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Clean screenshot captured')- except Exception as e:- self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Clean screenshot failed: {e}')- screenshot_b64 = None-- # Apply Python-based highlighting if both DOM and screenshot are available- if (screenshot_b64 and content and content.selector_map and- self.browser_session.browser_profile.highlight_elements):- try:- self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🎨 Applying Python-based highlighting...')- from browser_use.browser.python_highlights import create_highlighted_screenshot_async-- # Get CDP session for viewport info- cdp_session = await self.browser_session.get_or_create_cdp_session()-- screenshot_b64 = await create_highlighted_screenshot_async(- screenshot_b64, content.selector_map, cdp_session+ # Execute DOM building and screenshot capture in parallel+ dom_task = None+ screenshot_task = None++ # Start DOM building task if requested+ if event.include_dom:+ self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🌳 Starting DOM tree build task...')++ previous_state = (+ self.browser_session._cached_browser_state_summary.dom_state+ if self.browser_session._cached_browser_state_summary+ else None)- self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Applied highlights to {len(content.selector_map)} elements')- except Exception as e:- self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Python highlighting failed: {e}')-- # Ensure we have valid content- if not content:- content = SerializedDOMState(_root=None, selector_map={})++ dom_task = asyncio.create_task(self._build_dom_tree_without_highlights(previous_state))++ # Start clean screenshot task if requested (without JS highlights)+ if event.include_screenshot:+ self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Starting clean screenshot task...')+ screenshot_task = asyncio.create_task(self._capture_clean_screenshot())++ # Wait for both tasks to complete+ content = None+ screenshot_b64 = None++ if dom_task:+ try:+ content = await dom_task+ self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ DOM tree build completed')+ except Exception as e:+ self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: DOM build failed: {e}, using minimal state')+ content = SerializedDOMState(_root=None, selector_map={})+ else:+ content = SerializedDOMState(_root=None, selector_map={})++ if screenshot_task:+ try:+ screenshot_b64 = await screenshot_task+ self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Clean screenshot captured')+ except Exception as e:+ self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Clean screenshot failed: {e}')+ screenshot_b64 = None++ # Apply Python-based highlighting if both DOM and screenshot are available+ if screenshot_b64 and content and content.selector_map and self.browser_session.browser_profile.highlight_elements:+ try:+ self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🎨 Applying Python-based highlighting...')+ from browser_use.browser.python_highlights import create_highlighted_screenshot_async++ # Get CDP session for viewport info+ cdp_session = await self.browser_session.get_or_create_cdp_session()++ screenshot_b64 = await create_highlighted_screenshot_async(screenshot_b64, content.selector_map, cdp_session)+ self.logger.debug(+ f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Applied highlights to {len(content.selector_map)} elements'+ )+ except Exception as e:+ self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Python highlighting failed: {e}')++ # Ensure we have valid content+ if not content:+ content = SerializedDOMState(_root=None, selector_map={})# Tabs info already fetched at the beginning@@ -466,6 +465,87 @@ class DOMWatchdog(BaseWatchdog):)raise+ async def _build_dom_tree_without_highlights(self, previous_state: SerializedDOMState | None = None) -> SerializedDOMState:+ """Build DOM tree without injecting JavaScript highlights (for parallel execution)."""+ try:+ self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: STARTING DOM tree build')++ # Create or reuse DOM service+ if self._dom_service is None:+ self._dom_service = DomService(browser_session=self.browser_session, logger=self.logger)++ # Get serialized DOM tree using the service+ self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Calling DomService.get_serialized_dom_tree...')+ start = time.time()+ self.current_dom_state, self.enhanced_dom_tree, timing_info = await self._dom_service.get_serialized_dom_tree(+ previous_cached_state=previous_state,+ )+ end = time.time()+ self.logger.debug(+ '🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ DomService.get_serialized_dom_tree completed'+ )++ self.logger.debug(f'Time taken to get DOM tree: {end - start} seconds')+ self.logger.debug(f'Timing breakdown: {timing_info}')++ # Update selector map for other watchdogs+ self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Updating selector maps...')+ self.selector_map = self.current_dom_state.selector_map+ # Update BrowserSession's cached selector map+ if self.browser_session:+ self.browser_session.update_cached_selector_map(self.selector_map)+ self.logger.debug(+ f'🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ Selector maps updated, {len(self.selector_map)} elements'+ )++ # Skip JavaScript highlighting injection - Python highlighting will be applied later+ self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ COMPLETED DOM tree build (no JS highlights)')+ return self.current_dom_state++ except Exception as e:+ self.logger.error(f'Failed to build DOM tree without highlights: {e}')+ self.event_bus.dispatch(+ BrowserErrorEvent(+ error_type='DOMBuildFailed',+ message=str(e),+ )+ )+ raise++ async def _capture_clean_screenshot(self) -> str:+ """Capture a clean screenshot without JavaScript highlights."""+ try:+ self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: Capturing clean screenshot...')++ # Ensure we have a focused CDP session+ assert self.browser_session.agent_focus is not None, 'No current target ID'+ await self.browser_session.get_or_create_cdp_session(target_id=self.browser_session.agent_focus.target_id, focus=True)++ # Check if handler is registered+ handlers = self.event_bus.handlers.get('ScreenshotEvent', [])+ handler_names = [getattr(h, '__name__', str(h)) for h in handlers]+ self.logger.debug(f'📸 ScreenshotEvent handlers registered: {len(handlers)} - {handler_names}')++ screenshot_event = self.event_bus.dispatch(ScreenshotEvent(full_page=False))+ self.logger.debug('📸 Dispatched ScreenshotEvent, waiting for event to complete...')++ # Wait for the event itself to complete (this waits for all handlers)+ await screenshot_event++ # Get the single handler result+ screenshot_b64 = await screenshot_event.event_result(raise_if_any=True, raise_if_none=True)+ if screenshot_b64 is None:+ raise RuntimeError('Screenshot handler returned None')+ self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: ✅ Clean screenshot captured successfully')+ return str(screenshot_b64)++ except TimeoutError:+ self.logger.warning('📸 Clean screenshot timed out after 6 seconds - no handler registered or slow page?')+ raise+ except Exception as e:+ self.logger.warning(f'📸 Clean screenshot failed: {type(e).__name__}: {e}')+ raise+async def _wait_for_stable_network(self):"""Wait for page stability - simplified for CDP-only branch."""start_time = time.time()diff --git a/browser_use/browser/python_highlights.py b/browser_use/browser/python_highlights.pyindex 357532c66..a07f59aaa 100644--- a/browser_use/browser/python_highlights.py+++ b/browser_use/browser/python_highlights.py@@ -6,240 +6,261 @@ to draw bounding boxes around interactive elements directly on screenshots.import base64import io-from typing import Dict, Any, Optional, Tuple-from PIL import Image, ImageDraw, ImageFontimport logging+from typing import Optional, Tuple++from PIL import Image, ImageDraw, ImageFont-from browser_use.dom.serializer.types import DOMSelectorMap+from browser_use.dom.views import DOMSelectorMap+from browser_use.observability import observe_debuglogger = logging.getLogger(__name__)# Color scheme for different element typesELEMENT_COLORS = {- 'button': '#FF6B6B', # Red for buttons- 'input': '#4ECDC4', # Teal for inputs- 'select': '#45B7D1', # Blue for dropdowns- 'a': '#96CEB4', # Green for links- 'textarea': '#FFEAA7', # Yellow for text areas- 'default': '#DDA0DD', # Light purple for other interactive elements+ 'button': '#FF6B6B', # Red for buttons+ 'input': '#4ECDC4', # Teal for inputs+ 'select': '#45B7D1', # Blue for dropdowns+ 'a': '#96CEB4', # Green for links+ 'textarea': '#FFEAA7', # Yellow for text areas+ 'default': '#DDA0DD', # Light purple for other interactive elements}# Element type mappingsELEMENT_TYPE_MAP = {- 'button': 'button',- 'input': 'input',- 'select': 'select',- 'a': 'a',- 'textarea': 'textarea',+ 'button': 'button',+ 'input': 'input',+ 'select': 'select',+ 'a': 'a',+ 'textarea': 'textarea',}+def get_element_color(tag_name: str, element_type: Optional[str] = None) -> str:- """Get color for element based on tag name and type."""- # Check input type first- if tag_name == 'input' and element_type:- if element_type in ['button', 'submit']:- return ELEMENT_COLORS['button']-- # Use tag-based color- return ELEMENT_COLORS.get(tag_name.lower(), ELEMENT_COLORS['default'])+ """Get color for element based on tag name and type."""+ # Check input type first+ if tag_name == 'input' and element_type:+ if element_type in ['button', 'submit']:+ return ELEMENT_COLORS['button']++ # Use tag-based color+ return ELEMENT_COLORS.get(tag_name.lower(), ELEMENT_COLORS['default'])+def should_show_text_overlay(text: Optional[str]) -> bool:- """Determine if text overlay should be shown based on length."""- if not text:- return False- return len(text.strip()) <= 10+ """Determine if text overlay should be shown based on length."""+ if not text:+ return False+ return len(text.strip()) <= 10+def draw_bounding_box_with_text(- draw: ImageDraw.Draw,- bbox: Tuple[int, int, int, int],- color: str,- text: Optional[str] = None,- font: Optional[ImageFont.FreeTypeFont] = None+ draw, # ImageDraw.Draw - avoiding type annotation due to PIL typing issues+ bbox: Tuple[int, int, int, int],+ color: str,+ text: Optional[str] = None,+ font: Optional[ImageFont.FreeTypeFont] = None,) -> None:- """Draw a bounding box with optional text overlay."""- x1, y1, x2, y2 = bbox-- # Draw bounding box with 2px width- for i in range(2):- draw.rectangle([x1 + i, y1 + i, x2 - i, y2 - i], outline=color, fill=None)-- # Draw text overlay if provided and short enough- if text and should_show_text_overlay(text):- try:- # Get text size- if font:- bbox_text = draw.textbbox((0, 0), text, font=font)- text_width = bbox_text[2] - bbox_text[0]- text_height = bbox_text[3] - bbox_text[1]- else:- # Fallback for default font- bbox_text = draw.textbbox((0, 0), text)- text_width = bbox_text[2] - bbox_text[0]- text_height = bbox_text[3] - bbox_text[1]-- # Position text at top-left of bounding box- text_x = max(0, x1)- text_y = max(0, y1 - text_height - 2) # Above the box-- # Draw background rectangle for text- draw.rectangle(- [text_x - 2, text_y - 2, text_x + text_width + 2, text_y + text_height + 2],- fill=color,- outline=None- )-- # Draw text- draw.text((text_x, text_y), text, fill='white', font=font)-- except Exception as e:- logger.debug(f"Failed to draw text overlay: {e}")+ """Draw a bounding box with optional text overlay."""+ x1, y1, x2, y2 = bbox++ # Draw bounding box with 2px width+ for i in range(2):+ draw.rectangle([x1 + i, y1 + i, x2 - i, y2 - i], outline=color, fill=None)++ # Draw text overlay if provided and short enough+ if text and should_show_text_overlay(text):+ try:+ # Get text size+ if font:+ bbox_text = draw.textbbox((0, 0), text, font=font)+ text_width = bbox_text[2] - bbox_text[0]+ text_height = bbox_text[3] - bbox_text[1]+ else:+ # Fallback for default font+ bbox_text = draw.textbbox((0, 0), text)+ text_width = bbox_text[2] - bbox_text[0]+ text_height = bbox_text[3] - bbox_text[1]++ # Position text at top-left of bounding box+ text_x = max(0, x1)+ text_y = max(0, y1 - text_height - 2) # Above the box++ # Draw background rectangle for text+ draw.rectangle([text_x - 2, text_y - 2, text_x + text_width + 2, text_y + text_height + 2], fill=color, outline=None)++ # Draw text+ draw.text((text_x, text_y), text, fill='white', font=font)+ except Exception as e:+ logger.debug(f'Failed to draw text overlay: {e}')+++@observe_debug(ignore_input=True, ignore_output=True, name='create_highlighted_screenshot')def create_highlighted_screenshot(- screenshot_b64: str,- selector_map: DOMSelectorMap,- device_pixel_ratio: float = 1.0,- viewport_offset_x: int = 0,- viewport_offset_y: int = 0-) -> str:- """Create a highlighted screenshot with bounding boxes around interactive elements.-- Args:- screenshot_b64: Base64 encoded screenshot- selector_map: Map of interactive elements with their positions- device_pixel_ratio: Device pixel ratio for scaling coordinates- viewport_offset_x: X offset for viewport positioning- viewport_offset_y: Y offset for viewport positioning-- Returns:- Base64 encoded highlighted screenshot- """- try:- # Decode screenshot- screenshot_data = base64.b64decode(screenshot_b64)- image = Image.open(io.BytesIO(screenshot_data)).convert('RGBA')-- # Create drawing context- draw = ImageDraw.Draw(image)-- # Try to load a font, fall back to default if not available- font = None- try:- font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 12)- except:- try:- font = ImageFont.truetype("arial.ttf", 12)- except:- font = None # Use default font-- # Process each interactive element- for element_id, element in selector_map.items():- try:- # Get element bounds- bounds = element.bounds- if not bounds:- continue-- # Scale coordinates by device pixel ratio and apply viewport offset- x1 = int((bounds.x + viewport_offset_x) * device_pixel_ratio)- y1 = int((bounds.y + viewport_offset_y) * device_pixel_ratio)- x2 = int((bounds.x + bounds.width + viewport_offset_x) * device_pixel_ratio)- y2 = int((bounds.y + bounds.height + viewport_offset_y) * device_pixel_ratio)-- # Ensure coordinates are within image bounds- img_width, img_height = image.size- x1 = max(0, min(x1, img_width))- y1 = max(0, min(y1, img_height))- x2 = max(x1, min(x2, img_width))- y2 = max(y1, min(y2, img_height))-- # Skip if bounding box is too small or invalid- if x2 - x1 < 2 or y2 - y1 < 2:- continue-- # Get element color based on type- tag_name = element.tag_name if hasattr(element, 'tag_name') else 'div'- element_type = None- if hasattr(element, 'attributes') and element.attributes:- element_type = element.attributes.get('type')-- color = get_element_color(tag_name, element_type)-- # Get text for overlay (if short enough)- text = None- if hasattr(element, 'text') and element.text:- text = element.text.strip()- elif hasattr(element, 'attributes') and element.attributes:- # Try to get meaningful text from attributes- text = (element.attributes.get('aria-label') or- element.attributes.get('title') or- element.attributes.get('placeholder') or- element.attributes.get('value', ''))-- # Draw bounding box with optional text- draw_bounding_box_with_text(- draw, (x1, y1, x2, y2), color, text, font- )-- except Exception as e:- logger.debug(f"Failed to draw highlight for element {element_id}: {e}")- continue-- # Convert back to base64- output_buffer = io.BytesIO()- image.save(output_buffer, format='PNG')- output_buffer.seek(0)-- highlighted_b64 = base64.b64encode(output_buffer.getvalue()).decode('utf-8')-- logger.debug(f"Successfully created highlighted screenshot with {len(selector_map)} elements")- return highlighted_b64-- except Exception as e:- logger.error(f"Failed to create highlighted screenshot: {e}")- # Return original screenshot on error- return screenshot_b64--def get_viewport_info_from_cdp(cdp_session) -> Tuple[float, int, int]:- """Get viewport information from CDP session.-- Returns:- Tuple of (device_pixel_ratio, viewport_offset_x, viewport_offset_y)- """- # This is a placeholder - in real implementation, you'd get this from CDP- # For now, return sensible defaults- return 1.0, 0, 0--async def create_highlighted_screenshot_async(- screenshot_b64: str,- selector_map: DOMSelectorMap,- cdp_session = None+ screenshot_b64: str,+ selector_map: DOMSelectorMap,+ device_pixel_ratio: float = 1.0,+ viewport_offset_x: int = 0,+ viewport_offset_y: int = 0,) -> str:- """Async wrapper for creating highlighted screenshots.-- Args:- screenshot_b64: Base64 encoded screenshot- selector_map: Map of interactive elements- cdp_session: CDP session for getting viewport info-- Returns:- Base64 encoded highlighted screenshot- """- # Get viewport information if CDP session is available- device_pixel_ratio = 1.0- viewport_offset_x = 0- viewport_offset_y = 0-- if cdp_session:- try:- device_pixel_ratio, viewport_offset_x, viewport_offset_y = get_viewport_info_from_cdp(cdp_session)- except Exception as e:- logger.debug(f"Failed to get viewport info from CDP: {e}")-- # Create highlighted screenshot (run in thread pool if needed for performance)- return create_highlighted_screenshot(- screenshot_b64,- selector_map,- device_pixel_ratio,- viewport_offset_x,- viewport_offset_y- )\ No newline at end of file+ """Create a highlighted screenshot with bounding boxes around interactive elements.++ Args:+ screenshot_b64: Base64 encoded screenshot+ selector_map: Map of interactive elements with their positions+ device_pixel_ratio: Device pixel ratio for scaling coordinates+ viewport_offset_x: X offset for viewport positioning+ viewport_offset_y: Y offset for viewport positioning++ Returns:+ Base64 encoded highlighted screenshot+ """+ try:+ # Decode screenshot+ screenshot_data = base64.b64decode(screenshot_b64)+ image = Image.open(io.BytesIO(screenshot_data)).convert('RGBA')++ # Create drawing context+ draw = ImageDraw.Draw(image)++ # Try to load a font, fall back to default if not available+ font = None+ try:+ font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf', 12)+ except:+ try:+ font = ImageFont.truetype('arial.ttf', 12)+ except:+ font = None # Use default font++ # Process each interactive element+ for element_id, element in selector_map.items():+ try:+ # Use snapshot bounds (document coordinates) if available, otherwise absolute_position+ bounds = None+ if element.snapshot_node and element.snapshot_node.bounds:+ bounds = element.snapshot_node.bounds+ elif element.absolute_position:+ bounds = element.absolute_position++ if not bounds:+ continue++ # Convert from CSS pixels to device pixels for screenshot coordinates+ # Note: bounds are already in CSS pixels, screenshot is in device pixels+ x1 = int((bounds.x - viewport_offset_x) * device_pixel_ratio)+ y1 = int((bounds.y - viewport_offset_y) * device_pixel_ratio)+ x2 = int((bounds.x + bounds.width - viewport_offset_x) * device_pixel_ratio)+ y2 = int((bounds.y + bounds.height - viewport_offset_y) * device_pixel_ratio)++ # Ensure coordinates are within image bounds+ img_width, img_height = image.size+ x1 = max(0, min(x1, img_width))+ y1 = max(0, min(y1, img_height))+ x2 = max(x1, min(x2, img_width))+ y2 = max(y1, min(y2, img_height))++ # Skip if bounding box is too small or invalid+ if x2 - x1 < 2 or y2 - y1 < 2:+ continue++ # Get element color based on type+ tag_name = element.tag_name if hasattr(element, 'tag_name') else 'div'+ element_type = None+ if hasattr(element, 'attributes') and element.attributes:+ element_type = element.attributes.get('type')++ color = get_element_color(tag_name, element_type)++ # Get text for overlay (if short enough)+ text = None+ if hasattr(element, 'node_value') and element.node_value:+ text = element.node_value.strip()+ elif hasattr(element, 'attributes') and element.attributes:+ # Try to get meaningful text from attributes+ text = (+ element.attributes.get('aria-label')+ or element.attributes.get('title')+ or element.attributes.get('placeholder')+ or element.attributes.get('value', '')+ )++ # Draw bounding box with optional text+ draw_bounding_box_with_text(draw, (x1, y1, x2, y2), color, text, font)++ except Exception as e:+ logger.debug(f'Failed to draw highlight for element {element_id}: {e}')+ continue++ # Convert back to base64+ output_buffer = io.BytesIO()+ image.save(output_buffer, format='PNG')+ output_buffer.seek(0)++ highlighted_b64 = base64.b64encode(output_buffer.getvalue()).decode('utf-8')++ logger.debug(f'Successfully created highlighted screenshot with {len(selector_map)} elements')+ return highlighted_b64++ except Exception as e:+ logger.error(f'Failed to create highlighted screenshot: {e}')+ # Return original screenshot on error+ return screenshot_b64+++async def get_viewport_info_from_cdp(cdp_session) -> Tuple[float, int, int]:+ """Get viewport information from CDP session.++ Returns:+ Tuple of (device_pixel_ratio, scroll_x, scroll_y)+ """+ try:+ # Get layout metrics which includes viewport info and device pixel ratio+ metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)++ # Extract viewport information+ visual_viewport = metrics.get('visualViewport', {})+ css_visual_viewport = metrics.get('cssVisualViewport', {})+ css_layout_viewport = metrics.get('cssLayoutViewport', {})++ # Calculate device pixel ratio+ css_width = css_visual_viewport.get('clientWidth', css_layout_viewport.get('clientWidth', 1280.0))+ device_width = visual_viewport.get('clientWidth', css_width)+ device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0++ # Get scroll position in CSS pixels+ scroll_x = int(css_visual_viewport.get('pageX', 0))+ scroll_y = int(css_visual_viewport.get('pageY', 0))++ return float(device_pixel_ratio), scroll_x, scroll_y++ except Exception as e:+ logger.debug(f'Failed to get viewport info from CDP: {e}')+ return 1.0, 0, 0+++async def create_highlighted_screenshot_async(screenshot_b64: str, selector_map: DOMSelectorMap, cdp_session=None) -> str:+ """Async wrapper for creating highlighted screenshots.++ Args:+ screenshot_b64: Base64 encoded screenshot+ selector_map: Map of interactive elements+ cdp_session: CDP session for getting viewport info++ Returns:+ Base64 encoded highlighted screenshot+ """+ # Get viewport information if CDP session is available+ device_pixel_ratio = 1.0+ viewport_offset_x = 0+ viewport_offset_y = 0++ if cdp_session:+ try:+ device_pixel_ratio, viewport_offset_x, viewport_offset_y = await get_viewport_info_from_cdp(cdp_session)+ except Exception as e:+ logger.debug(f'Failed to get viewport info from CDP: {e}')++ # Create highlighted screenshot (run in thread pool if needed for performance)+ return create_highlighted_screenshot(screenshot_b64, selector_map, device_pixel_ratio, viewport_offset_x, viewport_offset_y)diff --git a/browser_use/browser/session.py b/browser_use/browser/session.pyindex 78747141f..2e28f17a4 100644--- a/browser_use/browser/session.py+++ b/browser_use/browser/session.py@@ -35,6 +35,7 @@ from browser_use.browser.events import (from browser_use.browser.profile import BrowserProfilefrom browser_use.browser.views import BrowserStateSummary, TabInfofrom browser_use.dom.views import EnhancedDOMTreeNode, TargetInfo+from browser_use.observability import observe_debugfrom browser_use.utils import _log_pretty_url, is_new_tab_pageDEFAULT_BROWSER_PROFILE = BrowserProfile()@@ -737,7 +738,7 @@ class BrowserSession(BaseModel):return self.agent_focus.session_id if self.agent_focus else None# ========== Helper Methods ==========-+ @observe_debug(ignore_input=True, ignore_output=True, name='get_browser_state_summary')async def get_browser_state_summary(self,cache_clickable_elements_hashes: bool = True,