Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

highlight in parallel

Magnus Müllerce0876adb7fb
diff --git a/browser_use/browser/python_highlights.py b/browser_use/browser/python_highlights.py
index 5be9f1bc9..47a7a1ddb 100644
--- a/browser_use/browser/python_highlights.py
+++ b/browser_use/browser/python_highlights.py
@@ -4,6 +4,7 @@ This module replaces JavaScript-based highlighting with fast Python image proces
to draw bounding boxes around interactive elements directly on screenshots.
"""
+import asyncio
import base64
import io
import logging
@@ -12,6 +13,7 @@ from PIL import Image, ImageDraw, ImageFont
from browser_use.dom.views import DOMSelectorMap
from browser_use.observability import observe_debug
+from browser_use.utils import time_execution_async
logger = logging.getLogger(__name__)
@@ -284,8 +286,74 @@ def draw_bounding_box_with_text(
logger.debug(f'Failed to draw text overlay: {e}')
+async def process_element_highlight(
+ element_id: int,
+ element,
+ draw,
+ device_pixel_ratio: float,
+ font,
+ filter_highlight_ids: bool,
+ image_size: tuple[int, int],
+) -> None:
+ """Process a single element for highlighting in parallel."""
+ try:
+ # Use absolute_position coordinates directly
+ if not element.absolute_position:
+ return
+
+ bounds = element.absolute_position
+
+ # Scale coordinates from CSS pixels to device pixels for screenshot
+ # The screenshot is captured at device pixel resolution, but coordinates are in CSS pixels
+ x1 = int(bounds.x * device_pixel_ratio)
+ y1 = int(bounds.y * device_pixel_ratio)
+ x2 = int((bounds.x + bounds.width) * device_pixel_ratio)
+ y2 = int((bounds.y + bounds.height) * device_pixel_ratio)
+
+ # Ensure coordinates are within image bounds
+ img_width, img_height = image_size
+ x1 = max(0, min(x1, img_width))
+ y1 = max(0, min(y1, img_height))
+ x2 = max(x1, min(x2, img_width))
+ y2 = max(y1, min(y2, img_height))
+
+ # Skip if bounding box is too small or invalid
+ if x2 - x1 < 2 or y2 - y1 < 2:
+ return
+
+ # Get element color based on type
+ tag_name = element.tag_name if hasattr(element, 'tag_name') else 'div'
+ element_type = None
+ if hasattr(element, 'attributes') and element.attributes:
+ element_type = element.attributes.get('type')
+
+ color = get_element_color(tag_name, element_type)
+
+ # Get element index for overlay and apply filtering
+ element_index = getattr(element, 'element_index', None)
+ index_text = None
+
+ if element_index is not None:
+ if filter_highlight_ids:
+ # Use the meaningful text that matches what the LLM sees
+ meaningful_text = element.get_meaningful_text_for_llm()
+ # Show ID only if meaningful text is less than 5 characters
+ if len(meaningful_text) < 5:
+ index_text = str(element_index)
+ else:
+ # Always show ID when filter is disabled
+ index_text = str(element_index)
+
+ # Draw enhanced bounding box with bigger index
+ draw_enhanced_bounding_box_with_text(draw, (x1, y1, x2, y2), color, index_text, font, tag_name, image_size)
+
+ except Exception as e:
+ logger.debug(f'Failed to draw highlight for element {element_id}: {e}')
+
+
@observe_debug(ignore_input=True, ignore_output=True, name='create_highlighted_screenshot')
-def create_highlighted_screenshot(
+@time_execution_async('create_highlighted_screenshot')
+async def create_highlighted_screenshot(
screenshot_b64: str,
selector_map: DOMSelectorMap,
device_pixel_ratio: float = 1.0,
@@ -323,62 +391,17 @@ def create_highlighted_screenshot(
except OSError:
font = None # Use default font
- # Process each interactive element
+ # Process elements in parallel for better performance
+ tasks = []
for element_id, element in selector_map.items():
- try:
- # Use absolute_position coordinates directly
- if not element.absolute_position:
- continue
-
- bounds = element.absolute_position
-
- # Scale coordinates from CSS pixels to device pixels for screenshot
- # The screenshot is captured at device pixel resolution, but coordinates are in CSS pixels
- x1 = int(bounds.x * device_pixel_ratio)
- y1 = int(bounds.y * device_pixel_ratio)
- x2 = int((bounds.x + bounds.width) * device_pixel_ratio)
- y2 = int((bounds.y + bounds.height) * device_pixel_ratio)
-
- # Ensure coordinates are within image bounds
- img_width, img_height = image.size
- x1 = max(0, min(x1, img_width))
- y1 = max(0, min(y1, img_height))
- x2 = max(x1, min(x2, img_width))
- y2 = max(y1, min(y2, img_height))
-
- # Skip if bounding box is too small or invalid
- if x2 - x1 < 2 or y2 - y1 < 2:
- continue
-
- # Get element color based on type
- tag_name = element.tag_name if hasattr(element, 'tag_name') else 'div'
- element_type = None
- if hasattr(element, 'attributes') and element.attributes:
- element_type = element.attributes.get('type')
-
- color = get_element_color(tag_name, element_type)
-
- # Get element index for overlay and apply filtering
- element_index = getattr(element, 'element_index', None)
- index_text = None
-
- if element_index is not None:
- if filter_highlight_ids:
- # Use the meaningful text that matches what the LLM sees
- meaningful_text = element.get_meaningful_text_for_llm()
- # Show ID only if meaningful text is less than 5 characters
- if len(meaningful_text) < 5:
- index_text = str(element_index)
- else:
- # Always show ID when filter is disabled
- index_text = str(element_index)
-
- # Draw enhanced bounding box with bigger index
- draw_enhanced_bounding_box_with_text(draw, (x1, y1, x2, y2), color, index_text, font, tag_name, image.size)
-
- except Exception as e:
- logger.debug(f'Failed to draw highlight for element {element_id}: {e}')
- continue
+ task = process_element_highlight(
+ element_id, element, draw, device_pixel_ratio, font, filter_highlight_ids, image.size
+ )
+ tasks.append(task)
+
+ # Execute all element processing tasks in parallel
+ if tasks:
+ await asyncio.gather(*tasks, return_exceptions=True)
# Convert back to base64
output_buffer = io.BytesIO()
@@ -428,6 +451,7 @@ async def get_viewport_info_from_cdp(cdp_session) -> tuple[float, int, int]:
@observe_debug(ignore_input=True, ignore_output=True, name='create_highlighted_screenshot_async')
+@time_execution_async('create_highlighted_screenshot_async')
async def create_highlighted_screenshot_async(
screenshot_b64: str, selector_map: DOMSelectorMap, cdp_session=None, filter_highlight_ids: bool = True
) -> str:
@@ -452,7 +476,7 @@ async def create_highlighted_screenshot_async(
except Exception as e:
logger.debug(f'Failed to get viewport info from CDP: {e}')
- # Create highlighted screenshot (run in thread pool if needed for performance)
- return create_highlighted_screenshot(
+ # Create highlighted screenshot with async processing
+ return await create_highlighted_screenshot(
screenshot_b64, selector_map, device_pixel_ratio, viewport_offset_x, viewport_offset_y, filter_highlight_ids
)
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 60e8dbd64..5e6e9d578 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -40,7 +40,7 @@ from browser_use.browser.profile import BrowserProfile, ProxySettings
from browser_use.browser.views import BrowserStateSummary, TabInfo
from browser_use.dom.views import EnhancedDOMTreeNode, TargetInfo
from browser_use.observability import observe_debug
-from browser_use.utils import _log_pretty_url, is_new_tab_page
+from browser_use.utils import _log_pretty_url, is_new_tab_page, time_execution_async
DEFAULT_BROWSER_PROFILE = BrowserProfile()
@@ -813,6 +813,7 @@ class BrowserSession(BaseModel):
assert self._cdp_client_root is not None, 'CDP client not initialized - browser may not be connected yet'
return self._cdp_client_root
+ @time_execution_async('get_or_create_cdp_session')
@observe_debug(ignore_input=True, ignore_output=True, name='get_or_create_cdp_session')
async def get_or_create_cdp_session(
self, target_id: TargetID | None = None, focus: bool = True, new_socket: bool | None = None
diff --git a/browser_use/browser/watchdogs/dom_watchdog.py b/browser_use/browser/watchdogs/dom_watchdog.py
index e3605d5c5..af6205211 100644
--- a/browser_use/browser/watchdogs/dom_watchdog.py
+++ b/browser_use/browser/watchdogs/dom_watchdog.py
@@ -4,6 +4,8 @@ import asyncio
import time
from typing import TYPE_CHECKING
+from utils import time_execution_async
+
from browser_use.browser.events import (
BrowserErrorEvent,
BrowserStateRequestEvent,
@@ -232,7 +234,7 @@ class DOMWatchdog(BaseWatchdog):
# Get CDP session for viewport info
cdp_session = await self.browser_session.get_or_create_cdp_session()
-
+ start = time.time()
screenshot_b64 = await create_highlighted_screenshot_async(
screenshot_b64,
content.selector_map,
@@ -240,7 +242,7 @@ class DOMWatchdog(BaseWatchdog):
self.browser_session.browser_profile.filter_highlight_ids,
)
self.logger.debug(
- f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Applied highlights to {len(content.selector_map)} elements'
+ f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Applied highlights to {len(content.selector_map)} elements in {time.time() - start:.2f}s'
)
except Exception as e:
self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Python highlighting failed: {e}')
@@ -425,6 +427,7 @@ class DOMWatchdog(BaseWatchdog):
)
raise
+ @time_execution_async('build_dom_tree_without_highlights')
@observe_debug(ignore_input=True, ignore_output=True, name='build_dom_tree_without_highlights')
async def _build_dom_tree_without_highlights(self, previous_state: SerializedDOMState | None = None) -> SerializedDOMState:
"""Build DOM tree without injecting JavaScript highlights (for parallel execution)."""
@@ -477,6 +480,7 @@ class DOMWatchdog(BaseWatchdog):
)
raise
+ @time_execution_async('capture_clean_screenshot')
@observe_debug(ignore_input=True, ignore_output=True, name='capture_clean_screenshot')
async def _capture_clean_screenshot(self) -> str:
"""Capture a clean screenshot without JavaScript highlights."""