Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Refactor bounding box drawing and enhance LLM text representation

Magnus Müllera31ae9dffb83
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index c75663c58..98b01e8c3 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -599,7 +599,6 @@ class Agent(Generic[Context, AgentStructuredOutput]):
if hasattr(self, 'cloud_sync') and self.cloud_sync and self.enable_cloud_sync:
self.eventbus.on('*', self.cloud_sync.handle_event)
- @observe_debug(ignore_input=True, ignore_output=True, name='_raise_if_stopped_or_paused')
async def _raise_if_stopped_or_paused(self) -> None:
"""Utility function that raises an InterruptedError if the agent is stopped or paused."""
diff --git a/browser_use/browser/python_highlights.py b/browser_use/browser/python_highlights.py
index 6a64540e2..d22dc303d 100644
--- a/browser_use/browser/python_highlights.py
+++ b/browser_use/browser/python_highlights.py
@@ -93,24 +93,24 @@ def draw_enhanced_bounding_box_with_text(
# Draw much bigger index overlay if we have index text
if text:
try:
- # Use much bigger font size for index (5x bigger base)
- huge_font = None
- font_size = 40 # Much bigger than the original 16
+ # Use much bigger font size for visible index boxes
+ big_font = None
+ font_size = 30 # Much bigger, more visible size
try:
- huge_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf', font_size)
+ big_font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf', font_size)
except (OSError, IOError):
try:
- huge_font = ImageFont.truetype('arial.ttf', font_size)
+ big_font = ImageFont.truetype('arial.ttf', font_size)
except (OSError, IOError):
# Try system fonts on different platforms
try:
- huge_font = ImageFont.truetype('Arial Bold.ttf', font_size)
+ big_font = ImageFont.truetype('Arial Bold.ttf', font_size)
except (OSError, IOError):
- huge_font = font # Fallback to original font
+ big_font = font # Fallback to original font
- # Get text size with much bigger font
- if huge_font:
- bbox_text = draw.textbbox((0, 0), text, font=huge_font)
+ # Get text size with bigger font
+ if big_font:
+ bbox_text = draw.textbbox((0, 0), text, font=big_font)
text_width = bbox_text[2] - bbox_text[0]
text_height = bbox_text[3] - bbox_text[1]
else:
@@ -119,41 +119,60 @@ def draw_enhanced_bounding_box_with_text(
text_width = bbox_text[2] - bbox_text[0]
text_height = bbox_text[3] - bbox_text[1]
- # No padding - container fits exactly around the number
- padding = 10
+ # Bigger padding for more prominent index boxes
+ padding = 8
element_width = x2 - x1
element_height = y2 - y1
- # Simple positioning logic: always top-left
- # Inside if element is big enough, outside if too small
- min_container_width = text_width + padding * 2
- min_container_height = text_height + padding * 2
+ # Container dimensions
+ container_width = text_width + padding * 2
+ container_height = text_height + padding * 2
- if element_width >= min_container_width and element_height >= min_container_height:
+ # Position in top-left corner (inside if fits, outside if too small)
+ if element_width >= container_width and element_height >= container_height:
# Place inside top-left corner
- text_x = x1 + padding
- text_y = y1 + padding
+ bg_x1 = x1 + 2 # Small offset from edge
+ bg_y1 = y1 + 2
else:
# Place outside top-left corner
- text_x = x1
- text_y = max(0, y1 - min_container_height)
+ bg_x1 = x1
+ bg_y1 = max(0, y1 - container_height)
- # Ensure text stays within image bounds using actual image dimensions
- img_width, img_height = image_size
- text_x = max(0, min(text_x, img_width - min_container_width))
- text_y = max(0, min(text_y, img_height - min_container_height))
+ bg_x2 = bg_x1 + container_width
+ bg_y2 = bg_y1 + container_height
- # Draw much bigger background rectangle (5x bigger)
- bg_x1 = text_x - padding
- bg_y1 = text_y - padding
- bg_x2 = text_x + text_width + padding
- bg_y2 = text_y + text_height + padding
-
- # Use element color as background with white text for high contrast
- draw.rectangle([bg_x1, bg_y1, bg_x2, bg_y2], fill=color, outline='white', width=3)
+ # Center the number within the index box
+ text_x = bg_x1 + (container_width - text_width) // 2
+ text_y = bg_y1 + (container_height - text_height) // 2
- # Draw white text on colored background for maximum visibility
- draw.text((text_x, text_y), text, fill='white', font=huge_font or font)
+ # Ensure container stays within image bounds
+ img_width, img_height = image_size
+ if bg_x1 < 0:
+ offset = -bg_x1
+ bg_x1 += offset
+ bg_x2 += offset
+ text_x += offset
+ if bg_y1 < 0:
+ offset = -bg_y1
+ bg_y1 += offset
+ bg_y2 += offset
+ text_y += offset
+ if bg_x2 > img_width:
+ offset = bg_x2 - img_width
+ bg_x1 -= offset
+ bg_x2 -= offset
+ text_x -= offset
+ if bg_y2 > img_height:
+ offset = bg_y2 - img_height
+ bg_y1 -= offset
+ bg_y2 -= offset
+ text_y -= offset
+
+ # Draw bigger background rectangle with thicker border
+ draw.rectangle([bg_x1, bg_y1, bg_x2, bg_y2], fill=color, outline='white', width=2)
+
+ # Draw white text centered in the index box
+ draw.text((text_x, text_y), text, fill='white', font=big_font or font)
except Exception as e:
logger.debug(f'Failed to draw enhanced text overlay: {e}')
@@ -345,9 +364,10 @@ def create_highlighted_screenshot(
if element_index is not None:
if filter_highlight_ids:
- # Only show ID if llm_representation is less than 10 characters (elements with little text need visual ID)
- element_text = element.get_all_children_text()
- if len(element_text) < 5:
+ # Use the meaningful text that matches what the LLM sees
+ meaningful_text = element.get_meaningful_text_for_llm()
+ # Show ID only if meaningful text is less than 5 characters
+ if len(meaningful_text) < 5:
index_text = str(element_index)
else:
# Always show ID when filter is disabled
diff --git a/browser_use/dom/views.py b/browser_use/dom/views.py
index 0f182888f..92d3eaea1 100644
--- a/browser_use/dom/views.py
+++ b/browser_use/dom/views.py
@@ -12,6 +12,7 @@ from cdp_use.cdp.target.types import SessionID, TargetID, TargetInfo
from uuid_extensions import uuid7str
from browser_use.dom.utils import cap_text_length
+from browser_use.observability import observe_debug
# Serializer types
DEFAULT_INCLUDE_ATTRIBUTES = [
@@ -412,6 +413,25 @@ class EnhancedDOMTreeNode:
return f'<{self.tag_name}>{cap_text_length(self.get_all_children_text(), max_text_length) or ""}'
+ def get_meaningful_text_for_llm(self) -> str:
+ """
+ Get the meaningful text content that the LLM actually sees for this element.
+ This matches exactly what goes into the DOMTreeSerializer output.
+ """
+ meaningful_text = ''
+ if hasattr(self, 'attributes') and self.attributes:
+ # Priority order: value, aria-label, title, placeholder, alt, text content
+ for attr in ['value', 'aria-label', 'title', 'placeholder', 'alt']:
+ if attr in self.attributes and self.attributes[attr]:
+ meaningful_text = self.attributes[attr]
+ break
+
+ # Fallback to text content if no meaningful attributes
+ if not meaningful_text:
+ meaningful_text = self.get_all_children_text()
+
+ return meaningful_text.strip()
+
@property
def is_actually_scrollable(self) -> bool:
"""
@@ -677,6 +697,7 @@ class SerializedDOMState:
selector_map: DOMSelectorMap
+ @observe_debug(ignore_input=True, ignore_output=True, name='llm_representation')
def llm_representation(
self,
include_attributes: list[str] | None = None,