Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

when detecting iframe elements, tell LLM exactly how far to scroll to reach each Get pixel location of each iframe element that's out of the viewport, convert from pixels to number of page lengths, and provide that to the LLM as context

Laith Weinberger9027991bae65
diff --git a/browser_use/dom/serializer/serializer.py b/browser_use/dom/serializer/serializer.py
index 565f2311d..716a0094d 100644
--- a/browser_use/dom/serializer/serializer.py
+++ b/browser_use/dom/serializer/serializer.py
@@ -1043,18 +1043,18 @@ class DOMTreeSerializer:
if child_text:
formatted_text.append(child_text)
- # Add hidden content hint for iframes
+ # Add hidden content hint for iframes with element details
if (
node.original_node.node_type == NodeType.ELEMENT_NODE
and node.original_node.tag_name
and node.original_node.tag_name.upper() in ('IFRAME', 'FRAME')
+ and node.original_node.hidden_elements_info
):
- if node.original_node.hidden_interactive_count > 0:
- hint = f'{depth_str}... ({node.original_node.hidden_interactive_count} more interactive elements below - scroll to reveal)'
- formatted_text.append(hint)
- elif node.original_node.has_hidden_content:
- hint = f'{depth_str}... (more content below viewport - scroll to reveal)'
- formatted_text.append(hint)
+ hidden = node.original_node.hidden_elements_info
+ hint_lines = [f'{depth_str}... ({len(hidden)} more elements below - scroll to reveal):']
+ for elem in hidden:
+ hint_lines.append(f'{depth_str} <{elem["tag"]}> "{elem["text"]}" ~{elem["pages"]} pages down')
+ formatted_text.extend(hint_lines)
return '\n'.join(formatted_text)
diff --git a/browser_use/dom/service.py b/browser_use/dom/service.py
index 50c455177..06ef3e82c 100644
--- a/browser_use/dom/service.py
+++ b/browser_use/dom/service.py
@@ -1,7 +1,7 @@
import asyncio
import logging
import time
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
from cdp_use.cdp.accessibility.commands import GetFullAXTreeReturns
from cdp_use.cdp.accessibility.types import AXNode
@@ -67,11 +67,10 @@ class DomService:
pass # no need to cleanup anything, browser_session auto handles cleaning up session cache
def _count_hidden_elements_in_iframes(self, node: EnhancedDOMTreeNode) -> None:
- """Count hidden elements in iframes for LLM hints.
+ """Collect hidden interactive elements in iframes for LLM hints.
- For each iframe, tracks:
- - hidden_interactive_count: interactive elements hidden by viewport threshold
- - has_hidden_content: whether any content is hidden by viewport threshold
+ For each iframe, collects details of hidden interactive elements including
+ tag, text/name, and scroll distance in pages so the agent knows how far to scroll.
"""
interactive_tags = {
'A',
@@ -106,49 +105,64 @@ class DomService:
return not css_hidden
- def analyze_subtree(subtree_root: EnhancedDOMTreeNode) -> tuple[int, bool]:
- """Analyze subtree for hidden content. Returns (interactive_count, has_any_content)."""
- interactive_count = 0
- has_content = False
+ def collect_hidden_elements(
+ subtree_root: EnhancedDOMTreeNode, viewport_height: float
+ ) -> list[dict[str, Any]]:
+ """Collect hidden interactive elements from subtree."""
+ hidden: list[dict[str, Any]] = []
if subtree_root.node_type == NodeType.ELEMENT_NODE:
tag = (subtree_root.tag_name or '').upper()
role = subtree_root.ax_node.role if subtree_root.ax_node else None
is_interactive = tag in interactive_tags or subtree_root.has_js_click_listener or role in interactive_roles
- if is_hidden_by_threshold(subtree_root):
- has_content = True
- if is_interactive:
- interactive_count += 1
-
- elif subtree_root.node_type == NodeType.TEXT_NODE:
- text = (subtree_root.node_value or '').strip()
- if text and len(text) > 1 and is_hidden_by_threshold(subtree_root):
- has_content = True
+ if is_interactive and is_hidden_by_threshold(subtree_root):
+ # Get element text/name
+ text = ''
+ if subtree_root.ax_node and subtree_root.ax_node.name:
+ text = subtree_root.ax_node.name[:40]
+ elif subtree_root.attributes:
+ text = (
+ subtree_root.attributes.get('placeholder', '')
+ or subtree_root.attributes.get('title', '')
+ or subtree_root.attributes.get('aria-label', '')
+ )[:40]
+
+ # Get y position and convert to pages
+ y_pos = subtree_root.snapshot_node.bounds.y if subtree_root.snapshot_node.bounds else 0
+ pages_down = round(y_pos / viewport_height, 1) if viewport_height > 0 else 0
+
+ hidden.append({
+ 'tag': subtree_root.tag_name or '?',
+ 'text': text or '(no label)',
+ 'pages': pages_down,
+ })
for child in subtree_root.children_nodes or []:
- child_interactive, child_content = analyze_subtree(child)
- interactive_count += child_interactive
- has_content = has_content or child_content
+ hidden.extend(collect_hidden_elements(child, viewport_height))
for shadow_root in subtree_root.shadow_roots or []:
- shadow_interactive, shadow_content = analyze_subtree(shadow_root)
- interactive_count += shadow_interactive
- has_content = has_content or shadow_content
+ hidden.extend(collect_hidden_elements(shadow_root, viewport_height))
- return interactive_count, has_content
+ return hidden
def process_node(current_node: EnhancedDOMTreeNode) -> None:
- """Process node and descendants, analyzing hidden content for iframes."""
+ """Process node and descendants, collecting hidden elements for iframes."""
if (
current_node.node_type == NodeType.ELEMENT_NODE
and current_node.tag_name
and current_node.tag_name.upper() in ('IFRAME', 'FRAME')
and current_node.content_document
):
- interactive_count, has_content = analyze_subtree(current_node.content_document)
- current_node.hidden_interactive_count = interactive_count
- current_node.has_hidden_content = has_content
+ # Get viewport height from iframe's client rect
+ viewport_height = 0.0
+ if current_node.snapshot_node and current_node.snapshot_node.clientRects:
+ viewport_height = current_node.snapshot_node.clientRects.height
+
+ hidden = collect_hidden_elements(current_node.content_document, viewport_height)
+ # Sort by pages and limit to avoid bloating context
+ hidden.sort(key=lambda x: x['pages'])
+ current_node.hidden_elements_info = hidden[:10] # Limit to 10
for child in current_node.children_nodes or []:
process_node(child)
diff --git a/browser_use/dom/views.py b/browser_use/dom/views.py
index c89455e68..3db61bb2b 100644
--- a/browser_use/dom/views.py
+++ b/browser_use/dom/views.py
@@ -445,16 +445,11 @@ class EnhancedDOMTreeNode:
Used to identify clicks that don't use native interactive HTML tags
"""
- hidden_interactive_count: int = 0
+ hidden_elements_info: list[dict[str, Any]] = field(default_factory=list)
"""
- Number of interactive elements hidden due to viewport threshold (for iframes)
- Used to show a specific count hint in the LLM representation
- """
-
- has_hidden_content: bool = False
- """
- Whether the iframe has any content hidden due to viewport threshold
- Used to show a generic hint when there's content but no interactive elements
+ Details of interactive elements hidden due to viewport threshold (for iframes)
+ Each dict contains: tag, text, y_position
+ Used to show specific element info in the LLM representation
"""
uuid: str = field(default_factory=uuid7str)