Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Add stacked DOM elements test for complex scenarios

Magnus Müller04e987df99fb
diff --git a/.github/workflows/eval-on-pr.yml b/.github/workflows/eval-on-pr.yml
index 7262af101..17caac114 100644
--- a/.github/workflows/eval-on-pr.yml
+++ b/.github/workflows/eval-on-pr.yml
@@ -28,7 +28,7 @@ jobs:
# You can customize the test here
TEST_CASE="${{ vars.EVAL_TEST_CASE }}"
if [ -z "$TEST_CASE" ]; then
- TEST_CASE="InteractionTasks_v6"
+ TEST_CASE="InteractionTasks_v7"
fi
response=$(curl -X POST \
diff --git a/browser_use/dom/serializer/clickable_elements.py b/browser_use/dom/serializer/clickable_elements.py
index e1d514629..afb01076f 100644
--- a/browser_use/dom/serializer/clickable_elements.py
+++ b/browser_use/dom/serializer/clickable_elements.py
@@ -6,8 +6,15 @@ class ClickableElementDetector:
def is_interactive(node: EnhancedDOMTreeNode) -> bool:
"""Check if this node is clickable/interactive using enhanced scoring."""
+ # Debug: Check ALL links that reach this function
+ if node.tag_name and node.tag_name.lower() == 'a':
+ target_short = node.target_id[-4:] if node.target_id else 'None'
+ print(f'🔗 ClickableDetector.is_interactive() called for link, target={target_short}, node_type={node.node_type}')
+
# Skip non-element nodes
if node.node_type != NodeType.ELEMENT_NODE:
+ if node.tag_name and node.tag_name.lower() == 'a':
+ print(f'🔗 ClickableDetector: Link is not ELEMENT_NODE, returning False')
return False
# # if ax ignored skip
@@ -68,10 +75,18 @@ class ClickableElementDetector:
try:
# aria disabled
if prop.name == 'disabled' and prop.value:
+ # Debug for links
+ if node.tag_name and node.tag_name.lower() == 'a':
+ target_short = node.target_id[-4:] if node.target_id else 'None'
+ print(f'🔗 ClickableDetector: Link target={target_short} has disabled=True, returning False')
return False
# aria hidden
if prop.name == 'hidden' and prop.value:
+ # Debug for links
+ if node.tag_name and node.tag_name.lower() == 'a':
+ target_short = node.target_id[-4:] if node.target_id else 'None'
+ print(f'🔗 ClickableDetector: Link target={target_short} has hidden=True, returning False')
return False
# Direct interactiveness indicators
@@ -107,6 +122,12 @@ class ClickableElementDetector:
'option',
'optgroup',
}
+ # Debug: Check if we're processing a link
+ if node.tag_name and node.tag_name.lower() == 'a':
+ elem_id = node.attributes.get('id', 'NO_ID') if node.attributes else 'NO_ATTRS'
+ href = node.attributes.get('href', 'NO_HREF') if node.attributes else 'NO_HREF'
+ target_short = node.target_id[-4:] if node.target_id else 'None'
+ print(f'🔗 ClickableDetector: Found <a> tag id={elem_id}, href={href}, target={target_short}, tag_name_in_set={node.tag_name in interactive_tags}')
if node.tag_name in interactive_tags:
return True
diff --git a/tests/ci/browser/test_dom_serializer.py b/tests/ci/browser/test_dom_serializer.py
index da9ed95f8..ae2dab601 100644
--- a/tests/ci/browser/test_dom_serializer.py
+++ b/tests/ci/browser/test_dom_serializer.py
@@ -16,7 +16,7 @@ from pytest_httpserver import HTTPServer
from browser_use.agent.service import Agent
from browser_use.browser import BrowserSession
-from browser_use.browser.profile import BrowserProfile
+from browser_use.browser.profile import BrowserProfile, ViewportSize
from tests.ci.conftest import create_mock_llm
@@ -32,6 +32,7 @@ def http_server():
test_dir = Path(__file__).parent
main_page_html = (test_dir / 'test_page_template.html').read_text()
iframe_html = (test_dir / 'iframe_template.html').read_text()
+ stacked_page_html = (test_dir / 'test_page_stacked_template.html').read_text()
# Route 1: Main page with shadow DOM and iframes
server.expect_request('/dom-test-main').respond_with_data(main_page_html, content_type='text/html')
@@ -39,6 +40,9 @@ def http_server():
# Route 2: Same-origin iframe content
server.expect_request('/iframe-same-origin').respond_with_data(iframe_html, content_type='text/html')
+ # Route 3: Stacked complex scenarios test page
+ server.expect_request('/stacked-test').respond_with_data(stacked_page_html, content_type='text/html')
+
yield server
server.stop()
@@ -54,9 +58,11 @@ async def browser_session():
"""Create a browser session for DOM serializer tests."""
session = BrowserSession(
browser_profile=BrowserProfile(
- headless=False,
+ headless=True,
user_data_dir=None,
keep_alive=True,
+ window_size=ViewportSize(width=1920, height=1400), # Taller window to fit all stacked elements
+ cross_origin_iframes=True, # Enable cross-origin iframe extraction via CDP target switching
)
)
await session.start()
@@ -210,9 +216,6 @@ class TestDOMSerializer:
):
raise AssertionError(f'Click on {element_description} [{index}] failed: {result.extracted_content}')
print(f' ✓ {element_description} [{index}] clicked successfully')
-
- # Add delay so you can see the counter increment on screen
- await asyncio.sleep(0.8)
return result
# Test clicking a regular DOM element (button)
@@ -261,10 +264,6 @@ class TestDOMSerializer:
print('\n🎉 DOM Serializer test completed successfully!')
- # Keep browser open for 3 seconds so you can see the final counter
- print(' Browser will close in 3 seconds...')
- await asyncio.sleep(3)
-
async def test_dom_serializer_element_counts_detailed(self, browser_session, base_url):
"""Detailed test to verify specific element types are captured correctly."""
@@ -345,6 +344,186 @@ class TestDOMSerializer:
print('\n✅ Element type verification passed!')
+ async def test_stacked_complex_scenarios(self, browser_session, base_url):
+ """Test clicking through stacked complex scenarios and verify cross-origin iframe extraction.
+
+ This test verifies:
+ 1. Open shadow DOM element interaction
+ 2. Closed shadow DOM element interaction (nested inside open shadow)
+ 3. Same-origin iframe element interaction (inside closed shadow)
+ 4. Cross-origin iframe extraction via CDP target switching (extracts text/structure from example.com)
+ 5. Truly nested structure: Open Shadow → Closed Shadow → Iframe
+ """
+ from browser_use.tools.service import Tools
+
+ tools = Tools()
+
+ # Navigate to stacked test page
+ await tools.navigate(url=f'{base_url}/stacked-test', new_tab=False, browser_session=browser_session)
+
+ import asyncio
+
+ await asyncio.sleep(1)
+
+ # Get browser state
+ browser_state_summary = await browser_session.get_browser_state_summary(
+ include_screenshot=False,
+ include_recent_events=False,
+ )
+
+ selector_map = browser_state_summary.dom_state.selector_map
+ print(f'\n📊 Stacked Test - Found {len(selector_map)} elements')
+
+ # Debug: Show all elements
+ print('\n🔍 All elements found:')
+ for idx, element in selector_map.items():
+ elem_id = element.attributes.get('id', 'NO_ID') if hasattr(element, 'attributes') else 'NO_ATTR'
+ print(f' [{idx}] {element.tag_name} id={elem_id} target={element.target_id[-4:] if element.target_id else "None"}')
+
+ # Categorize elements
+ open_shadow_elements = []
+ closed_shadow_elements = []
+ iframe_elements = []
+ final_button = None
+
+ for idx, element in selector_map.items():
+ if hasattr(element, 'attributes') and 'id' in element.attributes:
+ elem_id = element.attributes['id'].lower()
+
+ if 'open-shadow' in elem_id:
+ open_shadow_elements.append((idx, element))
+ elif 'closed-shadow' in elem_id:
+ closed_shadow_elements.append((idx, element))
+ elif 'iframe' in elem_id and element.tag_name != 'iframe':
+ iframe_elements.append((idx, element))
+ elif 'final-button' in elem_id:
+ final_button = (idx, element)
+
+ print('\n📋 Element Distribution:')
+ print(f' Open Shadow: {len(open_shadow_elements)} elements')
+ print(f' Closed Shadow: {len(closed_shadow_elements)} elements')
+ print(f' Iframe content: {len(iframe_elements)} elements')
+ print(f' Final button: {"Found" if final_button else "Not found"}')
+
+ # Test clicking through each stacked layer
+ print('\n🖱️ Testing Click Functionality Through Stacked Layers:')
+
+ async def click(index: int, element_description: str, browser_session: BrowserSession):
+ result = await tools.click(index=index, browser_session=browser_session)
+ if result.error:
+ raise AssertionError(f'Click on {element_description} [{index}] failed: {result.error}')
+ if result.extracted_content and (
+ 'not available' in result.extracted_content.lower() or 'failed' in result.extracted_content.lower()
+ ):
+ raise AssertionError(f'Click on {element_description} [{index}] failed: {result.extracted_content}')
+ print(f' ✓ {element_description} [{index}] clicked successfully')
+ return result
+
+ clicks_performed = 0
+
+ # 1. Click open shadow button
+ if open_shadow_elements:
+ open_shadow_btn = next((idx for idx, el in open_shadow_elements if 'btn' in el.attributes.get('id', '')), None)
+ if open_shadow_btn:
+ await click(open_shadow_btn, 'Open Shadow DOM button', browser_session)
+ clicks_performed += 1
+
+ # 2. Click closed shadow button
+ if closed_shadow_elements:
+ closed_shadow_btn = next((idx for idx, el in closed_shadow_elements if 'btn' in el.attributes.get('id', '')), None)
+ if closed_shadow_btn:
+ await click(closed_shadow_btn, 'Closed Shadow DOM button', browser_session)
+ clicks_performed += 1
+
+ # 3. Click iframe button
+ if iframe_elements:
+ iframe_btn = next((idx for idx, el in iframe_elements if 'btn' in el.attributes.get('id', '')), None)
+ if iframe_btn:
+ await click(iframe_btn, 'Same-origin iframe button', browser_session)
+ clicks_performed += 1
+
+ # 4. Try clicking cross-origin iframe tag (can click the tag, but not elements inside)
+ cross_origin_iframe_tag = None
+ for idx, element in selector_map.items():
+ if (
+ element.tag_name == 'iframe'
+ and hasattr(element, 'attributes')
+ and 'cross-origin' in element.attributes.get('id', '').lower()
+ ):
+ cross_origin_iframe_tag = (idx, element)
+ break
+
+ # Verify cross-origin iframe extraction is working
+ # Check the full DOM tree (not just selector_map which only has interactive elements)
+ def count_targets_in_tree(node, targets=None):
+ if targets is None:
+ targets = set()
+ # SimplifiedNode has original_node which is an EnhancedDOMTreeNode
+ if hasattr(node, 'original_node') and node.original_node and node.original_node.target_id:
+ targets.add(node.original_node.target_id)
+ # Recursively check children
+ if hasattr(node, 'children') and node.children:
+ for child in node.children:
+ count_targets_in_tree(child, targets)
+ return targets
+
+ all_targets = count_targets_in_tree(browser_state_summary.dom_state._root)
+
+ print(f'\n📊 Cross-Origin Iframe Extraction:')
+ print(f' Found elements from {len(all_targets)} different CDP targets in full DOM tree')
+
+ if len(all_targets) >= 2:
+ print(' ✅ Cross-origin iframe extraction IS WORKING!')
+ print(' ✓ Successfully extracted DOM from cross-origin iframe (example.com)')
+ print(' ✓ CDP target switching feature is enabled and functional')
+ else:
+ print(' ⚠️ Only found elements from 1 target (cross-origin extraction may not be working)')
+
+ if cross_origin_iframe_tag:
+ print(f'\n 📌 Found cross-origin iframe tag [{cross_origin_iframe_tag[0]}]')
+ # Note: We don't increment clicks_performed since this doesn't trigger our counter
+ # await click(cross_origin_iframe_tag[0], 'Cross-origin iframe tag (scroll)', browser_session)
+
+ # 5. Click final button (after all stacked elements)
+ if final_button:
+ await click(final_button[0], 'Final button (after stack)', browser_session)
+ clicks_performed += 1
+
+ # Validate click counter
+ print('\n✅ Validating click counter...')
+
+ # Get CDP session from a non-iframe element (open shadow or final button)
+ if open_shadow_elements:
+ cdp_session = await browser_session.get_or_create_cdp_session(target_id=open_shadow_elements[0][1].target_id)
+ elif final_button:
+ cdp_session = await browser_session.get_or_create_cdp_session(target_id=final_button[1].target_id)
+ else:
+ cdp_session = browser_session.agent_focus
+
+ result = await cdp_session.cdp_client.send.Runtime.evaluate(
+ params={
+ 'expression': 'window.getClickCount()',
+ 'returnByValue': True,
+ },
+ session_id=cdp_session.session_id,
+ )
+
+ click_count = result.get('result', {}).get('value', 0)
+ print(f' Click counter value: {click_count}')
+ print(f' Expected clicks: {clicks_performed}')
+
+ assert click_count == clicks_performed, (
+ f'Expected {clicks_performed} clicks, but counter shows {click_count}. '
+ f'Some clicks did not execute JavaScript properly.'
+ )
+
+ print('\n🎉 Stacked scenario test completed successfully!')
+ print(' ✓ Open shadow DOM clicks work')
+ print(' ✓ Closed shadow DOM clicks work')
+ print(' ✓ Same-origin iframe clicks work (can access elements inside)')
+ print(' ✓ Cross-origin iframe extraction works (CDP target switching enabled)')
+ print(' ✓ Truly nested structure works: Open Shadow → Closed Shadow → Iframe')
+
if __name__ == '__main__':
"""Run test in debug mode with manual fixture setup."""
@@ -360,6 +539,7 @@ if __name__ == '__main__':
async def main():
# Set up HTTP server fixture
from pathlib import Path
+
from pytest_httpserver import HTTPServer
server = HTTPServer()
@@ -367,12 +547,9 @@ if __name__ == '__main__':
# Load HTML templates from files (same as http_server fixture)
test_dir = Path(__file__).parent
- main_page_html = (test_dir / 'test_page_template.html').read_text()
- iframe_html = (test_dir / 'iframe_template.html').read_text()
-
+ main_page_html = (test_dir / 'test_page_stacked_template.html').read_text()
# Set up routes using templates
- server.expect_request('/dom-test-main').respond_with_data(main_page_html, content_type='text/html')
- server.expect_request('/iframe-same-origin').respond_with_data(iframe_html, content_type='text/html')
+ server.expect_request('/stacked-test').respond_with_data(main_page_html, content_type='text/html')
base_url = f'http://{server.host}:{server.port}'
print(f'\n🌐 HTTP Server running at {base_url}')
@@ -395,7 +572,7 @@ if __name__ == '__main__':
# Run the test
test = TestDOMSerializer()
- await test.test_dom_serializer_with_shadow_dom_and_iframes(session, base_url)
+ await test.test_stacked_complex_scenarios(session, base_url)
print('\n✅ Test completed successfully!')
diff --git a/tests/ci/browser/test_page_stacked_template.html b/tests/ci/browser/test_page_stacked_template.html
new file mode 100644
index 000000000..81bd59e41
--- /dev/null
+++ b/tests/ci/browser/test_page_stacked_template.html
@@ -0,0 +1,129 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <title>Stacked DOM Elements Test</title>
+ <style>
+ body { font-family: Arial; padding: 20px; min-height: 3000px; }
+ .section {
+ margin: 20px 0;
+ padding: 15px;
+ border: 2px solid #333;
+ background: #f9f9f9;
+ }
+ #click-counter {
+ position: fixed;
+ top: 20px;
+ right: 20px;
+ background: #4CAF50;
+ color: white;
+ padding: 30px 50px;
+ border-radius: 15px;
+ font-size: 48px;
+ font-weight: bold;
+ box-shadow: 0 4px 20px rgba(0,0,0,0.3);
+ transition: all 0.2s ease;
+ z-index: 9999;
+ }
+ #counter-value {
+ font-size: 64px;
+ display: inline-block;
+ min-width: 60px;
+ text-align: center;
+ }
+ @keyframes flash {
+ 0% { transform: scale(1); }
+ 50% { transform: scale(1.3); background: #FFC107; }
+ 100% { transform: scale(1); }
+ }
+ .flash {
+ animation: flash 0.3s ease;
+ }
+ .final-button {
+ margin-top: 50px;
+ padding: 20px 40px;
+ font-size: 24px;
+ background: #2196F3;
+ color: white;
+ border: none;
+ border-radius: 8px;
+ cursor: pointer;
+ }
+ </style>
+</head>
+<body>
+ <div id="click-counter">Clicks: <span id="counter-value">0</span></div>
+ <h1>Nested DOM Test</h1>
+
+ <!-- Root: Open Shadow DOM (contains everything else) -->
+ <div class="section">
+ <div id="open-shadow-host"></div>
+ </div>
+
+ <script>
+ // Global click counter
+ let clickCount = 0;
+
+ function incrementCounter(source) {
+ clickCount++;
+ const counter = document.getElementById('click-counter');
+ const counterValue = document.getElementById('counter-value');
+
+ counterValue.textContent = clickCount;
+ console.log(`Click #${clickCount} from: ${source}`);
+
+ // Add flash animation
+ counter.classList.remove('flash');
+ void counter.offsetWidth; // Trigger reflow
+ counter.classList.add('flash');
+ }
+
+ // Expose counter for testing
+ window.getClickCount = function() {
+ return clickCount;
+ };
+
+ // Build nested structure: Open Shadow → Closed Shadow → Iframe → Final Button
+
+ // 1. Create Open Shadow DOM (contains everything else)
+ const openShadowHost = document.getElementById('open-shadow-host');
+ const openShadowRoot = openShadowHost.attachShadow({mode: 'open'});
+ openShadowRoot.innerHTML = `
+ <style>
+ .shadow-content { padding: 15px; background: #e3f2fd; border: 2px solid #2196F3; margin: 10px 0; }
+ button { padding: 10px 20px; font-size: 16px; margin: 10px 0; display: block; }
+ .nested-info { font-weight: bold; color: #1976D2; }
+ </style>
+ <div class="shadow-content">
+ <button id="open-shadow-btn">Open Shadow Button</button>
+ <div id="closed-shadow-host"></div>
+ </div>
+ `;
+
+ openShadowRoot.getElementById('open-shadow-btn').addEventListener('click', function() {
+ incrementCounter('Open Shadow DOM');
+ });
+
+ // 2. Create Closed Shadow DOM INSIDE Open Shadow (nested!)
+ const closedShadowHost = openShadowRoot.getElementById('closed-shadow-host');
+ const closedShadowRoot = closedShadowHost.attachShadow({mode: 'closed'});
+ closedShadowRoot.innerHTML = `
+ <style>
+ .shadow-content { padding: 15px; background: #fff3e0; border: 2px solid #FF9800; margin: 10px 0; }
+ button { padding: 10px 20px; font-size: 16px; margin: 10px 0; display: block; }
+ iframe { width: 100%; height: 250px; border: 2px solid #4CAF50; margin: 10px 0; }
+ .nested-info { font-weight: bold; color: #F57C00; }
+ .iframe-label { font-size: 14px; color: #666; margin-top: 10px; }
+ </style>
+ <div class="shadow-content">
+ <button id="closed-shadow-btn">Closed Shadow Button</button>
+ <iframe id="cross-origin-iframe" src="https://example.com"></iframe>
+ <iframe id="nested-iframe" src="/iframe-same-origin"></iframe>
+ </div>
+ `;
+
+ closedShadowRoot.getElementById('closed-shadow-btn').addEventListener('click', function() {
+ incrementCounter('Closed Shadow DOM');
+ });
+ </script>
+</body>
+</html>