Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Remove old

Magnus Müller90e4569125fc
diff --git a/tests/ci/test_browser_session_element_cache.py b/tests/ci/test_browser_session_element_cache.py
deleted file mode 100644
index fadadfcf4..000000000
--- a/tests/ci/test_browser_session_element_cache.py
+++ /dev/null
@@ -1,251 +0,0 @@
-"""
-Systematic debugging of the selector map issue.
-Test each assumption step by step to isolate the problem.
-"""
-
-import pytest
-
-from browser_use.browser import BrowserSession
-from browser_use.browser.profile import BrowserProfile
-from browser_use.tools.service import Tools
-
-
-def test_server(httpserver):
- """Create and provide a test HTTP server with properly configured routes."""
- # Clear any existing expectations to avoid conflicts
- httpserver.clear()
-
- # Add routes for test pages
- httpserver.expect_request('/').respond_with_data(
- """<html>
- <head><title>Test Home Page</title></head>
- <body>
- <h1>Test Home Page</h1>
- <a href="/page1" id="link1">Link 1</a>
- <button id="button1">Button 1</button>
- <input type="text" id="input1" />
- <div id="div1" class="clickable">Clickable Div</div>
- </body>
- </html>""",
- content_type='text/html',
- )
-
- httpserver.expect_request('/page1').respond_with_data(
- """<html>
- <head><title>Test Page 1</title></head>
- <body>
- <h1>Test Page 1</h1>
- <p>This is test page 1</p>
- <a href="/">Back to home</a>
- </body>
- </html>""",
- content_type='text/html',
- )
-
- httpserver.expect_request('/simple').respond_with_data(
- """<html>
- <head><title>Simple Page</title></head>
- <body>
- <h1>Simple Page</h1>
- <p>This is a simple test page</p>
- <a href="/">Home</a>
- </body>
- </html>""",
- content_type='text/html',
- )
-
- return httpserver
-
-
-async def browser_session():
- """Create a real browser session for testing."""
- session = BrowserSession(
- browser_profile=BrowserProfile(
- user_data_dir=None, # Use temporary profile
- headless=True,
- )
- )
- await session.start()
- yield session
- await session.stop()
-
-
-def tools():
- """Create a tools instance."""
- return Tools()
-
-
-async def test_assumption_1_dom_processing_works(browser_session, test_server):
- """Test assumption 1: DOM processing works and finds elements."""
- # Go to a simple page using CDP events
- from browser_use.browser.events import NavigateToUrlEvent
-
- event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=test_server.url_for('/')))
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Trigger DOM processing
- state = await browser_session.get_browser_state_summary()
-
- print('DOM processing result:')
- print(f' - Elements found: {len(state.dom_state.selector_map)}')
- print(f' - Element indices: {list(state.dom_state.selector_map.keys())}')
-
- # Verify DOM processing works
- assert len(state.dom_state.selector_map) > 0, 'DOM processing should find interactive elements'
-
-
-async def test_assumption_2_cached_selector_map_persists(browser_session, test_server):
- """Test assumption 2: Cached selector map persists after get_state_summary."""
- # Go to a simple page using CDP events
- from browser_use.browser.events import NavigateToUrlEvent
-
- event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=test_server.url_for('/')))
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Trigger DOM processing and cache
- state = await browser_session.get_browser_state_summary()
- initial_selector_map = dict(state.dom_state.selector_map)
-
- # Check if cached selector map is still available
- cached_selector_map = await browser_session.get_selector_map()
-
- print('Selector map persistence:')
- print(f' - Initial elements: {len(initial_selector_map)}')
- print(f' - Cached elements: {len(cached_selector_map)}')
- print(f' - Maps are identical: {initial_selector_map.keys() == cached_selector_map.keys()}')
-
- # Verify the cached map persists
- assert len(cached_selector_map) > 0, 'Cached selector map should persist'
- assert initial_selector_map.keys() == cached_selector_map.keys(), 'Cached map should match initial map'
-
-
-async def test_assumption_3_action_gets_same_selector_map(browser_session, tools, test_server):
- """Test assumption 3: Action gets the same selector map as cached."""
- # Go to a simple page using CDP events
- from browser_use.browser.events import NavigateToUrlEvent
-
- event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=test_server.url_for('/')))
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Trigger DOM processing and cache
- await browser_session.get_browser_state_summary()
- cached_selector_map = await browser_session.get_selector_map()
-
- print('Pre-action state:')
- print(f' - Cached elements: {len(cached_selector_map)}')
- print(f' - Element 0 exists in cache: {0 in cached_selector_map}')
-
- # Create a test action that checks the selector map it receives
- @tools.registry.action('Test: Check selector map')
- async def test_check_selector_map(browser_session: BrowserSession):
- from browser_use import ActionResult
-
- action_selector_map = await browser_session.get_selector_map()
- return ActionResult(
- extracted_content=f'Action sees {len(action_selector_map)} elements, index 0 exists: {0 in action_selector_map}',
- include_in_memory=False,
- )
-
- # Execute the test action
- result = await tools.registry.execute_action('test_check_selector_map', {}, browser_session=browser_session)
-
- print(f'Action result: {result.extracted_content}')
-
- # Verify the action sees the same selector map
- assert 'index 0 exists: False' in result.extracted_content, 'Element 0 should not exist (elements start at 1)'
-
-
-async def test_assumption_4_click_action_specific_issue(browser_session, tools, test_server):
- """Test assumption 4: Specific issue with click action."""
- # Go to a simple page using CDP events
- from browser_use.browser.events import NavigateToUrlEvent
-
- event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=test_server.url_for('/')))
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Trigger DOM processing and cache
- await browser_session.get_browser_state_summary()
- cached_selector_map = await browser_session.get_selector_map()
-
- print('Pre-click state:')
- print(f' - Cached elements: {len(cached_selector_map)}')
- print(f' - Element 0 exists: {0 in cached_selector_map}')
-
- # Create a test action that replicates click logic
- @tools.registry.action('Test: Debug click logic')
- async def test_debug_click_logic(browser_session: BrowserSession):
- from browser_use import ActionResult
-
- # Get the selector map and inspect its structure
- selector_map = await browser_session.get_selector_map()
-
- print(f' - Action selector map size: {len(selector_map)}')
- print(f' - Action selector map keys (first 10): {list(selector_map.keys())[:10]}')
-
- if len(selector_map) == 0:
- return ActionResult(
- error='Debug: Selector map is empty',
- include_in_memory=False,
- )
-
- # Get the first element from the map (by iterating, not indexing by 0)
- first_key = next(iter(selector_map.keys()))
- first_element = selector_map[first_key]
-
- print(f' - First element key: {first_key}')
- print(f' - First element backend_node_id: {first_element.backend_node_id}')
- print(f' - First element in map: {first_key in selector_map}')
-
- return ActionResult(
- extracted_content=f'Debug: Found element with key {first_key} in map of size {len(selector_map)}',
- include_in_memory=False,
- )
-
- # Test with index 1 (elements start at 1, not 0)
- result = await tools.registry.execute_action('test_debug_click_logic', params={}, browser_session=browser_session)
-
- print(f'Debug click result: {result.extracted_content or result.error}')
-
- # This will help us see exactly what the click action sees
- if result.error:
- pytest.fail(f'Click logic debug failed: {result.error}')
-
-
-async def test_assumption_5_multiple_get_selector_map_calls(browser_session, test_server):
- """Test assumption 5: Multiple calls to get_selector_map return consistent results."""
- # Go to a simple page using CDP events
- from browser_use.browser.events import NavigateToUrlEvent
-
- event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=test_server.url_for('/')))
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Trigger DOM processing and cache
- await browser_session.get_browser_state_summary()
-
- # Call get_selector_map multiple times
- map1 = await browser_session.get_selector_map()
- map2 = await browser_session.get_selector_map()
- map3 = await browser_session.get_selector_map()
-
- print('Multiple selector map calls:')
- print(f' - Call 1: {len(map1)} elements')
- print(f' - Call 2: {len(map2)} elements')
- print(f' - Call 3: {len(map3)} elements')
- print(f' - All calls identical: {map1.keys() == map2.keys() == map3.keys()}')
-
- # Verify consistency
- assert len(map1) == len(map2) == len(map3), 'Multiple calls should return same size'
- assert map1.keys() == map2.keys() == map3.keys(), 'Multiple calls should return same elements'
diff --git a/tests/ci/test_browser_watchdog_crash.py b/tests/ci/test_browser_watchdog_crash.py
deleted file mode 100644
index 42180ddca..000000000
--- a/tests/ci/test_browser_watchdog_crash.py
+++ /dev/null
@@ -1,328 +0,0 @@
-"""Test CrashWatchdog functionality."""
-
-import asyncio
-from typing import cast
-
-import pytest
-
-from browser_use.browser.events import (
- BrowserConnectedEvent,
- BrowserErrorEvent,
- BrowserStartEvent,
- BrowserStopEvent,
- BrowserStoppedEvent,
- NavigateToUrlEvent,
-)
-from browser_use.browser.profile import BrowserProfile
-from browser_use.browser.session import BrowserSession
-from browser_use.utils import logger
-
-
[email protected]('CrashWatchdog not implemented in current CDP architecture')
-async def test_crash_watchdog_network_timeout():
- """Test that CrashWatchdog detects network timeouts by monitoring actual network requests."""
-
- # Create browser session
- profile = BrowserProfile(headless=True)
- session = BrowserSession(browser_profile=profile)
-
- try:
- # Start browser using event system
- session.event_bus.dispatch(BrowserStartEvent())
- await session.event_bus.expect(BrowserConnectedEvent, timeout=10.0)
-
- logger.info('[TEST] Browser started, configuring watchdog timeout')
-
- # Configure crash watchdog with very short timeout for testing
- if hasattr(session, '_crash_watchdog') and session._crash_watchdog:
- session._crash_watchdog.network_timeout_seconds = 1.0 # Very short timeout
- session._crash_watchdog.check_interval_seconds = 0.2 # Check frequently
-
- # Try to navigate to a non-existent slow server that will hang
- # This will create a real network request that will timeout
- slow_url = 'http://192.0.2.1:8080/timeout-test' # RFC5737 TEST-NET-1 - non-routable
- logger.info(f'[TEST] Navigating to non-routable URL via events: {slow_url}')
- session.event_bus.dispatch(NavigateToUrlEvent(url=slow_url))
-
- # Wait for the network timeout error via event bus
- logger.info('[TEST] Waiting for NetworkTimeout event via event bus...')
- try:
- timeout_error = cast(
- BrowserErrorEvent,
- await session.event_bus.expect(
- BrowserErrorEvent, predicate=lambda e: cast(BrowserErrorEvent, e).error_type == 'NetworkTimeout', timeout=8.0
- ),
- )
-
- # Verify the timeout event details
- assert 'timeout-test' in timeout_error.details['url']
- assert timeout_error.details['elapsed_seconds'] >= 0.8 # Should be at least close to our timeout
- assert timeout_error.message.startswith('Network request timed out after')
-
- logger.info(f'[TEST] Successfully detected network timeout: {timeout_error.message}')
- except TimeoutError:
- # Network timeout detection can be flaky in test environment
- logger.warning('[TEST] NetworkTimeout event not received - this is expected in some test environments')
-
- # Verify the crash watchdog is running and configured correctly
- assert session._crash_watchdog is not None, 'CrashWatchdog should exist'
- assert session._crash_watchdog.network_timeout_seconds == 1.0, 'Network timeout should be configured'
- assert session._crash_watchdog._monitoring_task is not None, 'Monitoring task should be running'
- assert not session._crash_watchdog._monitoring_task.done(), 'Monitoring task should still be active'
-
- logger.info('[TEST] Crash watchdog is properly configured and running - test passes')
-
- finally:
- # Clean shutdown
- try:
- session.event_bus.dispatch(BrowserStopEvent())
- await session.event_bus.expect(BrowserStoppedEvent, timeout=3.0)
- except Exception:
- # If graceful shutdown fails, force cleanup
- await session.kill()
-
-
[email protected]('CrashWatchdog not implemented in current CDP architecture')
-async def test_crash_watchdog_browser_disconnect():
- """Test that CrashWatchdog detects browser disconnection through monitoring."""
- profile = BrowserProfile(headless=True)
- session = BrowserSession(browser_profile=profile)
-
- try:
- # Start browser
- start_event = session.event_bus.dispatch(BrowserStartEvent())
- await start_event
- # Ensure any exceptions from the event handler are propagated
- await start_event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Wait for browser to be fully started
- await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
-
- # Browser disconnection detection is now handled by the crash watchdog
- # No configuration needed
-
- # Mock browser disconnection by overriding is_connected
- # This simulates what would happen if the browser process crashed
- if session._cdp_client_root:
- # Simulate disconnection by setting _cdp_client_root to None
- original_cdp_client = session._cdp_client_root
- session._cdp_client_root = None
-
- try:
- # Wait for watchdog to detect disconnection
- disconnect_error: BrowserErrorEvent = cast(
- BrowserErrorEvent,
- await session.event_bus.expect(
- BrowserErrorEvent,
- predicate=lambda e: cast(BrowserErrorEvent, e).error_type == 'BrowserDisconnected',
- timeout=2.0,
- ),
- )
- assert 'disconnected unexpectedly' in disconnect_error.message
- finally:
- # Restore original CDP client
- session._cdp_client_root = original_cdp_client
-
- finally:
- # Force stop even if browser is marked as disconnected
- try:
- session.event_bus.dispatch(BrowserStopEvent(force=True))
- await asyncio.sleep(0.5) # Give it time to stop
- except Exception:
- pass # Browser might already be stopped
-
-
[email protected]('CrashWatchdog not implemented in current CDP architecture')
-async def test_crash_watchdog_lifecycle():
- """Test that CrashWatchdog starts and stops with browser session."""
- profile = BrowserProfile(headless=True)
- session = BrowserSession(browser_profile=profile)
-
- # Start browser via event and wait for BrowserConnectedEvent
- start_event = session.event_bus.dispatch(BrowserStartEvent())
- await start_event # Wait for the event and all handlers to complete
-
- started_event: BrowserConnectedEvent = cast(
- BrowserConnectedEvent, await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
- )
- assert started_event.cdp_url is not None
-
- # Verify crash watchdog is running
- assert hasattr(session, '_crash_watchdog'), 'CrashWatchdog should be created'
- assert session._crash_watchdog is not None, 'CrashWatchdog should not be None'
-
- # Check monitoring task is active
- assert session._crash_watchdog._monitoring_task is not None
- assert not session._crash_watchdog._monitoring_task.done()
-
- # Stop browser via event
- session.event_bus.dispatch(BrowserStopEvent())
-
- # Wait for browser stopped event
- try:
- stopped_event: BrowserStoppedEvent = cast(
- BrowserStoppedEvent, await session.event_bus.expect(BrowserStoppedEvent, timeout=3.0)
- )
- assert stopped_event.reason is not None
- except TimeoutError:
- # Browser stop can be flaky in test environment
- logger.warning('[TEST] BrowserStoppedEvent timeout - this is expected in some test environments')
- # Just verify the crash watchdog exists
- assert session._crash_watchdog is not None
-
- # Verify monitoring task was stopped
- await asyncio.sleep(0.1) # Give it a moment to clean up
- if session._crash_watchdog._monitoring_task:
- assert session._crash_watchdog._monitoring_task.done()
-
-
[email protected](reason='Browser initialization timeout in test environment - timing issue')
-async def test_infinite_loop_page_blocking():
- """Test that pages with infinite JavaScript loops are detected as unresponsive."""
- from pytest_httpserver import HTTPServer
-
- # Create HTTP server with blocking page
- httpserver = HTTPServer()
- httpserver.start()
-
- # Add route that serves permanently blocking JavaScript
- httpserver.expect_request('/infinite-loop').respond_with_data(
- '<html><body><h1>Loading...</h1><script>while(true){}</script></body></html>', content_type='text/html'
- )
-
- profile = BrowserProfile(headless=True)
- session = BrowserSession(browser_profile=profile)
-
- try:
- # Start browser
- session.event_bus.dispatch(BrowserStartEvent())
- await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
-
- # Navigate to blocking page
- blocking_url = httpserver.url_for('/infinite-loop')
- session.event_bus.dispatch(NavigateToUrlEvent(url=blocking_url))
-
- # The navigation should timeout or trigger an error
- # We don't expect NavigationCompleteEvent since the page blocks
- await asyncio.sleep(2) # Give it time to detect the issue
-
- # Try to interact with the page via CDP - should still work at protocol level
- cdp_session = await session.get_or_create_cdp_session()
-
- # CDP commands should still work even if page is blocked
- version_result = await session.cdp_client.send.Browser.getVersion()
- assert version_result is not None
-
- # Close the blocking tab to recover
- await session.cdp_client.send.Target.closeTarget(params={'targetId': cdp_session.target_id})
-
- finally:
- httpserver.stop()
- session.event_bus.dispatch(BrowserStopEvent())
- await asyncio.sleep(0.5)
-
-
-# @pytest.mark.asyncio
-# async def test_transient_blocking_recovery():
-# """Test recovery from temporarily blocking JavaScript."""
-# from pytest_httpserver import HTTPServer
-
-# httpserver = HTTPServer()
-# httpserver.start()
-
-# # Page that blocks for 1 second then recovers
-# httpserver.expect_request('/transient-block').respond_with_data(
-# """<html><body>
-# <h1 id="status">Blocking...</h1>
-# <script>
-# const start = Date.now();
-# while (Date.now() - start < 1000) {} // Block for 1 second
-# document.getElementById('status').textContent = 'Recovered!';
-# </script>
-# </body></html>""",
-# content_type='text/html',
-# )
-
-# profile = BrowserProfile(headless=True)
-# session = BrowserSession(browser_profile=profile)
-
-# try:
-# # Start browser
-# session.event_bus.dispatch(BrowserStartEvent())
-# await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
-
-# # Navigate to transiently blocking page
-# url = httpserver.url_for('/transient-block')
-# session.event_bus.dispatch(NavigateToUrlEvent(url=url))
-
-# # Wait for the blocking to end
-# await asyncio.sleep(2)
-
-# # Verify page recovered and we can interact with it
-# cdp_session = await session.get_or_create_cdp_session()
-# result = await session.cdp_client.send.Runtime.evaluate(
-# params={'expression': 'document.getElementById("status").textContent', 'returnByValue': True},
-# session_id=cdp_session.session_id,
-# )
-
-# status_text = result.get('result', {}).get('value', '')
-# assert status_text == 'Recovered!', f"Expected 'Recovered!' but got '{status_text}'"
-
-# finally:
-# httpserver.stop()
-# session.event_bus.dispatch(BrowserStopEvent())
-# await asyncio.sleep(0.5)
-
-
[email protected](reason='Browser initialization timeout in test environment - timing issue')
-async def test_browser_process_kill_detection():
- """Test that killing the browser process is detected."""
- import os
- import signal
-
- profile = BrowserProfile(headless=True)
- session = BrowserSession(browser_profile=profile)
-
- try:
- # Start browser
- session.event_bus.dispatch(BrowserStartEvent())
- await session.event_bus.expect(BrowserConnectedEvent, timeout=5.0)
-
- # Get browser process PID
- browser_pid = None
- if session._local_browser_watchdog and session._local_browser_watchdog._subprocess:
- browser_pid = session._local_browser_watchdog._subprocess.pid
-
- if browser_pid:
- # Kill the browser process
- try:
- os.kill(browser_pid, signal.SIGKILL)
- except ProcessLookupError:
- pass # Process might already be gone
-
- # Wait for crash detection
- try:
- error_event = cast(
- BrowserErrorEvent,
- await session.event_bus.expect(
- BrowserErrorEvent,
- predicate=lambda e: 'disconnect' in cast(BrowserErrorEvent, e).message.lower(),
- timeout=5.0,
- ),
- )
- assert error_event is not None
- except TimeoutError:
- # Crash detection might not trigger in all environments
- pass
-
- finally:
- # Force cleanup
- try:
- await session.kill()
- except Exception:
- pass
diff --git a/tests/ci/test_browser_watchdog_screenshots.py b/tests/ci/test_browser_watchdog_screenshots.py
deleted file mode 100644
index ec8900c5a..000000000
--- a/tests/ci/test_browser_watchdog_screenshots.py
+++ /dev/null
@@ -1,403 +0,0 @@
-"""
-Test that screenshots work correctly in headless browser mode.
-"""
-
-import asyncio
-import base64
-import time
-
-import pytest
-
-from browser_use.browser import BrowserProfile, BrowserSession
-from browser_use.browser.events import NavigateToUrlEvent, ScreenshotEvent
-
-
-class TestHeadlessScreenshots:
- """Test screenshot functionality specifically in headless browsers"""
-
- @pytest.mark.skip(reason='TODO: fix')
- async def test_screenshot_works_in_headless_mode(self, httpserver):
- """Explicitly test that screenshots can be captured in headless=True mode"""
- # Create a browser session with headless=True
- browser_session = BrowserSession(
- browser_profile=BrowserProfile(
- headless=True, # Explicitly set headless mode
- user_data_dir=None,
- keep_alive=False,
- )
- )
-
- try:
- # Start the session
- await browser_session.start()
- assert browser_session._cdp_client_root is not None
-
- # Set up test page with visible content
- httpserver.expect_request('/').respond_with_data(
- """<html>
- <head><title>Headless Screenshot Test</title></head>
- <body style="background: white; padding: 20px;">
- <h1 style="color: black;">This is a test page</h1>
- <p style="color: blue;">Testing screenshot capture in headless mode</p>
- <div style="width: 200px; height: 100px; background: red;">Red Box</div>
- </body>
- </html>""",
- content_type='text/html',
- )
-
- # Navigate to test page
- event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Take screenshot
- screenshot_event = browser_session.event_bus.dispatch(ScreenshotEvent())
- await screenshot_event
- screenshot_b64 = await screenshot_event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Verify screenshot was captured
- assert screenshot_b64 is not None
- assert isinstance(screenshot_b64, str)
- assert len(screenshot_b64) > 0
-
- # Decode and validate the screenshot
- screenshot_bytes = base64.b64decode(screenshot_b64)
-
- # Verify PNG signature
- assert screenshot_bytes.startswith(b'\x89PNG\r\n\x1a\n')
- # Should be a reasonable size (not just a blank image)
- assert len(screenshot_bytes) > 5000, f'Screenshot too small: {len(screenshot_bytes)} bytes'
-
- # Test full page screenshot
- screenshot_event = browser_session.event_bus.dispatch(ScreenshotEvent(full_page=True))
- await screenshot_event
- full_page_screenshot = await screenshot_event.event_result(raise_if_any=True, raise_if_none=False)
- assert full_page_screenshot is not None
- full_page_bytes = base64.b64decode(full_page_screenshot)
- assert full_page_bytes.startswith(b'\x89PNG\r\n\x1a\n')
- assert len(full_page_bytes) > 5000
-
- finally:
- await browser_session.kill()
-
- @pytest.mark.skip(reason='TODO: fix')
- async def test_screenshot_with_state_summary_in_headless(self, httpserver):
- """Test that get_state_summary includes screenshots in headless mode"""
- browser_session = BrowserSession(
- browser_profile=BrowserProfile(
- headless=True,
- user_data_dir=None,
- keep_alive=False,
- )
- )
-
- try:
- await browser_session.start()
-
- # Set up test page
- httpserver.expect_request('/').respond_with_data(
- '<html><body><h1>State Summary Test</h1></body></html>',
- content_type='text/html',
- )
- event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/')))
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Get state summary
- state = await browser_session.get_browser_state_summary()
-
- # Verify screenshot is included
- assert state.screenshot is not None
- assert isinstance(state.screenshot, str)
- assert len(state.screenshot) > 0
-
- # Decode and validate
- screenshot_bytes = base64.b64decode(state.screenshot)
- assert screenshot_bytes.startswith(b'\x89PNG\r\n\x1a\n')
- assert len(screenshot_bytes) > 1000
-
- finally:
- await browser_session.kill()
-
- @pytest.mark.skip(reason='TODO: fix')
- async def test_screenshot_graceful_handling_in_headless(self, httpserver):
- """Test that screenshot handling works correctly in headless mode even with closed pages"""
- # Set up test page
- httpserver.expect_request('/test').respond_with_data(
- '<html><body><h1>Test Page</h1></body></html>', content_type='text/html'
- )
-
- browser_session = BrowserSession(
- browser_profile=BrowserProfile(
- headless=True,
- user_data_dir=None,
- keep_alive=False,
- )
- )
-
- try:
- await browser_session.start()
-
- # Skip complex page manipulation - CDP doesn't have direct pages access
- pytest.skip('CDP pages access pattern needs refactoring')
-
- # Browser should auto-create a new page on about:blank with animation
- # With AboutBlankWatchdog, about:blank pages now have animated content, so they should have screenshots
- state = await browser_session.get_browser_state_summary()
- assert state.screenshot is not None, 'Screenshot should not be None for animated about:blank pages'
- assert state.url == 'about:blank' or state.url.startswith('chrome://'), f'Expected empty page but got {state.url}'
-
- # Now navigate to a real page and verify screenshot works
- event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/test')))
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Get state with screenshot
- state = await browser_session.get_browser_state_summary()
- # Should have a screenshot now
- assert state.screenshot is not None, 'Screenshot should not be None for real pages'
- assert isinstance(state.screenshot, str)
- assert len(state.screenshot) > 100, 'Screenshot should have substantial content'
- assert 'test' in state.url.lower()
-
- finally:
- await browser_session.kill()
-
- @pytest.mark.skip(reason='TODO: fix')
- async def test_parallel_screenshots_long_page(self, httpserver):
- """Test screenshots in a highly parallel environment with a very long page"""
-
- # Generate a very long page (50,000px+)
- long_content = []
- long_content.append('<html><head><title>Very Long Page</title></head>')
- long_content.append('<body style="margin: 0; padding: 0;">')
-
- # Add many div elements to create a 50,000px+ long page
- # Each div is 500px tall, so we need 100+ divs
- for i in range(120):
- color = f'rgb({i % 256}, {(i * 2) % 256}, {(i * 3) % 256})'
- long_content.append(
- f'<div style="height: 500px; background: {color}; '
- f'display: flex; align-items: center; justify-content: center; '
- f'font-size: 48px; color: white; text-shadow: 2px 2px 4px rgba(0,0,0,0.5);">'
- f'Section {i + 1} - Testing Parallel Screenshots'
- f'</div>'
- )
-
- long_content.append('</body></html>')
- html_content = ''.join(long_content)
-
- # Set up the test page
- httpserver.expect_request('/longpage').respond_with_data(
- html_content,
- content_type='text/html',
- )
- test_url = httpserver.url_for('/longpage')
-
- # Create 10 browser sessions
- browser_sessions = []
- for i in range(10):
- session = BrowserSession(
- browser_profile=BrowserProfile(
- headless=True,
- user_data_dir=None,
- keep_alive=False,
- )
- )
- browser_sessions.append(session)
-
- try:
- # Start all sessions sequentially to avoid playwright_global_object semaphore contention
- # The playwright global object semaphore only allows 1 concurrent initialization
- print('Starting 10 browser sessions sequentially...')
- for i, session in enumerate(browser_sessions):
- print(f'Starting session {i + 1}/10...')
- await session.start()
-
- # Navigate all sessions to the long page in parallel
- print('Navigating all sessions to the long test page...')
- await asyncio.gather(*[session.navigate(test_url) for session in browser_sessions])
-
- # Take screenshots from all sessions
- # Due to semaphore_limit=1, these will execute sequentially
- print('Taking screenshots from all 10 sessions...')
- start_time = time.time()
- screenshot_tasks = [session.take_screenshot() for session in browser_sessions]
-
- # Use return_exceptions=True to handle any failures gracefully
- results = await asyncio.gather(*screenshot_tasks, return_exceptions=True)
- total_time = time.time() - start_time
-
- # Verify timing - with semaphore_limit=1, screenshots execute sequentially
- # Each screenshot should take ~1.5s, so 10 × 1.5s = 15s, allow up to 30s for overhead
- assert total_time < 30, f'Screenshots took too long: {total_time:.1f}s (should be < 30s)'
- print(f'All screenshot attempts completed in {total_time:.1f}s')
-
- # Separate successful screenshots from failures
- screenshots = []
- failures = []
- for i, result in enumerate(results):
- if isinstance(result, Exception):
- failures.append((i, result))
- print(f'Session {i} failed: {type(result).__name__}: {result}')
- else:
- screenshots.append(result)
- print(f'Session {i} screenshot completed successfully')
-
- # ALL screenshots must succeed
- assert len(failures) == 0, (
- f'{len(failures)} screenshots failed: {[(i, type(e).__name__, str(e)) for i, e in failures]}'
- )
- assert len(screenshots) == 10, f'Expected 10 successful screenshots, got {len(screenshots)}'
- print('✅ All 10 screenshots captured successfully!')
-
- # Verify all screenshots are valid
- print('Verifying all 10 screenshots...')
- for i, screenshot in enumerate(screenshots):
- # Should not be None
- assert screenshot is not None, f'Screenshot {i} returned None'
- assert isinstance(screenshot, str), f'Screenshot {i} is not a string'
- assert len(screenshot) > 0, f'Screenshot {i} is empty'
-
- # Decode and validate
- try:
- screenshot_bytes = base64.b64decode(screenshot)
- except Exception as e:
- raise AssertionError(f'Screenshot {i} is not valid base64: {e}')
-
- # Verify PNG signature
- assert screenshot_bytes.startswith(b'\x89PNG\r\n\x1a\n'), f'Screenshot {i} is not a valid PNG'
-
- # Full page screenshot should be reasonably large
- # Due to our 6,000px height limit, expect at least 5KB
- assert len(screenshot_bytes) > 20, f'Screenshot {i} too small: {len(screenshot_bytes)} bytes'
- if len(screenshot_bytes) < 500:
- print(
- f'⚠️ Screenshot {i} failed to be taken in time, it returned a blank image instead: {len(screenshot_bytes)} bytes, perhaps the page failed to load in time?'
- )
-
- print('✅ All 10 screenshots validated successfully!')
-
- # Also test taking regular (viewport) screenshots
- print('\nTaking viewport screenshots from all sessions...')
- start_time = time.time()
- viewport_results = await asyncio.gather(
- *[session.take_screenshot() for session in browser_sessions], return_exceptions=True
- )
- viewport_time = time.time() - start_time
- assert viewport_time < 30, f'Viewport screenshots took too long: {viewport_time:.1f}s (should be < 30s)'
- print(f'All viewport screenshot attempts completed in {viewport_time:.1f}s')
-
- # Check for failures
- viewport_screenshots = []
- viewport_failures = []
- for i, result in enumerate(viewport_results):
- if isinstance(result, Exception):
- viewport_failures.append((i, result))
- print(f'Session {i} viewport failed: {type(result).__name__}: {result}')
- else:
- viewport_screenshots.append(result)
- print(f'Session {i} viewport screenshot completed successfully')
-
- # ALL viewport screenshots must succeed
- assert len(viewport_failures) == 0, (
- f'{len(viewport_failures)} viewport screenshots failed: {[(i, type(e).__name__, str(e)) for i, e in viewport_failures]}'
- )
- assert len(viewport_screenshots) == 10, (
- f'Expected 10 successful viewport screenshots, got {len(viewport_screenshots)}'
- )
- print('✅ All 10 viewport screenshots captured successfully!')
-
- # Verify all 10 viewport screenshots
- print('Verifying all 10 viewport screenshots...')
- for i, screenshot in enumerate(viewport_screenshots):
- assert screenshot is not None, f'Viewport screenshot {i} is None'
- screenshot_bytes = base64.b64decode(screenshot)
- assert screenshot_bytes.startswith(b'\x89PNG\r\n\x1a\n'), f'Viewport screenshot {i} is not a valid PNG'
- # Viewport screenshots should be reasonably sized
- assert len(screenshot_bytes) > 10, f'Viewport screenshot {i} too small: {len(screenshot_bytes)} bytes'
- print('✅ All 10 viewport screenshots validated successfully!')
-
- finally:
- # Kill all sessions in parallel
- print('Killing all browser sessions...')
- # Use return_exceptions=True to prevent one failed kill from affecting others
- # This prevents "Future exception was never retrieved" errors
- results = await asyncio.gather(*[session.kill() for session in browser_sessions], return_exceptions=True)
-
- # Check that no exceptions were raised during cleanup
- for i, result in enumerate(results):
- if isinstance(result, Exception):
- print(f'Warning: Session {i} kill raised exception: {type(result).__name__}: {result}')
-
- @pytest.mark.skip(reason='TODO: fix')
- async def test_screenshot_at_bottom_of_page(self, httpserver):
- """Test screenshot capture when scrolled to bottom of page (regression test for clipping issue)"""
- browser_session = BrowserSession(
- browser_profile=BrowserProfile(
- headless=True,
- user_data_dir=None,
- keep_alive=False,
- )
- )
-
- try:
- await browser_session.start()
-
- # Create a page with scrollable content
- httpserver.expect_request('/scrollable').respond_with_data(
- """<html>
- <head><title>Scrollable Page Test</title></head>
- <body style="margin: 0; padding: 0;">
- <div style="height: 3000px; background: linear-gradient(to bottom, red, yellow, green, blue);">
- <div style="position: absolute; top: 0; left: 10px; font-size: 24px;">Top of page</div>
- <div style="position: absolute; top: 50%; left: 10px; font-size: 24px;">Middle of page</div>
- <div style="position: absolute; bottom: 10px; left: 10px; font-size: 24px;">Bottom of page</div>
- </div>
- </body>
- </html>""",
- content_type='text/html',
- )
-
- # Navigate to test page
- event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/scrollable')))
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
- # Skip - get_current_tab doesn't exist in CDP session
- pytest.skip('get_current_tab method not available in CDP session')
-
- finally:
- await browser_session.kill()
-
-
-class TestScreenshotEventSystem:
- """Tests for NEW event-driven screenshot infrastructure only."""
-
- @pytest.mark.skip(reason='TODO: fix')
- async def test_screenshot_response_event_dispatching(self, httpserver):
- """Test that ScreenshotResponseEvent is properly dispatched by event handlers."""
- from browser_use.browser.events import ScreenshotEvent
-
- browser_session = BrowserSession(browser_profile=BrowserProfile(headless=True, user_data_dir=None, keep_alive=False))
-
- try:
- await browser_session.start()
-
- # Set up test page
- httpserver.expect_request('/event-test').respond_with_data(
- '<html><body><h1>Screenshot Event Test</h1></body></html>',
- content_type='text/html',
- )
- event = browser_session.event_bus.dispatch(NavigateToUrlEvent(url=httpserver.url_for('/event-test')))
- await event
- await event.event_result(raise_if_any=True, raise_if_none=False)
-
- # Test the NEW event-driven path: direct event dispatching
- event = browser_session.event_bus.dispatch(ScreenshotEvent(full_page=False))
- screenshot_b64 = await event.event_result()
- assert screenshot_b64 is not None
- assert isinstance(screenshot_b64, str)
- assert len(base64.b64decode(screenshot_b64)) > 5000
-
- finally:
- await browser_session.kill()
diff --git a/tests/ci/test_llm_custom_structured_ouput.py b/tests/ci/test_llm_custom_structured_ouput.py
deleted file mode 100644
index 92ef1b62c..000000000
--- a/tests/ci/test_llm_custom_structured_ouput.py
+++ /dev/null
@@ -1,90 +0,0 @@
-import json
-import os
-
-import tiktoken
-
-from browser_use.agent.views import AgentOutput
-from browser_use.llm.schema import SchemaOptimizer
-from browser_use.tools.service import Tools
-
-
-def test_optimized_schema():
- """Test the optimized schema generation and save to file."""
-
- # Create tools and get all registered actions
- tools = Tools()
- ActionModel = tools.registry.create_action_model()
-
- # Create the agent output model with custom actions
- agent_output_model = AgentOutput.type_with_custom_actions(ActionModel)
-
- # Get original schema for comparison
- original_schema = agent_output_model.model_json_schema()
-
- # Create the optimized schema
- optimized_schema = SchemaOptimizer.create_optimized_json_schema(agent_output_model)
-
- # Create tmp directory if it doesn't exist
- os.makedirs('./tmp', exist_ok=True)
-
- # Save optimized schema
- with open('./tmp/optimized_schema.json', 'w') as f:
- json.dump(optimized_schema, f, separators=(',', ':'), indent=2)
-
- print('✅ Optimized schema generated and saved to ./tmp/optimized_schema.json')
-
- # Compare token counts of both
- try:
- enc = tiktoken.encoding_for_model('gpt-4o')
- except KeyError:
- enc = tiktoken.get_encoding('cl100k_base')
-
- original_tokens = len(enc.encode(json.dumps(original_schema)))
- optimized_tokens = len(enc.encode(json.dumps(optimized_schema, separators=(',', ':'))))
-
- savings = original_tokens - optimized_tokens
- savings_percentage = (savings / original_tokens * 100) if original_tokens > 0 else 0
-
- print('\n📊 Token Count Comparison:')
- print(f' Original schema: {original_tokens:,} tokens')
- print(f' Optimized schema: {optimized_tokens:,} tokens')
- print(f' Token savings: {savings:,} tokens ({savings_percentage:.1f}% reduction)')
-
- # Count tokens per action in optimized schema
- print('\n🔍 Tokens per Action in Optimized Schema:')
-
- if 'properties' in optimized_schema and 'action' in optimized_schema['properties']:
- action_prop = optimized_schema['properties']['action']
- if 'items' in action_prop and 'anyOf' in action_prop['items']:
- actions = action_prop['items']['anyOf']
-
- total_action_tokens = 0
- for i, action in enumerate(actions):
- action_json = json.dumps(action, separators=(',', ':'))
- action_tokens = len(enc.encode(action_json))
- total_action_tokens += action_tokens
-
- # Try to get action name from the schema
- action_name = 'Unknown'
- if 'properties' in action:
- # Get the first property that's not common ones like 'index', 'reasoning'
- for prop_name in action['properties'].keys():
- if prop_name not in ['index', 'reasoning']:
- action_name = prop_name
- break
-
- print(f' Action {i + 1} ({action_name}): {action_tokens:,} tokens')
-
- print('\n📈 Summary:')
- print(f' Total actions: {len(actions)}')
- print(f' Total action tokens: {total_action_tokens:,} tokens')
- print(f' Average tokens per action: {total_action_tokens // len(actions):,} tokens')
- print(f' Action tokens as % of total: {(total_action_tokens / optimized_tokens * 100):.1f}%')
- else:
- print(' No actions found in expected schema structure')
- else:
- print(' No action property found in optimized schema')
-
-
-if __name__ == '__main__':
- test_optimized_schema()
diff --git a/tests/ci/test_sync_client_legacy.py.disabled b/tests/ci/test_sync_client_legacy.py.disabled
deleted file mode 100644
index fe23b9945..000000000
--- a/tests/ci/test_sync_client_legacy.py.disabled
+++ /dev/null
@@ -1,603 +0,0 @@
-"""Tests for CloudSync client machinery - retry logic, event handling, backend communication."""
-
-import os
-import tempfile
-from pathlib import Path
-
-import httpx
-import pytest
-from bubus import BaseEvent
-from pytest_httpserver import HTTPServer
-
-from browser_use.agent.cloud_events import CreateAgentTaskEvent
-from browser_use.sync.auth import TEMP_USER_ID, DeviceAuthClient
-from browser_use.sync.service import CloudSync
-
-
-def temp_config_dir():
- """Create temporary config directory for tests."""
- with tempfile.TemporaryDirectory() as tmpdir:
- temp_dir = Path(tmpdir) / '.config' / 'browseruse'
- temp_dir.mkdir(parents=True, exist_ok=True)
-
- os.environ['BROWSER_USE_CONFIG_DIR'] = str(temp_dir)
-
- yield temp_dir
-
-
-async def http_client(httpserver: HTTPServer):
- """Create a real HTTP client pointed at the test server."""
- async with httpx.AsyncClient(base_url=httpserver.url_for('')) as client:
- yield client
-
-
-class TestCloudSyncInit:
- """Test CloudSync initialization and configuration."""
-
- async def test_init_with_auth_enabled(self, temp_config_dir):
- """Test CloudSync initialization with auth enabled."""
- service = CloudSync(base_url='http://localhost:8000')
-
- assert service.base_url == 'http://localhost:8000'
- assert service.auth_client is not None
- assert isinstance(service.auth_client, DeviceAuthClient)
- assert service.session_id is None
-
- async def test_init_basic(self, temp_config_dir):
- """Test CloudSync basic initialization."""
- service = CloudSync(base_url='http://localhost:8000')
-
- assert service.base_url == 'http://localhost:8000'
- assert service.auth_client is not None # Always has auth_client in current version
- assert isinstance(service.auth_client, DeviceAuthClient)
-
-
-class TestCloudSyncEventHandling:
- """Test CloudSync event validation and processing."""
-
- @pytest.fixture
- def authenticated_sync(self, httpserver: HTTPServer, http_client, temp_config_dir):
- """Create authenticated CloudSync service."""
- auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
- auth.auth_config.api_token = 'test-api-key'
- auth.auth_config.user_id = 'test-user-123'
-
- service = CloudSync(base_url=httpserver.url_for(''), enable_auth=True)
- service.auth_client = auth
- service.session_id = 'test-session-id'
- return service
-
- @pytest.fixture
- def unauthenticated_sync(self, httpserver: HTTPServer, temp_config_dir):
- """Create unauthenticated CloudSync service."""
- service = CloudSync(base_url=httpserver.url_for(''), enable_auth=True)
- service.session_id = 'test-session-id'
- return service
-
- async def test_event_forwarding_authenticated(self, httpserver: HTTPServer, authenticated_sync):
- """Test event forwarding when authenticated."""
- # Capture requests
- requests = []
-
- def capture_request(request):
- requests.append(request.get_json())
- from werkzeug.wrappers import Response
-
- return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
-
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
-
- # Send event
- await authenticated_sync.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Test task',
- user_id='test-user-123',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- # Verify forwarding
- assert len(requests) == 1
- event_batch = requests[0]
- assert len(event_batch['events']) == 1
-
- event = event_batch['events'][0]
- assert event['event_type'] == 'CreateAgentTaskEvent'
- assert event['user_id'] == 'test-user-123'
- # BaseEvent creates event_type attribute, plus our custom data as attributes
- assert event['task'] == 'Test task'
-
- async def test_event_queueing_unauthenticated(self, httpserver: HTTPServer, unauthenticated_sync):
- """Test event queueing when unauthenticated."""
- # Server returns 401
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_json({'error': 'unauthorized'}, status=401)
-
- # Send event
- await unauthenticated_sync.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Queued task',
- user_id=TEMP_USER_ID,
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- # Event should be queued
- assert len(unauthenticated_sync.pending_events) == 1
- queued_event = unauthenticated_sync.pending_events[0]
- assert queued_event.event_type == 'CreateAgentTaskEvent'
- assert queued_event.user_id == TEMP_USER_ID
- assert queued_event.task == 'Queued task'
-
- async def test_event_user_id_injection_pre_auth(self, httpserver: HTTPServer, unauthenticated_sync):
- """Test that temp user ID is injected for pre-auth events."""
- requests = []
-
- def capture_request(request):
- requests.append(request.get_json())
- from werkzeug.wrappers import Response
-
- return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
-
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
-
- # Send event without user_id
- await unauthenticated_sync.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Pre-auth task',
- user_id=TEMP_USER_ID,
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- # Verify temp user ID was injected
- assert len(requests) == 1
- event = requests[0]['events'][0]
- assert event['user_id'] == TEMP_USER_ID
-
-
-class TestCloudSyncRetryLogic:
- """Test CloudSync retry and error handling logic."""
-
- @pytest.fixture
- def sync_with_auth(self, httpserver: HTTPServer, http_client, temp_config_dir):
- """Create CloudSync with auth."""
- auth = DeviceAuthClient(base_url=httpserver.url_for(''), http_client=http_client)
- auth.auth_config.api_token = 'test-api-key'
- auth.auth_config.user_id = 'test-user-123'
-
- service = CloudSync(base_url=httpserver.url_for(''), enable_auth=True)
- service.auth_client = auth
- service.session_id = 'test-session-id'
- return service
-
- async def test_pending_event_resending(self, httpserver: HTTPServer, sync_with_auth):
- """Test resending of pending events after authentication."""
- requests = []
-
- def capture_request(request):
- requests.append(request.get_json())
- from werkzeug.wrappers import Response
-
- return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
-
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
-
- # Manually add pending events (simulating 401 scenario)
- sync_with_auth.pending_events.extend(
- [
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Pending task 1',
- user_id=TEMP_USER_ID,
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- ),
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Pending task 2',
- user_id=TEMP_USER_ID,
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- ),
- ]
- )
-
- # Resend pending events
- await sync_with_auth._resend_pending_events()
-
- # Should send all pending events with updated user ID
- assert len(requests) == 2
- for i, request in enumerate(requests):
- event = request['events'][0]
- assert event['user_id'] == 'test-user-123' # Updated from temp ID
- assert f'Pending task {i + 1}' == event['task']
-
- # Pending events should be cleared
- assert len(sync_with_auth.pending_events) == 0
-
- async def test_backend_error_resilience(self, httpserver: HTTPServer, sync_with_auth):
- """Test resilience to backend errors."""
- # Server returns 500 error
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_data('Internal Server Error', status=500)
-
- # Should not raise exception
- await sync_with_auth.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Task during outage',
- user_id='test-user-123',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- # Events should not be queued for 500 errors (only 401)
- assert len(sync_with_auth.pending_events) == 0
-
- async def test_network_error_resilience(self, sync_with_auth):
- """Test resilience to network errors."""
- # No server running - will get connection error
- sync_with_auth.base_url = 'http://localhost:99999' # Invalid port
-
- # Should not raise exception
- await sync_with_auth.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Task during network error',
- user_id='test-user-123',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- # Should handle gracefully without crashing
-
- async def test_concurrent_event_sending(self, httpserver: HTTPServer, sync_with_auth):
- """Test handling of concurrent event sending."""
- import asyncio
-
- requests = []
-
- def capture_request(request):
- requests.append(request.get_json())
- from werkzeug.wrappers import Response
-
- return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
-
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
-
- # Send multiple events concurrently
- tasks = []
- for i in range(5):
- task = sync_with_auth.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task=f'Concurrent task {i}',
- user_id='test-user-123',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
- tasks.append(task)
-
- await asyncio.gather(*tasks)
-
- # All events should be sent
- assert len(requests) == 5
- # Just verify all events have task data - order may vary due to concurrency
- task_values = [req['events'][0]['task'] for req in requests]
- expected_tasks = [f'Concurrent task {i}' for i in range(5)]
- assert sorted(task_values) == sorted(expected_tasks)
-
-
-class TestCloudSyncBackendCommunication:
- """Test CloudSync backend communication patterns."""
-
- async def test_request_format_validation(self, httpserver: HTTPServer, temp_config_dir):
- """Test that requests are formatted correctly for backend."""
- requests = []
-
- def capture_request(request):
- # Validate request structure
- assert request.content_type == 'application/json'
- data = request.get_json()
- requests.append(data)
-
- # Validate batch structure
- assert 'events' in data
- assert isinstance(data['events'], list)
- assert len(data['events']) == 1
-
- event = data['events'][0]
- required_fields = ['event_type', 'event_id', 'event_created_at', 'event_schema', 'user_id']
- for field in required_fields:
- assert field in event, f'Missing required field: {field}'
-
- from werkzeug.wrappers import Response
-
- return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
-
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
-
- # Create authenticated service
- auth = DeviceAuthClient(base_url=httpserver.url_for(''))
- auth.auth_config.api_token = 'test-api-key'
- auth.auth_config.user_id = 'test-user-123'
-
- service = CloudSync(base_url=httpserver.url_for(''), enable_auth=True)
- service.auth_client = auth
- service.session_id = 'test-session-id'
-
- await service.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Format validation test',
- user_id='test-user-123',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- assert len(requests) == 1
-
- async def test_auth_header_handling(self, httpserver: HTTPServer, temp_config_dir):
- """Test proper auth header handling."""
- requests = []
-
- def capture_request(request):
- requests.append(
- {
- 'headers': dict(request.headers),
- 'json': request.get_json(),
- }
- )
- from werkzeug.wrappers import Response
-
- return Response('{"processed": 1, "failed": 0}', status=200, mimetype='application/json')
-
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(capture_request)
-
- # Test authenticated request
- auth = DeviceAuthClient(base_url=httpserver.url_for(''))
- auth.auth_config.api_token = 'test-api-key'
- auth.auth_config.user_id = 'test-user-123'
-
- service = CloudSync(base_url=httpserver.url_for(''), enable_auth=True)
- service.auth_client = auth
-
- await service.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Auth header test',
- user_id='test-user-123',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- # Check auth header was included
- assert len(requests) == 1
- headers = requests[0]['headers']
- assert 'Authorization' in headers
- assert headers['Authorization'] == 'Bearer test-api-key'
-
- # Test unauthenticated request
- requests.clear()
- service.auth_client = DeviceAuthClient(base_url=httpserver.url_for('')) # No credentials
-
- await service.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='No auth test',
- user_id='',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- # Check no auth header
- assert len(requests) == 1
- headers = requests[0]['headers']
- assert 'Authorization' not in headers
-
-
-class TestCloudSyncErrorHandling:
- """Test CloudSync error handling doesn't crash the agent."""
-
- @pytest.fixture
- def sync_service(self, httpserver: HTTPServer, temp_config_dir):
- """Create CloudSync service."""
- return CloudSync(base_url=httpserver.url_for(''), enable_auth=False)
-
- async def test_timeout_error_handling(self, sync_service):
- """Test that timeout errors are handled gracefully."""
- # Use a URL that will timeout
- sync_service.base_url = 'http://10.255.255.1' # Non-routable IP for timeout
-
- # Should not raise exception
- await sync_service.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Timeout test',
- user_id='test-user-123',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- async def test_malformed_event_handling(self, httpserver: HTTPServer, sync_service):
- """Test handling of events that can't be serialized."""
-
- class BadEvent(BaseEvent):
- """Event that will fail to serialize."""
-
- event_type: str = 'BadEvent'
-
- def model_dump(self, **kwargs):
- raise ValueError('Serialization failed')
-
- # Should not raise exception
- await sync_service.handle_event(BadEvent())
-
- async def test_http_error_responses(self, httpserver: HTTPServer, sync_service):
- """Test various HTTP error responses don't crash the service."""
- error_codes = [400, 403, 404, 429, 500, 502, 503]
-
- for status_code in error_codes:
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_json(
- {'error': f'Test error {status_code}'}, status=status_code
- )
-
- # Should not raise exception
- await sync_service.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task=f'Error {status_code} test',
- user_id='test-user-123',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- async def test_invalid_response_handling(self, httpserver: HTTPServer, sync_service):
- """Test handling of invalid server responses."""
- # Return invalid JSON
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_data('Not JSON', status=200)
-
- # Should not raise exception
- await sync_service.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task='Invalid response test',
- user_id='test-user-123',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
-
- async def test_event_with_restricted_attributes(self, httpserver: HTTPServer, sync_service):
- """Test handling events that don't allow user_id attribute."""
- from pydantic import ConfigDict
-
- class RestrictedEvent(BaseEvent):
- """Event that doesn't allow extra attributes."""
-
- model_config = ConfigDict(extra='forbid')
- event_type: str = 'RestrictedEvent'
- data: str = 'test'
-
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_json({'processed': 1}, status=200)
-
- # Should not raise exception - will log debug message about not being able to set user_id
- await sync_service.handle_event(RestrictedEvent())
-
- async def test_concurrent_error_resilience(self, httpserver: HTTPServer, sync_service):
- """Test that concurrent errors don't affect other events."""
- import asyncio
-
- successful_requests = []
- request_count = 0
-
- def handler(request):
- nonlocal request_count
- request_count += 1
- # Every 3rd request fails
- if request_count % 3 == 0:
- from werkzeug.wrappers import Response
-
- return Response('Server Error', status=500)
- else:
- successful_requests.append(request.get_json())
- from werkzeug.wrappers import Response
-
- return Response('{"processed": 1}', status=200, mimetype='application/json')
-
- httpserver.expect_request('/api/v1/events', method='POST').respond_with_handler(handler)
-
- # Send 10 events concurrently
- tasks = []
- for i in range(10):
- task = sync_service.handle_event(
- CreateAgentTaskEvent(
- agent_session_id='test-session',
- llm_model='test-model',
- task=f'Concurrent error test {i}',
- user_id='test-user-123',
- device_id='test-device-id',
- done_output=None,
- user_feedback_type=None,
- user_comment=None,
- gif_url=None,
- )
- )
- tasks.append(task)
-
- # All should complete without raising
- await asyncio.gather(*tasks)
-
- # ~7 should succeed (10 total, ~3 fail)
- assert len(successful_requests) >= 6