Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Recover OpenAI browser observations and preserve usable task files

Magnus Muelleraadf9ecd9162
diff --git a/browser_use/integrations/anthropic/bash.py b/browser_use/integrations/anthropic/bash.py
index e27b7cd12..49e461199 100644
--- a/browser_use/integrations/anthropic/bash.py
+++ b/browser_use/integrations/anthropic/bash.py
@@ -52,6 +52,8 @@ async def run_bash(
output_dir: str | Path,
timeout_seconds: float = 120,
max_output_bytes: int = 50_000,
+ python_executable: str | Path | None = None,
+ home_dir: str | Path | None = None,
) -> str:
"""Run Bash with a stripped environment, bounded output, and a hard timeout."""
if os.name != 'posix':
@@ -62,6 +64,14 @@ async def run_bash(
raise ValueError('max_output_bytes must be positive')
root, tmp = await asyncio.to_thread(_prepare_output_dir, output_dir)
+ home = Path(home_dir).expanduser().absolute() if home_dir is not None else root
+ await asyncio.to_thread(home.mkdir, parents=True, exist_ok=True)
+ path = '/usr/local/bin:/usr/bin:/bin'
+ if python_executable is not None:
+ python = Path(python_executable).expanduser().absolute()
+ if not python.is_file() or not os.access(python, os.X_OK):
+ raise ValueError('python_executable must name an executable file')
+ path = f'{python.parent}:{path}'
protocol = _BashOutput(max_output_bytes)
transport, _ = await asyncio.get_running_loop().subprocess_exec(
lambda: protocol,
@@ -72,10 +82,10 @@ async def run_bash(
command,
cwd=root,
env={
- 'HOME': str(root),
+ 'HOME': str(home),
'LANG': 'C.UTF-8',
'LC_ALL': 'C.UTF-8',
- 'PATH': '/usr/local/bin:/usr/bin:/bin',
+ 'PATH': path,
'PYTHONNOUSERSITE': '1',
'TMPDIR': str(tmp),
},
diff --git a/browser_use/integrations/openai/bash.py b/browser_use/integrations/openai/bash.py
index ab0b69c23..fcb444e02 100644
--- a/browser_use/integrations/openai/bash.py
+++ b/browser_use/integrations/openai/bash.py
@@ -1,5 +1,6 @@
"""Expose the shared Bash executor through OpenAI's tool runner."""
+import sys
from pathlib import Path
from pydantic import BaseModel, ConfigDict
@@ -18,16 +19,21 @@ def Bash(
*,
output_dir: str | Path = 'outputs',
timeout_seconds: float = 120,
- max_output_bytes: int = 50_000,
+ max_output_bytes: int = 10_000,
+ python_executable: str | Path | None = None,
) -> AsyncRegisteredFunction:
"""Create an OpenAI Bash tool with bounded execution and output."""
+ root = Path(output_dir).expanduser().absolute()
+
async def execute(args: BashInput) -> str:
return await run_bash(
args.command,
output_dir=output_dir,
timeout_seconds=timeout_seconds,
max_output_bytes=max_output_bytes,
+ python_executable=python_executable or sys.executable,
+ home_dir=root / '.tmp' / 'home',
)
return AsyncRegisteredFunction.from_tool(
diff --git a/browser_use/integrations/openai/browser_use.py b/browser_use/integrations/openai/browser_use.py
index 2b74c8cd3..e6347ba65 100644
--- a/browser_use/integrations/openai/browser_use.py
+++ b/browser_use/integrations/openai/browser_use.py
@@ -17,15 +17,20 @@ from PIL import Image
from pydantic import BaseModel
from browser_use.actor.page import Page
+from browser_use.browser._cdp_timeout import TimeoutWrappedCDPClient
from browser_use.browser.session import BrowserSession
from openai.lib import browser as api
+_STATE_TIMEOUT_S = 15.0
+_DIAGNOSTIC_TIMEOUT_S = 5.0
+_OBSERVATION_TIMEOUT_S = 20.0
+
def _bounded_text(text: str) -> str:
- if len(text) <= 50000:
+ if len(text) <= 10000:
return text
notice = '\n[Output truncated; request a smaller section or return fewer script results.]'
- return text[: 50000 - len(notice)] + notice
+ return text[: 10000 - len(notice)] + notice
class Reference(BaseModel):
@@ -57,6 +62,7 @@ class BrowserUse:
self._refs: dict[str, Reference] = {}
self._next_ref = 0
self._active: str | None = None
+ self._crashed_targets: set[str] = set()
self._scopes: dict[str, Reference] = {}
self._grants: dict[str, FileGrant] = {}
self._diagnostic_client: CDPClient | None = None
@@ -99,11 +105,32 @@ class BrowserUse:
self._refs.clear()
self._closed = True
+ def _target_crashed(self, event: Any, session_id: str | None = None) -> None:
+ self._crashed_targets.add(event['targetId'])
+
async def state(self) -> api.BrowserState:
"""Report current browser tabs without retaining stale tab state."""
if self._closed:
raise api.BrowserError('The driver is closed.')
- result = await self.browser.cdp_client.send.Target.getTargets()
+ for attempt in range(2):
+ try:
+ async with asyncio.timeout(_STATE_TIMEOUT_S):
+ result = await self.browser.cdp_client.send.Target.getTargets()
+ break
+ except Exception as exc:
+ if attempt:
+ raise api.BrowserError('Browser state is unavailable after reconnecting.') from exc
+ try:
+ async with asyncio.timeout(_STATE_TIMEOUT_S):
+ # Suppress automatic reconnect when this reconnect closes the old socket.
+ stopping = self.browser._intentional_stop
+ self.browser._intentional_stop = True
+ try:
+ await self.browser.reconnect()
+ finally:
+ self.browser._intentional_stop = stopping
+ except Exception as reconnect_error:
+ raise api.BrowserError('Browser reconnection failed.') from reconnect_error
tabs = [t for t in result['targetInfos'] if t['type'] == 'page']
ids = {t['targetId'] for t in tabs}
if self._active not in ids:
@@ -127,9 +154,18 @@ class BrowserUse:
target = tab_id or self._active
if target is None or target not in {tab['id'] for tab in state['tabs']}:
raise api.BrowserError('Unknown tab.')
- session = await self.browser.get_or_create_cdp_session(target, focus=True)
+ if target in self._crashed_targets:
+ raise api.BrowserError('The page crashed; open the page in a new tab.')
+ try:
+ async with asyncio.timeout(_STATE_TIMEOUT_S):
+ session = await self.browser.get_or_create_cdp_session(target, focus=True)
+ except Exception as exc:
+ raise api.BrowserError('The page is unavailable; select another tab or open it again.') from exc
self._active = target
- await self._start_diagnostics(target)
+ try:
+ await self._start_diagnostics(target)
+ except api.BrowserError:
+ pass
return Page(self.browser, target, session.session_id)
async def _element(self, page: Page, target: str):
@@ -158,10 +194,21 @@ class BrowserUse:
return await self._read_page(args)
async def _read_page(self, args: api.ReadPageInput, query: str | None = None) -> str:
+ attempt = 0
+ while True:
+ try:
+ async with asyncio.timeout(_OBSERVATION_TIMEOUT_S):
+ return await self._read_page_once(args, query)
+ except Exception as exc:
+ if attempt:
+ raise api.BrowserError('Page reading failed; select another tab or try again.') from exc
+ attempt += 1
+
+ async def _read_page_once(self, args: api.ReadPageInput, query: str | None = None) -> str:
terms = query.casefold().split() if query else []
page = await self._page(args['tab_id'])
await self.browser.cdp_client.send.DOM.getDocument({'depth': 0}, session_id=await page.session_id)
- self._refs = {name: ref for name, ref in self._refs.items() if ref.tab != page._target_id}
+ refs = {ref.backend: name for name, ref in self._refs.items() if ref.tab == page._target_id}
result = await self.browser.cdp_client.send.Accessibility.getFullAXTree(session_id=await page.session_id)
interactive = {
'button',
@@ -179,12 +226,13 @@ class BrowserUse:
'listbox',
'switch',
}
- lines: list[str] = []
+ identity = json.loads(await page.evaluate('() => ({title: document.title, url: location.href})'))
+ lines = [f'Title: {identity["title"][:2000]}', f'URL: {identity["url"][:2000]}']
nodes = {node['nodeId']: node for node in result['nodes']}
children = {child for node in result['nodes'] for child in node.get('childIds', [])}
pending = [(node, 0) for node in reversed(result['nodes']) if node['nodeId'] not in children]
seen: set[str] = set()
- length = 0
+ length = sum(len(line) + 1 for line in lines)
notice = '\n[Page truncated; use read_page_scoped, find, or interactive reads for specific content.]'
while pending:
node, depth = pending.pop()
@@ -204,9 +252,12 @@ class BrowserUse:
prefix = ''
backend = node.get('backendDOMNodeId')
if backend and role in interactive:
- self._next_ref += 1
- ref = f'ref_{self._next_ref}'
- self._refs[ref] = Reference(tab=page._target_id, backend=backend)
+ ref = refs.get(backend)
+ if ref is None:
+ self._next_ref += 1
+ ref = f'ref_{self._next_ref}'
+ self._refs[ref] = Reference(tab=page._target_id, backend=backend)
+ refs[backend] = ref
prefix = f'[{ref}] '
states = [
f'{prop["name"]}={json.dumps(prop["value"].get("value"))}'
@@ -222,7 +273,7 @@ class BrowserUse:
line = f'{" " * depth}{prefix}{role} {json.dumps(name, ensure_ascii=False)}{suffix}'
if not all(term in line.casefold() for term in terms):
continue
- remaining = 50000 - len(notice) - length
+ remaining = 10000 - len(notice) - length
if len(line) + 1 > remaining:
return '\n'.join(lines) + notice
lines.append(line)
@@ -299,17 +350,26 @@ class BrowserUse:
async def screenshot(self, args: api.TabInput) -> dict[str, str]:
"""Capture the selected viewport as a native image input."""
- page = await self._page(args['tab_id'])
- metrics = await self.browser.cdp_client.send.Page.getLayoutMetrics(session_id=await page.session_id)
- viewport = metrics['cssVisualViewport']
- size = (round(viewport['clientWidth']), round(viewport['clientHeight']))
- data = await page.screenshot()
- with Image.open(BytesIO(base64.b64decode(data))) as image:
- if image.size != size:
- output = BytesIO()
- image.resize(size, Image.Resampling.LANCZOS).save(output, format='PNG')
- data = base64.b64encode(output.getvalue()).decode()
- return {'type': 'input_image', 'image_url': 'data:image/png;base64,' + data, 'detail': 'original'}
+ attempt = 0
+ while True:
+ try:
+ async with asyncio.timeout(_OBSERVATION_TIMEOUT_S):
+ page = await self._page(args['tab_id'])
+ metrics = await self.browser.cdp_client.send.Page.getLayoutMetrics(session_id=await page.session_id)
+ viewport = metrics['cssVisualViewport']
+ size = (round(viewport['clientWidth']), round(viewport['clientHeight']))
+ data = await page.screenshot()
+ with Image.open(BytesIO(base64.b64decode(data))) as image:
+ image.load()
+ if image.size != size:
+ output = BytesIO()
+ image.resize(size, Image.Resampling.LANCZOS).save(output, format='PNG')
+ data = base64.b64encode(output.getvalue()).decode()
+ return {'type': 'input_image', 'image_url': 'data:image/png;base64,' + data, 'detail': 'original'}
+ except Exception as exc:
+ if attempt:
+ raise api.BrowserError('Screenshot capture failed; select another tab or try again.') from exc
+ attempt += 1
async def zoom(self, args: api.ZoomInput) -> dict[str, str]:
"""Crop a screenshot without changing the pointer coordinate origin."""
@@ -417,20 +477,40 @@ class BrowserUse:
return _bounded_text(value if isinstance(value, str) else json.dumps(value))
async def _start_diagnostics(self, target: str) -> None:
- if self._diagnostic_client is None:
- self._diagnostic_client = CDPClient(
- self.browser.cdp_client.url, additional_headers=self.browser.cdp_client.additional_headers
- )
- await self._diagnostic_client.start()
- self._diagnostic_client.register.Runtime.consoleAPICalled(self._console_event)
- self._diagnostic_client.register.Network.responseReceived(self._network_event)
- if target not in self._diagnostic_sessions:
- attached = await self._diagnostic_client.send.Target.attachToTarget({'targetId': target, 'flatten': True})
- session = attached['sessionId']
- self._diagnostic_sessions[target] = session
- self._diagnostics[session] = {'console': deque(maxlen=100), 'network': deque(maxlen=100)}
- await self._diagnostic_client.send.Runtime.enable(session_id=session)
- await self._diagnostic_client.send.Network.enable(session_id=session)
+ if target in self._diagnostic_sessions:
+ return
+ try:
+ async with asyncio.timeout(_DIAGNOSTIC_TIMEOUT_S):
+ if self._diagnostic_client is None:
+ self._diagnostic_client = TimeoutWrappedCDPClient(
+ self.browser.cdp_client.url,
+ additional_headers=self.browser.cdp_client.additional_headers,
+ cdp_request_timeout_s=_DIAGNOSTIC_TIMEOUT_S,
+ )
+ await self._diagnostic_client.start()
+ self._diagnostic_client.register.Target.targetCrashed(self._target_crashed)
+ await self._diagnostic_client.send.Target.setDiscoverTargets({'discover': True})
+ self._diagnostic_client.register.Runtime.consoleAPICalled(self._console_event)
+ self._diagnostic_client.register.Network.responseReceived(self._network_event)
+ attached = await self._diagnostic_client.send.Target.attachToTarget({'targetId': target, 'flatten': True})
+ session = attached['sessionId']
+ self._diagnostics[session] = {'console': deque(maxlen=100), 'network': deque(maxlen=100)}
+ await self._diagnostic_client.send.Runtime.enable(session_id=session)
+ await self._diagnostic_client.send.Network.enable(session_id=session)
+ self._diagnostic_sessions[target] = session
+ except BaseException as exc:
+ client, self._diagnostic_client = self._diagnostic_client, None
+ self._diagnostic_sessions.clear()
+ self._diagnostics.clear()
+ if client is not None:
+ try:
+ async with asyncio.timeout(_DIAGNOSTIC_TIMEOUT_S):
+ await client.stop()
+ except Exception:
+ pass
+ if not isinstance(exc, Exception):
+ raise
+ raise api.BrowserError('Page diagnostics are unavailable.') from exc
def _console_event(self, event: Any, session_id: str | None) -> None:
if session_id in self._diagnostics:
@@ -448,6 +528,8 @@ class BrowserUse:
async def read_diagnostics(self, args: api.ReadDiagnosticsInput) -> api.BrowserReadV1:
"""Return bounded diagnostics without network headers or request bodies."""
page = await self._page(args['tab_id'])
+ if page._target_id not in self._diagnostic_sessions:
+ raise api.BrowserError('Page diagnostics are unavailable; try again after the page responds.')
records = self._diagnostics[self._diagnostic_sessions[page._target_id]][args['source']]
return {
'version': 1,
@@ -472,8 +554,18 @@ class BrowserUse:
):
key = 'scope_' + uuid4().hex
self._scopes[key] = Reference(tab=page._target_id, backend=int(node.get('backendDOMNodeId') or 0))
- scopes.append({'ref': key, 'label': str(node.get('name', {}).get('value') or role)[:200]})
- return {'version': 1, 'text': '', 'elements': [], 'omitted': [], 'scopes': scopes[:100]}
+ scopes.append(
+ {
+ 'ref': key,
+ 'label': str(node.get('name', {}).get('value') or role)
+ .encode('utf-8')[:256]
+ .decode('utf-8', errors='ignore'),
+ }
+ )
+ if len(json.dumps(scopes)) > 9800 or len(scopes) > 100:
+ scopes.pop()
+ return {'version': 1, 'text': '', 'elements': [], 'omitted': ['report_limit'], 'scopes': scopes}
+ return {'version': 1, 'text': '', 'elements': [], 'omitted': [], 'scopes': scopes}
ref = self._scopes.get(root)
if ref is None or ref.tab != page._target_id:
raise api.BrowserError('Unknown or stale read scope.')
@@ -483,12 +575,15 @@ class BrowserUse:
queue = [(node, 0)]
lines, elements = [], []
visited = 0
+ truncated = False
while queue and len(lines) < 200 and visited < 2000:
node, depth = queue.pop(0)
visited += 1
role = node.get('role', {}).get('value', '')
name = str(node.get('name', {}).get('value', ''))
prefix = ''
+ previous_elements = len(elements)
+ previous_lines = len(lines)
if not node.get('ignored') and role != 'InlineTextBox':
if role in {'button', 'textbox', 'combobox', 'checkbox', 'radio', 'slider', 'listbox', 'link'} and node.get(
'backendDOMNodeId'
@@ -498,7 +593,8 @@ class BrowserUse:
self._refs[key] = Reference(tab=page._target_id, backend=int(node.get('backendDOMNodeId') or 0))
prefix = f'[{key}] '
if role != 'link':
- elements.append({'ref': key, 'role': role, 'label': name[:200]})
+ label = (name.strip() or f'Unnamed {role}').encode('utf-8')[:256].decode('utf-8', errors='ignore')
+ elements.append({'ref': key, 'role': role, 'label': label})
if args['filter'] == 'all' or role in {
'button',
'textbox',
@@ -510,13 +606,25 @@ class BrowserUse:
'link',
}:
lines.append(f'{prefix}{role} {name[:500]}')
+ if (
+ len(
+ json.dumps(
+ {'version': 1, 'text': '\n'.join(lines), 'elements': elements, 'omitted': ['report_limit'], 'scopes': []}
+ )
+ )
+ > 10000
+ ):
+ del lines[previous_lines:]
+ del elements[previous_elements:]
+ truncated = True
+ break
if depth < args['depth']:
queue.extend((nodes[key], depth + 1) for key in node.get('childIds', []) if key in nodes)
return {
'version': 1,
'text': '\n'.join(lines),
'elements': elements,
- 'omitted': ['report_limit'] if queue else [],
+ 'omitted': ['report_limit'] if queue or truncated else [],
'scopes': [],
}
diff --git a/examples/integrations/openai/quickstart.py b/examples/integrations/openai/quickstart.py
index 7ae1130e8..ca493c342 100644
--- a/examples/integrations/openai/quickstart.py
+++ b/examples/integrations/openai/quickstart.py
@@ -1,48 +1,47 @@
-"""Read a page through Browser Use actions and the OpenAI tool runner."""
+"""Use OpenAI browser tools and Bash with a Cloud browser."""
import asyncio
import os
+from pathlib import Path
from urllib.parse import urlsplit
from openai import AsyncOpenAI
-from openai.lib.browser import AsyncBrowserToolset
+from openai.lib.browser import BROWSER_MEMBERS, AsyncBrowserToolset
from openai.lib.responses_tool_runner import AsyncResponsesToolRunner
-from browser_use.integrations.openai import BrowserUse
+from browser_use.integrations.openai import Bash, BrowserUse
async def allow_url(context, url):
parsed = urlsplit(url)
- return parsed.scheme == 'https' and parsed.hostname == 'example.com' and not parsed.username and not parsed.password
+ return parsed.scheme in {'http', 'https'} and not parsed.username and not parsed.password
async def confirm(context):
- return context['member'] in {'navigate', 'new_tab'}
+ return True
async def main():
model = os.environ['OPENAI_MODEL']
- async with AsyncOpenAI(max_retries=0, timeout=60) as client, BrowserUse(use_cloud=True) as driver:
+ async with AsyncOpenAI(max_retries=5, timeout=180) as client, BrowserUse(use_cloud=True) as driver:
tools = AsyncBrowserToolset(
driver,
- members=['new_tab', 'navigate', 'get_page_text'],
+ members=BROWSER_MEMBERS,
confirm=confirm,
url_policy=allow_url,
)
- runner = AsyncResponsesToolRunner(browser=tools)
+ runner = AsyncResponsesToolRunner(browser=tools, functions=[Bash(output_dir=Path('outputs'))])
try:
response = await runner.run(
client,
{
'model': model,
'store': False,
- 'input': 'Open https://example.com and read its heading and explanation. Do not follow links.',
+ 'input': 'Open https://example.com, follow Learn more, and save a source-backed summary in summary.md using Bash.',
+ 'context_management': [{'type': 'compaction', 'compact_threshold': 60000}],
},
- max_steps=6,
+ max_steps=100,
)
- text = await driver.get_page_text({'tab_id': None})
- if 'documentation examples' not in text:
- raise RuntimeError('The browser did not show the expected page.')
print(response.output_text)
finally:
runner.close()
diff --git a/tests/ci/integrations/test_openai_browser_use.py b/tests/ci/integrations/test_openai_browser_use.py
index 5852aff3b..fb8f1f47a 100644
--- a/tests/ci/integrations/test_openai_browser_use.py
+++ b/tests/ci/integrations/test_openai_browser_use.py
@@ -3,6 +3,8 @@ from __future__ import annotations
import base64
import json
import re
+import sys
+import venv
from collections.abc import Awaitable, Callable
from io import BytesIO
from typing import cast
@@ -65,23 +67,40 @@ async def test_driver_reads_and_changes_real_form(driver):
@pytest.mark.asyncio
[email protected]('change', ['navigate', 'detach', 'new_tab', 'reread'])
[email protected]('change', ['navigate', 'page_navigation', 'detach', 'new_tab'])
async def test_driver_rejects_stale_or_cross_tab_reference(driver, change):
page = await fixture_page(driver, '<button id="button">Original</button>')
text = await driver.read_page({'tab_id': None, 'filter': 'interactive'})
target = reference(text, 'Original')
if change == 'navigate':
await driver.navigate({'tab_id': None, 'url': 'about:blank'})
+ elif change == 'page_navigation':
+ await page.goto('about:blank?next')
elif change == 'detach':
await page.evaluate("() => document.querySelector('#button').remove()")
elif change == 'new_tab':
await driver.new_tab({'url': 'about:blank'})
- else:
- await driver.read_page({'tab_id': None, 'filter': 'interactive'})
with pytest.raises(BrowserError):
await driver.click({'tab_id': None, 'target': target})
+async def test_reads_preserve_current_references_and_report_page_identity(driver):
+ page = await fixture_page(
+ driver, '<button onclick="document.title=&quot;Clicked&quot;">Original</button><button>Other</button>'
+ )
+ await page.evaluate("() => { document.title = 'Fixture'; location.hash = 'full-path'; }")
+ first = await driver.read_page({'tab_id': None, 'filter': 'interactive'})
+ target = reference(first, 'Original')
+ assert 'Fixture' in first
+ assert 'about:blank#full-path' in first
+ await driver.find({'tab_id': None, 'query': 'Other'})
+ second = await driver.read_page({'tab_id': None, 'filter': 'interactive'})
+ assert reference(second, 'Original') == target
+ await driver.click({'tab_id': None, 'target': target})
+ assert await page.evaluate('() => document.title') == 'Clicked'
+
+
@pytest.mark.asyncio
async def test_toolset_denies_click_without_changing_page(driver):
page = await fixture_page(driver, '<button onclick="this.textContent=\'Changed\'">Original</button>')
@@ -227,17 +246,30 @@ async def test_screenshot_and_pointer_share_css_pixels_at_double_scale(driver):
@pytest.mark.asyncio
-async def test_openai_bash_creates_deliverable_without_inherited_secrets(tmp_path, monkeypatch):
[email protected]('custom_python', [False, True])
+async def test_openai_bash_creates_deliverable_without_inherited_secrets(tmp_path, monkeypatch, custom_python):
from browser_use.integrations.openai import Bash
from browser_use.integrations.openai.bash import BashInput
monkeypatch.setenv('OPENAI_API_KEY', 'private-test-value')
- tool = Bash(output_dir=tmp_path)
+ python = None
+ expected_prefix = sys.prefix
+ if custom_python:
+ environment = tmp_path / 'python-env'
+ venv.EnvBuilder().create(environment)
+ python = environment / 'bin' / 'python'
+ expected_prefix = str(environment)
+ tool = Bash(output_dir=tmp_path, python_executable=python)
execute = cast(Callable[[BashInput], Awaitable[str]], tool.execute)
result = json.loads(await execute(BashInput(command='printf result > report.txt; printenv OPENAI_API_KEY || true')))
assert result['exit_code'] == 0
assert result['output'] == ''
assert (tmp_path / 'report.txt').read_text() == 'result'
+ result = json.loads(await execute(BashInput(command="python -c 'import sys; print(sys.prefix)'")))
+ assert result['exit_code'] == 0, result
+ assert result['output'].strip() == expected_prefix
+ result = json.loads(await execute(BashInput(command='printenv HOME')))
+ assert result['output'].strip() == str(tmp_path / '.tmp' / 'home')
async def dispatch(driver, member, arguments):
@@ -264,6 +296,32 @@ async def dispatch(driver, member, arguments):
tools.close()
[email protected](('label', 'scope_label'), [('', 'main'), ('猫' * 100, 'main'), ('Save', '猫' * 100)])
+async def test_scoped_reads_accept_unnamed_and_multibyte_controls(driver, label, scope_label):
+ await fixture_page(driver, '<main aria-label="' + scope_label + '"><button>' + label + '</button></main>')
+ args = {'tab_id': None, 'root': None, 'depth': 8, 'filter': 'all'}
+ discovered = await dispatch(driver, 'read_page_scoped', args)
+ assert all(0 < len(scope['label'].encode()) <= 256 for scope in discovered['scopes'])
+ args['root'] = next(scope['ref'] for scope in discovered['scopes'] if scope['label'].startswith(scope_label[:1]))
+ read = await dispatch(driver, 'read_page_scoped', args)
+ assert len(read['elements']) == 1
+ element = read['elements'][0]
+ assert 0 < len(element['label'].encode()) <= 256
+ assert element['label'] == (label or 'Unnamed button').encode()[:256].decode('utf-8', errors='ignore')
+ await driver.click({'tab_id': None, 'target': element['ref']})
+
+
+async def test_scoped_read_bounds_text_and_elements_together(driver):
+ await fixture_page(driver, '<main>' + ('<button>' + 'large label ' * 30 + '</button>') * 100 + '</main>')
+ args = {'tab_id': None, 'root': None, 'depth': 8, 'filter': 'all'}
+ discovered = await dispatch(driver, 'read_page_scoped', args)
+ args['root'] = next(scope['ref'] for scope in discovered['scopes'] if scope['label'] == 'main')
+ read = await dispatch(driver, 'read_page_scoped', args)
+ assert len(json.dumps(read)) <= 10000
+ assert read['omitted'] == ['report_limit']
+ assert read['elements']
+
+
@pytest.mark.asyncio
async def test_scoped_read_limits_region_and_rejects_stale_scope(driver):
await fixture_page(driver, '<main><button>Inside</button></main><nav><button>Outside</button></nav>')
@@ -367,13 +425,13 @@ async def test_page_read_bounds_large_observations(driver):
+ '<button>Last</button></main>',
)
text = await driver.read_page({'tab_id': None, 'filter': 'all'})
- assert len(text) <= 50000
+ assert len(text) <= 10000
assert 'truncated' in text.lower()
plain = await driver.get_page_text({'tab_id': None})
- assert len(plain) <= 50000
+ assert len(plain) <= 10000
assert 'truncated' in plain.lower()
script = await driver.run_script({'tab_id': None, 'source': 'document.body.innerText', 'timeout_ms': 1000})
- assert len(script) <= 50000
+ assert len(script) <= 10000
assert 'truncated' in script.lower()
assert await driver.run_script({'tab_id': None, 'source': '1 + 2', 'timeout_ms': 1000}) == '3'
controls = await driver.find({'tab_id': None, 'query': 'Last'})
@@ -385,3 +443,90 @@ async def test_page_read_does_not_expose_password_value(driver):
text = await driver.read_page({'tab_id': None, 'filter': 'all'})
assert 'private-password-value' not in text
assert 'Password' in text
+
+
+async def test_state_recovers_after_temporary_transport_failure(driver, monkeypatch):
+ page = await fixture_page(driver, '<p>Preserved</p>')
+ target = driver.browser.cdp_client.send.Target
+ original = target.getTargets
+
+ async def fail_once(*args, **kwargs):
+ monkeypatch.setattr(target, 'getTargets', original)
+ raise TimeoutError('transport stalled')
+
+ monkeypatch.setattr(target, 'getTargets', fail_once)
+ state = await driver.state()
+ assert page._target_id in {tab['id'] for tab in state['tabs']}
+ assert await driver.get_page_text({'tab_id': page._target_id}) == 'Preserved'
+
+
+async def test_diagnostics_timeout_does_not_block_page_read(driver, monkeypatch):
+ import asyncio
+
+ import browser_use.integrations.openai.browser_use as implementation
+
+ page = await fixture_page(driver, '<p>Readable without diagnostics</p>')
+ driver._diagnostic_sessions.pop(page._target_id)
+ client = driver._diagnostic_client
+ cancelled = asyncio.Event()
+
+ async def never_attach(*args, **kwargs):
+ try:
+ await asyncio.Event().wait()
+ finally:
+ cancelled.set()
+
+ monkeypatch.setattr(implementation, '_DIAGNOSTIC_TIMEOUT_S', 0.05)
+ monkeypatch.setattr(client.send.Target, 'attachToTarget', never_attach)
+ assert await driver.get_page_text({'tab_id': page._target_id}) == 'Readable without diagnostics'
+ assert cancelled.is_set()
+ assert driver._diagnostic_client is None
+ assert not driver._diagnostic_sessions
+ assert not driver._diagnostics
+
+
[email protected]('failure', ['timeout', 'corrupt'])
+async def test_screenshot_retries_temporary_capture_failure(driver, monkeypatch, failure):
+ page = await fixture_page(driver, '<p>Screenshot retry</p>')
+ target = driver.browser.cdp_client.send.Page
+ original = target.captureScreenshot
+
+ async def fail_once(*args, **kwargs):
+ monkeypatch.setattr(target, 'captureScreenshot', original)
+ if failure == 'corrupt':
+ return {'data': base64.b64encode(b'not an image').decode()}
+ raise TimeoutError('capture stalled')
+
+ monkeypatch.setattr(target, 'captureScreenshot', fail_once)
+ result = await driver.screenshot({'tab_id': page._target_id})
+ with Image.open(BytesIO(base64.b64decode(result['image_url'].split(',', 1)[1]))) as image:
+ assert image.width > 0
+ assert image.height > 0
+
+
+async def test_crashed_page_returns_error_and_other_tabs_remain_usable(driver):
+ page = await fixture_page(driver, '<p>Lost page</p>')
+ driver._target_crashed({'targetId': page._target_id})
+ with pytest.raises(BrowserError, match='page crashed'):
+ await driver.get_page_text({'tab_id': page._target_id})
+ other = await fixture_page(driver, '<p>Still usable</p>')
+ assert await driver.get_page_text({'tab_id': other._target_id}) == 'Still usable'
+
+
+async def test_page_read_retries_temporary_accessibility_failure(driver, monkeypatch):
+ await fixture_page(driver, '<button>Recovered control</button>')
+ target = driver.browser.cdp_client.send.Accessibility
+ original = target.getFullAXTree
+
+ async def fail_once(*args, **kwargs):
+ monkeypatch.setattr(target, 'getFullAXTree', original)
+ raise TimeoutError('accessibility stalled')
+
+ monkeypatch.setattr(target, 'getFullAXTree', fail_once)
+ text = await driver.read_page({'tab_id': None, 'filter': 'interactive'})
+ assert reference(text, 'Recovered control')