Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
feat: implement remaining OpenAI browser tools
Magnus Mueller44a38ca26a6d
diff --git a/browser_use/integrations/openai/browser_use.py b/browser_use/integrations/openai/browser_use.pyindex 1accea61c..e6fd72539 100644--- a/browser_use/integrations/openai/browser_use.py+++ b/browser_use/integrations/openai/browser_use.py@@ -5,9 +5,14 @@ from __future__ import annotationsimport asyncioimport base64import json+from collections import dequefrom io import BytesIO+from pathlib import Pathfrom typing import Any+from urllib.parse import urlsplit+from uuid import uuid4+from cdp_use import CDPClientfrom PIL import Imagefrom pydantic import BaseModel@@ -21,6 +26,18 @@ class Reference(BaseModel):backend: int+class FileGrant(BaseModel):+ """Bind one approved transfer to its original browser document."""++ operation: str+ tab: str+ document: int+ path: Path+ target: int | None = None+ url: str | None = None+ content: bytes = b''++class BrowserUse:"""Own a browser session or borrow an already connected session."""@@ -33,6 +50,11 @@ class BrowserUse:self._refs: dict[str, Reference] = {}self._next_ref = 0self._active: str | None = None+ self._scopes: dict[str, Reference] = {}+ self._grants: dict[str, FileGrant] = {}+ self._diagnostic_client: CDPClient | None = None+ self._diagnostic_sessions: dict[str, str] = {}+ self._diagnostics: dict[str, dict[str, deque[str]]] = {}async def __aenter__(self) -> BrowserUse:if self._closed:@@ -53,6 +75,11 @@ class BrowserUse:async def close(self) -> None:"""Close only the browser that this driver created."""if not self._closed:+ if self._diagnostic_client:+ await self._diagnostic_client.stop()+ self._diagnostic_client = None+ self._grants.clear()+ self._scopes.clear()if self._owns_browser:try:cloud = self.browser._cloud_browser_client@@ -95,6 +122,7 @@ class BrowserUse:raise api.BrowserError('Unknown tab.')session = await self.browser.get_or_create_cdp_session(target, focus=True)self._active = target+ await self._start_diagnostics(target)return Page(self.browser, target, session.session_id)async def _element(self, page: Page, target: str):@@ -352,3 +380,209 @@ class BrowserUse:raise api.BrowserError('Page script failed.')value = result.get('result', {}).get('value')return value if isinstance(value, str) else json.dumps(value)++ async def _start_diagnostics(self, target: str) -> None:+ if self._diagnostic_client is None:+ self._diagnostic_client = CDPClient(+ self.browser.cdp_client.url, additional_headers=self.browser.cdp_client.additional_headers+ )+ await self._diagnostic_client.start()+ self._diagnostic_client.register.Runtime.consoleAPICalled(self._console_event)+ self._diagnostic_client.register.Network.responseReceived(self._network_event)+ if target not in self._diagnostic_sessions:+ attached = await self._diagnostic_client.send.Target.attachToTarget({'targetId': target, 'flatten': True})+ session = attached['sessionId']+ self._diagnostic_sessions[target] = session+ self._diagnostics[session] = {'console': deque(maxlen=100), 'network': deque(maxlen=100)}+ await self._diagnostic_client.send.Runtime.enable(session_id=session)+ await self._diagnostic_client.send.Network.enable(session_id=session)++ def _console_event(self, event: Any, session_id: str | None) -> None:+ if session_id in self._diagnostics:+ values = [str(arg.get('value', arg.get('type', '')))[:512] for arg in event.get('args', [])[:10]]+ self._diagnostics[session_id]['console'].append(event['type'] + ': ' + ' '.join(values))++ def _network_event(self, event: Any, session_id: str | None) -> None:+ if session_id in self._diagnostics:+ response = event['response']+ url = urlsplit(response['url'])+ self._diagnostics[session_id]['network'].append(+ f'{response["status"]} {url.scheme}://{url.hostname or ""}{url.path}'[:1024]+ )++ async def read_diagnostics(self, args: api.ReadDiagnosticsInput) -> api.BrowserReadV1:+ """Return bounded diagnostics without network headers or request bodies."""+ page = await self._page(args['tab_id'])+ records = self._diagnostics[self._diagnostic_sessions[page._target_id]][args['source']]+ return {+ 'version': 1,+ 'text': '\n'.join(list(records)[-args['limit'] :]),+ 'elements': [],+ 'omitted': ['report_limit'] if len(records) > args['limit'] else [],+ }++ async def read_page_scoped(self, args: api.ReadPageScopedInput) -> api.BrowserScopedReadV1:+ """Discover semantic regions or read their current accessibility subtree."""+ page = await self._page(args['tab_id'])+ result = await self.browser.cdp_client.send.Accessibility.getFullAXTree(session_id=await page.session_id)+ nodes = {node['nodeId']: node for node in result['nodes']}+ root = args['root']+ if root is None:+ self._scopes = {key: ref for key, ref in self._scopes.items() if ref.tab != page._target_id}+ scopes = []+ for node in nodes.values():+ role = node.get('role', {}).get('value')+ if role in {'RootWebArea', 'main', 'navigation', 'region', 'complementary', 'contentinfo', 'banner'} and node.get(+ 'backendDOMNodeId'+ ):+ key = 'scope_' + uuid4().hex+ self._scopes[key] = Reference(tab=page._target_id, backend=int(node.get('backendDOMNodeId') or 0))+ scopes.append({'ref': key, 'label': str(node.get('name', {}).get('value') or role)[:200]})+ return {'version': 1, 'text': '', 'elements': [], 'omitted': [], 'scopes': scopes[:100]}+ ref = self._scopes.get(root)+ if ref is None or ref.tab != page._target_id:+ raise api.BrowserError('Unknown or stale read scope.')+ node = next((n for n in nodes.values() if n.get('backendDOMNodeId') == ref.backend), None)+ if node is None:+ raise api.BrowserError('Unknown or stale read scope.')+ queue = [(node, 0)]+ lines, elements = [], []+ visited = 0+ while queue and len(lines) < 200 and visited < 2000:+ node, depth = queue.pop(0)+ visited += 1+ role = node.get('role', {}).get('value', '')+ name = str(node.get('name', {}).get('value', ''))+ prefix = ''+ if not node.get('ignored') and role != 'InlineTextBox':+ if role in {'button', 'textbox', 'combobox', 'checkbox', 'radio', 'slider', 'listbox', 'link'} and node.get(+ 'backendDOMNodeId'+ ):+ self._next_ref += 1+ key = f'ref_{self._next_ref}'+ self._refs[key] = Reference(tab=page._target_id, backend=int(node.get('backendDOMNodeId') or 0))+ prefix = f'[{key}] '+ if role != 'link':+ elements.append({'ref': key, 'role': role, 'label': name[:200]})+ if args['filter'] == 'all' or role in {+ 'button',+ 'textbox',+ 'combobox',+ 'checkbox',+ 'radio',+ 'slider',+ 'listbox',+ 'link',+ }:+ lines.append(f'{prefix}{role} {name[:500]}')+ if depth < args['depth']:+ queue.extend((nodes[key], depth + 1) for key in node.get('childIds', []) if key in nodes)+ return {+ 'version': 1,+ 'text': '\n'.join(lines),+ 'elements': elements,+ 'omitted': ['report_limit'] if queue else [],+ 'scopes': [],+ }++ async def grant_upload(self, path: str | Path, *, target: str, tab_id: str | None = None) -> str:+ """Approve one file snapshot for a specific input and document."""+ page = await self._page(tab_id)+ element = await self._element(page, target)+ if await element.get_attribute('type') != 'file':+ raise api.BrowserError('Upload target must be a file input.')+ path = Path(path).resolve()+ with path.open('rb') as source:+ content = source.read(10_000_001)+ if len(content) > 10_000_000:+ raise api.BrowserError('File exceeds the 10 MB transfer limit.')+ document = await self.browser.cdp_client.send.DOM.getDocument(session_id=await page.session_id)+ key = uuid4().hex+ self._grants[key] = FileGrant(+ operation='upload',+ tab=page._target_id,+ document=document['root']['backendNodeId'],+ path=path,+ target=self._refs[target].backend,+ content=content,+ )+ return key++ async def grant_download(self, url: str, path: str | Path, *, tab_id: str | None = None) -> str:+ """Approve one browser fetch into a new application-side file."""+ if urlsplit(url).scheme not in {'http', 'https', 'blob'}:+ raise api.BrowserError('Download requires an HTTP, HTTPS, or blob URL.')+ page = await self._page(tab_id)+ document = await self.browser.cdp_client.send.DOM.getDocument(session_id=await page.session_id)+ path = Path(path).resolve()+ if path.exists():+ raise api.BrowserError('Download destination already exists.')+ key = uuid4().hex+ self._grants[key] = FileGrant(+ operation='download', tab=page._target_id, document=document['root']['backendNodeId'], path=path, url=url+ )+ return key++ async def _consume_grant(self, key: str, operation: str) -> tuple[FileGrant, Page]:+ grant = self._grants.pop(key, None)+ if grant is None or grant.operation != operation:+ raise api.BrowserError('Unknown, consumed, or mismatched file grant.')+ page = await self._page(grant.tab)+ document = await self.browser.cdp_client.send.DOM.getDocument(session_id=await page.session_id)+ if document['root']['backendNodeId'] != grant.document:+ raise api.BrowserError('File grant belongs to an earlier document.')+ return grant, page++ async def upload_file(self, args: api.BrowserFileInput) -> api.BrowserFileOutcome:+ """Transfer approved bytes into the remote browser file input."""+ grant, page = await self._consume_grant(args['grant_id'], 'upload')+ if grant.target is None:+ raise api.BrowserError('Upload grant has no target.')+ element = await page.get_element(grant.target)+ await element.evaluate(+ """(encoded, name) => {+ if (!this.isConnected || this.type !== 'file') throw new Error('Invalid upload target');+ const bytes = Uint8Array.from(atob(encoded), c => c.charCodeAt(0));+ const transfer = new DataTransfer();+ transfer.items.add(new File([bytes], name));+ this.files = transfer.files;+ this.dispatchEvent(new Event('input', {bubbles: true}));+ this.dispatchEvent(new Event('change', {bubbles: true}));+ }""",+ base64.b64encode(grant.content).decode(),+ grant.path.name,+ )+ return {'version': 1, 'status': 'completed', 'effect': 'completed', 'cleanup': 'completed'}++ async def download_file(self, args: api.BrowserFileInput) -> api.BrowserFileOutcome:+ """Fetch approved bytes through the browser and create the destination."""+ grant, page = await self._consume_grant(args['grant_id'], 'download')+ encoded = await page.evaluate(+ """(url) => (async () => {+ const response = await fetch(url, {redirect: 'error', signal: AbortSignal.timeout(30000)});+ if (!response.ok) throw new Error('Download request failed');+ const reader = response.body.getReader();+ const chunks = [];+ let size = 0;+ try {+ while (true) {+ const {done, value} = await reader.read();+ if (done) break;+ size += value.length;+ if (size > 10000000) throw new Error('Download exceeds 10 MB');+ chunks.push(value);+ }+ } finally { await reader.cancel(); }+ let binary = '';+ for (const chunk of chunks) {+ for (let i = 0; i < chunk.length; i += 8192)+ binary += String.fromCharCode(...chunk.subarray(i, i + 8192));+ }+ return btoa(binary);+ })()""",+ grant.url,+ )+ content = base64.b64decode(encoded, validate=True)+ with grant.path.open('xb') as destination:+ destination.write(content)+ return {'version': 1, 'status': 'completed', 'effect': 'completed', 'cleanup': 'completed'}diff --git a/tests/ci/integrations/test_openai_browser_use.py b/tests/ci/integrations/test_openai_browser_use.pyindex ce7adfc50..28595320a 100644--- a/tests/ci/integrations/test_openai_browser_use.py+++ b/tests/ci/integrations/test_openai_browser_use.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport base64import jsonimport re@@ -198,6 +199,8 @@ async def test_sdk_accepts_uncommitted_tab_without_url(driver, monkeypatch):)assert isinstance(output['output'], list)assert output['output'][0]['type'] == 'input_text'+ assert isinstance(output['output'], list)+ assert output['output'][0]['type'] == 'input_text'result = json.loads(output['output'][0]['text'])assert result['ok'] is Trueassert result['state']['tabs']@@ -236,3 +239,116 @@ async def test_openai_bash_creates_deliverable_without_inherited_secrets(tmp_patassert result['exit_code'] == 0assert result['output'] == ''assert (tmp_path / 'report.txt').read_text() == 'result'+++async def dispatch(driver, member, arguments):+ async def approve(context):+ return True++ tools = AsyncBrowserToolset(driver, members=[member], confirm=approve)+ try:+ output = await tools.execute(+ ResponseFunctionToolCall(+ id='fc_' + member,+ call_id='call_' + member,+ type='function_call',+ name='browser_' + member,+ arguments=json.dumps(arguments),+ )+ )+ assert isinstance(output['output'], list)+ assert output['output'][0]['type'] == 'input_text'+ result = json.loads(output['output'][0]['text'])+ assert result['ok'], result+ return result['result']+ finally:+ tools.close()+++async def test_scoped_read_limits_region_and_rejects_stale_scope(driver):+ await fixture_page(driver, '<main><button>Inside</button></main><nav><button>Outside</button></nav>')+ discovered = await dispatch(driver, 'read_page_scoped', {'tab_id': None, 'root': None, 'depth': 8, 'filter': 'all'})+ root = next(scope['ref'] for scope in discovered['scopes'] if scope['label'] == 'main')+ read = await dispatch(driver, 'read_page_scoped', {'tab_id': None, 'root': root, 'depth': 8, 'filter': 'all'})+ assert 'Inside' in read['text']+ assert 'Outside' not in read['text']+ await driver.click({'tab_id': None, 'target': read['elements'][0]['ref']})+ with pytest.raises(BrowserError):+ await driver.click({'tab_id': None, 'target': root})+ await driver.navigate({'tab_id': None, 'url': 'about:blank'})+ with pytest.raises(BrowserError):+ await driver.read_page_scoped({'tab_id': None, 'root': root, 'depth': 8, 'filter': 'all'})+++async def test_diagnostics_reads_console_and_bounds_output(driver):+ page = await fixture_page(driver, '<p>Diagnostics</p>')+ await page.evaluate("() => {console.log('first'); console.warn('second');}")+ await driver._diagnostic_client.send.Runtime.evaluate(+ {'expression': '0'}, session_id=driver._diagnostic_sessions[page._target_id]+ )+ read = await dispatch(driver, 'read_diagnostics', {'tab_id': None, 'source': 'console', 'limit': 1})+ assert 'second' in read['text']+ assert 'first' not in read['text']+ assert 'report_limit' in read['omitted']+++async def test_file_grants_transfer_real_bytes_and_reject_reuse(driver, tmp_path):+ page = await fixture_page(driver, '<label>Upload<input type="file" id="file"></label>')+ source = tmp_path / 'source.txt'+ source.write_bytes(b'approved bytes\x00')+ text = await driver.read_page({'tab_id': None, 'filter': 'interactive'})+ grant = await driver.grant_upload(source, target=reference(text, 'Upload'))+ source.write_text('changed after approval')+ await dispatch(driver, 'upload_file', {'grant_id': grant})+ assert await page.evaluate("() => document.querySelector('#file').files[0].text()") == 'approved bytes\x00'+ with pytest.raises(BrowserError):+ await driver.upload_file({'grant_id': grant})+ url = await page.evaluate("() => URL.createObjectURL(new Blob(['download bytes']))")+ destination = tmp_path / 'download.txt'+ grant = await driver.grant_download(url, destination)+ await dispatch(driver, 'download_file', {'grant_id': grant})+ assert destination.read_bytes() == b'download bytes'+ with pytest.raises(BrowserError):+ await driver.download_file({'grant_id': grant})+++async def test_file_grant_rejects_navigation_and_existing_destination(driver, tmp_path):+ page = await fixture_page(driver, '<p>Download</p>')+ url = await page.evaluate("() => URL.createObjectURL(new Blob(['bytes']))")+ destination = tmp_path / 'download.txt'+ grant = await driver.grant_download(url, destination)+ await driver.navigate({'tab_id': None, 'url': 'about:blank'})+ with pytest.raises(BrowserError):+ await driver.download_file({'grant_id': grant})+ assert not destination.exists()+ destination.write_text('keep')+ with pytest.raises(BrowserError):+ await driver.grant_download(url, destination)+ assert destination.read_text() == 'keep'+++async def test_network_diagnostics_strip_query_and_capture_status(driver):+ async def respond(reader, writer):+ try:+ await reader.readuntil(b'\r\n\r\n')+ writer.write(b'HTTP/1.1 200 OK\r\nContent-Length: 2\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\nok')+ await writer.drain()+ finally:+ writer.close()+ await writer.wait_closed()++ server = await asyncio.start_server(respond, '127.0.0.1', 0)+ async with server:+ port = server.sockets[0].getsockname()[1]+ await fixture_page(driver, '<p>Before</p>')+ await driver.navigate({'tab_id': None, 'url': f'http://127.0.0.1:{port}/diagnostic?secret=hidden'})+ read = await dispatch(driver, 'read_diagnostics', {'tab_id': None, 'source': 'network', 'limit': 100})+ assert '200' in read['text']+ assert '/diagnostic' in read['text']+ assert 'secret' not in read['text']+ assert 'hidden' not in read['text']