Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Add OpenAI browser tool driver using Browser Use Actor
Magnus Mueller8173a07fd766
diff --git a/browser_use/integrations/openai/__init__.py b/browser_use/integrations/openai/__init__.pynew file mode 100644index 000000000..e41399a5d--- /dev/null+++ b/browser_use/integrations/openai/__init__.py@@ -0,0 +1,5 @@+"""Connect OpenAI browser tools to Browser Use actions."""++from .browser_use import BrowserUse++__all__ = ['BrowserUse']diff --git a/browser_use/integrations/openai/browser_use.py b/browser_use/integrations/openai/browser_use.pynew file mode 100644index 000000000..6e9c26db8--- /dev/null+++ b/browser_use/integrations/openai/browser_use.py@@ -0,0 +1,321 @@+"""Run OpenAI browser tools through Browser Use Actor methods."""++from __future__ import annotations++import asyncio+import base64+import json+from io import BytesIO+from typing import Any++from PIL import Image+from pydantic import BaseModel++from browser_use.actor.page import Page+from browser_use.browser.session import BrowserSession+from openai.lib import browser as api+++class Reference(BaseModel):+ tab: str+ document: int+ backend: int+++class BrowserUse:+ """Own a browser session or borrow an already connected session."""++ def __init__(self, browser: BrowserSession | None = None, *, use_cloud: bool = False):+ if browser is not None and use_cloud:+ raise ValueError('Pass a browser or use_cloud=True, not both.')+ self.browser = browser if browser is not None else BrowserSession(use_cloud=use_cloud)+ self._owns_browser = browser is None+ self._closed = False+ self._refs: dict[str, Reference] = {}+ self._next_ref = 0+ self._active: str | None = None++ async def __aenter__(self) -> BrowserUse:+ if self._closed:+ raise RuntimeError('The driver is closed.')+ try:+ if self._owns_browser:+ await self.browser.start()+ elif not self.browser.is_cdp_connected:+ raise RuntimeError('Start the borrowed browser before entering the driver.')+ return self+ except BaseException:+ await self.close()+ raise++ async def __aexit__(self, *_: Any) -> None:+ await self.close()++ async def close(self) -> None:+ """Close only the browser that this driver created."""+ if not self._closed:+ if self._owns_browser:+ await self.browser.kill()+ self._refs.clear()+ self._closed = True++ async def state(self) -> api.BrowserState:+ """Report current browser tabs without retaining stale tab state."""+ if self._closed:+ raise api.BrowserError('The driver is closed.')+ result = await self.browser.cdp_client.send.Target.getTargets()+ tabs = [t for t in result['targetInfos'] if t['type'] == 'page']+ ids = {t['targetId'] for t in tabs}+ if self._active not in ids:+ focus = self.browser.agent_focus_target_id+ self._active = focus if focus in ids else next(iter(ids), None)+ self._refs = {name: ref for name, ref in self._refs.items() if ref.tab in ids}+ return {+ 'tabs': [+ {'id': t['targetId'], 'url': t['url'], 'title': t['title'], 'active': t['targetId'] == self._active} for t in tabs+ ]+ }++ async def _page(self, tab_id: str | None) -> Page:+ state = await self.state()+ target = tab_id or self._active+ if target is None or target not in {tab['id'] for tab in state['tabs']}:+ raise api.BrowserError('Unknown tab.')+ session = await self.browser.get_or_create_cdp_session(target, focus=True)+ self._active = target+ return Page(self.browser, target, session.session_id)++ async def _document(self, page: Page) -> int:+ result = await self.browser.cdp_client.send.DOM.getDocument({'depth': 0}, session_id=await page.session_id)+ return result['root']['backendNodeId']++ async def _element(self, page: Page, target: str):+ ref = self._refs.get(target)+ if ref is None or ref.tab != page._target_id or ref.document != await self._document(page):+ raise api.BrowserError('Stale element reference; read the page again.')+ element = await page.get_element(ref.backend)+ connected = await element.evaluate('() => this.isConnected')+ if connected.casefold() != 'true':+ raise api.BrowserError('Detached element; read the page again.')+ return element++ async def navigate(self, args: api.NavigateInput) -> None:+ """Invalidate element references before navigating the selected tab."""+ page = await self._page(args['tab_id'])+ self._refs = {name: ref for name, ref in self._refs.items() if ref.tab != page._target_id}+ await page.goto(args['url'])++ async def read_page(self, args: api.ReadPageInput) -> str:+ """Read the accessibility tree and issue tab-bound element references."""+ page = await self._page(args['tab_id'])+ document = await self._document(page)+ self._refs = {name: ref for name, ref in self._refs.items() if ref.tab != page._target_id}+ result = await self.browser.cdp_client.send.Accessibility.getFullAXTree(session_id=await page.session_id)+ interactive = {+ 'button',+ 'link',+ 'textbox',+ 'searchbox',+ 'combobox',+ 'checkbox',+ 'radio',+ 'slider',+ 'spinbutton',+ 'menuitem',+ 'tab',+ 'option',+ 'listbox',+ 'switch',+ }+ lines = []+ for node in result['nodes']:+ role = node.get('role', {}).get('value', '')+ if node.get('ignored') or role == 'InlineTextBox':+ continue+ if args['filter'] == 'interactive' and role not in interactive:+ continue+ name = node.get('name', {}).get('value', '')+ prefix = ''+ backend = node.get('backendDOMNodeId')+ if backend and role in interactive:+ self._next_ref += 1+ ref = f'ref_{self._next_ref}'+ self._refs[ref] = Reference(tab=page._target_id, document=document, backend=backend)+ prefix = f'[{ref}] '+ lines.append(f'{prefix}{role} {json.dumps(name, ensure_ascii=False)}')+ return '\n'.join(lines)++ async def find(self, args: api.FindInput) -> str:+ """Find current interactive elements by visible label text."""+ text = await self.read_page({'tab_id': args['tab_id'], 'filter': 'interactive'})+ terms = args['query'].casefold().split()+ return '\n'.join(line for line in text.splitlines() if all(term in line.casefold() for term in terms))++ async def get_page_text(self, args: api.TabInput) -> str:+ """Read the selected page without returning form values."""+ page = await self._page(args['tab_id'])+ return await page.evaluate("() => document.body?.innerText || ''")++ async def click(self, args: api.ClickInput) -> None:+ """Click an element from the latest page read."""+ page = await self._page(args['tab_id'])+ await (await self._element(page, args['target'])).click()++ async def hover(self, args: api.ClickInput) -> None:+ """Move the pointer over a referenced element."""+ page = await self._page(args['tab_id'])+ await (await self._element(page, args['target'])).hover()++ async def scroll_to(self, args: api.ClickInput) -> None:+ """Scroll a referenced element into view."""+ page = await self._page(args['tab_id'])+ await (await self._element(page, args['target'])).scroll_into_view()++ async def type(self, args: api.TypeInput) -> None:+ """Fill a referenced element or type at the current focus."""+ page = await self._page(args['tab_id'])+ if args['target'] is not None:+ await (await self._element(page, args['target'])).fill(args['text'])+ else:+ await self.browser.cdp_client.send.Input.insertText({'text': args['text']}, session_id=await page.session_id)++ async def key(self, args: api.KeyInput) -> None:+ """Press a key combination in the selected tab."""+ page = await self._page(args['tab_id'])+ await page.press('+'.join(args['keys']))++ async def select_option(self, args: api.SelectOptionInput) -> None:+ """Select values in a referenced native select element."""+ page = await self._page(args['tab_id'])+ await (await self._element(page, args['target'])).select_option(args['values'])++ async def scroll(self, args: api.ScrollInput) -> None:+ """Scroll by the requested number of pixels."""+ page = await self._page(args['tab_id'])+ amount = args['amount'] * (-1 if args['direction'] in {'up', 'left'} else 1)+ horizontal = args['direction'] in {'left', 'right'}+ await (await page.mouse).scroll(delta_x=amount if horizontal else 0, delta_y=0 if horizontal else amount)++ async def wait_for(self, args: api.WaitForInput) -> None:+ """Wait for visible text within the requested deadline."""+ page = await self._page(args['tab_id'])+ result = await page.evaluate(+ """(text, timeout) => new Promise(resolve => {+ let timer;+ const observer = new MutationObserver(check);+ function finish(found) { observer.disconnect(); clearTimeout(timer); resolve(found); }+ function check() { if (document.body?.innerText.includes(text)) finish(true); }+ observer.observe(document, {subtree: true, childList: true, characterData: true, attributes: true});+ timer = setTimeout(() => finish(false), timeout);+ check();+ })""",+ args['text'],+ args['timeout_ms'],+ )+ if result.casefold() != 'true':+ raise api.BrowserError('Visible text did not appear before the deadline.')++ async def screenshot(self, args: api.TabInput) -> dict[str, str]:+ """Capture the selected viewport as a native image input."""+ page = await self._page(args['tab_id'])+ return {'type': 'input_image', 'image_url': 'data:image/png;base64,' + await page.screenshot(), 'detail': 'original'}++ async def zoom(self, args: api.ZoomInput) -> dict[str, str]:+ """Crop a screenshot without changing the pointer coordinate origin."""+ result = await self.screenshot({'tab_id': args['tab_id']})+ with Image.open(BytesIO(base64.b64decode(result['image_url'].split(',', 1)[1]))) as image:+ left, top, right, bottom = args['region']+ if left < 0 or top < 0 or right > image.width or bottom > image.height or right <= left or bottom <= top:+ raise api.BrowserError('Crop exceeds the screenshot bounds.')+ output = BytesIO()+ image.crop((left, top, right, bottom)).save(output, format='PNG')+ result['image_url'] = 'data:image/png;base64,' + base64.b64encode(output.getvalue()).decode()+ return result++ async def new_tab(self, args: api.NewTabInput) -> None:+ """Open a tab and select it for subsequent actions."""+ page = await self.browser.new_page(args['url'])+ self._active = page._target_id++ async def switch_tab(self, args: api.TabInput) -> None:+ """Activate an existing tab."""+ page = await self._page(args['tab_id'])+ await self.browser.cdp_client.send.Target.activateTarget({'targetId': page._target_id})++ async def close_tab(self, args: api.TabInput) -> None:+ """Close only the selected tab."""+ page = await self._page(args['tab_id'])+ await self.browser.close_page(page)++ async def _bounds(self, page: Page, points: list[tuple[int, int]]) -> None:+ metrics = await self.browser.cdp_client.send.Page.getLayoutMetrics(session_id=await page.session_id)+ viewport = metrics['cssVisualViewport']+ if any(x < 0 or y < 0 or x >= viewport['clientWidth'] or y >= viewport['clientHeight'] for x, y in points):+ raise api.BrowserError('Pointer coordinates exceed the viewport.')++ async def click_at(self, args: api.ClickAtInput, *, cancelled=None) -> None:+ """Click full-viewport coordinates with the requested modifiers."""+ page = await self._page(args['tab_id'])+ await self._bounds(page, [(args['x'], args['y'])])+ if cancelled and cancelled():+ raise asyncio.CancelledError()+ await (await page.mouse).click(+ args['x'], args['y'], button=args['button'], click_count=args['click_count'], modifiers=args['keys']+ )++ async def hover_at(self, args: api.HoverAtInput) -> None:+ """Move the pointer to full-viewport coordinates."""+ page = await self._page(args['tab_id'])+ await self._bounds(page, [(args['x'], args['y'])])+ await (await page.mouse).move(args['x'], args['y'])++ async def drag_path(self, args: api.DragPathInput, *, cancelled=None) -> None:+ """Drag through a path and release the pointer after failure."""+ page = await self._page(args['tab_id'])+ points = [(point['x'], point['y']) for point in args['path']]+ await self._bounds(page, points)+ mouse = await page.mouse+ await mouse.move(*points[0], modifiers=args['keys'])+ try:+ if cancelled and cancelled():+ raise asyncio.CancelledError()+ await mouse.down(modifiers=args['keys'])+ for x, y in points[1:]:+ if cancelled and cancelled():+ raise asyncio.CancelledError()+ await mouse.move(x, y, modifiers=args['keys'])+ finally:+ await mouse.up(modifiers=args['keys'])++ async def hold_keys(self, args: api.HoldKeysInput, *, cancelled=None) -> None:+ """Hold a key combination for the requested duration."""+ page = await self._page(args['tab_id'])+ if cancelled and cancelled():+ raise asyncio.CancelledError()+ await page.hold_key('+'.join(args['keys']), args['duration_ms'] / 1000)++ async def set_checked(self, args: api.SetCheckedInput) -> None:+ """Set a native checkbox without toggling an already matching state."""+ page = await self._page(args['tab_id'])+ element = await self._element(page, args['target'])+ kind = await element.get_attribute('type')+ if kind not in {'checkbox', 'radio'}:+ raise api.BrowserError('The target is not a native checkbox or radio button.')+ if kind == 'radio' and not args['checked']:+ raise api.BrowserError('Clear a radio button by selecting another option.')+ current = await element.evaluate('() => this.checked')+ if (current.casefold() == 'true') != args['checked']:+ await element.click()++ async def run_script(self, args: api.RunScriptInput) -> str:+ """Run an explicitly enabled page script within the caller deadline."""+ page = await self._page(args['tab_id'])+ async with asyncio.timeout(args['timeout_ms'] / 1000):+ result = await self.browser.cdp_client.send.Runtime.evaluate(+ {'expression': args['source'], 'returnByValue': True, 'awaitPromise': True},+ session_id=await page.session_id,+ )+ if result.get('exceptionDetails'):+ raise api.BrowserError('Page script failed.')+ value = result.get('result', {}).get('value')+ return value if isinstance(value, str) else json.dumps(value)diff --git a/examples/integrations/openai/quickstart.py b/examples/integrations/openai/quickstart.pynew file mode 100644index 000000000..54fb3140e--- /dev/null+++ b/examples/integrations/openai/quickstart.py@@ -0,0 +1,53 @@+"""Read a page through Browser Use actions and the OpenAI tool runner."""++import asyncio+import os+from urllib.parse import urlsplit++from openai import AsyncOpenAI+from openai.lib.browser import AsyncBrowserToolset+from openai.lib.responses_tool_runner import AsyncResponsesToolRunner++from browser_use.integrations.openai import BrowserUse+++async def allow_url(context, url):+ parsed = urlsplit(url)+ return parsed.scheme == 'https' and parsed.hostname == 'example.com' and not parsed.username and not parsed.password+++async def confirm(context):+ return context['member'] in {'navigate', 'new_tab'}+++async def main():+ model = os.environ['OPENAI_MODEL']+ async with AsyncOpenAI(max_retries=0, timeout=60) as client, BrowserUse(use_cloud=True) as driver:+ tools = AsyncBrowserToolset(+ driver,+ members=['new_tab', 'navigate', 'get_page_text'],+ confirm=confirm,+ url_policy=allow_url,+ )+ runner = AsyncResponsesToolRunner(browser=tools)+ try:+ response = await runner.run(+ client,+ {+ 'model': model,+ 'store': False,+ 'input': 'Open https://example.com and read its heading and explanation. Do not follow links.',+ },+ max_steps=6,+ )+ text = await driver.get_page_text({'tab_id': None})+ if 'Example Domain' not in text:+ raise RuntimeError('The browser did not show the expected page.')+ print(response.output_text)+ finally:+ runner.close()+ tools.close()+++if __name__ == '__main__':+ asyncio.run(main())diff --git a/pyproject.toml b/pyproject.tomlindex dc5b6d520..cf482d95d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -30,7 +30,7 @@ dependencies = ["typing-extensions==4.15.0","uuid7==0.1.0","google-genai==1.65.0",- "openai==2.26.0",+ "openai>=2.26.0,<4.0.0","anthropic>=0.76.0,<2.0.0","groq==1.0.0","ollama==0.6.1",@@ -59,6 +59,7 @@ dependencies = [# click: used for command-line argument parsing[project.optional-dependencies]+openai-tools = ["openai>=3.31.0,<4.0.0"]cli = []core = ["browser-use-core==0.13.3; sys_platform == 'darwin' and platform_machine == 'arm64'",diff --git a/tests/ci/integrations/test_openai_browser_use.py b/tests/ci/integrations/test_openai_browser_use.pynew file mode 100644index 000000000..28fdd33dd--- /dev/null+++ b/tests/ci/integrations/test_openai_browser_use.py@@ -0,0 +1,138 @@+from __future__ import annotations++import base64+import json+import re+from io import BytesIO++import pytest+from openai.lib.browser import AsyncBrowserToolset, BrowserError+from openai.types.responses import ResponseFunctionToolCall+from PIL import Image+from playwright.async_api import async_playwright++from browser_use import BrowserSession+from browser_use.integrations.openai import BrowserUse+++async def driver():+ async with async_playwright() as playwright:+ browser = BrowserSession(+ headless=True,+ executable_path=playwright.chromium.executable_path,+ enable_default_extensions=False,+ user_data_dir=None,+ )+ await browser.start()+ try:+ async with BrowserUse(browser) as value:+ yield value+ assert browser.is_cdp_connected+ finally:+ await browser.kill()+++async def fixture_page(driver, html):+ await driver.new_tab({'url': 'about:blank'})+ page = await driver._page(None)+ await page.evaluate('(html) => { document.body.innerHTML = html; }', html)+ return page+++def reference(text, label):+ for line in text.splitlines():+ match = re.search(r'\[(ref_\d+)\]', line)+ if label in line and match:+ return match.group(1)+ raise AssertionError(f'Missing reference for {label!r}: {text}')+++async def test_driver_reads_and_changes_real_form(driver):+ page = await fixture_page(+ driver,+ '<label>Name<input id="name"></label><label>Ready<input type="checkbox" id="ready"></label><button onclick="document.querySelector(\'#result\').textContent=document.querySelector(\'#name\').value">Save</button><p id="result"></p>',+ )+ text = await driver.read_page({'tab_id': None, 'filter': 'interactive'})+ await driver.type({'tab_id': None, 'target': reference(text, 'Name'), 'text': 'OpenAI driver'})+ await driver.set_checked({'tab_id': None, 'target': reference(text, 'Ready'), 'checked': True})+ await driver.set_checked({'tab_id': None, 'target': reference(text, 'Ready'), 'checked': True})+ await driver.click({'tab_id': None, 'target': reference(text, 'Save')})+ assert await page.evaluate("() => document.querySelector('#result').textContent") == 'OpenAI driver'+ assert await page.evaluate("() => document.querySelector('#ready').checked") == 'True'+ image = await driver.zoom({'tab_id': None, 'region': [0, 0, 100, 80]})+ with Image.open(BytesIO(base64.b64decode(image['image_url'].split(',', 1)[1]))) as crop:+ assert crop.size == (100, 80)++[email protected]('change', ['navigate', 'detach', 'new_tab', 'reread'])+async def test_driver_rejects_stale_or_cross_tab_reference(driver, change):+ page = await fixture_page(driver, '<button id="button">Original</button>')+ text = await driver.read_page({'tab_id': None, 'filter': 'interactive'})+ target = reference(text, 'Original')+ if change == 'navigate':+ await driver.navigate({'tab_id': None, 'url': 'about:blank'})+ elif change == 'detach':+ await page.evaluate("() => document.querySelector('#button').remove()")+ elif change == 'new_tab':+ await driver.new_tab({'url': 'about:blank'})+ else:+ await driver.read_page({'tab_id': None, 'filter': 'interactive'})+ with pytest.raises((BrowserError, RuntimeError)):+ await driver.click({'tab_id': None, 'target': target})+++async def test_toolset_denies_click_without_changing_page(driver):+ page = await fixture_page(driver, '<button onclick="this.textContent=\'Changed\'">Original</button>')+ text = await driver.read_page({'tab_id': None, 'filter': 'interactive'})++ async def deny(context):+ return False++ tools = AsyncBrowserToolset(driver, members=['click'], confirm=deny)+ try:+ output = await tools.execute(+ ResponseFunctionToolCall(+ id='fc_1',+ call_id='call_1',+ type='function_call',+ status='completed',+ name='browser_click',+ arguments=json.dumps({'target': reference(text, 'Original'), 'tab_id': None}),+ )+ )+ assert 'Original' in await page.evaluate('() => document.body.innerText')+ assert 'Changed' not in await page.evaluate('() => document.body.innerText')+ assert output.get('call_id') == 'call_1'+ finally:+ tools.close()+++async def test_tab_selection_and_close_follow_actual_targets(driver):+ await driver.new_tab({'url': 'about:blank'})+ first = next(t['id'] for t in (await driver.state())['tabs'] if t['active'])+ await driver.new_tab({'url': 'about:blank'})+ second = next(t['id'] for t in (await driver.state())['tabs'] if t['active'])+ assert first != second+ await driver.switch_tab({'tab_id': first})+ assert next(t['id'] for t in (await driver.state())['tabs'] if t['active']) == first+ await driver.close_tab({'tab_id': first})+ assert first not in {t['id'] for t in (await driver.state())['tabs']}+ with pytest.raises(BrowserError):+ await driver.get_page_text({'tab_id': first})+++async def test_driver_wait_and_pointer_bounds(driver):+ await fixture_page(driver, '<p>Present</p>')+ await driver.wait_for({'tab_id': None, 'text': 'Present', 'timeout_ms': 100})+ with pytest.raises(BrowserError):+ await driver.wait_for({'tab_id': None, 'text': 'Missing', 'timeout_ms': 10})+ with pytest.raises(BrowserError):+ await driver.click_at({'tab_id': None, 'x': 999999, 'y': 0, 'button': 'left', 'click_count': 1, 'keys': []})+ with pytest.raises(BrowserError):+ await driver.zoom({'tab_id': None, 'region': [0, 0, 999999, 100]})