Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Add OpenAI browser tool driver using Browser Use Actor

Magnus Mueller8173a07fd766
diff --git a/browser_use/integrations/openai/__init__.py b/browser_use/integrations/openai/__init__.py
new file mode 100644
index 000000000..e41399a5d
--- /dev/null
+++ b/browser_use/integrations/openai/__init__.py
@@ -0,0 +1,5 @@
+"""Connect OpenAI browser tools to Browser Use actions."""
+
+from .browser_use import BrowserUse
+
+__all__ = ['BrowserUse']
diff --git a/browser_use/integrations/openai/browser_use.py b/browser_use/integrations/openai/browser_use.py
new file mode 100644
index 000000000..6e9c26db8
--- /dev/null
+++ b/browser_use/integrations/openai/browser_use.py
@@ -0,0 +1,321 @@
+"""Run OpenAI browser tools through Browser Use Actor methods."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+from io import BytesIO
+from typing import Any
+
+from PIL import Image
+from pydantic import BaseModel
+
+from browser_use.actor.page import Page
+from browser_use.browser.session import BrowserSession
+from openai.lib import browser as api
+
+
+class Reference(BaseModel):
+ tab: str
+ document: int
+ backend: int
+
+
+class BrowserUse:
+ """Own a browser session or borrow an already connected session."""
+
+ def __init__(self, browser: BrowserSession | None = None, *, use_cloud: bool = False):
+ if browser is not None and use_cloud:
+ raise ValueError('Pass a browser or use_cloud=True, not both.')
+ self.browser = browser if browser is not None else BrowserSession(use_cloud=use_cloud)
+ self._owns_browser = browser is None
+ self._closed = False
+ self._refs: dict[str, Reference] = {}
+ self._next_ref = 0
+ self._active: str | None = None
+
+ async def __aenter__(self) -> BrowserUse:
+ if self._closed:
+ raise RuntimeError('The driver is closed.')
+ try:
+ if self._owns_browser:
+ await self.browser.start()
+ elif not self.browser.is_cdp_connected:
+ raise RuntimeError('Start the borrowed browser before entering the driver.')
+ return self
+ except BaseException:
+ await self.close()
+ raise
+
+ async def __aexit__(self, *_: Any) -> None:
+ await self.close()
+
+ async def close(self) -> None:
+ """Close only the browser that this driver created."""
+ if not self._closed:
+ if self._owns_browser:
+ await self.browser.kill()
+ self._refs.clear()
+ self._closed = True
+
+ async def state(self) -> api.BrowserState:
+ """Report current browser tabs without retaining stale tab state."""
+ if self._closed:
+ raise api.BrowserError('The driver is closed.')
+ result = await self.browser.cdp_client.send.Target.getTargets()
+ tabs = [t for t in result['targetInfos'] if t['type'] == 'page']
+ ids = {t['targetId'] for t in tabs}
+ if self._active not in ids:
+ focus = self.browser.agent_focus_target_id
+ self._active = focus if focus in ids else next(iter(ids), None)
+ self._refs = {name: ref for name, ref in self._refs.items() if ref.tab in ids}
+ return {
+ 'tabs': [
+ {'id': t['targetId'], 'url': t['url'], 'title': t['title'], 'active': t['targetId'] == self._active} for t in tabs
+ ]
+ }
+
+ async def _page(self, tab_id: str | None) -> Page:
+ state = await self.state()
+ target = tab_id or self._active
+ if target is None or target not in {tab['id'] for tab in state['tabs']}:
+ raise api.BrowserError('Unknown tab.')
+ session = await self.browser.get_or_create_cdp_session(target, focus=True)
+ self._active = target
+ return Page(self.browser, target, session.session_id)
+
+ async def _document(self, page: Page) -> int:
+ result = await self.browser.cdp_client.send.DOM.getDocument({'depth': 0}, session_id=await page.session_id)
+ return result['root']['backendNodeId']
+
+ async def _element(self, page: Page, target: str):
+ ref = self._refs.get(target)
+ if ref is None or ref.tab != page._target_id or ref.document != await self._document(page):
+ raise api.BrowserError('Stale element reference; read the page again.')
+ element = await page.get_element(ref.backend)
+ connected = await element.evaluate('() => this.isConnected')
+ if connected.casefold() != 'true':
+ raise api.BrowserError('Detached element; read the page again.')
+ return element
+
+ async def navigate(self, args: api.NavigateInput) -> None:
+ """Invalidate element references before navigating the selected tab."""
+ page = await self._page(args['tab_id'])
+ self._refs = {name: ref for name, ref in self._refs.items() if ref.tab != page._target_id}
+ await page.goto(args['url'])
+
+ async def read_page(self, args: api.ReadPageInput) -> str:
+ """Read the accessibility tree and issue tab-bound element references."""
+ page = await self._page(args['tab_id'])
+ document = await self._document(page)
+ self._refs = {name: ref for name, ref in self._refs.items() if ref.tab != page._target_id}
+ result = await self.browser.cdp_client.send.Accessibility.getFullAXTree(session_id=await page.session_id)
+ interactive = {
+ 'button',
+ 'link',
+ 'textbox',
+ 'searchbox',
+ 'combobox',
+ 'checkbox',
+ 'radio',
+ 'slider',
+ 'spinbutton',
+ 'menuitem',
+ 'tab',
+ 'option',
+ 'listbox',
+ 'switch',
+ }
+ lines = []
+ for node in result['nodes']:
+ role = node.get('role', {}).get('value', '')
+ if node.get('ignored') or role == 'InlineTextBox':
+ continue
+ if args['filter'] == 'interactive' and role not in interactive:
+ continue
+ name = node.get('name', {}).get('value', '')
+ prefix = ''
+ backend = node.get('backendDOMNodeId')
+ if backend and role in interactive:
+ self._next_ref += 1
+ ref = f'ref_{self._next_ref}'
+ self._refs[ref] = Reference(tab=page._target_id, document=document, backend=backend)
+ prefix = f'[{ref}] '
+ lines.append(f'{prefix}{role} {json.dumps(name, ensure_ascii=False)}')
+ return '\n'.join(lines)
+
+ async def find(self, args: api.FindInput) -> str:
+ """Find current interactive elements by visible label text."""
+ text = await self.read_page({'tab_id': args['tab_id'], 'filter': 'interactive'})
+ terms = args['query'].casefold().split()
+ return '\n'.join(line for line in text.splitlines() if all(term in line.casefold() for term in terms))
+
+ async def get_page_text(self, args: api.TabInput) -> str:
+ """Read the selected page without returning form values."""
+ page = await self._page(args['tab_id'])
+ return await page.evaluate("() => document.body?.innerText || ''")
+
+ async def click(self, args: api.ClickInput) -> None:
+ """Click an element from the latest page read."""
+ page = await self._page(args['tab_id'])
+ await (await self._element(page, args['target'])).click()
+
+ async def hover(self, args: api.ClickInput) -> None:
+ """Move the pointer over a referenced element."""
+ page = await self._page(args['tab_id'])
+ await (await self._element(page, args['target'])).hover()
+
+ async def scroll_to(self, args: api.ClickInput) -> None:
+ """Scroll a referenced element into view."""
+ page = await self._page(args['tab_id'])
+ await (await self._element(page, args['target'])).scroll_into_view()
+
+ async def type(self, args: api.TypeInput) -> None:
+ """Fill a referenced element or type at the current focus."""
+ page = await self._page(args['tab_id'])
+ if args['target'] is not None:
+ await (await self._element(page, args['target'])).fill(args['text'])
+ else:
+ await self.browser.cdp_client.send.Input.insertText({'text': args['text']}, session_id=await page.session_id)
+
+ async def key(self, args: api.KeyInput) -> None:
+ """Press a key combination in the selected tab."""
+ page = await self._page(args['tab_id'])
+ await page.press('+'.join(args['keys']))
+
+ async def select_option(self, args: api.SelectOptionInput) -> None:
+ """Select values in a referenced native select element."""
+ page = await self._page(args['tab_id'])
+ await (await self._element(page, args['target'])).select_option(args['values'])
+
+ async def scroll(self, args: api.ScrollInput) -> None:
+ """Scroll by the requested number of pixels."""
+ page = await self._page(args['tab_id'])
+ amount = args['amount'] * (-1 if args['direction'] in {'up', 'left'} else 1)
+ horizontal = args['direction'] in {'left', 'right'}
+ await (await page.mouse).scroll(delta_x=amount if horizontal else 0, delta_y=0 if horizontal else amount)
+
+ async def wait_for(self, args: api.WaitForInput) -> None:
+ """Wait for visible text within the requested deadline."""
+ page = await self._page(args['tab_id'])
+ result = await page.evaluate(
+ """(text, timeout) => new Promise(resolve => {
+ let timer;
+ const observer = new MutationObserver(check);
+ function finish(found) { observer.disconnect(); clearTimeout(timer); resolve(found); }
+ function check() { if (document.body?.innerText.includes(text)) finish(true); }
+ observer.observe(document, {subtree: true, childList: true, characterData: true, attributes: true});
+ timer = setTimeout(() => finish(false), timeout);
+ check();
+ })""",
+ args['text'],
+ args['timeout_ms'],
+ )
+ if result.casefold() != 'true':
+ raise api.BrowserError('Visible text did not appear before the deadline.')
+
+ async def screenshot(self, args: api.TabInput) -> dict[str, str]:
+ """Capture the selected viewport as a native image input."""
+ page = await self._page(args['tab_id'])
+ return {'type': 'input_image', 'image_url': 'data:image/png;base64,' + await page.screenshot(), 'detail': 'original'}
+
+ async def zoom(self, args: api.ZoomInput) -> dict[str, str]:
+ """Crop a screenshot without changing the pointer coordinate origin."""
+ result = await self.screenshot({'tab_id': args['tab_id']})
+ with Image.open(BytesIO(base64.b64decode(result['image_url'].split(',', 1)[1]))) as image:
+ left, top, right, bottom = args['region']
+ if left < 0 or top < 0 or right > image.width or bottom > image.height or right <= left or bottom <= top:
+ raise api.BrowserError('Crop exceeds the screenshot bounds.')
+ output = BytesIO()
+ image.crop((left, top, right, bottom)).save(output, format='PNG')
+ result['image_url'] = 'data:image/png;base64,' + base64.b64encode(output.getvalue()).decode()
+ return result
+
+ async def new_tab(self, args: api.NewTabInput) -> None:
+ """Open a tab and select it for subsequent actions."""
+ page = await self.browser.new_page(args['url'])
+ self._active = page._target_id
+
+ async def switch_tab(self, args: api.TabInput) -> None:
+ """Activate an existing tab."""
+ page = await self._page(args['tab_id'])
+ await self.browser.cdp_client.send.Target.activateTarget({'targetId': page._target_id})
+
+ async def close_tab(self, args: api.TabInput) -> None:
+ """Close only the selected tab."""
+ page = await self._page(args['tab_id'])
+ await self.browser.close_page(page)
+
+ async def _bounds(self, page: Page, points: list[tuple[int, int]]) -> None:
+ metrics = await self.browser.cdp_client.send.Page.getLayoutMetrics(session_id=await page.session_id)
+ viewport = metrics['cssVisualViewport']
+ if any(x < 0 or y < 0 or x >= viewport['clientWidth'] or y >= viewport['clientHeight'] for x, y in points):
+ raise api.BrowserError('Pointer coordinates exceed the viewport.')
+
+ async def click_at(self, args: api.ClickAtInput, *, cancelled=None) -> None:
+ """Click full-viewport coordinates with the requested modifiers."""
+ page = await self._page(args['tab_id'])
+ await self._bounds(page, [(args['x'], args['y'])])
+ if cancelled and cancelled():
+ raise asyncio.CancelledError()
+ await (await page.mouse).click(
+ args['x'], args['y'], button=args['button'], click_count=args['click_count'], modifiers=args['keys']
+ )
+
+ async def hover_at(self, args: api.HoverAtInput) -> None:
+ """Move the pointer to full-viewport coordinates."""
+ page = await self._page(args['tab_id'])
+ await self._bounds(page, [(args['x'], args['y'])])
+ await (await page.mouse).move(args['x'], args['y'])
+
+ async def drag_path(self, args: api.DragPathInput, *, cancelled=None) -> None:
+ """Drag through a path and release the pointer after failure."""
+ page = await self._page(args['tab_id'])
+ points = [(point['x'], point['y']) for point in args['path']]
+ await self._bounds(page, points)
+ mouse = await page.mouse
+ await mouse.move(*points[0], modifiers=args['keys'])
+ try:
+ if cancelled and cancelled():
+ raise asyncio.CancelledError()
+ await mouse.down(modifiers=args['keys'])
+ for x, y in points[1:]:
+ if cancelled and cancelled():
+ raise asyncio.CancelledError()
+ await mouse.move(x, y, modifiers=args['keys'])
+ finally:
+ await mouse.up(modifiers=args['keys'])
+
+ async def hold_keys(self, args: api.HoldKeysInput, *, cancelled=None) -> None:
+ """Hold a key combination for the requested duration."""
+ page = await self._page(args['tab_id'])
+ if cancelled and cancelled():
+ raise asyncio.CancelledError()
+ await page.hold_key('+'.join(args['keys']), args['duration_ms'] / 1000)
+
+ async def set_checked(self, args: api.SetCheckedInput) -> None:
+ """Set a native checkbox without toggling an already matching state."""
+ page = await self._page(args['tab_id'])
+ element = await self._element(page, args['target'])
+ kind = await element.get_attribute('type')
+ if kind not in {'checkbox', 'radio'}:
+ raise api.BrowserError('The target is not a native checkbox or radio button.')
+ if kind == 'radio' and not args['checked']:
+ raise api.BrowserError('Clear a radio button by selecting another option.')
+ current = await element.evaluate('() => this.checked')
+ if (current.casefold() == 'true') != args['checked']:
+ await element.click()
+
+ async def run_script(self, args: api.RunScriptInput) -> str:
+ """Run an explicitly enabled page script within the caller deadline."""
+ page = await self._page(args['tab_id'])
+ async with asyncio.timeout(args['timeout_ms'] / 1000):
+ result = await self.browser.cdp_client.send.Runtime.evaluate(
+ {'expression': args['source'], 'returnByValue': True, 'awaitPromise': True},
+ session_id=await page.session_id,
+ )
+ if result.get('exceptionDetails'):
+ raise api.BrowserError('Page script failed.')
+ value = result.get('result', {}).get('value')
+ return value if isinstance(value, str) else json.dumps(value)
diff --git a/examples/integrations/openai/quickstart.py b/examples/integrations/openai/quickstart.py
new file mode 100644
index 000000000..54fb3140e
--- /dev/null
+++ b/examples/integrations/openai/quickstart.py
@@ -0,0 +1,53 @@
+"""Read a page through Browser Use actions and the OpenAI tool runner."""
+
+import asyncio
+import os
+from urllib.parse import urlsplit
+
+from openai import AsyncOpenAI
+from openai.lib.browser import AsyncBrowserToolset
+from openai.lib.responses_tool_runner import AsyncResponsesToolRunner
+
+from browser_use.integrations.openai import BrowserUse
+
+
+async def allow_url(context, url):
+ parsed = urlsplit(url)
+ return parsed.scheme == 'https' and parsed.hostname == 'example.com' and not parsed.username and not parsed.password
+
+
+async def confirm(context):
+ return context['member'] in {'navigate', 'new_tab'}
+
+
+async def main():
+ model = os.environ['OPENAI_MODEL']
+ async with AsyncOpenAI(max_retries=0, timeout=60) as client, BrowserUse(use_cloud=True) as driver:
+ tools = AsyncBrowserToolset(
+ driver,
+ members=['new_tab', 'navigate', 'get_page_text'],
+ confirm=confirm,
+ url_policy=allow_url,
+ )
+ runner = AsyncResponsesToolRunner(browser=tools)
+ try:
+ response = await runner.run(
+ client,
+ {
+ 'model': model,
+ 'store': False,
+ 'input': 'Open https://example.com and read its heading and explanation. Do not follow links.',
+ },
+ max_steps=6,
+ )
+ text = await driver.get_page_text({'tab_id': None})
+ if 'Example Domain' not in text:
+ raise RuntimeError('The browser did not show the expected page.')
+ print(response.output_text)
+ finally:
+ runner.close()
+ tools.close()
+
+
+if __name__ == '__main__':
+ asyncio.run(main())
diff --git a/pyproject.toml b/pyproject.toml
index dc5b6d520..cf482d95d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,7 +30,7 @@ dependencies = [
"typing-extensions==4.15.0",
"uuid7==0.1.0",
"google-genai==1.65.0",
- "openai==2.26.0",
+ "openai>=2.26.0,<4.0.0",
"anthropic>=0.76.0,<2.0.0",
"groq==1.0.0",
"ollama==0.6.1",
@@ -59,6 +59,7 @@ dependencies = [
# click: used for command-line argument parsing
[project.optional-dependencies]
+openai-tools = ["openai>=3.31.0,<4.0.0"]
cli = []
core = [
"browser-use-core==0.13.3; sys_platform == 'darwin' and platform_machine == 'arm64'",
diff --git a/tests/ci/integrations/test_openai_browser_use.py b/tests/ci/integrations/test_openai_browser_use.py
new file mode 100644
index 000000000..28fdd33dd
--- /dev/null
+++ b/tests/ci/integrations/test_openai_browser_use.py
@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import base64
+import json
+import re
+from io import BytesIO
+
+import pytest
+from openai.lib.browser import AsyncBrowserToolset, BrowserError
+from openai.types.responses import ResponseFunctionToolCall
+from PIL import Image
+from playwright.async_api import async_playwright
+
+from browser_use import BrowserSession
+from browser_use.integrations.openai import BrowserUse
+
+
+async def driver():
+ async with async_playwright() as playwright:
+ browser = BrowserSession(
+ headless=True,
+ executable_path=playwright.chromium.executable_path,
+ enable_default_extensions=False,
+ user_data_dir=None,
+ )
+ await browser.start()
+ try:
+ async with BrowserUse(browser) as value:
+ yield value
+ assert browser.is_cdp_connected
+ finally:
+ await browser.kill()
+
+
+async def fixture_page(driver, html):
+ await driver.new_tab({'url': 'about:blank'})
+ page = await driver._page(None)
+ await page.evaluate('(html) => { document.body.innerHTML = html; }', html)
+ return page
+
+
+def reference(text, label):
+ for line in text.splitlines():
+ match = re.search(r'\[(ref_\d+)\]', line)
+ if label in line and match:
+ return match.group(1)
+ raise AssertionError(f'Missing reference for {label!r}: {text}')
+
+
+async def test_driver_reads_and_changes_real_form(driver):
+ page = await fixture_page(
+ driver,
+ '<label>Name<input id="name"></label><label>Ready<input type="checkbox" id="ready"></label><button onclick="document.querySelector(\'#result\').textContent=document.querySelector(\'#name\').value">Save</button><p id="result"></p>',
+ )
+ text = await driver.read_page({'tab_id': None, 'filter': 'interactive'})
+ await driver.type({'tab_id': None, 'target': reference(text, 'Name'), 'text': 'OpenAI driver'})
+ await driver.set_checked({'tab_id': None, 'target': reference(text, 'Ready'), 'checked': True})
+ await driver.set_checked({'tab_id': None, 'target': reference(text, 'Ready'), 'checked': True})
+ await driver.click({'tab_id': None, 'target': reference(text, 'Save')})
+ assert await page.evaluate("() => document.querySelector('#result').textContent") == 'OpenAI driver'
+ assert await page.evaluate("() => document.querySelector('#ready').checked") == 'True'
+ image = await driver.zoom({'tab_id': None, 'region': [0, 0, 100, 80]})
+ with Image.open(BytesIO(base64.b64decode(image['image_url'].split(',', 1)[1]))) as crop:
+ assert crop.size == (100, 80)
+
+
[email protected]('change', ['navigate', 'detach', 'new_tab', 'reread'])
+async def test_driver_rejects_stale_or_cross_tab_reference(driver, change):
+ page = await fixture_page(driver, '<button id="button">Original</button>')
+ text = await driver.read_page({'tab_id': None, 'filter': 'interactive'})
+ target = reference(text, 'Original')
+ if change == 'navigate':
+ await driver.navigate({'tab_id': None, 'url': 'about:blank'})
+ elif change == 'detach':
+ await page.evaluate("() => document.querySelector('#button').remove()")
+ elif change == 'new_tab':
+ await driver.new_tab({'url': 'about:blank'})
+ else:
+ await driver.read_page({'tab_id': None, 'filter': 'interactive'})
+ with pytest.raises((BrowserError, RuntimeError)):
+ await driver.click({'tab_id': None, 'target': target})
+
+
+async def test_toolset_denies_click_without_changing_page(driver):
+ page = await fixture_page(driver, '<button onclick="this.textContent=\'Changed\'">Original</button>')
+ text = await driver.read_page({'tab_id': None, 'filter': 'interactive'})
+
+ async def deny(context):
+ return False
+
+ tools = AsyncBrowserToolset(driver, members=['click'], confirm=deny)
+ try:
+ output = await tools.execute(
+ ResponseFunctionToolCall(
+ id='fc_1',
+ call_id='call_1',
+ type='function_call',
+ status='completed',
+ name='browser_click',
+ arguments=json.dumps({'target': reference(text, 'Original'), 'tab_id': None}),
+ )
+ )
+ assert 'Original' in await page.evaluate('() => document.body.innerText')
+ assert 'Changed' not in await page.evaluate('() => document.body.innerText')
+ assert output.get('call_id') == 'call_1'
+ finally:
+ tools.close()
+
+
+async def test_tab_selection_and_close_follow_actual_targets(driver):
+ await driver.new_tab({'url': 'about:blank'})
+ first = next(t['id'] for t in (await driver.state())['tabs'] if t['active'])
+ await driver.new_tab({'url': 'about:blank'})
+ second = next(t['id'] for t in (await driver.state())['tabs'] if t['active'])
+ assert first != second
+ await driver.switch_tab({'tab_id': first})
+ assert next(t['id'] for t in (await driver.state())['tabs'] if t['active']) == first
+ await driver.close_tab({'tab_id': first})
+ assert first not in {t['id'] for t in (await driver.state())['tabs']}
+ with pytest.raises(BrowserError):
+ await driver.get_page_text({'tab_id': first})
+
+
+async def test_driver_wait_and_pointer_bounds(driver):
+ await fixture_page(driver, '<p>Present</p>')
+ await driver.wait_for({'tab_id': None, 'text': 'Present', 'timeout_ms': 100})
+ with pytest.raises(BrowserError):
+ await driver.wait_for({'tab_id': None, 'text': 'Missing', 'timeout_ms': 10})
+ with pytest.raises(BrowserError):
+ await driver.click_at({'tab_id': None, 'x': 999999, 'y': 0, 'button': 'left', 'click_count': 1, 'keys': []})
+ with pytest.raises(BrowserError):
+ await driver.zoom({'tab_id': None, 'region': [0, 0, 999999, 100]})