Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Reconstruct Rust terminal screenshots
Magnus Müller1a598f9a3987
diff --git a/RUST_AGENT_INTEGRATION_FEATURES.md b/RUST_AGENT_INTEGRATION_FEATURES.mdindex 7b2ea4ecb..d1354757c 100644--- a/RUST_AGENT_INTEGRATION_FEATURES.md+++ b/RUST_AGENT_INTEGRATION_FEATURES.md@@ -649,10 +649,15 @@ Terminal core branch: `magnus/browser-use-rust-main-integration` at terminal mai- This keeps `history.errors()` and final action errors actionable instead of falling back to the generic "Rust terminal session did not produce a final result" message after the terminal core already recorded the failure.- Proof: `test_rust_history_surfaces_terminal_tool_failure_message`.+128. Rust terminal screenshot path reconstruction+ - Terminal browser-script image events now populate Browser Use `BrowserStateHistory.screenshot_path`.+ - This makes `history.screenshot_paths()` and `history.screenshots()` work for Rust-backed histories when terminal current-main records `tool.image` events or image paths on `tool.output`.+ - Proof: `test_rust_history_reconstructs_terminal_screenshot_paths`.+## Current Verification- `python3 -m py_compile browser_use/agent/service.py browser_use/rust/service.py browser_use/rust/__init__.py browser_use/__init__.py tests/ci/test_rust_agent.py examples/rust_agent/basic.py examples/rust_agent/real_v8_smoke.py`-- `uv run pytest -q tests/ci/test_rust_agent.py` (99 tests)+- `uv run pytest -q tests/ci/test_rust_agent.py` (100 tests)- `cargo build -q -p browser-use-cli` on terminal branch `magnus/browser-use-rust-main-integration`- `cargo test -q -p browser-use-cli run_codex_session_command_accepts_task_id_and_model -- --nocapture`- `CARGO_INCREMENTAL=0 cargo test -q -p browser-use-agent selected_remote_cdp_mode_allows_remote_cdp_connect -- --nocapture`diff --git a/browser_use/rust/service.py b/browser_use/rust/service.pyindex 7ec330150..d3b13cdc7 100644--- a/browser_use/rust/service.py+++ b/browser_use/rust/service.py@@ -1033,6 +1033,40 @@ def _browser_state_candidates(value: Any) -> list[tuple[str, str, str]]:return candidates+def _image_path(value: Any) -> str | None:+ if not isinstance(value, dict):+ return None+ path = value.get('path')+ if isinstance(path, str) and path:+ return path+ url = value.get('url')+ if isinstance(url, str) and url.startswith('file://'):+ return url.removeprefix('file://')+ return None+++def _screenshot_path_from_events(events: list[dict[str, Any]]) -> str | None:+ screenshot_path = None+ for event in events:+ event_type = _event_type(event)+ payload = _event_payload(event)+ if event_type == 'tool.image':+ image_path = _image_path(payload.get('image'))+ if image_path:+ screenshot_path = image_path+ continue+ if event_type not in ('tool.output', 'tool.failed'):+ continue+ images = payload.get('images')+ if not isinstance(images, list):+ continue+ for image in images:+ image_path = _image_path(image)+ if image_path:+ screenshot_path = image_path+ return screenshot_path++def _browser_script_navigation_candidates(payload: dict[str, Any]) -> list[tuple[str, str, str]]:if payload.get('name') != 'browser_script':return []@@ -1088,7 +1122,13 @@ def _browser_state_from_events(events: list[dict[str, Any]]) -> BrowserStateHisttabs = next_tabsif not tabs and (url or title):tabs = [TabInfo(url=url, title=title, target_id='tab-0')]- return BrowserStateHistory(url=url, title=title, tabs=tabs, interacted_element=[])+ return BrowserStateHistory(+ url=url,+ title=title,+ tabs=tabs,+ interacted_element=[],+ screenshot_path=_screenshot_path_from_events(events),+ )def _int_value(value: Any) -> int:diff --git a/tests/ci/test_rust_agent.py b/tests/ci/test_rust_agent.pyindex c588db5c0..a420f6678 100644--- a/tests/ci/test_rust_agent.py+++ b/tests/ci/test_rust_agent.py@@ -318,6 +318,46 @@ def test_rust_history_reconstructs_terminal_browser_script_urls():assert navigation_history.urls() == ['https://example.com']+def test_rust_history_reconstructs_terminal_screenshot_paths(tmp_path):+ import base64++ from browser_use.rust.service import _history_from_events++ screenshot_path = tmp_path / 'terminal-shot.png'+ screenshot_bytes = b'\x89PNG\r\n\x1a\nterminal-shot'+ screenshot_path.write_bytes(screenshot_bytes)++ history = _history_from_events(+ [+ {+ 'type': 'tool.output',+ 'payload': {+ 'name': 'browser_script',+ 'tool_call_id': 'call-browser',+ 'images': [{'path': str(screenshot_path), 'mime_type': 'image/png'}],+ },+ },+ {+ 'type': 'tool.image',+ 'payload': {+ 'name': 'browser_script',+ 'tool_call_id': 'call-browser',+ 'image': {'path': str(screenshot_path), 'mime_type': 'image/png'},+ },+ },+ {'event_type': 'session.done', 'payload': {'result': 'final answer'}},+ ],+ model='gpt-test',+ started=1.0,+ finished=2.0,+ output_model_schema=None,+ process_error=None,+ )++ assert history.screenshot_paths() == [str(screenshot_path)]+ assert history.screenshots() == [base64.b64encode(screenshot_bytes).decode('utf-8')]++def test_rust_history_reconstructs_terminal_tool_call_actions():from browser_use.rust.service import _history_from_events