Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Reconstruct Rust terminal screenshots

Magnus Müller1a598f9a3987
diff --git a/RUST_AGENT_INTEGRATION_FEATURES.md b/RUST_AGENT_INTEGRATION_FEATURES.md
index 7b2ea4ecb..d1354757c 100644
--- a/RUST_AGENT_INTEGRATION_FEATURES.md
+++ b/RUST_AGENT_INTEGRATION_FEATURES.md
@@ -649,10 +649,15 @@ Terminal core branch: `magnus/browser-use-rust-main-integration` at terminal mai
- This keeps `history.errors()` and final action errors actionable instead of falling back to the generic "Rust terminal session did not produce a final result" message after the terminal core already recorded the failure.
- Proof: `test_rust_history_surfaces_terminal_tool_failure_message`.
+128. Rust terminal screenshot path reconstruction
+ - Terminal browser-script image events now populate Browser Use `BrowserStateHistory.screenshot_path`.
+ - This makes `history.screenshot_paths()` and `history.screenshots()` work for Rust-backed histories when terminal current-main records `tool.image` events or image paths on `tool.output`.
+ - Proof: `test_rust_history_reconstructs_terminal_screenshot_paths`.
+
## Current Verification
- `python3 -m py_compile browser_use/agent/service.py browser_use/rust/service.py browser_use/rust/__init__.py browser_use/__init__.py tests/ci/test_rust_agent.py examples/rust_agent/basic.py examples/rust_agent/real_v8_smoke.py`
-- `uv run pytest -q tests/ci/test_rust_agent.py` (99 tests)
+- `uv run pytest -q tests/ci/test_rust_agent.py` (100 tests)
- `cargo build -q -p browser-use-cli` on terminal branch `magnus/browser-use-rust-main-integration`
- `cargo test -q -p browser-use-cli run_codex_session_command_accepts_task_id_and_model -- --nocapture`
- `CARGO_INCREMENTAL=0 cargo test -q -p browser-use-agent selected_remote_cdp_mode_allows_remote_cdp_connect -- --nocapture`
diff --git a/browser_use/rust/service.py b/browser_use/rust/service.py
index 7ec330150..d3b13cdc7 100644
--- a/browser_use/rust/service.py
+++ b/browser_use/rust/service.py
@@ -1033,6 +1033,40 @@ def _browser_state_candidates(value: Any) -> list[tuple[str, str, str]]:
return candidates
+def _image_path(value: Any) -> str | None:
+ if not isinstance(value, dict):
+ return None
+ path = value.get('path')
+ if isinstance(path, str) and path:
+ return path
+ url = value.get('url')
+ if isinstance(url, str) and url.startswith('file://'):
+ return url.removeprefix('file://')
+ return None
+
+
+def _screenshot_path_from_events(events: list[dict[str, Any]]) -> str | None:
+ screenshot_path = None
+ for event in events:
+ event_type = _event_type(event)
+ payload = _event_payload(event)
+ if event_type == 'tool.image':
+ image_path = _image_path(payload.get('image'))
+ if image_path:
+ screenshot_path = image_path
+ continue
+ if event_type not in ('tool.output', 'tool.failed'):
+ continue
+ images = payload.get('images')
+ if not isinstance(images, list):
+ continue
+ for image in images:
+ image_path = _image_path(image)
+ if image_path:
+ screenshot_path = image_path
+ return screenshot_path
+
+
def _browser_script_navigation_candidates(payload: dict[str, Any]) -> list[tuple[str, str, str]]:
if payload.get('name') != 'browser_script':
return []
@@ -1088,7 +1122,13 @@ def _browser_state_from_events(events: list[dict[str, Any]]) -> BrowserStateHist
tabs = next_tabs
if not tabs and (url or title):
tabs = [TabInfo(url=url, title=title, target_id='tab-0')]
- return BrowserStateHistory(url=url, title=title, tabs=tabs, interacted_element=[])
+ return BrowserStateHistory(
+ url=url,
+ title=title,
+ tabs=tabs,
+ interacted_element=[],
+ screenshot_path=_screenshot_path_from_events(events),
+ )
def _int_value(value: Any) -> int:
diff --git a/tests/ci/test_rust_agent.py b/tests/ci/test_rust_agent.py
index c588db5c0..a420f6678 100644
--- a/tests/ci/test_rust_agent.py
+++ b/tests/ci/test_rust_agent.py
@@ -318,6 +318,46 @@ def test_rust_history_reconstructs_terminal_browser_script_urls():
assert navigation_history.urls() == ['https://example.com']
+def test_rust_history_reconstructs_terminal_screenshot_paths(tmp_path):
+ import base64
+
+ from browser_use.rust.service import _history_from_events
+
+ screenshot_path = tmp_path / 'terminal-shot.png'
+ screenshot_bytes = b'\x89PNG\r\n\x1a\nterminal-shot'
+ screenshot_path.write_bytes(screenshot_bytes)
+
+ history = _history_from_events(
+ [
+ {
+ 'type': 'tool.output',
+ 'payload': {
+ 'name': 'browser_script',
+ 'tool_call_id': 'call-browser',
+ 'images': [{'path': str(screenshot_path), 'mime_type': 'image/png'}],
+ },
+ },
+ {
+ 'type': 'tool.image',
+ 'payload': {
+ 'name': 'browser_script',
+ 'tool_call_id': 'call-browser',
+ 'image': {'path': str(screenshot_path), 'mime_type': 'image/png'},
+ },
+ },
+ {'event_type': 'session.done', 'payload': {'result': 'final answer'}},
+ ],
+ model='gpt-test',
+ started=1.0,
+ finished=2.0,
+ output_model_schema=None,
+ process_error=None,
+ )
+
+ assert history.screenshot_paths() == [str(screenshot_path)]
+ assert history.screenshots() == [base64.b64encode(screenshot_bytes).decode('utf-8')]
+
+
def test_rust_history_reconstructs_terminal_tool_call_actions():
from browser_use.rust.service import _history_from_events