Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Attach Rust terminal tool images to actions
Magnus Müllera585e13376b5
diff --git a/RUST_AGENT_INTEGRATION_FEATURES.md b/RUST_AGENT_INTEGRATION_FEATURES.mdindex a085773f0..84e19e246 100644--- a/RUST_AGENT_INTEGRATION_FEATURES.md+++ b/RUST_AGENT_INTEGRATION_FEATURES.md@@ -774,10 +774,15 @@ Terminal core branch: `magnus/browser-use-rust-main-integration` at terminal mai- This preserves current-main exec-specific streamed `chunk` payloads on Python-visible `ActionResult.extracted_content`, `long_term_memory`, and `action_history()` while avoiding duplicate text when terminal also emits the paired generic `tool.output_delta`.- Proof: `test_rust_history_reconstructs_terminal_exec_command_output_deltas`.+153. Rust terminal tool-image action attachments+ - Terminal `tool.image` events and tool-output `images` payloads now attach their image paths to the matching Python-visible `ActionResult`.+ - This preserves current-main screenshot/image artifacts on the per-action Browser Use API surface while keeping final `done.files_to_display` limited to file/result attachments.+ - Proof: `test_rust_history_attaches_terminal_tool_images_to_actions`.+## Current Verification- `python3 -m py_compile browser_use/agent/service.py browser_use/rust/service.py browser_use/rust/__init__.py browser_use/__init__.py tests/ci/test_rust_agent.py examples/rust_agent/basic.py examples/rust_agent/real_v8_smoke.py`-- `uv run pytest -q tests/ci/test_rust_agent.py` (124 tests)+- `uv run pytest -q tests/ci/test_rust_agent.py` (125 tests)- `cargo build -q -p browser-use-cli` on terminal branch `magnus/browser-use-rust-main-integration`- `cargo test -q -p browser-use-cli run_codex_session_command_accepts_task_id_and_model -- --nocapture`- `CARGO_INCREMENTAL=0 cargo test -q -p browser-use-agent selected_remote_cdp_mode_allows_remote_cdp_connect -- --nocapture`diff --git a/browser_use/rust/service.py b/browser_use/rust/service.pyindex b0e9f3eac..e13e19b4b 100644--- a/browser_use/rust/service.py+++ b/browser_use/rust/service.py@@ -1409,6 +1409,25 @@ def _matching_unkeyed_tool_result_index(return None+def _matching_unkeyed_tool_attachment_index(+ attachments: list[tuple[str, list[str]]],+ used_indices: set[int],+ name: Any,+ *,+ allow_any: bool,+) -> int | None:+ if isinstance(name, str) and name:+ for index, (attachment_name, _paths) in enumerate(attachments):+ if index not in used_indices and attachment_name == name:+ return index+ if not allow_any:+ return None+ for index in range(len(attachments)):+ if index not in used_indices:+ return index+ return None++def _response_input_item_tool_payload(payload: dict[str, Any]) -> dict[str, Any]:item = payload.get('item')if not isinstance(item, dict):@@ -1616,10 +1635,14 @@ def _action_results_from_tool_calls() -> list[ActionResult]:tool_results = _tool_results_by_call_id(events)unkeyed_results = _unkeyed_tool_results(events)+ image_attachments_by_call_id = _tool_image_attachments_by_call_id(events)+ unkeyed_image_attachments = _tool_image_attachments_by_name(events)used_unkeyed_result_indices: set[int] = set()+ used_unkeyed_image_attachment_indices: set[int] = set()action_results: list[ActionResult] = []for tool_call in tool_calls:- event_type, payload = tool_results.get(str(tool_call.get('tool_call_id')), ('', {}))+ tool_call_id = str(tool_call.get('tool_call_id'))+ event_type, payload = tool_results.get(tool_call_id, ('', {}))if not event_type:unkeyed_result_index = _matching_unkeyed_tool_result_index(unkeyed_results,@@ -1630,6 +1653,17 @@ def _action_results_from_tool_calls(if unkeyed_result_index is not None:used_unkeyed_result_indices.add(unkeyed_result_index)event_type, payload = unkeyed_results[unkeyed_result_index]+ tool_attachments = image_attachments_by_call_id.get(tool_call_id)+ if not tool_attachments:+ unkeyed_attachment_index = _matching_unkeyed_tool_attachment_index(+ unkeyed_image_attachments,+ used_unkeyed_image_attachment_indices,+ tool_call.get('name'),+ allow_any=not tool_call.get('_has_explicit_tool_call_id', True),+ )+ if unkeyed_attachment_index is not None:+ used_unkeyed_image_attachment_indices.add(unkeyed_attachment_index)+ tool_attachments = unkeyed_image_attachments[unkeyed_attachment_index][1]text = _tool_result_text(payload)if event_type == 'tool.finished' and not text:name = payload.get('name') or tool_call.get('name') or 'tool'@@ -1645,6 +1679,7 @@ def _action_results_from_tool_calls(action_results.append(ActionResult(error=error,+ attachments=list(tool_attachments) if tool_attachments else None,extracted_content=text if event_type in _TEXTUAL_TOOL_RESULT_EVENTS else None,long_term_memory=text if event_type in _TEXTUAL_TOOL_RESULT_EVENTS else None,)@@ -1775,6 +1810,60 @@ def _image_path(value: Any) -> str | None:return None+def _append_unique_attachment(attachments: list[str], pointer: str | None) -> None:+ if pointer and pointer not in attachments:+ attachments.append(pointer)+++def _tool_image_attachments_by_call_id(events: list[dict[str, Any]]) -> dict[str, list[str]]:+ attachments: dict[str, list[str]] = {}+ for event in events:+ event_type = _event_type(event)+ payload = _event_payload(event)+ if event_type == 'tool.image':+ image_path = _image_path(payload.get('image'))+ call_id = payload.get('tool_call_id') or payload.get('call_id')+ if image_path and call_id:+ _append_unique_attachment(attachments.setdefault(str(call_id), []), image_path)+ continue+ if event_type not in ('tool.output', 'tool.failed'):+ continue+ call_id = payload.get('tool_call_id') or payload.get('call_id')+ if not call_id:+ continue+ images = payload.get('images')+ if not isinstance(images, list):+ continue+ for image in images:+ image_path = _image_path(image)+ if image_path:+ _append_unique_attachment(attachments.setdefault(str(call_id), []), image_path)+ return attachments+++def _tool_image_attachments_by_name(events: list[dict[str, Any]]) -> list[tuple[str, list[str]]]:+ attachments: list[tuple[str, list[str]]] = []+ for event in events:+ event_type = _event_type(event)+ payload = _event_payload(event)+ if payload.get('tool_call_id') or payload.get('call_id'):+ continue+ name = payload.get('name')+ if not isinstance(name, str) or not name:+ continue+ image_paths: list[str] = []+ if event_type == 'tool.image':+ _append_unique_attachment(image_paths, _image_path(payload.get('image')))+ elif event_type in ('tool.output', 'tool.failed'):+ images = payload.get('images')+ if isinstance(images, list):+ for image in images:+ _append_unique_attachment(image_paths, _image_path(image))+ if image_paths:+ attachments.append((name, image_paths))+ return attachments++def _screenshot_path_from_events(events: list[dict[str, Any]]) -> str | None:screenshot_path = Nonefor event in events:diff --git a/tests/ci/test_rust_agent.py b/tests/ci/test_rust_agent.pyindex a7c1bc118..42a26306f 100644--- a/tests/ci/test_rust_agent.py+++ b/tests/ci/test_rust_agent.py@@ -384,6 +384,57 @@ def test_rust_history_reconstructs_terminal_screenshot_paths(tmp_path):assert history.screenshots() == [base64.b64encode(screenshot_bytes).decode('utf-8')]+def test_rust_history_attaches_terminal_tool_images_to_actions(tmp_path):+ from browser_use.rust.service import _history_from_events++ screenshot_path = tmp_path / 'tool-image.png'+ screenshot_path.write_bytes(b'\x89PNG\r\n\x1a\nterminal-image')++ history = _history_from_events(+ [+ {+ 'type': 'tool.started',+ 'payload': {+ 'name': 'browser_script',+ 'tool_call_id': 'call-browser',+ 'arguments': {'code': 'observe_page()'},+ },+ },+ {+ 'type': 'tool.output',+ 'payload': {+ 'name': 'browser_script',+ 'tool_call_id': 'call-browser',+ 'text': 'Observed page.',+ 'images': [{'path': str(screenshot_path), 'mime_type': 'image/png'}],+ },+ },+ {+ 'type': 'tool.image',+ 'payload': {+ 'name': 'browser_script',+ 'tool_call_id': 'call-browser',+ 'image': {'path': str(screenshot_path), 'mime_type': 'image/png'},+ },+ },+ {'event_type': 'session.done', 'payload': {'result': 'final answer'}},+ ],+ model='gpt-test',+ started=1.0,+ finished=2.0,+ output_model_schema=None,+ process_error=None,+ )++ results = history.action_results()++ assert history.action_names() == ['browser_script', 'done']+ assert results[0].attachments == [str(screenshot_path)]+ assert results[0].extracted_content == 'Observed page.'+ assert results[-1].attachments is None+ assert history.screenshot_paths() == [str(screenshot_path)]++def test_rust_history_reconstructs_terminal_tool_call_actions():from browser_use.rust.service import _history_from_events