Agent Zero AI framework
Expose computer-use captures as vision messages
Alessandrocee9abfde44f
diff --git a/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md b/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.mdindex 683cfbf52..e5d7bb2bd 100644--- a/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md+++ b/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md@@ -10,7 +10,7 @@ If the tool reports no CLI, disabled computer use, or `COMPUTER_USE_REARM_REQUIRCall `start_session` before screen-driven tasks. Use `status` for state only, `capture` for screenshots without an action, and `stop_session` when the desktop task is complete. Interactive actions should use normalized global-screen coordinates from the most recent capture.-State-changing actions automatically attach a fresh screen after they run. Treat key presses, clicks, scrolling, typing, and window-manager shortcuts as attempts, not success: inspect the latest attached screen, or one explicit `capture` if it is unclear or unchanged, before saying the requested outcome happened. For Ubuntu/GNOME/Wayland hide-window tasks, prefer `Super+H` (`{"action":"key","keys":["Super","H"]}`) for the active window; do not use `Alt+F9` as the primary hide/minimize shortcut because it often leaves the window visible. A `type` result only proves keystrokes were sent; it does not prove the window was hidden or that text landed in the intended place.+State-changing actions automatically attach a fresh screen after they run. Treat key presses, clicks, scrolling, typing, and window-manager shortcuts as attempts, not success: inspect the latest attached screen, or one explicit `capture` if it is unclear or unchanged, before saying the requested outcome happened. If the tool says a screen was attached but you cannot actually inspect the image, stop and report that visual verification is unavailable; do not continue by assuming the host state. For Ubuntu/GNOME/Wayland hide-window tasks, prefer `Super+H` (`{"action":"key","keys":["Super","H"]}`) for the active window; do not use `Alt+F9` as the primary hide/minimize shortcut because it often leaves the window visible. A `type` result only proves keystrokes were sent; it does not prove the window was hidden or that text landed in the intended place.```json{diff --git a/plugins/_a0_connector/skills/host-computer-use/SKILL.md b/plugins/_a0_connector/skills/host-computer-use/SKILL.mdindex 479823936..20b1e7eaf 100644--- a/plugins/_a0_connector/skills/host-computer-use/SKILL.md+++ b/plugins/_a0_connector/skills/host-computer-use/SKILL.md@@ -80,6 +80,7 @@ If any tool result contains `COMPUTER_USE_REARM_REQUIRED` or `status=rearm requi## Operating Rules- Only the latest screenshot or a definitive tool result counts as evidence.+- If a tool result says a screenshot was attached but you cannot actually see the image, stop and report that visual verification is unavailable. Do not continue with another action from an assumed host state.- The current API uses normalized global screen coordinates; do not assume window ids, element indexes, background-safe input, or semantic click targets unless the runtime explicitly advertises them.- Prefer accessibility and semantic UI paths first: shortcuts, command palettes, menu accelerators, address/search bars, focus traversal, and other keyboard-accessible controls.- Prefer `key` and `type` over pointer actions whenever a reliable keyboard path exists.diff --git a/plugins/_a0_connector/tools/computer_use_remote.py b/plugins/_a0_connector/tools/computer_use_remote.pyindex 494a2f72d..f0e60332e 100644--- a/plugins/_a0_connector/tools/computer_use_remote.py+++ b/plugins/_a0_connector/tools/computer_use_remote.py@@ -6,6 +6,7 @@ from pathlib import Pathimport uuidfrom typing import Any+from helpers import historyfrom helpers.print_style import PrintStylefrom helpers.tool import Response, Toolfrom helpers.ws import NAMESPACE@@ -24,7 +25,9 @@ COMPUTER_USE_OP_EVENT = "connector_computer_use_op"CAPTURE_TOKENS_ESTIMATE = 1500MAX_CAPTURE_ARTIFACT_SIZE_BYTES = 25 * 1024 * 1024CAPTURE_VERIFICATION_NOTE = (- "Do not claim success unless this screen visibly confirms the requested outcome."+ "Inspect the attached screenshot before the next action; do not claim or proceed "+ "from assumed state. If you cannot see the screenshot, stop and report that visual "+ "verification is unavailable.")REARM_REQUIRED_DEFAULT_MESSAGE = ("Computer use is configured, but the installed desktop-control backend is not armed."@@ -144,16 +147,21 @@ class ComputerUseRemote(Tool):text = _sanitize_tool_text(response.message.strip())additional = dict(response.additional)+ raw_content = additional.pop("raw_content", None)+ preview = str(additional.pop("preview", "") or "").strip() or texttoken_estimate = self._coerce_token_estimate(additional.pop("_tokens", CAPTURE_TOKENS_ESTIMATE))log_id = str(getattr(getattr(self, "log", None), "id", "") or "")- message = self.agent.hist_add_tool_result(+ self.agent.hist_add_tool_result(self.name,text,id=log_id,**additional,)- if hasattr(message, "tokens"):- message.tokens = token_estimate+ self.agent.hist_add_message(+ False,+ content=history.RawMessage(raw_content=raw_content, preview=preview),+ tokens=token_estimate,+ )agent_name = str(getattr(self.agent, "agent_name", "Agent Zero") or "Agent Zero")PrintStyle(@@ -472,6 +480,14 @@ class ComputerUseRemote(Tool):message.tokens = message.calculate_tokens()def _resolve_capture_ref(self, data: dict[str, Any]) -> tuple[str, str]:+ path_error: FileNotFoundError | None = None+ try:+ image_path, display_path = self._resolve_capture_path(data)+ except FileNotFoundError as exc:+ path_error = exc+ else:+ return display_path, image_path.stem+artifact = data.get("artifact")if isinstance(artifact, dict) and str(artifact.get("encoding", "")).strip().lower() == "base64":encoded = str(artifact.get("data") or "")@@ -488,8 +504,9 @@ class ComputerUseRemote(Tool):filename = _safe_filename(str(artifact.get("filename") or "computer-use-capture.png"))return f"data:{mime};base64,{encoded}", Path(filename).stem- image_path, display_path = self._resolve_capture_path(data)- return display_path, image_path.stem+ if path_error is not None:+ raise path_error+ raise FileNotFoundError("Capture artifact was not found in the tool response.")def _collect_capture_messages(self, history_obj: Any) -> list[Any]:messages: list[Any] = []diff --git a/tests/test_a0_connector_prompt_gating.py b/tests/test_a0_connector_prompt_gating.pyindex d83dda224..3e7c54476 100644--- a/tests/test_a0_connector_prompt_gating.py+++ b/tests/test_a0_connector_prompt_gating.py@@ -136,9 +136,12 @@ def test_computer_use_remote_prompt_requires_visual_verification_after_actions()assert "Treat key presses, clicks, scrolling, typing" in promptassert "attempts, not success" in prompt+ assert "visual verification is unavailable" in prompt+ assert "do not continue by assuming the host state" in promptassert "Super+H" in promptassert '["Super","H"]' in promptassert "Alt+F9" in prompt+ assert "cannot actually see the image" in skillassert "Do not use `Alt+F9` as the primary hide/minimize shortcut" in skillassert "A `type` tool result only confirms keystrokes were sent" in skillassert "do not type follow-up text into the active field" in skilldiff --git a/tests/test_tool_action_contracts.py b/tests/test_tool_action_contracts.pyindex 9da8b5cf3..8ded5a19b 100644--- a/tests/test_tool_action_contracts.py+++ b/tests/test_tool_action_contracts.py@@ -568,5 +568,6 @@ def test_computer_use_remote_is_runtime_checked_standard_tool():assert '"tool_name": "computer_use_remote"' in standard_prompt_textassert "not scoped to a single chat context" in standard_prompt_textassert "checked when the tool runs" in standard_prompt_text+ assert "visual verification is unavailable" in standard_prompt_textassert '"tool_name": "computer_use_remote"' in skill_textassert "Beta desktop control" in skill_text