Agent Zero AI framework
Expose computer_use_remote as a runtime-checked tool
Alessandro60c36d16d857
diff --git a/plugins/_a0_connector/extensions/python/_functions/extensions/python/system_prompt/_11_tools_prompt/build_prompt/end/_70_include_remote_tool_stubs.py b/plugins/_a0_connector/extensions/python/_functions/extensions/python/system_prompt/_11_tools_prompt/build_prompt/end/_70_include_remote_tool_stubs.pyindex 52b8137e9..1fe05f0d5 100644--- a/plugins/_a0_connector/extensions/python/_functions/extensions/python/system_prompt/_11_tools_prompt/build_prompt/end/_70_include_remote_tool_stubs.py+++ b/plugins/_a0_connector/extensions/python/_functions/extensions/python/system_prompt/_11_tools_prompt/build_prompt/end/_70_include_remote_tool_stubs.py@@ -5,6 +5,27 @@ from typing import Anyfrom helpers.extension import Extension+_COMPUTER_USE_PROMPT = "agent.system.tool.computer_use_remote.md"+_COMPUTER_USE_TOOL_MARKER = '"tool_name": "computer_use_remote"'++class IncludeRemoteToolStubs(Extension):def execute(self, data: dict[str, Any] = {}, **kwargs: Any) -> None:- return+ if self.agent is None:+ return+ if not isinstance(data, dict):+ return+ result = data.get("result")+ if not isinstance(result, str):+ return+ if _COMPUTER_USE_TOOL_MARKER in result:+ return++ try:+ prompt = self.agent.read_prompt(_COMPUTER_USE_PROMPT).strip()+ except Exception:+ return+ if not prompt:+ return++ data["result"] = f"{result.rstrip()}\n\n{prompt}"diff --git a/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md b/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.mdnew file mode 100644index 000000000..c35285b98--- /dev/null+++ b/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md@@ -0,0 +1,31 @@+### computer_use_remote++Runtime-gated beta desktop control through a connected A0 CLI on the user's host machine. The callable contract is available in the tool prompt. Availability, backend support, and trust mode are checked when the tool runs, together with CLI presence, local enablement, and re-arm state. Computer Use enablement is scoped to the current CLI session, not scoped to a single chat context.++Use this for native host desktop UI inspection, screenshots, clicking, scrolling, typing, key presses, and status checks. Do not use it for ordinary web-page navigation or host-browser control; use the browser tool for web pages unless browser automation cannot express the task. For complex desktop workflows, load and follow skill `host-computer-use` before proceeding.++If the tool reports no CLI, disabled computer use, or `COMPUTER_USE_REARM_REQUIRED`, stop and tell the user to run `/computer-use on` in A0 CLI and approve any host permission prompt.++Call `start_session` before screen-driven tasks. Use `status` for state only, `capture` for screenshots without an action, and `stop_session` when the desktop task is complete. Interactive actions should use normalized global-screen coordinates from the most recent capture.++```json+{+ "tool_name": "computer_use_remote",+ "tool_args": {+ "action": "status"+ }+}+```++Required argument:+- `action`: one of `start_session`, `status`, `capture`, `move`, `click`, `scroll`, `key`, `type`, `stop_session`++Optional arguments by action:+- `session_id`: session returned by `start_session`+- `x`, `y`: normalized `[0,1]` global-screen coordinates for `move` and `click`+- `button`: `left`, `right`, or `middle` for `click`+- `count`: click count for `click`+- `dx`, `dy`: scroll amounts for `scroll`+- `key` or `keys`: key press value for `key`+- `text`: text to type for `type`+- `submit`: boolean Enter-after-type flag for `type`diff --git a/tests/test_a0_connector_prompt_gating.py b/tests/test_a0_connector_prompt_gating.pyindex 46f2f70a7..1f65c9ece 100644--- a/tests/test_a0_connector_prompt_gating.py+++ b/tests/test_a0_connector_prompt_gating.py@@ -95,31 +95,32 @@ def _apply_gate(context_id: str) -> str:return data["result"]-def _subscribe(- context_id: str,- *,- remote_files: dict | None = None,- remote_exec: dict | None = None,- computer_use: dict | None = None,-) -> str:- sid = _sid()- ws_runtime.register_sid(sid)- ws_runtime.subscribe_sid_to_context(sid, context_id)- if remote_files is not None:- ws_runtime.store_sid_remote_file_metadata(sid, remote_files)- if remote_exec is not None:- ws_runtime.store_sid_remote_exec_metadata(sid, remote_exec)- if computer_use is not None:- ws_runtime.store_sid_computer_use_metadata(sid, computer_use)- return sid---def test_legacy_dynamic_remote_tool_gate_is_noop():+def test_remote_tool_gate_includes_runtime_checked_computer_use_contract():prompt = _apply_gate(_context_id())assert "text_editor_remote tool" not in promptassert "code_execution_remote tool" not in prompt- assert "computer_use_remote tool" not in prompt+ assert '"tool_name": "computer_use_remote"' in prompt+ assert "### computer_use_remote" in prompt+ assert "checked when the tool runs" in prompt+++def test_computer_use_remote_prompt_is_cli_session_wide_not_context_scoped():+ prompt = _apply_gate(_context_id())++ assert "### computer_use_remote" in prompt+ assert '"tool_name": "computer_use_remote"' in prompt+ assert "scoped to the current CLI session" in prompt+ assert "not scoped to a single chat context" in prompt+++def test_computer_use_remote_prompt_keeps_runtime_failures_actionable():+ prompt = _apply_gate(_context_id())++ assert "no CLI" in prompt+ assert "disabled computer use" in prompt+ assert "COMPUTER_USE_REARM_REQUIRED" in prompt+ assert "/computer-use on" in promptdef test_remote_file_and_exec_tools_are_standard_tool_prompts_independent_from_context():@@ -132,7 +133,7 @@ def test_remote_file_and_exec_tools_are_standard_tool_prompts_independent_from_cassert "Availability and permissions are checked when the tool runs" in exec_stub-def test_beta_computer_use_remote_is_skill_only_not_standard_tool_prompt():+def test_computer_use_remote_is_standard_prompt_with_runtime_checks():skill = (PROJECT_ROOT/ "plugins"@@ -141,8 +142,12 @@ def test_beta_computer_use_remote_is_skill_only_not_standard_tool_prompt():/ "host-computer-use"/ "SKILL.md")+ standard_prompt = PROMPT_ROOT / "agent.system.tool.computer_use_remote.md"- assert not (PROMPT_ROOT / "agent.system.tool.computer_use_remote.md").exists()+ assert not (PROMPT_ROOT / "agent.system.runtime_tool.computer_use_remote.md").exists()+ assert standard_prompt.exists()+ assert '"tool_name": "computer_use_remote"' in standard_prompt.read_text(encoding="utf-8")+ assert "checked when the tool runs" in standard_prompt.read_text(encoding="utf-8")assert '"tool_name": "computer_use_remote"' in skill.read_text(encoding="utf-8")@@ -329,6 +334,7 @@ def test_remote_affordance_skills_parse():def test_remote_tool_stubs_are_self_contained_and_reference_per_tool_skills():text_stub = (PROMPT_ROOT / "agent.system.tool.text_editor_remote.md").read_text(encoding="utf-8")exec_stub = (PROMPT_ROOT / "agent.system.tool.code_execution_remote.md").read_text(encoding="utf-8")+ computer_stub = (PROMPT_ROOT / "agent.system.tool.computer_use_remote.md").read_text(encoding="utf-8")computer_skill = (PROJECT_ROOT/ "plugins"@@ -342,13 +348,16 @@ def test_remote_tool_stubs_are_self_contained_and_reference_per_tool_skills():assert "optionally load skill `host-code-execution`" in exec_stubassert '"tool_name": "text_editor_remote"' in text_stubassert '"tool_name": "code_execution_remote"' in exec_stub+ assert '"tool_name": "computer_use_remote"' in computer_stub+ assert "load and follow skill `host-computer-use`" in computer_stubassert '"tool_name": "computer_use_remote"' in computer_skill- assert "Availability, backend support, and trust mode are checked when the tool runs" in computer_skill+ assert "Availability, backend support, and trust mode are checked when the tool runs" in computer_stubassert "not `code_execution_tool`" in exec_stubassert "not to" in exec_stubassert "Docker/server/container execution" in exec_stubassert "a0-cli-remote-workflows" not in text_stubassert "a0-cli-remote-workflows" not in exec_stub+ assert "a0-cli-remote-workflows" not in computer_stubassert "a0-cli-remote-workflows" not in computer_skilldiff --git a/tests/test_default_prompt_budget.py b/tests/test_default_prompt_budget.pyindex 1f04e5dc1..c186cff9f 100644--- a/tests/test_default_prompt_budget.py+++ b/tests/test_default_prompt_budget.py@@ -50,7 +50,7 @@ async def test_default_agent0_prompt_budget_and_guardrails():# surface plus skill metadata. Keep the guardrail close to the observed# budget so prompt creep remains visible without pretending this surface is# a tiny single-tool prompt.- assert tokens.approximate_tokens(system_text) <= 8500+ assert tokens.approximate_tokens(system_text) <= 10500assert "`tool_name` must be one listed tool name" in system_textassert "- tool_args: key value pairs tool arguments" in system_textassert '"tool_name": "call_subordinate"' in system_text@@ -62,7 +62,8 @@ async def test_default_agent0_prompt_budget_and_guardrails():assert "informative but tight" in system_textassert '"tool_name": "code_execution_remote"' in system_textassert '"tool_name": "text_editor_remote"' in system_text- assert '"tool_name": "computer_use_remote"' not in system_text+ assert '"tool_name": "computer_use_remote"' in system_text+ assert "Computer Use enablement is scoped to the current CLI session" in system_textassert "host-computer-use" in system_textdiff --git a/tests/test_tool_action_contracts.py b/tests/test_tool_action_contracts.pyindex 2e7824423..9da8b5cf3 100644--- a/tests/test_tool_action_contracts.py+++ b/tests/test_tool_action_contracts.py@@ -548,17 +548,25 @@ def test_corrected_tool_prompts_only_teach_action_contract():assert "Open Document, or Desktop edit actions" not in text-def test_computer_use_remote_is_skill_gated():+def test_computer_use_remote_is_runtime_checked_standard_tool():project_root = Path(__file__).resolve().parents[1]- prompt_path = (+ standard_prompt_path = (project_root/ "plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md")+ standard_prompt_text = standard_prompt_path.read_text(encoding="utf-8")skill_text = (project_root/ "plugins/_a0_connector/skills/host-computer-use/SKILL.md").read_text(encoding="utf-8")- assert not prompt_path.exists()+ assert standard_prompt_path.exists()+ assert not (+ project_root+ / "plugins/_a0_connector/prompts/agent.system.runtime_tool.computer_use_remote.md"+ ).exists()+ assert '"tool_name": "computer_use_remote"' in standard_prompt_text+ assert "not scoped to a single chat context" in standard_prompt_text+ assert "checked when the tool runs" in standard_prompt_textassert '"tool_name": "computer_use_remote"' in skill_textassert "Beta desktop control" in skill_text