Skip to content

Agent Zero AI framework

add backend-aware computer-use and inline capture support

Alessandrobdf9cad447bc
diff --git a/plugins/_a0_connector/api/v1/capabilities.py b/plugins/_a0_connector/api/v1/capabilities.py
index cc7a2a296..2ae2e87a9 100644
--- a/plugins/_a0_connector/api/v1/capabilities.py
+++ b/plugins/_a0_connector/api/v1/capabilities.py
@@ -21,6 +21,7 @@ _BASE_FEATURES = [
"projects",
"text_editor_remote",
"code_execution_remote",
+ "computer_use_remote",
"remote_file_tree",
"token_status",
]
diff --git a/plugins/_a0_connector/api/ws_connector.py b/plugins/_a0_connector/api/ws_connector.py
index 30d4a1f82..9bbb3bd5c 100644
--- a/plugins/_a0_connector/api/ws_connector.py
+++ b/plugins/_a0_connector/api/ws_connector.py
@@ -8,16 +8,20 @@ from helpers.print_style import PrintStyle
from helpers.ws import WsHandler
from helpers.ws_manager import WsResult
-from plugins._a0_connector.helpers.event_bridge import get_context_log_entries
from plugins._a0_connector.helpers.exec_config import build_exec_config
+from plugins._a0_connector.helpers.event_bridge import get_context_log_entries
from plugins._a0_connector.helpers.ws_runtime import (
clear_remote_tree_snapshot,
+ clear_sid_computer_use_metadata,
+ fail_pending_computer_use_ops_for_sid,
fail_pending_file_ops_for_sid,
fail_pending_exec_ops_for_sid,
register_sid,
- resolve_pending_file_op,
+ resolve_pending_computer_use_op,
resolve_pending_exec_op,
+ resolve_pending_file_op,
store_remote_tree_snapshot,
+ store_sid_computer_use_metadata,
subscribe_sid_to_context,
subscribed_contexts_for_sid,
subscribed_sids_for_context,
@@ -36,6 +40,7 @@ WS_FEATURES = [
"text_editor_remote",
"remote_file_tree",
"code_execution_remote",
+ "computer_use_remote",
]
@@ -71,6 +76,11 @@ class WsConnector(WsHandler):
sid,
error="CLI disconnected before completing the requested remote execution",
)
+ fail_pending_computer_use_ops_for_sid(
+ sid,
+ error="CLI disconnected before completing the requested computer-use operation",
+ )
+ clear_sid_computer_use_metadata(sid)
PrintStyle.debug(f"[a0-connector] /ws disconnected: {sid}")
async def process(
@@ -80,6 +90,11 @@ class WsConnector(WsHandler):
sid: str,
) -> dict[str, Any] | WsResult | None:
if event == "connector_hello":
+ computer_use = data.get("computer_use")
+ if isinstance(computer_use, dict):
+ store_sid_computer_use_metadata(sid, computer_use)
+ else:
+ clear_sid_computer_use_metadata(sid)
return {
"protocol": PROTOCOL_VERSION,
"features": WS_FEATURES,
@@ -104,6 +119,9 @@ class WsConnector(WsHandler):
if event == "connector_exec_op_result":
return self._handle_exec_op_result(data, sid)
+ if event == "connector_computer_use_op_result":
+ return self._handle_computer_use_op_result(data, sid)
+
if event.startswith("connector_"):
return WsResult.error(
code="UNKNOWN_EVENT",
@@ -347,6 +365,28 @@ class WsConnector(WsHandler):
return {"op_id": op_id, "accepted": True}
+ def _handle_computer_use_op_result(
+ self,
+ data: dict[str, Any],
+ sid: str,
+ ) -> dict[str, Any] | WsResult:
+ op_id = str(data.get("op_id", "")).strip()
+ if not op_id:
+ return WsResult.error(
+ code="MISSING_OP_ID",
+ message="op_id is required",
+ correlation_id=data.get("correlationId"),
+ )
+
+ if not resolve_pending_computer_use_op(op_id, sid=sid, payload=data):
+ return WsResult.error(
+ code="UNKNOWN_OP_ID",
+ message=f"No pending computer-use operation for op_id '{op_id}'",
+ correlation_id=data.get("correlationId"),
+ )
+
+ return {"op_id": op_id, "accepted": True}
+
async def _resolve_context(
self,
*,
diff --git a/plugins/_a0_connector/helpers/exec_config.py b/plugins/_a0_connector/helpers/exec_config.py
index fe29f1de1..0c41bba18 100644
--- a/plugins/_a0_connector/helpers/exec_config.py
+++ b/plugins/_a0_connector/helpers/exec_config.py
@@ -2,95 +2,35 @@ from __future__ import annotations
from typing import Any
-_TIMEOUT_KEYS = (
- "first_output_timeout",
- "between_output_timeout",
- "max_exec_timeout",
- "dialog_timeout",
-)
-_DEFAULT_CODE_EXEC_TIMEOUTS = {
- "first_output_timeout": 30,
- "between_output_timeout": 15,
- "max_exec_timeout": 180,
- "dialog_timeout": 5,
-}
-_DEFAULT_OUTPUT_TIMEOUTS = {
- "first_output_timeout": 90,
- "between_output_timeout": 45,
- "max_exec_timeout": 300,
- "dialog_timeout": 5,
-}
-_DEFAULT_PROMPT_PATTERNS = [
- r"(\(venv\)).+[$#] ?$",
- r"root@[^:]+:[^#]+# ?$",
- r"[a-zA-Z0-9_.-]+@[^:]+:[^$#]+[$#] ?$",
- r"\(?.*\)?\s*PS\s+[^>]+> ?$",
-]
-_DEFAULT_DIALOG_PATTERNS = [
- r"Y/N",
- r"yes/no",
- r":\s*$",
- r"\?\s*$",
-]
+_TIMEOUT_KEYS = ("first_output_timeout", "between_output_timeout", "max_exec_timeout", "dialog_timeout")
-def _coerce_timeout_group(raw: Any, defaults: dict[str, int]) -> dict[str, int]:
- group = raw if isinstance(raw, dict) else {}
- result: dict[str, int] = {}
- for key in _TIMEOUT_KEYS:
- value = group.get(key, defaults[key])
- try:
- result[key] = int(value)
- except (TypeError, ValueError):
- result[key] = defaults[key]
- return result
-
-
-def _pattern_lines(raw: Any, defaults: list[str]) -> list[str]:
- if isinstance(raw, list):
- values = raw
- elif isinstance(raw, str):
- values = raw.splitlines()
+def _parse_patterns(value: object) -> list[str]:
+ if isinstance(value, str):
+ items = value.splitlines()
+ elif isinstance(value, (list, tuple)):
+ items = value
else:
- values = defaults
+ return []
+ return [str(item).strip() for item in items if str(item).strip()]
+
- patterns = [str(value).strip() for value in values if str(value).strip()]
- return patterns or list(defaults)
+def _parse_timeouts(cfg: dict[str, Any], prefix: str, defaults: tuple[int, ...]) -> dict[str, int]:
+ return {
+ key: int(cfg.get(f"{prefix}_{key}", default))
+ for key, default in zip(_TIMEOUT_KEYS, defaults)
+ }
-def build_exec_config() -> dict[str, Any]:
+def build_exec_config(*, agent: object | None = None) -> dict[str, Any]:
from helpers import plugins
- try:
- config = plugins.get_plugin_config("_code_execution") or {}
- except Exception:
- config = {}
-
+ cfg = plugins.get_plugin_config("_code_execution", agent=agent) or {}
return {
"version": 1,
- "code_exec_timeouts": _coerce_timeout_group(
- config.get("code_exec_timeouts") or {
- key: config.get(f"code_exec_{key}")
- for key in _TIMEOUT_KEYS
- if f"code_exec_{key}" in config
- },
- _DEFAULT_CODE_EXEC_TIMEOUTS,
- ),
- "output_timeouts": _coerce_timeout_group(
- config.get("output_timeouts") or {
- key: config.get(f"output_{key}")
- for key in _TIMEOUT_KEYS
- if f"output_{key}" in config
- },
- _DEFAULT_OUTPUT_TIMEOUTS,
- ),
- "prompt_patterns": _pattern_lines(
- config.get("prompt_patterns"),
- _DEFAULT_PROMPT_PATTERNS,
- ),
- "dialog_patterns": _pattern_lines(
- config.get("dialog_patterns"),
- _DEFAULT_DIALOG_PATTERNS,
- ),
+ "code_exec_timeouts": _parse_timeouts(cfg, "code_exec", (30, 15, 180, 5)),
+ "output_timeouts": _parse_timeouts(cfg, "output", (90, 45, 300, 5)),
+ "prompt_patterns": _parse_patterns(cfg.get("prompt_patterns", "")),
+ "dialog_patterns": _parse_patterns(cfg.get("dialog_patterns", "")),
}
diff --git a/plugins/_a0_connector/helpers/ws_runtime.py b/plugins/_a0_connector/helpers/ws_runtime.py
index 164e97e38..0efed71a7 100644
--- a/plugins/_a0_connector/helpers/ws_runtime.py
+++ b/plugins/_a0_connector/helpers/ws_runtime.py
@@ -23,6 +23,14 @@ class PendingExecOperation:
context_id: str | None = None
+@dataclass
+class PendingComputerUseOperation:
+ sid: str
+ loop: asyncio.AbstractEventLoop
+ future: asyncio.Future[dict[str, Any]]
+ context_id: str | None = None
+
+
@dataclass(frozen=True)
class RemoteTreeSnapshot:
sid: str
@@ -30,11 +38,26 @@ class RemoteTreeSnapshot:
updated_at: float
+@dataclass(frozen=True)
+class ComputerUseMetadata:
+ supported: bool
+ enabled: bool
+ trust_mode: str
+ artifact_root: str
+ backend_id: str
+ backend_family: str
+ features: tuple[str, ...]
+ support_reason: str
+ updated_at: float
+
+
_context_subscriptions: dict[str, set[str]] = {}
_sid_contexts: dict[str, set[str]] = {}
_pending_file_ops: dict[str, PendingFileOperation] = {}
_pending_exec_ops: dict[str, PendingExecOperation] = {}
+_pending_computer_use_ops: dict[str, PendingComputerUseOperation] = {}
_remote_tree_snapshots: dict[str, RemoteTreeSnapshot] = {}
+_sid_computer_use_metadata: dict[str, ComputerUseMetadata] = {}
_state_lock = threading.RLock()
@@ -47,6 +70,7 @@ def unregister_sid(sid: str) -> set[str]:
with _state_lock:
contexts = _sid_contexts.pop(sid, set())
_remote_tree_snapshots.pop(sid, None)
+ _sid_computer_use_metadata.pop(sid, None)
for context_id in contexts:
subscribers = _context_subscriptions.get(context_id)
if not subscribers:
@@ -143,6 +167,61 @@ def select_target_sid(context_id: str) -> str | None:
return sorted(subscribers)[0]
+def store_sid_computer_use_metadata(sid: str, payload: dict[str, Any]) -> ComputerUseMetadata:
+ features_value = payload.get("features")
+ if isinstance(features_value, (list, tuple)):
+ features = tuple(str(item).strip() for item in features_value if str(item).strip())
+ else:
+ features = ()
+ metadata = ComputerUseMetadata(
+ supported=bool(payload.get("supported")),
+ enabled=bool(payload.get("supported")) and bool(payload.get("enabled")),
+ trust_mode=str(payload.get("trust_mode", "") or "").strip(),
+ artifact_root=str(payload.get("artifact_root", "") or "").strip(),
+ backend_id=str(payload.get("backend_id", "") or "").strip(),
+ backend_family=str(payload.get("backend_family", "") or "").strip(),
+ features=features,
+ support_reason=str(payload.get("support_reason", "") or "").strip(),
+ updated_at=time.time(),
+ )
+ with _state_lock:
+ _sid_computer_use_metadata[sid] = metadata
+ return metadata
+
+
+def clear_sid_computer_use_metadata(sid: str) -> None:
+ with _state_lock:
+ _sid_computer_use_metadata.pop(sid, None)
+
+
+def computer_use_metadata_for_sid(sid: str) -> dict[str, Any] | None:
+ with _state_lock:
+ metadata = _sid_computer_use_metadata.get(sid)
+ if metadata is None:
+ return None
+ return {
+ "supported": metadata.supported,
+ "enabled": metadata.enabled,
+ "trust_mode": metadata.trust_mode,
+ "artifact_root": metadata.artifact_root,
+ "backend_id": metadata.backend_id,
+ "backend_family": metadata.backend_family,
+ "features": list(metadata.features),
+ "support_reason": metadata.support_reason,
+ "updated_at": metadata.updated_at,
+ }
+
+
+def select_computer_use_target_sid(context_id: str) -> str | None:
+ with _state_lock:
+ subscribers = sorted(_context_subscriptions.get(context_id, set()))
+ for sid in subscribers:
+ metadata = _sid_computer_use_metadata.get(sid)
+ if metadata and metadata.supported and metadata.enabled:
+ return sid
+ return None
+
+
def store_pending_file_op(
op_id: str,
*,
@@ -171,14 +250,7 @@ def resolve_pending_file_op(
sid: str,
payload: dict[str, Any],
) -> bool:
- with _state_lock:
- pending = _pending_file_ops.get(op_id)
- if pending is None or pending.sid != sid:
- return False
- _pending_file_ops.pop(op_id, None)
-
- pending.loop.call_soon_threadsafe(_set_future_result, pending.future, dict(payload))
- return True
+ return _resolve_pending(_pending_file_ops, op_id, sid=sid, payload=payload)
def fail_pending_file_op(
@@ -187,32 +259,11 @@ def fail_pending_file_op(
sid: str | None = None,
error: str,
) -> bool:
- with _state_lock:
- pending = _pending_file_ops.get(op_id)
- if pending is None:
- return False
- if sid is not None and pending.sid != sid:
- return False
- _pending_file_ops.pop(op_id, None)
-
- payload = {"op_id": op_id, "ok": False, "error": error}
- pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)
- return True
+ return _fail_pending(_pending_file_ops, op_id, sid=sid, error=error)
def fail_pending_file_ops_for_sid(sid: str, *, error: str) -> None:
- with _state_lock:
- matches = [
- (op_id, pending)
- for op_id, pending in _pending_file_ops.items()
- if pending.sid == sid
- ]
- for op_id, _pending in matches:
- _pending_file_ops.pop(op_id, None)
-
- for op_id, pending in matches:
- payload = {"op_id": op_id, "ok": False, "error": error}
- pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)
+ _fail_pending_for_sid(_pending_file_ops, sid=sid, error=error)
def store_pending_exec_op(
@@ -243,48 +294,127 @@ def resolve_pending_exec_op(
sid: str,
payload: dict[str, Any],
) -> bool:
+ return _resolve_pending(_pending_exec_ops, op_id, sid=sid, payload=payload)
+
+
+def fail_pending_exec_op(
+ op_id: str,
+ *,
+ sid: str | None = None,
+ error: str,
+) -> bool:
+ return _fail_pending(_pending_exec_ops, op_id, sid=sid, error=error)
+
+
+def fail_pending_exec_ops_for_sid(sid: str, *, error: str) -> None:
+ _fail_pending_for_sid(_pending_exec_ops, sid=sid, error=error)
+
+
+def store_pending_computer_use_op(
+ op_id: str,
+ *,
+ sid: str,
+ future: asyncio.Future[dict[str, Any]],
+ loop: asyncio.AbstractEventLoop,
+ context_id: str | None = None,
+) -> None:
+ with _state_lock:
+ _pending_computer_use_ops[op_id] = PendingComputerUseOperation(
+ sid=sid,
+ loop=loop,
+ future=future,
+ context_id=context_id,
+ )
+
+
+def clear_pending_computer_use_op(op_id: str) -> None:
with _state_lock:
- pending = _pending_exec_ops.get(op_id)
+ _pending_computer_use_ops.pop(op_id, None)
+
+
+def resolve_pending_computer_use_op(
+ op_id: str,
+ *,
+ sid: str,
+ payload: dict[str, Any],
+) -> bool:
+ return _resolve_pending(_pending_computer_use_ops, op_id, sid=sid, payload=payload)
+
+
+def fail_pending_computer_use_op(
+ op_id: str,
+ *,
+ sid: str | None = None,
+ error: str,
+) -> bool:
+ return _fail_pending(_pending_computer_use_ops, op_id, sid=sid, error=error)
+
+
+def fail_pending_computer_use_ops_for_sid(sid: str, *, error: str) -> None:
+ _fail_pending_for_sid(_pending_computer_use_ops, sid=sid, error=error)
+
+
+def _resolve_pending(
+ registry: dict[str, PendingFileOperation | PendingExecOperation | PendingComputerUseOperation],
+ op_id: str,
+ *,
+ sid: str,
+ payload: dict[str, Any],
+) -> bool:
+ with _state_lock:
+ pending = registry.get(op_id)
if pending is None or pending.sid != sid:
return False
- _pending_exec_ops.pop(op_id, None)
+ registry.pop(op_id, None)
pending.loop.call_soon_threadsafe(_set_future_result, pending.future, dict(payload))
return True
-def fail_pending_exec_op(
+def _fail_pending(
+ registry: dict[str, PendingFileOperation | PendingExecOperation | PendingComputerUseOperation],
op_id: str,
*,
- sid: str | None = None,
+ sid: str | None,
error: str,
) -> bool:
with _state_lock:
- pending = _pending_exec_ops.get(op_id)
+ pending = registry.get(op_id)
if pending is None:
return False
if sid is not None and pending.sid != sid:
return False
- _pending_exec_ops.pop(op_id, None)
+ registry.pop(op_id, None)
- payload = {"op_id": op_id, "ok": False, "error": error}
- pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)
+ pending.loop.call_soon_threadsafe(
+ _set_future_result,
+ pending.future,
+ {"op_id": op_id, "ok": False, "error": error},
+ )
return True
-def fail_pending_exec_ops_for_sid(sid: str, *, error: str) -> None:
+def _fail_pending_for_sid(
+ registry: dict[str, PendingFileOperation | PendingExecOperation | PendingComputerUseOperation],
+ *,
+ sid: str,
+ error: str,
+) -> None:
with _state_lock:
matches = [
(op_id, pending)
- for op_id, pending in _pending_exec_ops.items()
+ for op_id, pending in registry.items()
if pending.sid == sid
]
for op_id, _pending in matches:
- _pending_exec_ops.pop(op_id, None)
+ registry.pop(op_id, None)
for op_id, pending in matches:
- payload = {"op_id": op_id, "ok": False, "error": error}
- pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)
+ pending.loop.call_soon_threadsafe(
+ _set_future_result,
+ pending.future,
+ {"op_id": op_id, "ok": False, "error": error},
+ )
def _set_future_result(
diff --git a/plugins/_a0_connector/plugin.yaml b/plugins/_a0_connector/plugin.yaml
index b5ff1f51a..5a4e6685b 100644
--- a/plugins/_a0_connector/plugin.yaml
+++ b/plugins/_a0_connector/plugin.yaml
@@ -1,7 +1,7 @@
name: _a0_connector
title: A0 Connector
description: Current Agent Zero connector plugin for HTTP plus /ws integration, using session auth and handler activation through auth.handlers.
-version: 1.4
+version: 1.5.0
settings_sections:
- external
- developer
diff --git a/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md b/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md
new file mode 100644
index 000000000..59a1d5ddc
--- /dev/null
+++ b/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md
@@ -0,0 +1,34 @@
+# computer_use_remote tool
+
+Use the connected CLI host machine as a local desktop target.
+
+## Preferred Scope
+- Use this for local desktop and native UI tasks on the connected machine.
+- For ordinary website browsing, search, form filling, and web downloads, prefer `browser_agent`.
+- If the user is flexible and the task is browser-only, briefly guide them toward browser tools because they are usually more reliable and token-efficient than screenshot-driven computer use.
+- Before doing real computer-use work, load the `computer-use-remote` skill and follow it.
+
+## Requirements
+- A CLI client must be connected to this context via the shared `/ws` namespace.
+- The CLI must advertise `computer_use_remote` support and local computer use must be enabled there.
+- In `free_run`, do not expect a fresh approval prompt. If restore is no longer valid, the tool will surface `COMPUTER_USE_REARM_REQUIRED`.
+
+## Minimal Rules
+- Treat user interventions as high-priority control signals.
+- If the user says `stop`, `pause`, `abort`, `hold`, `don't continue`, or equivalent, halt immediately and do not use computer-use tools again until the user explicitly resumes.
+- Call `start_session` first. It automatically attaches the current screen.
+- Decide from the latest screenshot, not from memory.
+- Interactive actions (`move`, `click`, `scroll`, `key`, `type`) automatically attach a fresh screenshot after they run.
+- Use `capture` only when you need another screen refresh without taking an action.
+- Prefer keyboard actions over pointer actions whenever a reliable keyboard path exists.
+
+## Arguments
+- `action`: one of `start_session`, `status`, `capture`, `move`, `click`, `scroll`, `key`, `type`, `stop_session`
+- `session_id`: optional for actions after `start_session`
+
+Action-specific fields:
+- `move`: `x`, `y` normalized to `[0,1]`
+- `click`: optional `x`, `y`, plus optional `button` (`left`, `right`, `middle`) and `count`
+- `scroll`: `dx`, `dy`
+- `key`: `key` or `keys`
+- `type`: `text`, optional `submit` boolean
diff --git a/plugins/_a0_connector/tools/computer_use_remote.py b/plugins/_a0_connector/tools/computer_use_remote.py
new file mode 100644
index 000000000..efbe7916b
--- /dev/null
+++ b/plugins/_a0_connector/tools/computer_use_remote.py
@@ -0,0 +1,433 @@
+"""computer_use_remote tool — drive the CLI host machine through the connected frontend."""
+from __future__ import annotations
+
+import asyncio
+import base64
+from pathlib import Path
+import uuid
+from typing import Any
+
+from helpers import history
+from helpers.tool import Response, Tool
+from helpers.ws import NAMESPACE
+from helpers.ws_manager import ConnectionNotFoundError, get_shared_ws_manager
+
+from plugins._a0_connector.helpers.ws_runtime import (
+ clear_pending_computer_use_op,
+ select_computer_use_target_sid,
+ store_pending_computer_use_op,
+)
+
+
+COMPUTER_USE_OP_TIMEOUT = 180.0
+COMPUTER_USE_OP_EVENT = "connector_computer_use_op"
+CAPTURE_TOKENS_ESTIMATE = 1500
+_AUTO_CAPTURE_ACTIONS = {
+ "start_session",
+ "move",
+ "click",
+ "scroll",
+ "key",
+ "type",
+}
+_SETTLE_DELAY_START_SESSION = 0.2
+_SETTLE_DELAY_GLOBAL_FOCUS = 0.45
+_SETTLE_DELAY_PLAIN_ENTER = 0.3
+_SETTLE_DELAY_SUBMIT = 0.45
+_SUPPORTED_ACTIONS = {
+ "start_session",
+ "status",
+ "capture",
+ "move",
+ "click",
+ "scroll",
+ "key",
+ "type",
+ "stop_session",
+}
+
+
+class ComputerUseRemote(Tool):
+ async def execute(self, **kwargs: Any) -> Response:
+ action = str(self.args.get("action") or "").strip().lower()
+ if action not in _SUPPORTED_ACTIONS:
+ return Response(
+ message=(
+ "action is required and must be one of: "
+ "start_session, status, capture, move, click, scroll, key, type, stop_session"
+ ),
+ break_loop=False,
+ )
+
+ context_id = self.agent.context.id
+ sid = select_computer_use_target_sid(context_id)
+ if not sid:
+ return Response(
+ message=(
+ "computer_use_remote: no subscribed CLI in this context currently advertises "
+ "enabled local computer use. Enable it in the CLI with F2 and choose a trust mode first."
+ ),
+ break_loop=False,
+ )
+
+ try:
+ payload = self._build_payload(op_id=str(uuid.uuid4()), context_id=context_id, action=action)
+ result = await self._dispatch_payload(sid=sid, payload=payload)
+ capture_note = await self._maybe_attach_latest_capture(
+ action=action,
+ sid=sid,
+ context_id=context_id,
+ result=result,
+ )
+ except ValueError as exc:
+ return Response(
+ message=f"computer_use_remote: {exc}",
+ break_loop=False,
+ )
+ except ConnectionNotFoundError:
+ return Response(
+ message=(
+ "computer_use_remote: the selected CLI disconnected before the request "
+ "could be delivered."
+ ),
+ break_loop=False,
+ )
+ except asyncio.TimeoutError:
+ return Response(
+ message=f"computer_use_remote: timed out waiting for action={action!r}",
+ break_loop=False,
+ )
+ except Exception as exc:
+ return Response(
+ message=f"computer_use_remote: error sending action={action!r}: {exc}",
+ break_loop=False,
+ )
+
+ message = self._extract_result(action, result)
+ if capture_note:
+ message = f"{message} {capture_note}".strip()
+
+ return Response(
+ message=message,
+ break_loop=False,
+ )
+
+ async def _dispatch_payload(self, *, sid: str, payload: dict[str, Any]) -> dict[str, Any]:
+ op_id = str(payload.get("op_id") or "").strip()
+ loop = asyncio.get_running_loop()
+ future: asyncio.Future[dict[str, Any]] = loop.create_future()
+ store_pending_computer_use_op(
+ op_id,
+ sid=sid,
+ future=future,
+ loop=loop,
+ context_id=str(payload.get("context_id") or "").strip() or None,
+ )
+
+ try:
+ await get_shared_ws_manager().emit_to(
+ NAMESPACE,
+ sid,
+ COMPUTER_USE_OP_EVENT,
+ payload,
+ handler_id=f"{self.__class__.__module__}.{self.__class__.__name__}",
+ )
+ result = await asyncio.wait_for(future, timeout=COMPUTER_USE_OP_TIMEOUT)
+ finally:
+ clear_pending_computer_use_op(op_id)
+
+ if isinstance(result, dict):
+ return result
+ raise RuntimeError(f"Unexpected response format from CLI: {result!r}")
+
+ async def _maybe_attach_latest_capture(
+ self,
+ *,
+ action: str,
+ sid: str,
+ context_id: str,
+ result: dict[str, Any],
+ ) -> str:
+ if action not in _AUTO_CAPTURE_ACTIONS or not bool(result.get("ok")):
+ return ""
+
+ data = result.get("result")
+ result_data = dict(data) if isinstance(data, dict) else {}
+ session_id = str(result_data.get("session_id") or self.args.get("session_id") or "").strip()
+ if not session_id:
+ return ""
+
+ settle_seconds = self._auto_capture_settle_seconds(action)
+ if settle_seconds > 0:
+ await asyncio.sleep(settle_seconds)
+
+ capture_result = await self._dispatch_payload(
+ sid=sid,
+ payload={
+ "op_id": str(uuid.uuid4()),
+ "context_id": context_id,
+ "action": "capture",
+ "session_id": session_id,
+ },
+ )
+ if not bool(capture_result.get("ok")):
+ return f"Automatic screen refresh failed: {self._format_error(capture_result)}"
+
+ capture_data = capture_result.get("result")
+ if not isinstance(capture_data, dict):
+ return "Automatic screen refresh failed: missing capture payload."
+
+ self._record_capture(capture_data)
+ return "Latest screen attached."
+
+ def _auto_capture_settle_seconds(self, action: str) -> float:
+ if action == "start_session":
+ return _SETTLE_DELAY_START_SESSION
+ if action == "type" and self._coerce_bool(self.args.get("submit")):
+ return _SETTLE_DELAY_SUBMIT
+ if action != "key":
+ return 0.0
+
+ keyset = {key.lower() for key in self._requested_keys()}
+ if "super" in keyset or ("alt" in keyset and "tab" in keyset):
+ return _SETTLE_DELAY_GLOBAL_FOCUS
+ if keyset == {"enter"}:
+ return _SETTLE_DELAY_PLAIN_ENTER
+ return 0.0
+
+ def _requested_keys(self) -> list[str]:
+ keys_value = self.args.get("keys")
+ if isinstance(keys_value, (list, tuple)):
+ return [str(item).strip() for item in keys_value if str(item).strip()]
+ raw = str(keys_value or self.args.get("key", "") or "").strip()
+ if not raw:
+ return []
+ return [part.strip() for part in raw.split("+") if part.strip()]
+
+ def _build_payload(self, *, op_id: str, context_id: str, action: str) -> dict[str, Any]:
+ payload: dict[str, Any] = {
+ "op_id": op_id,
+ "context_id": context_id,
+ "action": action,
+ }
+ session_id = str(self.args.get("session_id", "") or "").strip()
+ if session_id:
+ payload["session_id"] = session_id
+
+ if action == "move":
+ payload["x"] = self.args.get("x")
+ payload["y"] = self.args.get("y")
+ elif action == "click":
+ if "x" in self.args:
+ payload["x"] = self.args.get("x")
+ if "y" in self.args:
+ payload["y"] = self.args.get("y")
+ payload["button"] = self.args.get("button", "left")
+ payload["count"] = self._coerce_int(self.args.get("count", 1), name="count")
+ elif action == "scroll":
+ payload["dx"] = self._coerce_int(self.args.get("dx", self.args.get("delta_x", 0)), name="dx")
+ payload["dy"] = self._coerce_int(self.args.get("dy", self.args.get("delta_y", 0)), name="dy")
+ elif action == "key":
+ if "keys" in self.args:
+ payload["keys"] = self.args.get("keys")
+ elif "key" in self.args:
+ payload["key"] = self.args.get("key")
+ elif action == "type":
+ payload["text"] = self.args.get("text", "")
+ if self._coerce_bool(self.args.get("submit")):
+ payload["submit"] = True
+
+ return payload
+
+ def _extract_result(self, action: str, result: Any) -> str:
+ if not isinstance(result, dict):
+ return f"Unexpected response format from CLI: {result!r}"
+
+ ok = bool(result.get("ok"))
+ data = result.get("result")
+
+ if not ok:
+ return self._format_error(result)
+
+ if not isinstance(data, dict):
+ data = {}
+
+ if action == "capture":
+ self._record_capture(data)
+ return "Current screen attached."
+ if action == "status":
+ return self._format_status(data)
+ if action == "start_session":
+ return (
+ f"Computer-use session started: session_id={data.get('session_id', '?')} "
+ f"size={data.get('width', '?')}x{data.get('height', '?')}"
+ )
+ if action == "stop_session":
+ return "Computer-use session stopped."
+ if action == "move":
+ return f"Pointer moved to x={data.get('x')} y={data.get('y')}."
+ if action == "click":
+ return f"Clicked {data.get('button', 'left')} button {data.get('count', 1)} time(s)."
+ if action == "scroll":
+ return f"Scrolled dx={data.get('dx', 0)} dy={data.get('dy', 0)}."
+ if action == "key":
+ keys = data.get("keys") or []
+ return f"Sent keys: {keys!r}."
+ if action == "type":
+ text = str(data.get("text", "") or "")
+ if data.get("submitted"):
+ return f"Typed {len(text)} character(s) and submitted."
+ return f"Typed {len(text)} character(s)."
+ return str(data)
+
+ def _format_error(self, result: dict[str, Any]) -> str:
+ error = str(result.get("error") or "Unknown error")
+ code = str(result.get("code") or "")
+ if code:
+ return f"{code}: {error}"
+ return error
+
+ def _format_status(self, data: dict[str, Any]) -> str:
+ status = str(data.get("status", "unknown") or "unknown")
+ trust_mode = str(data.get("trust_mode", "") or "")
+ backend_id = str(data.get("backend_id", "") or "").strip()
+ backend_family = str(data.get("backend_family", "") or "").strip()
+ active_contexts = data.get("active_contexts") or []
+ active_text = ", ".join(str(item) for item in active_contexts) if active_contexts else "none"
+ backend_text = ""
+ if backend_id:
+ backend_text = backend_id
+ if backend_family:
+ backend_text = f"{backend_text}/{backend_family}"
+ if backend_text:
+ return (
+ f"Computer use status={status}, trust_mode={trust_mode or 'unknown'}, "
+ f"backend={backend_text}, active_contexts={active_text}."
+ )
+ return f"Computer use status={status}, trust_mode={trust_mode or 'unknown'}, active_contexts={active_text}."
+
+ def _record_capture(self, data: dict[str, Any]) -> str:
+ image_b64 = self._capture_image_base64(data)
+ width = data.get("width", "?")
+ height = data.get("height", "?")
+ summary = f"Computer-use capture {width}x{height}."
+ content = [
+ {"type": "text", "text": summary},
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_b64}"}},
+ ]
+ raw_message = history.RawMessage(raw_content=content, preview=summary)
+ self.agent.hist_add_message(False, content=raw_message, tokens=CAPTURE_TOKENS_ESTIMATE)
+ self._prune_prior_capture_history()
+ return summary
+
+ def _prune_prior_capture_history(self) -> None:
+ history_obj = getattr(self.agent, "history", None)
+ if history_obj is None:
+ return
+
+ capture_messages = self._collect_capture_messages(history_obj)
+ if len(capture_messages) <= 1:
+ return
+
+ latest = capture_messages[-1]
+ for message in capture_messages[:-1]:
+ if message is latest:
+ continue
+ preview = self._capture_preview_from_message(message)
+ if not preview:
+ continue
+ message.content = f"{preview} [embedded image removed]"
+ if hasattr(message, "summary"):
+ message.summary = ""
+ if hasattr(message, "calculate_tokens"):
+ message.tokens = message.calculate_tokens()
+
+ def _collect_capture_messages(self, history_obj: Any) -> list[Any]:
+ messages: list[Any] = []
+
+ def collect_topic(topic: Any) -> None:
+ topic_messages = getattr(topic, "messages", None)
+ if isinstance(topic_messages, list):
+ for message in topic_messages:
+ if self._capture_preview_from_message(message):
+ messages.append(message)
+
+ bulks = getattr(history_obj, "bulks", None)
+ if isinstance(bulks, list):
+ for bulk in bulks:
+ self._collect_capture_messages_from_record(bulk, messages)
+
+ topics = getattr(history_obj, "topics", None)
+ if isinstance(topics, list):
+ for topic in topics:
+ collect_topic(topic)
+
+ current = getattr(history_obj, "current", None)
+ if current is not None:
+ collect_topic(current)
+
+ return messages
+
+ def _collect_capture_messages_from_record(self, record: Any, messages: list[Any]) -> None:
+ topic_messages = getattr(record, "messages", None)
+ if isinstance(topic_messages, list):
+ for message in topic_messages:
+ if self._capture_preview_from_message(message):
+ messages.append(message)
+ return
+
+ nested_records = getattr(record, "records", None)
+ if isinstance(nested_records, list):
+ for nested in nested_records:
+ self._collect_capture_messages_from_record(nested, messages)
+
+ def _capture_preview_from_message(self, message: Any) -> str:
+ content = getattr(message, "content", None)
+ if not isinstance(content, dict):
+ return ""
+ raw_content = content.get("raw_content")
+ preview = content.get("preview")
+ if raw_content is None or not isinstance(preview, str):
+ return ""
+ if preview.startswith("Computer-use capture "):
+ return preview
+ return ""
+
+ def _capture_image_base64(self, data: dict[str, Any]) -> str:
+ inline_payload = str(data.get("png_base64", "") or "").strip()
+ if inline_payload:
+ try:
+ base64.b64decode(inline_payload, validate=True)
+ except Exception:
+ pass
+ else:
+ return inline_payload
+
+ image_path, _display_path = self._resolve_capture_path(data)
+ return base64.b64encode(image_path.read_bytes()).decode("utf-8")
+
+ def _resolve_capture_path(self, data: dict[str, Any]) -> tuple[Path, str]:
+ candidates = [
+ str(data.get("capture_path", "") or "").strip(),
+ str(data.get("container_path", "") or "").strip(),
+ str(data.get("host_path", "") or "").strip(),
+ ]
+ for candidate in candidates:
+ if candidate and Path(candidate).exists():
+ return Path(candidate), candidate
+ raise FileNotFoundError(
+ f"Capture artifact was not found in any advertised path: {candidates!r}"
+ )
+
+ def _coerce_int(self, value: object, *, name: str) -> int:
+ try:
+ return int(value or 0)
+ except (TypeError, ValueError) as exc:
+ raise ValueError(f"{name} must be an integer") from exc
+
+ def _coerce_bool(self, value: object) -> bool:
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, (int, float)):
+ return bool(value)
+ return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
diff --git a/skills/computer-use-remote/SKILL.md b/skills/computer-use-remote/SKILL.md
new file mode 100644
index 000000000..8675ed53c
--- /dev/null
+++ b/skills/computer-use-remote/SKILL.md
@@ -0,0 +1,50 @@
+---
+name: computer-use-remote
+description: Detailed operating guide for using computer_use_remote on the connected local machine. Load this skill before using computer_use_remote for desktop control, screenshots, menus, browser chrome, or other native UI tasks.
+version: 1.0.0
+author: Agent Zero Team
+tags: ["computer-use", "desktop", "local-ui", "screenshots", "native-ui"]
+trigger_patterns:
+ - "computer use"
+ - "computer-use"
+ - "computer_use_remote"
+ - "local desktop control"
+ - "control local browser"
+ - "click on screen"
+ - "native ui"
+allowed_tools:
+ - computer_use_remote
+ - code_execution_remote
+---
+
+# Computer Use Remote
+
+## When to Use
+
+Load this skill before using `computer_use_remote` for local desktop and native UI tasks on the connected machine.
+
+For ordinary website browsing, search, form filling, and web downloads, prefer `browser_agent` instead. If the user is flexible and the task is browser-only, guide them toward browser tools because they are usually more reliable and token-efficient than screenshot-driven computer use.
+
+## Core Loop
+
+1. Call `start_session` first.
+2. Decide from the latest screenshot, not from memory.
+3. Interactive actions (`move`, `click`, `scroll`, `key`, `type`) already attach a fresh screenshot after they run.
+4. Use `capture` only when you need another screenshot without taking an action.
+
+## Operating Rules
+
+- Only the latest screenshot or a definitive tool result counts as evidence.
+- Prefer keyboard actions over pointer actions whenever a reliable keyboard path exists.
+- When a menu or popup is open, treat it as the active UI and prefer keyboard navigation over clicking small transient rows by coordinate.
+- If a click dismisses a menu or popup without producing the expected next UI, treat that attempt as failed.
+- If the same approach has already failed twice without visible progress, switch strategy instead of repeating it.
+- Do not infer focus or task completion from chat logs, sidebars, tool summaries, or status text.
+- For browser-navigation tasks done through this tool, only claim success if the browser content area visibly shows the destination page or result.
+- Use `type(..., submit=true)` only for URL or navigation-style entry where Enter should fire immediately after typing.
+- Do not use `submit=true` for ordinary text fields. Type first, then send `enter` separately if needed.
+
+## Control Signals
+
+- Treat user interventions as high-priority control signals.
+- If the user says `stop`, `pause`, `abort`, `hold`, `don't continue`, or equivalent, halt immediately and do not use computer-use tools again until the user explicitly resumes.