Agent Zero AI framework
add backend-aware computer-use and inline capture support
Alessandrobdf9cad447bc
diff --git a/plugins/_a0_connector/api/v1/capabilities.py b/plugins/_a0_connector/api/v1/capabilities.pyindex cc7a2a296..2ae2e87a9 100644--- a/plugins/_a0_connector/api/v1/capabilities.py+++ b/plugins/_a0_connector/api/v1/capabilities.py@@ -21,6 +21,7 @@ _BASE_FEATURES = ["projects","text_editor_remote","code_execution_remote",+ "computer_use_remote","remote_file_tree","token_status",]diff --git a/plugins/_a0_connector/api/ws_connector.py b/plugins/_a0_connector/api/ws_connector.pyindex 30d4a1f82..9bbb3bd5c 100644--- a/plugins/_a0_connector/api/ws_connector.py+++ b/plugins/_a0_connector/api/ws_connector.py@@ -8,16 +8,20 @@ from helpers.print_style import PrintStylefrom helpers.ws import WsHandlerfrom helpers.ws_manager import WsResult-from plugins._a0_connector.helpers.event_bridge import get_context_log_entriesfrom plugins._a0_connector.helpers.exec_config import build_exec_config+from plugins._a0_connector.helpers.event_bridge import get_context_log_entriesfrom plugins._a0_connector.helpers.ws_runtime import (clear_remote_tree_snapshot,+ clear_sid_computer_use_metadata,+ fail_pending_computer_use_ops_for_sid,fail_pending_file_ops_for_sid,fail_pending_exec_ops_for_sid,register_sid,- resolve_pending_file_op,+ resolve_pending_computer_use_op,resolve_pending_exec_op,+ resolve_pending_file_op,store_remote_tree_snapshot,+ store_sid_computer_use_metadata,subscribe_sid_to_context,subscribed_contexts_for_sid,subscribed_sids_for_context,@@ -36,6 +40,7 @@ WS_FEATURES = ["text_editor_remote","remote_file_tree","code_execution_remote",+ "computer_use_remote",]@@ -71,6 +76,11 @@ class WsConnector(WsHandler):sid,error="CLI disconnected before completing the requested remote execution",)+ fail_pending_computer_use_ops_for_sid(+ sid,+ error="CLI disconnected before completing the requested computer-use operation",+ )+ clear_sid_computer_use_metadata(sid)PrintStyle.debug(f"[a0-connector] /ws disconnected: {sid}")async def process(@@ -80,6 +90,11 @@ class WsConnector(WsHandler):sid: str,) -> dict[str, Any] | WsResult | None:if event == "connector_hello":+ computer_use = data.get("computer_use")+ if isinstance(computer_use, dict):+ store_sid_computer_use_metadata(sid, computer_use)+ else:+ clear_sid_computer_use_metadata(sid)return {"protocol": PROTOCOL_VERSION,"features": WS_FEATURES,@@ -104,6 +119,9 @@ class WsConnector(WsHandler):if event == "connector_exec_op_result":return self._handle_exec_op_result(data, sid)+ if event == "connector_computer_use_op_result":+ return self._handle_computer_use_op_result(data, sid)+if event.startswith("connector_"):return WsResult.error(code="UNKNOWN_EVENT",@@ -347,6 +365,28 @@ class WsConnector(WsHandler):return {"op_id": op_id, "accepted": True}+ def _handle_computer_use_op_result(+ self,+ data: dict[str, Any],+ sid: str,+ ) -> dict[str, Any] | WsResult:+ op_id = str(data.get("op_id", "")).strip()+ if not op_id:+ return WsResult.error(+ code="MISSING_OP_ID",+ message="op_id is required",+ correlation_id=data.get("correlationId"),+ )++ if not resolve_pending_computer_use_op(op_id, sid=sid, payload=data):+ return WsResult.error(+ code="UNKNOWN_OP_ID",+ message=f"No pending computer-use operation for op_id '{op_id}'",+ correlation_id=data.get("correlationId"),+ )++ return {"op_id": op_id, "accepted": True}+async def _resolve_context(self,*,diff --git a/plugins/_a0_connector/helpers/exec_config.py b/plugins/_a0_connector/helpers/exec_config.pyindex fe29f1de1..0c41bba18 100644--- a/plugins/_a0_connector/helpers/exec_config.py+++ b/plugins/_a0_connector/helpers/exec_config.py@@ -2,95 +2,35 @@ from __future__ import annotationsfrom typing import Any-_TIMEOUT_KEYS = (- "first_output_timeout",- "between_output_timeout",- "max_exec_timeout",- "dialog_timeout",-)-_DEFAULT_CODE_EXEC_TIMEOUTS = {- "first_output_timeout": 30,- "between_output_timeout": 15,- "max_exec_timeout": 180,- "dialog_timeout": 5,-}-_DEFAULT_OUTPUT_TIMEOUTS = {- "first_output_timeout": 90,- "between_output_timeout": 45,- "max_exec_timeout": 300,- "dialog_timeout": 5,-}-_DEFAULT_PROMPT_PATTERNS = [- r"(\(venv\)).+[$#] ?$",- r"root@[^:]+:[^#]+# ?$",- r"[a-zA-Z0-9_.-]+@[^:]+:[^$#]+[$#] ?$",- r"\(?.*\)?\s*PS\s+[^>]+> ?$",-]-_DEFAULT_DIALOG_PATTERNS = [- r"Y/N",- r"yes/no",- r":\s*$",- r"\?\s*$",-]+_TIMEOUT_KEYS = ("first_output_timeout", "between_output_timeout", "max_exec_timeout", "dialog_timeout")-def _coerce_timeout_group(raw: Any, defaults: dict[str, int]) -> dict[str, int]:- group = raw if isinstance(raw, dict) else {}- result: dict[str, int] = {}- for key in _TIMEOUT_KEYS:- value = group.get(key, defaults[key])- try:- result[key] = int(value)- except (TypeError, ValueError):- result[key] = defaults[key]- return result---def _pattern_lines(raw: Any, defaults: list[str]) -> list[str]:- if isinstance(raw, list):- values = raw- elif isinstance(raw, str):- values = raw.splitlines()+def _parse_patterns(value: object) -> list[str]:+ if isinstance(value, str):+ items = value.splitlines()+ elif isinstance(value, (list, tuple)):+ items = valueelse:- values = defaults+ return []+ return [str(item).strip() for item in items if str(item).strip()]+- patterns = [str(value).strip() for value in values if str(value).strip()]- return patterns or list(defaults)+def _parse_timeouts(cfg: dict[str, Any], prefix: str, defaults: tuple[int, ...]) -> dict[str, int]:+ return {+ key: int(cfg.get(f"{prefix}_{key}", default))+ for key, default in zip(_TIMEOUT_KEYS, defaults)+ }-def build_exec_config() -> dict[str, Any]:+def build_exec_config(*, agent: object | None = None) -> dict[str, Any]:from helpers import plugins- try:- config = plugins.get_plugin_config("_code_execution") or {}- except Exception:- config = {}-+ cfg = plugins.get_plugin_config("_code_execution", agent=agent) or {}return {"version": 1,- "code_exec_timeouts": _coerce_timeout_group(- config.get("code_exec_timeouts") or {- key: config.get(f"code_exec_{key}")- for key in _TIMEOUT_KEYS- if f"code_exec_{key}" in config- },- _DEFAULT_CODE_EXEC_TIMEOUTS,- ),- "output_timeouts": _coerce_timeout_group(- config.get("output_timeouts") or {- key: config.get(f"output_{key}")- for key in _TIMEOUT_KEYS- if f"output_{key}" in config- },- _DEFAULT_OUTPUT_TIMEOUTS,- ),- "prompt_patterns": _pattern_lines(- config.get("prompt_patterns"),- _DEFAULT_PROMPT_PATTERNS,- ),- "dialog_patterns": _pattern_lines(- config.get("dialog_patterns"),- _DEFAULT_DIALOG_PATTERNS,- ),+ "code_exec_timeouts": _parse_timeouts(cfg, "code_exec", (30, 15, 180, 5)),+ "output_timeouts": _parse_timeouts(cfg, "output", (90, 45, 300, 5)),+ "prompt_patterns": _parse_patterns(cfg.get("prompt_patterns", "")),+ "dialog_patterns": _parse_patterns(cfg.get("dialog_patterns", "")),}diff --git a/plugins/_a0_connector/helpers/ws_runtime.py b/plugins/_a0_connector/helpers/ws_runtime.pyindex 164e97e38..0efed71a7 100644--- a/plugins/_a0_connector/helpers/ws_runtime.py+++ b/plugins/_a0_connector/helpers/ws_runtime.py@@ -23,6 +23,14 @@ class PendingExecOperation:context_id: str | None = None+@dataclass+class PendingComputerUseOperation:+ sid: str+ loop: asyncio.AbstractEventLoop+ future: asyncio.Future[dict[str, Any]]+ context_id: str | None = None++@dataclass(frozen=True)class RemoteTreeSnapshot:sid: str@@ -30,11 +38,26 @@ class RemoteTreeSnapshot:updated_at: float+@dataclass(frozen=True)+class ComputerUseMetadata:+ supported: bool+ enabled: bool+ trust_mode: str+ artifact_root: str+ backend_id: str+ backend_family: str+ features: tuple[str, ...]+ support_reason: str+ updated_at: float++_context_subscriptions: dict[str, set[str]] = {}_sid_contexts: dict[str, set[str]] = {}_pending_file_ops: dict[str, PendingFileOperation] = {}_pending_exec_ops: dict[str, PendingExecOperation] = {}+_pending_computer_use_ops: dict[str, PendingComputerUseOperation] = {}_remote_tree_snapshots: dict[str, RemoteTreeSnapshot] = {}+_sid_computer_use_metadata: dict[str, ComputerUseMetadata] = {}_state_lock = threading.RLock()@@ -47,6 +70,7 @@ def unregister_sid(sid: str) -> set[str]:with _state_lock:contexts = _sid_contexts.pop(sid, set())_remote_tree_snapshots.pop(sid, None)+ _sid_computer_use_metadata.pop(sid, None)for context_id in contexts:subscribers = _context_subscriptions.get(context_id)if not subscribers:@@ -143,6 +167,61 @@ def select_target_sid(context_id: str) -> str | None:return sorted(subscribers)[0]+def store_sid_computer_use_metadata(sid: str, payload: dict[str, Any]) -> ComputerUseMetadata:+ features_value = payload.get("features")+ if isinstance(features_value, (list, tuple)):+ features = tuple(str(item).strip() for item in features_value if str(item).strip())+ else:+ features = ()+ metadata = ComputerUseMetadata(+ supported=bool(payload.get("supported")),+ enabled=bool(payload.get("supported")) and bool(payload.get("enabled")),+ trust_mode=str(payload.get("trust_mode", "") or "").strip(),+ artifact_root=str(payload.get("artifact_root", "") or "").strip(),+ backend_id=str(payload.get("backend_id", "") or "").strip(),+ backend_family=str(payload.get("backend_family", "") or "").strip(),+ features=features,+ support_reason=str(payload.get("support_reason", "") or "").strip(),+ updated_at=time.time(),+ )+ with _state_lock:+ _sid_computer_use_metadata[sid] = metadata+ return metadata+++def clear_sid_computer_use_metadata(sid: str) -> None:+ with _state_lock:+ _sid_computer_use_metadata.pop(sid, None)+++def computer_use_metadata_for_sid(sid: str) -> dict[str, Any] | None:+ with _state_lock:+ metadata = _sid_computer_use_metadata.get(sid)+ if metadata is None:+ return None+ return {+ "supported": metadata.supported,+ "enabled": metadata.enabled,+ "trust_mode": metadata.trust_mode,+ "artifact_root": metadata.artifact_root,+ "backend_id": metadata.backend_id,+ "backend_family": metadata.backend_family,+ "features": list(metadata.features),+ "support_reason": metadata.support_reason,+ "updated_at": metadata.updated_at,+ }+++def select_computer_use_target_sid(context_id: str) -> str | None:+ with _state_lock:+ subscribers = sorted(_context_subscriptions.get(context_id, set()))+ for sid in subscribers:+ metadata = _sid_computer_use_metadata.get(sid)+ if metadata and metadata.supported and metadata.enabled:+ return sid+ return None++def store_pending_file_op(op_id: str,*,@@ -171,14 +250,7 @@ def resolve_pending_file_op(sid: str,payload: dict[str, Any],) -> bool:- with _state_lock:- pending = _pending_file_ops.get(op_id)- if pending is None or pending.sid != sid:- return False- _pending_file_ops.pop(op_id, None)-- pending.loop.call_soon_threadsafe(_set_future_result, pending.future, dict(payload))- return True+ return _resolve_pending(_pending_file_ops, op_id, sid=sid, payload=payload)def fail_pending_file_op(@@ -187,32 +259,11 @@ def fail_pending_file_op(sid: str | None = None,error: str,) -> bool:- with _state_lock:- pending = _pending_file_ops.get(op_id)- if pending is None:- return False- if sid is not None and pending.sid != sid:- return False- _pending_file_ops.pop(op_id, None)-- payload = {"op_id": op_id, "ok": False, "error": error}- pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)- return True+ return _fail_pending(_pending_file_ops, op_id, sid=sid, error=error)def fail_pending_file_ops_for_sid(sid: str, *, error: str) -> None:- with _state_lock:- matches = [- (op_id, pending)- for op_id, pending in _pending_file_ops.items()- if pending.sid == sid- ]- for op_id, _pending in matches:- _pending_file_ops.pop(op_id, None)-- for op_id, pending in matches:- payload = {"op_id": op_id, "ok": False, "error": error}- pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)+ _fail_pending_for_sid(_pending_file_ops, sid=sid, error=error)def store_pending_exec_op(@@ -243,48 +294,127 @@ def resolve_pending_exec_op(sid: str,payload: dict[str, Any],) -> bool:+ return _resolve_pending(_pending_exec_ops, op_id, sid=sid, payload=payload)+++def fail_pending_exec_op(+ op_id: str,+ *,+ sid: str | None = None,+ error: str,+) -> bool:+ return _fail_pending(_pending_exec_ops, op_id, sid=sid, error=error)+++def fail_pending_exec_ops_for_sid(sid: str, *, error: str) -> None:+ _fail_pending_for_sid(_pending_exec_ops, sid=sid, error=error)+++def store_pending_computer_use_op(+ op_id: str,+ *,+ sid: str,+ future: asyncio.Future[dict[str, Any]],+ loop: asyncio.AbstractEventLoop,+ context_id: str | None = None,+) -> None:+ with _state_lock:+ _pending_computer_use_ops[op_id] = PendingComputerUseOperation(+ sid=sid,+ loop=loop,+ future=future,+ context_id=context_id,+ )+++def clear_pending_computer_use_op(op_id: str) -> None:with _state_lock:- pending = _pending_exec_ops.get(op_id)+ _pending_computer_use_ops.pop(op_id, None)+++def resolve_pending_computer_use_op(+ op_id: str,+ *,+ sid: str,+ payload: dict[str, Any],+) -> bool:+ return _resolve_pending(_pending_computer_use_ops, op_id, sid=sid, payload=payload)+++def fail_pending_computer_use_op(+ op_id: str,+ *,+ sid: str | None = None,+ error: str,+) -> bool:+ return _fail_pending(_pending_computer_use_ops, op_id, sid=sid, error=error)+++def fail_pending_computer_use_ops_for_sid(sid: str, *, error: str) -> None:+ _fail_pending_for_sid(_pending_computer_use_ops, sid=sid, error=error)+++def _resolve_pending(+ registry: dict[str, PendingFileOperation | PendingExecOperation | PendingComputerUseOperation],+ op_id: str,+ *,+ sid: str,+ payload: dict[str, Any],+) -> bool:+ with _state_lock:+ pending = registry.get(op_id)if pending is None or pending.sid != sid:return False- _pending_exec_ops.pop(op_id, None)+ registry.pop(op_id, None)pending.loop.call_soon_threadsafe(_set_future_result, pending.future, dict(payload))return True-def fail_pending_exec_op(+def _fail_pending(+ registry: dict[str, PendingFileOperation | PendingExecOperation | PendingComputerUseOperation],op_id: str,*,- sid: str | None = None,+ sid: str | None,error: str,) -> bool:with _state_lock:- pending = _pending_exec_ops.get(op_id)+ pending = registry.get(op_id)if pending is None:return Falseif sid is not None and pending.sid != sid:return False- _pending_exec_ops.pop(op_id, None)+ registry.pop(op_id, None)- payload = {"op_id": op_id, "ok": False, "error": error}- pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)+ pending.loop.call_soon_threadsafe(+ _set_future_result,+ pending.future,+ {"op_id": op_id, "ok": False, "error": error},+ )return True-def fail_pending_exec_ops_for_sid(sid: str, *, error: str) -> None:+def _fail_pending_for_sid(+ registry: dict[str, PendingFileOperation | PendingExecOperation | PendingComputerUseOperation],+ *,+ sid: str,+ error: str,+) -> None:with _state_lock:matches = [(op_id, pending)- for op_id, pending in _pending_exec_ops.items()+ for op_id, pending in registry.items()if pending.sid == sid]for op_id, _pending in matches:- _pending_exec_ops.pop(op_id, None)+ registry.pop(op_id, None)for op_id, pending in matches:- payload = {"op_id": op_id, "ok": False, "error": error}- pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)+ pending.loop.call_soon_threadsafe(+ _set_future_result,+ pending.future,+ {"op_id": op_id, "ok": False, "error": error},+ )def _set_future_result(diff --git a/plugins/_a0_connector/plugin.yaml b/plugins/_a0_connector/plugin.yamlindex b5ff1f51a..5a4e6685b 100644--- a/plugins/_a0_connector/plugin.yaml+++ b/plugins/_a0_connector/plugin.yaml@@ -1,7 +1,7 @@name: _a0_connectortitle: A0 Connectordescription: Current Agent Zero connector plugin for HTTP plus /ws integration, using session auth and handler activation through auth.handlers.-version: 1.4+version: 1.5.0settings_sections:- external- developerdiff --git a/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md b/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.mdnew file mode 100644index 000000000..59a1d5ddc--- /dev/null+++ b/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md@@ -0,0 +1,34 @@+# computer_use_remote tool++Use the connected CLI host machine as a local desktop target.++## Preferred Scope+- Use this for local desktop and native UI tasks on the connected machine.+- For ordinary website browsing, search, form filling, and web downloads, prefer `browser_agent`.+- If the user is flexible and the task is browser-only, briefly guide them toward browser tools because they are usually more reliable and token-efficient than screenshot-driven computer use.+- Before doing real computer-use work, load the `computer-use-remote` skill and follow it.++## Requirements+- A CLI client must be connected to this context via the shared `/ws` namespace.+- The CLI must advertise `computer_use_remote` support and local computer use must be enabled there.+- In `free_run`, do not expect a fresh approval prompt. If restore is no longer valid, the tool will surface `COMPUTER_USE_REARM_REQUIRED`.++## Minimal Rules+- Treat user interventions as high-priority control signals.+- If the user says `stop`, `pause`, `abort`, `hold`, `don't continue`, or equivalent, halt immediately and do not use computer-use tools again until the user explicitly resumes.+- Call `start_session` first. It automatically attaches the current screen.+- Decide from the latest screenshot, not from memory.+- Interactive actions (`move`, `click`, `scroll`, `key`, `type`) automatically attach a fresh screenshot after they run.+- Use `capture` only when you need another screen refresh without taking an action.+- Prefer keyboard actions over pointer actions whenever a reliable keyboard path exists.++## Arguments+- `action`: one of `start_session`, `status`, `capture`, `move`, `click`, `scroll`, `key`, `type`, `stop_session`+- `session_id`: optional for actions after `start_session`++Action-specific fields:+- `move`: `x`, `y` normalized to `[0,1]`+- `click`: optional `x`, `y`, plus optional `button` (`left`, `right`, `middle`) and `count`+- `scroll`: `dx`, `dy`+- `key`: `key` or `keys`+- `type`: `text`, optional `submit` booleandiff --git a/plugins/_a0_connector/tools/computer_use_remote.py b/plugins/_a0_connector/tools/computer_use_remote.pynew file mode 100644index 000000000..efbe7916b--- /dev/null+++ b/plugins/_a0_connector/tools/computer_use_remote.py@@ -0,0 +1,433 @@+"""computer_use_remote tool — drive the CLI host machine through the connected frontend."""+from __future__ import annotations++import asyncio+import base64+from pathlib import Path+import uuid+from typing import Any++from helpers import history+from helpers.tool import Response, Tool+from helpers.ws import NAMESPACE+from helpers.ws_manager import ConnectionNotFoundError, get_shared_ws_manager++from plugins._a0_connector.helpers.ws_runtime import (+ clear_pending_computer_use_op,+ select_computer_use_target_sid,+ store_pending_computer_use_op,+)+++COMPUTER_USE_OP_TIMEOUT = 180.0+COMPUTER_USE_OP_EVENT = "connector_computer_use_op"+CAPTURE_TOKENS_ESTIMATE = 1500+_AUTO_CAPTURE_ACTIONS = {+ "start_session",+ "move",+ "click",+ "scroll",+ "key",+ "type",+}+_SETTLE_DELAY_START_SESSION = 0.2+_SETTLE_DELAY_GLOBAL_FOCUS = 0.45+_SETTLE_DELAY_PLAIN_ENTER = 0.3+_SETTLE_DELAY_SUBMIT = 0.45+_SUPPORTED_ACTIONS = {+ "start_session",+ "status",+ "capture",+ "move",+ "click",+ "scroll",+ "key",+ "type",+ "stop_session",+}+++class ComputerUseRemote(Tool):+ async def execute(self, **kwargs: Any) -> Response:+ action = str(self.args.get("action") or "").strip().lower()+ if action not in _SUPPORTED_ACTIONS:+ return Response(+ message=(+ "action is required and must be one of: "+ "start_session, status, capture, move, click, scroll, key, type, stop_session"+ ),+ break_loop=False,+ )++ context_id = self.agent.context.id+ sid = select_computer_use_target_sid(context_id)+ if not sid:+ return Response(+ message=(+ "computer_use_remote: no subscribed CLI in this context currently advertises "+ "enabled local computer use. Enable it in the CLI with F2 and choose a trust mode first."+ ),+ break_loop=False,+ )++ try:+ payload = self._build_payload(op_id=str(uuid.uuid4()), context_id=context_id, action=action)+ result = await self._dispatch_payload(sid=sid, payload=payload)+ capture_note = await self._maybe_attach_latest_capture(+ action=action,+ sid=sid,+ context_id=context_id,+ result=result,+ )+ except ValueError as exc:+ return Response(+ message=f"computer_use_remote: {exc}",+ break_loop=False,+ )+ except ConnectionNotFoundError:+ return Response(+ message=(+ "computer_use_remote: the selected CLI disconnected before the request "+ "could be delivered."+ ),+ break_loop=False,+ )+ except asyncio.TimeoutError:+ return Response(+ message=f"computer_use_remote: timed out waiting for action={action!r}",+ break_loop=False,+ )+ except Exception as exc:+ return Response(+ message=f"computer_use_remote: error sending action={action!r}: {exc}",+ break_loop=False,+ )++ message = self._extract_result(action, result)+ if capture_note:+ message = f"{message} {capture_note}".strip()++ return Response(+ message=message,+ break_loop=False,+ )++ async def _dispatch_payload(self, *, sid: str, payload: dict[str, Any]) -> dict[str, Any]:+ op_id = str(payload.get("op_id") or "").strip()+ loop = asyncio.get_running_loop()+ future: asyncio.Future[dict[str, Any]] = loop.create_future()+ store_pending_computer_use_op(+ op_id,+ sid=sid,+ future=future,+ loop=loop,+ context_id=str(payload.get("context_id") or "").strip() or None,+ )++ try:+ await get_shared_ws_manager().emit_to(+ NAMESPACE,+ sid,+ COMPUTER_USE_OP_EVENT,+ payload,+ handler_id=f"{self.__class__.__module__}.{self.__class__.__name__}",+ )+ result = await asyncio.wait_for(future, timeout=COMPUTER_USE_OP_TIMEOUT)+ finally:+ clear_pending_computer_use_op(op_id)++ if isinstance(result, dict):+ return result+ raise RuntimeError(f"Unexpected response format from CLI: {result!r}")++ async def _maybe_attach_latest_capture(+ self,+ *,+ action: str,+ sid: str,+ context_id: str,+ result: dict[str, Any],+ ) -> str:+ if action not in _AUTO_CAPTURE_ACTIONS or not bool(result.get("ok")):+ return ""++ data = result.get("result")+ result_data = dict(data) if isinstance(data, dict) else {}+ session_id = str(result_data.get("session_id") or self.args.get("session_id") or "").strip()+ if not session_id:+ return ""++ settle_seconds = self._auto_capture_settle_seconds(action)+ if settle_seconds > 0:+ await asyncio.sleep(settle_seconds)++ capture_result = await self._dispatch_payload(+ sid=sid,+ payload={+ "op_id": str(uuid.uuid4()),+ "context_id": context_id,+ "action": "capture",+ "session_id": session_id,+ },+ )+ if not bool(capture_result.get("ok")):+ return f"Automatic screen refresh failed: {self._format_error(capture_result)}"++ capture_data = capture_result.get("result")+ if not isinstance(capture_data, dict):+ return "Automatic screen refresh failed: missing capture payload."++ self._record_capture(capture_data)+ return "Latest screen attached."++ def _auto_capture_settle_seconds(self, action: str) -> float:+ if action == "start_session":+ return _SETTLE_DELAY_START_SESSION+ if action == "type" and self._coerce_bool(self.args.get("submit")):+ return _SETTLE_DELAY_SUBMIT+ if action != "key":+ return 0.0++ keyset = {key.lower() for key in self._requested_keys()}+ if "super" in keyset or ("alt" in keyset and "tab" in keyset):+ return _SETTLE_DELAY_GLOBAL_FOCUS+ if keyset == {"enter"}:+ return _SETTLE_DELAY_PLAIN_ENTER+ return 0.0++ def _requested_keys(self) -> list[str]:+ keys_value = self.args.get("keys")+ if isinstance(keys_value, (list, tuple)):+ return [str(item).strip() for item in keys_value if str(item).strip()]+ raw = str(keys_value or self.args.get("key", "") or "").strip()+ if not raw:+ return []+ return [part.strip() for part in raw.split("+") if part.strip()]++ def _build_payload(self, *, op_id: str, context_id: str, action: str) -> dict[str, Any]:+ payload: dict[str, Any] = {+ "op_id": op_id,+ "context_id": context_id,+ "action": action,+ }+ session_id = str(self.args.get("session_id", "") or "").strip()+ if session_id:+ payload["session_id"] = session_id++ if action == "move":+ payload["x"] = self.args.get("x")+ payload["y"] = self.args.get("y")+ elif action == "click":+ if "x" in self.args:+ payload["x"] = self.args.get("x")+ if "y" in self.args:+ payload["y"] = self.args.get("y")+ payload["button"] = self.args.get("button", "left")+ payload["count"] = self._coerce_int(self.args.get("count", 1), name="count")+ elif action == "scroll":+ payload["dx"] = self._coerce_int(self.args.get("dx", self.args.get("delta_x", 0)), name="dx")+ payload["dy"] = self._coerce_int(self.args.get("dy", self.args.get("delta_y", 0)), name="dy")+ elif action == "key":+ if "keys" in self.args:+ payload["keys"] = self.args.get("keys")+ elif "key" in self.args:+ payload["key"] = self.args.get("key")+ elif action == "type":+ payload["text"] = self.args.get("text", "")+ if self._coerce_bool(self.args.get("submit")):+ payload["submit"] = True++ return payload++ def _extract_result(self, action: str, result: Any) -> str:+ if not isinstance(result, dict):+ return f"Unexpected response format from CLI: {result!r}"++ ok = bool(result.get("ok"))+ data = result.get("result")++ if not ok:+ return self._format_error(result)++ if not isinstance(data, dict):+ data = {}++ if action == "capture":+ self._record_capture(data)+ return "Current screen attached."+ if action == "status":+ return self._format_status(data)+ if action == "start_session":+ return (+ f"Computer-use session started: session_id={data.get('session_id', '?')} "+ f"size={data.get('width', '?')}x{data.get('height', '?')}"+ )+ if action == "stop_session":+ return "Computer-use session stopped."+ if action == "move":+ return f"Pointer moved to x={data.get('x')} y={data.get('y')}."+ if action == "click":+ return f"Clicked {data.get('button', 'left')} button {data.get('count', 1)} time(s)."+ if action == "scroll":+ return f"Scrolled dx={data.get('dx', 0)} dy={data.get('dy', 0)}."+ if action == "key":+ keys = data.get("keys") or []+ return f"Sent keys: {keys!r}."+ if action == "type":+ text = str(data.get("text", "") or "")+ if data.get("submitted"):+ return f"Typed {len(text)} character(s) and submitted."+ return f"Typed {len(text)} character(s)."+ return str(data)++ def _format_error(self, result: dict[str, Any]) -> str:+ error = str(result.get("error") or "Unknown error")+ code = str(result.get("code") or "")+ if code:+ return f"{code}: {error}"+ return error++ def _format_status(self, data: dict[str, Any]) -> str:+ status = str(data.get("status", "unknown") or "unknown")+ trust_mode = str(data.get("trust_mode", "") or "")+ backend_id = str(data.get("backend_id", "") or "").strip()+ backend_family = str(data.get("backend_family", "") or "").strip()+ active_contexts = data.get("active_contexts") or []+ active_text = ", ".join(str(item) for item in active_contexts) if active_contexts else "none"+ backend_text = ""+ if backend_id:+ backend_text = backend_id+ if backend_family:+ backend_text = f"{backend_text}/{backend_family}"+ if backend_text:+ return (+ f"Computer use status={status}, trust_mode={trust_mode or 'unknown'}, "+ f"backend={backend_text}, active_contexts={active_text}."+ )+ return f"Computer use status={status}, trust_mode={trust_mode or 'unknown'}, active_contexts={active_text}."++ def _record_capture(self, data: dict[str, Any]) -> str:+ image_b64 = self._capture_image_base64(data)+ width = data.get("width", "?")+ height = data.get("height", "?")+ summary = f"Computer-use capture {width}x{height}."+ content = [+ {"type": "text", "text": summary},+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_b64}"}},+ ]+ raw_message = history.RawMessage(raw_content=content, preview=summary)+ self.agent.hist_add_message(False, content=raw_message, tokens=CAPTURE_TOKENS_ESTIMATE)+ self._prune_prior_capture_history()+ return summary++ def _prune_prior_capture_history(self) -> None:+ history_obj = getattr(self.agent, "history", None)+ if history_obj is None:+ return++ capture_messages = self._collect_capture_messages(history_obj)+ if len(capture_messages) <= 1:+ return++ latest = capture_messages[-1]+ for message in capture_messages[:-1]:+ if message is latest:+ continue+ preview = self._capture_preview_from_message(message)+ if not preview:+ continue+ message.content = f"{preview} [embedded image removed]"+ if hasattr(message, "summary"):+ message.summary = ""+ if hasattr(message, "calculate_tokens"):+ message.tokens = message.calculate_tokens()++ def _collect_capture_messages(self, history_obj: Any) -> list[Any]:+ messages: list[Any] = []++ def collect_topic(topic: Any) -> None:+ topic_messages = getattr(topic, "messages", None)+ if isinstance(topic_messages, list):+ for message in topic_messages:+ if self._capture_preview_from_message(message):+ messages.append(message)++ bulks = getattr(history_obj, "bulks", None)+ if isinstance(bulks, list):+ for bulk in bulks:+ self._collect_capture_messages_from_record(bulk, messages)++ topics = getattr(history_obj, "topics", None)+ if isinstance(topics, list):+ for topic in topics:+ collect_topic(topic)++ current = getattr(history_obj, "current", None)+ if current is not None:+ collect_topic(current)++ return messages++ def _collect_capture_messages_from_record(self, record: Any, messages: list[Any]) -> None:+ topic_messages = getattr(record, "messages", None)+ if isinstance(topic_messages, list):+ for message in topic_messages:+ if self._capture_preview_from_message(message):+ messages.append(message)+ return++ nested_records = getattr(record, "records", None)+ if isinstance(nested_records, list):+ for nested in nested_records:+ self._collect_capture_messages_from_record(nested, messages)++ def _capture_preview_from_message(self, message: Any) -> str:+ content = getattr(message, "content", None)+ if not isinstance(content, dict):+ return ""+ raw_content = content.get("raw_content")+ preview = content.get("preview")+ if raw_content is None or not isinstance(preview, str):+ return ""+ if preview.startswith("Computer-use capture "):+ return preview+ return ""++ def _capture_image_base64(self, data: dict[str, Any]) -> str:+ inline_payload = str(data.get("png_base64", "") or "").strip()+ if inline_payload:+ try:+ base64.b64decode(inline_payload, validate=True)+ except Exception:+ pass+ else:+ return inline_payload++ image_path, _display_path = self._resolve_capture_path(data)+ return base64.b64encode(image_path.read_bytes()).decode("utf-8")++ def _resolve_capture_path(self, data: dict[str, Any]) -> tuple[Path, str]:+ candidates = [+ str(data.get("capture_path", "") or "").strip(),+ str(data.get("container_path", "") or "").strip(),+ str(data.get("host_path", "") or "").strip(),+ ]+ for candidate in candidates:+ if candidate and Path(candidate).exists():+ return Path(candidate), candidate+ raise FileNotFoundError(+ f"Capture artifact was not found in any advertised path: {candidates!r}"+ )++ def _coerce_int(self, value: object, *, name: str) -> int:+ try:+ return int(value or 0)+ except (TypeError, ValueError) as exc:+ raise ValueError(f"{name} must be an integer") from exc++ def _coerce_bool(self, value: object) -> bool:+ if isinstance(value, bool):+ return value+ if isinstance(value, (int, float)):+ return bool(value)+ return str(value or "").strip().lower() in {"1", "true", "yes", "on"}diff --git a/skills/computer-use-remote/SKILL.md b/skills/computer-use-remote/SKILL.mdnew file mode 100644index 000000000..8675ed53c--- /dev/null+++ b/skills/computer-use-remote/SKILL.md@@ -0,0 +1,50 @@+---+name: computer-use-remote+description: Detailed operating guide for using computer_use_remote on the connected local machine. Load this skill before using computer_use_remote for desktop control, screenshots, menus, browser chrome, or other native UI tasks.+version: 1.0.0+author: Agent Zero Team+tags: ["computer-use", "desktop", "local-ui", "screenshots", "native-ui"]+trigger_patterns:+ - "computer use"+ - "computer-use"+ - "computer_use_remote"+ - "local desktop control"+ - "control local browser"+ - "click on screen"+ - "native ui"+allowed_tools:+ - computer_use_remote+ - code_execution_remote+---++# Computer Use Remote++## When to Use++Load this skill before using `computer_use_remote` for local desktop and native UI tasks on the connected machine.++For ordinary website browsing, search, form filling, and web downloads, prefer `browser_agent` instead. If the user is flexible and the task is browser-only, guide them toward browser tools because they are usually more reliable and token-efficient than screenshot-driven computer use.++## Core Loop++1. Call `start_session` first.+2. Decide from the latest screenshot, not from memory.+3. Interactive actions (`move`, `click`, `scroll`, `key`, `type`) already attach a fresh screenshot after they run.+4. Use `capture` only when you need another screenshot without taking an action.++## Operating Rules++- Only the latest screenshot or a definitive tool result counts as evidence.+- Prefer keyboard actions over pointer actions whenever a reliable keyboard path exists.+- When a menu or popup is open, treat it as the active UI and prefer keyboard navigation over clicking small transient rows by coordinate.+- If a click dismisses a menu or popup without producing the expected next UI, treat that attempt as failed.+- If the same approach has already failed twice without visible progress, switch strategy instead of repeating it.+- Do not infer focus or task completion from chat logs, sidebars, tool summaries, or status text.+- For browser-navigation tasks done through this tool, only claim success if the browser content area visibly shows the destination page or result.+- Use `type(..., submit=true)` only for URL or navigation-style entry where Enter should fire immediately after typing.+- Do not use `submit=true` for ordinary text fields. Type first, then send `enter` separately if needed.++## Control Signals++- Treat user interventions as high-priority control signals.+- If the user says `stop`, `pause`, `abort`, `hold`, `don't continue`, or equivalent, halt immediately and do not use computer-use tools again until the user explicitly resumes.