Agent Zero AI framework
Preserve computer-use capability metadata
Alessandrod8710c3f8145
diff --git a/plugins/_a0_connector/helpers/ws_runtime.py b/plugins/_a0_connector/helpers/ws_runtime.pyindex 94711a2ba..1987e5350 100644--- a/plugins/_a0_connector/helpers/ws_runtime.py+++ b/plugins/_a0_connector/helpers/ws_runtime.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import copyimport threadingimport timefrom dataclasses import dataclass@@ -58,6 +59,8 @@ class ComputerUseMetadata:backend_id: strbackend_family: strfeatures: tuple[str, ...]+ contract_version: int+ capabilities: dict[str, Any]support_reason: strupdated_at: float@@ -337,6 +340,12 @@ def store_sid_computer_use_metadata(sid: str, payload: dict[str, Any]) -> Computfeatures = tuple(str(item).strip() for item in features_value if str(item).strip())else:features = ()+ capabilities_value = payload.get("capabilities")+ capabilities = copy.deepcopy(capabilities_value) if isinstance(capabilities_value, dict) else {}+ try:+ contract_version = int(payload.get("contract_version") or 0)+ except (TypeError, ValueError):+ contract_version = 0metadata = ComputerUseMetadata(supported=bool(payload.get("supported")),enabled=bool(payload.get("supported")) and bool(payload.get("enabled")),@@ -348,6 +357,8 @@ def store_sid_computer_use_metadata(sid: str, payload: dict[str, Any]) -> Computbackend_id=str(payload.get("backend_id", "") or "").strip(),backend_family=str(payload.get("backend_family", "") or "").strip(),features=features,+ contract_version=contract_version,+ capabilities=capabilities,support_reason=str(payload.get("support_reason", "") or "").strip(),updated_at=time.time(),)@@ -377,6 +388,8 @@ def computer_use_metadata_for_sid(sid: str) -> dict[str, Any] | None:"backend_id": metadata.backend_id,"backend_family": metadata.backend_family,"features": list(metadata.features),+ "contract_version": metadata.contract_version,+ "capabilities": copy.deepcopy(metadata.capabilities),"support_reason": metadata.support_reason,"updated_at": metadata.updated_at,}diff --git a/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md b/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.mdindex c0003090d..163f23db4 100644--- a/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md+++ b/plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md@@ -8,7 +8,7 @@ This is the only desktop-control path for the user's connected host/local computIf the tool reports no CLI, disabled computer use, or `COMPUTER_USE_REARM_REQUIRED`, stop and tell the user to run `/computer-use on` in A0 CLI and approve any host permission prompt.-Call `start_session` before screen-driven tasks. Use `status` for state only, `capture` for screenshots without an action, and `stop_session` when the desktop task is complete. When the backend advertises native window and element-index features, prefer `list_windows` -> `get_window_state` -> `element_action` with `dispatch: "background"` before using global coordinates. Interactive coordinate actions should use normalized global-screen coordinates from the most recent capture.+Call `start_session` before screen-driven tasks. Use `status` for state only, `capture` for screenshots without an action, and `stop_session` when the desktop task is complete. Read `backend_id`, `backend_family`, `features`, and the structured `capabilities` object in status/session results. When capabilities report native windows, window state, element indexes, and background dispatch, prefer `list_windows` -> `get_window_state` -> `element_action` with `dispatch: "background"` before using global coordinates. Interactive coordinate actions should use normalized global-screen coordinates from the most recent capture.Some actions are backend-specific and intentionally documented only in backend skills. If `status` or `start_session` reports backend-specific features or tells you to load a backend skill, load and follow that skill before using those backend-only actions. For structural targeting details, load and follow the backend-specific skill such as `host-computer-use-macos` or `host-computer-use-windows`; do not apply one backend's guidance to another backend.@@ -39,3 +39,5 @@ Optional arguments by action:- `key` or `keys`: key press value for `key`- `text`: text to type for `type`- `submit`: boolean Enter-after-type flag for `type`++Status/session results may include `contract_version` and `capabilities`. Treat `capabilities.identity.pid`, `capabilities.identity.window_id`, `capabilities.identity.element_index`, and `capabilities.dispatch.background` as the authoritative cross-platform contract for whether the native background loop is available. Use `features` for backend-specific refinements and skill selection.diff --git a/plugins/_a0_connector/skills/host-computer-use/SKILL.md b/plugins/_a0_connector/skills/host-computer-use/SKILL.mdindex b8815c5d9..3d4cc543b 100644--- a/plugins/_a0_connector/skills/host-computer-use/SKILL.md+++ b/plugins/_a0_connector/skills/host-computer-use/SKILL.md@@ -75,14 +75,15 @@ If any tool result contains `COMPUTER_USE_REARM_REQUIRED` or `status=rearm requi## Core Loop1. Call `start_session` first.-2. Read the returned `backend_id`, `backend_family`, and `features`; load a backend-specific Computer Use skill when the task needs backend-only affordances.-3. If the backend advertises `native-window-list`, call `list_windows` before using coordinates.-4. If the backend advertises `window-state` and `element-index-targeting`, call `get_window_state` for the target `pid`/`window_id`, then use `element_action` with `dispatch: "background"` by default.-5. If `element_action` reports `background_unavailable`, use `dispatch: "auto"` or `dispatch: "foreground"` only when foreground control is acceptable for the user/task.-6. Decide final success from the latest screenshot or a definitive structural result, not from memory.-7. Interactive actions already attach a fresh screenshot after they run; inspect it before claiming the requested outcome succeeded.-8. Use `status` for state without starting a session.-9. Use `capture` only when you need another screenshot without taking an action.+2. Read the returned `backend_id`, `backend_family`, `features`, `contract_version`, and `capabilities`; load a backend-specific Computer Use skill when the task needs backend-only affordances.+3. Prefer the structured `capabilities` object over guessing from OS names. Use `capabilities.identity.pid`, `capabilities.identity.window_id`, `capabilities.identity.element_index`, and `capabilities.dispatch.background` as the portable contract for the native background loop.+4. If the backend advertises native window listing through capabilities or `native-window-list`, call `list_windows` before using coordinates.+5. If the backend advertises window state and element-index targeting through capabilities or features, call `get_window_state` for the target `pid`/`window_id`, then use `element_action` with `dispatch: "background"` by default.+6. If `element_action` reports `background_unavailable`, use `dispatch: "auto"` or `dispatch: "foreground"` only when foreground control is acceptable for the user/task.+7. Decide final success from the latest screenshot or a definitive structural result, not from memory.+8. Interactive actions already attach a fresh screenshot after they run; inspect it before claiming the requested outcome succeeded.+9. Use `status` for state without starting a session.+10. Use `capture` only when you need another screenshot without taking an action.## Backend Skillsdiff --git a/plugins/_a0_connector/tools/computer_use_remote.py b/plugins/_a0_connector/tools/computer_use_remote.pyindex e8a03168c..f1d5d2c5a 100644--- a/plugins/_a0_connector/tools/computer_use_remote.py+++ b/plugins/_a0_connector/tools/computer_use_remote.py@@ -505,6 +505,11 @@ class ComputerUseRemote(Tool):if backend_family:backend_text = f"{backend_text}/{backend_family}"parts.append(f"backend={backend_text}")+ contract_version = data.get("contract_version")+ if not contract_version and isinstance(data.get("capabilities"), dict):+ contract_version = data["capabilities"].get("contract_version")+ if contract_version:+ parts.append(f"contract=v{contract_version}")if features:parts.append(f"features={', '.join(features)}")return ", ".join(parts)diff --git a/tests/test_a0_connector_computer_use_metadata.py b/tests/test_a0_connector_computer_use_metadata.pynew file mode 100644index 000000000..f803b939e--- /dev/null+++ b/tests/test_a0_connector_computer_use_metadata.py@@ -0,0 +1,52 @@+from __future__ import annotations++from plugins._a0_connector.helpers import ws_runtime+++def test_computer_use_metadata_preserves_structured_capabilities() -> None:+ sid = "sid-capabilities"+ ws_runtime.clear_sid_computer_use_metadata(sid)++ ws_runtime.store_sid_computer_use_metadata(+ sid,+ {+ "supported": True,+ "enabled": True,+ "trust_mode": "allow",+ "status": "active",+ "last_error": "",+ "restore_token_present": True,+ "artifact_root": "/a0/tmp/_a0_connector/computer_use",+ "backend_id": "windows",+ "backend_family": "windows",+ "features": ["native-window-list", "element-index-targeting"],+ "contract_version": 1,+ "capabilities": {+ "contract_version": 1,+ "identity": {+ "pid": True,+ "window_id": True,+ "element_index": True,+ },+ "dispatch": {+ "default": "background",+ "background": True,+ },+ },+ "support_reason": "Windows UIA backend is available.",+ },+ )++ metadata = ws_runtime.computer_use_metadata_for_sid(sid)++ assert metadata is not None+ assert metadata["contract_version"] == 1+ assert metadata["capabilities"]["identity"]["element_index"] is True+ assert metadata["capabilities"]["dispatch"]["default"] == "background"++ metadata["capabilities"]["dispatch"]["default"] = "foreground"+ fresh_metadata = ws_runtime.computer_use_metadata_for_sid(sid)+ assert fresh_metadata is not None+ assert fresh_metadata["capabilities"]["dispatch"]["default"] == "background"++ ws_runtime.clear_sid_computer_use_metadata(sid)