Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(computer): add JavaScript SDK runner (#2633)

Maxa1231210ebed
diff --git a/core/computer_cli.py b/core/computer_cli.py
index 70efa0df6..ef9ff29e9 100644
--- a/core/computer_cli.py
+++ b/core/computer_cli.py
@@ -163,25 +163,51 @@ def _error_payload(result: Mapping[str, Any]) -> dict[str, Any]:
_COMPUTER_ERROR_I18N_KEYS = {
"active_turn_required": "computer.cli.error.activeTurnRequired",
+ "ambiguous": "computer.cli.error.scriptAmbiguous",
+ "application_not_found": "computer.cli.error.scriptTargetNotFound",
+ "computer_script_request_failed": "computer.cli.error.scriptControllerRejected",
+ "computer_script_request_too_large": "computer.cli.error.scriptRequestTooLarge",
+ "computer_script_descriptor_required": "computer.cli.error.scriptInvalidDescriptor",
+ "computer_script_protocol_error": "computer.cli.error.scriptRuntimeInvalid",
"computer_use_unavailable": "computer.cli.error.computerUseUnavailable",
"computer_use_error": "computer.cli.error.genericDriverError",
+ "controller_timeout": "computer.cli.error.scriptOperationTimedOut",
+ "controller_unavailable": "computer.cli.error.scriptControllerUnavailable",
"desktop_busy": "computer.cli.error.desktopBusy",
"desktop_lease_lost": "computer.cli.error.desktopLeaseLost",
+ "element_not_actionable": "computer.cli.error.scriptElementNotActionable",
+ "element_not_found": "computer.cli.error.scriptTargetNotFound",
"focus_shortcut_forbidden": "computer.cli.error.focusShortcutForbidden",
+ "foreground_consent_required": "computer.cli.error.foregroundConsentRequired",
"foreground_forbidden": "computer.cli.error.foregroundForbidden",
"invalid_arguments": "computer.cli.error.invalidDriverArguments",
"invalid_computer_use_request": "computer.cli.error.requestRejected",
+ "invalid_computer_use_response": "computer.cli.error.scriptInvalidControllerResponse",
+ "invalid_computer_script_descriptor": "computer.cli.error.scriptInvalidDescriptor",
+ "invalid_selector": "computer.cli.error.scriptInvalidSelector",
+ "invalid_timeout": "computer.cli.error.scriptInvalidTimeout",
+ "not_found": "computer.cli.error.scriptTargetNotFound",
"observe_first": "computer.cli.error.observeFirst",
"platform_unsupported": "computer.cli.error.platformUnsupported",
+ "script_connection_invalid": "computer.cli.error.scriptConnectionInvalid",
+ "script_timeout": "computer.cli.error.scriptDeadlineExpired",
+ "script_turn_ended": "computer.cli.error.scriptTurnEnded",
"session_archived": "computer.cli.error.sessionArchived",
"session_argument_forbidden": "computer.cli.error.sessionArgumentForbidden",
"session_required": "computer.cli.error.missingCallerSession",
"session_unavailable": "computer.cli.error.sessionUnavailable",
+ "snapshot_not_found": "computer.cli.error.scriptTargetNotFound",
+ "stale_app_handle": "computer.cli.error.scriptStaleAppHandle",
+ "stale_window_handle": "computer.cli.error.scriptStaleWindowHandle",
+ "timeout": "computer.cli.error.scriptOperationTimedOut",
"unknown_tool": "computer.cli.error.unknownTool",
"upstream_error": "computer.cli.error.upstreamError",
"upstream_protocol_error": "computer.cli.error.upstreamProtocolError",
"upstream_tool_error": "computer.cli.error.upstreamToolError",
"upstream_unavailable": "computer.cli.error.upstreamUnavailable",
+ "window_id_not_found": "computer.cli.error.scriptTargetNotFound",
+ "window_not_found": "computer.cli.error.scriptTargetNotFound",
+ "window_order_unavailable": "computer.cli.error.scriptWindowOrderUnavailable",
"window_target_required": "computer.cli.error.windowTargetRequired",
}
@@ -227,14 +253,16 @@ def translate_computer_error(
error: Mapping[str, Any],
*,
translate: Callable[..., str],
+ fallback: str | None = None,
) -> str:
"""Render a Controller or driver error without trusting prose from it."""
code = str(error.get("code") or "computer_use_error")
- key = _COMPUTER_ERROR_I18N_KEYS.get(
- code,
- "computer.cli.error.genericDriverError",
- )
+ key = _COMPUTER_ERROR_I18N_KEYS.get(code)
+ if key is None:
+ if fallback is not None:
+ return fallback
+ key = "computer.cli.error.genericDriverError"
params = error.get("params")
if code == "unknown_tool" and isinstance(params, Mapping):
return translate(key, tool=str(params.get("tool") or ""))
diff --git a/core/computer_script_handles.py b/core/computer_script_handles.py
new file mode 100644
index 000000000..26934365f
--- /dev/null
+++ b/core/computer_script_handles.py
@@ -0,0 +1,223 @@
+"""Immutable handle descriptors for Controller-owned Computer Use scripts."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable, Mapping
+
+
+class ComputerScriptHandleError(RuntimeError):
+ """A script handle descriptor is invalid or no longer names its target."""
+
+ def __init__(self, code: str, *, params: Mapping[str, Any] | None = None) -> None:
+ self.code = code
+ self.params = dict(params or {})
+ super().__init__(code)
+
+
+@dataclass(frozen=True)
+class AppHandleDescriptor:
+ pid: int | None
+ bundle_id: str | None
+ name: str | None
+
+ def matches(self, record: Mapping[str, Any]) -> bool:
+ if record.get("pid") != self.pid:
+ return False
+ if self.bundle_id is not None:
+ return record.get("bundle_id") == self.bundle_id
+ return record.get("name") == self.name
+
+
+@dataclass(frozen=True)
+class WindowHandleDescriptor:
+ pid: int
+ window_id: int
+ app: AppHandleDescriptor
+
+
+HandleDescriptor = AppHandleDescriptor | WindowHandleDescriptor
+DiscoveryCall = Callable[
+ [str, Mapping[str, Any]],
+ Awaitable[dict[str, Any]],
+]
+
+_GLOBAL_SCRIPT_TOOLS = frozenset(
+ {
+ "list_apps",
+ "clipboard_read",
+ "clipboard_write",
+ }
+)
+_APP_HANDLE_TOOLS = frozenset(
+ {
+ "launch_app",
+ "kill_app",
+ "list_windows",
+ }
+)
+_WINDOW_HANDLE_TOOLS = frozenset(
+ {
+ "get_window_state",
+ "zoom",
+ "click",
+ "double_click",
+ "type_text",
+ "press_key",
+ "hotkey",
+ "scroll",
+ "drag",
+ "set_value",
+ }
+)
+
+
+def script_tool_requires_handle(tool: str) -> bool:
+ """Return whether the shipped SDK must supply a handle for this tool."""
+
+ return tool not in _GLOBAL_SCRIPT_TOOLS
+
+
+def parse_handle_descriptor(value: Any) -> HandleDescriptor | None:
+ """Parse the exact descriptor shape accepted from the shipped SDK."""
+
+ if value is None:
+ return None
+ if not isinstance(value, dict):
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ kind = value.get("kind")
+ if kind == "app":
+ if set(value) != {"kind", "pid", "bundle_id", "name"}:
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ pid = value.get("pid")
+ if pid is not None and (
+ isinstance(pid, bool) or not isinstance(pid, int) or pid <= 0
+ ):
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ bundle_id = _optional_nonempty_string(value.get("bundle_id"))
+ name = _optional_nonempty_string(value.get("name"))
+ if (bundle_id is None) == (name is None):
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ return AppHandleDescriptor(
+ pid=pid,
+ bundle_id=bundle_id,
+ name=name,
+ )
+ if kind == "window":
+ if set(value) != {"kind", "pid", "window_id", "app"}:
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ pid = value.get("pid")
+ window_id = value.get("window_id")
+ if (
+ isinstance(pid, bool)
+ or not isinstance(pid, int)
+ or pid <= 0
+ or isinstance(window_id, bool)
+ or not isinstance(window_id, int)
+ or window_id < 0
+ ):
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ app = parse_handle_descriptor(value.get("app"))
+ if not isinstance(app, AppHandleDescriptor) or app.pid != pid:
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ return WindowHandleDescriptor(pid=pid, window_id=window_id, app=app)
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+
+
+def bind_handle_to_request(
+ descriptor: HandleDescriptor | None,
+ tool: str,
+ arguments: Mapping[str, Any],
+) -> None:
+ """Require the forwarded target to be exactly the supplied descriptor."""
+
+ if descriptor is None:
+ if script_tool_requires_handle(tool):
+ raise ComputerScriptHandleError("computer_script_descriptor_required")
+ return
+ if isinstance(descriptor, AppHandleDescriptor):
+ if tool not in _APP_HANDLE_TOOLS:
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ if tool == "launch_app":
+ identity_key = "bundle_id" if descriptor.bundle_id is not None else "name"
+ identity_value = descriptor.bundle_id or descriptor.name
+ if arguments.get(identity_key) != identity_value:
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ other_key = "name" if identity_key == "bundle_id" else "bundle_id"
+ if arguments.get(other_key) is not None:
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ return
+ if arguments.get("pid") != descriptor.pid:
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ return
+ if tool not in _WINDOW_HANDLE_TOOLS:
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ if (
+ arguments.get("pid") != descriptor.pid
+ or arguments.get("window_id") != descriptor.window_id
+ ):
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+
+
+async def validate_handle_is_current(
+ descriptor: HandleDescriptor | None,
+ tool: str,
+ discover: DiscoveryCall,
+) -> dict[str, Any] | None:
+ """Validate one descriptor against current Controller discovery state."""
+
+ if descriptor is None:
+ return None
+ app = descriptor.app if isinstance(descriptor, WindowHandleDescriptor) else descriptor
+ if app.pid is None:
+ if tool == "launch_app":
+ return None
+ raise ComputerScriptHandleError("stale_app_handle")
+
+ apps_result = await discover("list_apps", {})
+ if apps_result.get("isError") is True:
+ return apps_result
+ apps = _structured_list(apps_result, "apps")
+ if not any(app.matches(record) for record in apps):
+ raise ComputerScriptHandleError(
+ "stale_app_handle",
+ params={"pid": app.pid},
+ )
+
+ if not isinstance(descriptor, WindowHandleDescriptor):
+ return None
+ windows_result = await discover("list_windows", {"pid": descriptor.pid})
+ if windows_result.get("isError") is True:
+ return windows_result
+ windows = _structured_list(windows_result, "windows")
+ if not any(
+ record.get("pid") == descriptor.pid
+ and record.get("window_id") == descriptor.window_id
+ for record in windows
+ ):
+ raise ComputerScriptHandleError(
+ "stale_window_handle",
+ params={
+ "pid": descriptor.pid,
+ "window_id": descriptor.window_id,
+ },
+ )
+ return None
+
+
+def _optional_nonempty_string(value: Any) -> str | None:
+ if value is None:
+ return None
+ if not isinstance(value, str) or not value:
+ raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+ return value
+
+
+def _structured_list(result: Mapping[str, Any], field: str) -> list[Mapping[str, Any]]:
+ structured = result.get("structuredContent")
+ if not isinstance(structured, Mapping):
+ return []
+ records = structured.get(field)
+ if not isinstance(records, list):
+ return []
+ return [record for record in records if isinstance(record, Mapping)]
diff --git a/core/computer_script_sessions.py b/core/computer_script_sessions.py
new file mode 100644
index 000000000..3946e346e
--- /dev/null
+++ b/core/computer_script_sessions.py
@@ -0,0 +1,181 @@
+"""Controller-owned Computer Use script connections.
+
+A script connection is a capability for one exact live Agent Turn. It is not a
+Session capability: once that Turn ends, a later Turn in the same Session
+cannot reuse the connection token.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+import secrets
+from typing import TYPE_CHECKING, Any
+
+from core.computer_cli import COMPUTER_CLI_TOOLS
+
+if TYPE_CHECKING:
+ from core.session_turns import SessionTurnManager, Turn
+
+
+_MAX_SCRIPT_TIMEOUT_SECONDS = 3600.0
+_INVALID_BINDING_RETENTION_SECONDS = 60.0
+_SCRIPT_TOOLS = frozenset({*COMPUTER_CLI_TOOLS.values(), "double_click"})
+
+
+class ComputerScriptSessionError(RuntimeError):
+ """A script connection request cannot be admitted."""
+
+ def __init__(self, code: str, *, status_code: int = 409) -> None:
+ self.code = code
+ self.status_code = status_code
+ super().__init__(code)
+
+
+@dataclass
+class _ScriptBinding:
+ token: str
+ session_id: str
+ turn: "Turn"
+ logical_turn_id: str | None
+ deadline: float
+ timeout_handle: asyncio.TimerHandle
+ turn_done_callback: Any
+ invalid_code: str | None = None
+ cleanup_handle: asyncio.TimerHandle | None = None
+
+
+class ComputerScriptSessionManager:
+ """Bind script calls to the exact Turn that opened the connection."""
+
+ def __init__(self, turn_manager: "SessionTurnManager") -> None:
+ self._turn_manager = turn_manager
+ self._bindings: dict[str, _ScriptBinding] = {}
+
+ def open(self, caller_session_id: str, timeout_seconds: float) -> dict[str, Any]:
+ if (
+ isinstance(timeout_seconds, bool)
+ or not isinstance(timeout_seconds, (int, float))
+ or timeout_seconds <= 0
+ or timeout_seconds > _MAX_SCRIPT_TIMEOUT_SECONDS
+ ):
+ raise ComputerScriptSessionError(
+ "invalid_computer_script_timeout",
+ status_code=400,
+ )
+
+ admission, admission_error = self._turn_manager.begin_computer_use_request(
+ caller_session_id
+ )
+ if admission is None:
+ raise ComputerScriptSessionError(admission_error)
+
+ loop = asyncio.get_running_loop()
+ token = secrets.token_urlsafe(32)
+
+ def turn_done(_task: asyncio.Task[Any]) -> None:
+ self._invalidate(token, "script_turn_ended")
+
+ timeout_handle = loop.call_later(
+ float(timeout_seconds),
+ self._invalidate,
+ token,
+ "script_timeout",
+ )
+ binding = _ScriptBinding(
+ token=token,
+ session_id=admission.session_id,
+ turn=admission.turn,
+ logical_turn_id=admission.logical_turn_id,
+ deadline=loop.time() + float(timeout_seconds),
+ timeout_handle=timeout_handle,
+ turn_done_callback=turn_done,
+ )
+ self._bindings[token] = binding
+ admission.turn.task.add_done_callback(turn_done)
+ return {
+ "connection_token": token,
+ "session_id": admission.session_id,
+ "timeout_ms": int(float(timeout_seconds) * 1000),
+ }
+
+ def admit(self, token: str) -> _ScriptBinding:
+ binding = self._bindings.get(token)
+ if binding is None:
+ raise ComputerScriptSessionError("script_connection_invalid")
+ if binding.invalid_code is not None:
+ raise ComputerScriptSessionError(binding.invalid_code)
+
+ loop = asyncio.get_running_loop()
+ if loop.time() >= binding.deadline:
+ self._invalidate(token, "script_timeout")
+ raise ComputerScriptSessionError("script_timeout")
+
+ admission, _error = self._turn_manager.begin_computer_use_request(
+ binding.session_id
+ )
+ if (
+ admission is None
+ or admission.turn is not binding.turn
+ or admission.logical_turn_id != binding.logical_turn_id
+ ):
+ self._invalidate(token, "script_turn_ended")
+ raise ComputerScriptSessionError("script_turn_ended")
+ return binding
+
+ def prepare_tool_request(
+ self,
+ token: str,
+ tool: str,
+ arguments: dict[str, Any],
+ ) -> tuple[str, dict[str, Any]]:
+ binding = self.admit(token)
+ if tool not in _SCRIPT_TOOLS:
+ raise ComputerScriptSessionError(
+ "invalid_computer_script_tool",
+ status_code=400,
+ )
+ if "session" in arguments:
+ raise ComputerScriptSessionError(
+ "session_argument_forbidden",
+ status_code=400,
+ )
+ return binding.session_id, {**arguments, "session": binding.session_id}
+
+ def close(self, token: str) -> bool:
+ binding = self._bindings.pop(token, None)
+ if binding is None:
+ return False
+ self._cancel_binding_handles(binding)
+ return True
+
+ def close_all(self) -> None:
+ for token in list(self._bindings):
+ self.close(token)
+
+ def invalidate_turn(self, turn: "Turn") -> None:
+ """Invalidate every script capability bound to one exact Turn."""
+
+ for token, binding in list(self._bindings.items()):
+ if binding.turn is turn:
+ self._invalidate(token, "script_turn_ended")
+
+ def _invalidate(self, token: str, code: str) -> None:
+ binding = self._bindings.get(token)
+ if binding is None or binding.invalid_code is not None:
+ return
+ binding.invalid_code = code
+ binding.timeout_handle.cancel()
+ binding.turn.task.remove_done_callback(binding.turn_done_callback)
+ binding.cleanup_handle = asyncio.get_running_loop().call_later(
+ _INVALID_BINDING_RETENTION_SECONDS,
+ self.close,
+ token,
+ )
+
+ @staticmethod
+ def _cancel_binding_handles(binding: _ScriptBinding) -> None:
+ binding.timeout_handle.cancel()
+ if binding.cleanup_handle is not None:
+ binding.cleanup_handle.cancel()
+ binding.turn.task.remove_done_callback(binding.turn_done_callback)
diff --git a/core/computer_server.py b/core/computer_server.py
index 970c8285d..e0845cce6 100644
--- a/core/computer_server.py
+++ b/core/computer_server.py
@@ -42,6 +42,12 @@ from core.computer_use import (
effective_computer_use_status,
read_computer_use_state,
)
+from core.computer_script_handles import (
+ ComputerScriptHandleError,
+ bind_handle_to_request,
+ parse_handle_descriptor,
+ validate_handle_is_current,
+)
_LEASE_TTL_SECONDS = 60.0
_LEASE_HEARTBEAT_SECONDS = 10.0
@@ -54,6 +60,7 @@ _UPSTREAM_DETAIL_LIMIT = 2048
_UPSTREAM_DIAGNOSTIC_SCAN_LIMIT = 8192
_UPSTREAM_DIAGNOSTIC_LOG_LIMIT = 20
_ADMISSION_RETRY_LIMIT = 1
+_NO_SCRIPT_HANDLE = object()
_PROXY_PROCESS_ENV_KEYS = (
"HOME",
"LANG",
@@ -80,6 +87,7 @@ _INPUT_TOOLS = frozenset(
)
_OBSERVE_TOOLS = frozenset({"get_window_state", "zoom"})
_ALLOWED_TARGET_KEYS = frozenset({"kind", "pid", "window_id"})
+_UPSTREAM_NOT_DISPATCHED_CODES = frozenset({"foreground_consent_required"})
_COMMAND_MODIFIERS = frozenset({"cmd", "command", "meta"})
_SHIFT_MODIFIERS = frozenset({"shift"})
@@ -727,12 +735,17 @@ def _error_result(
message: str | None = None,
*,
detail: str | bytes | None = None,
+ dispatched: bool | None = None,
result_params: Mapping[str, Any] | None = None,
**params: Any,
) -> dict[str, Any]:
+ if dispatched is not None and dispatched is not False:
+ raise ValueError("computer-use errors may only declare dispatched=false")
payload_params = dict(result_params) if result_params is not None else {}
payload_params.update(params)
payload = {"code": code, "params": payload_params}
+ if dispatched is False:
+ payload["dispatched"] = False
bounded_detail = _bounded_diagnostic(detail)
if bounded_detail:
payload["detail"] = bounded_detail
@@ -748,6 +761,23 @@ def _error_result(
}
+def _not_dispatched_error_result(result: Mapping[str, Any]) -> dict[str, Any]:
+ structured = result.get("structuredContent")
+ if not isinstance(structured, Mapping):
+ return _error_result("computer_use_error", dispatched=False)
+ code = structured.get("code")
+ return _error_result(
+ code if isinstance(code, str) else "computer_use_error",
+ detail=structured.get("detail"),
+ dispatched=False,
+ result_params=(
+ structured.get("params")
+ if isinstance(structured.get("params"), Mapping)
+ else None
+ ),
+ )
+
+
def _fold_tool_result(
message: dict[str, Any],
*,
@@ -774,14 +804,18 @@ def _fold_tool_result(
structured = result.pop("structuredContent", None)
if result.get("isError") is True:
if isinstance(structured, dict) and isinstance(structured.get("code"), str):
+ code = structured["code"]
detail = (
_diagnostic_from_mapping(structured)
or _diagnostic_from_mapping(structured.get("params"))
or _diagnostic_from_content(result.get("content"))
)
return _error_result(
- structured["code"],
+ code,
detail=detail,
+ dispatched=(
+ False if code in _UPSTREAM_NOT_DISPATCHED_CODES else None
+ ),
result_params=_driver_error_params(structured),
)
return _error_result(
@@ -1169,25 +1203,43 @@ class ComputerUseServer:
self,
name: str,
arguments: Mapping[str, Any],
+ *,
+ handle_descriptor: Any = _NO_SCRIPT_HANDLE,
) -> dict[str, Any]:
session = arguments.get("session")
if not isinstance(session, str) or not session.strip():
return _error_result(
"session_required",
"Every computer-use call requires the current Avibe session id.",
+ dispatched=False,
)
session = session.strip()
if name not in COMPUTER_USE_TOOL_NAMES:
- return _error_result("unknown_tool", tool=name)
+ return _error_result("unknown_tool", dispatched=False, tool=name)
if not isinstance(arguments, dict):
- return _error_result("invalid_arguments", "Tool arguments must be an object.")
+ return _error_result(
+ "invalid_arguments",
+ "Tool arguments must be an object.",
+ dispatched=False,
+ )
+ parsed_handle = None
+ if handle_descriptor is not _NO_SCRIPT_HANDLE:
+ try:
+ parsed_handle = parse_handle_descriptor(handle_descriptor)
+ bind_handle_to_request(parsed_handle, name, arguments)
+ except ComputerScriptHandleError as exc:
+ return _error_result(
+ exc.code,
+ dispatched=False,
+ result_params=exc.params,
+ )
lock = self._session_locks.setdefault(session, asyncio.Lock())
async with lock:
try:
_validate_input(name, arguments)
except ComputerServerError as exc:
- return _error_result(exc.code, str(exc))
+ return _error_result(exc.code, str(exc), dispatched=False)
for admission_attempt in range(_ADMISSION_RETRY_LIMIT + 1):
acquisition: LeaseAcquisition | None = None
@@ -1200,6 +1252,7 @@ class ComputerUseServer:
return _error_result(
"computer_use_unavailable",
"Computer use is not ready.",
+ dispatched=False,
status=status.status,
reason=status.reason,
)
@@ -1241,6 +1294,40 @@ class ComputerUseServer:
explicit_start=name == "start_session",
),
)
+ if handle_descriptor is not _NO_SCRIPT_HANDLE:
+ async def discover(
+ discovery_tool: str,
+ discovery_arguments: Mapping[str, Any],
+ ) -> dict[str, Any]:
+ forwarded_discovery = {
+ **discovery_arguments,
+ "session": session,
+ }
+ if discovery_tool not in self._upstream_accepts_session:
+ forwarded_discovery.pop("session", None)
+ return await self._upstream_call(
+ upstream,
+ discovery_tool,
+ forwarded_discovery,
+ )
+
+ try:
+ validation_error = await self._run_while_lease_owned(
+ heartbeat,
+ validate_handle_is_current(
+ parsed_handle,
+ name,
+ discover,
+ ),
+ )
+ except ComputerScriptHandleError as exc:
+ return _error_result(
+ exc.code,
+ dispatched=False,
+ result_params=exc.params,
+ )
+ if validation_error is not None:
+ return _not_dispatched_error_result(validation_error)
self._require_observation(name, arguments, session_state)
await self._run_while_lease_owned(
heartbeat,
@@ -1303,6 +1390,7 @@ class ComputerUseServer:
return _error_result(
"desktop_lease_lost",
"The desktop lease changed before computer use completed.",
+ dispatched=False if pre_forward else None,
)
except _DaemonGenerationChanged:
if upstream is not None:
@@ -1321,6 +1409,7 @@ class ComputerUseServer:
return _error_result(
"computer_use_unavailable",
"Computer use changed while the request was being admitted.",
+ dispatched=False,
status="unavailable",
reason="daemon_generation_changed",
)
@@ -1334,7 +1423,11 @@ class ComputerUseServer:
self._lease_manager.release,
acquisition.lease,
)
- return _error_result(exc.code, str(exc))
+ return _error_result(
+ exc.code,
+ str(exc),
+ dispatched=False if pre_forward else None,
+ )
except asyncio.CancelledError:
if (
acquisition is not None
@@ -1368,6 +1461,7 @@ class ComputerUseServer:
"upstream_unavailable",
"The computer-use proxy exited. Observe again before retrying.",
detail=detail,
+ dispatched=False if pre_forward else None,
)
except Exception as exc:
if (
@@ -1389,6 +1483,7 @@ class ComputerUseServer:
return _error_result(
"computer_use_error",
detail=detail,
+ dispatched=False if pre_forward else None,
error_type=type(exc).__name__,
)
finally:
diff --git a/core/controller.py b/core/controller.py
index a0380552a..6fa11fea3 100644
--- a/core/controller.py
+++ b/core/controller.py
@@ -264,6 +264,7 @@ class Controller:
# Keep the driver proxy lazy so an enabled desktop toggle does not
# create a driver process until an active Turn requests it.
self.computer_use_server = None
+ self.computer_script_sessions = None
self.runtime_ownership = RuntimeOwnershipProvider(
self.session_turns._sqlite_engine()
)
@@ -2571,6 +2572,10 @@ class Controller:
self._internal_server_task = None
async def _close_computer_use_server() -> None:
+ script_sessions = getattr(self, "computer_script_sessions", None)
+ self.computer_script_sessions = None
+ if script_sessions is not None:
+ script_sessions.close_all()
server = getattr(self, "computer_use_server", None)
self.computer_use_server = None
if server is not None:
diff --git a/core/internal_server.py b/core/internal_server.py
index b0dee9360..37015cb3c 100644
--- a/core/internal_server.py
+++ b/core/internal_server.py
@@ -55,6 +55,11 @@ logger = logging.getLogger(__name__)
_PROCESSING_RECORD_CURSOR_RE = re.compile(r"[A-Za-z0-9_-]{1,256}\Z")
_PROCESSING_RECORD_ENTRY_ID_RE = re.compile(r"[A-Za-z0-9_.:-]{1,256}\Z")
+_COMPUTER_SCRIPT_REQUEST_MAX_BYTES = 8 * 1024 * 1024
+
+
+class _RequestBodyTooLarge(RuntimeError):
+ """An internal request exceeded its route-specific hard limit."""
def _create_controller_loop_server(config: Any) -> Any:
@@ -189,6 +194,16 @@ def create_app(
controller.session_turns = manager
manager.bind_context(_build_session_context)
+ from core.computer_script_sessions import (
+ ComputerScriptSessionError,
+ ComputerScriptSessionManager,
+ )
+
+ computer_script_sessions = getattr(controller, "computer_script_sessions", None)
+ if not isinstance(computer_script_sessions, ComputerScriptSessionManager):
+ computer_script_sessions = ComputerScriptSessionManager(manager)
+ controller.computer_script_sessions = computer_script_sessions
+
# The turn registry (``session_id -> Turn``) is owned by the manager and tests
# inspect it via ``app.state``. Flush intents live on each ``Turn`` (set by
# ``manager.cancel`` / ``manager.send_now``), not in side sets here.
@@ -728,6 +743,170 @@ def create_app(
"result": result,
}
+ def _computer_script_error(
+ exc: ComputerScriptSessionError,
+ *,
+ dispatched: bool | None = None,
+ ) -> JSONResponse:
+ content: dict[str, Any] = {
+ "ok": False,
+ "error": exc.code,
+ "params": {},
+ }
+ if dispatched is False:
+ content["dispatched"] = False
+ return JSONResponse(
+ status_code=exc.status_code,
+ content=content,
+ )
+
+ @app.post("/internal/computer-script/open")
+ async def _computer_script_open(request: Request) -> Any:
+ payload = await _safe_json(request)
+ if set(payload) != {"caller_session_id", "timeout_seconds"}:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "ok": False,
+ "error": "invalid_computer_script_open_request",
+ "params": {},
+ },
+ )
+ caller_session_id = payload.get("caller_session_id")
+ if not isinstance(caller_session_id, str) or not caller_session_id.strip():
+ return JSONResponse(
+ status_code=400,
+ content={
+ "ok": False,
+ "error": "invalid_computer_script_open_request",
+ "params": {},
+ },
+ )
+ try:
+ opened = computer_script_sessions.open(
+ caller_session_id.strip(),
+ payload.get("timeout_seconds"),
+ )
+ except ComputerScriptSessionError as exc:
+ return _computer_script_error(exc)
+ return {"ok": True, **opened}
+
+ @app.post("/internal/computer-script/request")
+ async def _computer_script_request(request: Request) -> Any:
+ try:
+ payload = await _bounded_json_object(
+ request,
+ limit=_COMPUTER_SCRIPT_REQUEST_MAX_BYTES,
+ )
+ except _RequestBodyTooLarge:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "ok": False,
+ "error": "computer_script_request_too_large",
+ "params": {
+ "limit_bytes": _COMPUTER_SCRIPT_REQUEST_MAX_BYTES,
+ },
+ "dispatched": False,
+ },
+ )
+ if set(payload) != {
+ "connection_token",
+ "tool",
+ "arguments",
+ "descriptor",
+ }:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "ok": False,
+ "error": "invalid_computer_script_request",
+ "params": {},
+ "dispatched": False,
+ },
+ )
+ connection_token = payload.get("connection_token")
+ tool = payload.get("tool")
+ arguments = payload.get("arguments")
+ descriptor = payload.get("descriptor")
+ if (
+ not isinstance(connection_token, str)
+ or not connection_token
+ or not isinstance(tool, str)
+ or not isinstance(arguments, dict)
+ ):
+ return JSONResponse(
+ status_code=400,
+ content={
+ "ok": False,
+ "error": "invalid_computer_script_request",
+ "params": {},
+ "dispatched": False,
+ },
+ )
+ try:
+ caller_session_id, forwarded = (
+ computer_script_sessions.prepare_tool_request(
+ connection_token,
+ tool,
+ arguments,
+ )
+ )
+ except ComputerScriptSessionError as exc:
+ return _computer_script_error(exc, dispatched=False)
+
+ computer_server = getattr(controller, "computer_use_server", None)
+ if computer_server is None:
+ from core.computer_server import ComputerUseServer
+
+ try:
+ computer_server = ComputerUseServer()
+ except Exception:
+ logger.exception("Computer Use server initialization failed")
+ return JSONResponse(
+ status_code=503,
+ content={
+ "ok": False,
+ "error": "computer_use_unavailable",
+ "params": {},
+ "dispatched": False,
+ },
+ )
+ controller.computer_use_server = computer_server
+
+ result = await computer_server.call_tool(
+ tool,
+ forwarded,
+ handle_descriptor=descriptor,
+ )
+ return {
+ "ok": result.get("isError") is not True,
+ "tool": tool,
+ "session_id": caller_session_id,
+ "result": result,
+ }
+
+ @app.post("/internal/computer-script/close")
+ async def _computer_script_close(request: Request) -> Any:
+ payload = await _safe_json(request)
+ connection_token = payload.get("connection_token")
+ if set(payload) != {"connection_token"} or not isinstance(
+ connection_token,
+ str,
+ ):
+ return JSONResponse(
+ status_code=400,
+ content={
+ "ok": False,
+ "error": "invalid_computer_script_close_request",
+ "params": {},
+ },
+ )
+ return {
+ "ok": True,
+ "closed": computer_script_sessions.close(connection_token),
+ }
+
@app.post("/internal/show-access/settings-read")
async def _show_access_settings_read(request: Request) -> Any:
from core.show_pages import ShowPageError, ShowPageStore, show_access_payload
@@ -1580,6 +1759,32 @@ async def _safe_json(request: Request) -> dict[str, Any]:
return body if isinstance(body, dict) else {}
+async def _bounded_json_object(
+ request: Request,
+ *,
+ limit: int,
+) -> dict[str, Any]:
+ content_length = request.headers.get("content-length")
+ if content_length is not None:
+ try:
+ declared_length = int(content_length)
+ except ValueError:
+ declared_length = -1
+ if declared_length > limit:
+ raise _RequestBodyTooLarge
+
+ body = bytearray()
+ async for chunk in request.stream():
+ if len(body) + len(chunk) > limit:
+ raise _RequestBodyTooLarge
+ body.extend(chunk)
+ try:
+ payload = await asyncio.to_thread(json.loads, bytes(body))
+ except (TypeError, ValueError, RecursionError):
+ return {}
+ return payload if isinstance(payload, dict) else {}
+
+
async def _build_dispatch_payload(payload: dict[str, Any]) -> tuple[str, MessageContext]:
"""Translate the JSON payload into a ``(text, MessageContext)`` pair.
diff --git a/core/session_turns.py b/core/session_turns.py
index df4ee1fe0..0f79e8b10 100644
--- a/core/session_turns.py
+++ b/core/session_turns.py
@@ -552,6 +552,8 @@ class ComputerUseAdmission:
"""One Computer Use request admitted for one live Avibe Session."""
session_id: str
+ turn: Turn = field(repr=False, compare=False)
+ logical_turn_id: str | None = None
@dataclass(frozen=True)
@@ -3898,6 +3900,7 @@ class SessionTurnManager:
):
return {"state": "reconciling", "reason": "prewrite_owner_changed"}
+ self._close_turn_computer_use_admission(projected)
projected.cancel_settled_by = SETTLED_BY_STOPPED
mark_prewrite_user_stop(projected.context)
projected.task.cancel()
@@ -3957,6 +3960,8 @@ class SessionTurnManager:
or turn.task.done()
):
runtime_turn = None
+ if runtime_turn is not None:
+ self._close_turn_computer_use_admission(runtime_turn)
stop_context = runtime_turn.context if runtime_turn is not None else None
unavailable_reason = "runtime_owner_unavailable"
if stop_context is None and logical_turn_id:
@@ -4514,10 +4519,19 @@ class SessionTurnManager:
return (
ComputerUseAdmission(
session_id=normalized_session_id,
+ turn=turn,
+ logical_turn_id=turn.logical_turn_id,
),
"",
)
+ def _close_turn_computer_use_admission(self, turn: Turn) -> None:
+ turn.computer_use_admission_closed = True
+ script_sessions = getattr(self.controller, "computer_script_sessions", None)
+ invalidate_turn = getattr(script_sessions, "invalidate_turn", None)
+ if callable(invalidate_turn):
+ invalidate_turn(turn)
+
def _delivery_runtime_owner_factory(
self,
session_id: str,
@@ -8620,6 +8634,7 @@ class SessionTurnManager:
return {"ok": False, "code": "not_in_flight", "session_id": session_id}
if turn.task.done():
return {"ok": True, "session_id": session_id, "status": "already_finished"}
+ self._close_turn_computer_use_admission(turn)
turn.cancel_settled_by = SETTLED_BY_STOPPED
if turn.context.platform_specific is None:
turn.context.platform_specific = {}
diff --git a/docs/plans/desktop-computer-use-js-sdk-v1-implementation.md b/docs/plans/desktop-computer-use-js-sdk-v1-implementation.md
new file mode 100644
index 000000000..0b8ab22e1
--- /dev/null
+++ b/docs/plans/desktop-computer-use-js-sdk-v1-implementation.md
@@ -0,0 +1,70 @@
+# Computer Use JavaScript SDK v1 Implementation
+
+**Status:** In progress
+**Date:** 2026-10-14
+**Authority:** `docs/plans/desktop-computer-use-js-sdk-design.md`
+
+## Background
+
+The shipped `vibe computer` command already routes each desktop operation
+through Controller admission and `ComputerUseServer`. JavaScript scripts need
+the same boundary, a stable object model, and a runner that uses Avibe's private
+desktop Node runtime without resolving packages from the working directory.
+
+The complete v1, including Web/IM confirmation ownership, is expected to exceed
+the implementation brief's approximate 1,800-line split threshold. Delivery is
+therefore divided into two independent PRs:
+
+1. SDK core, runner, Controller script Session, and exact Turn binding.
+2. `avibe.confirm`, `avibe.notify`, and expected-approver binding.
+
+The second PR starts only after the first PR is delivered through its normal
+review and CI gates.
+
+## Change Contract
+
+### Intended behavior
+
+- `vibe computer run <file.mjs> [--arg k=v | --arg k=@file]` runs a normal ESM
+ module on Avibe's verified private Node runtime.
+- The runner injects resolution for `@avibe/computer`; project-local and global
+ npm state cannot replace the shipped module.
+- One script uses one keep-alive Controller connection for SDK calls and one
+ total deadline.
+- The SDK exposes the `computer -> app -> window -> element` model, declarative
+ selectors, hidden element tokens, one safe re-observation for stale tokens,
+ waits, expectations, typed errors, and an effect result for every action.
+- The Controller binds a script connection to the exact live Turn object that
+ launched it. Turn completion, Stop, or the script deadline invalidates the
+ connection, so a later Turn in the same Session cannot reuse it.
+- Script desktop operations continue through `ComputerUseServer`; neither the
+ runner nor the SDK connects to the driver socket.
+- Failures identify the current SDK step and include the most recent screenshot
+ path when an observation produced one.
+
+### Affected boundaries
+
+- Controller internal HTTP API: script open, request, and close endpoints.
+- Session Turn admission: expose the exact admitted Turn in the internal
+ admission record.
+- Python CLI/runtime packaging: script runner and shipped JavaScript assets.
+- Built-in Computer Use Skill: script authoring and invocation guidance.
+
+### Deferred behavior
+
+- `avibe.confirm`, `avibe.notify`, pending-question ownership, and the
+ verify-confirm-reverify-act continuation are delivered in the second PR.
+- Existing single-operation `vibe computer` leaves are not rebased onto the SDK.
+- Scheduled or unattended runs, automatic Skill saving, browser support,
+ Windows desktop actions, and a persistent REPL remain out of scope.
+
+## Validation
+
+- Parser-backed coverage for every documented CLI example.
+- Hermetic Controller route tests for exact-Turn binding, deadline invalidation,
+ and stale connection rejection.
+- Hermetic Node tests against a fake loopback Controller for module injection,
+ selectors, stale-token re-observation, effects, waits, expectations, typed
+ errors, failing-step reporting, and connection reuse.
+- Focused pytest, Ruff on changed Python files, wheel content inspection, and
+ exact-head review plus CI before close-out.
diff --git a/skills/computer-use/SKILL.md b/skills/computer-use/SKILL.md
index f1f1817b3..05015802f 100644
--- a/skills/computer-use/SKILL.md
+++ b/skills/computer-use/SKILL.md
@@ -10,6 +10,7 @@ version: 1.0.0
Use the Controller-owned CLI for desktop work:
```text
+vibe computer run ./inspect-mail.mjs --arg mailbox=Inbox --arg query=@/tmp/query.txt
vibe computer observe apps
vibe computer observe windows
vibe computer observe window --arg pid=123 --arg window_id=456 --output /tmp/window.png
@@ -27,11 +28,47 @@ vibe computer launch --arg bundle_id=com.example.App
vibe computer force-quit --arg pid=123
```
-Every leaf accepts `--args-json OBJECT`, `--args-json @FILE`, repeatable
-`--arg NAME=VALUE`, and `--json`. The two screenshot operations also accept
-`--output PATH`; the CLI writes or confirms the caller-selected PNG and prints
-the resulting path. `observe window` and `zoom` are the observation operations
-that establish the exact window state required before input.
+For multi-step work, write a normal ESM file and import the shipped SDK:
+
+```js
+import { computer } from "@avibe/computer";
+
+export default async ({ mailbox, query }) => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.waitFor({ role: "AXWindow", labelContains: mailbox });
+ const search = window.element({ role: "textbox", labelContains: "Search" });
+ await search.fill(query);
+ await window
+ .expect({ role: "textbox", labelContains: "Search" })
+ .toHaveValue(query);
+ return { mailbox, query };
+};
+```
+
+Run it with `vibe computer run ./inspect-mail.mjs --arg mailbox=Inbox
+--arg query=@/tmp/query.txt`. Ordinary argument values remain strings; `@file`
+loads the file as UTF-8 text. The default-exported function receives one
+argument object. Its return value becomes the JSON result; an omitted return is
+reported as `null`, and any other return must be JSON-serializable.
+
+The runner uses Avibe's shipped Node runtime and injects `@avibe/computer`;
+never install or resolve another package with that name. The SDK keeps element
+tokens private, re-observes once when the Controller explicitly reports a stale
+observation, and stops on ambiguous action outcomes.
+App and window handles are immutable process descriptors; after an app
+relaunches, acquire fresh handles with `computer.app(...)` before continuing.
+
+Scripts that send, submit, pay, delete, or perform another irreversible action
+must follow verify -> confirm -> re-verify -> act. If the owner-confirmation
+primitive is unavailable, stop before the irreversible action and use an
+interactive flow. Save a proven script as a reusable Skill only after the owner
+explicitly approves saving it.
+
+Every single-operation leaf accepts `--args-json OBJECT`, `--args-json @FILE`,
+repeatable `--arg NAME=VALUE`, and `--json`. The two screenshot operations also
+accept `--output PATH`; the CLI writes or confirms the caller-selected PNG and
+prints the resulting path. `observe window` and `zoom` are the observation
+operations that establish the exact window state required before input.
`--json` keeps the structured driver object at `result.structuredContent`.
The legacy `content` text block remains available for existing callers.
diff --git a/tests/test_computer_cli.py b/tests/test_computer_cli.py
index 02c71e7a1..97346e34d 100644
--- a/tests/test_computer_cli.py
+++ b/tests/test_computer_cli.py
@@ -58,6 +58,7 @@ def _bind_socket(path: Path) -> socket.socket:
def test_parser_covers_the_allowlisted_driver_operations_without_session_id() -> None:
commands = [
+ (["computer", "run", "./script.mjs"], ("run",)),
(["computer", "observe", "apps"], ("observe", "apps")),
(["computer", "observe", "windows"], ("observe", "windows")),
(["computer", "observe", "window"], ("observe", "window")),
@@ -85,7 +86,8 @@ def test_parser_covers_the_allowlisted_driver_operations_without_session_id() ->
else:
command_path = (args.computer_command,)
assert command_path == path
- assert tool_for_command(path)
+ if path != ("run",):
+ assert tool_for_command(path)
with pytest.raises(SystemExit):
parser.parse_args(["computer", "click", "--session-id", "manual"])
@@ -117,6 +119,9 @@ def test_packaged_skill_examples_parse_and_match_the_pinned_driver_schema() -> N
argv = shlex.split(command)
assert argv[:2] == ["vibe", "computer"]
args = parser.parse_args(argv[1:])
+ if args.computer_command == "run":
+ assert args.file.endswith(".mjs")
+ continue
if args.computer_command == "observe":
path = ("observe", args.computer_observe_command)
elif args.computer_command == "clipboard":
@@ -258,8 +263,126 @@ def test_cmd_computer_binds_the_caller_session_and_prints_structured_result(
assert payload["result"]["content"][0]["type"] == "text"
+def test_cmd_computer_run_passes_string_arguments_and_prints_json(
+ monkeypatch: pytest.MonkeyPatch,
+ capsys,
+) -> None:
+ monkeypatch.setenv("AVIBE_SESSION_ID", "ses-cli")
+ captured: dict = {}
+
+ def fake_run_computer_script(**kwargs):
+ captured.update(kwargs)
+ return {"mailbox": "Inbox", "count": "01"}
+
+ monkeypatch.setattr(
+ "vibe.computer_script_runner.run_computer_script",
+ fake_run_computer_script,
+ )
+ args = cli.build_parser().parse_args(
+ [
+ "computer",
+ "run",
+ "./inspect.mjs",
+ "--arg",
+ "mailbox=Inbox",
+ "--arg",
+ "count=01",
+ ]
+ )
+ assert cli.cmd_computer(args) == 0
+ assert captured == {
+ "caller_session_id": "ses-cli",
+ "script_file": "./inspect.mjs",
+ "arg_values": ["mailbox=Inbox", "count=01"],
+ }
+ assert json.loads(capsys.readouterr().out) == {
+ "count": "01",
+ "mailbox": "Inbox",
+ }
+
+
+def test_cmd_computer_run_localizes_missing_private_node(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from vibe.computer_script_runner import ComputerScriptRunnerError
+
+ monkeypatch.setenv("AVIBE_SESSION_ID", "ses-cli")
+ monkeypatch.setattr(cli, "_configured_cli_language", lambda: "zh")
+
+ def unavailable(**_kwargs):
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptNodeUnavailable",
+ code="computer_script_unavailable",
+ )
+
+ monkeypatch.setattr(
+ "vibe.computer_script_runner.run_computer_script",
+ unavailable,
+ )
+ args = cli.build_parser().parse_args(["computer", "run", "./inspect.mjs"])
+ with pytest.raises(
+ cli.TaskCliError,
+ match="Avibe 桌面应用随附的 Node runtime",
+ ) as exc_info:
+ cli.cmd_computer(args)
+ assert exc_info.value.code == "computer_script_unavailable"
+
+
+def test_cmd_computer_run_localizes_sdk_error_details(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from vibe.computer_script_runner import ComputerScriptRunnerError
+
+ monkeypatch.setenv("AVIBE_SESSION_ID", "ses-cli")
+ monkeypatch.setattr(cli, "_configured_cli_language", lambda: "zh")
+
+ def invalid_selector(**_kwargs):
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptFailed",
+ code="invalid_selector",
+ params={
+ "step": "computer.app",
+ "error": "Application selector contains unknown field 'text'.",
+ },
+ structured_error={
+ "code": "invalid_selector",
+ "params": {"field": "text"},
+ },
+ )
+
+ monkeypatch.setattr(
+ "vibe.computer_script_runner.run_computer_script",
+ invalid_selector,
+ )
+ args = cli.build_parser().parse_args(["computer", "run", "./inspect.mjs"])
+
+ with pytest.raises(cli.TaskCliError) as exc_info:
+ cli.cmd_computer(args)
+
+ assert str(exc_info.value) == (
+ "Computer Use 脚本在 computer.app 失败:脚本使用了无效选择器。"
+ )
+ assert "Application selector" not in str(exc_info.value)
+
+
def test_computer_cli_error_translations_exist_in_all_supported_languages() -> None:
- keys = {*_COMPUTER_ERROR_I18N_KEYS.values(), "computer.cli.error.screenshotMissing"}
+ keys = {
+ *_COMPUTER_ERROR_I18N_KEYS.values(),
+ "computer.cli.error.screenshotMissing",
+ "computer.cli.error.scriptArgNameValue",
+ "computer.cli.error.scriptArgDuplicate",
+ "computer.cli.error.scriptArgFile",
+ "computer.cli.error.scriptFile",
+ "computer.cli.error.scriptFileType",
+ "computer.cli.error.scriptNodeUnavailable",
+ "computer.cli.error.scriptSdkUnavailable",
+ "computer.cli.error.scriptOpenRejected",
+ "computer.cli.error.scriptRuntime",
+ "computer.cli.error.scriptRuntimeExit",
+ "computer.cli.error.scriptRuntimeInvalid",
+ "computer.cli.error.scriptTimedOut",
+ "computer.cli.error.scriptFailed",
+ }
for catalog_path in sorted(Path("vibe/i18n").glob("*.json")):
catalog = json.loads(catalog_path.read_text(encoding="utf-8"))
for key in keys:
diff --git a/tests/test_computer_script_sdk.py b/tests/test_computer_script_sdk.py
new file mode 100644
index 000000000..65405f845
--- /dev/null
+++ b/tests/test_computer_script_sdk.py
@@ -0,0 +1,2420 @@
+from __future__ import annotations
+
+import base64
+from contextlib import contextmanager
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import json
+import os
+from pathlib import Path
+import signal
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from typing import Iterator
+from unittest.mock import patch
+
+import httpx
+import pytest
+
+from vibe import cli, computer_script_runner
+from vibe.computer_script_runner import (
+ ComputerScriptRunnerError,
+ parse_script_arguments,
+)
+from vibe.internal_client import (
+ computer_script_close_sync,
+ computer_script_endpoint_payload,
+ computer_script_open_sync,
+ resolve_computer_script_endpoint,
+)
+
+
+NODE = shutil.which("node")
+SDK_DIRECTORY = Path("vibe/computer_sdk").resolve()
+
+
+def _result(payload: dict, *, is_error: bool = False, image: bool = False) -> dict:
+ content: list[dict] = [
+ {"type": "text", "text": "Structured content:\n" + json.dumps(payload)}
+ ]
+ if image:
+ content.append(
+ {
+ "type": "image",
+ "data": base64.b64encode(b"png-fixture").decode("ascii"),
+ "mimeType": "image/png",
+ }
+ )
+ return {
+ "ok": not is_error,
+ "result": {
+ "isError": is_error,
+ "structuredContent": payload,
+ "content": content,
+ },
+ }
+
+
+@contextmanager
+def _fake_controller(
+ *,
+ click_error: str | None = None,
+ initial_click_error: str = "observe_first",
+ duplicate_within: bool = False,
+ dynamic_elements_after_action: bool = False,
+ response_delay_by_tool: dict[str, float] | None = None,
+ apps: list[dict] | None = None,
+ app_responses: list[list[dict]] | None = None,
+ windows: list[dict] | None = None,
+ launch_pid: int | None = 41,
+) -> Iterator[tuple[ThreadingHTTPServer, list[dict]]]:
+ requests: list[dict] = []
+ observations = 0
+ clicks = 0
+ app_lookups = 0
+ connection_ids: set[int] = set()
+
+ class Handler(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+
+ def do_POST(self) -> None: # noqa: N802
+ nonlocal observations, clicks, app_lookups
+ length = int(self.headers.get("Content-Length", "0"))
+ payload = json.loads(self.rfile.read(length))
+ connection_ids.add(id(self.connection))
+ requests.append(
+ {
+ "path": self.path,
+ "payload": payload,
+ "authorization": self.headers.get("Authorization"),
+ }
+ )
+ if self.path == "/internal/computer-script/close":
+ body = {"ok": True, "closed": True}
+ else:
+ tool = payload["tool"]
+ delay = (response_delay_by_tool or {}).get(tool, 0)
+ if delay:
+ time.sleep(delay)
+ if tool == "list_apps":
+ if app_responses is not None:
+ app_records = app_responses[
+ min(app_lookups, len(app_responses) - 1)
+ ]
+ app_lookups += 1
+ else:
+ app_records = apps
+ body = _result(
+ {
+ "apps": app_records
+ if app_records is not None
+ else [
+ {
+ "pid": 41,
+ "name": "Mail",
+ "bundle_id": "com.apple.mail",
+ "running": True,
+ "active": False,
+ }
+ ]
+ }
+ )
+ elif tool == "list_windows":
+ body = _result(
+ {
+ "windows": windows
+ or [
+ {
+ "pid": 41,
+ "window_id": 9,
+ "app_name": "Mail",
+ "title": "New Message",
+ "bounds": {
+ "x": 0,
+ "y": 0,
+ "width": 800,
+ "height": 600,
+ },
+ "is_on_screen": True,
+ "z_index": 1,
+ }
+ ],
+ }
+ )
+ elif tool == "get_window_state":
+ observations += 1
+ elements = [
+ {
+ "element_index": 1,
+ "element_token": f"group-{observations}",
+ "role": "AXGroup",
+ "label": "Composer",
+ "value": None,
+ "parent_index": None,
+ "depth": 0,
+ },
+ {
+ "element_index": 2,
+ "element_token": f"send-{observations}",
+ "role": "AXButton",
+ "label": "Send",
+ "value": None,
+ "parent_index": 1,
+ "depth": 1,
+ },
+ {
+ "element_index": 3,
+ "element_token": f"subject-{observations}",
+ "role": "AXTextField",
+ "label": "Subject",
+ "value": "Quarterly report",
+ "parent_index": 1,
+ "depth": 1,
+ },
+ {
+ "element_index": 4,
+ "element_token": f"send-alt-{observations}",
+ "role": "AXButton",
+ "label": "Send",
+ "value": None,
+ "parent_index": 1,
+ "depth": 1,
+ },
+ ]
+ if dynamic_elements_after_action:
+ elements = [
+ {
+ "element_index": 1,
+ "element_token": f"group-{observations}",
+ "role": "AXGroup",
+ "label": "Items",
+ "value": None,
+ "parent_index": None,
+ "depth": 0,
+ },
+ {
+ "element_index": 2,
+ "element_token": f"add-{observations}",
+ "role": "AXButton",
+ "label": "Add",
+ "value": None,
+ "parent_index": 1,
+ "depth": 1,
+ },
+ {
+ "element_index": 3,
+ "element_token": f"delete-a-{observations}",
+ "role": "AXButton",
+ "label": "Delete",
+ "value": None,
+ "parent_index": 1,
+ "depth": 1,
+ },
+ ]
+ if clicks >= 2:
+ elements.append(
+ {
+ "element_index": 4,
+ "element_token": f"delete-b-{observations}",
+ "role": "AXButton",
+ "label": "Delete",
+ "value": None,
+ "parent_index": 1,
+ "depth": 1,
+ }
+ )
+ if duplicate_within:
+ elements = [
+ {
+ "element_index": 1,
+ "element_token": f"group-a-{observations}",
+ "role": "AXGroup",
+ "label": "Composer",
+ "value": None,
+ "parent_index": None,
+ "depth": 0,
+ },
+ {
+ "element_index": 2,
+ "element_token": f"group-b-{observations}",
+ "role": "AXGroup",
+ "label": "Composer",
+ "value": None,
+ "parent_index": None,
+ "depth": 0,
+ },
+ {
+ "element_index": 3,
+ "element_token": f"send-{observations}",
+ "role": "AXButton",
+ "label": "Send",
+ "value": None,
+ "parent_index": 1,
+ "depth": 1,
+ },
+ ]
+ if (
+ payload["arguments"].get("include_accessibility_tree")
+ is False
+ ):
+ elements = None
+ body = _result(
+ {
+ "pid": 41,
+ "window_id": 9,
+ **({"elements": elements} if elements is not None else {}),
+ },
+ image=True,
+ )
+ elif tool == "click":
+ clicks += 1
+ if click_error is not None:
+ body = _result(
+ {"code": click_error, "params": {}},
+ is_error=True,
+ )
+ elif clicks == 1:
+ body = _result(
+ {
+ "code": initial_click_error,
+ "params": {},
+ "dispatched": False,
+ },
+ is_error=True,
+ )
+ else:
+ body = _result(
+ {
+ "effect": "confirmed",
+ "route": "accessibility",
+ }
+ )
+ elif tool == "double_click":
+ body = _result(
+ {
+ "effect": "confirmed",
+ "route": "accessibility",
+ }
+ )
+ elif tool == "kill_app":
+ body = _result(
+ {
+ "effect": "confirmed",
+ "route": "system_api",
+ }
+ )
+ elif tool == "launch_app":
+ launch_payload = {
+ "effect": "confirmed",
+ "route": "system_api",
+ }
+ if launch_pid is not None:
+ launch_payload["pid"] = launch_pid
+ body = _result(launch_payload)
+ else:
+ body = _result(
+ {"code": "unknown_tool", "params": {"tool": tool}},
+ is_error=True,
+ )
+ encoded = json.dumps(body).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(encoded)))
+ self.send_header("X-Avibe-Control-Instance", "fixture-instance")
+ self.end_headers()
+ try:
+ self.wfile.write(encoded)
+ except (BrokenPipeError, ConnectionResetError):
+ pass
+
+ def log_message(self, _format: str, *_args) -> None:
+ return
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ server.daemon_threads = True
+ server.connection_ids = connection_ids # type: ignore[attr-defined]
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ yield server, requests
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=5)
+
+
+def _run_node_script(
+ tmp_path: Path,
+ source: str,
+ *,
+ click_error: str | None = None,
+ initial_click_error: str = "observe_first",
+ duplicate_within: bool = False,
+ dynamic_elements_after_action: bool = False,
+ response_delay_by_tool: dict[str, float] | None = None,
+ apps: list[dict] | None = None,
+ app_responses: list[list[dict]] | None = None,
+ windows: list[dict] | None = None,
+ launch_pid: int | None = 41,
+) -> tuple[dict, ThreadingHTTPServer, list[dict]]:
+ if NODE is None:
+ pytest.skip("Node.js is unavailable")
+ script = tmp_path / "script.mjs"
+ script.write_text(source, encoding="utf-8")
+ shadow = tmp_path / "node_modules" / "@avibe" / "computer"
+ shadow.mkdir(parents=True)
+ (shadow / "package.json").write_text(
+ '{"name":"@avibe/computer","type":"module","exports":"./index.mjs"}',
+ encoding="utf-8",
+ )
+ (shadow / "index.mjs").write_text(
+ 'throw new Error("project-local shadow loaded");',
+ encoding="utf-8",
+ )
+ artifacts = tmp_path / "artifacts"
+ artifacts.mkdir()
+ result_path = artifacts / "result.json"
+
+ with _fake_controller(
+ click_error=click_error,
+ initial_click_error=initial_click_error,
+ duplicate_within=duplicate_within,
+ dynamic_elements_after_action=dynamic_elements_after_action,
+ response_delay_by_tool=response_delay_by_tool,
+ apps=apps,
+ app_responses=app_responses,
+ windows=windows,
+ launch_pid=launch_pid,
+ ) as (server, requests):
+ request = {
+ "script_url": script.resolve().as_uri(),
+ "sdk_url": (SDK_DIRECTORY / "index.mjs").as_uri(),
+ "arguments": {"subject": "Quarterly report"},
+ "endpoint": {
+ "transport": "tcp",
+ "host": "127.0.0.1",
+ "port": server.server_address[1],
+ "headers": {"Authorization": "Bearer fixture-token"},
+ "instance_id": "fixture-instance",
+ "instance_header": "X-Avibe-Control-Instance",
+ },
+ "connection_token": "script-token",
+ "artifact_directory": str(artifacts),
+ "runtime_state_path": str(artifacts / "runtime-state.json"),
+ "result_path": str(result_path),
+ }
+ started = time.monotonic()
+ completed = computer_script_runner._run_node_process(
+ Path(NODE),
+ SDK_DIRECTORY / "runner.mjs",
+ request,
+ 10,
+ )
+ server.subprocess_elapsed = time.monotonic() - started # type: ignore[attr-defined]
+ server.completed = completed # type: ignore[attr-defined]
+ assert completed.returncode == 0, completed.stderr
+ result_read = computer_script_runner._read_capped_json(
+ result_path,
+ computer_script_runner._SCRIPT_RESULT_MAX_BYTES,
+ )
+ assert result_read.error is None
+ assert isinstance(result_read.value, dict)
+ return result_read.value, server, requests
+
+
+def _run_python_deadline_script(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ source: str,
+ *,
+ response_delay_by_tool: dict[str, float] | None = None,
+ timeout_seconds: float = 0.25,
+) -> tuple[ComputerScriptRunnerError, list[dict], float]:
+ if NODE is None:
+ pytest.skip("Node.js is unavailable")
+ script = tmp_path / "deadline-script.mjs"
+ script.write_text(source, encoding="utf-8")
+ endpoint = object()
+
+ with _fake_controller(
+ response_delay_by_tool=response_delay_by_tool,
+ ) as (server, requests):
+ monkeypatch.setattr(
+ computer_script_runner,
+ "private_desktop_node_bin",
+ lambda: Path(NODE),
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "resolve_computer_script_endpoint",
+ lambda: endpoint,
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "computer_script_open_sync",
+ lambda **_kwargs: {
+ "body": {
+ "ok": True,
+ "connection_token": "script-token",
+ }
+ },
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "computer_script_endpoint_payload",
+ lambda value: {
+ "transport": "tcp",
+ "host": "127.0.0.1",
+ "port": server.server_address[1],
+ "headers": {"Authorization": "Bearer fixture-token"},
+ "instance_id": "fixture-instance",
+ "instance_header": "X-Avibe-Control-Instance",
+ }
+ if value is endpoint
+ else {},
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "computer_script_close_sync",
+ lambda **_kwargs: None,
+ )
+ started = time.monotonic()
+ with pytest.raises(ComputerScriptRunnerError) as exc_info:
+ computer_script_runner.run_computer_script(
+ caller_session_id="ses-script",
+ script_file=str(script),
+ arg_values=[],
+ timeout_seconds=timeout_seconds,
+ )
+ elapsed = time.monotonic() - started
+ return exc_info.value, requests, elapsed
+
+
+def _pid_exists(pid: int) -> bool:
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return False
+ return True
+
+
+def _wait_for_pid_exit(pid: int, *, timeout: float = 3.0) -> None:
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if not _pid_exists(pid):
+ return
+ time.sleep(0.02)
+ pytest.fail(f"process {pid} survived the script supervisor")
+
+
+def _blocking_node_runner(tmp_path: Path) -> tuple[Path, Path]:
+ pid_path = tmp_path / "node.pid"
+ runner = tmp_path / "blocking-runner.mjs"
+ runner.write_text(
+ (
+ 'import fs from "node:fs";\n'
+ f"fs.writeFileSync({json.dumps(str(pid_path))}, String(process.pid));\n"
+ "while (true) {}\n"
+ ),
+ encoding="utf-8",
+ )
+ return runner, pid_path
+
+
+def _assert_supervisor_wait_failure_kills_node(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ failure: BaseException,
+) -> None:
+ if NODE is None:
+ pytest.skip("Node.js is unavailable")
+ if os.name != "posix":
+ pytest.skip("process-group supervision is POSIX-only")
+
+ runner, pid_path = _blocking_node_runner(tmp_path)
+ real_popen = subprocess.Popen
+ captured: dict[str, subprocess.Popen] = {}
+
+ class FailingWaitProcess:
+ def __init__(self, *args, **kwargs) -> None:
+ self._process = real_popen(*args, **kwargs)
+ self.pid = self._process.pid
+ self._failed = False
+ captured["process"] = self._process
+
+ def wait(self, timeout=None):
+ if not self._failed:
+ self._failed = True
+ deadline = time.monotonic() + 3
+ while not pid_path.exists() and time.monotonic() < deadline:
+ time.sleep(0.01)
+ assert pid_path.exists()
+ raise failure
+ return self._process.wait(timeout=timeout)
+
+ def poll(self):
+ return self._process.poll()
+
+ def kill(self) -> None:
+ self._process.kill()
+
+ monkeypatch.setattr(
+ computer_script_runner.subprocess,
+ "Popen",
+ FailingWaitProcess,
+ )
+ request = {"result_path": str(tmp_path / "result.json")}
+
+ with pytest.raises(type(failure)):
+ computer_script_runner._run_node_process(
+ Path(NODE),
+ runner,
+ request,
+ 30,
+ )
+
+ node_pid = int(pid_path.read_text(encoding="utf-8"))
+ assert captured["process"].poll() is not None
+ _wait_for_pid_exit(node_pid)
+
+
+def test_keyboard_interrupt_during_wait_kills_the_node_process(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ _assert_supervisor_wait_failure_kills_node(
+ monkeypatch,
+ tmp_path,
+ KeyboardInterrupt(),
+ )
+
+
+def test_supervisor_exception_during_wait_kills_the_node_process(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ _assert_supervisor_wait_failure_kills_node(
+ monkeypatch,
+ tmp_path,
+ RuntimeError("supervisor failed"),
+ )
+
+
+def test_sigterm_to_the_supervisor_kills_the_node_process(tmp_path: Path) -> None:
+ if NODE is None:
+ pytest.skip("Node.js is unavailable")
+ if os.name != "posix":
+ pytest.skip("SIGTERM supervision is POSIX-only")
+
+ runner, pid_path = _blocking_node_runner(tmp_path)
+ supervisor = tmp_path / "supervisor.py"
+ supervisor.write_text(
+ "\n".join(
+ [
+ "from pathlib import Path",
+ "from vibe.computer_script_runner import _run_node_process",
+ (
+ f"_run_node_process(Path({str(NODE)!r}), "
+ f"Path({str(runner)!r}), "
+ f"{{'result_path': {str(tmp_path / 'result.json')!r}}}, 30)"
+ ),
+ ]
+ ),
+ encoding="utf-8",
+ )
+ environment = os.environ.copy()
+ environment["PYTHONPATH"] = os.pathsep.join(
+ filter(None, [str(Path.cwd()), environment.get("PYTHONPATH", "")])
+ )
+ process = subprocess.Popen(
+ [sys.executable, str(supervisor)],
+ cwd=Path.cwd(),
+ env=environment,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ node_pid: int | None = None
+ try:
+ deadline = time.monotonic() + 5
+ while not pid_path.exists() and time.monotonic() < deadline:
+ assert process.poll() is None
+ time.sleep(0.02)
+ assert pid_path.exists()
+ node_pid = int(pid_path.read_text(encoding="utf-8"))
+
+ os.kill(process.pid, signal.SIGTERM)
+ assert process.wait(timeout=5) != 0
+ _wait_for_pid_exit(node_pid)
+ finally:
+ if process.poll() is None:
+ process.kill()
+ process.wait(timeout=3)
+ if node_pid is not None and _pid_exists(node_pid):
+ os.kill(node_pid, signal.SIGKILL)
+
+
+def test_sdk_injects_module_resolution_and_reobserves_one_stale_token(
+ tmp_path: Path,
+) -> None:
+ payload, server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async ({ subject }) => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.waitFor(
+ { role: "AXWindow", labelContains: "New Message" },
+ { timeout: 500 },
+ );
+ const observation = await window.observe();
+ const send = window.element({
+ role: "button",
+ label: "Send",
+ within: { role: "group", label: "Composer" },
+ index: 0,
+ });
+ const effect = await send.click();
+ await window.expect({ role: "textbox", label: "Subject" }).toHaveValue(
+ subject,
+ { timeout: 500 },
+ );
+ return {
+ effect: effect.status,
+ title: window.title,
+ tokenHidden: !Object.hasOwn(observation.elements[0], "element_token"),
+ };
+};
+""",
+ initial_click_error="stale_element_token",
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {
+ "effect": "confirmed",
+ "title": "New Message",
+ "tokenHidden": True,
+ },
+ }
+ tools = [
+ request["payload"].get("tool")
+ for request in requests
+ if request["path"].endswith("/request")
+ ]
+ assert tools.count("click") == 2
+ assert tools.count("get_window_state") >= 3
+ click_tokens = [
+ request["payload"]["arguments"]["element_token"]
+ for request in requests
+ if request["payload"].get("tool") == "click"
+ ]
+ assert click_tokens == ["send-2", "send-3"]
+ assert requests[-1]["path"] == "/internal/computer-script/close"
+ assert all(
+ request["authorization"] == "Bearer fixture-token" for request in requests
+ )
+ assert len(server.connection_ids) == 1 # type: ignore[attr-defined]
+
+
+def test_sdk_failure_reports_step_and_last_screenshot(tmp_path: Path) -> None:
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ await window.element({ role: "button", label: "Missing" }).click();
+ return {};
+};
+""",
+ )
+
+ assert payload["ok"] is False
+ assert payload["error"]["name"] == "NotFound"
+ assert payload["error"]["code"] == "not_found"
+ assert payload["error"]["step"] == "element.click"
+ screenshot = Path(payload["error"]["screenshot_path"])
+ assert screenshot.read_bytes() == b"png-fixture"
+
+
+def test_plain_script_error_keeps_last_completed_step_separate(
+ tmp_path: Path,
+) -> None:
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ await computer.app({ bundleId: "com.apple.mail" });
+ throw new Error("plain script failure");
+};
+""",
+ )
+
+ assert payload["ok"] is False
+ assert payload["error"]["code"] == "computer_script_failed"
+ assert payload["error"]["step"] == "script"
+ assert payload["error"]["last_completed_step"] == "computer.app"
+ assert payload["error"]["ambiguous"] is False
+
+
+def test_failure_envelope_survives_runner_python_and_cli(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ capsys,
+) -> None:
+ screenshot_path = tmp_path / "failure.png"
+ screenshot_path.write_bytes(b"fixture")
+ expected = {
+ "name": "FixtureFailure",
+ "code": "invalid_selector",
+ "message": "Application selector contains unknown field 'text'.",
+ "params": {"field": "text"},
+ "step": "computer.app",
+ "last_completed_step": "computer.app",
+ "screenshot_path": str(screenshot_path),
+ "ambiguous": True,
+ }
+ assert tuple(expected) == computer_script_runner._failure_envelope_fields()
+
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ f"""
+import {{ computer }} from "@avibe/computer";
+
+export default async () => {{
+ await computer.app({{ bundleId: "com.apple.mail" }});
+ const error = new Error({json.dumps(expected["message"])});
+ error.name = {json.dumps(expected["name"])};
+ error.code = {json.dumps(expected["code"])};
+ error.params = {json.dumps(expected["params"])};
+ error.step = {json.dumps(expected["step"])};
+ error.screenshotPath = {json.dumps(expected["screenshot_path"])};
+ error.ambiguous = true;
+ throw error;
+}};
+""",
+ )
+ assert payload == {"ok": False, "error": expected}
+
+ script = tmp_path / "python-hop.mjs"
+ script.write_text("export default async () => null;", encoding="utf-8")
+ endpoint = object()
+ monkeypatch.setattr(
+ computer_script_runner,
+ "private_desktop_node_bin",
+ lambda: Path("/private/node"),
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "resolve_computer_script_endpoint",
+ lambda: endpoint,
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "computer_script_open_sync",
+ lambda **_kwargs: {
+ "body": {"ok": True, "connection_token": "script-token"}
+ },
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "computer_script_endpoint_payload",
+ lambda value: {"endpoint": value is endpoint},
+ )
+ def return_failure(_node, _runner, request, _timeout_seconds):
+ Path(request["result_path"]).write_text(
+ json.dumps(payload),
+ encoding="utf-8",
+ )
+ Path(request["runtime_state_path"]).write_text(
+ json.dumps(
+ {
+ "step": expected["step"],
+ "last_completed_step": expected["last_completed_step"],
+ "screenshot_path": expected["screenshot_path"],
+ "diagnostics": "",
+ }
+ ),
+ encoding="utf-8",
+ )
+ return subprocess.CompletedProcess(
+ args=["node", "runner.mjs"],
+ returncode=0,
+ stdout="",
+ stderr="",
+ )
+
+ monkeypatch.setattr(
+ computer_script_runner,
+ "_run_node_process",
+ return_failure,
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "computer_script_close_sync",
+ lambda **_kwargs: None,
+ )
+
+ with pytest.raises(ComputerScriptRunnerError) as runner_exc_info:
+ computer_script_runner.run_computer_script(
+ caller_session_id="ses-script",
+ script_file=str(script),
+ arg_values=[],
+ )
+ runner_error = runner_exc_info.value
+ assert runner_error.details == expected
+ assert runner_error.structured_error == expected
+
+ monkeypatch.setenv("AVIBE_SESSION_ID", "ses-script")
+ monkeypatch.setattr(cli, "_configured_cli_language", lambda: "zh")
+
+ def fail_script(**_kwargs):
+ raise runner_error
+
+ monkeypatch.setattr(
+ computer_script_runner,
+ "run_computer_script",
+ fail_script,
+ )
+ args = cli.build_parser().parse_args(["computer", "run", str(script)])
+ with pytest.raises(cli.TaskCliError) as cli_exc_info:
+ cli.cmd_computer(args)
+ cli_error = cli_exc_info.value
+ assert cli_error.details == expected
+ assert expected["message"] not in str(cli_error)
+
+ cli._print_task_error(cli_error)
+ rendered = json.loads(capsys.readouterr().err)
+ assert rendered["details"] == expected
+
+
+def test_sdk_controller_error_ambiguity_table(tmp_path: Path) -> None:
+ if NODE is None:
+ pytest.skip("Node.js is unavailable")
+
+ not_dispatched_codes = [
+ "computer_script_request_too_large",
+ "invalid_computer_script_request",
+ "active_turn_required",
+ "session_archived",
+ "script_connection_invalid",
+ "script_timeout",
+ "script_turn_ended",
+ "invalid_computer_script_tool",
+ "session_argument_forbidden",
+ "computer_use_unavailable",
+ "session_required",
+ "unknown_tool",
+ "invalid_arguments",
+ "invalid_computer_script_descriptor",
+ "computer_script_descriptor_required",
+ "stale_app_handle",
+ "stale_window_handle",
+ "desktop_busy",
+ "platform_unsupported",
+ "foreground_forbidden",
+ "focus_shortcut_forbidden",
+ "foreground_consent_required",
+ "window_target_required",
+ "observe_first",
+ "desktop_lease_lost",
+ "session_unavailable",
+ ]
+ post_send_codes = [
+ "upstream_unavailable",
+ "upstream_error",
+ "upstream_protocol_error",
+ "upstream_tool_error",
+ "computer_use_error",
+ "application_not_found",
+ "element_not_found",
+ "snapshot_not_found",
+ "window_id_not_found",
+ "window_not_found",
+ "stale_element_token",
+ "capture_failed",
+ "tool_invocation_failed",
+ "controller_unavailable",
+ "timeout",
+ "invalid_computer_use_response",
+ ]
+ cases = [
+ *[
+ {
+ "code": code,
+ "requestWasSent": True,
+ "controllerPayload": {"dispatched": False},
+ "expected": False,
+ }
+ for code in not_dispatched_codes
+ ],
+ *[
+ {
+ "code": code,
+ "requestWasSent": True,
+ "controllerPayload": {},
+ "expected": True,
+ }
+ for code in post_send_codes
+ ],
+ {
+ "code": "timeout",
+ "requestWasSent": False,
+ "controllerPayload": {},
+ "expected": False,
+ },
+ {
+ "code": "controller_unavailable",
+ "requestWasSent": False,
+ "controllerPayload": {},
+ "expected": False,
+ },
+ ]
+ script = tmp_path / "ambiguity-table.mjs"
+ script.write_text(
+ (
+ "import { controllerFailureAmbiguous } from "
+ f"{json.dumps((SDK_DIRECTORY / 'runtime.mjs').as_uri())};\n"
+ f"const cases = {json.dumps(cases)};\n"
+ "const actual = cases.map((item) => ({\n"
+ " code: item.code,\n"
+ " expected: item.expected,\n"
+ " actual: controllerFailureAmbiguous(\n"
+ " item.requestWasSent,\n"
+ " item.controllerPayload,\n"
+ " ),\n"
+ "}));\n"
+ "process.stdout.write(JSON.stringify(actual));\n"
+ ),
+ encoding="utf-8",
+ )
+
+ completed = subprocess.run(
+ [NODE, str(script)],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ actual = json.loads(completed.stdout)
+ assert all(row["actual"] == row["expected"] for row in actual), actual
+
+ runtime_source = (SDK_DIRECTORY / "runtime.mjs").read_text(encoding="utf-8")
+ assert runtime_source.count("new ControllerFault(") == 1
+ assert runtime_source.count("controllerPayload.dispatched === false") == 1
+ assert "hasAmbiguousInFlightRequest" not in runtime_source
+
+
+ "controller_code",
+ ["stale_element_token", "upstream_unavailable"],
+)
+def test_sdk_never_retries_a_mutation_without_explicit_not_dispatched(
+ tmp_path: Path,
+ controller_code: str,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ f"""
+import {{ computer }} from "@avibe/computer";
+
+export default async () => {{
+ const mail = await computer.app({{ bundleId: "com.apple.mail" }});
+ const window = await mail.window({{ labelContains: "New Message" }});
+ try {{
+ await window
+ .element({{ role: "button", label: "Send", index: 0 }})
+ .click();
+ }} catch (error) {{
+ return {{ code: error.code, ambiguous: error.ambiguous }};
+ }}
+ throw new Error("Controller failure was not raised");
+}};
+""",
+ click_error=controller_code,
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {"code": controller_code, "ambiguous": True},
+ }
+ assert [
+ request["payload"]["tool"]
+ for request in requests
+ if request["path"].endswith("/request")
+ and request["payload"].get("tool") == "click"
+ ] == ["click"]
+
+
+def test_sdk_has_one_mutating_action_retry_gate() -> None:
+ source = (SDK_DIRECTORY / "index.mjs").read_text(encoding="utf-8")
+
+ assert source.count("function shouldRetryElementAction(error)") == 1
+ assert source.count("shouldRetryElementAction(error)") == 2
+ assert source.count("STALE_ELEMENT_CODES.has(error.code)") == 1
+ assert source.count("return effectResult(await invoke());") == 2
+
+
+def test_sdk_marks_post_forward_upstream_failure_as_ambiguous(
+ tmp_path: Path,
+) -> None:
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ try {
+ await window
+ .element({ role: "button", label: "Send", index: 0 })
+ .click();
+ } catch (error) {
+ return { code: error.code, ambiguous: error.ambiguous };
+ }
+ throw new Error("upstream_unavailable was not raised");
+};
+""",
+ click_error="upstream_unavailable",
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {
+ "code": "upstream_unavailable",
+ "ambiguous": True,
+ },
+ }
+
+
+def test_sdk_marks_post_send_response_decode_failure_as_ambiguous(
+ tmp_path: Path,
+) -> None:
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ await computer.app({ index: 0 });
+};
+""",
+ apps=[
+ {
+ "pid": 41,
+ "name": "",
+ "bundle_id": "",
+ "running": True,
+ "active": False,
+ }
+ ],
+ )
+
+ assert payload["ok"] is False
+ assert payload["error"]["code"] == "invalid_computer_use_response"
+ assert payload["error"]["ambiguous"] is True
+
+
+def test_sdk_maps_stable_controller_codes_to_typed_errors(tmp_path: Path) -> None:
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ """
+import { computer, DesktopBusy } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ try {
+ await window
+ .element({ role: "button", label: "Send", index: 0 })
+ .click();
+ } catch (error) {
+ return { typed: error instanceof DesktopBusy, code: error.code };
+ }
+ throw new Error("desktop_busy was not raised");
+};
+""",
+ click_error="desktop_busy",
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {"typed": True, "code": "desktop_busy"},
+ }
+
+
+ ("controller_code", "error_class"),
+ [
+ ("element_not_found", "NotFound"),
+ ("foreground_forbidden", "ForegroundConsentRequired"),
+ ("computer_use_unavailable", "Unavailable"),
+ ],
+)
+def test_sdk_maps_controller_error_families(
+ tmp_path: Path,
+ controller_code: str,
+ error_class: str,
+) -> None:
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ f"""
+import {{
+ computer,
+ NotFound,
+ ForegroundConsentRequired,
+ Unavailable,
+}} from "@avibe/computer";
+
+export default async () => {{
+ const types = {{ NotFound, ForegroundConsentRequired, Unavailable }};
+ const mail = await computer.app({{ bundleId: "com.apple.mail" }});
+ const window = await mail.window({{ labelContains: "New Message" }});
+ try {{
+ await window
+ .element({{ role: "button", label: "Send", index: 0 }})
+ .click();
+ }} catch (error) {{
+ return {{ typed: error instanceof types.{error_class}, code: error.code }};
+ }}
+ throw new Error("typed Controller error was not raised");
+}};
+""",
+ click_error=controller_code,
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {"typed": True, "code": controller_code},
+ }
+
+
+def test_sdk_rejects_an_ambiguous_within_boundary(tmp_path: Path) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ await window
+ .element({
+ role: "button",
+ label: "Send",
+ within: { role: "group", label: "Composer" },
+ })
+ .click();
+ return {};
+};
+""",
+ duplicate_within=True,
+ )
+
+ assert payload["ok"] is False
+ assert payload["error"]["name"] == "Ambiguous"
+ assert payload["error"]["step"] == "element.click"
+ assert all(
+ request["payload"].get("tool") != "click"
+ for request in requests
+ if request["path"].endswith("/request")
+ )
+
+
+def test_sdk_reobserves_before_each_action_and_rejects_new_ambiguity(
+ tmp_path: Path,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ await window.element({ role: "button", label: "Add" }).click();
+ await window.element({ role: "button", label: "Delete" }).click();
+ return {};
+};
+""",
+ dynamic_elements_after_action=True,
+ )
+
+ assert payload["ok"] is False
+ assert payload["error"]["name"] == "Ambiguous"
+ assert payload["error"]["step"] == "element.click"
+ click_requests = [
+ request
+ for request in requests
+ if request["payload"].get("tool") == "click"
+ ]
+ assert len(click_requests) == 2
+ assert {
+ request["payload"]["arguments"]["element_token"]
+ for request in click_requests
+ } == {"add-1", "add-2"}
+
+
+def test_sdk_action_ignores_a_cached_screenshot_only_observation(
+ tmp_path: Path,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ await window.observe({ include_accessibility_tree: false });
+ const effect = await window
+ .element({ role: "button", label: "Send", index: 0 })
+ .click();
+ return { effect: effect.status };
+};
+""",
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {"effect": "confirmed"},
+ }
+ observations = [
+ request
+ for request in requests
+ if request["payload"].get("tool") == "get_window_state"
+ ]
+ assert observations[0]["payload"]["arguments"][
+ "include_accessibility_tree"
+ ] is False
+ assert len(observations) == 3
+
+
+ ("source", "expected_step"),
+ [
+ (
+ """
+import { computer } from "@avibe/computer";
+export default async () => {
+ await computer.app({ bundleId: "com.apple.mail", text: "Mail" });
+};
+""",
+ "computer.app",
+ ),
+ (
+ """
+import { computer } from "@avibe/computer";
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ await mail.window({ labelContains: "New Message", name: "New Message" });
+};
+""",
+ "app.window",
+ ),
+ (
+ """
+import { computer } from "@avibe/computer";
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ await window.element({ role: "button", name: "Send" }).click();
+};
+""",
+ "element.click",
+ ),
+ ],
+)
+def test_sdk_rejects_unknown_selector_fields_without_acting(
+ tmp_path: Path,
+ source: str,
+ expected_step: str,
+) -> None:
+ payload, _server, requests = _run_node_script(tmp_path, source)
+
+ assert payload["ok"] is False
+ assert payload["error"]["name"] == "ControllerError"
+ assert payload["error"]["code"] == "invalid_selector"
+ assert payload["error"]["step"] == expected_step
+ assert all(
+ request["payload"].get("tool") != "click"
+ for request in requests
+ if request["path"].endswith("/request")
+ )
+
+
+def test_sdk_wait_does_not_turn_an_invalid_selector_into_a_timeout(
+ tmp_path: Path,
+) -> None:
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ await computer.waitFor(
+ { bundleId: "com.apple.mail", text: "Mail" },
+ { timeout: 1000 },
+ );
+};
+""",
+ )
+
+ assert payload["ok"] is False
+ assert payload["error"]["code"] == "invalid_selector"
+ assert payload["error"]["step"] == "computer.waitFor"
+
+
+def test_sdk_expect_does_not_turn_an_invalid_index_into_a_timeout(
+ tmp_path: Path,
+) -> None:
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ await window
+ .expect({ role: "button", index: -1 })
+ .toExist({ timeout: 1000 });
+};
+""",
+ )
+
+ assert payload["ok"] is False
+ assert payload["error"]["code"] == "invalid_selector"
+ assert payload["error"]["step"] == "expect.toExist"
+
+
+def test_sdk_zero_timeout_checks_each_current_state_once(tmp_path: Path) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.waitFor(
+ { bundleId: "com.apple.mail" },
+ { timeout: 0 },
+ );
+ const window = await mail.waitFor(
+ { labelContains: "New Message" },
+ { timeout: 0 },
+ );
+ await window
+ .expect({ role: "textbox", label: "Subject" })
+ .toHaveValue("Quarterly report", { timeout: 0 });
+ return { pid: mail.pid, windowId: window.windowId };
+};
+""",
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {"pid": 41, "windowId": 9},
+ }
+ tools = [
+ request["payload"].get("tool")
+ for request in requests
+ if request["path"].endswith("/request")
+ ]
+ assert tools.count("list_apps") == 1
+ assert tools.count("list_windows") == 1
+ assert tools.count("get_window_state") == 1
+
+
+def test_sdk_selected_app_descriptor_keeps_the_selected_positive_pid(
+ tmp_path: Path,
+) -> None:
+ apps = [
+ {
+ "pid": pid,
+ "name": "Mail",
+ "bundle_id": "com.apple.mail",
+ "running": True,
+ "active": pid == 42,
+ }
+ for pid in (41, 42)
+ ]
+ windows = [
+ {
+ "pid": 42,
+ "window_id": 10,
+ "app_name": "Mail",
+ "title": "Selected Process",
+ "bounds": {"x": 0, "y": 0, "width": 800, "height": 600},
+ "is_on_screen": True,
+ "z_index": 1,
+ }
+ ]
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({
+ bundleId: "com.apple.mail",
+ index: 1,
+ });
+ const window = await mail.window({ label: "Selected Process" });
+ return { appPid: mail.pid, windowPid: window.pid };
+};
+""",
+ apps=apps,
+ windows=windows,
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {"appPid": 42, "windowPid": 42},
+ }
+ list_window_requests = [
+ request
+ for request in requests
+ if request["payload"].get("tool") == "list_windows"
+ ]
+ assert [request["payload"]["arguments"] for request in list_window_requests] == [
+ {"pid": 42}
+ ]
+ assert list_window_requests[0]["payload"]["descriptor"] == {
+ "kind": "app",
+ "pid": 42,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ }
+
+
+def test_sdk_app_handle_never_refreshes_or_rebinds_before_an_action(
+ tmp_path: Path,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ await mail.forceQuit();
+};
+""",
+ )
+
+ assert payload == {"ok": True, "result": None}
+ tool_requests = [
+ request
+ for request in requests
+ if request["path"].endswith("/request")
+ ]
+ assert [request["payload"]["tool"] for request in tool_requests] == [
+ "list_apps",
+ "kill_app",
+ ]
+ assert tool_requests[-1]["payload"]["descriptor"] == {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ }
+
+
+def test_sdk_window_handle_never_refreshes_or_rebinds_its_application(
+ tmp_path: Path,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ label: "New Message" });
+ await window.observe({ include_screenshot: false });
+};
+""",
+ )
+
+ assert payload == {"ok": True, "result": None}
+ tool_requests = [
+ request
+ for request in requests
+ if request["path"].endswith("/request")
+ ]
+ assert [request["payload"]["tool"] for request in tool_requests] == [
+ "list_apps",
+ "list_windows",
+ "get_window_state",
+ ]
+ assert tool_requests[-1]["payload"]["descriptor"] == {
+ "kind": "window",
+ "pid": 41,
+ "window_id": 9,
+ "app": {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ },
+ }
+
+
+def test_sdk_launch_with_pid_zero_resolves_one_fresh_app_descriptor(
+ tmp_path: Path,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const target = await computer.app({ bundleId: "com.apple.mail" });
+ const launched = await target.launch();
+ return {
+ oldPid: target.pid,
+ newPid: launched.app.pid,
+ effect: launched.status,
+ };
+};
+""",
+ app_responses=[
+ [
+ {
+ "pid": 0,
+ "name": "Mail",
+ "bundle_id": "com.apple.mail",
+ "running": False,
+ "active": False,
+ }
+ ],
+ [
+ {
+ "pid": 57,
+ "name": "Mail",
+ "bundle_id": "com.apple.mail",
+ "running": True,
+ "active": True,
+ }
+ ],
+ ],
+ launch_pid=0,
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {
+ "oldPid": None,
+ "newPid": 57,
+ "effect": "confirmed",
+ },
+ }
+ tool_requests = [
+ request
+ for request in requests
+ if request["path"].endswith("/request")
+ ]
+ assert [request["payload"]["tool"] for request in tool_requests] == [
+ "list_apps",
+ "launch_app",
+ "list_apps",
+ ]
+ assert tool_requests[1]["payload"]["descriptor"] == {
+ "kind": "app",
+ "pid": None,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ }
+
+
+def test_sdk_element_double_click_uses_the_dedicated_tool(tmp_path: Path) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ label: "New Message" });
+ const effect = await window
+ .element({ role: "button", label: "Send", index: 0 })
+ .doubleClick();
+ return effect.status;
+};
+""",
+ )
+
+ assert payload == {"ok": True, "result": "confirmed"}
+ action_requests = [
+ request
+ for request in requests
+ if request["payload"].get("tool") in {"click", "double_click"}
+ ]
+ assert [request["payload"]["tool"] for request in action_requests] == [
+ "double_click",
+ ]
+ assert "count" not in action_requests[0]["payload"]["arguments"]
+
+
+def test_sdk_window_index_uses_descending_z_order(tmp_path: Path) -> None:
+ windows = [
+ {
+ "pid": 41,
+ "window_id": 9,
+ "app_name": "Mail",
+ "title": "Back",
+ "bounds": {"x": 0, "y": 0, "width": 800, "height": 600},
+ "is_on_screen": True,
+ "z_index": 1,
+ },
+ {
+ "pid": 41,
+ "window_id": 10,
+ "app_name": "Mail",
+ "title": "Front",
+ "bounds": {"x": 10, "y": 10, "width": 800, "height": 600},
+ "is_on_screen": True,
+ "z_index": 5,
+ },
+ ]
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ role: "window", index: 0 });
+ return { id: window.windowId, title: window.title };
+};
+""",
+ windows=windows,
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {"id": 10, "title": "Front"},
+ }
+
+
+def test_sdk_window_index_refuses_unobservable_order(tmp_path: Path) -> None:
+ windows = [
+ {
+ "pid": 41,
+ "window_id": window_id,
+ "app_name": "Mail",
+ "title": title,
+ "bounds": {"x": 0, "y": 0, "width": 800, "height": 600},
+ "is_on_screen": True,
+ "z_index": z_index,
+ }
+ for window_id, title, z_index in [(9, "One", None), (10, "Two", 5)]
+ ]
+ payload, _server, _requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ await mail.window({ role: "window", index: 0 });
+};
+""",
+ windows=windows,
+ )
+
+ assert payload["ok"] is False
+ assert payload["error"]["code"] == "window_order_unavailable"
+ assert payload["error"]["step"] == "app.window"
+
+
+def test_runner_returns_null_and_exits_with_user_timers_still_open(
+ tmp_path: Path,
+) -> None:
+ payload, server, _requests = _run_node_script(
+ tmp_path,
+ """
+export default async () => {
+ setInterval(() => {}, 1000);
+};
+""",
+ )
+
+ assert payload == {"ok": True, "result": None}
+ assert server.subprocess_elapsed < 1.0 # type: ignore[attr-defined]
+
+
+def test_runtime_state_retains_only_a_bounded_console_diagnostic_tail(
+ tmp_path: Path,
+) -> None:
+ payload, server, _requests = _run_node_script(
+ tmp_path,
+ """
+export default async () => {
+ console.log("discarded-marker");
+ console.log("x".repeat(128 * 1024));
+ console.log("retained-tail");
+};
+""",
+ )
+
+ completed = server.completed # type: ignore[attr-defined]
+ state_read = computer_script_runner._read_capped_json(
+ tmp_path / "artifacts" / "runtime-state.json",
+ computer_script_runner._SCRIPT_RUNTIME_STATE_MAX_BYTES,
+ )
+
+ assert payload == {"ok": True, "result": None}
+ assert completed.returncode == 0
+ assert completed.stdout == ""
+ assert completed.stderr == ""
+ assert state_read.error is None
+ assert isinstance(state_read.value, dict)
+ diagnostics = state_read.value["diagnostics"]
+ assert len(diagnostics.encode("utf-8")) <= (
+ computer_script_runner._SCRIPT_DIAGNOSTIC_TAIL_BYTES
+ )
+ assert "discarded-marker" not in diagnostics
+ assert diagnostics.endswith("retained-tail\n")
+ assert {path.name for path in (tmp_path / "artifacts").iterdir()} == {
+ "result.json",
+ "runtime-state.json",
+ }
+ assert (
+ tmp_path / "artifacts" / "runtime-state.json"
+ ).stat().st_size <= computer_script_runner._SCRIPT_RUNTIME_STATE_MAX_BYTES
+
+
+def test_continuous_console_logs_leave_no_capture_files_after_deadline(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ temp_root = tmp_path / "runner-temp"
+ temp_root.mkdir()
+ real_mkdtemp = tempfile.mkdtemp
+ created_directories: list[Path] = []
+
+ def tracked_mkdtemp(*args, **kwargs):
+ kwargs["dir"] = temp_root
+ created = Path(real_mkdtemp(*args, **kwargs))
+ created_directories.append(created)
+ return str(created)
+
+ monkeypatch.setattr(computer_script_runner.tempfile, "mkdtemp", tracked_mkdtemp)
+
+ error, _requests, elapsed = _run_python_deadline_script(
+ monkeypatch,
+ tmp_path,
+ """
+export default function () {
+ while (true) {
+ console.log("x".repeat(4096));
+ }
+};
+""",
+ )
+
+ diagnostics = error.details["params"]["diagnostics"]
+ assert len(diagnostics.encode("utf-8")) <= (
+ computer_script_runner._SCRIPT_DIAGNOSTIC_TAIL_BYTES
+ )
+ assert elapsed < 1.0
+ assert created_directories
+ assert all(not path.exists() for path in created_directories)
+ assert list(temp_root.iterdir()) == []
+
+
+def test_python_runner_rejects_an_oversized_result_with_canonical_error(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ error, requests, elapsed = _run_python_deadline_script(
+ monkeypatch,
+ tmp_path,
+ f"""
+export default async () => "x".repeat(
+ {computer_script_runner._SCRIPT_RESULT_MAX_BYTES}
+);
+""",
+ timeout_seconds=5,
+ )
+
+ assert error.details == {
+ "name": "ControllerError",
+ "code": "computer_script_protocol_error",
+ "message": "The script result exceeded the protocol size limit.",
+ "params": {
+ "limit_bytes": computer_script_runner._SCRIPT_RESULT_MAX_BYTES,
+ },
+ "step": "script",
+ "last_completed_step": None,
+ "screenshot_path": None,
+ "ambiguous": True,
+ }
+ assert [
+ request
+ for request in requests
+ if request["path"].endswith("/request")
+ ] == []
+ assert elapsed < 2
+
+
+def test_python_deadline_is_not_held_by_a_detached_descendant_stdout(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ temp_root = tmp_path / "runner-temp"
+ temp_root.mkdir()
+ real_mkdtemp = tempfile.mkdtemp
+ created_directories: list[Path] = []
+
+ def tracked_mkdtemp(*args, **kwargs):
+ kwargs["dir"] = temp_root
+ created = Path(real_mkdtemp(*args, **kwargs))
+ created_directories.append(created)
+ return str(created)
+
+ monkeypatch.setattr(computer_script_runner.tempfile, "mkdtemp", tracked_mkdtemp)
+
+ error, requests, elapsed = _run_python_deadline_script(
+ monkeypatch,
+ tmp_path,
+ """
+import { spawn } from "node:child_process";
+
+export default async () => {
+ const child = spawn(
+ process.execPath,
+ [
+ "-e",
+ "const end=Date.now()+750; const chunk='x'.repeat(8192); " +
+ "while(Date.now()<end){process.stdout.write(chunk)}",
+ ],
+ { detached: true, stdio: ["ignore", "inherit", "inherit"] },
+ );
+ child.unref();
+ await new Promise(() => {});
+};
+""",
+ )
+
+ assert error.details == {
+ "name": "Timeout",
+ "code": "script_timeout",
+ "message": "The script deadline expired.",
+ "params": {},
+ "step": "script",
+ "last_completed_step": None,
+ "screenshot_path": None,
+ "ambiguous": True,
+ }
+ assert requests == []
+ assert elapsed < 1.0
+ assert created_directories
+ assert all(not path.exists() for path in created_directories)
+ assert list(temp_root.iterdir()) == []
+
+
+ ("source", "response_delay_by_tool", "expected_step", "expected_tool"),
+ [
+ (
+ """
+export default function () {
+ while (true) {}
+};
+""",
+ None,
+ "script",
+ None,
+ ),
+ (
+ """
+export default async () => {
+ await new Promise(() => {});
+};
+""",
+ None,
+ "script",
+ None,
+ ),
+ (
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ await computer.clipboard.write("deadline fixture");
+};
+""",
+ {"clipboard_write": 1.5},
+ "computer.clipboard.write",
+ "clipboard_write",
+ ),
+ ],
+)
+def test_python_parent_owns_the_canonical_total_deadline(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ source: str,
+ response_delay_by_tool: dict[str, float] | None,
+ expected_step: str,
+ expected_tool: str | None,
+) -> None:
+ error, requests, elapsed = _run_python_deadline_script(
+ monkeypatch,
+ tmp_path,
+ source,
+ response_delay_by_tool=response_delay_by_tool,
+ )
+
+ assert error.details == {
+ "name": "Timeout",
+ "code": "script_timeout",
+ "message": "The script deadline expired.",
+ "params": {},
+ "step": expected_step,
+ "last_completed_step": None,
+ "screenshot_path": None,
+ "ambiguous": True,
+ }
+ tools = [
+ request["payload"].get("tool")
+ for request in requests
+ if request["path"].endswith("/request")
+ ]
+ if expected_tool is None:
+ assert tools == []
+ else:
+ assert expected_tool in tools
+ assert elapsed < 1.0
+
+
+def test_python_total_deadline_preserves_the_last_screenshot(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ error, requests, _elapsed = _run_python_deadline_script(
+ monkeypatch,
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ await window.element({ role: "button", label: "Send", index: 0 }).click();
+};
+""",
+ response_delay_by_tool={"click": 2.0},
+ timeout_seconds=0.75,
+ )
+
+ screenshot_path = error.details["screenshot_path"]
+ assert isinstance(screenshot_path, str)
+ screenshot = Path(screenshot_path)
+ try:
+ assert screenshot.read_bytes() == b"png-fixture"
+ assert error.details["step"] == "element.click"
+ assert error.details["last_completed_step"] == "app.window"
+ assert error.details["ambiguous"] is True
+ assert any(
+ request["payload"].get("tool") == "click"
+ for request in requests
+ if request["path"].endswith("/request")
+ )
+ finally:
+ shutil.rmtree(screenshot.parent, ignore_errors=True)
+
+
+ ("termination", "expected_reason"),
+ [
+ ('process.kill(process.pid, "SIGKILL");', "missing"),
+ (
+ 'fs.writeFileSync(process.env.AVIBE_COMPUTER_RESULT_PATH, "{bad"); '
+ "process.exit(7);",
+ "invalid_json",
+ ),
+ (
+ "fs.rmSync(process.env.AVIBE_COMPUTER_RESULT_PATH, { force: true }); "
+ "process.exit(8);",
+ "missing",
+ ),
+ ],
+)
+def test_finalize_recovers_step_and_screenshot_from_runtime_state(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ termination: str,
+ expected_reason: str,
+) -> None:
+ error, _requests, _elapsed = _run_python_deadline_script(
+ monkeypatch,
+ tmp_path,
+ f"""
+import fs from "node:fs";
+import {{ computer }} from "@avibe/computer";
+
+export default async () => {{
+ const mail = await computer.app({{ bundleId: "com.apple.mail" }});
+ const window = await mail.window({{ labelContains: "New Message" }});
+ await window.observe();
+ {termination}
+}};
+""",
+ timeout_seconds=5,
+ )
+
+ assert error.details["code"] == "computer_script_protocol_error"
+ assert error.details["step"] == "script"
+ assert error.details["last_completed_step"] == "window.observe"
+ assert error.details["params"]["reason"] == expected_reason
+ screenshot_path = error.details["screenshot_path"]
+ assert isinstance(screenshot_path, str)
+ screenshot = Path(screenshot_path)
+ try:
+ assert screenshot.read_bytes() == b"png-fixture"
+ assert list(screenshot.parent.iterdir()) == [screenshot]
+ finally:
+ shutil.rmtree(screenshot.parent, ignore_errors=True)
+
+
+def test_python_failure_retains_only_the_referenced_screenshot(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ error, _requests, _elapsed = _run_python_deadline_script(
+ monkeypatch,
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ await window.observe();
+ await window.observe();
+ await window.element({ role: "button", label: "Missing" }).click();
+};
+""",
+ timeout_seconds=5,
+ )
+
+ screenshot_path = error.details["screenshot_path"]
+ assert isinstance(screenshot_path, str)
+ screenshot = Path(screenshot_path)
+ try:
+ assert screenshot.read_bytes() == b"png-fixture"
+ assert list(screenshot.parent.iterdir()) == [screenshot]
+ finally:
+ shutil.rmtree(screenshot.parent, ignore_errors=True)
+
+
+def test_failure_artifact_retention_is_bounded(tmp_path: Path) -> None:
+ retained: list[Path] = []
+ for index in range(computer_script_runner._MAX_RETAINED_ARTIFACT_RUNS + 3):
+ directory = (
+ tmp_path
+ / f"{computer_script_runner._ARTIFACT_DIRECTORY_PREFIX}{index:02d}"
+ )
+ directory.mkdir()
+ (directory / "failure.png").write_bytes(b"png")
+ os.utime(directory, (index + 1, index + 1))
+ retained.append(directory)
+
+ computer_script_runner._prune_retained_artifact_runs(tmp_path)
+
+ survivors = [directory for directory in retained if directory.exists()]
+ assert len(survivors) == computer_script_runner._MAX_RETAINED_ARTIFACT_RUNS
+ assert survivors == retained[-computer_script_runner._MAX_RETAINED_ARTIFACT_RUNS :]
+
+
+ ("source", "delayed_tool", "expected_step"),
+ [
+ (
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ await computer.waitFor(
+ { bundleId: "com.apple.mail" },
+ { timeout: 100 },
+ );
+ return {};
+};
+""",
+ "list_apps",
+ "computer.waitFor",
+ ),
+ (
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ labelContains: "New Message" });
+ await window
+ .expect({ role: "textfield", label: "Subject" })
+ .toHaveValue("Quarterly report", { timeout: 100 });
+ return {};
+};
+""",
+ "get_window_state",
+ "expect.toHaveValue",
+ ),
+ ],
+)
+def test_sdk_local_timeout_aborts_an_in_flight_controller_request(
+ tmp_path: Path,
+ source: str,
+ delayed_tool: str,
+ expected_step: str,
+) -> None:
+ payload, server, requests = _run_node_script(
+ tmp_path,
+ source,
+ response_delay_by_tool={delayed_tool: 1.5},
+ )
+
+ assert payload["ok"] is False
+ assert payload["error"]["name"] == "Timeout"
+ assert payload["error"]["code"] == "timeout"
+ assert payload["error"]["step"] == expected_step
+ assert payload["error"]["ambiguous"] is True
+ assert server.subprocess_elapsed < 1.0 # type: ignore[attr-defined]
+ assert all(
+ request["payload"].get("tool") != "click"
+ for request in requests
+ if request["path"].endswith("/request")
+ )
+
+
+def test_script_arguments_keep_strings_and_load_utf8_files(tmp_path: Path) -> None:
+ source = tmp_path / "body.txt"
+ source.write_text("hello\n世界\n", encoding="utf-8")
+ assert parse_script_arguments(
+ [" count =01", f"body=@{source}", "json={\"x\":1}"]
+ ) == {
+ "count": "01",
+ "body": "hello\n世界\n",
+ "json": '{"x":1}',
+ }
+
+
+def test_script_arguments_reject_duplicate_names() -> None:
+ with pytest.raises(ComputerScriptRunnerError) as exc_info:
+ parse_script_arguments(["name=first", "name=second"])
+ assert exc_info.value.code == "duplicate_computer_script_argument"
+
+
+def test_python_runner_uses_one_deadline_for_open_node_and_close(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ script = tmp_path / "script.mjs"
+ script.write_text("export default async () => ({ ok: true });", encoding="utf-8")
+ endpoint = object()
+ captured: dict[str, object] = {}
+ ticks = iter([100.0, 102.0, 105.0, 106.0])
+
+ monkeypatch.setattr(
+ computer_script_runner,
+ "private_desktop_node_bin",
+ lambda: Path("/private/node"),
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "resolve_computer_script_endpoint",
+ lambda: endpoint,
+ )
+
+ def open_connection(**kwargs):
+ captured["open"] = kwargs
+ return {
+ "body": {
+ "ok": True,
+ "connection_token": "token",
+ }
+ }
+
+ monkeypatch.setattr(
+ computer_script_runner,
+ "computer_script_open_sync",
+ open_connection,
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "computer_script_endpoint_payload",
+ lambda value: {"endpoint": value is endpoint},
+ )
+
+ def run_node(node, runner, request, timeout_seconds):
+ captured["node"] = node
+ captured["runner"] = runner
+ captured["request"] = request
+ captured["timeout_seconds"] = timeout_seconds
+ Path(request["result_path"]).write_text(
+ '{"ok":true,"result":{"done":true}}',
+ encoding="utf-8",
+ )
+ return subprocess.CompletedProcess(
+ args=[str(node), str(runner)],
+ returncode=0,
+ stdout="",
+ stderr="",
+ )
+
+ monkeypatch.setattr(computer_script_runner, "_run_node_process", run_node)
+ monkeypatch.setattr(
+ computer_script_runner,
+ "computer_script_close_sync",
+ lambda **kwargs: captured.update(close=kwargs),
+ )
+ monkeypatch.setattr(computer_script_runner.time, "monotonic", lambda: next(ticks))
+
+ assert computer_script_runner.run_computer_script(
+ caller_session_id="ses-script",
+ script_file=str(script),
+ arg_values=[],
+ timeout_seconds=20,
+ ) == {"done": True}
+
+ assert captured["open"] == {
+ "endpoint": endpoint,
+ "caller_session_id": "ses-script",
+ "timeout_seconds": 20,
+ "request_timeout": 10.0,
+ }
+ assert captured["timeout_seconds"] == 15.0
+ request = captured["request"]
+ assert isinstance(request, dict)
+ assert "timeout_ms" not in request
+ assert "deadline_unix_ms" not in request
+ assert request["runtime_state_path"].endswith("/runtime-state.json")
+ assert request["result_path"].endswith("/result.json")
+ assert captured["close"] == {
+ "endpoint": endpoint,
+ "connection_token": "token",
+ "timeout": 1.0,
+ }
+
+
+ ("code", "key"),
+ [
+ ("active_turn_required", "computer.cli.error.missingCallerTurn"),
+ ("session_archived", "computer.cli.error.sessionArchived"),
+ ("other_rejection", "computer.cli.error.scriptOpenRejected"),
+ ],
+)
+def test_python_runner_localizes_known_open_rejections(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ code: str,
+ key: str,
+) -> None:
+ script = tmp_path / "script.mjs"
+ script.write_text("export default async () => null;", encoding="utf-8")
+ monkeypatch.setattr(
+ computer_script_runner,
+ "private_desktop_node_bin",
+ lambda: Path("/private/node"),
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "resolve_computer_script_endpoint",
+ lambda: object(),
+ )
+ monkeypatch.setattr(
+ computer_script_runner,
+ "computer_script_open_sync",
+ lambda **_kwargs: {"body": {"ok": False, "error": code}},
+ )
+
+ with pytest.raises(ComputerScriptRunnerError) as exc_info:
+ computer_script_runner.run_computer_script(
+ caller_session_id="ses-script",
+ script_file=str(script),
+ arg_values=[],
+ )
+
+ assert exc_info.value.code == code
+ assert exc_info.value.key == key
+ assert exc_info.value.params == (
+ {"code": code} if code == "other_rejection" else {}
+ )
+
+
+def test_python_runner_formats_integral_timeout_without_decimal() -> None:
+ assert computer_script_runner._display_seconds(300.0) == 300
+ assert computer_script_runner._display_seconds(0.25) == 0.25
+
+
+def test_python_runner_uses_the_verified_controller_endpoint_for_open_and_close() -> None:
+ with tempfile.TemporaryDirectory(prefix="avibe-script-uds-", dir="/tmp") as directory:
+ socket_path = Path(directory) / "dispatch.sock"
+ listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ listener.bind(str(socket_path))
+ listener.listen(1)
+ socket_path.chmod(0o600)
+ captured: list[tuple[str, dict]] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ payload = json.loads(request.content)
+ captured.append((request.url.path, payload))
+ if request.url.path.endswith("/open"):
+ return httpx.Response(
+ 200,
+ json={
+ "ok": True,
+ "connection_token": "token",
+ "session_id": "ses-script",
+ "timeout_ms": 30000,
+ },
+ )
+ return httpx.Response(200, json={"ok": True, "closed": True})
+
+ try:
+ with patch(
+ "vibe.internal_client.httpx.HTTPTransport",
+ return_value=httpx.MockTransport(handler),
+ ):
+ endpoint = resolve_computer_script_endpoint(socket_path)
+ opened = computer_script_open_sync(
+ endpoint=endpoint,
+ caller_session_id="ses-script",
+ timeout_seconds=30,
+ )
+ closed = computer_script_close_sync(
+ endpoint=endpoint,
+ connection_token="token",
+ )
+ finally:
+ listener.close()
+
+ assert computer_script_endpoint_payload(endpoint) == {
+ "transport": "unix",
+ "socket_path": str(socket_path),
+ }
+ assert opened["body"]["connection_token"] == "token"
+ assert closed["body"]["closed"] is True
+ assert captured == [
+ (
+ "/internal/computer-script/open",
+ {"caller_session_id": "ses-script", "timeout_seconds": 30},
+ ),
+ (
+ "/internal/computer-script/close",
+ {"connection_token": "token"},
+ ),
+ ]
diff --git a/tests/test_computer_script_sessions.py b/tests/test_computer_script_sessions.py
new file mode 100644
index 000000000..abc1303c8
--- /dev/null
+++ b/tests/test_computer_script_sessions.py
@@ -0,0 +1,422 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from types import SimpleNamespace
+
+import httpx
+
+from core import internal_server, session_turns
+from core.computer_script_sessions import (
+ ComputerScriptSessionError,
+ ComputerScriptSessionManager,
+)
+from modules.im import MessageContext
+
+
+def _structured(payload: dict, *, is_error: bool = False) -> dict:
+ return {
+ "isError": is_error,
+ "structuredContent": payload,
+ "content": [{"type": "text", "text": json.dumps(payload)}],
+ }
+
+
+def _context(turn_id: str) -> MessageContext:
+ return MessageContext(
+ user_id="owner",
+ channel_id="channel",
+ platform="avibe",
+ platform_specific={
+ "agent_session_id": "ses-script",
+ "turn_token": turn_id,
+ },
+ )
+
+
+def test_script_manager_binds_to_the_exact_turn_and_retains_terminal_code() -> None:
+ async def exercise() -> tuple[str, str]:
+ controller = SimpleNamespace(
+ _session_id_from_context=lambda context: context.platform_specific[
+ "agent_session_id"
+ ],
+ )
+ turn_manager = session_turns.SessionTurnManager(controller)
+ first_task = asyncio.create_task(asyncio.sleep(60))
+ first_turn = session_turns.Turn(
+ task=first_task,
+ context=_context("turn-1"),
+ logical_turn_id="turn-1",
+ )
+ turn_manager.in_flight["ses-script"] = first_turn
+ manager = ComputerScriptSessionManager(turn_manager)
+ opened = manager.open("ses-script", 30)
+ token = opened["connection_token"]
+
+ second_task = asyncio.create_task(asyncio.sleep(60))
+ turn_manager.in_flight["ses-script"] = session_turns.Turn(
+ task=second_task,
+ context=_context("turn-2"),
+ logical_turn_id="turn-2",
+ )
+ try:
+ try:
+ manager.admit(token)
+ except ComputerScriptSessionError as exc:
+ first_code = exc.code
+ try:
+ manager.admit(token)
+ except ComputerScriptSessionError as exc:
+ second_code = exc.code
+ return first_code, second_code
+ finally:
+ manager.close_all()
+ first_task.cancel()
+ second_task.cancel()
+ for task in (first_task, second_task):
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+
+ assert asyncio.run(exercise()) == ("script_turn_ended", "script_turn_ended")
+
+
+def test_script_manager_invalidates_when_the_bound_turn_finishes() -> None:
+ async def exercise() -> str:
+ controller = SimpleNamespace(
+ _session_id_from_context=lambda context: context.platform_specific[
+ "agent_session_id"
+ ],
+ )
+ turn_manager = session_turns.SessionTurnManager(controller)
+ task = asyncio.create_task(asyncio.sleep(60))
+ turn_manager.in_flight["ses-script"] = session_turns.Turn(
+ task=task,
+ context=_context("turn-1"),
+ logical_turn_id="turn-1",
+ )
+ manager = ComputerScriptSessionManager(turn_manager)
+ token = manager.open("ses-script", 30)["connection_token"]
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+ await asyncio.sleep(0)
+ try:
+ manager.admit(token)
+ except ComputerScriptSessionError as exc:
+ return exc.code
+ finally:
+ manager.close_all()
+ raise AssertionError("finished Turn was admitted")
+
+ assert asyncio.run(exercise()) == "script_turn_ended"
+
+
+def test_script_manager_refuses_requests_while_stop_waits_for_the_backend() -> None:
+ async def exercise() -> tuple[str, dict]:
+ stop_started = asyncio.Event()
+ release_stop = asyncio.Event()
+
+ class CommandHandler:
+ async def handle_stop(self, _context: MessageContext) -> bool:
+ stop_started.set()
+ await release_stop.wait()
+ return True
+
+ controller = SimpleNamespace(
+ command_handler=CommandHandler(),
+ computer_script_sessions=None,
+ _session_id_from_context=lambda context: context.platform_specific[
+ "agent_session_id"
+ ],
+ )
+ turn_manager = session_turns.SessionTurnManager(controller)
+ task = asyncio.create_task(asyncio.sleep(60))
+ turn = session_turns.Turn(
+ task=task,
+ context=_context("turn-1"),
+ logical_turn_id="turn-1",
+ )
+ turn_manager.in_flight["ses-script"] = turn
+ manager = ComputerScriptSessionManager(turn_manager)
+ controller.computer_script_sessions = manager
+ token = manager.open("ses-script", 30)["connection_token"]
+ cancel_task = asyncio.create_task(
+ turn_manager._cancel_legacy_turn("ses-script", turn)
+ )
+ await stop_started.wait()
+ try:
+ try:
+ manager.admit(token)
+ except ComputerScriptSessionError as exc:
+ refusal = exc.code
+ else:
+ refusal = "admitted"
+ finally:
+ release_stop.set()
+ result = await cancel_task
+ manager.close_all()
+ await asyncio.gather(task, return_exceptions=True)
+ return refusal, result
+
+ refusal, result = asyncio.run(exercise())
+ assert refusal == "script_turn_ended"
+ assert result["status"] == "cancel_requested"
+
+
+def test_script_manager_enforces_one_total_deadline() -> None:
+ async def exercise() -> str:
+ controller = SimpleNamespace(
+ _session_id_from_context=lambda context: context.platform_specific[
+ "agent_session_id"
+ ],
+ )
+ turn_manager = session_turns.SessionTurnManager(controller)
+ task = asyncio.create_task(asyncio.sleep(60))
+ turn_manager.in_flight["ses-script"] = session_turns.Turn(
+ task=task,
+ context=_context("turn-1"),
+ logical_turn_id="turn-1",
+ )
+ manager = ComputerScriptSessionManager(turn_manager)
+ token = manager.open("ses-script", 0.01)["connection_token"]
+ await asyncio.sleep(0.02)
+ try:
+ manager.admit(token)
+ except ComputerScriptSessionError as exc:
+ return exc.code
+ finally:
+ manager.close_all()
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+ raise AssertionError("expired script was admitted")
+
+ assert asyncio.run(exercise()) == "script_timeout"
+
+
+def test_script_manager_keeps_the_cli_allowlist_and_session_binding() -> None:
+ async def exercise() -> tuple[str, str, str]:
+ controller = SimpleNamespace(
+ _session_id_from_context=lambda context: context.platform_specific[
+ "agent_session_id"
+ ],
+ )
+ turn_manager = session_turns.SessionTurnManager(controller)
+ task = asyncio.create_task(asyncio.sleep(60))
+ turn_manager.in_flight["ses-script"] = session_turns.Turn(
+ task=task,
+ context=_context("turn-1"),
+ logical_turn_id="turn-1",
+ )
+ manager = ComputerScriptSessionManager(turn_manager)
+ token = manager.open("ses-script", 30)["connection_token"]
+ try:
+ try:
+ manager.prepare_tool_request(token, "start_session", {})
+ except ComputerScriptSessionError as exc:
+ tool_code = exc.code
+ try:
+ manager.prepare_tool_request(
+ token,
+ "list_apps",
+ {"session": "other-session"},
+ )
+ except ComputerScriptSessionError as exc:
+ session_code = exc.code
+ prepared_session, prepared_arguments = manager.prepare_tool_request(
+ token,
+ "double_click",
+ {
+ "pid": 41,
+ "window_id": 9,
+ "element_token": "el-1",
+ },
+ )
+ return tool_code, session_code, (
+ f"{prepared_session}:{prepared_arguments['element_token']}"
+ )
+ finally:
+ manager.close_all()
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+
+ assert asyncio.run(exercise()) == (
+ "invalid_computer_script_tool",
+ "session_argument_forbidden",
+ "ses-script:el-1",
+ )
+
+
+def test_script_request_body_is_bounded_before_admission(
+ monkeypatch,
+) -> None:
+ async def exercise(chunked: bool) -> tuple[int, dict, list[tuple[str, dict]]]:
+ calls: list[tuple[str, dict]] = []
+
+ class ComputerServer:
+ async def call_tool(self, name: str, arguments: dict, **_kwargs) -> dict:
+ calls.append((name, arguments))
+ return _structured({})
+
+ controller = SimpleNamespace(
+ computer_use_server=ComputerServer(),
+ computer_script_sessions=None,
+ controller_id="controller-test",
+ _session_id_from_context=lambda _context: None,
+ )
+ app = internal_server.create_app(controller)
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(
+ transport=transport,
+ base_url="http://test",
+ ) as client:
+ if chunked:
+
+ async def chunks():
+ yield b"{"
+ yield b"x" * 64
+
+ response = await client.post(
+ "/internal/computer-script/request",
+ content=chunks(),
+ headers={"Content-Type": "application/json"},
+ )
+ else:
+ response = await client.post(
+ "/internal/computer-script/request",
+ content=b"x" * 65,
+ headers={"Content-Type": "application/json"},
+ )
+ return response.status_code, response.json(), calls
+
+ monkeypatch.setattr(
+ internal_server,
+ "_COMPUTER_SCRIPT_REQUEST_MAX_BYTES",
+ 64,
+ )
+ for chunked in (False, True):
+ status, payload, calls = asyncio.run(exercise(chunked))
+ assert status == 413
+ assert payload == {
+ "ok": False,
+ "error": "computer_script_request_too_large",
+ "params": {"limit_bytes": 64},
+ "dispatched": False,
+ }
+ assert calls == []
+
+
+def test_script_routes_reject_an_old_turn_before_reaching_computer_server() -> None:
+ async def exercise() -> tuple[dict, dict, list[tuple[str, dict, object]]]:
+ calls: list[tuple[str, dict, object]] = []
+
+ class ComputerServer:
+ async def call_tool(
+ self,
+ name: str,
+ arguments: dict,
+ *,
+ handle_descriptor=None,
+ ) -> dict:
+ calls.append((name, arguments, handle_descriptor))
+ return _structured({"windows": []})
+
+ controller = SimpleNamespace(
+ computer_use_server=ComputerServer(),
+ computer_script_sessions=None,
+ controller_id="controller-test",
+ _session_id_from_context=lambda context: context.platform_specific[
+ "agent_session_id"
+ ],
+ )
+ app = internal_server.create_app(controller)
+ first_task = asyncio.create_task(asyncio.sleep(60))
+ controller.session_turns.in_flight["ses-script"] = session_turns.Turn(
+ task=first_task,
+ context=_context("turn-1"),
+ logical_turn_id="turn-1",
+ )
+ transport = httpx.ASGITransport(app=app)
+ second_task = asyncio.create_task(asyncio.sleep(60))
+ try:
+ async with httpx.AsyncClient(
+ transport=transport,
+ base_url="http://test",
+ ) as client:
+ opened = await client.post(
+ "/internal/computer-script/open",
+ json={
+ "caller_session_id": "ses-script",
+ "timeout_seconds": 30,
+ },
+ )
+ token = opened.json()["connection_token"]
+ success = await client.post(
+ "/internal/computer-script/request",
+ json={
+ "connection_token": token,
+ "tool": "list_windows",
+ "arguments": {"pid": 41},
+ "descriptor": {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ },
+ },
+ )
+ controller.session_turns.in_flight["ses-script"] = session_turns.Turn(
+ task=second_task,
+ context=_context("turn-2"),
+ logical_turn_id="turn-2",
+ )
+ stale = await client.post(
+ "/internal/computer-script/request",
+ json={
+ "connection_token": token,
+ "tool": "list_windows",
+ "arguments": {"pid": 41},
+ "descriptor": {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ },
+ },
+ )
+ return success.json(), stale.json(), calls
+ finally:
+ controller.computer_script_sessions.close_all()
+ first_task.cancel()
+ second_task.cancel()
+ for task in (first_task, second_task):
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+
+ success, stale, calls = asyncio.run(exercise())
+ assert success["ok"] is True
+ assert stale["error"] == "script_turn_ended"
+ assert stale["dispatched"] is False
+ assert calls == [
+ (
+ "list_windows",
+ {"pid": 41, "session": "ses-script"},
+ {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ },
+ ),
+ ]
diff --git a/tests/test_computer_server.py b/tests/test_computer_server.py
index 2731c4e7d..173c9bd12 100644
--- a/tests/test_computer_server.py
+++ b/tests/test_computer_server.py
@@ -446,6 +446,7 @@ async def test_non_ready_call_names_status_and_never_spawns_upstream(
payload = json.loads(result["content"][0]["text"])
assert payload == {
"code": "computer_use_unavailable",
+ "dispatched": False,
"params": {
"reason": reason,
"status": status,
@@ -454,6 +455,232 @@ async def test_non_ready_call_names_status_and_never_spawns_upstream(
assert starts == 0
+ (
+ "descriptor",
+ "tool",
+ "arguments",
+ "apps",
+ "windows",
+ "expected_code",
+ "expected_tools",
+ ),
+ [
+ (
+ {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ },
+ "kill_app",
+ {"pid": 41, "session": "ses-script"},
+ [
+ {
+ "pid": 41,
+ "bundle_id": "com.apple.notes",
+ "name": "Notes",
+ }
+ ],
+ [],
+ "stale_app_handle",
+ ["start_session", "list_apps"],
+ ),
+ (
+ {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ },
+ "list_windows",
+ {"pid": 41, "session": "ses-script"},
+ [
+ {
+ "pid": 57,
+ "bundle_id": "com.apple.mail",
+ "name": "Mail",
+ }
+ ],
+ [],
+ "stale_app_handle",
+ ["start_session", "list_apps"],
+ ),
+ (
+ {
+ "kind": "window",
+ "pid": 41,
+ "window_id": 9,
+ "app": {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ },
+ },
+ "get_window_state",
+ {"pid": 41, "window_id": 9, "session": "ses-script"},
+ [
+ {
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": "Mail",
+ }
+ ],
+ [
+ {
+ "pid": 57,
+ "window_id": 9,
+ "app_name": "Mail",
+ }
+ ],
+ "stale_window_handle",
+ ["start_session", "list_apps", "list_windows"],
+ ),
+ ],
+)
+async def test_script_handle_descriptors_are_validated_before_the_action(
+ tmp_path: Path,
+ descriptor: dict[str, Any],
+ tool: str,
+ arguments: dict[str, Any],
+ apps: list[dict[str, Any]],
+ windows: list[dict[str, Any]],
+ expected_code: str,
+ expected_tools: list[str],
+) -> None:
+ state_path, state = _state(tmp_path)
+
+ class HandleUpstream(FakeUpstream):
+ async def request(
+ self,
+ method: str,
+ params: Mapping[str, Any],
+ ) -> dict[str, Any]:
+ assert method == "tools/call"
+ copied = json.loads(json.dumps(params))
+ self.calls.append((method, copied))
+ name = params["name"]
+ if name == "list_apps":
+ structured = {"apps": apps}
+ elif name == "list_windows":
+ structured = {"windows": windows}
+ else:
+ structured = {"effect": "confirmed", "route": "system_api"}
+ return {
+ "jsonrpc": "2.0",
+ "id": 1,
+ "result": {
+ "content": [],
+ "structuredContent": structured,
+ },
+ }
+
+ upstream = HandleUpstream()
+ server = ComputerUseServer(
+ state_path=state_path,
+ status_reader=lambda: ComputerUseStatus("ready", None, state),
+ upstream_factory=lambda _state: asyncio.sleep(0, result=upstream),
+ lease_manager=FakeLeaseManager(), # type: ignore[arg-type]
+ )
+
+ result = await server.call_tool(
+ tool,
+ arguments,
+ handle_descriptor=descriptor,
+ )
+
+ assert result["structuredContent"]["code"] == expected_code
+ assert [params["name"] for _method, params in upstream.calls] == expected_tools
+
+
+async def test_current_script_window_descriptor_is_not_forwarded_to_driver(
+ tmp_path: Path,
+) -> None:
+ state_path, state = _state(tmp_path)
+
+ class HandleUpstream(FakeUpstream):
+ async def request(
+ self,
+ method: str,
+ params: Mapping[str, Any],
+ ) -> dict[str, Any]:
+ assert method == "tools/call"
+ copied = json.loads(json.dumps(params))
+ self.calls.append((method, copied))
+ name = params["name"]
+ if name == "list_apps":
+ structured = {
+ "apps": [
+ {
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": "Mail",
+ }
+ ]
+ }
+ elif name == "list_windows":
+ structured = {
+ "windows": [
+ {
+ "pid": 41,
+ "window_id": 9,
+ "app_name": "Mail",
+ }
+ ]
+ }
+ else:
+ structured = {"pid": 41, "window_id": 9, "elements": []}
+ return {
+ "jsonrpc": "2.0",
+ "id": 1,
+ "result": {
+ "content": [],
+ "structuredContent": structured,
+ },
+ }
+
+ upstream = HandleUpstream()
+ server = ComputerUseServer(
+ state_path=state_path,
+ status_reader=lambda: ComputerUseStatus("ready", None, state),
+ upstream_factory=lambda _state: asyncio.sleep(0, result=upstream),
+ lease_manager=FakeLeaseManager(), # type: ignore[arg-type]
+ )
+ descriptor = {
+ "kind": "window",
+ "pid": 41,
+ "window_id": 9,
+ "app": {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ },
+ }
+
+ result = await server.call_tool(
+ "get_window_state",
+ {"pid": 41, "window_id": 9, "session": "ses-script"},
+ handle_descriptor=descriptor,
+ )
+
+ assert result.get("isError") is not True
+ assert [params["name"] for _method, params in upstream.calls] == [
+ "start_session",
+ "list_apps",
+ "list_windows",
+ "get_window_state",
+ ]
+ assert upstream.calls[-1][1]["arguments"] == {
+ "pid": 41,
+ "window_id": 9,
+ "session": "ses-script",
+ }
+
+
@pytest.mark.asyncio
async def test_daemon_generation_and_child_exit_each_replace_proxy_once(
tmp_path: Path,
@@ -551,6 +778,29 @@ def test_result_folding_preserves_driver_error_code_params_and_detail() -> None:
assert result["structuredContent"] == payload
+def test_result_folding_marks_consent_refusal_not_dispatched() -> None:
+ result = _fold_tool_result(
+ {
+ "result": {
+ "content": [],
+ "structuredContent": {
+ "code": "foreground_consent_required",
+ "params": {},
+ },
+ "isError": True,
+ }
+ }
+ )
+
+ payload = json.loads(result["content"][0]["text"])
+ assert payload == {
+ "code": "foreground_consent_required",
+ "dispatched": False,
+ "params": {},
+ }
+ assert result["structuredContent"] == payload
+
+
def test_result_folding_preserves_bounded_detail_inside_driver_params() -> None:
result = _fold_tool_result(
{
@@ -841,11 +1091,36 @@ async def test_call_tool_preserves_upstream_unavailable_detail_and_logs(
assert payload == {
"code": "upstream_unavailable",
"detail": "transport failed",
+ "dispatched": False,
"params": {},
}
assert "transport failed" in caplog.text
+async def test_post_forward_upstream_failure_does_not_claim_not_dispatched(
+ tmp_path: Path,
+) -> None:
+ state_path, state = _state(tmp_path)
+ upstream = FakeUpstream()
+ server = ComputerUseServer(
+ state_path=state_path,
+ status_reader=lambda: ComputerUseStatus("ready", None, state),
+ upstream_factory=lambda _state: asyncio.sleep(0, result=upstream),
+ lease_manager=FakeLeaseManager(), # type: ignore[arg-type]
+ )
+
+ started = await server.call_tool("start_session", {"session": "ses-a"})
+ assert started.get("isError") is not True
+
+ upstream.fail_transport = True
+ result = await server.call_tool("list_apps", {"session": "ses-a"})
+ payload = result["structuredContent"]
+
+ assert payload["code"] == "upstream_unavailable"
+ assert "dispatched" not in payload
+
+
@pytest.mark.asyncio
async def test_real_stdio_proxy_preserves_large_window_state_response(
tmp_path: Path,
diff --git a/tests/test_distribution_artifacts.py b/tests/test_distribution_artifacts.py
index b0228a6c4..64caa1c60 100644
--- a/tests/test_distribution_artifacts.py
+++ b/tests/test_distribution_artifacts.py
@@ -56,8 +56,20 @@ def _assert_members_and_metadata(members, metadata):
assert "memory" not in parsed.get_all("Provides-Extra", [])
assert all("avibe-memory" not in item.lower().replace("_", "-")
for item in parsed.get_all("Requires-Dist", []))
- for required in ("vibe/cli.py", "core/controller.py", "vibe/show_runtime_manifest.json",
- "vibe/show_router.tsx", "vibe/ui/dist/index.html"):
+ for required in (
+ "vibe/cli.py",
+ "core/controller.py",
+ "vibe/show_runtime_manifest.json",
+ "vibe/show_router.tsx",
+ "vibe/ui/dist/index.html",
+ "vibe/computer_sdk/index.mjs",
+ "vibe/computer_sdk/index.d.ts",
+ "vibe/computer_sdk/failure-envelope.json",
+ "vibe/computer_sdk/loader.mjs",
+ "vibe/computer_sdk/package.json",
+ "vibe/computer_sdk/runner.mjs",
+ "vibe/computer_sdk/runtime.mjs",
+ ):
assert required in members, required
diff --git a/vibe/cli.py b/vibe/cli.py
index 30adcfc63..a41a0a6aa 100644
--- a/vibe/cli.py
+++ b/vibe/cli.py
@@ -756,6 +756,34 @@ def cmd_computer(args) -> int:
hint=translate("computer.cli.hint.missingCallerSession"),
help_command="vibe computer --help",
)
+ if args.computer_command == "run":
+ from vibe.computer_script_runner import (
+ ComputerScriptRunnerError,
+ run_computer_script,
+ )
+
+ try:
+ result = run_computer_script(
+ caller_session_id=caller_session_id,
+ script_file=args.file,
+ arg_values=args.arg,
+ )
+ except ComputerScriptRunnerError as exc:
+ params = dict(exc.params)
+ if exc.structured_error is not None:
+ params["error"] = translate_computer_error(
+ exc.structured_error,
+ translate=translate,
+ fallback=str(params.get("error") or ""),
+ )
+ raise TaskCliError(
+ translate(exc.key, **params),
+ code=exc.code,
+ details=exc.details,
+ help_command="vibe computer run --help",
+ ) from exc
+ print(json.dumps(result, ensure_ascii=False, sort_keys=True))
+ return 0
if args.computer_command == "observe":
command_path = ("observe", args.computer_observe_command)
elif args.computer_command == "clipboard":
@@ -17390,10 +17418,33 @@ def build_parser():
)
computer_subparsers = computer_parser.add_subparsers(
dest="computer_command",
- metavar="{observe,zoom,click,type,key,hotkey,scroll,drag,set-value,clipboard,launch,force-quit}",
+ metavar="{run,observe,zoom,click,type,key,hotkey,scroll,drag,set-value,clipboard,launch,force-quit}",
)
computer_subparsers.required = True
+ computer_run_parser = computer_subparsers.add_parser(
+ "run",
+ help=i18n_t("computer.cli.help.run", computer_help_language),
+ description=i18n_t(
+ "computer.cli.help.runDescription",
+ computer_help_language,
+ ),
+ error_help_command="vibe computer run --help",
+ error_hint=i18n_t("computer.cli.hint.invalidArguments", computer_help_language),
+ error_i18n_key="computer.cli.error.invalidArguments",
+ )
+ computer_run_parser.add_argument(
+ "file",
+ help=i18n_t("computer.cli.help.scriptFile", computer_help_language),
+ )
+ computer_run_parser.add_argument(
+ "--arg",
+ action="append",
+ default=[],
+ metavar="NAME=VALUE",
+ help=i18n_t("computer.cli.help.scriptArg", computer_help_language),
+ )
+
def _add_computer_options(command_parser) -> None:
command_parser.add_argument(
"--args-json",
@@ -19390,6 +19441,7 @@ _CLI_COMMAND_FLOORS: dict[tuple[str, ...], Optional[str]] = {
("computer", "observe", "apps"): "member",
("computer", "observe", "windows"): "member",
("computer", "observe", "window"): "member",
+ ("computer", "run"): "member",
("computer", "zoom"): "member",
("computer", "click"): "member",
("computer", "type"): "member",
diff --git a/vibe/computer_script_runner.py b/vibe/computer_script_runner.py
new file mode 100644
index 000000000..580e9e749
--- /dev/null
+++ b/vibe/computer_script_runner.py
@@ -0,0 +1,752 @@
+"""Python owner for ``vibe computer run``."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass
+import json
+import os
+from pathlib import Path
+import signal
+import shutil
+import subprocess
+import tempfile
+import threading
+import time
+from typing import Any, Iterable
+
+from vibe.desktop_runtime import private_desktop_node_bin
+from vibe.internal_client import (
+ InternalServerTimeout,
+ InternalServerUnavailable,
+ computer_script_close_sync,
+ computer_script_endpoint_payload,
+ computer_script_open_sync,
+ resolve_computer_script_endpoint,
+)
+
+
+DEFAULT_SCRIPT_TIMEOUT_SECONDS = 300.0
+_SCRIPT_DIAGNOSTIC_TAIL_BYTES = 64 * 1024
+_SCRIPT_RESULT_MAX_BYTES = 8 * 1024 * 1024
+_SCRIPT_RUNTIME_STATE_MAX_BYTES = _SCRIPT_RESULT_MAX_BYTES
+_SCRIPT_RESULT_PATH_ENV = "AVIBE_COMPUTER_RESULT_PATH"
+_PROCESS_REAP_TIMEOUT_SECONDS = 0.25
+_ARTIFACT_DIRECTORY_PREFIX = "avibe-computer-script-"
+_SUPERVISOR_DIRECTORY_PREFIX = "avibe-computer-script-runner-"
+_MAX_RETAINED_ARTIFACT_RUNS = 20
+_FAILURE_ENVELOPE_FIELDS: tuple[str, ...] | None = None
+
+_SCRIPT_OPEN_ERROR_KEYS = {
+ "active_turn_required": "computer.cli.error.missingCallerTurn",
+ "session_archived": "computer.cli.error.sessionArchived",
+}
+
+
+class ComputerScriptRunnerError(RuntimeError):
+ """A script could not be prepared or did not complete successfully."""
+
+ def __init__(
+ self,
+ key: str,
+ *,
+ code: str,
+ params: dict[str, Any] | None = None,
+ details: dict[str, Any] | None = None,
+ structured_error: dict[str, Any] | None = None,
+ ) -> None:
+ self.key = key
+ self.code = code
+ self.params = params or {}
+ self.details = details or {}
+ self.structured_error = structured_error
+ super().__init__(key)
+
+
+class _SupervisorTerminated(RuntimeError):
+ """The script supervisor received SIGTERM while owning a child."""
+
+
+@dataclass(frozen=True)
+class _CappedJsonRead:
+ value: Any = None
+ error: str | None = None
+
+
+@dataclass
+class _ScriptRun:
+ artifact_directory: Path
+ runtime_state_path: Path
+ result_path: Path
+ node_started: bool = False
+ completed: subprocess.CompletedProcess[str] | None = None
+
+
+@dataclass(frozen=True)
+class _FinalizedRun:
+ result: Any = None
+ error: ComputerScriptRunnerError | None = None
+ cause: Exception | None = None
+
+
+def parse_script_arguments(values: Iterable[str]) -> dict[str, str]:
+ """Parse repeatable ``NAME=VALUE`` arguments without guessing value types."""
+
+ parsed: dict[str, str] = {}
+ for raw in values:
+ name, separator, value = raw.partition("=")
+ name = name.strip()
+ if separator != "=" or not name:
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptArgNameValue",
+ code="invalid_computer_script_argument",
+ )
+ if name in parsed:
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptArgDuplicate",
+ code="duplicate_computer_script_argument",
+ params={"name": name},
+ )
+ if value.startswith("@"):
+ source = Path(value[1:]).expanduser()
+ try:
+ value = source.read_text(encoding="utf-8")
+ except (OSError, UnicodeError) as exc:
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptArgFile",
+ code="computer_script_argument_file_unreadable",
+ params={"path": str(source), "error": str(exc)},
+ ) from exc
+ parsed[name] = value
+ return parsed
+
+
+def _script_path(raw_path: str) -> Path:
+ path = Path(raw_path).expanduser()
+ try:
+ resolved = path.resolve(strict=True)
+ except OSError as exc:
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptFile",
+ code="computer_script_file_unreadable",
+ params={"path": str(path), "error": str(exc)},
+ ) from exc
+ if not resolved.is_file() or resolved.suffix.lower() != ".mjs":
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptFileType",
+ code="computer_script_file_unreadable",
+ params={"path": str(path)},
+ )
+ return resolved
+
+
+def _sdk_directory() -> Path:
+ return Path(__file__).resolve().parent / "computer_sdk"
+
+
+def _failure_envelope_fields() -> tuple[str, ...]:
+ global _FAILURE_ENVELOPE_FIELDS
+ if _FAILURE_ENVELOPE_FIELDS is not None:
+ return _FAILURE_ENVELOPE_FIELDS
+ contract_path = _sdk_directory() / "failure-envelope.json"
+ try:
+ contract = json.loads(contract_path.read_text(encoding="utf-8"))
+ fields = contract["fields"]
+ except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc:
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptSdkUnavailable",
+ code="computer_script_unavailable",
+ ) from exc
+ if (
+ not isinstance(fields, list)
+ or not fields
+ or any(not isinstance(field, str) or not field for field in fields)
+ or len(set(fields)) != len(fields)
+ ):
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptSdkUnavailable",
+ code="computer_script_unavailable",
+ )
+ _FAILURE_ENVELOPE_FIELDS = tuple(fields)
+ return _FAILURE_ENVELOPE_FIELDS
+
+
+def _failure_envelope(**values: Any) -> dict[str, Any]:
+ fields = _failure_envelope_fields()
+ if set(values) != set(fields):
+ raise RuntimeError("failure envelope implementation does not match its contract")
+ return {field: values[field] for field in fields}
+
+
+def _read_capped_json(path: Path, limit: int) -> _CappedJsonRead:
+ try:
+ size = path.stat().st_size
+ except FileNotFoundError:
+ return _CappedJsonRead(error="missing")
+ except OSError:
+ return _CappedJsonRead(error="unreadable")
+ if size > limit:
+ return _CappedJsonRead(error="too_large")
+ try:
+ with path.open("rb") as stream:
+ encoded = stream.read(limit + 1)
+ except OSError:
+ return _CappedJsonRead(error="unreadable")
+ if len(encoded) > limit:
+ return _CappedJsonRead(error="too_large")
+ try:
+ return _CappedJsonRead(value=json.loads(encoded.decode("utf-8")))
+ except (UnicodeError, json.JSONDecodeError):
+ return _CappedJsonRead(error="invalid_json")
+
+
+def _terminate_and_reap_process_group(process: subprocess.Popen[Any]) -> None:
+ try:
+ if os.name == "posix":
+ os.killpg(process.pid, signal.SIGKILL)
+ elif process.poll() is None:
+ process.kill()
+ except ProcessLookupError:
+ pass
+ try:
+ process.wait(timeout=_PROCESS_REAP_TIMEOUT_SECONDS)
+ except subprocess.TimeoutExpired:
+ pass
+
+
+@contextmanager
+def _sigterm_as_supervisor_exception():
+ if (
+ os.name != "posix"
+ or threading.current_thread() is not threading.main_thread()
+ ):
+ yield
+ return
+
+ previous_handler = signal.getsignal(signal.SIGTERM)
+
+ def terminate(_signum, _frame) -> None:
+ raise _SupervisorTerminated("computer script supervisor terminated")
+
+ signal.signal(signal.SIGTERM, terminate)
+ try:
+ yield
+ finally:
+ signal.signal(signal.SIGTERM, previous_handler)
+
+
+def _run_node_process(
+ node: Path,
+ runner: Path,
+ request: dict[str, Any],
+ timeout_seconds: float,
+) -> subprocess.CompletedProcess[str]:
+ command = [str(node), str(runner)]
+ supervisor_directory = Path(
+ tempfile.mkdtemp(prefix=_SUPERVISOR_DIRECTORY_PREFIX)
+ ).resolve()
+ try:
+ try:
+ os.chmod(supervisor_directory, 0o700)
+ except OSError:
+ pass
+ request_path = supervisor_directory / "request.json"
+ result_path = Path(str(request["result_path"]))
+ request_path.write_text(
+ json.dumps(request, ensure_ascii=False),
+ encoding="utf-8",
+ )
+ try:
+ os.chmod(request_path, 0o600)
+ except OSError:
+ pass
+ child_environment = os.environ.copy()
+ child_environment[_SCRIPT_RESULT_PATH_ENV] = str(result_path)
+
+ with (
+ _sigterm_as_supervisor_exception(),
+ request_path.open("r", encoding="utf-8") as request_input,
+ ):
+ process: subprocess.Popen[Any] | None = None
+ try:
+ process = subprocess.Popen(
+ command,
+ stdin=request_input,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ env=child_environment,
+ start_new_session=os.name == "posix",
+ )
+ try:
+ returncode = process.wait(timeout=timeout_seconds)
+ except subprocess.TimeoutExpired as exc:
+ raise subprocess.TimeoutExpired(
+ command,
+ timeout_seconds,
+ ) from exc
+ finally:
+ if process is not None:
+ _terminate_and_reap_process_group(process)
+
+ return subprocess.CompletedProcess(
+ command,
+ returncode,
+ "",
+ "",
+ )
+ finally:
+ shutil.rmtree(supervisor_directory, ignore_errors=True)
+
+
+def _local_screenshot_path(
+ error: dict[str, Any] | None,
+ artifact_directory: Path,
+) -> Path | None:
+ if not isinstance(error, dict):
+ return None
+ screenshot_path = error.get("screenshot_path")
+ if not isinstance(screenshot_path, str) or not screenshot_path:
+ return None
+ try:
+ resolved = Path(screenshot_path).resolve(strict=True)
+ resolved.relative_to(artifact_directory)
+ except (OSError, RuntimeError, ValueError):
+ return None
+ return resolved if resolved.is_file() else None
+
+
+def _retain_only_screenshot(
+ artifact_directory: Path,
+ screenshot_path: Path,
+) -> None:
+ keep_directories = set(screenshot_path.parents)
+ for root, directories, files in os.walk(
+ artifact_directory,
+ topdown=False,
+ followlinks=False,
+ ):
+ root_path = Path(root)
+ for name in files:
+ candidate = root_path / name
+ try:
+ is_kept = candidate.resolve(strict=True) == screenshot_path
+ except OSError:
+ is_kept = False
+ if not is_kept:
+ candidate.unlink(missing_ok=True)
+ for name in directories:
+ candidate = root_path / name
+ if candidate in keep_directories:
+ continue
+ if candidate.is_symlink():
+ candidate.unlink(missing_ok=True)
+ else:
+ shutil.rmtree(candidate, ignore_errors=True)
+
+
+def _prune_retained_artifact_runs(parent: Path) -> None:
+ candidates: list[tuple[float, Path]] = []
+ try:
+ entries = list(parent.iterdir())
+ except OSError:
+ return
+ for candidate in entries:
+ if (
+ not candidate.name.startswith(_ARTIFACT_DIRECTORY_PREFIX)
+ or candidate.name.startswith(_SUPERVISOR_DIRECTORY_PREFIX)
+ or not candidate.is_dir()
+ or candidate.is_symlink()
+ or (candidate / ".active").exists()
+ ):
+ continue
+ try:
+ candidates.append((candidate.stat().st_mtime, candidate))
+ except OSError:
+ continue
+ candidates.sort(key=lambda item: item[0], reverse=True)
+ for _modified_at, candidate in candidates[_MAX_RETAINED_ARTIFACT_RUNS:]:
+ shutil.rmtree(candidate, ignore_errors=True)
+
+
+def _runtime_context(read: _CappedJsonRead) -> dict[str, Any]:
+ payload = read.value if isinstance(read.value, dict) else {}
+ step = payload.get("step")
+ last_completed_step = payload.get("last_completed_step")
+ screenshot_path = payload.get("screenshot_path")
+ diagnostics = payload.get("diagnostics")
+ if not isinstance(step, str) or not step:
+ step = "script"
+ if not isinstance(last_completed_step, str) or not last_completed_step:
+ last_completed_step = None
+ if not isinstance(screenshot_path, str) or not screenshot_path:
+ screenshot_path = None
+ if not isinstance(diagnostics, str):
+ diagnostics = ""
+ encoded_diagnostics = diagnostics.encode("utf-8", errors="replace")
+ diagnostics = encoded_diagnostics[-_SCRIPT_DIAGNOSTIC_TAIL_BYTES:].decode(
+ "utf-8",
+ errors="replace",
+ )
+ return {
+ "step": step,
+ "last_completed_step": last_completed_step,
+ "screenshot_path": screenshot_path,
+ "diagnostics": diagnostics,
+ "read_error": read.error,
+ }
+
+
+def _failure_params(
+ context: dict[str, Any],
+ values: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+ params = dict(values or {})
+ if context["diagnostics"]:
+ params["diagnostics"] = context["diagnostics"]
+ if context["read_error"] is not None:
+ params["runtime_state_error"] = context["read_error"]
+ return params
+
+
+def _protocol_failure_envelope(
+ context: dict[str, Any],
+ *,
+ reason: str,
+ returncode: int | None,
+) -> dict[str, Any]:
+ if reason == "too_large":
+ message = "The script result exceeded the protocol size limit."
+ params = {"limit_bytes": _SCRIPT_RESULT_MAX_BYTES}
+ else:
+ message = "The script runtime did not produce a valid protocol result."
+ params = {"reason": reason}
+ if returncode is not None:
+ params["status"] = returncode
+ return _failure_envelope(
+ name="ControllerError",
+ code="computer_script_protocol_error",
+ message=message,
+ params=_failure_params(context, params),
+ step=context["step"],
+ last_completed_step=context["last_completed_step"],
+ screenshot_path=context["screenshot_path"],
+ ambiguous=True,
+ )
+
+
+def _merge_failure_context(
+ error: dict[str, Any],
+ context: dict[str, Any],
+) -> dict[str, Any]:
+ merged = dict(error)
+ if context["step"] != "script" or not merged.get("step"):
+ merged["step"] = context["step"]
+ if context["last_completed_step"] is not None:
+ merged["last_completed_step"] = context["last_completed_step"]
+ if context["screenshot_path"] is not None:
+ merged["screenshot_path"] = context["screenshot_path"]
+ error_params = merged.get("params")
+ merged["params"] = _failure_params(
+ context,
+ error_params if isinstance(error_params, dict) else {},
+ )
+ return merged
+
+
+def _finalize_run(
+ run: _ScriptRun,
+ pending_error: Exception | None,
+) -> _FinalizedRun:
+ state_read = _read_capped_json(
+ run.runtime_state_path,
+ _SCRIPT_RUNTIME_STATE_MAX_BYTES,
+ )
+ context = _runtime_context(state_read)
+ failure_envelope: dict[str, Any] | None = None
+ result: Any = None
+ cause = pending_error
+
+ if not run.node_started:
+ finalized_error = (
+ pending_error
+ if isinstance(pending_error, ComputerScriptRunnerError)
+ else None
+ )
+ if finalized_error is None and pending_error is not None:
+ finalized_error = ComputerScriptRunnerError(
+ "computer.cli.error.scriptRuntime",
+ code="computer_script_runtime_failed",
+ params={"error": str(pending_error)},
+ )
+ elif isinstance(pending_error, subprocess.TimeoutExpired):
+ failure_envelope = _failure_envelope(
+ name="Timeout",
+ code="script_timeout",
+ message="The script deadline expired.",
+ params=_failure_params(context),
+ step=context["step"],
+ last_completed_step=context["last_completed_step"],
+ screenshot_path=context["screenshot_path"],
+ ambiguous=True,
+ )
+ finalized_error = _script_failure(failure_envelope)
+ elif pending_error is not None:
+ failure_envelope = _failure_envelope(
+ name="ControllerError",
+ code="computer_script_runtime_failed",
+ message="The script runtime failed.",
+ params=_failure_params(context, {"error": str(pending_error)}),
+ step=context["step"],
+ last_completed_step=context["last_completed_step"],
+ screenshot_path=context["screenshot_path"],
+ ambiguous=True,
+ )
+ finalized_error = _script_failure(failure_envelope)
+ else:
+ result_read = _read_capped_json(run.result_path, _SCRIPT_RESULT_MAX_BYTES)
+ payload = result_read.value
+ if result_read.error is not None:
+ failure_envelope = _protocol_failure_envelope(
+ context,
+ reason=result_read.error,
+ returncode=run.completed.returncode if run.completed else None,
+ )
+ finalized_error = _script_failure(failure_envelope)
+ elif not isinstance(payload, dict) or not isinstance(payload.get("ok"), bool):
+ failure_envelope = _protocol_failure_envelope(
+ context,
+ reason="invalid_shape",
+ returncode=run.completed.returncode if run.completed else None,
+ )
+ finalized_error = _script_failure(failure_envelope)
+ elif payload["ok"] is True:
+ finalized_error = None
+ result = payload.get("result")
+ else:
+ error = payload.get("error")
+ if (
+ not isinstance(error, dict)
+ or set(error) != set(_failure_envelope_fields())
+ ):
+ failure_envelope = _protocol_failure_envelope(
+ context,
+ reason="invalid_failure_envelope",
+ returncode=run.completed.returncode if run.completed else None,
+ )
+ else:
+ failure_envelope = _merge_failure_context(error, context)
+ finalized_error = _script_failure(failure_envelope)
+
+ retained_error = (
+ failure_envelope
+ if failure_envelope is not None
+ else (
+ finalized_error.structured_error
+ if isinstance(finalized_error, ComputerScriptRunnerError)
+ else None
+ )
+ )
+ local_screenshot = _local_screenshot_path(
+ retained_error,
+ run.artifact_directory,
+ )
+ if local_screenshot is None:
+ shutil.rmtree(run.artifact_directory, ignore_errors=True)
+ else:
+ _retain_only_screenshot(run.artifact_directory, local_screenshot)
+ try:
+ os.utime(run.artifact_directory)
+ except OSError:
+ pass
+ _prune_retained_artifact_runs(run.artifact_directory.parent)
+
+ return _FinalizedRun(
+ result=result,
+ error=finalized_error,
+ cause=cause,
+ )
+
+
+def _display_seconds(value: float) -> int | float:
+ return int(value) if float(value).is_integer() else value
+
+
+def _script_failure(error: dict[str, Any]) -> ComputerScriptRunnerError:
+ if set(error) != set(_failure_envelope_fields()):
+ return ComputerScriptRunnerError(
+ "computer.cli.error.scriptRuntimeInvalid",
+ code="computer_script_runtime_failed",
+ )
+ step = str(error.get("step") or "script")
+ message = str(error.get("message") or "the script failed")
+ code = str(error.get("code") or "computer_script_failed")
+ failure_envelope = dict(error)
+ return ComputerScriptRunnerError(
+ "computer.cli.error.scriptFailed",
+ code=code,
+ params={"step": step, "error": message},
+ details=failure_envelope,
+ structured_error=failure_envelope,
+ )
+
+
+def run_computer_script(
+ *,
+ caller_session_id: str,
+ script_file: str,
+ arg_values: Iterable[str],
+ timeout_seconds: float = DEFAULT_SCRIPT_TIMEOUT_SECONDS,
+) -> Any:
+ """Run one ESM script with an exact-Turn-bound Controller connection."""
+
+ node = private_desktop_node_bin()
+ if node is None:
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptNodeUnavailable",
+ code="computer_script_unavailable",
+ )
+ script_path = _script_path(script_file)
+ arguments = parse_script_arguments(arg_values)
+
+ sdk_directory = _sdk_directory()
+ runner = sdk_directory / "runner.mjs"
+ index = sdk_directory / "index.mjs"
+ if not runner.is_file() or not index.is_file():
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptSdkUnavailable",
+ code="computer_script_unavailable",
+ )
+
+ deadline = time.monotonic() + timeout_seconds
+ display_seconds = _display_seconds(timeout_seconds)
+ artifact_directory = Path(
+ tempfile.mkdtemp(prefix=_ARTIFACT_DIRECTORY_PREFIX)
+ ).resolve()
+ try:
+ os.chmod(artifact_directory, 0o700)
+ except OSError:
+ pass
+ try:
+ (artifact_directory / ".active").touch(mode=0o600, exist_ok=False)
+ except OSError:
+ pass
+
+ run = _ScriptRun(
+ artifact_directory=artifact_directory,
+ runtime_state_path=artifact_directory / "runtime-state.json",
+ result_path=artifact_directory / "result.json",
+ )
+ endpoint = None
+ connection_token: str | None = None
+ pending_error: Exception | None = None
+ try:
+ try:
+ endpoint = resolve_computer_script_endpoint()
+ open_timeout = deadline - time.monotonic()
+ if open_timeout <= 0:
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptTimedOut",
+ code="script_timeout",
+ params={"seconds": display_seconds},
+ details={"step": "script"},
+ )
+ opened = computer_script_open_sync(
+ endpoint=endpoint,
+ caller_session_id=caller_session_id,
+ timeout_seconds=timeout_seconds,
+ request_timeout=min(10.0, open_timeout),
+ )
+ except InternalServerUnavailable as exc:
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.serviceUnavailable",
+ code="controller_unavailable",
+ params={"error": str(exc)},
+ ) from exc
+ except InternalServerTimeout as exc:
+ if time.monotonic() >= deadline:
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptTimedOut",
+ code="script_timeout",
+ params={"seconds": display_seconds},
+ details={"step": "script"},
+ ) from exc
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.requestTimedOut",
+ code="computer_script_open_timeout",
+ params={"error": str(exc)},
+ ) from exc
+
+ body = opened.get("body")
+ if not isinstance(body, dict):
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.invalidResponse",
+ code="invalid_computer_script_response",
+ )
+ candidate_token = body.get("connection_token")
+ if (
+ body.get("ok") is not True
+ or not isinstance(candidate_token, str)
+ or not candidate_token
+ ):
+ code = str(body.get("error") or "computer_script_open_rejected")
+ key = _SCRIPT_OPEN_ERROR_KEYS.get(
+ code,
+ "computer.cli.error.scriptOpenRejected",
+ )
+ raise ComputerScriptRunnerError(
+ key,
+ code=code,
+ params={} if code in _SCRIPT_OPEN_ERROR_KEYS else {"code": code},
+ )
+ connection_token = candidate_token
+
+ remaining_seconds = deadline - time.monotonic()
+ if remaining_seconds <= 0:
+ raise ComputerScriptRunnerError(
+ "computer.cli.error.scriptTimedOut",
+ code="script_timeout",
+ params={"seconds": display_seconds},
+ details={"step": "script"},
+ )
+ request = {
+ "script_url": script_path.as_uri(),
+ "sdk_url": index.as_uri(),
+ "arguments": arguments,
+ "endpoint": computer_script_endpoint_payload(endpoint),
+ "connection_token": connection_token,
+ "artifact_directory": str(artifact_directory),
+ "runtime_state_path": str(run.runtime_state_path),
+ "result_path": str(run.result_path),
+ }
+ run.node_started = True
+ run.completed = _run_node_process(
+ node,
+ runner,
+ request,
+ remaining_seconds,
+ )
+ except Exception as exc:
+ pending_error = exc
+ finally:
+ close_timeout = deadline - time.monotonic()
+ if (
+ endpoint is not None
+ and connection_token is not None
+ and close_timeout > 0
+ ):
+ try:
+ computer_script_close_sync(
+ endpoint=endpoint,
+ connection_token=connection_token,
+ timeout=min(1.0, close_timeout),
+ )
+ except (InternalServerUnavailable, InternalServerTimeout):
+ pass
+
+ finalized = _finalize_run(run, pending_error)
+
+ if finalized.error is not None:
+ if finalized.cause is not None and finalized.cause is not finalized.error:
+ raise finalized.error from finalized.cause
+ raise finalized.error
+ return finalized.result
diff --git a/vibe/computer_sdk/failure-envelope.json b/vibe/computer_sdk/failure-envelope.json
new file mode 100644
index 000000000..7e7f01759
--- /dev/null
+++ b/vibe/computer_sdk/failure-envelope.json
@@ -0,0 +1,12 @@
+{
+ "fields": [
+ "name",
+ "code",
+ "message",
+ "params",
+ "step",
+ "last_completed_step",
+ "screenshot_path",
+ "ambiguous"
+ ]
+}
diff --git a/vibe/computer_sdk/index.d.ts b/vibe/computer_sdk/index.d.ts
new file mode 100644
index 000000000..b940f0e02
--- /dev/null
+++ b/vibe/computer_sdk/index.d.ts
@@ -0,0 +1,139 @@
+export type EffectStatus =
+ | "confirmed"
+ | "partial"
+ | "unverifiable"
+ | "suspected_noop"
+ | "refused";
+
+export interface Effect {
+ status: EffectStatus;
+ effect: EffectStatus;
+ route?: string;
+ summary?: string | null;
+ delivery?: {
+ mode: "background" | "foreground" | "not_applicable" | "unknown";
+ delivered_count?: number | null;
+ } | null;
+ evidence?: Array<{ kind: string; detail?: string | null }> | null;
+ escalation?: { target: string; reason: string } | null;
+ error?: { code: string; hint?: string | null } | null;
+ [key: string]: unknown;
+}
+
+export interface WaitOptions {
+ timeout?: number;
+}
+
+export interface BaseSelector {
+ role?: string;
+ label?: string;
+ labelContains?: string;
+ value?: string | null;
+ index?: number;
+}
+
+export interface Selector extends BaseSelector {
+ within?: Selector;
+}
+
+export interface AppSelector extends BaseSelector {
+ bundleId?: string;
+ name?: string;
+ nameContains?: string;
+}
+
+export interface WindowSelector extends BaseSelector {
+ title?: string;
+ titleContains?: string;
+}
+
+export interface ActionOptions {
+ [key: string]: unknown;
+}
+
+export interface ElementExpectation {
+ toExist(options?: WaitOptions | number): Promise<void>;
+ toHaveValue(
+ value: string | null,
+ options?: WaitOptions | number,
+ ): Promise<void>;
+ toHaveLabel(label: string, options?: WaitOptions | number): Promise<void>;
+ toBeEnabled(options?: WaitOptions | number): Promise<void>;
+ toBeSelected(options?: WaitOptions | number): Promise<void>;
+}
+
+export interface ElementHandle {
+ expect(): ElementExpectation;
+ click(options?: ActionOptions): Promise<Effect>;
+ doubleClick(options?: ActionOptions): Promise<Effect>;
+ rightClick(options?: ActionOptions): Promise<Effect>;
+ fill(value: string): Promise<Effect>;
+ type(text: string, options?: ActionOptions): Promise<Effect>;
+ press(key: string, options?: ActionOptions): Promise<Effect>;
+ hotkey(keys: string[], options?: ActionOptions): Promise<Effect>;
+ scroll(direction: string, options?: ActionOptions): Promise<Effect>;
+}
+
+export interface WindowHandle {
+ readonly pid: number;
+ readonly windowId: number;
+ readonly title: string;
+ element(selector: Selector): ElementHandle;
+ expect(selector: Selector): ElementExpectation;
+ waitFor(
+ selector: Selector,
+ options?: WaitOptions | number,
+ ): Promise<ElementHandle>;
+ observe(options?: ActionOptions): Promise<Record<string, unknown>>;
+ click(options: ActionOptions): Promise<Effect>;
+ type(text: string, options?: ActionOptions): Promise<Effect>;
+ press(key: string, options?: ActionOptions): Promise<Effect>;
+ hotkey(keys: string[], options?: ActionOptions): Promise<Effect>;
+ scroll(direction: string, options?: ActionOptions): Promise<Effect>;
+ drag(options: ActionOptions): Promise<Effect>;
+}
+
+export interface AppHandle {
+ readonly pid: number | null;
+ readonly name: string;
+ readonly bundleId: string | null;
+ launch(options?: ActionOptions): Promise<LaunchEffect>;
+ forceQuit(): Promise<Effect>;
+ window(selector?: WindowSelector): Promise<WindowHandle>;
+ waitFor(
+ selector?: WindowSelector,
+ options?: WaitOptions | number,
+ ): Promise<WindowHandle>;
+}
+
+export interface LaunchEffect extends Effect {
+ app: AppHandle;
+}
+
+export interface Computer {
+ readonly clipboard: {
+ read(options?: ActionOptions): Promise<Record<string, unknown>>;
+ write(text: string): Promise<Effect>;
+ };
+ app(selector?: AppSelector): Promise<AppHandle>;
+ waitFor(
+ selector?: AppSelector,
+ options?: WaitOptions | number,
+ ): Promise<AppHandle>;
+}
+
+export class ControllerError extends Error {
+ readonly code: string;
+ readonly params: Record<string, unknown>;
+ readonly ambiguous: boolean;
+ readonly step?: string;
+}
+
+export class NotFound extends ControllerError {}
+export class Ambiguous extends ControllerError {}
+export class Timeout extends ControllerError {}
+export class DesktopBusy extends ControllerError {}
+export class ForegroundConsentRequired extends ControllerError {}
+export class Unavailable extends ControllerError {}
+
+export const computer: Computer;
diff --git a/vibe/computer_sdk/index.mjs b/vibe/computer_sdk/index.mjs
new file mode 100644
index 000000000..5fc4a8cb8
--- /dev/null
+++ b/vibe/computer_sdk/index.mjs
@@ -0,0 +1,967 @@
+import {
+ ControllerFault,
+ callTool,
+ controllerFailureAmbiguous,
+ runStep,
+ sleepWithinDeadline,
+} from "./runtime.mjs";
+
+const DEFAULT_WAIT_MS = 5000;
+const POLL_INTERVAL_MS = 100;
+const EFFECT_STATUSES = new Set([
+ "confirmed",
+ "partial",
+ "unverifiable",
+ "suspected_noop",
+ "refused",
+]);
+const STALE_ELEMENT_CODES = new Set([
+ "observe_first",
+ "element_not_found",
+ "element_token_invalid",
+ "element_token_stale",
+ "stale_element_token",
+ "snapshot_not_found",
+ "snapshot_stale",
+]);
+const NOT_FOUND_CODES = new Set([
+ "application_not_found",
+ "element_not_found",
+ "snapshot_not_found",
+ "window_id_not_found",
+ "window_not_found",
+]);
+const BASE_SELECTOR_FIELDS = new Set([
+ "role",
+ "label",
+ "labelContains",
+ "value",
+ "index",
+]);
+const APP_SELECTOR_FIELDS = new Set([
+ ...BASE_SELECTOR_FIELDS,
+ "bundleId",
+ "name",
+ "nameContains",
+]);
+const WINDOW_SELECTOR_FIELDS = new Set([
+ ...BASE_SELECTOR_FIELDS,
+ "title",
+ "titleContains",
+]);
+const ELEMENT_SELECTOR_FIELDS = new Set([
+ ...BASE_SELECTOR_FIELDS,
+ "within",
+]);
+const windowState = new WeakMap();
+
+function invalidHandleResponse(message) {
+ throw new ControllerError(message, {
+ code: "invalid_computer_use_response",
+ ambiguous: controllerFailureAmbiguous(true),
+ });
+}
+
+function appDescriptor(record) {
+ const bundleId =
+ typeof record.bundle_id === "string" && record.bundle_id
+ ? record.bundle_id
+ : null;
+ const name =
+ bundleId === null && typeof record.name === "string" && record.name
+ ? record.name
+ : null;
+ if (bundleId === null && name === null) {
+ invalidHandleResponse(
+ "The Controller returned an application without a stable identity.",
+ );
+ }
+ return Object.freeze({
+ kind: "app",
+ pid: Number.isInteger(record.pid) && record.pid > 0 ? record.pid : null,
+ bundle_id: bundleId,
+ name,
+ });
+}
+
+function windowDescriptor(app, record) {
+ const descriptor = app._descriptor;
+ if (
+ !descriptor ||
+ !Number.isInteger(descriptor.pid) ||
+ descriptor.pid <= 0 ||
+ record.pid !== descriptor.pid ||
+ !Number.isInteger(record.window_id) ||
+ record.window_id < 0
+ ) {
+ invalidHandleResponse(
+ "The Controller returned a window outside the selected application.",
+ );
+ }
+ return Object.freeze({
+ kind: "window",
+ pid: descriptor.pid,
+ window_id: record.window_id,
+ app: descriptor,
+ });
+}
+
+export class ControllerError extends Error {
+ constructor(message, options = {}) {
+ super(message);
+ this.name = "ControllerError";
+ this.code = options.code || "controller_error";
+ this.params = sanitizePublic(options.params || {});
+ this.ambiguous = options.ambiguous === true;
+ }
+}
+
+export class NotFound extends ControllerError {
+ constructor(message, options = {}) {
+ super(message, { ...options, code: options.code || "not_found" });
+ this.name = "NotFound";
+ }
+}
+
+export class Ambiguous extends ControllerError {
+ constructor(message, options = {}) {
+ super(message, { ...options, code: options.code || "ambiguous" });
+ this.name = "Ambiguous";
+ }
+}
+
+export class Timeout extends ControllerError {
+ constructor(message, options = {}) {
+ super(message, { ...options, code: options.code || "timeout" });
+ this.name = "Timeout";
+ }
+}
+
+export class DesktopBusy extends ControllerError {
+ constructor(message, options = {}) {
+ super(message, { ...options, code: "desktop_busy" });
+ this.name = "DesktopBusy";
+ }
+}
+
+export class ForegroundConsentRequired extends ControllerError {
+ constructor(message, options = {}) {
+ super(message, {
+ ...options,
+ code: options.code || "foreground_consent_required",
+ });
+ this.name = "ForegroundConsentRequired";
+ }
+}
+
+export class Unavailable extends ControllerError {
+ constructor(message, options = {}) {
+ super(message, { ...options, code: options.code || "unavailable" });
+ this.name = "Unavailable";
+ }
+}
+
+function publicError(error) {
+ if (error instanceof ControllerError) {
+ return error;
+ }
+ if (!(error instanceof ControllerFault)) {
+ return error;
+ }
+ const options = {
+ code: error.code,
+ params: error.params,
+ ambiguous: error.ambiguous,
+ };
+ if (error.code === "desktop_busy") {
+ return new DesktopBusy(error.message, options);
+ }
+ if (
+ error.code === "foreground_forbidden" ||
+ error.code === "focus_shortcut_forbidden" ||
+ error.code === "foreground_consent_required"
+ ) {
+ return new ForegroundConsentRequired(error.message, options);
+ }
+ if (
+ error.code === "timeout" ||
+ error.code === "script_timeout" ||
+ error.code === "controller_timeout"
+ ) {
+ return new Timeout(error.message, options);
+ }
+ if (NOT_FOUND_CODES.has(error.code)) {
+ return new NotFound(error.message, options);
+ }
+ if (
+ error.code === "computer_use_unavailable" ||
+ error.code === "platform_unsupported" ||
+ error.code === "upstream_unavailable" ||
+ error.code === "controller_unavailable"
+ ) {
+ return new Unavailable(error.message, options);
+ }
+ return new ControllerError(error.message, options);
+}
+
+async function sdkStep(label, operation) {
+ try {
+ return await runStep(label, operation);
+ } catch (error) {
+ const mapped = publicError(error);
+ if (mapped && typeof mapped === "object" && !mapped.step) {
+ mapped.step = label;
+ }
+ throw mapped;
+ }
+}
+
+function structured(result) {
+ return result &&
+ result.structuredContent &&
+ typeof result.structuredContent === "object"
+ ? result.structuredContent
+ : {};
+}
+
+function effectResult(result) {
+ const payload = sanitizePublic(structured(result));
+ const status = EFFECT_STATUSES.has(payload.effect)
+ ? payload.effect
+ : "unverifiable";
+ return { ...payload, effect: status, status };
+}
+
+function publicObservation(result) {
+ return sanitizePublic(structured(result));
+}
+
+function sanitizePublic(value, seen = new WeakSet()) {
+ if (Array.isArray(value)) {
+ if (seen.has(value)) {
+ return "[Circular]";
+ }
+ seen.add(value);
+ const result = value.map((item) => sanitizePublic(item, seen));
+ seen.delete(value);
+ return result;
+ }
+ if (!value || typeof value !== "object") {
+ return value;
+ }
+ if (seen.has(value)) {
+ return "[Circular]";
+ }
+ seen.add(value);
+ const result = Object.fromEntries(
+ Object.entries(value)
+ .filter(([key]) => key !== "element_token")
+ .map(([key, item]) => [key, sanitizePublic(item, seen)]),
+ );
+ seen.delete(value);
+ return result;
+}
+
+function normalizeRole(value) {
+ return String(value || "")
+ .replace(/^AX/i, "")
+ .toLowerCase();
+}
+
+function roleMatches(actual, expected) {
+ if (expected === undefined) {
+ return true;
+ }
+ const actualRole = normalizeRole(actual);
+ const expectedRole = normalizeRole(expected);
+ if (expectedRole === "textbox") {
+ return actualRole === "textfield" || actualRole === "textarea";
+ }
+ return actualRole === expectedRole;
+}
+
+function invalidSelector(noun, reason, field) {
+ throw new ControllerError(`${noun} selector ${reason}.`, {
+ code: "invalid_selector",
+ params: field === undefined ? {} : { field },
+ });
+}
+
+function validateSelector(selector, allowedFields, noun, depth = 0) {
+ if (
+ !selector ||
+ typeof selector !== "object" ||
+ Array.isArray(selector)
+ ) {
+ invalidSelector(noun, "must be an object");
+ }
+ if (depth > 8) {
+ invalidSelector(noun, "exceeds the supported within depth");
+ }
+ for (const field of Object.keys(selector)) {
+ if (!allowedFields.has(field)) {
+ invalidSelector(noun, `contains unknown field '${field}'`, field);
+ }
+ }
+ if (
+ selector.index !== undefined &&
+ (!Number.isInteger(selector.index) || selector.index < 0)
+ ) {
+ invalidSelector(noun, "index must be a non-negative integer", "index");
+ }
+ if (selector.within !== undefined) {
+ validateSelector(
+ selector.within,
+ ELEMENT_SELECTOR_FIELDS,
+ "Element",
+ depth + 1,
+ );
+ }
+ return selector;
+}
+
+function exactString(actual, expected) {
+ return expected === undefined || actual === expected;
+}
+
+function containsString(actual, expected) {
+ return (
+ expected === undefined ||
+ String(actual ?? "").toLowerCase().includes(String(expected).toLowerCase())
+ );
+}
+
+function baseMatches(record, selector, fields) {
+ return (
+ roleMatches(fields.role(record), selector.role) &&
+ exactString(fields.label(record), selector.label) &&
+ containsString(fields.label(record), selector.labelContains) &&
+ exactString(fields.value(record), selector.value)
+ );
+}
+
+function selectOne(records, selector, noun) {
+ let matches = records;
+ if (selector.index !== undefined) {
+ if (!Number.isInteger(selector.index) || selector.index < 0) {
+ invalidSelector(noun, "index must be a non-negative integer", "index");
+ }
+ matches =
+ selector.index < matches.length ? [matches[selector.index]] : [];
+ }
+ if (matches.length === 0) {
+ throw new NotFound(`No ${noun} matched the selector.`, {
+ params: { selector },
+ });
+ }
+ if (matches.length > 1) {
+ throw new Ambiguous(`${matches.length} ${noun}s matched the selector.`, {
+ params: { selector, count: matches.length },
+ });
+ }
+ return matches[0];
+}
+
+function appMatches(record, selector) {
+ return (
+ baseMatches(record, selector, {
+ role: () => "application",
+ label: (item) => item.name,
+ value: (item) => item.bundle_id,
+ }) &&
+ exactString(record.bundle_id, selector.bundleId) &&
+ exactString(record.name, selector.name) &&
+ containsString(record.name, selector.nameContains)
+ );
+}
+
+function windowMatches(record, selector) {
+ return (
+ baseMatches(record, selector, {
+ role: () => "AXWindow",
+ label: (item) => item.title,
+ value: (item) => item.title,
+ }) &&
+ exactString(record.title, selector.title) &&
+ containsString(record.title, selector.titleContains)
+ );
+}
+
+function selectWindow(records, selector) {
+ const matches = records.filter((item) => windowMatches(item, selector));
+ if (selector.index === undefined) {
+ return selectOne(matches, {}, "window");
+ }
+ if (selector.index >= matches.length) {
+ return selectOne([], {}, "window");
+ }
+ if (matches.length === 1) {
+ return matches[0];
+ }
+ const zIndexes = matches.map((item) => item.z_index);
+ if (
+ zIndexes.some((value) => !Number.isInteger(value)) ||
+ new Set(zIndexes).size !== zIndexes.length
+ ) {
+ throw new ControllerError(
+ "Window order is unavailable for this selector.",
+ {
+ code: "window_order_unavailable",
+ params: { count: matches.length },
+ },
+ );
+ }
+ return [...matches].sort((left, right) => right.z_index - left.z_index)[
+ selector.index
+ ];
+}
+
+function matchingElements(elements, selector, depth = 0) {
+ validateSelector(selector, ELEMENT_SELECTOR_FIELDS, "Element", depth);
+ const byIndex = new Map(
+ elements
+ .filter((item) => Number.isInteger(item.element_index))
+ .map((item) => [item.element_index, item]),
+ );
+ let withinIndexes;
+ if (selector.within !== undefined) {
+ const boundary = selectOne(
+ matchingElements(elements, selector.within, depth + 1),
+ {},
+ "within boundary",
+ );
+ withinIndexes = new Set([boundary.element_index]);
+ }
+ let matches = elements.filter((item) => {
+ if (
+ !baseMatches(item, selector, {
+ role: (record) => record.role,
+ label: (record) => record.label,
+ value: (record) => record.value,
+ })
+ ) {
+ return false;
+ }
+ if (!withinIndexes) {
+ return true;
+ }
+ const visited = new Set();
+ let parentIndex = item.parent_index;
+ while (Number.isInteger(parentIndex) && !visited.has(parentIndex)) {
+ if (withinIndexes.has(parentIndex)) {
+ return true;
+ }
+ visited.add(parentIndex);
+ parentIndex = byIndex.get(parentIndex)?.parent_index;
+ }
+ return false;
+ });
+ if (selector.index !== undefined) {
+ matches =
+ Number.isInteger(selector.index) &&
+ selector.index >= 0 &&
+ selector.index < matches.length
+ ? [matches[selector.index]]
+ : [];
+ }
+ return matches;
+}
+
+function selectElement(elements, selector) {
+ return selectOne(matchingElements(elements, selector), {}, "element");
+}
+
+function timeoutMs(value) {
+ const candidate =
+ typeof value === "number"
+ ? value
+ : value && typeof value === "object"
+ ? value.timeout
+ : undefined;
+ if (candidate === undefined) {
+ return DEFAULT_WAIT_MS;
+ }
+ if (!Number.isFinite(candidate) || candidate < 0) {
+ throw new Timeout("Timeout must be a non-negative number.", {
+ code: "invalid_timeout",
+ });
+ }
+ return candidate;
+}
+
+async function poll(label, timeoutValue, attempt) {
+ const duration = timeoutMs(timeoutValue);
+ const deadline = performance.now() + duration;
+ let firstAttempt = true;
+ while (true) {
+ const localRemaining = deadline - performance.now();
+ if (!firstAttempt && localRemaining <= 0) {
+ throw new Timeout(`${label} timed out after ${duration} ms.`, {
+ params: { timeout: duration },
+ });
+ }
+ const attemptDeadline =
+ firstAttempt && duration === 0 ? undefined : deadline;
+ firstAttempt = false;
+ try {
+ return await attempt(attemptDeadline);
+ } catch (error) {
+ if (!(error instanceof NotFound)) {
+ throw error;
+ }
+ }
+ await sleepWithinDeadline(
+ Math.min(POLL_INTERVAL_MS, Math.max(0, deadline - performance.now())),
+ );
+ }
+}
+
+async function callToolBefore(tool, argumentsValue, deadline, options = {}) {
+ if (deadline === undefined) {
+ return callTool(tool, argumentsValue, options);
+ }
+ const remaining = deadline - performance.now();
+ if (remaining <= 0) {
+ throw new Timeout("The operation deadline expired.");
+ }
+ return callTool(tool, argumentsValue, { ...options, timeoutMs: remaining });
+}
+
+function callHandleTool(handle, tool, argumentsValue, deadline) {
+ return callToolBefore(tool, argumentsValue, deadline, {
+ descriptor: handle._descriptor,
+ });
+}
+
+function shouldRetryElementAction(error) {
+ return (
+ error instanceof ControllerFault &&
+ error.dispatched === false &&
+ STALE_ELEMENT_CODES.has(error.code)
+ );
+}
+
+function windowArguments(handle, options = {}) {
+ return { ...options, pid: handle.pid, window_id: handle.windowId };
+}
+
+class Computer {
+ constructor() {
+ this.clipboard = Object.freeze({
+ read: (options = {}) =>
+ sdkStep("computer.clipboard.read", async () =>
+ structured(await callTool("clipboard_read", options)),
+ ),
+ write: (text) =>
+ sdkStep("computer.clipboard.write", async () =>
+ effectResult(await callTool("clipboard_write", { text })),
+ ),
+ });
+ }
+
+ async app(selector = {}) {
+ return sdkStep("computer.app", async () => this._resolveApp(selector));
+ }
+
+ async waitFor(selector = {}, options) {
+ return sdkStep("computer.waitFor", async () =>
+ poll("Application wait", options, (deadline) =>
+ this._resolveApp(selector, deadline),
+ ),
+ );
+ }
+
+ async _resolveApp(selector = {}, deadline) {
+ validateSelector(selector, APP_SELECTOR_FIELDS, "Application");
+ const apps = structured(await callToolBefore("list_apps", {}, deadline)).apps;
+ const records = Array.isArray(apps) ? apps : [];
+ const record = selectOne(
+ records.filter((item) => appMatches(item, selector)),
+ selector,
+ "application",
+ );
+ return new AppHandle(record);
+ }
+}
+
+class AppHandle {
+ constructor(record) {
+ const descriptor = appDescriptor(record);
+ this.pid = descriptor.pid;
+ this.name =
+ typeof record.name === "string" && record.name
+ ? record.name
+ : descriptor.name;
+ this.bundleId = descriptor.bundle_id;
+ Object.defineProperty(this, "_descriptor", { value: descriptor });
+ Object.freeze(this);
+ }
+
+ async launch(options = {}) {
+ return sdkStep("app.launch", async () => {
+ const descriptor = this._descriptor;
+ const argumentsValue = { ...options };
+ if (this.bundleId) {
+ argumentsValue.bundle_id = this.bundleId;
+ } else {
+ argumentsValue.name = this.name;
+ }
+ const result = await callHandleTool(this, "launch_app", argumentsValue);
+ const payload = structured(result);
+ const pid =
+ Number.isInteger(payload.pid) && payload.pid > 0 ? payload.pid : null;
+ let record;
+ if (pid === null) {
+ const apps = structured(await callTool("list_apps", {})).apps;
+ const records = Array.isArray(apps) ? apps : [];
+ record = selectOne(
+ records.filter(
+ (item) =>
+ Number.isInteger(item.pid) &&
+ item.pid > 0 &&
+ (descriptor.bundle_id !== null
+ ? item.bundle_id === descriptor.bundle_id
+ : item.name === descriptor.name),
+ ),
+ {},
+ "launched application",
+ );
+ } else {
+ record = {
+ pid,
+ bundle_id: descriptor.bundle_id,
+ name:
+ typeof payload.name === "string" && payload.name
+ ? payload.name
+ : this.name,
+ };
+ }
+ return {
+ ...effectResult(result),
+ app: new AppHandle(record),
+ };
+ });
+ }
+
+ async forceQuit() {
+ return sdkStep("app.forceQuit", async () => {
+ const descriptor = this._descriptor;
+ return effectResult(
+ await callHandleTool(this, "kill_app", { pid: descriptor.pid }),
+ );
+ });
+ }
+
+ async window(selector = {}) {
+ return sdkStep("app.window", async () => this._resolveWindow(selector));
+ }
+
+ async waitFor(selector = {}, options) {
+ return sdkStep("app.waitFor", async () =>
+ poll("Window wait", options, (deadline) =>
+ this._resolveWindow(selector, deadline),
+ ),
+ );
+ }
+
+ async _resolveWindow(selector, deadline) {
+ validateSelector(selector, WINDOW_SELECTOR_FIELDS, "Window");
+ const descriptor = this._descriptor;
+ const windows = structured(
+ await callHandleTool(this, "list_windows", { pid: descriptor.pid }, deadline),
+ ).windows;
+ const records = Array.isArray(windows) ? windows : [];
+ const record = selectWindow(records, selector);
+ return new WindowHandle(this, selector, record);
+ }
+}
+
+class WindowHandle {
+ constructor(app, _selector, record) {
+ windowState.set(this, { snapshot: undefined, snapshotKey: undefined });
+ const descriptor = windowDescriptor(app, record);
+ this.pid = descriptor.pid;
+ this.windowId = descriptor.window_id;
+ this.title = record.title;
+ Object.defineProperty(this, "_descriptor", { value: descriptor });
+ Object.freeze(this);
+ }
+
+ element(selector) {
+ return new ElementHandle(this, selector);
+ }
+
+ expect(selector) {
+ return new ElementExpectation(this, selector);
+ }
+
+ async waitFor(selector, options) {
+ return sdkStep("window.waitFor", async () => {
+ await poll("Element wait", options, (deadline) =>
+ resolveWindowElement(this, selector, true, deadline),
+ );
+ return new ElementHandle(this, selector);
+ });
+ }
+
+ async observe(options = {}) {
+ return sdkStep("window.observe", async () =>
+ publicObservation(await observeWindow(this, true, options)),
+ );
+ }
+
+ async click(options) {
+ return this._windowAction("window.click", "click", options);
+ }
+
+ async type(text, options = {}) {
+ return this._windowAction("window.type", "type_text", {
+ ...options,
+ text,
+ });
+ }
+
+ async press(key, options = {}) {
+ return this._windowAction("window.press", "press_key", {
+ ...options,
+ key,
+ });
+ }
+
+ async hotkey(keys, options = {}) {
+ return this._windowAction("window.hotkey", "hotkey", {
+ ...options,
+ keys,
+ });
+ }
+
+ async scroll(direction, options = {}) {
+ return this._windowAction("window.scroll", "scroll", {
+ ...options,
+ direction,
+ });
+ }
+
+ async drag(options) {
+ return this._windowAction("window.drag", "drag", options);
+ }
+
+ async _windowAction(label, tool, options = {}) {
+ return sdkStep(label, async () => {
+ await observeWindow(this, false);
+ return effectResult(
+ await callHandleTool(this, tool, windowArguments(this, options)),
+ );
+ });
+ }
+}
+
+async function observeWindow(handle, force, options = {}, deadline) {
+ const state = windowState.get(handle);
+ const descriptor = handle._descriptor;
+ const key = `${descriptor.pid}:${descriptor.window_id}`;
+ if (!force && state.snapshot && state.snapshotKey === key) {
+ return state.snapshot;
+ }
+ state.snapshot = await callHandleTool(
+ handle,
+ "get_window_state",
+ windowArguments(handle, options),
+ deadline,
+ );
+ state.snapshotKey = key;
+ return state.snapshot;
+}
+
+async function resolveWindowElement(handle, selector, force, deadline) {
+ validateSelector(selector, ELEMENT_SELECTOR_FIELDS, "Element");
+ const observation = await observeWindow(handle, force, {}, deadline);
+ const elements = structured(observation).elements;
+ return selectElement(Array.isArray(elements) ? elements : [], selector);
+}
+
+async function runElementAction(handle, selector, label, tool, options = {}) {
+ return sdkStep(label, async () => {
+ let element = await resolveWindowElement(handle, selector, true);
+ if (typeof element.element_token !== "string" || !element.element_token) {
+ throw new NotFound("The matched element is not actionable.", {
+ code: "element_not_actionable",
+ });
+ }
+ const invoke = () =>
+ callHandleTool(
+ handle,
+ tool,
+ windowArguments(handle, {
+ ...options,
+ element_token: element.element_token,
+ }),
+ );
+ try {
+ return effectResult(await invoke());
+ } catch (error) {
+ if (!shouldRetryElementAction(error)) {
+ throw error;
+ }
+ element = await resolveWindowElement(handle, selector, true);
+ if (typeof element.element_token !== "string" || !element.element_token) {
+ throw new NotFound("The matched element is not actionable.", {
+ code: "element_not_actionable",
+ });
+ }
+ return effectResult(await invoke());
+ }
+ });
+}
+
+class ElementHandle {
+ constructor(window, selector) {
+ this._window = window;
+ this._selector = selector;
+ }
+
+ expect() {
+ return new ElementExpectation(this._window, this._selector);
+ }
+
+ click(options = {}) {
+ return runElementAction(
+ this._window,
+ this._selector,
+ "element.click",
+ "click",
+ options,
+ );
+ }
+
+ doubleClick(options = {}) {
+ return runElementAction(
+ this._window,
+ this._selector,
+ "element.doubleClick",
+ "double_click",
+ options,
+ );
+ }
+
+ rightClick(options = {}) {
+ return runElementAction(
+ this._window,
+ this._selector,
+ "element.rightClick",
+ "click",
+ { ...options, button: "right" },
+ );
+ }
+
+ fill(value) {
+ return runElementAction(
+ this._window,
+ this._selector,
+ "element.fill",
+ "set_value",
+ { value },
+ );
+ }
+
+ type(text, options = {}) {
+ return runElementAction(
+ this._window,
+ this._selector,
+ "element.type",
+ "type_text",
+ { ...options, text },
+ );
+ }
+
+ press(key, options = {}) {
+ return runElementAction(
+ this._window,
+ this._selector,
+ "element.press",
+ "press_key",
+ { ...options, key },
+ );
+ }
+
+ hotkey(keys, options = {}) {
+ return runElementAction(
+ this._window,
+ this._selector,
+ "element.hotkey",
+ "hotkey",
+ { ...options, keys },
+ );
+ }
+
+ scroll(direction, options = {}) {
+ return runElementAction(
+ this._window,
+ this._selector,
+ "element.scroll",
+ "scroll",
+ { ...options, direction },
+ );
+ }
+}
+
+class ElementExpectation {
+ constructor(window, selector) {
+ this._window = window;
+ this._selector = selector;
+ }
+
+ toExist(options) {
+ return this._check("expect.toExist", options, () => true);
+ }
+
+ toHaveValue(value, options) {
+ return this._check(
+ "expect.toHaveValue",
+ options,
+ (element) => element.value === value,
+ );
+ }
+
+ toHaveLabel(label, options) {
+ return this._check(
+ "expect.toHaveLabel",
+ options,
+ (element) => String(element.label ?? "") === String(label),
+ );
+ }
+
+ toBeEnabled(options) {
+ return this._check(
+ "expect.toBeEnabled",
+ options,
+ (element) => element.enabled === true,
+ );
+ }
+
+ toBeSelected(options) {
+ return this._check(
+ "expect.toBeSelected",
+ options,
+ (element) => element.selected === true,
+ );
+ }
+
+ async _check(label, options, predicate) {
+ return sdkStep(label, async () =>
+ poll("Expectation", options, async (deadline) => {
+ const element = await resolveWindowElement(
+ this._window,
+ this._selector,
+ true,
+ deadline,
+ );
+ if (!predicate(element)) {
+ throw new NotFound("The expectation is not satisfied yet.");
+ }
+ }),
+ );
+ }
+}
+
+export const computer = Object.freeze(new Computer());
diff --git a/vibe/computer_sdk/loader.mjs b/vibe/computer_sdk/loader.mjs
new file mode 100644
index 000000000..ef5234d96
--- /dev/null
+++ b/vibe/computer_sdk/loader.mjs
@@ -0,0 +1,15 @@
+let sdkUrl;
+
+export function initialize(data) {
+ sdkUrl = data?.sdkUrl;
+ if (typeof sdkUrl !== "string" || !sdkUrl.startsWith("file:")) {
+ throw new Error("invalid @avibe/computer module URL");
+ }
+}
+
+export async function resolve(specifier, context, nextResolve) {
+ if (specifier === "@avibe/computer") {
+ return { url: sdkUrl, shortCircuit: true };
+ }
+ return nextResolve(specifier, context);
+}
diff --git a/vibe/computer_sdk/package.json b/vibe/computer_sdk/package.json
new file mode 100644
index 000000000..799625f5f
--- /dev/null
+++ b/vibe/computer_sdk/package.json
@@ -0,0 +1,8 @@
+{
+ "name": "@avibe/computer",
+ "version": "1.0.0",
+ "type": "module",
+ "exports": "./index.mjs",
+ "types": "./index.d.ts",
+ "private": true
+}
diff --git a/vibe/computer_sdk/runner.mjs b/vibe/computer_sdk/runner.mjs
new file mode 100644
index 000000000..439428a4e
--- /dev/null
+++ b/vibe/computer_sdk/runner.mjs
@@ -0,0 +1,96 @@
+import { register } from "node:module";
+import * as runtime from "./runtime.mjs";
+
+const resultPath = process.env.AVIBE_COMPUTER_RESULT_PATH;
+let output;
+
+try {
+ if (typeof resultPath !== "string" || !resultPath) {
+ throw new Error("missing Computer Use runner result path");
+ }
+ const input = await readStdin();
+ register(new URL("./loader.mjs", import.meta.url), {
+ data: { sdkUrl: input.sdk_url },
+ });
+
+ runtime.configureRuntime(input);
+
+ try {
+ const keepAlive = setInterval(() => {}, 1000);
+ let result;
+ try {
+ const module = await import(input.script_url);
+ if (typeof module.default !== "function") {
+ throw new Error("Computer Use scripts must default-export a function.");
+ }
+ result = await module.default(input.arguments);
+ } finally {
+ clearInterval(keepAlive);
+ }
+ const serializedResult = JSON.stringify(result ?? null);
+ if (serializedResult === undefined) {
+ throw new Error("Computer Use scripts must return a JSON value.");
+ }
+ output = { ok: true, result: JSON.parse(serializedResult) };
+ } catch (error) {
+ output = {
+ ok: false,
+ error: runtime.failureEnvelope(error),
+ };
+ } finally {
+ await runtime.closeScriptConnection().catch(() => {});
+ }
+} catch (error) {
+ output = {
+ ok: false,
+ error: runtime.failureEnvelope(error, {
+ defaultCode: "computer_script_runtime_failed",
+ }),
+ };
+}
+
+try {
+ runtime.writeCappedJsonFile(resultPath, output);
+ process.exit(0);
+} catch (error) {
+ if (error instanceof runtime.ProtocolFileTooLargeError) {
+ const protocolError = new Error(
+ "The script result exceeded the protocol size limit.",
+ );
+ protocolError.name = "ControllerError";
+ protocolError.code = "computer_script_protocol_error";
+ protocolError.params = { limit_bytes: error.limitBytes };
+ protocolError.ambiguous = true;
+ try {
+ runtime.writeCappedJsonFile(resultPath, {
+ ok: false,
+ error: runtime.failureEnvelope(protocolError),
+ });
+ process.exit(0);
+ } catch (fallbackError) {
+ console.error(fallbackError);
+ }
+ } else {
+ console.error(error);
+ }
+ process.exit(1);
+}
+
+async function readStdin() {
+ const chunks = [];
+ for await (const chunk of process.stdin) {
+ chunks.push(chunk);
+ }
+ const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+ if (
+ !parsed ||
+ typeof parsed !== "object" ||
+ typeof parsed.script_url !== "string" ||
+ typeof parsed.sdk_url !== "string" ||
+ !parsed.arguments ||
+ typeof parsed.arguments !== "object"
+ ) {
+ throw new Error("invalid Computer Use runner input");
+ }
+ return parsed;
+}
diff --git a/vibe/computer_sdk/runtime.mjs b/vibe/computer_sdk/runtime.mjs
new file mode 100644
index 000000000..da81bfcca
--- /dev/null
+++ b/vibe/computer_sdk/runtime.mjs
@@ -0,0 +1,616 @@
+import fs from "node:fs";
+import http from "node:http";
+import path from "node:path";
+import { inspect } from "node:util";
+
+const MAX_RESPONSE_BYTES = 128 * 1024 * 1024;
+export const PROTOCOL_FILE_MAX_BYTES = 8 * 1024 * 1024;
+export const DIAGNOSTIC_TAIL_MAX_BYTES = 64 * 1024;
+
+const failureEnvelopeContract = JSON.parse(
+ fs.readFileSync(new URL("./failure-envelope.json", import.meta.url), "utf8"),
+);
+if (
+ !Array.isArray(failureEnvelopeContract.fields) ||
+ failureEnvelopeContract.fields.some(
+ (field) => typeof field !== "string" || !field,
+ ) ||
+ new Set(failureEnvelopeContract.fields).size !==
+ failureEnvelopeContract.fields.length
+) {
+ throw new Error("invalid failure envelope contract");
+}
+export const FAILURE_ENVELOPE_FIELDS = Object.freeze(
+ failureEnvelopeContract.fields,
+);
+
+let state;
+let agent;
+let currentStep = "script";
+let lastCompletedStep;
+let lastScreenshotPath;
+let lastOwnedScreenshotPath;
+let screenshotSequence = 0;
+let diagnosticTail = Buffer.alloc(0);
+let consoleDiagnosticsInstalled = false;
+
+export class ControllerFault extends Error {
+ constructor(code, message, params = {}, options = {}) {
+ super(message);
+ this.name = "ControllerFault";
+ this.code = code;
+ this.params = params;
+ this.ambiguous = options.ambiguous === true;
+ this.dispatched = options.dispatched === false ? false : undefined;
+ }
+}
+
+function controllerFailureState(requestWasSent, controllerPayload) {
+ const explicitlyNotDispatched =
+ controllerPayload &&
+ typeof controllerPayload === "object" &&
+ controllerPayload.dispatched === false;
+ return {
+ ambiguous: requestWasSent === true && !explicitlyNotDispatched,
+ dispatched: explicitlyNotDispatched ? false : undefined,
+ };
+}
+
+export function controllerFailureAmbiguous(
+ requestWasSent,
+ controllerPayload,
+) {
+ return controllerFailureState(requestWasSent, controllerPayload).ambiguous;
+}
+
+function controllerFault(
+ code,
+ message,
+ params = {},
+ { requestWasSent = false, controllerPayload } = {},
+) {
+ const failureState = controllerFailureState(
+ requestWasSent,
+ controllerPayload,
+ );
+ return new ControllerFault(code, message, params, {
+ ...failureState,
+ });
+}
+
+export function configureRuntime(config) {
+ if (state !== undefined) {
+ throw new Error("computer SDK runtime is already configured");
+ }
+ if (
+ !config ||
+ typeof config.connection_token !== "string" ||
+ !config.connection_token ||
+ typeof config.artifact_directory !== "string" ||
+ typeof config.runtime_state_path !== "string"
+ ) {
+ throw new Error("invalid computer SDK runtime configuration");
+ }
+ validateEndpoint(config.endpoint);
+ state = {
+ endpoint: config.endpoint,
+ connectionToken: config.connection_token,
+ artifactDirectory: config.artifact_directory,
+ runtimeStatePath: config.runtime_state_path,
+ };
+ agent = new http.Agent({
+ keepAlive: true,
+ maxSockets: 1,
+ maxFreeSockets: 1,
+ scheduling: "fifo",
+ });
+ installConsoleDiagnostics();
+ persistRuntimeState();
+}
+
+function validateEndpoint(endpoint) {
+ if (!endpoint || typeof endpoint !== "object") {
+ throw new Error("invalid Controller endpoint");
+ }
+ if (
+ endpoint.transport === "unix" &&
+ typeof endpoint.socket_path === "string" &&
+ endpoint.socket_path
+ ) {
+ return;
+ }
+ if (
+ endpoint.transport === "tcp" &&
+ typeof endpoint.host === "string" &&
+ Number.isInteger(endpoint.port) &&
+ endpoint.port > 0 &&
+ endpoint.headers &&
+ typeof endpoint.headers === "object" &&
+ typeof endpoint.instance_id === "string" &&
+ typeof endpoint.instance_header === "string"
+ ) {
+ return;
+ }
+ throw new Error("invalid Controller endpoint");
+}
+
+export async function sleepWithinDeadline(milliseconds) {
+ await new Promise((resolve) =>
+ setTimeout(resolve, Math.max(0, milliseconds)),
+ );
+}
+
+export async function runStep(label, operation) {
+ currentStep = label;
+ persistRuntimeState();
+ try {
+ const result = await operation();
+ lastCompletedStep = label;
+ return result;
+ } catch (error) {
+ if (error && typeof error === "object" && !error.step) {
+ error.step = label;
+ }
+ throw error;
+ } finally {
+ currentStep = "script";
+ persistRuntimeState();
+ }
+}
+
+export async function callTool(tool, argumentsValue, options = {}) {
+ const body = await postJson(
+ "/internal/computer-script/request",
+ {
+ connection_token: state.connectionToken,
+ tool,
+ arguments: argumentsValue,
+ descriptor: options.descriptor ?? null,
+ },
+ options,
+ );
+ const result = body?.result;
+ if (result && typeof result === "object") {
+ recordScreenshot(result);
+ if (result.isError === true) {
+ const structured = result.structuredContent;
+ const code =
+ structured && typeof structured.code === "string"
+ ? structured.code
+ : "computer_use_error";
+ const params =
+ structured && structured.params && typeof structured.params === "object"
+ ? structured.params
+ : {};
+ throw controllerFault(code, controllerMessage(code, structured), params, {
+ requestWasSent: true,
+ controllerPayload: structured,
+ });
+ }
+ return result;
+ }
+ if (body?.ok === false) {
+ const code =
+ typeof body.error === "string" ? body.error : "computer_script_request_failed";
+ throw controllerFault(code, controllerMessage(code, body), body.params, {
+ requestWasSent: true,
+ controllerPayload: body,
+ });
+ }
+ throw controllerFault(
+ "invalid_computer_use_response",
+ "The Controller returned an invalid Computer Use response.",
+ {},
+ { requestWasSent: true },
+ );
+}
+
+function controllerMessage(code, payload) {
+ if (payload && typeof payload.detail === "string" && payload.detail) {
+ return payload.detail;
+ }
+ return `Controller error: ${code}`;
+}
+
+function recordScreenshot(result) {
+ const structured = result.structuredContent;
+ if (
+ structured &&
+ typeof structured === "object" &&
+ typeof structured.screenshot_file_path === "string" &&
+ structured.screenshot_file_path
+ ) {
+ replaceDiagnosticScreenshot(structured.screenshot_file_path);
+ return;
+ }
+ const image = Array.isArray(result.content)
+ ? result.content.find(
+ (item) =>
+ item &&
+ item.type === "image" &&
+ typeof item.data === "string" &&
+ (!item.mimeType || item.mimeType === "image/png"),
+ )
+ : undefined;
+ if (!image) {
+ return;
+ }
+ try {
+ screenshotSequence += 1;
+ const target = path.join(
+ state.artifactDirectory,
+ `observation-${screenshotSequence}.png`,
+ );
+ fs.writeFileSync(target, Buffer.from(image.data, "base64"), { mode: 0o600 });
+ replaceDiagnosticScreenshot(target, { owned: true });
+ } catch {
+ // A screenshot artifact is diagnostic only; the structured observation
+ // remains authoritative when the local file cannot be written.
+ }
+}
+
+function replaceDiagnosticScreenshot(target, options = {}) {
+ const previousOwned = lastOwnedScreenshotPath;
+ lastScreenshotPath = target;
+ lastOwnedScreenshotPath = options.owned === true ? target : undefined;
+ persistRuntimeState();
+ if (previousOwned && previousOwned !== lastOwnedScreenshotPath) {
+ try {
+ fs.unlinkSync(previousOwned);
+ } catch {
+ // The current screenshot remains valid even if an old diagnostic file
+ // was already removed or cannot be cleaned up.
+ }
+ }
+}
+
+async function postJson(requestPath, payload, requestOptions = {}) {
+ requireConfigured();
+ const localTimeout =
+ Number.isFinite(requestOptions.timeoutMs) && requestOptions.timeoutMs >= 0
+ ? requestOptions.timeoutMs
+ : undefined;
+ if (localTimeout !== undefined && localTimeout <= 0) {
+ throw requestTimeoutFault(false);
+ }
+ const encoded = Buffer.from(JSON.stringify(payload));
+ const endpoint = state.endpoint;
+ const headers = {
+ ...(endpoint.transport === "tcp" ? endpoint.headers : {}),
+ "Content-Type": "application/json",
+ "Content-Length": String(encoded.length),
+ Connection: "keep-alive",
+ };
+ const transportOptions =
+ endpoint.transport === "unix"
+ ? {
+ socketPath: endpoint.socket_path,
+ path: requestPath,
+ method: "POST",
+ headers,
+ agent,
+ }
+ : {
+ hostname: endpoint.host,
+ port: endpoint.port,
+ path: requestPath,
+ method: "POST",
+ headers,
+ agent,
+ };
+
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ let timer;
+ const requestState = { sent: false };
+ const settle = (callback, value) => {
+ if (settled) {
+ return;
+ }
+ settled = true;
+ clearTimeout(timer);
+ callback(value);
+ };
+ let request;
+ try {
+ request = http.request(transportOptions, (response) => {
+ if (endpoint.transport === "tcp") {
+ const actual = response.headers[endpoint.instance_header.toLowerCase()];
+ if (actual !== endpoint.instance_id) {
+ response.resume();
+ settle(
+ reject,
+ controllerFault(
+ "controller_unavailable",
+ "The Controller response came from a stale instance.",
+ {},
+ { requestWasSent: requestState.sent },
+ ),
+ );
+ return;
+ }
+ }
+ const chunks = [];
+ let size = 0;
+ response.on("data", (chunk) => {
+ size += chunk.length;
+ if (size > MAX_RESPONSE_BYTES) {
+ response.destroy(
+ controllerFault(
+ "invalid_computer_use_response",
+ "The Controller response exceeded the supported size.",
+ {},
+ { requestWasSent: requestState.sent },
+ ),
+ );
+ return;
+ }
+ chunks.push(chunk);
+ });
+ response.on("end", () => {
+ let body;
+ try {
+ body = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+ } catch {
+ settle(
+ reject,
+ controllerFault(
+ "invalid_computer_use_response",
+ "The Controller returned invalid JSON.",
+ {},
+ { requestWasSent: requestState.sent },
+ ),
+ );
+ return;
+ }
+ settle(resolve, body);
+ });
+ response.on("error", (error) =>
+ settle(
+ reject,
+ controllerTransportFault(error, requestState.sent),
+ ),
+ );
+ });
+ } catch (error) {
+ settle(reject, controllerTransportFault(error, requestState.sent));
+ return;
+ }
+ if (localTimeout !== undefined) {
+ timer = setTimeout(() => {
+ request.destroy(requestTimeoutFault(requestState.sent));
+ }, localTimeout);
+ }
+ request.on("finish", () => {
+ requestState.sent = true;
+ });
+ request.on("error", (error) => {
+ settle(reject, controllerTransportFault(error, requestState.sent));
+ });
+ request.end(encoded);
+ });
+}
+
+function controllerTransportFault(error, requestWasSent) {
+ if (error instanceof ControllerFault) {
+ return error;
+ }
+ const message =
+ error && typeof error === "object" && typeof error.message === "string"
+ ? error.message
+ : String(error);
+ return controllerFault(
+ "controller_unavailable",
+ `The Controller connection failed: ${message}`,
+ {},
+ { requestWasSent },
+ );
+}
+
+function requestTimeoutFault(requestWasSent) {
+ return controllerFault(
+ "timeout",
+ "The Computer Use request exceeded the operation timeout.",
+ {},
+ { requestWasSent },
+ );
+}
+
+export async function closeScriptConnection() {
+ if (!state || !agent) {
+ return;
+ }
+ try {
+ await postJson(
+ "/internal/computer-script/close",
+ {
+ connection_token: state.connectionToken,
+ },
+ { timeoutMs: 500 },
+ );
+ } finally {
+ agent.destroy();
+ }
+}
+
+function installConsoleDiagnostics() {
+ if (consoleDiagnosticsInstalled) {
+ return;
+ }
+ consoleDiagnosticsInstalled = true;
+ for (const name of ["debug", "error", "info", "log", "trace", "warn"]) {
+ const original = console[name];
+ if (typeof original !== "function") {
+ continue;
+ }
+ console[name] = (...args) => {
+ try {
+ original.apply(console, args);
+ } finally {
+ appendDiagnostic(args);
+ }
+ };
+ }
+}
+
+function appendDiagnostic(args) {
+ const line = args.map(formatDiagnosticValue).join(" ");
+ const encoded = Buffer.from(`${line}\n`, "utf8");
+ if (encoded.length >= DIAGNOSTIC_TAIL_MAX_BYTES) {
+ diagnosticTail = Buffer.from(
+ encoded.subarray(encoded.length - DIAGNOSTIC_TAIL_MAX_BYTES),
+ );
+ } else {
+ const combined = Buffer.concat([diagnosticTail, encoded]);
+ diagnosticTail = Buffer.from(
+ combined.subarray(
+ Math.max(0, combined.length - DIAGNOSTIC_TAIL_MAX_BYTES),
+ ),
+ );
+ }
+ persistRuntimeState();
+}
+
+function formatDiagnosticValue(value) {
+ if (typeof value === "string") {
+ return value.slice(-DIAGNOSTIC_TAIL_MAX_BYTES);
+ }
+ if (value instanceof Error) {
+ return String(value.stack || value.message || value);
+ }
+ try {
+ return inspect(value, {
+ breakLength: Infinity,
+ colors: false,
+ compact: true,
+ depth: 4,
+ maxArrayLength: 100,
+ maxStringLength: 4096,
+ });
+ } catch {
+ return "<unprintable>";
+ }
+}
+
+export class ProtocolFileTooLargeError extends Error {
+ constructor(limitBytes) {
+ super("Computer Use protocol file exceeded its size limit.");
+ this.name = "ProtocolFileTooLargeError";
+ this.code = "computer_script_protocol_error";
+ this.params = { limit_bytes: limitBytes };
+ this.limitBytes = limitBytes;
+ }
+}
+
+export function writeCappedJsonFile(
+ targetPath,
+ payload,
+ limitBytes = PROTOCOL_FILE_MAX_BYTES,
+) {
+ const serialized = JSON.stringify(payload);
+ if (serialized === undefined) {
+ throw new Error("Computer Use protocol payload is not JSON serializable.");
+ }
+ const encoded = Buffer.from(serialized, "utf8");
+ if (encoded.length > limitBytes) {
+ throw new ProtocolFileTooLargeError(limitBytes);
+ }
+ const temporaryPath = `${targetPath}.${process.pid}.tmp`;
+ try {
+ fs.writeFileSync(temporaryPath, encoded, {
+ flag: "w",
+ mode: 0o600,
+ });
+ fs.renameSync(temporaryPath, targetPath);
+ } catch (error) {
+ try {
+ fs.unlinkSync(temporaryPath);
+ } catch {
+ // The original write failure is authoritative.
+ }
+ throw error;
+ }
+}
+
+function persistRuntimeState() {
+ if (!state?.runtimeStatePath) {
+ return;
+ }
+ try {
+ writeCappedJsonFile(
+ state.runtimeStatePath,
+ {
+ step: currentStep || "script",
+ last_completed_step: lastCompletedStep ?? null,
+ screenshot_path: lastScreenshotPath ?? null,
+ diagnostics: diagnosticTail.toString("utf8"),
+ },
+ PROTOCOL_FILE_MAX_BYTES,
+ );
+ } catch {
+ // Runtime state is diagnostic; script execution remains authoritative.
+ }
+}
+
+export function failureContext(error) {
+ return {
+ step:
+ error && typeof error === "object" && typeof error.step === "string"
+ ? error.step
+ : currentStep || "script",
+ last_completed_step: lastCompletedStep ?? null,
+ screenshot_path:
+ error &&
+ typeof error === "object" &&
+ (typeof error.screenshot_path === "string" ||
+ typeof error.screenshotPath === "string")
+ ? error.screenshot_path || error.screenshotPath
+ : lastScreenshotPath,
+ };
+}
+
+export function failureEnvelope(error, options = {}) {
+ const context = failureContext(error);
+ const params =
+ error &&
+ typeof error === "object" &&
+ error.params &&
+ typeof error.params === "object" &&
+ !Array.isArray(error.params)
+ ? error.params
+ : {};
+ const values = {
+ name:
+ error && typeof error === "object" && typeof error.name === "string"
+ ? error.name
+ : "Error",
+ code:
+ error && typeof error === "object" && typeof error.code === "string"
+ ? error.code
+ : options.defaultCode || "computer_script_failed",
+ message:
+ error && typeof error === "object" && typeof error.message === "string"
+ ? error.message
+ : String(error),
+ params,
+ step: context.step,
+ last_completed_step: context.last_completed_step,
+ screenshot_path: context.screenshot_path ?? null,
+ ambiguous:
+ error && typeof error === "object" && error.ambiguous === true,
+ };
+ for (const field of FAILURE_ENVELOPE_FIELDS) {
+ if (!Object.hasOwn(values, field)) {
+ throw new Error(`failure envelope field is not implemented: ${field}`);
+ }
+ }
+ return Object.fromEntries(
+ FAILURE_ENVELOPE_FIELDS.map((field) => [field, values[field]]),
+ );
+}
+
+function requireConfigured() {
+ if (!state || !agent) {
+ throw new Error("computer SDK runtime is not configured");
+ }
+}
diff --git a/vibe/i18n/en.json b/vibe/i18n/en.json
index 0ab929719..594fb9152 100644
--- a/vibe/i18n/en.json
+++ b/vibe/i18n/en.json
@@ -862,6 +862,10 @@
"arg": "Override one driver argument; JSON values are parsed when valid.",
"output": "Caller-selected screenshot path for 'observe window'.",
"json": "Print the structured service response.",
+ "run": "Run a JavaScript Computer Use script",
+ "runDescription": "Run an ESM file with the shipped @avibe/computer module, bound to this exact Avibe Agent Turn.",
+ "scriptFile": "Path to a readable .mjs script.",
+ "scriptArg": "Pass one script argument as name=value or name=@file. Values remain strings.",
"observe": "Observe apps, windows, or one exact window",
"observeApps": "List visible applications",
"observeWindows": "List windows",
@@ -890,6 +894,36 @@
"unsupportedCommand": "Unsupported Computer Use command: {command}",
"screenshotMissing": "No screenshot is available for this result. The window may be hidden or minimized, or capture may be unavailable.",
"outputUnsupported": "--output is supported only by observe window or zoom.",
+ "scriptArgNameValue": "Script --arg values must use name=value.",
+ "scriptArgDuplicate": "Script argument '{name}' was provided more than once.",
+ "scriptArgFile": "Cannot read script argument file {path}: {error}",
+ "scriptFile": "Cannot run Computer Use script {path}: {error}",
+ "scriptFileType": "Computer Use script {path} must be a readable .mjs file.",
+ "scriptNodeUnavailable": "Computer Use scripts require the Node runtime shipped with the Avibe desktop app.",
+ "scriptSdkUnavailable": "The shipped @avibe/computer module is unavailable. Reinstall or update the Avibe desktop app.",
+ "scriptOpenRejected": "The Controller rejected the Computer Use script connection ({code}).",
+ "scriptRuntime": "The Computer Use script runtime failed: {error}",
+ "scriptRuntimeExit": "The Computer Use script runtime exited with status {status} without returning a result.",
+ "scriptRuntimeInvalid": "The shipped Computer Use script runtime returned an invalid result.",
+ "scriptTimedOut": "The Computer Use script exceeded its total {seconds}-second timeout.",
+ "scriptFailed": "Computer Use script failed at {step}: {error}",
+ "scriptAmbiguous": "The selector matched more than one target.",
+ "scriptConnectionInvalid": "The script's Controller connection is no longer valid.",
+ "scriptControllerRejected": "The Controller rejected the script request.",
+ "scriptRequestTooLarge": "The Computer Use script request exceeded the Controller size limit.",
+ "scriptControllerUnavailable": "The Controller is unavailable.",
+ "scriptDeadlineExpired": "The script deadline expired.",
+ "scriptElementNotActionable": "The matched element is not actionable.",
+ "scriptInvalidControllerResponse": "The Controller returned an invalid Computer Use response.",
+ "scriptInvalidDescriptor": "The Computer Use script used an invalid or missing handle descriptor.",
+ "scriptInvalidSelector": "The script used an invalid selector.",
+ "scriptInvalidTimeout": "The script used an invalid timeout.",
+ "scriptOperationTimedOut": "The Computer Use operation timed out.",
+ "scriptStaleAppHandle": "The selected application process is no longer available.",
+ "scriptStaleWindowHandle": "The selected application window is no longer available.",
+ "scriptTargetNotFound": "The requested Computer Use target was not found.",
+ "scriptTurnEnded": "The Avibe Turn that launched this script has ended.",
+ "scriptWindowOrderUnavailable": "Window order is unavailable for this selector.",
"missingCallerSession": "Computer Use requires an Avibe Agent caller Session.",
"missingCallerTurn": "Computer Use requires the caller's Avibe Session to have an active Turn.",
"serviceUnavailable": "Avibe service is unavailable: {error}",
@@ -902,6 +936,7 @@
"desktopLeaseLost": "The desktop lease was lost before Computer Use completed.",
"focusShortcutForbidden": "This focus shortcut is not allowed. Use launch with a URL instead.",
"foregroundForbidden": "Foreground input is not allowed.",
+ "foregroundConsentRequired": "Foreground input requires owner consent.",
"genericDriverError": "The Computer Use driver returned an error.",
"invalidDriverArguments": "The Computer Use arguments are invalid.",
"observeFirst": "Observe the exact window again before sending input.",
diff --git a/vibe/i18n/zh.json b/vibe/i18n/zh.json
index 3d3006d9e..e1388a7a2 100644
--- a/vibe/i18n/zh.json
+++ b/vibe/i18n/zh.json
@@ -862,6 +862,10 @@
"arg": "覆盖一个 driver 参数;有效 JSON 值会按 JSON 解析。",
"output": "调用方选择的截图路径,用于“observe window”。",
"json": "输出结构化服务响应。",
+ "run": "运行 JavaScript Computer Use 脚本",
+ "runDescription": "使用随 Avibe 发货的 @avibe/computer 模块运行 ESM 文件,并绑定到当前这个 Avibe Agent Turn。",
+ "scriptFile": "可读的 .mjs 脚本路径。",
+ "scriptArg": "以 name=value 或 name=@file 传入脚本参数;值保持为字符串。",
"observe": "观察应用、窗口或一个精确窗口",
"observeApps": "列出可见应用",
"observeWindows": "列出窗口",
@@ -890,6 +894,36 @@
"unsupportedCommand": "不支持的 Computer Use 命令:{command}",
"screenshotMissing": "此结果没有可用的截图。窗口可能已隐藏或最小化,也可能是截图不可用。",
"outputUnsupported": "--output 只支持 observe window 或 zoom。",
+ "scriptArgNameValue": "脚本 --arg 必须使用 name=value 格式。",
+ "scriptArgDuplicate": "脚本参数“{name}”被重复提供。",
+ "scriptArgFile": "无法读取脚本参数文件 {path}:{error}",
+ "scriptFile": "无法运行 Computer Use 脚本 {path}:{error}",
+ "scriptFileType": "Computer Use 脚本 {path} 必须是可读的 .mjs 文件。",
+ "scriptNodeUnavailable": "Computer Use 脚本需要 Avibe 桌面应用随附的 Node runtime。",
+ "scriptSdkUnavailable": "随 Avibe 发货的 @avibe/computer 模块不可用,请重新安装或更新 Avibe 桌面应用。",
+ "scriptOpenRejected": "Controller 拒绝了 Computer Use 脚本连接({code})。",
+ "scriptRuntime": "Computer Use 脚本 runtime 失败:{error}",
+ "scriptRuntimeExit": "Computer Use 脚本 runtime 以状态 {status} 退出,且未返回结果。",
+ "scriptRuntimeInvalid": "随 Avibe 发货的 Computer Use 脚本 runtime 返回了无效结果。",
+ "scriptTimedOut": "Computer Use 脚本超过了 {seconds} 秒的总超时。",
+ "scriptFailed": "Computer Use 脚本在 {step} 失败:{error}",
+ "scriptAmbiguous": "选择器匹配到多个目标。",
+ "scriptConnectionInvalid": "脚本的 Controller 连接已失效。",
+ "scriptControllerRejected": "Controller 拒绝了脚本请求。",
+ "scriptRequestTooLarge": "Computer Use 脚本请求超过了 Controller 的大小上限。",
+ "scriptControllerUnavailable": "Controller 当前不可用。",
+ "scriptDeadlineExpired": "脚本截止时间已到。",
+ "scriptElementNotActionable": "匹配到的元素无法执行操作。",
+ "scriptInvalidControllerResponse": "Controller 返回了无效的 Computer Use 响应。",
+ "scriptInvalidDescriptor": "Computer Use 脚本使用了无效或缺失的句柄描述符。",
+ "scriptInvalidSelector": "脚本使用了无效选择器。",
+ "scriptInvalidTimeout": "脚本使用了无效超时。",
+ "scriptOperationTimedOut": "Computer Use 操作超时。",
+ "scriptStaleAppHandle": "所选应用进程已失效。",
+ "scriptStaleWindowHandle": "所选应用窗口已失效。",
+ "scriptTargetNotFound": "未找到请求的 Computer Use 目标。",
+ "scriptTurnEnded": "启动此脚本的 Avibe Turn 已结束。",
+ "scriptWindowOrderUnavailable": "此选择器无法确定窗口顺序。",
"missingCallerSession": "Computer Use 需要 Avibe Agent 调用方 Session。",
"missingCallerTurn": "Computer Use 要求调用方 Avibe Session 当前有活动 Turn。",
"serviceUnavailable": "Avibe 服务不可用:{error}",
@@ -902,6 +936,7 @@
"desktopLeaseLost": "Computer Use 完成前桌面租约已丢失。",
"focusShortcutForbidden": "不允许使用可能切换焦点的快捷键,请改用带 URL 的 launch。",
"foregroundForbidden": "不允许使用前台输入。",
+ "foregroundConsentRequired": "前台输入需要所有者确认。",
"genericDriverError": "Computer Use driver 返回了错误。",
"invalidDriverArguments": "Computer Use 参数无效。",
"observeFirst": "发送输入前,请重新观察精确窗口。",
diff --git a/vibe/internal_client.py b/vibe/internal_client.py
index 4bf02a407..051591434 100644
--- a/vibe/internal_client.py
+++ b/vibe/internal_client.py
@@ -384,6 +384,106 @@ def computer_use_sync(
return {"status_code": response.status_code, "body": body}
+def resolve_computer_script_endpoint(
+ socket_path: Optional[Path] = None,
+) -> control_ipc.ControlIpcClientEndpoint:
+ """Resolve and verify the Controller endpoint before launching Node."""
+
+ return _resolve_endpoint(socket_path)
+
+
+def computer_script_endpoint_payload(
+ endpoint: control_ipc.ControlIpcClientEndpoint,
+) -> dict[str, Any]:
+ """Serialize one already-verified endpoint for the shipped Node runner."""
+
+ if endpoint.transport == "unix":
+ if endpoint.socket_path is None:
+ raise InternalServerUnavailable("control IPC Unix endpoint has no socket")
+ return {
+ "transport": "unix",
+ "socket_path": str(endpoint.socket_path),
+ }
+ descriptor = endpoint.descriptor
+ if descriptor is None:
+ raise InternalServerUnavailable("control IPC TCP endpoint has no descriptor")
+ return {
+ "transport": "tcp",
+ "host": descriptor.host,
+ "port": descriptor.port,
+ "headers": endpoint.headers,
+ "instance_id": descriptor.instance_id,
+ "instance_header": control_ipc.CONTROL_IPC_INSTANCE_HEADER,
+ }
+
+
+def _computer_script_post_sync(
+ endpoint: control_ipc.ControlIpcClientEndpoint,
+ path: str,
+ payload: dict[str, Any],
+ *,
+ timeout: float,
+) -> dict[str, Any]:
+ try:
+ with httpx.Client(
+ transport=_sync_transport(endpoint),
+ base_url=endpoint.base_url,
+ headers=endpoint.headers,
+ timeout=httpx.Timeout(timeout, connect=5.0),
+ ) as client:
+ response = client.post(path, json=payload)
+ _validate_response(response, endpoint)
+ try:
+ body = response.json() if response.content else {}
+ except (json.JSONDecodeError, UnicodeDecodeError) as exc:
+ raise InternalServerUnavailable(
+ f"{path} returned invalid JSON"
+ ) from exc
+ except _SOCKET_CONNECT_ERRORS as exc:
+ raise InternalServerUnavailable(str(exc)) from exc
+ except (httpx.TimeoutException, httpx.TransportError) as exc:
+ raise InternalServerTimeout(str(exc)) from exc
+ if not isinstance(body, dict):
+ raise InternalServerUnavailable(f"{path} returned invalid JSON")
+ return {"status_code": response.status_code, "body": body}
+
+
+def computer_script_open_sync(
+ *,
+ endpoint: control_ipc.ControlIpcClientEndpoint,
+ caller_session_id: str,
+ timeout_seconds: float,
+ request_timeout: float = 10.0,
+) -> dict[str, Any]:
+ """Open one exact-Turn-bound script connection."""
+
+ return _computer_script_post_sync(
+ endpoint,
+ "/internal/computer-script/open",
+ {
+ "caller_session_id": caller_session_id,
+ "timeout_seconds": timeout_seconds,
+ },
+ timeout=request_timeout,
+ )
+
+
+def computer_script_close_sync(
+ *,
+ endpoint: control_ipc.ControlIpcClientEndpoint,
+ connection_token: str,
+ timeout: float = 5.0,
+) -> dict[str, Any]:
+ """Best-effort close for a script connection known to the Python runner."""
+
+ return _computer_script_post_sync(
+ endpoint,
+ "/internal/computer-script/close",
+ {"connection_token": connection_token},
+ timeout=timeout,
+ )
+
+
async def dispatch_async(
payload: dict[str, Any],
*,