Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

perf(computer): validate window handles without scanning applications (#2898)

Maxc7bbcb8d780f
diff --git a/core/computer_script_handles.py b/core/computer_script_handles.py
index 26934365f..5bd033439 100644
--- a/core/computer_script_handles.py
+++ b/core/computer_script_handles.py
@@ -49,6 +49,10 @@ _GLOBAL_SCRIPT_TOOLS = frozenset(
"clipboard_write",
}
)
+# Window observations whose driver refuses a window_id that is closed or not
+# owned by the requested pid (``window_id_not_found`` /
+# ``window_owner_pid_mismatch``).
+_DRIVER_VALIDATED_TOOLS = frozenset({"get_window_state"})
_APP_HANDLE_TOOLS = frozenset(
{
"launch_app",
@@ -159,17 +163,57 @@ def bind_handle_to_request(
raise ComputerScriptHandleError("invalid_computer_script_descriptor")
+def handle_validated_by_driver(
+ descriptor: HandleDescriptor | None,
+ tool: str,
+) -> bool:
+ """Whether the driver itself refuses this call for a stale window handle.
+
+ ``get_window_state`` refuses unless ``window_id`` is live and owned by
+ ``pid``. macOS does not reuse window ids within a login session, so the
+ exact pair cannot name a different window. The Controller therefore
+ forwards this read-only observation without a discovery round trip and
+ classifies a refusal afterwards with ``validate_handle_is_current``.
+ """
+
+ return isinstance(descriptor, WindowHandleDescriptor) and tool in _DRIVER_VALIDATED_TOOLS
+
+
async def validate_handle_is_current(
descriptor: HandleDescriptor | None,
tool: str,
discover: DiscoveryCall,
) -> dict[str, Any] | None:
- """Validate one descriptor against current Controller discovery state."""
+ """Validate one descriptor against current Controller discovery state.
+
+ An app handle is checked against ``list_apps``. A window handle is checked
+ with one pid-scoped ``list_windows``: the exact ``(pid, window_id)`` pair
+ identifies the window for the whole login session, so the app list adds
+ no identity and costs a full application scan.
+ """
if descriptor is None:
return None
- app = descriptor.app if isinstance(descriptor, WindowHandleDescriptor) else descriptor
- if app.pid is None:
+ if isinstance(descriptor, WindowHandleDescriptor):
+ windows_result = await discover("list_windows", {"pid": descriptor.pid})
+ if windows_result.get("isError") is True:
+ return windows_result
+ windows = _structured_list(windows_result, "windows")
+ if not any(
+ record.get("pid") == descriptor.pid
+ and record.get("window_id") == descriptor.window_id
+ for record in windows
+ ):
+ raise ComputerScriptHandleError(
+ "stale_window_handle",
+ params={
+ "pid": descriptor.pid,
+ "window_id": descriptor.window_id,
+ },
+ )
+ return None
+
+ if descriptor.pid is None:
if tool == "launch_app":
return None
raise ComputerScriptHandleError("stale_app_handle")
@@ -178,29 +222,10 @@ async def validate_handle_is_current(
if apps_result.get("isError") is True:
return apps_result
apps = _structured_list(apps_result, "apps")
- if not any(app.matches(record) for record in apps):
+ if not any(descriptor.matches(record) for record in apps):
raise ComputerScriptHandleError(
"stale_app_handle",
- params={"pid": app.pid},
- )
-
- if not isinstance(descriptor, WindowHandleDescriptor):
- return None
- windows_result = await discover("list_windows", {"pid": descriptor.pid})
- if windows_result.get("isError") is True:
- return windows_result
- windows = _structured_list(windows_result, "windows")
- if not any(
- record.get("pid") == descriptor.pid
- and record.get("window_id") == descriptor.window_id
- for record in windows
- ):
- raise ComputerScriptHandleError(
- "stale_window_handle",
- params={
- "pid": descriptor.pid,
- "window_id": descriptor.window_id,
- },
+ params={"pid": descriptor.pid},
)
return None
diff --git a/core/computer_server.py b/core/computer_server.py
index 4b2cd06a2..f8668957e 100644
--- a/core/computer_server.py
+++ b/core/computer_server.py
@@ -41,6 +41,7 @@ from core.computer_use import (
from core.computer_script_handles import (
ComputerScriptHandleError,
bind_handle_to_request,
+ handle_validated_by_driver,
parse_handle_descriptor,
validate_handle_is_current,
)
@@ -1202,6 +1203,7 @@ class ComputerUseServer:
upstream: Upstream | None = None
session_state: _SessionState | None = None
pre_forward = True
+ driver_validates_handle = False
try:
status = await asyncio.to_thread(self._status_reader)
if status.status != "ready" or status.state is None:
@@ -1249,20 +1251,31 @@ class ComputerUseServer:
raise _DriverSessionEnded(discovery_tool)
return discovered
- try:
- validation_error = await validate_handle_is_current(
- parsed_handle,
- name,
- discover,
- )
- except ComputerScriptHandleError as exc:
- return _error_result(
- exc.code,
- dispatched=False,
- result_params=exc.params,
- )
- if validation_error is not None:
- return _not_dispatched_error_result(validation_error)
+ async def check_handle() -> dict[str, Any] | None:
+ try:
+ validation_error = await validate_handle_is_current(
+ parsed_handle,
+ name,
+ discover,
+ )
+ except ComputerScriptHandleError as exc:
+ return _error_result(
+ exc.code,
+ dispatched=False,
+ result_params=exc.params,
+ )
+ if validation_error is not None:
+ return _not_dispatched_error_result(validation_error)
+ return None
+
+ driver_validates_handle = handle_validated_by_driver(
+ parsed_handle,
+ name,
+ )
+ if not driver_validates_handle:
+ refusal = await check_handle()
+ if refusal is not None:
+ return refusal
self._require_observation(name, arguments, session_state)
forwarded = dict(arguments)
forwarded["session"] = session
@@ -1289,6 +1302,18 @@ class ComputerUseServer:
# outcome ambiguous.
return _not_dispatched_error_result(result)
+ if driver_validates_handle and result.get("isError") is True:
+ # A failed read-only observation changed nothing. When
+ # the handle no longer names a live window, report it
+ # exactly as the pre-forward check would; any other
+ # failure stays as the driver returned it.
+ refusal = await check_handle()
+ if (
+ refusal is not None
+ and refusal["structuredContent"]["code"]
+ == "stale_window_handle"
+ ):
+ return refusal
if name in _OBSERVE_TOOLS and not result.get("isError"):
self._record_observation(arguments, result, session_state)
if name == "start_session":
diff --git a/tests/test_computer_script_sdk.py b/tests/test_computer_script_sdk.py
index f498d0a81..5f8c2127a 100644
--- a/tests/test_computer_script_sdk.py
+++ b/tests/test_computer_script_sdk.py
@@ -332,7 +332,7 @@ def _fake_controller(
"route": "accessibility",
}
)
- elif tool == "double_click":
+ elif tool in {"double_click", "press_key", "drag"}:
body = _result(
{
"effect": "confirmed",
@@ -2607,3 +2607,83 @@ def test_python_runner_uses_the_verified_controller_endpoint_for_open_and_close(
{"connection_token": "token"},
),
]
+
+
+def _tool_requests(requests: list[dict]) -> list[tuple[str, dict]]:
+ return [
+ (request["payload"]["tool"], request["payload"]["arguments"])
+ for request in requests
+ if request["path"].endswith("/request")
+ ]
+
+
+def test_sdk_element_resolution_reads_the_tree_without_a_screenshot(
+ tmp_path: Path,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ label: "New Message" });
+ await window.element({ role: "button", label: "Send", index: 0 }).doubleClick();
+ await window.expect({ role: "textbox", label: "Subject" }).toExist();
+ await window.observe();
+};
+""",
+ )
+
+ assert payload == {"ok": True, "result": None}
+ observations = [
+ arguments
+ for tool, arguments in _tool_requests(requests)
+ if tool == "get_window_state"
+ ]
+ assert len(observations) == 3
+ # Element resolution (the action and the expectation) skips the capture.
+ assert [arguments.get("include_screenshot") for arguments in observations[:2]] == [
+ False,
+ False,
+ ]
+ # An explicit observation keeps the driver default, which includes it.
+ assert "include_screenshot" not in observations[2]
+
+
+def test_sdk_pixel_window_input_after_element_resolution_recaptures(
+ tmp_path: Path,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ label: "New Message" });
+ await window.element({ role: "button", label: "Send", index: 0 }).doubleClick();
+ await window.press("Return");
+ await window.drag({ from_x: 10, from_y: 20, to_x: 30, to_y: 40 });
+ await window.drag({ from_x: 11, from_y: 21, to_x: 31, to_y: 41 });
+};
+""",
+ )
+
+ assert payload == {"ok": True, "result": None}
+ sequence = [
+ (tool, arguments.get("include_screenshot"))
+ for tool, arguments in _tool_requests(requests)
+ if tool in {"get_window_state", "double_click", "press_key", "drag"}
+ ]
+ assert sequence == [
+ # Resolution leaves a tree-only snapshot ...
+ ("get_window_state", False),
+ ("double_click", None),
+ # ... which keyboard input can use as it is ...
+ ("press_key", None),
+ # ... while pixel input first recaptures once, then reuses it.
+ ("get_window_state", None),
+ ("drag", None),
+ ("drag", None),
+ ]
diff --git a/tests/test_computer_server.py b/tests/test_computer_server.py
index ab0161520..97a258f18 100644
--- a/tests/test_computer_server.py
+++ b/tests/test_computer_server.py
@@ -9,6 +9,7 @@ from typing import Any, Callable, Mapping
import pytest
+from core.computer_script_handles import _WINDOW_HANDLE_TOOLS
from core.computer_server import (
ComputerServerError,
ComputerUseServer,
@@ -610,8 +611,8 @@ async def test_non_ready_call_names_status_and_never_spawns_upstream(
"name": None,
},
},
- "get_window_state",
- {"pid": 41, "window_id": 9, "session": "ses-script"},
+ "click",
+ {"pid": 41, "window_id": 9, "x": 1, "y": 1, "session": "ses-script"},
[
{
"pid": 41,
@@ -627,7 +628,26 @@ async def test_non_ready_call_names_status_and_never_spawns_upstream(
}
],
"stale_window_handle",
- ["start_session", "list_apps", "list_windows"],
+ ["start_session", "list_windows"],
+ ),
+ (
+ {
+ "kind": "window",
+ "pid": 41,
+ "window_id": 9,
+ "app": {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ },
+ },
+ "click",
+ {"pid": 41, "window_id": 9, "x": 1, "y": 1, "session": "ses-script"},
+ [],
+ [],
+ "stale_window_handle",
+ ["start_session", "list_windows"],
),
],
)
@@ -759,8 +779,6 @@ async def test_current_script_window_descriptor_is_not_forwarded_to_driver(
assert result.get("isError") is not True
assert [params["name"] for _method, params in upstream.calls] == [
"start_session",
- "list_apps",
- "list_windows",
"get_window_state",
]
assert upstream.calls[-1][1]["arguments"] == {
@@ -770,6 +788,218 @@ async def test_current_script_window_descriptor_is_not_forwarded_to_driver(
}
+_MAIL_WINDOW_DESCRIPTOR = {
+ "kind": "window",
+ "pid": 41,
+ "window_id": 9,
+ "app": {
+ "kind": "app",
+ "pid": 41,
+ "bundle_id": "com.apple.mail",
+ "name": None,
+ },
+}
+
+
+class _WindowHandleUpstream(FakeUpstream):
+ """Driver double for one live Mail window that may close or fail to read."""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.window_open = True
+ self.observation_error: dict[str, Any] | None = None
+
+ async def request(
+ self,
+ method: str,
+ params: Mapping[str, Any],
+ ) -> dict[str, Any]:
+ assert method == "tools/call"
+ self.calls.append((method, json.loads(json.dumps(params))))
+ name = params["name"]
+ result: dict[str, Any]
+ if name == "list_apps":
+ result = {
+ "content": [],
+ "structuredContent": {
+ "apps": [
+ {"pid": 41, "bundle_id": "com.apple.mail", "name": "Mail"}
+ ]
+ },
+ }
+ elif name == "list_windows":
+ windows = (
+ [{"pid": 41, "window_id": 9, "app_name": "Mail"}]
+ if self.window_open
+ else []
+ )
+ result = {"content": [], "structuredContent": {"windows": windows}}
+ elif name == "get_window_state" and self.observation_error is not None:
+ result = {
+ "content": [],
+ "structuredContent": self.observation_error,
+ "isError": True,
+ }
+ elif name == "get_window_state":
+ result = {
+ "content": [],
+ "structuredContent": {
+ "pid": 41,
+ "window_id": 9,
+ "elements": [{"element_token": "s1:2", "role": "AXButton"}],
+ },
+ }
+ else:
+ result = {
+ "content": [],
+ "structuredContent": {"effect": "unverifiable"},
+ }
+ return {"jsonrpc": "2.0", "id": 1, "result": result}
+
+
+def _window_handle_server(
+ tmp_path: Path,
+) -> tuple[ComputerUseServer, _WindowHandleUpstream]:
+ state_path, state = _state(tmp_path)
+ upstream = _WindowHandleUpstream()
+ server = ComputerUseServer(
+ state_path=state_path,
+ status_reader=lambda: ComputerUseStatus("ready", None, state),
+ upstream_factory=lambda _state: asyncio.sleep(0, result=upstream),
+ )
+ return server, upstream
+
+
[email protected]("tool", sorted(_WINDOW_HANDLE_TOOLS))
+async def test_window_handle_calls_never_scan_applications(
+ tmp_path: Path,
+ tool: str,
+) -> None:
+ """A current window handle costs at most one pid-scoped window lookup."""
+
+ server, upstream = _window_handle_server(tmp_path)
+ observed = await server.call_tool(
+ "get_window_state",
+ {"pid": 41, "window_id": 9, "session": "ses-script"},
+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,
+ )
+ assert observed.get("isError") is not True
+ upstream.calls.clear()
+
+ result = await server.call_tool(
+ tool,
+ {"pid": 41, "window_id": 9, "session": "ses-script"},
+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,
+ )
+
+ assert result.get("isError") is not True, result
+ names = [params["name"] for _method, params in upstream.calls]
+ assert "list_apps" not in names
+ assert names.count("list_windows") <= 1
+ assert names[-1] == tool
+ discovery = [params for _method, params in upstream.calls if params["name"] == "list_windows"]
+ assert all(params["arguments"]["pid"] == 41 for params in discovery)
+
+
+async def test_element_click_hot_path_validates_its_window_once(
+ tmp_path: Path,
+) -> None:
+ """The SDK's resolve-then-act pair performs exactly one discovery call."""
+
+ server, upstream = _window_handle_server(tmp_path)
+
+ observed = await server.call_tool(
+ "get_window_state",
+ {
+ "pid": 41,
+ "window_id": 9,
+ "include_screenshot": False,
+ "session": "ses-script",
+ },
+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,
+ )
+ clicked = await server.call_tool(
+ "click",
+ {"pid": 41, "window_id": 9, "element_token": "s1:2", "session": "ses-script"},
+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,
+ )
+
+ assert observed.get("isError") is not True
+ assert clicked.get("isError") is not True
+ assert [params["name"] for _method, params in upstream.calls] == [
+ "start_session",
+ "get_window_state",
+ "list_windows",
+ "click",
+ ]
+
+
+async def test_closed_window_observation_reports_a_stale_handle(
+ tmp_path: Path,
+) -> None:
+ server, upstream = _window_handle_server(tmp_path)
+ upstream.window_open = False
+ upstream.observation_error = {
+ "code": "window_id_not_found",
+ "detail": "window_id 9 is not a live window",
+ }
+
+ result = await server.call_tool(
+ "get_window_state",
+ {"pid": 41, "window_id": 9, "session": "ses-script"},
+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,
+ )
+
+ assert result["isError"] is True
+ assert result["structuredContent"] == {
+ "code": "stale_window_handle",
+ "dispatched": False,
+ "params": {"pid": 41, "window_id": 9},
+ }
+ assert [params["name"] for _method, params in upstream.calls] == [
+ "start_session",
+ "get_window_state",
+ "list_windows",
+ ]
+ # A refused observation never authorizes input on the window.
+ clicked = await server.call_tool(
+ "click",
+ {"pid": 41, "window_id": 9, "x": 1, "y": 1, "session": "ses-script"},
+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,
+ )
+ assert clicked["structuredContent"]["code"] == "stale_window_handle"
+ assert clicked["structuredContent"]["dispatched"] is False
+ assert [params["name"] for _method, params in upstream.calls][-1] == "list_windows"
+
+
+async def test_failed_observation_of_a_live_window_keeps_the_driver_error(
+ tmp_path: Path,
+) -> None:
+ server, upstream = _window_handle_server(tmp_path)
+ upstream.observation_error = {
+ "code": "ax_timeout",
+ "detail": "the application did not answer",
+ }
+
+ result = await server.call_tool(
+ "get_window_state",
+ {"pid": 41, "window_id": 9, "session": "ses-script"},
+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,
+ )
+
+ assert result["isError"] is True
+ assert result["structuredContent"]["code"] == "ax_timeout"
+ assert [params["name"] for _method, params in upstream.calls] == [
+ "start_session",
+ "get_window_state",
+ "list_windows",
+ ]
+
+
@pytest.mark.asyncio
async def test_daemon_generation_and_child_exit_each_replace_proxy_once(
tmp_path: Path,
diff --git a/vibe/computer_sdk/index.mjs b/vibe/computer_sdk/index.mjs
index 36016b822..b8e37a9fc 100644
--- a/vibe/computer_sdk/index.mjs
+++ b/vibe/computer_sdk/index.mjs
@@ -705,7 +705,11 @@ class AppHandle {
class WindowHandle {
constructor(app, _selector, record) {
- windowState.set(this, { snapshot: undefined, snapshotKey: undefined });
+ windowState.set(this, {
+ snapshot: undefined,
+ snapshotKey: undefined,
+ snapshotHasScreenshot: false,
+ });
const descriptor = windowDescriptor(app, record);
this.pid = descriptor.pid;
this.windowId = descriptor.window_id;
@@ -775,7 +779,13 @@ class WindowHandle {
async _windowAction(label, tool, options = {}) {
return sdkStep(label, async () => {
- await observeWindow(this, false);
+ // The driver admits screenshot pixels only against a current snapshot
+ // that carries a screenshot; element resolution leaves a tree-only one.
+ await observeWindow(
+ this,
+ addressesPixels(options) &&
+ !windowState.get(this).snapshotHasScreenshot,
+ );
return effectResult(
await callHandleTool(this, tool, windowArguments(this, options)),
);
@@ -783,6 +793,12 @@ class WindowHandle {
}
}
+const PIXEL_FIELDS = ["x", "y", "from_x", "from_y", "to_x", "to_y"];
+
+function addressesPixels(options) {
+ return PIXEL_FIELDS.some((field) => options[field] !== undefined);
+}
+
async function observeWindow(handle, force, options = {}, deadline) {
const state = windowState.get(handle);
const descriptor = handle._descriptor;
@@ -797,12 +813,22 @@ async function observeWindow(handle, force, options = {}, deadline) {
deadline,
);
state.snapshotKey = key;
+ state.snapshotHasScreenshot = options.include_screenshot !== false;
return state.snapshot;
}
+// Element resolution reads only the accessibility tree. Element actions act
+// through element tokens, so they skip the screen capture and its encoding.
+const ELEMENT_RESOLUTION_OPTIONS = Object.freeze({ include_screenshot: false });
+
async function resolveWindowElement(handle, selector, force, deadline) {
validateSelector(selector, ELEMENT_SELECTOR_FIELDS, "Element");
- const observation = await observeWindow(handle, force, {}, deadline);
+ const observation = await observeWindow(
+ handle,
+ force,
+ ELEMENT_RESOLUTION_OPTIONS,
+ deadline,
+ );
const elements = structured(observation).elements;
return selectElement(Array.isArray(elements) ? elements : [], selector);
}