Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
perf(computer): validate window handles without scanning applications (#2898)
Maxc7bbcb8d780f
diff --git a/core/computer_script_handles.py b/core/computer_script_handles.pyindex 26934365f..5bd033439 100644--- a/core/computer_script_handles.py+++ b/core/computer_script_handles.py@@ -49,6 +49,10 @@ _GLOBAL_SCRIPT_TOOLS = frozenset("clipboard_write",})+# Window observations whose driver refuses a window_id that is closed or not+# owned by the requested pid (``window_id_not_found`` /+# ``window_owner_pid_mismatch``).+_DRIVER_VALIDATED_TOOLS = frozenset({"get_window_state"})_APP_HANDLE_TOOLS = frozenset({"launch_app",@@ -159,17 +163,57 @@ def bind_handle_to_request(raise ComputerScriptHandleError("invalid_computer_script_descriptor")+def handle_validated_by_driver(+ descriptor: HandleDescriptor | None,+ tool: str,+) -> bool:+ """Whether the driver itself refuses this call for a stale window handle.++ ``get_window_state`` refuses unless ``window_id`` is live and owned by+ ``pid``. macOS does not reuse window ids within a login session, so the+ exact pair cannot name a different window. The Controller therefore+ forwards this read-only observation without a discovery round trip and+ classifies a refusal afterwards with ``validate_handle_is_current``.+ """++ return isinstance(descriptor, WindowHandleDescriptor) and tool in _DRIVER_VALIDATED_TOOLS++async def validate_handle_is_current(descriptor: HandleDescriptor | None,tool: str,discover: DiscoveryCall,) -> dict[str, Any] | None:- """Validate one descriptor against current Controller discovery state."""+ """Validate one descriptor against current Controller discovery state.++ An app handle is checked against ``list_apps``. A window handle is checked+ with one pid-scoped ``list_windows``: the exact ``(pid, window_id)`` pair+ identifies the window for the whole login session, so the app list adds+ no identity and costs a full application scan.+ """if descriptor is None:return None- app = descriptor.app if isinstance(descriptor, WindowHandleDescriptor) else descriptor- if app.pid is None:+ if isinstance(descriptor, WindowHandleDescriptor):+ windows_result = await discover("list_windows", {"pid": descriptor.pid})+ if windows_result.get("isError") is True:+ return windows_result+ windows = _structured_list(windows_result, "windows")+ if not any(+ record.get("pid") == descriptor.pid+ and record.get("window_id") == descriptor.window_id+ for record in windows+ ):+ raise ComputerScriptHandleError(+ "stale_window_handle",+ params={+ "pid": descriptor.pid,+ "window_id": descriptor.window_id,+ },+ )+ return None++ if descriptor.pid is None:if tool == "launch_app":return Noneraise ComputerScriptHandleError("stale_app_handle")@@ -178,29 +222,10 @@ async def validate_handle_is_current(if apps_result.get("isError") is True:return apps_resultapps = _structured_list(apps_result, "apps")- if not any(app.matches(record) for record in apps):+ if not any(descriptor.matches(record) for record in apps):raise ComputerScriptHandleError("stale_app_handle",- params={"pid": app.pid},- )-- if not isinstance(descriptor, WindowHandleDescriptor):- return None- windows_result = await discover("list_windows", {"pid": descriptor.pid})- if windows_result.get("isError") is True:- return windows_result- windows = _structured_list(windows_result, "windows")- if not any(- record.get("pid") == descriptor.pid- and record.get("window_id") == descriptor.window_id- for record in windows- ):- raise ComputerScriptHandleError(- "stale_window_handle",- params={- "pid": descriptor.pid,- "window_id": descriptor.window_id,- },+ params={"pid": descriptor.pid},)return Nonediff --git a/core/computer_server.py b/core/computer_server.pyindex 4b2cd06a2..f8668957e 100644--- a/core/computer_server.py+++ b/core/computer_server.py@@ -41,6 +41,7 @@ from core.computer_use import (from core.computer_script_handles import (ComputerScriptHandleError,bind_handle_to_request,+ handle_validated_by_driver,parse_handle_descriptor,validate_handle_is_current,)@@ -1202,6 +1203,7 @@ class ComputerUseServer:upstream: Upstream | None = Nonesession_state: _SessionState | None = Nonepre_forward = True+ driver_validates_handle = Falsetry:status = await asyncio.to_thread(self._status_reader)if status.status != "ready" or status.state is None:@@ -1249,20 +1251,31 @@ class ComputerUseServer:raise _DriverSessionEnded(discovery_tool)return discovered- try:- validation_error = await validate_handle_is_current(- parsed_handle,- name,- discover,- )- except ComputerScriptHandleError as exc:- return _error_result(- exc.code,- dispatched=False,- result_params=exc.params,- )- if validation_error is not None:- return _not_dispatched_error_result(validation_error)+ async def check_handle() -> dict[str, Any] | None:+ try:+ validation_error = await validate_handle_is_current(+ parsed_handle,+ name,+ discover,+ )+ except ComputerScriptHandleError as exc:+ return _error_result(+ exc.code,+ dispatched=False,+ result_params=exc.params,+ )+ if validation_error is not None:+ return _not_dispatched_error_result(validation_error)+ return None++ driver_validates_handle = handle_validated_by_driver(+ parsed_handle,+ name,+ )+ if not driver_validates_handle:+ refusal = await check_handle()+ if refusal is not None:+ return refusalself._require_observation(name, arguments, session_state)forwarded = dict(arguments)forwarded["session"] = session@@ -1289,6 +1302,18 @@ class ComputerUseServer:# outcome ambiguous.return _not_dispatched_error_result(result)+ if driver_validates_handle and result.get("isError") is True:+ # A failed read-only observation changed nothing. When+ # the handle no longer names a live window, report it+ # exactly as the pre-forward check would; any other+ # failure stays as the driver returned it.+ refusal = await check_handle()+ if (+ refusal is not None+ and refusal["structuredContent"]["code"]+ == "stale_window_handle"+ ):+ return refusalif name in _OBSERVE_TOOLS and not result.get("isError"):self._record_observation(arguments, result, session_state)if name == "start_session":diff --git a/tests/test_computer_script_sdk.py b/tests/test_computer_script_sdk.pyindex f498d0a81..5f8c2127a 100644--- a/tests/test_computer_script_sdk.py+++ b/tests/test_computer_script_sdk.py@@ -332,7 +332,7 @@ def _fake_controller("route": "accessibility",})- elif tool == "double_click":+ elif tool in {"double_click", "press_key", "drag"}:body = _result({"effect": "confirmed",@@ -2607,3 +2607,83 @@ def test_python_runner_uses_the_verified_controller_endpoint_for_open_and_close({"connection_token": "token"},),]+++def _tool_requests(requests: list[dict]) -> list[tuple[str, dict]]:+ return [+ (request["payload"]["tool"], request["payload"]["arguments"])+ for request in requests+ if request["path"].endswith("/request")+ ]+++def test_sdk_element_resolution_reads_the_tree_without_a_screenshot(+ tmp_path: Path,+) -> None:+ payload, _server, requests = _run_node_script(+ tmp_path,+ """+import { computer } from "@avibe/computer";++export default async () => {+ const mail = await computer.app({ bundleId: "com.apple.mail" });+ const window = await mail.window({ label: "New Message" });+ await window.element({ role: "button", label: "Send", index: 0 }).doubleClick();+ await window.expect({ role: "textbox", label: "Subject" }).toExist();+ await window.observe();+};+""",+ )++ assert payload == {"ok": True, "result": None}+ observations = [+ arguments+ for tool, arguments in _tool_requests(requests)+ if tool == "get_window_state"+ ]+ assert len(observations) == 3+ # Element resolution (the action and the expectation) skips the capture.+ assert [arguments.get("include_screenshot") for arguments in observations[:2]] == [+ False,+ False,+ ]+ # An explicit observation keeps the driver default, which includes it.+ assert "include_screenshot" not in observations[2]+++def test_sdk_pixel_window_input_after_element_resolution_recaptures(+ tmp_path: Path,+) -> None:+ payload, _server, requests = _run_node_script(+ tmp_path,+ """+import { computer } from "@avibe/computer";++export default async () => {+ const mail = await computer.app({ bundleId: "com.apple.mail" });+ const window = await mail.window({ label: "New Message" });+ await window.element({ role: "button", label: "Send", index: 0 }).doubleClick();+ await window.press("Return");+ await window.drag({ from_x: 10, from_y: 20, to_x: 30, to_y: 40 });+ await window.drag({ from_x: 11, from_y: 21, to_x: 31, to_y: 41 });+};+""",+ )++ assert payload == {"ok": True, "result": None}+ sequence = [+ (tool, arguments.get("include_screenshot"))+ for tool, arguments in _tool_requests(requests)+ if tool in {"get_window_state", "double_click", "press_key", "drag"}+ ]+ assert sequence == [+ # Resolution leaves a tree-only snapshot ...+ ("get_window_state", False),+ ("double_click", None),+ # ... which keyboard input can use as it is ...+ ("press_key", None),+ # ... while pixel input first recaptures once, then reuses it.+ ("get_window_state", None),+ ("drag", None),+ ("drag", None),+ ]diff --git a/tests/test_computer_server.py b/tests/test_computer_server.pyindex ab0161520..97a258f18 100644--- a/tests/test_computer_server.py+++ b/tests/test_computer_server.py@@ -9,6 +9,7 @@ from typing import Any, Callable, Mappingimport pytest+from core.computer_script_handles import _WINDOW_HANDLE_TOOLSfrom core.computer_server import (ComputerServerError,ComputerUseServer,@@ -610,8 +611,8 @@ async def test_non_ready_call_names_status_and_never_spawns_upstream("name": None,},},- "get_window_state",- {"pid": 41, "window_id": 9, "session": "ses-script"},+ "click",+ {"pid": 41, "window_id": 9, "x": 1, "y": 1, "session": "ses-script"},[{"pid": 41,@@ -627,7 +628,26 @@ async def test_non_ready_call_names_status_and_never_spawns_upstream(}],"stale_window_handle",- ["start_session", "list_apps", "list_windows"],+ ["start_session", "list_windows"],+ ),+ (+ {+ "kind": "window",+ "pid": 41,+ "window_id": 9,+ "app": {+ "kind": "app",+ "pid": 41,+ "bundle_id": "com.apple.mail",+ "name": None,+ },+ },+ "click",+ {"pid": 41, "window_id": 9, "x": 1, "y": 1, "session": "ses-script"},+ [],+ [],+ "stale_window_handle",+ ["start_session", "list_windows"],),],)@@ -759,8 +779,6 @@ async def test_current_script_window_descriptor_is_not_forwarded_to_driver(assert result.get("isError") is not Trueassert [params["name"] for _method, params in upstream.calls] == ["start_session",- "list_apps",- "list_windows","get_window_state",]assert upstream.calls[-1][1]["arguments"] == {@@ -770,6 +788,218 @@ async def test_current_script_window_descriptor_is_not_forwarded_to_driver(}+_MAIL_WINDOW_DESCRIPTOR = {+ "kind": "window",+ "pid": 41,+ "window_id": 9,+ "app": {+ "kind": "app",+ "pid": 41,+ "bundle_id": "com.apple.mail",+ "name": None,+ },+}+++class _WindowHandleUpstream(FakeUpstream):+ """Driver double for one live Mail window that may close or fail to read."""++ def __init__(self) -> None:+ super().__init__()+ self.window_open = True+ self.observation_error: dict[str, Any] | None = None++ async def request(+ self,+ method: str,+ params: Mapping[str, Any],+ ) -> dict[str, Any]:+ assert method == "tools/call"+ self.calls.append((method, json.loads(json.dumps(params))))+ name = params["name"]+ result: dict[str, Any]+ if name == "list_apps":+ result = {+ "content": [],+ "structuredContent": {+ "apps": [+ {"pid": 41, "bundle_id": "com.apple.mail", "name": "Mail"}+ ]+ },+ }+ elif name == "list_windows":+ windows = (+ [{"pid": 41, "window_id": 9, "app_name": "Mail"}]+ if self.window_open+ else []+ )+ result = {"content": [], "structuredContent": {"windows": windows}}+ elif name == "get_window_state" and self.observation_error is not None:+ result = {+ "content": [],+ "structuredContent": self.observation_error,+ "isError": True,+ }+ elif name == "get_window_state":+ result = {+ "content": [],+ "structuredContent": {+ "pid": 41,+ "window_id": 9,+ "elements": [{"element_token": "s1:2", "role": "AXButton"}],+ },+ }+ else:+ result = {+ "content": [],+ "structuredContent": {"effect": "unverifiable"},+ }+ return {"jsonrpc": "2.0", "id": 1, "result": result}+++def _window_handle_server(+ tmp_path: Path,+) -> tuple[ComputerUseServer, _WindowHandleUpstream]:+ state_path, state = _state(tmp_path)+ upstream = _WindowHandleUpstream()+ server = ComputerUseServer(+ state_path=state_path,+ status_reader=lambda: ComputerUseStatus("ready", None, state),+ upstream_factory=lambda _state: asyncio.sleep(0, result=upstream),+ )+ return server, upstream++[email protected]("tool", sorted(_WINDOW_HANDLE_TOOLS))+async def test_window_handle_calls_never_scan_applications(+ tmp_path: Path,+ tool: str,+) -> None:+ """A current window handle costs at most one pid-scoped window lookup."""++ server, upstream = _window_handle_server(tmp_path)+ observed = await server.call_tool(+ "get_window_state",+ {"pid": 41, "window_id": 9, "session": "ses-script"},+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,+ )+ assert observed.get("isError") is not True+ upstream.calls.clear()++ result = await server.call_tool(+ tool,+ {"pid": 41, "window_id": 9, "session": "ses-script"},+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,+ )++ assert result.get("isError") is not True, result+ names = [params["name"] for _method, params in upstream.calls]+ assert "list_apps" not in names+ assert names.count("list_windows") <= 1+ assert names[-1] == tool+ discovery = [params for _method, params in upstream.calls if params["name"] == "list_windows"]+ assert all(params["arguments"]["pid"] == 41 for params in discovery)+++async def test_element_click_hot_path_validates_its_window_once(+ tmp_path: Path,+) -> None:+ """The SDK's resolve-then-act pair performs exactly one discovery call."""++ server, upstream = _window_handle_server(tmp_path)++ observed = await server.call_tool(+ "get_window_state",+ {+ "pid": 41,+ "window_id": 9,+ "include_screenshot": False,+ "session": "ses-script",+ },+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,+ )+ clicked = await server.call_tool(+ "click",+ {"pid": 41, "window_id": 9, "element_token": "s1:2", "session": "ses-script"},+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,+ )++ assert observed.get("isError") is not True+ assert clicked.get("isError") is not True+ assert [params["name"] for _method, params in upstream.calls] == [+ "start_session",+ "get_window_state",+ "list_windows",+ "click",+ ]+++async def test_closed_window_observation_reports_a_stale_handle(+ tmp_path: Path,+) -> None:+ server, upstream = _window_handle_server(tmp_path)+ upstream.window_open = False+ upstream.observation_error = {+ "code": "window_id_not_found",+ "detail": "window_id 9 is not a live window",+ }++ result = await server.call_tool(+ "get_window_state",+ {"pid": 41, "window_id": 9, "session": "ses-script"},+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,+ )++ assert result["isError"] is True+ assert result["structuredContent"] == {+ "code": "stale_window_handle",+ "dispatched": False,+ "params": {"pid": 41, "window_id": 9},+ }+ assert [params["name"] for _method, params in upstream.calls] == [+ "start_session",+ "get_window_state",+ "list_windows",+ ]+ # A refused observation never authorizes input on the window.+ clicked = await server.call_tool(+ "click",+ {"pid": 41, "window_id": 9, "x": 1, "y": 1, "session": "ses-script"},+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,+ )+ assert clicked["structuredContent"]["code"] == "stale_window_handle"+ assert clicked["structuredContent"]["dispatched"] is False+ assert [params["name"] for _method, params in upstream.calls][-1] == "list_windows"+++async def test_failed_observation_of_a_live_window_keeps_the_driver_error(+ tmp_path: Path,+) -> None:+ server, upstream = _window_handle_server(tmp_path)+ upstream.observation_error = {+ "code": "ax_timeout",+ "detail": "the application did not answer",+ }++ result = await server.call_tool(+ "get_window_state",+ {"pid": 41, "window_id": 9, "session": "ses-script"},+ handle_descriptor=_MAIL_WINDOW_DESCRIPTOR,+ )++ assert result["isError"] is True+ assert result["structuredContent"]["code"] == "ax_timeout"+ assert [params["name"] for _method, params in upstream.calls] == [+ "start_session",+ "get_window_state",+ "list_windows",+ ]++@pytest.mark.asyncioasync def test_daemon_generation_and_child_exit_each_replace_proxy_once(tmp_path: Path,diff --git a/vibe/computer_sdk/index.mjs b/vibe/computer_sdk/index.mjsindex 36016b822..b8e37a9fc 100644--- a/vibe/computer_sdk/index.mjs+++ b/vibe/computer_sdk/index.mjs@@ -705,7 +705,11 @@ class AppHandle {class WindowHandle {constructor(app, _selector, record) {- windowState.set(this, { snapshot: undefined, snapshotKey: undefined });+ windowState.set(this, {+ snapshot: undefined,+ snapshotKey: undefined,+ snapshotHasScreenshot: false,+ });const descriptor = windowDescriptor(app, record);this.pid = descriptor.pid;this.windowId = descriptor.window_id;@@ -775,7 +779,13 @@ class WindowHandle {async _windowAction(label, tool, options = {}) {return sdkStep(label, async () => {- await observeWindow(this, false);+ // The driver admits screenshot pixels only against a current snapshot+ // that carries a screenshot; element resolution leaves a tree-only one.+ await observeWindow(+ this,+ addressesPixels(options) &&+ !windowState.get(this).snapshotHasScreenshot,+ );return effectResult(await callHandleTool(this, tool, windowArguments(this, options)),);@@ -783,6 +793,12 @@ class WindowHandle {}}+const PIXEL_FIELDS = ["x", "y", "from_x", "from_y", "to_x", "to_y"];++function addressesPixels(options) {+ return PIXEL_FIELDS.some((field) => options[field] !== undefined);+}+async function observeWindow(handle, force, options = {}, deadline) {const state = windowState.get(handle);const descriptor = handle._descriptor;@@ -797,12 +813,22 @@ async function observeWindow(handle, force, options = {}, deadline) {deadline,);state.snapshotKey = key;+ state.snapshotHasScreenshot = options.include_screenshot !== false;return state.snapshot;}+// Element resolution reads only the accessibility tree. Element actions act+// through element tokens, so they skip the screen capture and its encoding.+const ELEMENT_RESOLUTION_OPTIONS = Object.freeze({ include_screenshot: false });+async function resolveWindowElement(handle, selector, force, deadline) {validateSelector(selector, ELEMENT_SELECTOR_FIELDS, "Element");- const observation = await observeWindow(handle, force, {}, deadline);+ const observation = await observeWindow(+ handle,+ force,+ ELEMENT_RESOLUTION_OPTIONS,+ deadline,+ );const elements = structured(observation).elements;return selectElement(Array.isArray(elements) ? elements : [], selector);}