👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
perf(linux): add opt-in lightweight hidden node details (#103)
draplater7cb8bb7cfb97
diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.pyindex e4d2971..3fc4c41 100644--- a/apps/OpenComputerUseLinux/runtime.py+++ b/apps/OpenComputerUseLinux/runtime.py@@ -268,37 +268,45 @@ def parse_text_limit(value, fallback=DEFAULT_TEXT_LIMIT):return positive_int(value, fallback)-def record_for(node, index, path, window_bounds, text_limit=DEFAULT_TEXT_LIMIT):- bounds = relative_frame(node, window_bounds)+def record_for(node, index, path, window_bounds, text_limit=DEFAULT_TEXT_LIMIT, lightweight_hidden=False):+ # Only an explicit non-SHOWING state qualifies. Errors/unknown state keep+ # full details; geometry alone cannot prove that a menu/control is hidden.+ showing = safe(lambda: node.get_state_set().contains(Atspi.StateType.SHOWING)) if lightweight_hidden else None+ omit_details = showing is not None and not bool(showing)+ bounds = None if omit_details else relative_frame(node, window_bounds)role = node_role(node)return {"index": index,"runtimeId": path[:],- "automationId": accessible_id(node),+ "automationId": "" if omit_details else accessible_id(node),"name": limit_text(node_name(node), text_limit=text_limit),"controlType": role,"localizedControlType": role,- "className": str(safe(node.get_toolkit_name, "") or ""),- "value": element_value(node, text_limit=text_limit),+ "className": "" if omit_details else str(safe(node.get_toolkit_name, "") or ""),+ "value": "" if omit_details else element_value(node, text_limit=text_limit),"nativeWindowHandle": 0,"frame": bounds,- "actions": action_names(node),+ "actions": [] if omit_details else action_names(node),+ **({"detailsOmitted": True} if omit_details else {}),}def render_tree(root, window_bounds, root_path, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEMENTS, max_tree_depth=MAX_DEPTH):records = []lines = []+ lightweight_hidden = os.environ.get("OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN", "").strip().lower() in {"1", "true", "yes", "on"}def visit(node, depth, path):if len(records) >= max_tree_nodes or depth > max_tree_depth or node is None:returnindex = len(records)- record = record_for(node, index, path, window_bounds, text_limit=text_limit)+ record = record_for(node, index, path, window_bounds, text_limit=text_limit, lightweight_hidden=lightweight_hidden)records.append(record)role = record["localizedControlType"] or record["controlType"] or "element"title = record["name"] or record["automationId"] or ""+ if record.get("detailsOmitted"):+ title += " [hidden details omitted]"value_segment = ""if record["value"] and record["value"] != title:safe_value = record["value"].replace("\r", "\\r").replace("\n", "\\n")diff --git a/apps/OpenComputerUseLinux/runtime_test.py b/apps/OpenComputerUseLinux/runtime_test.pyindex 0baa261..05280b0 100644--- a/apps/OpenComputerUseLinux/runtime_test.py+++ b/apps/OpenComputerUseLinux/runtime_test.py@@ -172,6 +172,91 @@ class RuntimeKeyTests(unittest.TestCase):self.assertEqual(self.events, [])+class HiddenDetailsTests(unittest.TestCase):+ def setUp(self):+ self.runtime = load_runtime()+ self.runtime.Atspi.StateType = types.SimpleNamespace(SHOWING="showing")++ def node(self, showing):+ node = mock.Mock()+ node.get_name.return_value = "Menu"+ node.get_role_name.return_value = "menu"+ node.get_child_count.return_value = 0+ node.get_state_set.return_value.contains.return_value = showing+ return node++ def test_hidden_record_retains_identity_and_skips_expensive_properties(self):+ node = self.node(False)+ with mock.patch.object(self.runtime, "relative_frame") as geometry, \+ mock.patch.object(self.runtime, "element_value") as value, \+ mock.patch.object(self.runtime, "action_names") as actions:+ record = self.runtime.record_for(node, 7, [0, 3], None, lightweight_hidden=True)+ self.assertEqual((record["index"], record["runtimeId"], record["name"]), (7, [0, 3], "Menu"))+ self.assertTrue(record["detailsOmitted"])+ for query in (geometry, value, actions):+ query.assert_not_called()+ node.get_toolkit_name.assert_not_called()+ node.get_accessible_id.assert_not_called()++ def test_default_and_unknown_or_showing_state_keep_full_details(self):+ for enabled, showing in ((False, False), (True, True), (True, None)):+ with self.subTest(enabled=enabled, showing=showing):+ node = self.node(showing)+ with mock.patch.object(self.runtime, "relative_frame", return_value=None), \+ mock.patch.object(self.runtime, "accessible_id", return_value="id"), \+ mock.patch.object(self.runtime, "element_value", return_value="value"), \+ mock.patch.object(self.runtime, "action_names", return_value=["open"]):+ record = self.runtime.record_for(node, 0, [0], None, lightweight_hidden=enabled)+ self.assertNotIn("detailsOmitted", record)+ self.assertEqual((record["value"], record["actions"]), ("value", ["open"]))+ if not enabled:+ node.get_state_set.assert_not_called()++ def test_hidden_parent_does_not_prune_visible_menu_children_or_renumber(self):+ parent = self.node(False)+ child = self.node(True)+ parent.get_child_count.return_value = 1+ parent.get_child_at_index.return_value = child+ with mock.patch.dict(self.runtime.os.environ, {"OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN": "1"}), \+ mock.patch.object(self.runtime, "relative_frame", return_value=None), \+ mock.patch.object(self.runtime, "accessible_id", return_value="id"), \+ mock.patch.object(self.runtime, "element_value", return_value="value"), \+ mock.patch.object(self.runtime, "action_names", return_value=[]):+ records, lines = self.runtime.render_tree(parent, None, [0])+ self.assertEqual([r["index"] for r in records], [0, 1])+ self.assertTrue(records[0]["detailsOmitted"])+ self.assertNotIn("detailsOmitted", records[1])+ self.assertIn("hidden details omitted", lines[0])+ self.assertEqual(records[1]["runtimeId"], [0, 0])++ def test_failed_state_reads_keep_full_details(self):+ for failure in ("state-set", "missing-state-set", "contains"):+ with self.subTest(failure=failure):+ node = self.node(False)+ if failure == "state-set":+ node.get_state_set.side_effect = RuntimeError("state unavailable")+ elif failure == "missing-state-set":+ node.get_state_set.return_value = None+ else:+ node.get_state_set.return_value.contains.side_effect = RuntimeError("state unavailable")+ with mock.patch.object(self.runtime, "relative_frame", return_value=None), \+ mock.patch.object(self.runtime, "element_value", return_value="retained"), \+ mock.patch.object(self.runtime, "action_names", return_value=["click"]):+ record = self.runtime.record_for(node, 0, [0], None, lightweight_hidden=True)+ self.assertNotIn("detailsOmitted", record)+ self.assertEqual((record["value"], record["actions"]), ("retained", ["click"]))++ def test_showing_node_with_negative_geometry_keeps_details(self):+ node = self.node(True)+ bounds = {"x": -100, "y": -200, "width": 30, "height": 40}+ with mock.patch.object(self.runtime, "relative_frame", return_value=bounds), \+ mock.patch.object(self.runtime, "element_value", return_value="visible"), \+ mock.patch.object(self.runtime, "action_names", return_value=["click"]):+ record = self.runtime.record_for(node, 0, [0], None, lightweight_hidden=True)+ self.assertNotIn("detailsOmitted", record)+ self.assertEqual((record["frame"], record["value"], record["actions"]), (bounds, "visible", ["click"]))++class FocusDetailsTests(unittest.TestCase):def setUp(self):self.runtime = load_runtime()diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 83254de..8fc5d79 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -139,6 +139,7 @@- Linux runtime 需要运行在已登录桌面用户 session 里。缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,Go runtime 会在启动 Python AT-SPI bridge 前尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session bus、display / Wayland 值;纯 SSH tty 如果找不到已登录桌面 session,可以启动二进制,但不能直接 inspect 或操作 GUI session。- `get_app_state` 的 accessibility tree 在 GTK/GNOME app 上可能很深,Linux bridge 使用与 macOS / Windows 一致的 1200 节点、64 层默认 tree budget,并支持显式提高 `max_tree_nodes` / `max_tree_depth`。截图通过 GDK root window best-effort capture;GNOME Wayland 可能返回黑图,bridge 会检测全黑采样并省略 image block。- 焦点摘要和选区在快照已经选定的窗口里共用一次查找,分别受 `max_tree_nodes` / `max_tree_depth` 约束;到深度边界后不再读取下一层子节点,查找未完成时追加明确提示。选中文字使用标准 AT-SPI `get_n_selections` / `get_selection` 接口读取。实现与隔离 GTK 验证见 [Linux 焦点查找](linux-bounded-focus.md)。+- `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN=1` 可省略明确非 SHOWING 节点的位置、值等详情,保留节点、索引及子树;默认关闭,未知状态继续完整读取。使用方式与真实 GTK 验证见 [Linux 隐藏控件详情](linux-hidden-details.md)。- 这 9 个 tool 的协议面与 macOS / Windows 保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime path metadata,coordinate action 使用 screenshot/window-relative 坐标。- Linux `click_method=accessibility` 映射到 AT-SPI action,`global` 映射到 AT-SPI mouse synthesis 并要求全局指针环境变量;AT-SPI 没有等价的进程定向 mouse dispatch,因此 `app_post` 和 macOS-only 的 `sky_click` 会在 snapshot lookup 前明确返回 unsupported,`key_method=sky_key` 也一样。`auto` 仍保持 AT-SPI action 优先、mouse synthesis fallback 的现有行为。- Linux 动作回读已有独立 GTK fixture,可在私有 D-Bus / Xvfb 中通过 native MCP 复测;运行方式见 [Linux 动作回读](linux-action-read-back.md)。当前 Linux 侧仍是功能性第一版:没有 visual cursor overlay、没有 installer/desktop entry,也没有覆盖全部工具的 Linux fixture。后续 TODO 记录在 `docs/exec-plans/active/20260422-linux-computer-use-runtime.md`。diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.mdindex d2bd90e..3bf1844 100644--- a/docs/RELIABILITY.md+++ b/docs/RELIABILITY.md@@ -18,6 +18,7 @@- macOS agent display 实机回归(会短暂创建并销毁一块 virtual display):`OPEN_COMPUTER_USE_RUN_AGENT_DISPLAY_LIVE_TEST=1 swift test --filter AgentDisplayLiveTests`- macOS 其他 Desktop 实机回归(需要第二个 Desktop,测试会短暂切换 Space 启动目标):`OPEN_COMPUTER_USE_RUN_CROSS_SPACE_LIVE_TEST=1 swift test --filter CrossSpaceLiveTests`- Linux runtime:`(cd apps/OpenComputerUseLinux && go test ./...)`、`./scripts/build-open-computer-use-linux.sh --arch arm64`+- Linux 隐藏控件实测:`bash scripts/run-linux-read-back-smoke.sh /path/to/linux/open-computer-use --hidden-checks`,在私有 D-Bus / Xvfb 中验证真实 GTK 隐藏状态、属性读取、索引及页面切换;依赖与范围见 [测试说明](linux-hidden-details.md)。- Windows runtime:`(cd apps/OpenComputerUseWindows && go test ./...)`、`./scripts/build-open-computer-use-windows.sh --arch amd64`;截图相关变更还要在 Windows 交互桌面验证正常窗口有 image、1x1/minimized/cloaked 窗口无 image。- 本地诊断:- `open-computer-use doctor`diff --git a/docs/exec-plans/completed/20261017-pr103-hidden-details.md b/docs/exec-plans/completed/20261017-pr103-hidden-details.mdnew file mode 100644index 0000000..6f3a195--- /dev/null+++ b/docs/exec-plans/completed/20261017-pr103-hidden-details.md@@ -0,0 +1,32 @@+# PR #103: Linux hidden-node details++## Goal++Integrate latest main into the contributor branch, review the opt-in hidden-node optimization, test real GTK/AT-SPI behavior in an isolated X11 session, fix issues and push follow-up commits to the same PR. Do not merge.++## Scope and constraints++- Default snapshots, node order, indexes, runtime paths and visible descendants must retain their behavior.+- Omit details only when AT-SPI explicitly reports a node as non-SHOWING. Unknown/failed state reads keep full details.+- Use a disposable container on the existing engine with private Xvfb/D-Bus, not a desktop VM or personal applications. Do not claim Wayland or whole-task performance coverage.+- Preserve the original contribution; update the existing history and feature documentation.++## Progress++- [x] Merge latest main; retain both test groups and release-note entries.+- [x] Review state handling, action compatibility and omitted property reads.+- [x] Add repeatable real GTK coverage and state-failure/negative-coordinate regressions; no functional fix required.+- [x] Run relevant Python/Go/native MCP checks and document evidence.+- [x] Synchronize remotes and prepare tested follow-up commits for the original #103 branch.++## Validation++Compare default and lightweight trees on real GTK controls with hidden/visible nodes, verify identity/index preservation and visible descendants, count actual property helper calls, and exercise native MCP actions with lightweight mode enabled. Include explicit unknown/failure fallbacks in unit tests. Run existing action read-back/focus smoke where compatible, Python and Linux Go suites, cross-builds, docs and whitespace checks. Container and fixture processes must be removed after testing.++## Evidence++- Main integrated through `eb501f4`, retaining original commit `917f2fe` and all upstream focus/read-back tests and release notes.+- Python 29/29; Linux Go 25/25; Linux ARM64/AMD64 builds and targeted vet passed. Docs/whitespace and shell/Python syntax checks passed.+- Real GTK 3 notebook on Debian 12 ARM64, private Xvfb/D-Bus and AT-SPI: 12 stable nodes, 3 hidden; four property-helper call totals 48 → 36. Helpers invoked real remote accessibles. These are helper-call counts, not D-Bus message counts or a latency benchmark.+- Native MCP compared disabled/opt-in output and indexes, clicked to reveal hidden controls, observed restored values/geometry, returned to the original page, then passed the existing action/no-read-back sequence. Separate default-action smoke and both disabled/enabled focus/Unicode-selection smokes passed.+- Temporary container, fixture processes and standalone build artifacts removed. No desktop VM, host personal-app actions, version bump, Wayland/other-toolkit validation or whole-task performance claim.diff --git a/docs/histories/2026-10/20261015-2035-linux-hidden-details.md b/docs/histories/2026-10/20261015-2035-linux-hidden-details.mdnew file mode 100644index 0000000..b9e74cd--- /dev/null+++ b/docs/histories/2026-10/20261015-2035-linux-hidden-details.md@@ -0,0 +1,28 @@+# Add opt-in lightweight details for hidden Linux nodes++Agent: Codex; runtime: Linux.++User request: submit scoped improvements for the remaining Linux performance paths.++With `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN=1`, snapshot rendering retains every node, index, runtime path, role and name, but skips geometry/value/action/toolkit/automation-ID reads for nodes explicitly reported as non-SHOWING by AT-SPI. Such rows are marked `[hidden details omitted]`.++This option is disabled by default. Unknown or failing state queries keep full details. The renderer never prunes a hidden parent's descendants: visible menu children retain their full properties and original numbering. Geometry alone is not evidence that a node is hidden, so off-window/negative coordinates are not used as a filtering rule.++Validation: Python bridge tests (13 passed), including skipped property calls, unchanged default output, unknown state fallback, visible children of hidden parents and index/path preservation. No live desktop benchmark or claim that all offscreen nodes are optimized.++Changed files: `apps/OpenComputerUseLinux/runtime.py`, `apps/OpenComputerUseLinux/runtime_test.py`, `docs/linux-hidden-details.md`, architecture and release notes.++## 2026-10-17:同步主线并补真实 GTK 验证++用户要求基于最新 main 解决冲突、检查测试并推送原 PR,补充真实 GTK 覆盖;本机没有完整 Linux 桌面,不启动桌面虚拟机。++- 合入主线 `eb501f4`,保留贡献者原提交,同时保留主线动作回读、焦点与选区测试以及发布记录。+- 检查后未发现需要修改功能实现的问题。增加状态集获取失败、缺失、contains 异常以及可见节点负坐标的回归,确保异常不会误省略详情。+- 扩展已有 GTK smoke 的 `--hidden-checks`:创建两页 notebook,直接读取真实远程 AT-SPI 对象,对比所有节点的索引 / 路径 / 名称 / 角色,验证隐藏节点省略、显示节点记录不变,并包装真实属性辅助函数计数。通过 native MCP 验证输出、切换页面后详情恢复及原有动作链,fixture 独立导出状态确认动作结果。+- 真实测试使用现有容器引擎上的临时 Debian 12 ARM64 容器、私有 Xvfb / D-Bus、GTK 3 / AT-SPI 2。不操作宿主个人应用,不启动桌面 VM。++验证:Python 29 项通过;Linux Go 测试 25 项通过;ARM64 / AMD64 构建和 Linux-targeted vet 通过;隐藏控件 native MCP smoke、默认动作回读 smoke,以及开关两种状态下的焦点 / 中文 emoji 选区回归均通过。真实树有 12 个节点、3 个隐藏节点,四类属性辅助函数调用 48 → 36 次;这不是 D-Bus 消息数或耗时基准。文档与 whitespace 检查通过。++范围:X11 / GTK / AT-SPI;没有做 Wayland、其他 toolkit 或全任务性能测试。每节点会增加 SHOWING 查询,因此保持默认关闭,不承诺所有界面都提速。无版本 bump。++验证完成后已删除临时容器及独立构建制品,未影响已有容器服务。diff --git a/docs/linux-hidden-details.md b/docs/linux-hidden-details.mdnew file mode 100644index 0000000..bd94099--- /dev/null+++ b/docs/linux-hidden-details.md@@ -0,0 +1,30 @@+# Linux 隐藏控件详情++默认读取完整详情。设置 `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN=1` 后,AT-SPI 明确标为非 `SHOWING` 的控件会省略位置、值、动作、toolkit 和 automation ID,行内显示 `[hidden details omitted]`。++```sh+OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN=1 ocu js \+ 'var app = await cua.getApp("gedit"); await app.getAXState();'+```++节点、索引、路径、角色和名称仍保留,也不会因为父节点隐藏就跳过子节点。状态读取失败或未知时继续读取完整详情;负坐标和窗口外位置不作为隐藏依据。控件重新显示后,下次观察会恢复详情。++开启后每个节点会额外读取一次 `SHOWING` 状态,适合隐藏节点较多的界面,不保证所有界面更快。它不改变焦点查找、截图或动作投递,也不会自动打开。++## 测试++```sh+python3 apps/OpenComputerUseLinux/runtime_test.py+(cd apps/OpenComputerUseLinux && go test ./...)+```++真实 GTK 测试复用 [私有 D-Bus / Xvfb 环境](linux-action-read-back.md),不需要完整桌面或虚拟机:++```sh+(cd apps/OpenComputerUseLinux && go build -o /tmp/ocu-hidden-test .)+bash scripts/run-linux-read-back-smoke.sh /tmp/ocu-hidden-test --hidden-checks+```++脚本创建两页 GTK 标签页,对比开关前后的完整树、索引、路径和属性辅助函数调用次数,再通过 native MCP 切换页面、读取恢复的详情,并验证点击、输入和动作回读。属性计数调用的仍是真实 AT-SPI 接口;单测另覆盖状态异常和隐藏父节点下的可见子节点。++本次 X11 / GTK 3 / AT-SPI 实测为 12 个节点、3 个隐藏节点,四类属性辅助函数共调用 48 → 36 次。这不是 D-Bus 消息计数或耗时基准,不代表 Wayland、其他 toolkit 或完整任务的速度。diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.mdindex 7b34ca3..c63ff06 100644--- a/docs/releases/feature-release-notes.md+++ b/docs/releases/feature-release-notes.md@@ -12,6 +12,7 @@| 2026-10-15 | Locked Use 安装与性能修复 | 减少认证和 IPC 开销,旧系统组件也能正常移除并重装。 | 非 UI 插件使用 privileged host,安装前验证真实加载;修复系统缓存卸载与转交 socket 的生命周期,降低空闲唤醒,status 提示组件版本差异。 || 2026-10-15 | Linux 动作回读 | REPL 批量操作可省去每个动作后被丢弃的完整界面观察。 | Linux 开始支持现有的 `OPEN_COMPUTER_USE_ACTION_READ_BACK=0`,成功动作返回 `ok`,保留最后一次显式观察的索引缓存;默认原生 MCP 回读与显式 `get_app_state` 保持不变。 || 2026-10-15 | Linux 焦点与选区 | 大型界面减少重复查找,预算不足时明确提示,选中文字能正确返回。 | 复用快照已选窗口,一次查找生成焦点摘要与选区;按节点和深度预算停止读取,使用标准 AT-SPI 选区接口,并补隔离 GTK 验证。 |+| 2026-10-15 | Linux 隐藏控件详情 | 隐藏节点较多时,可减少不需要的属性读取。 | 新增默认关闭的 `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN`;保留节点、索引和可见子节点,状态未知时完整读取,补充真实 GTK 标签页及 native MCP 回归。详见 [使用与验证](../linux-hidden-details.md)。 || 2026-10-14 | 1.1.0 正式发布 | 用户可直接安装 Locked Use,在锁屏遮罩期间继续 GUI 操作,也可用定向查询和持久截图配置减少工具开销。 | 统一版本为 1.1.0;签名 universal macOS App 内嵌服务、Guardian、安装器和授权插件,普通 enable 请求权限,补齐冷启动握手隔离和发布制品检查;新增 app.query()、doctor --json 与 ocu config。 || 2026-10-10 | macOS Locked Use 验证版 | 可在已登录 Mac 锁屏时,通过全屏遮罩继续 GUI 自动化,并在结束或本地接管后安全重锁。 | 正式组件采用 OCU Guardian / OCULockService / OCULockInstaller / OCULockAuth 短名称,开发 App 保留 Dev 区分;新增签名 Broker / 授权插件、双进程显示器保护、一次性许可、租约排空 / 恢复及构建和实机验证脚本;管理员显式启用 validation profile,生产 backend 在完整验证前保持关闭。 || 2026-10-10 | macOS 定向 AX 查询 | 知道控件文案时,可以直接查询并操作,不必先获取截图和完整 AX tree。 | 新增 native `query` 和 JS `app.query()`,按文字、角色和窗口查找元素,返回完整性元数据;查询索引仅在同一会话内有效,操作前重新验证目标和位置。 |diff --git a/scripts/run-linux-read-back-smoke.sh b/scripts/run-linux-read-back-smoke.shindex 3d4c7c2..8a9507f 100755--- a/scripts/run-linux-read-back-smoke.sh+++ b/scripts/run-linux-read-back-smoke.sh@@ -3,7 +3,7 @@ set -euo pipefailrepo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"if [[ "$(uname -s)" != Linux || $# -lt 1 ]]; then- echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use [--focus-checks]" >&2+ echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use [--focus-checks | --hidden-checks]" >&2exit 2fidiff --git a/scripts/tests/linux-read-back-smoke.py b/scripts/tests/linux-read-back-smoke.pyindex 9ae62f4..cebad77 100644--- a/scripts/tests/linux-read-back-smoke.py+++ b/scripts/tests/linux-read-back-smoke.py@@ -2,6 +2,8 @@"""Exercise native Linux MCP actions against an isolated real GTK fixture."""import argparse+from contextlib import ExitStack+import importlib.utilimport jsonimport osfrom pathlib import Path@@ -11,9 +13,10 @@ import subprocessimport sysimport tempfileimport time+from unittest import mock-def fixture(name, state_file, focus_checks=False):+def fixture(name, state_file, focus_checks=False, hidden_checks=False):import gigi.require_version("Gtk", "3.0")@@ -25,7 +28,22 @@ def fixture(name, state_file, focus_checks=False):window = Gtk.Window(title=name)window.set_default_size(500, 240)box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12, margin=20)- window.add(box)+ notebook = None+ if hidden_checks:+ notebook = Gtk.Notebook()+ window.add(notebook)+ notebook.append_page(box, Gtk.Label(label="Main page"))+ hidden_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12, margin=20)+ hidden_entry = Gtk.Entry(text="hidden-value-中文🙂")+ hidden_entry.get_accessible().set_name("Hidden input")+ hidden_button = Gtk.Button(label="Return page")+ hidden_box.pack_start(hidden_entry, False, False, 0)+ hidden_box.pack_start(hidden_button, False, False, 0)+ notebook.append_page(hidden_box, Gtk.Label(label="Other page"))+ reveal = Gtk.Button(label="Reveal page")+ box.pack_start(reveal, False, False, 0)+ else:+ window.add(box)entry = Gtk.Entry()entry.get_accessible().set_name("Read-back input")button = Gtk.Button(label="Increment")@@ -39,6 +57,8 @@ def fixture(name, state_file, focus_checks=False):state["focused"] = entry.has_focus()bounds = entry.get_selection_bounds()state["selected"] = entry.get_text()[bounds[0]:bounds[1]] if bounds else ""+ if notebook is not None:+ state["page"] = notebook.get_current_page()temporary = state_file.with_suffix(".tmp")temporary.write_text(json.dumps(state))temporary.replace(state_file)@@ -60,6 +80,10 @@ def fixture(name, state_file, focus_checks=False):entry.connect("notify::selection-bound", publish)entry.connect("notify::cursor-position", publish)entry.connect("notify::has-focus", publish)+ if notebook is not None:+ reveal.connect("clicked", lambda *_: notebook.set_current_page(1))+ hidden_button.connect("clicked", lambda *_: notebook.set_current_page(0))+ notebook.connect("switch-page", lambda *_: GLib.idle_add(publish))window.connect("destroy", Gtk.main_quit)window.show_all()entry.grab_focus()@@ -89,8 +113,9 @@ def wait_for(predicate, description):class MCP:- def __init__(self, native, read_back, stderr):+ def __init__(self, native, read_back, stderr, env_overrides=None):env = dict(os.environ)+ env.update(env_overrides or {})env.pop("OPEN_COMPUTER_USE_ACTION_READ_BACK", None)if read_back is not None:env["OPEN_COMPUTER_USE_ACTION_READ_BACK"] = read_back@@ -183,7 +208,84 @@ def check_focus(client, name, state_file):print("Focus smoke: node/depth limits, Unicode selection, text limit and focus summary passed.")-def smoke(native, focus_checks=False):+def check_hidden(client, native, name, state_file, stderr):+ # Import the production bridge and wrap its real property helpers. All+ # objects below are remote GTK/AT-SPI accessibles, not synthetic records.+ runtime_path = Path(__file__).resolve().parents[2] / "apps/OpenComputerUseLinux/runtime.py"+ spec = importlib.util.spec_from_file_location("gtk_smoke_runtime", runtime_path)+ runtime = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(runtime)+ app = runtime.resolve_app(name)+ window_index, window = runtime.main_window(app)+ bounds = runtime.extents(window)+ flag = "OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN"+ helpers = ("relative_frame", "accessible_id", "element_value", "action_names")++ def render(enabled):+ with ExitStack() as stack:+ stack.enter_context(mock.patch.dict(os.environ, {flag: "1" if enabled else "0"}))+ calls = [stack.enter_context(mock.patch.object(runtime, key, wraps=getattr(runtime, key)))+ for key in helpers]+ records, lines = runtime.render_tree(window, bounds, [window_index])+ return records, lines, [query.call_count for query in calls]++ full, _, full_calls = render(False)+ light, _, light_calls = render(True)+ identity = ("index", "runtimeId", "name", "controlType", "localizedControlType")+ assert [[row[key] for key in identity] for row in full] == [+ [row[key] for key in identity] for row in light], (full, light)+ omitted = [row for row in light if row.get("detailsOmitted")]+ assert any(row["name"] == "Hidden input" for row in omitted), light+ assert next(row for row in full if row["name"] == "Hidden input")["value"] == "hidden-value-中文🙂", full+ for original, row in zip(full, light):+ node = runtime.resolve_path(app, row["runtimeId"])+ showing = node.get_state_set().contains(runtime.Atspi.StateType.SHOWING)+ assert bool(row.get("detailsOmitted")) == (not showing), row+ if row.get("detailsOmitted"):+ assert row["frame"] is None and row["value"] == "" and row["actions"] == [], row+ assert row["className"] == "" and row["automationId"] == "", row+ else:+ assert row == original, (original, row)+ assert full_calls == [len(full)] * len(helpers), full_calls+ assert light_calls == [len(full) - len(omitted)] * len(helpers), light_calls+ print(f"Real AT-SPI tree: {len(full)} stable nodes, {len(omitted)} hidden; "+ f"property helpers {sum(full_calls)} → {sum(light_calls)} (not a latency benchmark).")++ baseline = MCP(native, "0", stderr, {flag: "0"})+ try:+ default = baseline.call("get_app_state", name)+ optimized = client.call("get_app_state", name)+ expect_snapshot(default)+ expect_snapshot(optimized)+ assert "[hidden details omitted]" not in text(default), text(default)+ assert "hidden-value-中文🙂" in text(default), text(default)+ assert "hidden-value-中文🙂" not in text(optimized), text(optimized)+ assert "[hidden details omitted]" in text(optimized), text(optimized)+ for title in ("Read-back input", "Increment", "Hidden input", "Return page", "Reveal page"):+ assert element_index(default, title) == element_index(optimized, title), title+ finally:+ baseline.close()++ def page_is(number):+ return json.loads(state_file.read_text())["page"] == number++ result = client.call("click", name, element_index=element_index(optimized, "Reveal page"),+ click_method="accessibility")+ assert not result.get("isError"), result+ wait_for(lambda: page_is(1), "real notebook reveal")+ shown = client.call("get_app_state", name)+ expect_snapshot(shown)+ row = next(line for line in text(shown).splitlines()+ if re.match(r"\s*\d+\s", line) and "Hidden input" in line)+ assert "hidden-value-中文🙂" in row and "[hidden details omitted]" not in row and "Frame:" in row, row+ result = client.call("click", name, element_index=element_index(shown, "Return page"),+ click_method="accessibility")+ assert not result.get("isError"), result+ wait_for(lambda: page_is(0), "real notebook return")+ print("Native MCP: default/opt-in output, stable indexes, page changes and full detail restoration passed.")+++def smoke(native, focus_checks=False, hidden_checks=False):with tempfile.TemporaryDirectory(prefix="ocu-read-back-smoke-") as directory:directory = Path(directory)state_file = directory / "state.json"@@ -192,17 +294,22 @@ def smoke(native, focus_checks=False):arguments = [sys.executable, __file__, "--fixture", name, "--state-file", str(state_file)]if focus_checks:arguments.append("--focus-checks")+ if hidden_checks:+ arguments.append("--hidden-checks")process = subprocess.Popen(arguments, stderr=stderr)clients = []try:wait_for(state_file.exists, "GTK fixture startup")- client = MCP(native, " OFF ", stderr)+ overrides = {"OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN": "1"} if hidden_checks else None+ client = MCP(native, " OFF ", stderr, overrides)clients.append(client)# AT-SPI registration follows GTK startup asynchronously.wait_for(lambda: not client.call("get_app_state", name).get("isError"), "AT-SPI registration")if focus_checks:check_focus(client, name, state_file)return+ if hidden_checks:+ check_hidden(client, native, name, state_file, stderr)observed = client.call("get_app_state", name)expect_snapshot(observed)input_index = element_index(observed, "Read-back input")@@ -271,12 +378,14 @@ if __name__ == "__main__":parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("--native")parser.add_argument("--fixture")- parser.add_argument("--focus-checks", action="store_true")+ checks = parser.add_mutually_exclusive_group()+ checks.add_argument("--focus-checks", action="store_true")+ checks.add_argument("--hidden-checks", action="store_true")parser.add_argument("--state-file", type=Path)args = parser.parse_args()if args.fixture:- fixture(args.fixture, args.state_file, args.focus_checks)+ fixture(args.fixture, args.state_file, args.focus_checks, args.hidden_checks)else:if not args.native:parser.error("--native is required")- smoke(str(Path(args.native).resolve()), args.focus_checks)+ smoke(str(Path(args.native).resolve()), args.focus_checks, args.hidden_checks)