Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

perf(linux): add opt-in lightweight hidden node details (#103)

draplater7cb8bb7cfb97
diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.py
index e4d2971..3fc4c41 100644
--- a/apps/OpenComputerUseLinux/runtime.py
+++ b/apps/OpenComputerUseLinux/runtime.py
@@ -268,37 +268,45 @@ def parse_text_limit(value, fallback=DEFAULT_TEXT_LIMIT):
return positive_int(value, fallback)
-def record_for(node, index, path, window_bounds, text_limit=DEFAULT_TEXT_LIMIT):
- bounds = relative_frame(node, window_bounds)
+def record_for(node, index, path, window_bounds, text_limit=DEFAULT_TEXT_LIMIT, lightweight_hidden=False):
+ # Only an explicit non-SHOWING state qualifies. Errors/unknown state keep
+ # full details; geometry alone cannot prove that a menu/control is hidden.
+ showing = safe(lambda: node.get_state_set().contains(Atspi.StateType.SHOWING)) if lightweight_hidden else None
+ omit_details = showing is not None and not bool(showing)
+ bounds = None if omit_details else relative_frame(node, window_bounds)
role = node_role(node)
return {
"index": index,
"runtimeId": path[:],
- "automationId": accessible_id(node),
+ "automationId": "" if omit_details else accessible_id(node),
"name": limit_text(node_name(node), text_limit=text_limit),
"controlType": role,
"localizedControlType": role,
- "className": str(safe(node.get_toolkit_name, "") or ""),
- "value": element_value(node, text_limit=text_limit),
+ "className": "" if omit_details else str(safe(node.get_toolkit_name, "") or ""),
+ "value": "" if omit_details else element_value(node, text_limit=text_limit),
"nativeWindowHandle": 0,
"frame": bounds,
- "actions": action_names(node),
+ "actions": [] if omit_details else action_names(node),
+ **({"detailsOmitted": True} if omit_details else {}),
}
def render_tree(root, window_bounds, root_path, text_limit=DEFAULT_TEXT_LIMIT, max_tree_nodes=MAX_ELEMENTS, max_tree_depth=MAX_DEPTH):
records = []
lines = []
+ lightweight_hidden = os.environ.get("OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN", "").strip().lower() in {"1", "true", "yes", "on"}
def visit(node, depth, path):
if len(records) >= max_tree_nodes or depth > max_tree_depth or node is None:
return
index = len(records)
- record = record_for(node, index, path, window_bounds, text_limit=text_limit)
+ record = record_for(node, index, path, window_bounds, text_limit=text_limit, lightweight_hidden=lightweight_hidden)
records.append(record)
role = record["localizedControlType"] or record["controlType"] or "element"
title = record["name"] or record["automationId"] or ""
+ if record.get("detailsOmitted"):
+ title += " [hidden details omitted]"
value_segment = ""
if record["value"] and record["value"] != title:
safe_value = record["value"].replace("\r", "\\r").replace("\n", "\\n")
diff --git a/apps/OpenComputerUseLinux/runtime_test.py b/apps/OpenComputerUseLinux/runtime_test.py
index 0baa261..05280b0 100644
--- a/apps/OpenComputerUseLinux/runtime_test.py
+++ b/apps/OpenComputerUseLinux/runtime_test.py
@@ -172,6 +172,91 @@ class RuntimeKeyTests(unittest.TestCase):
self.assertEqual(self.events, [])
+class HiddenDetailsTests(unittest.TestCase):
+ def setUp(self):
+ self.runtime = load_runtime()
+ self.runtime.Atspi.StateType = types.SimpleNamespace(SHOWING="showing")
+
+ def node(self, showing):
+ node = mock.Mock()
+ node.get_name.return_value = "Menu"
+ node.get_role_name.return_value = "menu"
+ node.get_child_count.return_value = 0
+ node.get_state_set.return_value.contains.return_value = showing
+ return node
+
+ def test_hidden_record_retains_identity_and_skips_expensive_properties(self):
+ node = self.node(False)
+ with mock.patch.object(self.runtime, "relative_frame") as geometry, \
+ mock.patch.object(self.runtime, "element_value") as value, \
+ mock.patch.object(self.runtime, "action_names") as actions:
+ record = self.runtime.record_for(node, 7, [0, 3], None, lightweight_hidden=True)
+ self.assertEqual((record["index"], record["runtimeId"], record["name"]), (7, [0, 3], "Menu"))
+ self.assertTrue(record["detailsOmitted"])
+ for query in (geometry, value, actions):
+ query.assert_not_called()
+ node.get_toolkit_name.assert_not_called()
+ node.get_accessible_id.assert_not_called()
+
+ def test_default_and_unknown_or_showing_state_keep_full_details(self):
+ for enabled, showing in ((False, False), (True, True), (True, None)):
+ with self.subTest(enabled=enabled, showing=showing):
+ node = self.node(showing)
+ with mock.patch.object(self.runtime, "relative_frame", return_value=None), \
+ mock.patch.object(self.runtime, "accessible_id", return_value="id"), \
+ mock.patch.object(self.runtime, "element_value", return_value="value"), \
+ mock.patch.object(self.runtime, "action_names", return_value=["open"]):
+ record = self.runtime.record_for(node, 0, [0], None, lightweight_hidden=enabled)
+ self.assertNotIn("detailsOmitted", record)
+ self.assertEqual((record["value"], record["actions"]), ("value", ["open"]))
+ if not enabled:
+ node.get_state_set.assert_not_called()
+
+ def test_hidden_parent_does_not_prune_visible_menu_children_or_renumber(self):
+ parent = self.node(False)
+ child = self.node(True)
+ parent.get_child_count.return_value = 1
+ parent.get_child_at_index.return_value = child
+ with mock.patch.dict(self.runtime.os.environ, {"OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN": "1"}), \
+ mock.patch.object(self.runtime, "relative_frame", return_value=None), \
+ mock.patch.object(self.runtime, "accessible_id", return_value="id"), \
+ mock.patch.object(self.runtime, "element_value", return_value="value"), \
+ mock.patch.object(self.runtime, "action_names", return_value=[]):
+ records, lines = self.runtime.render_tree(parent, None, [0])
+ self.assertEqual([r["index"] for r in records], [0, 1])
+ self.assertTrue(records[0]["detailsOmitted"])
+ self.assertNotIn("detailsOmitted", records[1])
+ self.assertIn("hidden details omitted", lines[0])
+ self.assertEqual(records[1]["runtimeId"], [0, 0])
+
+ def test_failed_state_reads_keep_full_details(self):
+ for failure in ("state-set", "missing-state-set", "contains"):
+ with self.subTest(failure=failure):
+ node = self.node(False)
+ if failure == "state-set":
+ node.get_state_set.side_effect = RuntimeError("state unavailable")
+ elif failure == "missing-state-set":
+ node.get_state_set.return_value = None
+ else:
+ node.get_state_set.return_value.contains.side_effect = RuntimeError("state unavailable")
+ with mock.patch.object(self.runtime, "relative_frame", return_value=None), \
+ mock.patch.object(self.runtime, "element_value", return_value="retained"), \
+ mock.patch.object(self.runtime, "action_names", return_value=["click"]):
+ record = self.runtime.record_for(node, 0, [0], None, lightweight_hidden=True)
+ self.assertNotIn("detailsOmitted", record)
+ self.assertEqual((record["value"], record["actions"]), ("retained", ["click"]))
+
+ def test_showing_node_with_negative_geometry_keeps_details(self):
+ node = self.node(True)
+ bounds = {"x": -100, "y": -200, "width": 30, "height": 40}
+ with mock.patch.object(self.runtime, "relative_frame", return_value=bounds), \
+ mock.patch.object(self.runtime, "element_value", return_value="visible"), \
+ mock.patch.object(self.runtime, "action_names", return_value=["click"]):
+ record = self.runtime.record_for(node, 0, [0], None, lightweight_hidden=True)
+ self.assertNotIn("detailsOmitted", record)
+ self.assertEqual((record["frame"], record["value"], record["actions"]), (bounds, "visible", ["click"]))
+
+
class FocusDetailsTests(unittest.TestCase):
def setUp(self):
self.runtime = load_runtime()
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 83254de..8fc5d79 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -139,6 +139,7 @@
- Linux runtime 需要运行在已登录桌面用户 session 里。缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,Go runtime 会在启动 Python AT-SPI bridge 前尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session bus、display / Wayland 值;纯 SSH tty 如果找不到已登录桌面 session,可以启动二进制,但不能直接 inspect 或操作 GUI session。
- `get_app_state` 的 accessibility tree 在 GTK/GNOME app 上可能很深,Linux bridge 使用与 macOS / Windows 一致的 1200 节点、64 层默认 tree budget,并支持显式提高 `max_tree_nodes` / `max_tree_depth`。截图通过 GDK root window best-effort capture;GNOME Wayland 可能返回黑图,bridge 会检测全黑采样并省略 image block。
- 焦点摘要和选区在快照已经选定的窗口里共用一次查找,分别受 `max_tree_nodes` / `max_tree_depth` 约束;到深度边界后不再读取下一层子节点,查找未完成时追加明确提示。选中文字使用标准 AT-SPI `get_n_selections` / `get_selection` 接口读取。实现与隔离 GTK 验证见 [Linux 焦点查找](linux-bounded-focus.md)。
+- `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN=1` 可省略明确非 SHOWING 节点的位置、值等详情,保留节点、索引及子树;默认关闭,未知状态继续完整读取。使用方式与真实 GTK 验证见 [Linux 隐藏控件详情](linux-hidden-details.md)。
- 这 9 个 tool 的协议面与 macOS / Windows 保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime path metadata,coordinate action 使用 screenshot/window-relative 坐标。
- Linux `click_method=accessibility` 映射到 AT-SPI action,`global` 映射到 AT-SPI mouse synthesis 并要求全局指针环境变量;AT-SPI 没有等价的进程定向 mouse dispatch,因此 `app_post` 和 macOS-only 的 `sky_click` 会在 snapshot lookup 前明确返回 unsupported,`key_method=sky_key` 也一样。`auto` 仍保持 AT-SPI action 优先、mouse synthesis fallback 的现有行为。
- Linux 动作回读已有独立 GTK fixture,可在私有 D-Bus / Xvfb 中通过 native MCP 复测;运行方式见 [Linux 动作回读](linux-action-read-back.md)。当前 Linux 侧仍是功能性第一版:没有 visual cursor overlay、没有 installer/desktop entry,也没有覆盖全部工具的 Linux fixture。后续 TODO 记录在 `docs/exec-plans/active/20260422-linux-computer-use-runtime.md`。
diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md
index d2bd90e..3bf1844 100644
--- a/docs/RELIABILITY.md
+++ b/docs/RELIABILITY.md
@@ -18,6 +18,7 @@
- macOS agent display 实机回归(会短暂创建并销毁一块 virtual display):`OPEN_COMPUTER_USE_RUN_AGENT_DISPLAY_LIVE_TEST=1 swift test --filter AgentDisplayLiveTests`
- macOS 其他 Desktop 实机回归(需要第二个 Desktop,测试会短暂切换 Space 启动目标):`OPEN_COMPUTER_USE_RUN_CROSS_SPACE_LIVE_TEST=1 swift test --filter CrossSpaceLiveTests`
- Linux runtime:`(cd apps/OpenComputerUseLinux && go test ./...)`、`./scripts/build-open-computer-use-linux.sh --arch arm64`
+- Linux 隐藏控件实测:`bash scripts/run-linux-read-back-smoke.sh /path/to/linux/open-computer-use --hidden-checks`,在私有 D-Bus / Xvfb 中验证真实 GTK 隐藏状态、属性读取、索引及页面切换;依赖与范围见 [测试说明](linux-hidden-details.md)。
- Windows runtime:`(cd apps/OpenComputerUseWindows && go test ./...)`、`./scripts/build-open-computer-use-windows.sh --arch amd64`;截图相关变更还要在 Windows 交互桌面验证正常窗口有 image、1x1/minimized/cloaked 窗口无 image。
- 本地诊断:
- `open-computer-use doctor`
diff --git a/docs/exec-plans/completed/20261017-pr103-hidden-details.md b/docs/exec-plans/completed/20261017-pr103-hidden-details.md
new file mode 100644
index 0000000..6f3a195
--- /dev/null
+++ b/docs/exec-plans/completed/20261017-pr103-hidden-details.md
@@ -0,0 +1,32 @@
+# PR #103: Linux hidden-node details
+
+## Goal
+
+Integrate latest main into the contributor branch, review the opt-in hidden-node optimization, test real GTK/AT-SPI behavior in an isolated X11 session, fix issues and push follow-up commits to the same PR. Do not merge.
+
+## Scope and constraints
+
+- Default snapshots, node order, indexes, runtime paths and visible descendants must retain their behavior.
+- Omit details only when AT-SPI explicitly reports a node as non-SHOWING. Unknown/failed state reads keep full details.
+- Use a disposable container on the existing engine with private Xvfb/D-Bus, not a desktop VM or personal applications. Do not claim Wayland or whole-task performance coverage.
+- Preserve the original contribution; update the existing history and feature documentation.
+
+## Progress
+
+- [x] Merge latest main; retain both test groups and release-note entries.
+- [x] Review state handling, action compatibility and omitted property reads.
+- [x] Add repeatable real GTK coverage and state-failure/negative-coordinate regressions; no functional fix required.
+- [x] Run relevant Python/Go/native MCP checks and document evidence.
+- [x] Synchronize remotes and prepare tested follow-up commits for the original #103 branch.
+
+## Validation
+
+Compare default and lightweight trees on real GTK controls with hidden/visible nodes, verify identity/index preservation and visible descendants, count actual property helper calls, and exercise native MCP actions with lightweight mode enabled. Include explicit unknown/failure fallbacks in unit tests. Run existing action read-back/focus smoke where compatible, Python and Linux Go suites, cross-builds, docs and whitespace checks. Container and fixture processes must be removed after testing.
+
+## Evidence
+
+- Main integrated through `eb501f4`, retaining original commit `917f2fe` and all upstream focus/read-back tests and release notes.
+- Python 29/29; Linux Go 25/25; Linux ARM64/AMD64 builds and targeted vet passed. Docs/whitespace and shell/Python syntax checks passed.
+- Real GTK 3 notebook on Debian 12 ARM64, private Xvfb/D-Bus and AT-SPI: 12 stable nodes, 3 hidden; four property-helper call totals 48 → 36. Helpers invoked real remote accessibles. These are helper-call counts, not D-Bus message counts or a latency benchmark.
+- Native MCP compared disabled/opt-in output and indexes, clicked to reveal hidden controls, observed restored values/geometry, returned to the original page, then passed the existing action/no-read-back sequence. Separate default-action smoke and both disabled/enabled focus/Unicode-selection smokes passed.
+- Temporary container, fixture processes and standalone build artifacts removed. No desktop VM, host personal-app actions, version bump, Wayland/other-toolkit validation or whole-task performance claim.
diff --git a/docs/histories/2026-10/20261015-2035-linux-hidden-details.md b/docs/histories/2026-10/20261015-2035-linux-hidden-details.md
new file mode 100644
index 0000000..b9e74cd
--- /dev/null
+++ b/docs/histories/2026-10/20261015-2035-linux-hidden-details.md
@@ -0,0 +1,28 @@
+# Add opt-in lightweight details for hidden Linux nodes
+
+Agent: Codex; runtime: Linux.
+
+User request: submit scoped improvements for the remaining Linux performance paths.
+
+With `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN=1`, snapshot rendering retains every node, index, runtime path, role and name, but skips geometry/value/action/toolkit/automation-ID reads for nodes explicitly reported as non-SHOWING by AT-SPI. Such rows are marked `[hidden details omitted]`.
+
+This option is disabled by default. Unknown or failing state queries keep full details. The renderer never prunes a hidden parent's descendants: visible menu children retain their full properties and original numbering. Geometry alone is not evidence that a node is hidden, so off-window/negative coordinates are not used as a filtering rule.
+
+Validation: Python bridge tests (13 passed), including skipped property calls, unchanged default output, unknown state fallback, visible children of hidden parents and index/path preservation. No live desktop benchmark or claim that all offscreen nodes are optimized.
+
+Changed files: `apps/OpenComputerUseLinux/runtime.py`, `apps/OpenComputerUseLinux/runtime_test.py`, `docs/linux-hidden-details.md`, architecture and release notes.
+
+## 2026-10-17:同步主线并补真实 GTK 验证
+
+用户要求基于最新 main 解决冲突、检查测试并推送原 PR,补充真实 GTK 覆盖;本机没有完整 Linux 桌面,不启动桌面虚拟机。
+
+- 合入主线 `eb501f4`,保留贡献者原提交,同时保留主线动作回读、焦点与选区测试以及发布记录。
+- 检查后未发现需要修改功能实现的问题。增加状态集获取失败、缺失、contains 异常以及可见节点负坐标的回归,确保异常不会误省略详情。
+- 扩展已有 GTK smoke 的 `--hidden-checks`:创建两页 notebook,直接读取真实远程 AT-SPI 对象,对比所有节点的索引 / 路径 / 名称 / 角色,验证隐藏节点省略、显示节点记录不变,并包装真实属性辅助函数计数。通过 native MCP 验证输出、切换页面后详情恢复及原有动作链,fixture 独立导出状态确认动作结果。
+- 真实测试使用现有容器引擎上的临时 Debian 12 ARM64 容器、私有 Xvfb / D-Bus、GTK 3 / AT-SPI 2。不操作宿主个人应用,不启动桌面 VM。
+
+验证:Python 29 项通过;Linux Go 测试 25 项通过;ARM64 / AMD64 构建和 Linux-targeted vet 通过;隐藏控件 native MCP smoke、默认动作回读 smoke,以及开关两种状态下的焦点 / 中文 emoji 选区回归均通过。真实树有 12 个节点、3 个隐藏节点,四类属性辅助函数调用 48 → 36 次;这不是 D-Bus 消息数或耗时基准。文档与 whitespace 检查通过。
+
+范围:X11 / GTK / AT-SPI;没有做 Wayland、其他 toolkit 或全任务性能测试。每节点会增加 SHOWING 查询,因此保持默认关闭,不承诺所有界面都提速。无版本 bump。
+
+验证完成后已删除临时容器及独立构建制品,未影响已有容器服务。
diff --git a/docs/linux-hidden-details.md b/docs/linux-hidden-details.md
new file mode 100644
index 0000000..bd94099
--- /dev/null
+++ b/docs/linux-hidden-details.md
@@ -0,0 +1,30 @@
+# Linux 隐藏控件详情
+
+默认读取完整详情。设置 `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN=1` 后,AT-SPI 明确标为非 `SHOWING` 的控件会省略位置、值、动作、toolkit 和 automation ID,行内显示 `[hidden details omitted]`。
+
+```sh
+OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN=1 ocu js \
+ 'var app = await cua.getApp("gedit"); await app.getAXState();'
+```
+
+节点、索引、路径、角色和名称仍保留,也不会因为父节点隐藏就跳过子节点。状态读取失败或未知时继续读取完整详情;负坐标和窗口外位置不作为隐藏依据。控件重新显示后,下次观察会恢复详情。
+
+开启后每个节点会额外读取一次 `SHOWING` 状态,适合隐藏节点较多的界面,不保证所有界面更快。它不改变焦点查找、截图或动作投递,也不会自动打开。
+
+## 测试
+
+```sh
+python3 apps/OpenComputerUseLinux/runtime_test.py
+(cd apps/OpenComputerUseLinux && go test ./...)
+```
+
+真实 GTK 测试复用 [私有 D-Bus / Xvfb 环境](linux-action-read-back.md),不需要完整桌面或虚拟机:
+
+```sh
+(cd apps/OpenComputerUseLinux && go build -o /tmp/ocu-hidden-test .)
+bash scripts/run-linux-read-back-smoke.sh /tmp/ocu-hidden-test --hidden-checks
+```
+
+脚本创建两页 GTK 标签页,对比开关前后的完整树、索引、路径和属性辅助函数调用次数,再通过 native MCP 切换页面、读取恢复的详情,并验证点击、输入和动作回读。属性计数调用的仍是真实 AT-SPI 接口;单测另覆盖状态异常和隐藏父节点下的可见子节点。
+
+本次 X11 / GTK 3 / AT-SPI 实测为 12 个节点、3 个隐藏节点,四类属性辅助函数共调用 48 → 36 次。这不是 D-Bus 消息计数或耗时基准,不代表 Wayland、其他 toolkit 或完整任务的速度。
diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.md
index 7b34ca3..c63ff06 100644
--- a/docs/releases/feature-release-notes.md
+++ b/docs/releases/feature-release-notes.md
@@ -12,6 +12,7 @@
| 2026-10-15 | Locked Use 安装与性能修复 | 减少认证和 IPC 开销,旧系统组件也能正常移除并重装。 | 非 UI 插件使用 privileged host,安装前验证真实加载;修复系统缓存卸载与转交 socket 的生命周期,降低空闲唤醒,status 提示组件版本差异。 |
| 2026-10-15 | Linux 动作回读 | REPL 批量操作可省去每个动作后被丢弃的完整界面观察。 | Linux 开始支持现有的 `OPEN_COMPUTER_USE_ACTION_READ_BACK=0`,成功动作返回 `ok`,保留最后一次显式观察的索引缓存;默认原生 MCP 回读与显式 `get_app_state` 保持不变。 |
| 2026-10-15 | Linux 焦点与选区 | 大型界面减少重复查找,预算不足时明确提示,选中文字能正确返回。 | 复用快照已选窗口,一次查找生成焦点摘要与选区;按节点和深度预算停止读取,使用标准 AT-SPI 选区接口,并补隔离 GTK 验证。 |
+| 2026-10-15 | Linux 隐藏控件详情 | 隐藏节点较多时,可减少不需要的属性读取。 | 新增默认关闭的 `OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN`;保留节点、索引和可见子节点,状态未知时完整读取,补充真实 GTK 标签页及 native MCP 回归。详见 [使用与验证](../linux-hidden-details.md)。 |
| 2026-10-14 | 1.1.0 正式发布 | 用户可直接安装 Locked Use,在锁屏遮罩期间继续 GUI 操作,也可用定向查询和持久截图配置减少工具开销。 | 统一版本为 1.1.0;签名 universal macOS App 内嵌服务、Guardian、安装器和授权插件,普通 enable 请求权限,补齐冷启动握手隔离和发布制品检查;新增 app.query()、doctor --json 与 ocu config。 |
| 2026-10-10 | macOS Locked Use 验证版 | 可在已登录 Mac 锁屏时,通过全屏遮罩继续 GUI 自动化,并在结束或本地接管后安全重锁。 | 正式组件采用 OCU Guardian / OCULockService / OCULockInstaller / OCULockAuth 短名称,开发 App 保留 Dev 区分;新增签名 Broker / 授权插件、双进程显示器保护、一次性许可、租约排空 / 恢复及构建和实机验证脚本;管理员显式启用 validation profile,生产 backend 在完整验证前保持关闭。 |
| 2026-10-10 | macOS 定向 AX 查询 | 知道控件文案时,可以直接查询并操作,不必先获取截图和完整 AX tree。 | 新增 native `query` 和 JS `app.query()`,按文字、角色和窗口查找元素,返回完整性元数据;查询索引仅在同一会话内有效,操作前重新验证目标和位置。 |
diff --git a/scripts/run-linux-read-back-smoke.sh b/scripts/run-linux-read-back-smoke.sh
index 3d4c7c2..8a9507f 100755
--- a/scripts/run-linux-read-back-smoke.sh
+++ b/scripts/run-linux-read-back-smoke.sh
@@ -3,7 +3,7 @@ set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
if [[ "$(uname -s)" != Linux || $# -lt 1 ]]; then
- echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use [--focus-checks]" >&2
+ echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use [--focus-checks | --hidden-checks]" >&2
exit 2
fi
diff --git a/scripts/tests/linux-read-back-smoke.py b/scripts/tests/linux-read-back-smoke.py
index 9ae62f4..cebad77 100644
--- a/scripts/tests/linux-read-back-smoke.py
+++ b/scripts/tests/linux-read-back-smoke.py
@@ -2,6 +2,8 @@
"""Exercise native Linux MCP actions against an isolated real GTK fixture."""
import argparse
+from contextlib import ExitStack
+import importlib.util
import json
import os
from pathlib import Path
@@ -11,9 +13,10 @@ import subprocess
import sys
import tempfile
import time
+from unittest import mock
-def fixture(name, state_file, focus_checks=False):
+def fixture(name, state_file, focus_checks=False, hidden_checks=False):
import gi
gi.require_version("Gtk", "3.0")
@@ -25,7 +28,22 @@ def fixture(name, state_file, focus_checks=False):
window = Gtk.Window(title=name)
window.set_default_size(500, 240)
box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12, margin=20)
- window.add(box)
+ notebook = None
+ if hidden_checks:
+ notebook = Gtk.Notebook()
+ window.add(notebook)
+ notebook.append_page(box, Gtk.Label(label="Main page"))
+ hidden_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12, margin=20)
+ hidden_entry = Gtk.Entry(text="hidden-value-中文🙂")
+ hidden_entry.get_accessible().set_name("Hidden input")
+ hidden_button = Gtk.Button(label="Return page")
+ hidden_box.pack_start(hidden_entry, False, False, 0)
+ hidden_box.pack_start(hidden_button, False, False, 0)
+ notebook.append_page(hidden_box, Gtk.Label(label="Other page"))
+ reveal = Gtk.Button(label="Reveal page")
+ box.pack_start(reveal, False, False, 0)
+ else:
+ window.add(box)
entry = Gtk.Entry()
entry.get_accessible().set_name("Read-back input")
button = Gtk.Button(label="Increment")
@@ -39,6 +57,8 @@ def fixture(name, state_file, focus_checks=False):
state["focused"] = entry.has_focus()
bounds = entry.get_selection_bounds()
state["selected"] = entry.get_text()[bounds[0]:bounds[1]] if bounds else ""
+ if notebook is not None:
+ state["page"] = notebook.get_current_page()
temporary = state_file.with_suffix(".tmp")
temporary.write_text(json.dumps(state))
temporary.replace(state_file)
@@ -60,6 +80,10 @@ def fixture(name, state_file, focus_checks=False):
entry.connect("notify::selection-bound", publish)
entry.connect("notify::cursor-position", publish)
entry.connect("notify::has-focus", publish)
+ if notebook is not None:
+ reveal.connect("clicked", lambda *_: notebook.set_current_page(1))
+ hidden_button.connect("clicked", lambda *_: notebook.set_current_page(0))
+ notebook.connect("switch-page", lambda *_: GLib.idle_add(publish))
window.connect("destroy", Gtk.main_quit)
window.show_all()
entry.grab_focus()
@@ -89,8 +113,9 @@ def wait_for(predicate, description):
class MCP:
- def __init__(self, native, read_back, stderr):
+ def __init__(self, native, read_back, stderr, env_overrides=None):
env = dict(os.environ)
+ env.update(env_overrides or {})
env.pop("OPEN_COMPUTER_USE_ACTION_READ_BACK", None)
if read_back is not None:
env["OPEN_COMPUTER_USE_ACTION_READ_BACK"] = read_back
@@ -183,7 +208,84 @@ def check_focus(client, name, state_file):
print("Focus smoke: node/depth limits, Unicode selection, text limit and focus summary passed.")
-def smoke(native, focus_checks=False):
+def check_hidden(client, native, name, state_file, stderr):
+ # Import the production bridge and wrap its real property helpers. All
+ # objects below are remote GTK/AT-SPI accessibles, not synthetic records.
+ runtime_path = Path(__file__).resolve().parents[2] / "apps/OpenComputerUseLinux/runtime.py"
+ spec = importlib.util.spec_from_file_location("gtk_smoke_runtime", runtime_path)
+ runtime = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(runtime)
+ app = runtime.resolve_app(name)
+ window_index, window = runtime.main_window(app)
+ bounds = runtime.extents(window)
+ flag = "OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN"
+ helpers = ("relative_frame", "accessible_id", "element_value", "action_names")
+
+ def render(enabled):
+ with ExitStack() as stack:
+ stack.enter_context(mock.patch.dict(os.environ, {flag: "1" if enabled else "0"}))
+ calls = [stack.enter_context(mock.patch.object(runtime, key, wraps=getattr(runtime, key)))
+ for key in helpers]
+ records, lines = runtime.render_tree(window, bounds, [window_index])
+ return records, lines, [query.call_count for query in calls]
+
+ full, _, full_calls = render(False)
+ light, _, light_calls = render(True)
+ identity = ("index", "runtimeId", "name", "controlType", "localizedControlType")
+ assert [[row[key] for key in identity] for row in full] == [
+ [row[key] for key in identity] for row in light], (full, light)
+ omitted = [row for row in light if row.get("detailsOmitted")]
+ assert any(row["name"] == "Hidden input" for row in omitted), light
+ assert next(row for row in full if row["name"] == "Hidden input")["value"] == "hidden-value-中文🙂", full
+ for original, row in zip(full, light):
+ node = runtime.resolve_path(app, row["runtimeId"])
+ showing = node.get_state_set().contains(runtime.Atspi.StateType.SHOWING)
+ assert bool(row.get("detailsOmitted")) == (not showing), row
+ if row.get("detailsOmitted"):
+ assert row["frame"] is None and row["value"] == "" and row["actions"] == [], row
+ assert row["className"] == "" and row["automationId"] == "", row
+ else:
+ assert row == original, (original, row)
+ assert full_calls == [len(full)] * len(helpers), full_calls
+ assert light_calls == [len(full) - len(omitted)] * len(helpers), light_calls
+ print(f"Real AT-SPI tree: {len(full)} stable nodes, {len(omitted)} hidden; "
+ f"property helpers {sum(full_calls)} → {sum(light_calls)} (not a latency benchmark).")
+
+ baseline = MCP(native, "0", stderr, {flag: "0"})
+ try:
+ default = baseline.call("get_app_state", name)
+ optimized = client.call("get_app_state", name)
+ expect_snapshot(default)
+ expect_snapshot(optimized)
+ assert "[hidden details omitted]" not in text(default), text(default)
+ assert "hidden-value-中文🙂" in text(default), text(default)
+ assert "hidden-value-中文🙂" not in text(optimized), text(optimized)
+ assert "[hidden details omitted]" in text(optimized), text(optimized)
+ for title in ("Read-back input", "Increment", "Hidden input", "Return page", "Reveal page"):
+ assert element_index(default, title) == element_index(optimized, title), title
+ finally:
+ baseline.close()
+
+ def page_is(number):
+ return json.loads(state_file.read_text())["page"] == number
+
+ result = client.call("click", name, element_index=element_index(optimized, "Reveal page"),
+ click_method="accessibility")
+ assert not result.get("isError"), result
+ wait_for(lambda: page_is(1), "real notebook reveal")
+ shown = client.call("get_app_state", name)
+ expect_snapshot(shown)
+ row = next(line for line in text(shown).splitlines()
+ if re.match(r"\s*\d+\s", line) and "Hidden input" in line)
+ assert "hidden-value-中文🙂" in row and "[hidden details omitted]" not in row and "Frame:" in row, row
+ result = client.call("click", name, element_index=element_index(shown, "Return page"),
+ click_method="accessibility")
+ assert not result.get("isError"), result
+ wait_for(lambda: page_is(0), "real notebook return")
+ print("Native MCP: default/opt-in output, stable indexes, page changes and full detail restoration passed.")
+
+
+def smoke(native, focus_checks=False, hidden_checks=False):
with tempfile.TemporaryDirectory(prefix="ocu-read-back-smoke-") as directory:
directory = Path(directory)
state_file = directory / "state.json"
@@ -192,17 +294,22 @@ def smoke(native, focus_checks=False):
arguments = [sys.executable, __file__, "--fixture", name, "--state-file", str(state_file)]
if focus_checks:
arguments.append("--focus-checks")
+ if hidden_checks:
+ arguments.append("--hidden-checks")
process = subprocess.Popen(arguments, stderr=stderr)
clients = []
try:
wait_for(state_file.exists, "GTK fixture startup")
- client = MCP(native, " OFF ", stderr)
+ overrides = {"OPEN_COMPUTER_USE_LINUX_LIGHTWEIGHT_HIDDEN": "1"} if hidden_checks else None
+ client = MCP(native, " OFF ", stderr, overrides)
clients.append(client)
# AT-SPI registration follows GTK startup asynchronously.
wait_for(lambda: not client.call("get_app_state", name).get("isError"), "AT-SPI registration")
if focus_checks:
check_focus(client, name, state_file)
return
+ if hidden_checks:
+ check_hidden(client, native, name, state_file, stderr)
observed = client.call("get_app_state", name)
expect_snapshot(observed)
input_index = element_index(observed, "Read-back input")
@@ -271,12 +378,14 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--native")
parser.add_argument("--fixture")
- parser.add_argument("--focus-checks", action="store_true")
+ checks = parser.add_mutually_exclusive_group()
+ checks.add_argument("--focus-checks", action="store_true")
+ checks.add_argument("--hidden-checks", action="store_true")
parser.add_argument("--state-file", type=Path)
args = parser.parse_args()
if args.fixture:
- fixture(args.fixture, args.state_file, args.focus_checks)
+ fixture(args.fixture, args.state_file, args.focus_checks, args.hidden_checks)
else:
if not args.native:
parser.error("--native is required")
- smoke(str(Path(args.native).resolve()), args.focus_checks)
+ smoke(str(Path(args.native).resolve()), args.focus_checks, args.hidden_checks)