👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
fix(linux): honor action read-back configuration (#100)
draplater1b6fd8f8354f
diff --git a/apps/OpenComputerUseLinux/main.go b/apps/OpenComputerUseLinux/main.goindex 2d9032c..41d8812 100644--- a/apps/OpenComputerUseLinux/main.go+++ b/apps/OpenComputerUseLinux/main.go@@ -161,6 +161,7 @@ type linuxRequest struct {TextLimit any `json:"text_limit,omitempty"`MaxTreeNodes int `json:"max_tree_nodes,omitempty"`MaxTreeDepth int `json:"max_tree_depth,omitempty"`+ SkipReadBack bool `json:"skip_read_back,omitempty"`}type textLimit struct {@@ -467,6 +468,18 @@ func (s *service) setValue(app, elementIndex, value string) toolCallResult {}func (s *service) actionResult(app string, request linuxRequest) toolCallResult {+ if !actionReadBackEnabled() {+ request.SkipReadBack = true+ response, err := runPython(request)+ if err != nil {+ return textResult(err.Error(), true)+ }+ if !response.OK {+ return textResult(response.Error, true)+ }+ // Keep the last explicitly observed indexes until get_app_state refreshes them.+ return textResult("ok", false)+ }snapshot, result := s.refreshSnapshot(app, request)if result.IsError {return result@@ -474,6 +487,16 @@ func (s *service) actionResult(app string, request linuxRequest) toolCallResultreturn snapshot.result()}+// Match the macOS runtime: only explicit false values disable action read-back.+func actionReadBackEnabled() bool {+ switch strings.ToLower(strings.TrimSpace(os.Getenv("OPEN_COMPUTER_USE_ACTION_READ_BACK"))) {+ case "0", "false", "no", "off":+ return false+ default:+ return true+ }+}+func (s *service) currentSnapshot(app string) *appSnapshot {return s.snapshots[strings.ToLower(app)]}diff --git a/apps/OpenComputerUseLinux/main_test.go b/apps/OpenComputerUseLinux/main_test.goindex 210d471..13c4f7c 100644--- a/apps/OpenComputerUseLinux/main_test.go+++ b/apps/OpenComputerUseLinux/main_test.go@@ -16,6 +16,108 @@ func TestToolDefinitionCount(t *testing.T) {}}+func TestActionReadBackEnabled(t *testing.T) {+ for _, value := range []string{"", "1", "true", "yes", "on", "unexpected"} {+ t.Setenv("OPEN_COMPUTER_USE_ACTION_READ_BACK", value)+ if !actionReadBackEnabled() {+ t.Errorf("read-back should be enabled for %q", value)+ }+ }+ for _, value := range []string{"0", "false", "no", "off", " OFF ", "False"} {+ t.Setenv("OPEN_COMPUTER_USE_ACTION_READ_BACK", value)+ if actionReadBackEnabled() {+ t.Errorf("read-back should be disabled for %q", value)+ }+ }+}++// Exercise the Go/bridge JSON boundary without requiring a live AT-SPI desktop.+func fakePythonResponse(t *testing.T, response string) string {+ t.Helper()+ dir := t.TempDir()+ requestPath := filepath.Join(dir, "request.json")+ script := "#!/bin/sh\ncat \"$2\" > \"$OCU_TEST_REQUEST\"\nprintf '%s\\n' \"$OCU_TEST_PYTHON_RESPONSE\"\n"+ if err := os.WriteFile(filepath.Join(dir, "python3"), []byte(script), 0o700); err != nil {+ t.Fatal(err)+ }+ t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))+ t.Setenv("OCU_TEST_REQUEST", requestPath)+ t.Setenv("OCU_TEST_PYTHON_RESPONSE", response)+ return requestPath+}++func TestActionWithoutReadBackPreservesObservedSnapshot(t *testing.T) {+ t.Setenv("OPEN_COMPUTER_USE_ACTION_READ_BACK", "0")+ requestPath := fakePythonResponse(t, `{"ok":true}`)+ s := newService()+ observed := &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}, Elements: []elementRecord{{Index: 7}}}+ s.rememberSnapshot("Editor", observed)+ result := s.pressKey("Editor", "Return", "auto")+ if result.IsError || len(result.Content) != 1 || result.Content[0].Text != "ok" {+ t.Fatalf("action result = %#v", result)+ }+ for _, alias := range []string{"Editor", "42"} {+ if s.currentSnapshot(alias) != observed {+ t.Fatalf("action replaced the observed snapshot for %s", alias)+ }+ }+ data, err := os.ReadFile(requestPath)+ if err != nil {+ t.Fatal(err)+ }+ var request linuxRequest+ if err := json.Unmarshal(data, &request); err != nil {+ t.Fatal(err)+ }+ if !request.SkipReadBack || request.Tool != "press_key" || request.Key != "Return" {+ t.Fatalf("bridge request = %#v", request)+ }+}++func TestActionReadBackStillRefreshesSnapshotByDefault(t *testing.T) {+ t.Setenv("OPEN_COMPUTER_USE_ACTION_READ_BACK", "")+ requestPath := fakePythonResponse(t, `{"ok":true,"snapshot":{"app":{"name":"Editor","pid":42},"treeLines":["updated"],"screenshotPngBase64":"aW1hZ2U="}}`)+ s := newService()+ s.rememberSnapshot("Editor", &appSnapshot{App: appDescriptor{Name: "Editor", PID: 42}})+ result := s.pressKey("Editor", "Return", "auto")+ if result.IsError || len(result.Content) != 2 || result.Content[1].Type != "image" {+ t.Fatalf("default action result = %#v", result)+ }+ if got := s.currentSnapshot("Editor").TreeLines; len(got) != 1 || got[0] != "updated" {+ t.Fatalf("snapshot was not refreshed: %#v", got)+ }+ data, err := os.ReadFile(requestPath)+ if err != nil {+ t.Fatal(err)+ }+ if strings.Contains(string(data), "skip_read_back") {+ t.Fatalf("default action unexpectedly requested no read-back: %s", data)+ }+}++func TestReadBackFlagDoesNotSuppressExplicitObservation(t *testing.T) {+ t.Setenv("OPEN_COMPUTER_USE_ACTION_READ_BACK", "0")+ fakePythonResponse(t, `{"ok":true,"snapshot":{"app":{"name":"Editor","pid":42}}}`)+ s := newService()+ _, result := s.refreshSnapshot("Editor", linuxRequest{Tool: "get_app_state", App: "Editor"})+ if result.IsError || s.currentSnapshot("Editor") == nil {+ t.Fatalf("explicit observation failed: %#v", result)+ }+}++func TestActionWithoutReadBackPropagatesBridgeErrors(t *testing.T) {+ for _, response := range []string{`{"ok":false,"error":"input failed"}`, `not json`} {+ t.Run(response, func(t *testing.T) {+ t.Setenv("OPEN_COMPUTER_USE_ACTION_READ_BACK", "0")+ fakePythonResponse(t, response)+ s := newService()+ if result := s.actionResult("Editor", linuxRequest{Tool: "press_key"}); !result.IsError {+ t.Fatalf("bridge failure reported as success: %#v", result)+ }+ })+ }+}+func TestClickMethodSchemaAndParser(t *testing.T) {tool := findToolDefinition(t, "click")properties := tool.InputSchema["properties"].(map[string]any)diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.pyindex 0759575..0aee2cc 100644--- a/apps/OpenComputerUseLinux/runtime.py+++ b/apps/OpenComputerUseLinux/runtime.py@@ -874,6 +874,9 @@ def perform_operation(operation):else:raise RuntimeError('unsupportedTool("{}")'.format(tool))+ if operation.get("skip_read_back", False):+ return {"ok": True}+time.sleep(0.12)return {"ok": True, "snapshot": build_snapshot(operation.get("app", ""))}diff --git a/apps/OpenComputerUseLinux/runtime_test.py b/apps/OpenComputerUseLinux/runtime_test.pyindex f52e8e3..fb7b922 100644--- a/apps/OpenComputerUseLinux/runtime_test.py+++ b/apps/OpenComputerUseLinux/runtime_test.py@@ -1,3 +1,4 @@+import contextlibimport importlib.utilimport pathlibimport sys@@ -171,5 +172,56 @@ class RuntimeKeyTests(unittest.TestCase):self.assertEqual(self.events, [])+class RuntimeActionReadBackTests(unittest.TestCase):+ def setUp(self):+ self.runtime = load_runtime()+ self.stack = contextlib.ExitStack()+ self.addCleanup(self.stack.close)+ self.stack.enter_context(mock.patch.object(self.runtime, "resolve_app"))+ self.stack.enter_context(mock.patch.object(self.runtime, "main_window", return_value=(0, object())))+ for name in ("extents", "find_element", "send_key", "send_drag", "send_mouse_click",+ "invoke_secondary_action", "scroll_element", "insert_text", "set_element_value"):+ self.stack.enter_context(mock.patch.object(self.runtime, name))+ self.snapshot = self.stack.enter_context(mock.patch.object(self.runtime, "build_snapshot", return_value={"elements": []}))+ self.sleep = self.stack.enter_context(mock.patch.object(self.runtime.time, "sleep"))++ def test_all_actions_skip_snapshot_and_settle_when_requested(self):+ for tool, arguments in (+ ("click", {"x": 1, "y": 2}),+ ("drag", {"from_x": 1, "from_y": 2, "to_x": 3, "to_y": 4}),+ ("press_key", {"key": "Return"}),+ ("type_text", {"text": "hello"}),+ ("set_value", {"value": "hello"}),+ ("perform_secondary_action", {"action": "activate"}),+ ("scroll", {"direction": "down", "pages": 1}),+ ):+ with self.subTest(tool=tool):+ with mock.patch.object(self.runtime, "screen_point", side_effect=[(1, 2), (3, 4)]):+ result = self.runtime.perform_operation({"tool": tool, "app": "Editor", "skip_read_back": True, **arguments})+ self.assertEqual(result, {"ok": True})+ self.snapshot.assert_not_called()+ self.sleep.assert_not_called()+ self.runtime.send_key.assert_called_once_with("Return")+ self.runtime.send_drag.assert_called_once_with(1, 2, 3, 4)+ self.runtime.insert_text.assert_called_once()++ def test_default_action_still_waits_and_returns_snapshot(self):+ result = self.runtime.perform_operation({"tool": "press_key", "app": "Editor", "key": "Return"})+ self.assertEqual(result, {"ok": True, "snapshot": {"elements": []}})+ self.sleep.assert_called_once_with(0.12)+ self.snapshot.assert_called_once_with("Editor")++ def test_explicit_observation_is_not_suppressed(self):+ result = self.runtime.perform_operation({"tool": "get_app_state", "app": "Editor", "skip_read_back": True})+ self.assertIn("snapshot", result)+ self.snapshot.assert_called_once()++ def test_action_errors_are_not_suppressed(self):+ self.runtime.send_key.side_effect = RuntimeError("input failed")+ with self.assertRaisesRegex(RuntimeError, "input failed"):+ self.runtime.perform_operation({"tool": "press_key", "app": "Editor", "key": "Return", "skip_read_back": True})+ self.snapshot.assert_not_called()++if __name__ == "__main__":unittest.main()diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 9f72683..6fd2d50 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -131,6 +131,7 @@### 7. Linux Runtime- Linux runtime 位于 `apps/OpenComputerUseLinux`,以 Go 维护 CLI、`call --calls` sequence、MCP JSON-RPC、tool schema 和进程内 snapshot cache。+- Linux 支持与 macOS 一致的 `OPEN_COMPUTER_USE_ACTION_READ_BACK`:默认保持动作后等待并返回完整快照;设置为 `0`、`false`、`no` 或 `off`(忽略大小写与首尾空白)时,成功动作只返回 `ok`,跳过回读等待、AX、焦点/选区和截图采集,也不覆盖最后一次显式观察的索引缓存。REPL adapter 已自动设置此开关;调用方应在需要观察时显式调用 `get_app_state` / `getAXState()`,并在界面变化后刷新索引。`ok` 仅表示动作调用完成,不代表界面已完成更新或任务成功。显式观察不受开关影响,仍返回完整快照。- 构建入口是 `scripts/build-open-computer-use-linux.sh --arch arm64|amd64`,默认输出到 `dist/linux/<arch>/open-computer-use`;npm release package 会把两个 Linux artifact 内置到已有 root/alias packages,Node launcher 按 `process.platform/process.arch` 自动选择。- Go runtime 通过 `go:embed` 带上 `runtime.py`,执行 tool call 时临时落盘并调用 `python3`。Python bridge 使用 GNOME/GObject Introspection 暴露的 AT-SPI2 接口做 app/window discovery、accessibility tree rendering、semantic action、editable text、value set,以及 best-effort 的 key/mouse fallback;文本能力通过 `Accessible.get_interfaces()` 检测 `Text` / `EditableText`,不依赖不同 PyGObject 版本未必存在的便捷属性。- Linux 上最接近 macOS AX 的是 AT-SPI2/D-Bus accessibility,而不是一套统一的后台键鼠输入模型。第一版优先使用元素暴露的 AT-SPI action、EditableText 和 Value 接口;coordinate `click` / `drag` 与 `press_key` 使用 AT-SPI event synthesis fallback,在 Wayland 下只能按 best-effort 处理。`press_key` 的具名键以 keysym 配合 `KeySynthType.SYM` 发送,修饰键以 X modifier mask 配合 `LOCKMODIFIERS` / `UNLOCKMODIFIERS` 保持;`PRESS` / `RELEASE` / `PRESSRELEASE` 只接受硬件 keycode,不能传 keysym。@@ -138,7 +139,7 @@- `get_app_state` 的 accessibility tree 在 GTK/GNOME app 上可能很深,Linux bridge 使用与 macOS / Windows 一致的 1200 节点、64 层默认 tree budget,并支持显式提高 `max_tree_nodes` / `max_tree_depth`。截图通过 GDK root window best-effort capture;GNOME Wayland 可能返回黑图,bridge 会检测全黑采样并省略 image block。- 这 9 个 tool 的协议面与 macOS / Windows 保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime path metadata,coordinate action 使用 screenshot/window-relative 坐标。- Linux `click_method=accessibility` 映射到 AT-SPI action,`global` 映射到 AT-SPI mouse synthesis 并要求全局指针环境变量;AT-SPI 没有等价的进程定向 mouse dispatch,因此 `app_post` 和 macOS-only 的 `sky_click` 会在 snapshot lookup 前明确返回 unsupported,`key_method=sky_key` 也一样。`auto` 仍保持 AT-SPI action 优先、mouse synthesis fallback 的现有行为。-- 当前 Linux 侧仍是功能性第一版:没有 visual cursor overlay、没有 installer/desktop entry,也没有独立 Linux fixture。后续 TODO 记录在 `docs/exec-plans/active/20260422-linux-computer-use-runtime.md`。+- Linux 动作回读已有独立 GTK fixture,可在私有 D-Bus / Xvfb 中通过 native MCP 复测;运行方式见 [Linux 动作回读](linux-action-read-back.md)。当前 Linux 侧仍是功能性第一版:没有 visual cursor overlay、没有 installer/desktop entry,也没有覆盖全部工具的 Linux fixture。后续 TODO 记录在 `docs/exec-plans/active/20260422-linux-computer-use-runtime.md`。## macOS Locked Use 实验diff --git a/docs/QUALITY_SCORE.md b/docs/QUALITY_SCORE.mdindex e680e7e..a6200bb 100644--- a/docs/QUALITY_SCORE.md+++ b/docs/QUALITY_SCORE.md@@ -13,7 +13,7 @@| --- | --- | --- | --- || 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导,以及一轮按官方 surface / result 行为收敛过的 9 个 tools。 | 继续收敛复杂 AX 场景下的 state rendering 细节、权限 UI 和更清晰的用户错误提示。 || Windows runtime | C | 已新增独立 Go `.exe`,通过 Windows UI Automation + Win32 window message 暴露同样 9 个 tools、MCP server 和 `call --calls`;默认不再自动启动 app、执行 `SetFocus`,或让 `type_text` 走可能抢前台的 UIA text fallback;截图会过滤隐藏、最小化、cloaked、离屏和退化窗口,并在 Go 层复核 PNG,但整体仍是功能性第一版。 | 把当前实机 screenshot 边界用例收敛成仓库内 Windows fixture / 交互式 smoke,再补 installer/signing,以及更原生的 Go UIA 实现或更稳定的 bridge。 |-| Linux runtime | C | 已新增独立 Go binary,通过 Python GI / AT-SPI2 暴露同样 9 个 tools、MCP server 和 `call --calls`;Ubuntu GNOME VM 已跑通 `list_apps`、MCP tools list 和 Text Editor 8-tool sequence,并已接入 npm bundled artifact 分发,但截图在 GNOME Wayland 下仍只能 best-effort,coordinate input 也不是通用后台模型。 | 补 Linux fixture、可重复 smoke runner、portal/compositor screenshot 路径,以及更原生的 Go D-Bus/libatspi bridge。 |+| Linux runtime | C | 已新增独立 Go binary,通过 Python GI / AT-SPI2 暴露同样 9 个 tools、MCP server 和 `call --calls`;Ubuntu GNOME VM 已跑通 `list_apps`、MCP tools list 和 Text Editor 8-tool sequence,并已接入 npm bundled artifact 分发,但截图在 GNOME Wayland 下仍只能 best-effort,coordinate input 也不是通用后台模型。 | 扩展已有的隔离 GTK 动作回读 smoke,补完整工具覆盖、portal/compositor screenshot 路径,以及更原生的 Go D-Bus/libatspi bridge。 || macOS Locked Use | D | 已有签名 Broker / 插件、双进程保护、原生租约与恢复入口,离线和遮罩 rehearsal 有回归;安装恢复和无许可双保护释放已实测,独立无遮罩 Return 解锁 / 重锁已实测,受保护自动解锁 / AX / SCK / 重锁及隔离 legacy Keychain 已有一轮实测通过,完整会话认证状态 / 故障链路尚未通过实测;正常启用改为管理员 enable 与签名校验,普通 enable / 正常 MCP 的完整 GUI 闭环已实测。 | 物理覆盖已确认,继续完成 Data Protection / 会话认证状态、Secure Input / 崩溃 / 显示器故障回归及正式发布制品检查。 || 架构文档 | B | 顶层结构、fixture bridge、app 模式和验证路径已经落文档。 | 后续补 release artifact、code signing / notarization 和 host 集成方式。 || 测试 | B | `swift test` + smoke suite 已覆盖 9 个 tools 的回归,并新增了针对“前台焦点是否被抢占”的手工对比样本沉淀。 | 增加更多普通 app 的录制回归,减少只依赖 fixture 和一次性手工检查。 |diff --git a/docs/exec-plans/active/20260422-linux-computer-use-runtime.md b/docs/exec-plans/active/20260422-linux-computer-use-runtime.mdindex ecf1a02..1c9ece5 100644--- a/docs/exec-plans/active/20260422-linux-computer-use-runtime.md+++ b/docs/exec-plans/active/20260422-linux-computer-use-runtime.md@@ -84,7 +84,7 @@- [x] 在 Ubuntu VM 中验证 8-tool sequence:`get_app_state`、`set_value`、`type_text`、`press_key`、`perform_secondary_action`、`click`、`scroll`、`drag` 均 `isError=false`。- [x] 在 Ubuntu VM 中验证 `0.1.36` 预发布二进制可在 `leo` 用户 `env -i` 下自动发现 session env,并跑通 MCP `tools/list`、`tools/call(list_apps)` 和 9-tool sequence。- [x] 在 Ubuntu VM 中验证 `npm i -g [email protected]` 后,npm launcher 选择 Linux arm64 binary,`codex mcp list` 显示 `open-computer-use mcp` enabled,raw MCP `tools/list` 返回 9 个 tools,`call list_apps` 返回 `isError=false`。-- [ ] 增加 Linux fixture 和可重复 smoke runner。+- [ ] 扩展 Linux fixture 和可重复 smoke runner 到完整 9-tool sequence(已有动作回读的隔离 GTK / Xvfb smoke,见 `docs/linux-action-read-back.md`)。- [ ] 评估 xdg-desktop-portal / compositor-specific screenshot 路径,补非黑图 capture。- [x] 将 Linux artifact 接入 npm release packaging,作为既有 npm root/alias packages 的 bundled artifacts 分发。- [ ] 评估用原生 Go D-Bus/libatspi 替换 Python GI bridge 的收益和风险。diff --git a/docs/exec-plans/completed/20261017-pr100-linux-read-back.md b/docs/exec-plans/completed/20261017-pr100-linux-read-back.mdnew file mode 100644index 0000000..da9b6fd--- /dev/null+++ b/docs/exec-plans/completed/20261017-pr100-linux-read-back.md@@ -0,0 +1,31 @@+# PR #100: Linux action read-back on current main++## Objective++Resolve #100 against latest main, review and repair any issues, run Linux regression and isolated real-widget tests, then push to the original contributor branch and report merge readiness. Do not merge.++## Constraints++- Reuse the existing worktree and running container engine. Do not start a desktop VM.+- Test only disposable fixture widgets on an isolated virtual display; do not touch personal applications.+- Preserve default native read-back, explicit observation, error propagation and previously observed indexes when read-back is disabled.+- Preserve the contributor's commits. Synchronize remote refs before a fast-forward push.++## Progress++- [x] Merge current main; resolve the release-note conflict while retaining both entries.+- [x] Review Go/Python branches and add regressions for any uncovered defects.+- [x] Run Go, Python and Node tests on Linux; validate real GTK/AT-SPI actions with and without read-back.+- [x] Update existing history and relevant documentation; push follow-ups and verify PR status.++## Validation++Use a temporary Linux container with a private D-Bus session, Xvfb display and GTK fixture. Verify successful actions return only an acknowledgment when disabled and actual widget state changes on the next explicit observation; default actions still return a snapshot, failures still surface, and repeated operations use the preserved index cache. Record any platform or scenario limits without claiming broader desktop performance.++## Evidence and outcome++- Production changes merged cleanly; the sole conflict was the release-note table, resolved by retaining both contributions.+- Full Linux Go test binary passed in Debian 12 ARM64; Python 14/14 and Node 24 36/36 passed. Linux-targeted vet, ARM64/AMD64 builds and repository checks passed.+- The native MCP/GTK test independently verified real widget state after short acknowledgments, repeated indexes, Unicode input, Return, secondary actions and errors. Default snapshots and explicit observations passed.+- Unmodified main failed the regression for all seven action branches. No further production-code defect was found.+- The reusable smoke starts Xvfb before the private D-Bus session so the accessibility registry inherits the display for key synthesis. Coverage is X11/GTK/AT-SPI; Wayland, other toolkits and whole-task performance remain outside this validation.diff --git a/docs/histories/2026-10/20261015-2012-linux-action-read-back.md b/docs/histories/2026-10/20261015-2012-linux-action-read-back.mdnew file mode 100644index 0000000..54fbf90--- /dev/null+++ b/docs/histories/2026-10/20261015-2012-linux-action-read-back.md@@ -0,0 +1,41 @@+## [2026-10-15 20:12] | Task: Honor action read-back configuration on Linux++### Execution Context+- Agent: Codex+- Runtime: Linux development environment++### User Request+Submit a focused upstream fix for Linux ignoring the existing action read-back switch.++### Changes+- Match macOS parsing of `OPEN_COMPUTER_USE_ACTION_READ_BACK`; preserve read-back by default.+- Pass an internal `skip_read_back` request to the Python bridge for actions only.+- Skip the post-action wait and snapshot when disabled, return `ok`, and retain the last explicitly observed index cache.+- Keep explicit observations and input error propagation unchanged.+- Add bridge and service regression tests, architecture documentation, and a release note.++### Design Intent+The persistent REPL already disables action read-back and discards action results. Linux previously ignored this setting and repeatedly generated AX trees, focus/selection summaries, and screenshots. This change implements the existing contract without changing tool schemas or adding a separate screenshot-only mode. Callers must explicitly observe updated state before reusing indexes after UI changes; action acknowledgment is not proof of UI settlement or task success.++### Files Modified+- `apps/OpenComputerUseLinux/main.go`+- `apps/OpenComputerUseLinux/main_test.go`+- `apps/OpenComputerUseLinux/runtime.py`+- `apps/OpenComputerUseLinux/runtime_test.py`+- `docs/ARCHITECTURE.md`+- `docs/releases/feature-release-notes.md`++### Validation Scope+Automated tests cover the switch contract, all seven Python action branches, default read-back, explicit observations, preserved cache aliases, and bridge error responses. No live desktop benchmark is claimed.+++### 2026-10-17: current-main review and Linux widget validation++- Merged main at `9bc2bd7` (including #95). Only the feature-release-notes table conflicted; retained both the Linux entry and main's release/cursor entries. The production implementation needed no further changes after review.+- Added `scripts/run-linux-read-back-smoke.sh` and `scripts/tests/linux-read-back-smoke.py`: a private D-Bus/Xvfb session, a disposable real GTK fixture and native MCP client. Updated the architecture, Linux plan and `docs/linux-action-read-back.md` with the scoped runner and its prerequisites.+- Used the already-running container engine with a temporary Debian 12 ARM64 container; no desktop VM was started. Copied only repository test inputs and compiled test artifacts into it.+- The full Linux Go test binary (Go 1.26.1 cross-compiled on the host) passed in Linux. Python bridge tests: 14 passed. Node 24 REPL/CLI/config tests: 36 passed. Linux-targeted `go vet`, formatting, ARM64/AMD64 builds, shell/Python syntax, documentation and whitespace checks passed.+- Real GTK/AT-SPI through native MCP passed set_value, repeated cached-index clicks, secondary action, Unicode insertion and Return delivery. Disabled read-back returned only `ok`; invalid widget actions still failed; explicit observation returned the new entry/counter state; the default action returned an updated snapshot. Fixture state was independently checked rather than inferred from the acknowledgment.+- Negative proof against unmodified main's Python bridge: the same regression suite failed for all seven action branches; the PR implementation passed all 14 tests.+- Xvfb must start before the private D-Bus session so the activated AT-SPI registry inherits its display and can synthesize keys. This is handled by the runner.+- Validation covers X11/GTK/AT-SPI. It does not establish Wayland/other-toolkit behavior or a whole-task speedup; no physical-desktop benchmark was run.diff --git a/docs/linux-action-read-back.md b/docs/linux-action-read-back.mdnew file mode 100644index 0000000..a349638--- /dev/null+++ b/docs/linux-action-read-back.md@@ -0,0 +1,28 @@+# Linux 动作回读++Linux 默认在动作后等待 120 ms,再返回完整快照。`OPEN_COMPUTER_USE_ACTION_READ_BACK=0` 会省去这一步,成功动作只返回 `ok`。`false`、`no`、`off` 也可以,忽略大小写和首尾空白。JS REPL 已自动设置这个变量。++关闭回读后,动作保留上次观察的索引缓存;界面变化后应通过 `get_app_state` / `getAXState()` 更新状态再选目标。显式观察照常返回快照,动作错误照常返回。`ok` 表示调用完成,不能代替界面结果验证。++## 测试++Linux 回归测试:++```sh+(cd apps/OpenComputerUseLinux && go test ./...)+python3 apps/OpenComputerUseLinux/runtime_test.py+node --test scripts/node-repl/*.test.mjs+```++真实控件测试使用私有 D-Bus 会话和 Xvfb 显示器,不操作已有桌面。Debian / Ubuntu 需要 `python3-gi`、`gir1.2-gtk-3.0`、`gir1.2-atspi-2.0`、`at-spi2-core`、`dbus-x11`、`xvfb` 和 `xauth`。++在 Linux 上构建 native binary,然后运行:++```sh+(cd apps/OpenComputerUseLinux && go build -o /tmp/ocu-read-back-test .)+bash scripts/run-linux-read-back-smoke.sh /tmp/ocu-read-back-test+```++脚本创建自己的 GTK 窗口,通过 native MCP 执行点击、secondary action、设值、中文 / emoji 输入和 Return,并检查控件的实际状态;同时验证关闭回读时的短响应、连续动作索引、错误传播、显式观察和默认快照。结束后清理测试进程、会话、显示器和临时文件。++这项测试覆盖 X11 / GTK / AT-SPI,不代表 GNOME Wayland、其他应用或完整任务的性能测试。diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.mdindex 33f366c..337ff98 100644--- a/docs/releases/feature-release-notes.md+++ b/docs/releases/feature-release-notes.md@@ -7,6 +7,7 @@| 2026-10-17 | macOS 软件光标窗口跟随 | 目标窗口移动后,行进、点击反馈和等待中的软件光标继续跟随,避免停在旧位置。 | 接入窗口相对锚点,修复 idle / pulse 的旧落点、行进中止后的可见位置及重复计时器;AX 帧读取改为后台串行缓存,限制为最多 30 Hz / 一个请求,并在清理时释放追踪。 || 2026-10-15 | 1.1.1 修复发布 | Locked Use 安装更新更可靠,降低认证与后台 IPC 开销。 | 统一版本为 1.1.1,发布授权宿主、安装加载检查、旧组件卸载与连接生命周期修复;完整双屏遮罩、自动解锁、点击验证和重锁实测通过。 || 2026-10-15 | Locked Use 安装与性能修复 | 减少认证和 IPC 开销,旧系统组件也能正常移除并重装。 | 非 UI 插件使用 privileged host,安装前验证真实加载;修复系统缓存卸载与转交 socket 的生命周期,降低空闲唤醒,status 提示组件版本差异。 |+| 2026-10-15 | Linux 动作回读 | REPL 批量操作可省去每个动作后被丢弃的完整界面观察。 | Linux 开始支持现有的 `OPEN_COMPUTER_USE_ACTION_READ_BACK=0`,成功动作返回 `ok`,保留最后一次显式观察的索引缓存;默认原生 MCP 回读与显式 `get_app_state` 保持不变。 || 2026-10-14 | 1.1.0 正式发布 | 用户可直接安装 Locked Use,在锁屏遮罩期间继续 GUI 操作,也可用定向查询和持久截图配置减少工具开销。 | 统一版本为 1.1.0;签名 universal macOS App 内嵌服务、Guardian、安装器和授权插件,普通 enable 请求权限,补齐冷启动握手隔离和发布制品检查;新增 app.query()、doctor --json 与 ocu config。 || 2026-10-10 | macOS Locked Use 验证版 | 可在已登录 Mac 锁屏时,通过全屏遮罩继续 GUI 自动化,并在结束或本地接管后安全重锁。 | 正式组件采用 OCU Guardian / OCULockService / OCULockInstaller / OCULockAuth 短名称,开发 App 保留 Dev 区分;新增签名 Broker / 授权插件、双进程显示器保护、一次性许可、租约排空 / 恢复及构建和实机验证脚本;管理员显式启用 validation profile,生产 backend 在完整验证前保持关闭。 || 2026-10-10 | macOS 定向 AX 查询 | 知道控件文案时,可以直接查询并操作,不必先获取截图和完整 AX tree。 | 新增 native `query` 和 JS `app.query()`,按文字、角色和窗口查找元素,返回完整性元数据;查询索引仅在同一会话内有效,操作前重新验证目标和位置。 |diff --git a/scripts/run-linux-read-back-smoke.sh b/scripts/run-linux-read-back-smoke.shnew file mode 100755index 0000000..e02f6e4--- /dev/null+++ b/scripts/run-linux-read-back-smoke.sh@@ -0,0 +1,14 @@+#!/usr/bin/env bash+set -euo pipefail++repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+if [[ "$(uname -s)" != Linux || $# != 1 ]]; then+ echo "Usage (Linux): bash scripts/run-linux-read-back-smoke.sh /path/to/open-computer-use" >&2+ exit 2+fi++runtime_dir="$(mktemp -d)"+trap 'rm -rf "${runtime_dir}"' EXIT+XDG_RUNTIME_DIR="${runtime_dir}" NO_AT_BRIDGE=0 GTK_A11Y=always \+ xvfb-run -a dbus-run-session -- python3 \+ "${repo_root}/scripts/tests/linux-read-back-smoke.py" --native "$1"diff --git a/scripts/tests/linux-read-back-smoke.py b/scripts/tests/linux-read-back-smoke.pynew file mode 100644index 0000000..71bf58f--- /dev/null+++ b/scripts/tests/linux-read-back-smoke.py@@ -0,0 +1,231 @@+#!/usr/bin/env python3+"""Exercise native Linux MCP actions against an isolated real GTK fixture."""++import argparse+import json+import os+from pathlib import Path+import re+import select+import subprocess+import sys+import tempfile+import time+++def fixture(name, state_file):+ import gi++ gi.require_version("Gtk", "3.0")+ gi.require_version("Gdk", "3.0")+ from gi.repository import Gdk, GLib, Gtk++ GLib.set_prgname(name)+ GLib.set_application_name(name)+ window = Gtk.Window(title=name)+ window.set_default_size(500, 240)+ box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12, margin=20)+ window.add(box)+ entry = Gtk.Entry()+ entry.get_accessible().set_name("Read-back input")+ button = Gtk.Button(label="Increment")+ label = Gtk.Label(label="Counter: 0")+ for widget in (entry, button, label):+ box.pack_start(widget, False, False, 0)+ state = {"count": 0, "text": "", "activated": 0}++ def publish(*_args):+ state["text"] = entry.get_text()+ state["focused"] = entry.has_focus()+ temporary = state_file.with_suffix(".tmp")+ temporary.write_text(json.dumps(state))+ temporary.replace(state_file)++ def increment(*_args):+ state["count"] += 1+ label.set_text("Counter: " + str(state["count"]))+ entry.grab_focus()+ window.get_window().focus(Gdk.CURRENT_TIME)+ publish()++ def activate(*_args):+ state["activated"] += 1+ publish()++ button.connect("clicked", increment)+ entry.connect("changed", publish)+ entry.connect("activate", activate)+ window.connect("destroy", Gtk.main_quit)+ window.show_all()+ entry.grab_focus()+ window.get_window().focus(Gdk.CURRENT_TIME)+ publish()+ Gtk.main()+++def wait_for(predicate, description):+ deadline = time.monotonic() + 10+ while time.monotonic() < deadline:+ if predicate():+ return+ time.sleep(0.03)+ raise AssertionError("Timed out: " + description)+++class MCP:+ def __init__(self, native, read_back, stderr):+ env = dict(os.environ)+ env.pop("OPEN_COMPUTER_USE_ACTION_READ_BACK", None)+ if read_back is not None:+ env["OPEN_COMPUTER_USE_ACTION_READ_BACK"] = read_back+ self.process = subprocess.Popen(+ [native, "mcp"], stdin=subprocess.PIPE, stdout=subprocess.PIPE,+ stderr=stderr, env=env, text=True,+ )+ self.next_id = 0+ try:+ self.request("initialize", {"protocolVersion": "2025-03-26", "capabilities": {},+ "clientInfo": {"name": "linux-read-back-smoke", "version": "1"}})+ self.process.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n")+ self.process.stdin.flush()+ except Exception:+ self.close()+ raise++ def request(self, method, params):+ self.next_id += 1+ self.process.stdin.write(json.dumps({"jsonrpc": "2.0", "id": self.next_id,+ "method": method, "params": params}) + "\n")+ self.process.stdin.flush()+ ready, _, _ = select.select([self.process.stdout], [], [], 40)+ if not ready:+ raise AssertionError("MCP response timed out: " + method)+ line = self.process.stdout.readline()+ if not line:+ raise AssertionError("Native MCP exited before responding: " + method)+ response = json.loads(line)+ assert response.get("id") == self.next_id, response+ assert "error" not in response, response+ return response["result"]++ def call(self, tool, app, **arguments):+ return self.request("tools/call", {"name": tool, "arguments": {"app": app, **arguments}})++ def close(self):+ try:+ self.process.stdin.close()+ except BrokenPipeError:+ pass+ try:+ self.process.wait(timeout=3)+ except subprocess.TimeoutExpired:+ self.process.kill()+ self.process.wait()+++def text(result):+ return "\n".join(item["text"] for item in result["content"] if item["type"] == "text")+++def expect_snapshot(result):+ assert not result.get("isError"), result+ assert "Window:" in text(result) and "Read-back input" in text(result), text(result)+++def element_index(result, name):+ matches = re.findall(r"^\s*(\d+)\s+[^\n]*" + re.escape(name) + r"(?:\s|$)", text(result), re.MULTILINE)+ assert len(matches) == 1, (name, text(result))+ return matches[0]+++def smoke(native):+ with tempfile.TemporaryDirectory(prefix="ocu-read-back-smoke-") as directory:+ directory = Path(directory)+ state_file = directory / "state.json"+ name = "OCUReadBackFixture" + str(os.getpid())+ with (directory / "stderr.log").open("w+") as stderr:+ process = subprocess.Popen([sys.executable, __file__, "--fixture", name,+ "--state-file", str(state_file)], stderr=stderr)+ clients = []+ try:+ wait_for(state_file.exists, "GTK fixture startup")+ client = MCP(native, " OFF ", stderr)+ clients.append(client)+ # AT-SPI registration follows GTK startup asynchronously.+ wait_for(lambda: not client.call("get_app_state", name).get("isError"), "AT-SPI registration")+ observed = client.call("get_app_state", name)+ expect_snapshot(observed)+ input_index = element_index(observed, "Read-back input")+ button_index = element_index(observed, "Increment")++ def state_is(**expected):+ actual = json.loads(state_file.read_text())+ return all(actual[key] == value for key, value in expected.items())++ def acknowledged(tool, **arguments):+ result = client.call(tool, name, **arguments)+ assert not result.get("isError"), result+ assert result["content"] == [{"type": "text", "text": "ok"}], result++ acknowledged("set_value", element_index=input_index, value="hello-中文-🙂")+ wait_for(lambda: state_is(text="hello-中文-🙂"), "real entry set_value")+ for count in (1, 2):+ acknowledged("click", element_index=button_index, click_method="accessibility")+ wait_for(lambda: state_is(count=count), "cached-index button click")+ acknowledged("perform_secondary_action", element_index=button_index, action="click")+ wait_for(lambda: state_is(count=3), "real secondary action")+ acknowledged("type_text", text="-typed")+ wait_for(lambda: state_is(text="hello-中文-🙂-typed"), "real entry insertion")+ acknowledged("press_key", key="Return")+ wait_for(lambda: state_is(activated=1), "real Return delivery")+ for tool, arguments in (+ ("set_value", {"element_index": button_index, "value": "invalid"}),+ ("perform_secondary_action", {"element_index": button_index, "action": "not-an-action"}),+ ):+ result = client.call(tool, name, **arguments)+ assert result.get("isError") and text(result) != "ok", result+ refreshed = client.call("get_app_state", name)+ expect_snapshot(refreshed)+ assert "hello-中文-🙂-typed" in text(refreshed), text(refreshed)+ assert "Counter: 3" in text(refreshed), text(refreshed)+ print("Disabled read-back: real actions, preserved indexes, errors and explicit observation passed.")++ default = MCP(native, None, stderr)+ clients.append(default)+ observed = default.call("get_app_state", name)+ expect_snapshot(observed)+ result = default.call("set_value", name, element_index=element_index(observed, "Read-back input"), value="default-read-back")+ expect_snapshot(result)+ assert "default-read-back" in text(result), text(result)+ wait_for(lambda: state_is(text="default-read-back"), "default action state")+ print("Default read-back: real action returns an updated snapshot.")+ except Exception:+ if state_file.exists():+ print("Fixture state:", state_file.read_text(), file=sys.stderr)+ stderr.flush()+ stderr.seek(0)+ print(stderr.read(), file=sys.stderr)+ raise+ finally:+ for client in clients:+ client.close()+ process.terminate()+ try:+ process.wait(timeout=3)+ except subprocess.TimeoutExpired:+ process.kill()+ process.wait()+++if __name__ == "__main__":+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--native")+ parser.add_argument("--fixture")+ parser.add_argument("--state-file", type=Path)+ args = parser.parse_args()+ if args.fixture:+ fixture(args.fixture, args.state_file)+ else:+ if not args.native:+ parser.error("--native is required")+ smoke(str(Path(args.native).resolve()))