👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
fix(linux): detect AT-SPI text interfaces safely
Kapish14ead48da2032c
diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.pyindex a323dd9..6c0f248 100644--- a/apps/OpenComputerUseLinux/runtime.py+++ b/apps/OpenComputerUseLinux/runtime.py@@ -50,6 +50,12 @@ def safe(call, default=None):return default+def supports_interface(node, interface_name):+ interfaces = safe(lambda: node.get_interfaces(), [])+ expected = str(interface_name).casefold()+ return any(str(interface).casefold() == expected for interface in interfaces)++def require_desktop_session():missing = []if not os.environ.get("XDG_RUNTIME_DIR"):@@ -217,7 +223,7 @@ def accessible_id(node):def text_value(node, text_limit=DEFAULT_TEXT_LIMIT):- if not bool(safe(node.is_text, False)):+ if not supports_interface(node, "Text"):return ""text_iface = safe(node.get_text_iface)if text_iface is None:@@ -407,7 +413,7 @@ def selected_text(app_pid, text_limit=DEFAULT_TEXT_LIMIT):focused = find_first(win, lambda node: state_contains(node, Atspi.StateType.FOCUSED))- if focused is None or not bool(safe(focused.is_text, False)):+ if focused is None or not supports_interface(focused, "Text"):return Nonetext_iface = safe(focused.get_text_iface)selections = safe(lambda: Atspi.Text.get_text_selections(text_iface), [])@@ -690,8 +696,8 @@ def send_text(text):def find_editable_text(root):def is_editable(node):- return bool(safe(node.is_editable_text, False)) and bool(- safe(node.is_text, False)+ return supports_interface(node, "EditableText") and supports_interface(+ node, "Text")return find_first(root, is_editable)@@ -717,7 +723,7 @@ def insert_text(root, text):def set_element_value(node, value):- if node is not None and bool(safe(node.is_editable_text, False)):+ if node is not None and supports_interface(node, "EditableText"):editable = safe(node.get_editable_text_iface)if editable is not None:return bool(diff --git a/apps/OpenComputerUseLinux/runtime_test.py b/apps/OpenComputerUseLinux/runtime_test.pynew file mode 100644index 0000000..c42f41e--- /dev/null+++ b/apps/OpenComputerUseLinux/runtime_test.py@@ -0,0 +1,90 @@+import importlib.util+import pathlib+import sys+import types+import unittest+from unittest import mock+++class FakeText:+ @staticmethod+ def get_character_count(_interface):+ return 5++ @staticmethod+ def get_text(_interface, _start, _end):+ return "hello"+++class FakeEditableText:+ @staticmethod+ def insert_text(_interface, _offset, _text, _length):+ return True++ @staticmethod+ def set_text_contents(_interface, _text):+ return True+++def load_runtime():+ gi = types.ModuleType("gi")+ gi.require_version = lambda *_args: None+ repository = types.ModuleType("gi.repository")+ repository.Atspi = types.SimpleNamespace(+ Text=FakeText,+ EditableText=FakeEditableText,+ )+ repository.Gdk = types.SimpleNamespace()+ gi.repository = repository++ runtime_path = pathlib.Path(__file__).with_name("runtime.py")+ spec = importlib.util.spec_from_file_location("open_computer_use_linux_runtime", runtime_path)+ module = importlib.util.module_from_spec(spec)+ with mock.patch.dict(+ sys.modules,+ {"gi": gi, "gi.repository": repository},+ ):+ spec.loader.exec_module(module)+ return module+++class InterfaceOnlyAccessible:+ def __init__(self, interfaces):+ self.interfaces = interfaces++ def get_interfaces(self):+ return self.interfaces++ def get_text_iface(self):+ return self++ def get_editable_text_iface(self):+ return self++ def get_child_count(self):+ return 0+++class RuntimeInterfaceDetectionTests(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.runtime = load_runtime()++ def test_text_value_uses_accessible_interfaces(self):+ node = InterfaceOnlyAccessible(["Accessible", "Text"])++ self.assertEqual(self.runtime.text_value(node), "hello")++ def test_insert_text_uses_accessible_interfaces(self):+ node = InterfaceOnlyAccessible(["Accessible", "Text", "EditableText"])++ self.assertTrue(self.runtime.insert_text(node, "hello"))++ def test_set_value_uses_accessible_interfaces(self):+ node = InterfaceOnlyAccessible(["Accessible", "Text", "EditableText"])++ self.assertTrue(self.runtime.set_element_value(node, "hello"))+++if __name__ == "__main__":+ unittest.main()diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex dbef39e..e4c400b 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -120,7 +120,7 @@- Linux runtime 位于 `apps/OpenComputerUseLinux`,以 Go 维护 CLI、`call --calls` sequence、MCP JSON-RPC、tool schema 和进程内 snapshot cache。- 构建入口是 `scripts/build-open-computer-use-linux.sh --arch arm64|amd64`,默认输出到 `dist/linux/<arch>/open-computer-use`;npm release package 会把两个 Linux artifact 内置到已有 root/alias packages,Node launcher 按 `process.platform/process.arch` 自动选择。-- Go runtime 通过 `go:embed` 带上 `runtime.py`,执行 tool call 时临时落盘并调用 `python3`。Python bridge 使用 GNOME/GObject Introspection 暴露的 AT-SPI2 接口做 app/window discovery、accessibility tree rendering、semantic action、editable text、value set,以及 best-effort 的 key/mouse fallback。+- Go runtime 通过 `go:embed` 带上 `runtime.py`,执行 tool call 时临时落盘并调用 `python3`。Python bridge 使用 GNOME/GObject Introspection 暴露的 AT-SPI2 接口做 app/window discovery、accessibility tree rendering、semantic action、editable text、value set,以及 best-effort 的 key/mouse fallback;文本能力通过 `Accessible.get_interfaces()` 检测 `Text` / `EditableText`,不依赖不同 PyGObject 版本未必存在的便捷属性。- Linux 上最接近 macOS AX 的是 AT-SPI2/D-Bus accessibility,而不是一套统一的后台键鼠输入模型。第一版优先使用元素暴露的 AT-SPI action、EditableText 和 Value 接口;coordinate `click` / `drag` 与 `press_key` 使用 AT-SPI event synthesis fallback,在 Wayland 下只能按 best-effort 处理。- Linux runtime 需要运行在已登录桌面用户 session 里。缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,Go runtime 会在启动 Python AT-SPI bridge 前尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session bus、display / Wayland 值;纯 SSH tty 如果找不到已登录桌面 session,可以启动二进制,但不能直接 inspect 或操作 GUI session。- `get_app_state` 的 accessibility tree 在 GTK/GNOME app 上可能很深,Linux bridge 使用与 macOS / Windows 一致的 1200 节点、64 层默认 tree budget,并支持显式提高 `max_tree_nodes` / `max_tree_depth`。截图通过 GDK root window best-effort capture;GNOME Wayland 可能返回黑图,bridge 会检测全黑采样并省略 image block。diff --git a/docs/histories/2026-08/20260808-0409-fix-linux-atspi-interface-detection.md b/docs/histories/2026-08/20260808-0409-fix-linux-atspi-interface-detection.mdnew file mode 100644index 0000000..7bde3f8--- /dev/null+++ b/docs/histories/2026-08/20260808-0409-fix-linux-atspi-interface-detection.md@@ -0,0 +1,28 @@+## [2026-08-08 04:09] | Task: Fix Linux AT-SPI interface detection++### 🤖 Execution Context+* **Agent ID**: `Codex`+* **Base Model**: `GPT-5.6`+* **Runtime**: `T3 Code / Codex harness`++### 📥 User Query+> 修复 Linux 文本编辑流程中发现的上游问题,并准备 Pull Request。++### 🛠 Changes Overview+**Scope:** Linux Computer Use runtime++**Key Actions:**+- **标准接口检测**: 使用 `Accessible.get_interfaces()` 判断 `Text` 与 `EditableText` 能力,移除对部分 PyGObject 环境不存在属性的访问。+- **回归覆盖**: 添加无桌面依赖的 Python 测试,并接入仓库基础 CI。+- **文档同步**: 更新 Linux 架构说明、troubleshooting 与用户可见功能记录。++### 🧠 Design Intent (Why)+Ubuntu 24.04 的 AT-SPI GI binding 不提供 `Accessible.is_text` 和 `Accessible.is_editable_text` 属性。原实现会在进入容错包装前抛出 `AttributeError`,导致快照和文本工具不可用。`get_interfaces()` 是实际 binding 提供的能力查询接口,并可在不同 toolkit 节点上安全降级。++### 📁 Files Modified+- `apps/OpenComputerUseLinux/runtime.py`+- `apps/OpenComputerUseLinux/runtime_test.py`+- `scripts/ci.sh`+- `docs/ARCHITECTURE.md`+- `skills/open-computer-use/references/troubleshooting.md`+- `docs/releases/feature-release-notes.md`diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.mdindex d222cb1..b4a6708 100644--- a/docs/releases/feature-release-notes.md+++ b/docs/releases/feature-release-notes.md@@ -1,5 +1,11 @@# 功能发布记录+## 2026-08++| 日期 | 功能域 | 用户价值 | 变更摘要 |+| --- | --- | --- | --- |+| 2026-08-08 | Linux AT-SPI 文本能力检测 | Ubuntu 24.04 等 PyGObject 环境中的 `get_app_state`、`type_text` 和 `set_value` 不再因缺少非标准 `Accessible.is_text` / `is_editable_text` 属性而崩溃。 | Linux bridge 改为通过标准 `Accessible.get_interfaces()` 检测 `Text` / `EditableText`,并新增不依赖真实桌面的 Python 回归测试。 |+## 2026-07| 日期 | 功能域 | 用户价值 | 变更摘要 |diff --git a/scripts/ci.sh b/scripts/ci.shindex f12dc11..870c125 100755--- a/scripts/ci.sh+++ b/scripts/ci.sh@@ -16,6 +16,11 @@ while IFS= read -r file; donode --check "$file"done < <(find "${repo_root}/scripts" -type f -name '*.mjs' | sort)+(+ cd "${repo_root}/apps/OpenComputerUseLinux"+ python3 -m unittest -v runtime_test.py+)+if command -v go >/dev/null 2>&1; then(cd "${repo_root}/apps/OpenComputerUseWindows"diff --git a/skills/open-computer-use/references/troubleshooting.md b/skills/open-computer-use/references/troubleshooting.mdindex 9bce680..7853816 100644--- a/skills/open-computer-use/references/troubleshooting.md+++ b/skills/open-computer-use/references/troubleshooting.md@@ -91,6 +91,10 @@ Windows UI Automation and Linux AT-SPI require a live user desktop. SSH sessionsIf desktop access is missing, ask the user to run the command from the logged-in desktop session or start the target app visibly in that session.+## Linux Accessible Interface Errors++If a Linux release reports that `Accessible` has no `is_text` or `is_editable_text` attribute, upgrade to a release containing the AT-SPI interface-detection fix. Current source checks `Accessible.get_interfaces()` for `Text` and `EditableText`, which is compatible with the PyGObject bindings used by Ubuntu 24.04.+## Permission And Safety Issues- Do not bypass macOS TCC prompts.