Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

fix(linux): detect AT-SPI text interfaces safely

Kapish14ead48da2032c
diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.py
index a323dd9..6c0f248 100644
--- a/apps/OpenComputerUseLinux/runtime.py
+++ b/apps/OpenComputerUseLinux/runtime.py
@@ -50,6 +50,12 @@ def safe(call, default=None):
return default
+def supports_interface(node, interface_name):
+ interfaces = safe(lambda: node.get_interfaces(), [])
+ expected = str(interface_name).casefold()
+ return any(str(interface).casefold() == expected for interface in interfaces)
+
+
def require_desktop_session():
missing = []
if not os.environ.get("XDG_RUNTIME_DIR"):
@@ -217,7 +223,7 @@ def accessible_id(node):
def text_value(node, text_limit=DEFAULT_TEXT_LIMIT):
- if not bool(safe(node.is_text, False)):
+ if not supports_interface(node, "Text"):
return ""
text_iface = safe(node.get_text_iface)
if text_iface is None:
@@ -407,7 +413,7 @@ def selected_text(app_pid, text_limit=DEFAULT_TEXT_LIMIT):
focused = find_first(
win, lambda node: state_contains(node, Atspi.StateType.FOCUSED)
)
- if focused is None or not bool(safe(focused.is_text, False)):
+ if focused is None or not supports_interface(focused, "Text"):
return None
text_iface = safe(focused.get_text_iface)
selections = safe(lambda: Atspi.Text.get_text_selections(text_iface), [])
@@ -690,8 +696,8 @@ def send_text(text):
def find_editable_text(root):
def is_editable(node):
- return bool(safe(node.is_editable_text, False)) and bool(
- safe(node.is_text, False)
+ return supports_interface(node, "EditableText") and supports_interface(
+ node, "Text"
)
return find_first(root, is_editable)
@@ -717,7 +723,7 @@ def insert_text(root, text):
def set_element_value(node, value):
- if node is not None and bool(safe(node.is_editable_text, False)):
+ if node is not None and supports_interface(node, "EditableText"):
editable = safe(node.get_editable_text_iface)
if editable is not None:
return bool(
diff --git a/apps/OpenComputerUseLinux/runtime_test.py b/apps/OpenComputerUseLinux/runtime_test.py
new file mode 100644
index 0000000..c42f41e
--- /dev/null
+++ b/apps/OpenComputerUseLinux/runtime_test.py
@@ -0,0 +1,90 @@
+import importlib.util
+import pathlib
+import sys
+import types
+import unittest
+from unittest import mock
+
+
+class FakeText:
+ @staticmethod
+ def get_character_count(_interface):
+ return 5
+
+ @staticmethod
+ def get_text(_interface, _start, _end):
+ return "hello"
+
+
+class FakeEditableText:
+ @staticmethod
+ def insert_text(_interface, _offset, _text, _length):
+ return True
+
+ @staticmethod
+ def set_text_contents(_interface, _text):
+ return True
+
+
+def load_runtime():
+ gi = types.ModuleType("gi")
+ gi.require_version = lambda *_args: None
+ repository = types.ModuleType("gi.repository")
+ repository.Atspi = types.SimpleNamespace(
+ Text=FakeText,
+ EditableText=FakeEditableText,
+ )
+ repository.Gdk = types.SimpleNamespace()
+ gi.repository = repository
+
+ runtime_path = pathlib.Path(__file__).with_name("runtime.py")
+ spec = importlib.util.spec_from_file_location("open_computer_use_linux_runtime", runtime_path)
+ module = importlib.util.module_from_spec(spec)
+ with mock.patch.dict(
+ sys.modules,
+ {"gi": gi, "gi.repository": repository},
+ ):
+ spec.loader.exec_module(module)
+ return module
+
+
+class InterfaceOnlyAccessible:
+ def __init__(self, interfaces):
+ self.interfaces = interfaces
+
+ def get_interfaces(self):
+ return self.interfaces
+
+ def get_text_iface(self):
+ return self
+
+ def get_editable_text_iface(self):
+ return self
+
+ def get_child_count(self):
+ return 0
+
+
+class RuntimeInterfaceDetectionTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.runtime = load_runtime()
+
+ def test_text_value_uses_accessible_interfaces(self):
+ node = InterfaceOnlyAccessible(["Accessible", "Text"])
+
+ self.assertEqual(self.runtime.text_value(node), "hello")
+
+ def test_insert_text_uses_accessible_interfaces(self):
+ node = InterfaceOnlyAccessible(["Accessible", "Text", "EditableText"])
+
+ self.assertTrue(self.runtime.insert_text(node, "hello"))
+
+ def test_set_value_uses_accessible_interfaces(self):
+ node = InterfaceOnlyAccessible(["Accessible", "Text", "EditableText"])
+
+ self.assertTrue(self.runtime.set_element_value(node, "hello"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index dbef39e..e4c400b 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -120,7 +120,7 @@
- Linux runtime 位于 `apps/OpenComputerUseLinux`,以 Go 维护 CLI、`call --calls` sequence、MCP JSON-RPC、tool schema 和进程内 snapshot cache。
- 构建入口是 `scripts/build-open-computer-use-linux.sh --arch arm64|amd64`,默认输出到 `dist/linux/<arch>/open-computer-use`;npm release package 会把两个 Linux artifact 内置到已有 root/alias packages,Node launcher 按 `process.platform/process.arch` 自动选择。
-- Go runtime 通过 `go:embed` 带上 `runtime.py`,执行 tool call 时临时落盘并调用 `python3`。Python bridge 使用 GNOME/GObject Introspection 暴露的 AT-SPI2 接口做 app/window discovery、accessibility tree rendering、semantic action、editable text、value set,以及 best-effort 的 key/mouse fallback。
+- Go runtime 通过 `go:embed` 带上 `runtime.py`,执行 tool call 时临时落盘并调用 `python3`。Python bridge 使用 GNOME/GObject Introspection 暴露的 AT-SPI2 接口做 app/window discovery、accessibility tree rendering、semantic action、editable text、value set,以及 best-effort 的 key/mouse fallback;文本能力通过 `Accessible.get_interfaces()` 检测 `Text` / `EditableText`,不依赖不同 PyGObject 版本未必存在的便捷属性。
- Linux 上最接近 macOS AX 的是 AT-SPI2/D-Bus accessibility,而不是一套统一的后台键鼠输入模型。第一版优先使用元素暴露的 AT-SPI action、EditableText 和 Value 接口;coordinate `click` / `drag` 与 `press_key` 使用 AT-SPI event synthesis fallback,在 Wayland 下只能按 best-effort 处理。
- Linux runtime 需要运行在已登录桌面用户 session 里。缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,Go runtime 会在启动 Python AT-SPI bridge 前尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session bus、display / Wayland 值;纯 SSH tty 如果找不到已登录桌面 session,可以启动二进制,但不能直接 inspect 或操作 GUI session。
- `get_app_state` 的 accessibility tree 在 GTK/GNOME app 上可能很深,Linux bridge 使用与 macOS / Windows 一致的 1200 节点、64 层默认 tree budget,并支持显式提高 `max_tree_nodes` / `max_tree_depth`。截图通过 GDK root window best-effort capture;GNOME Wayland 可能返回黑图,bridge 会检测全黑采样并省略 image block。
diff --git a/docs/histories/2026-08/20260808-0409-fix-linux-atspi-interface-detection.md b/docs/histories/2026-08/20260808-0409-fix-linux-atspi-interface-detection.md
new file mode 100644
index 0000000..7bde3f8
--- /dev/null
+++ b/docs/histories/2026-08/20260808-0409-fix-linux-atspi-interface-detection.md
@@ -0,0 +1,28 @@
+## [2026-08-08 04:09] | Task: Fix Linux AT-SPI interface detection
+
+### 🤖 Execution Context
+* **Agent ID**: `Codex`
+* **Base Model**: `GPT-5.6`
+* **Runtime**: `T3 Code / Codex harness`
+
+### 📥 User Query
+> 修复 Linux 文本编辑流程中发现的上游问题,并准备 Pull Request。
+
+### 🛠 Changes Overview
+**Scope:** Linux Computer Use runtime
+
+**Key Actions:**
+- **标准接口检测**: 使用 `Accessible.get_interfaces()` 判断 `Text` 与 `EditableText` 能力,移除对部分 PyGObject 环境不存在属性的访问。
+- **回归覆盖**: 添加无桌面依赖的 Python 测试,并接入仓库基础 CI。
+- **文档同步**: 更新 Linux 架构说明、troubleshooting 与用户可见功能记录。
+
+### 🧠 Design Intent (Why)
+Ubuntu 24.04 的 AT-SPI GI binding 不提供 `Accessible.is_text` 和 `Accessible.is_editable_text` 属性。原实现会在进入容错包装前抛出 `AttributeError`,导致快照和文本工具不可用。`get_interfaces()` 是实际 binding 提供的能力查询接口,并可在不同 toolkit 节点上安全降级。
+
+### 📁 Files Modified
+- `apps/OpenComputerUseLinux/runtime.py`
+- `apps/OpenComputerUseLinux/runtime_test.py`
+- `scripts/ci.sh`
+- `docs/ARCHITECTURE.md`
+- `skills/open-computer-use/references/troubleshooting.md`
+- `docs/releases/feature-release-notes.md`
diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.md
index d222cb1..b4a6708 100644
--- a/docs/releases/feature-release-notes.md
+++ b/docs/releases/feature-release-notes.md
@@ -1,5 +1,11 @@
# 功能发布记录
+## 2026-08
+
+| 日期 | 功能域 | 用户价值 | 变更摘要 |
+| --- | --- | --- | --- |
+| 2026-08-08 | Linux AT-SPI 文本能力检测 | Ubuntu 24.04 等 PyGObject 环境中的 `get_app_state`、`type_text` 和 `set_value` 不再因缺少非标准 `Accessible.is_text` / `is_editable_text` 属性而崩溃。 | Linux bridge 改为通过标准 `Accessible.get_interfaces()` 检测 `Text` / `EditableText`,并新增不依赖真实桌面的 Python 回归测试。 |
+
## 2026-07
| 日期 | 功能域 | 用户价值 | 变更摘要 |
diff --git a/scripts/ci.sh b/scripts/ci.sh
index f12dc11..870c125 100755
--- a/scripts/ci.sh
+++ b/scripts/ci.sh
@@ -16,6 +16,11 @@ while IFS= read -r file; do
node --check "$file"
done < <(find "${repo_root}/scripts" -type f -name '*.mjs' | sort)
+(
+ cd "${repo_root}/apps/OpenComputerUseLinux"
+ python3 -m unittest -v runtime_test.py
+)
+
if command -v go >/dev/null 2>&1; then
(
cd "${repo_root}/apps/OpenComputerUseWindows"
diff --git a/skills/open-computer-use/references/troubleshooting.md b/skills/open-computer-use/references/troubleshooting.md
index 9bce680..7853816 100644
--- a/skills/open-computer-use/references/troubleshooting.md
+++ b/skills/open-computer-use/references/troubleshooting.md
@@ -91,6 +91,10 @@ Windows UI Automation and Linux AT-SPI require a live user desktop. SSH sessions
If desktop access is missing, ask the user to run the command from the logged-in desktop session or start the target app visibly in that session.
+## Linux Accessible Interface Errors
+
+If a Linux release reports that `Accessible` has no `is_text` or `is_editable_text` attribute, upgrade to a release containing the AT-SPI interface-detection fix. Current source checks `Accessible.get_interfaces()` for `Text` and `EditableText`, which is compatible with the PyGObject bindings used by Ubuntu 24.04.
+
## Permission And Safety Issues
- Do not bypass macOS TCC prompts.