👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
fix(linux): send press_key named keys and modifiers as keysyms
Aaron Hammer40c12decc1ee
diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.pyindex 6c0f248..0759575 100644--- a/apps/OpenComputerUseLinux/runtime.py+++ b/apps/OpenComputerUseLinux/runtime.py@@ -644,24 +644,33 @@ KEY_ALIASES = {"next": "Page_Down",}-MODIFIER_KEYS = {- "ctrl": "Control_L",- "control": "Control_L",- "shift": "Shift_L",- "alt": "Alt_L",- "super": "Super_L",- "win": "Super_L",- "cmd": "Super_L",+# X11 modifier bits for LOCKMODIFIERS/UNLOCKMODIFIERS. Super is usually Mod4.+MODIFIER_BITS = {+ "ctrl": "CONTROL",+ "control": "CONTROL",+ "shift": "SHIFT",+ "alt": "ALT",+ "super": "META3",+ "win": "META3",+ "cmd": "META3",}+# GDK returns VoidSymbol for names it doesn't know.+VOID_SYMBOL = 0xFFFFFF++def keyval(name):- if Gdk is not None:- value = Gdk.keyval_from_name(name)- if value:- return int(value)+ # Characters have no name lookup in GDK ("/" is "slash"), so map them+ # from Unicode instead.if len(name) == 1:+ if Gdk is not None:+ return int(Gdk.unicode_to_keyval(ord(name)))return ord(name)+ if Gdk is not None:+ value = int(Gdk.keyval_from_name(name))+ if value and value != VOID_SYMBOL:+ return valueraise RuntimeError("Unsupported key: " + name)@@ -670,24 +679,29 @@ def send_key(key):if not parts:raise RuntimeError("Unsupported key: " + str(key))main = parts[-1]- modifiers = parts[:-1]- pressed = []- for modifier in modifiers:- name = MODIFIER_KEYS.get(modifier.lower())+ mask = 0+ for modifier in parts[:-1]:+ name = MODIFIER_BITS.get(modifier.lower())if name is None:- continue- value = keyval(name)- Atspi.generate_keyboard_event(value, None, Atspi.KeySynthType.PRESS)- pressed.append(value)+ raise RuntimeError("Unsupported modifier: " + modifier)+ mask |= 1 << int(getattr(Atspi.ModifierType, name))normalized = KEY_ALIASES.get(main.lower(), main)- if len(normalized) == 1:- Atspi.generate_keyboard_event(0, normalized, Atspi.KeySynthType.STRING)- else:- Atspi.generate_keyboard_event(- keyval(normalized), None, Atspi.KeySynthType.PRESSRELEASE- )- for value in reversed(pressed):- Atspi.generate_keyboard_event(value, None, Atspi.KeySynthType.RELEASE)+ if mask:+ Atspi.generate_keyboard_event(mask, None, Atspi.KeySynthType.LOCKMODIFIERS)+ try:+ if len(normalized) == 1 and not mask:+ Atspi.generate_keyboard_event(0, normalized, Atspi.KeySynthType.STRING)+ else:+ # SYM takes a keysym. PRESS, RELEASE, and PRESSRELEASE take a+ # hardware keycode, so passing a keysym there sends another key.+ Atspi.generate_keyboard_event(+ keyval(normalized), None, Atspi.KeySynthType.SYM+ )+ finally:+ if mask:+ Atspi.generate_keyboard_event(+ mask, None, Atspi.KeySynthType.UNLOCKMODIFIERS+ )def send_text(text):diff --git a/apps/OpenComputerUseLinux/runtime_test.py b/apps/OpenComputerUseLinux/runtime_test.pyindex c42f41e..f52e8e3 100644--- a/apps/OpenComputerUseLinux/runtime_test.py+++ b/apps/OpenComputerUseLinux/runtime_test.py@@ -86,5 +86,90 @@ class RuntimeInterfaceDetectionTests(unittest.TestCase):self.assertTrue(self.runtime.set_element_value(node, "hello"))+class FakeKeySynthType:+ PRESS = "PRESS"+ RELEASE = "RELEASE"+ PRESSRELEASE = "PRESSRELEASE"+ SYM = "SYM"+ STRING = "STRING"+ LOCKMODIFIERS = "LOCKMODIFIERS"+ UNLOCKMODIFIERS = "UNLOCKMODIFIERS"+++class FakeModifierType:+ SHIFT = 0+ CONTROL = 2+ ALT = 3+ META3 = 6+++KEYSYMS = {"Return": 0xFF0D, "Tab": 0xFF09, "Down": 0xFF54}+++class RuntimeKeyTests(unittest.TestCase):+ def setUp(self):+ self.runtime = load_runtime()+ self.events = []+ self.runtime.Atspi = types.SimpleNamespace(+ KeySynthType=FakeKeySynthType,+ ModifierType=FakeModifierType,+ generate_keyboard_event=lambda value, text, synth: self.events.append(+ (value, text, synth)+ ),+ )+ self.runtime.Gdk = types.SimpleNamespace(+ # Like GDK, unknown names map to VoidSymbol rather than 0.+ keyval_from_name=lambda name: KEYSYMS.get(name, 0xFFFFFF),+ unicode_to_keyval=lambda codepoint: codepoint,+ )++ def test_named_key_is_sent_as_keysym(self):+ self.runtime.send_key("Enter")++ self.assertEqual(self.events, [(0xFF0D, None, "SYM")])++ def test_modifiers_are_held_around_the_key(self):+ self.runtime.send_key("ctrl+shift+Tab")++ mask = (1 << FakeModifierType.CONTROL) | (1 << FakeModifierType.SHIFT)+ self.assertEqual(+ self.events,+ [+ (mask, None, "LOCKMODIFIERS"),+ (0xFF09, None, "SYM"),+ (mask, None, "UNLOCKMODIFIERS"),+ ],+ )++ def test_modified_character_is_sent_as_keysym(self):+ self.runtime.send_key("ctrl+a")++ mask = 1 << FakeModifierType.CONTROL+ self.assertEqual(self.events[1], (ord("a"), None, "SYM"))+ self.assertEqual(self.events[0], (mask, None, "LOCKMODIFIERS"))++ def test_modified_punctuation_is_sent_as_keysym(self):+ self.runtime.send_key("ctrl+/")++ self.assertEqual(self.events[1], (ord("/"), None, "SYM"))++ def test_unknown_key_name_is_rejected(self):+ with self.assertRaises(RuntimeError):+ self.runtime.send_key("NoSuchKey")++ self.assertEqual(self.events, [])++ def test_plain_character_is_typed(self):+ self.runtime.send_key("a")++ self.assertEqual(self.events, [(0, "a", "STRING")])++ def test_unknown_modifier_is_rejected(self):+ with self.assertRaises(RuntimeError):+ self.runtime.send_key("hyper+a")++ self.assertEqual(self.events, [])++if __name__ == "__main__":unittest.main()diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 7042651..2ed833b 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -128,7 +128,7 @@- Linux runtime 位于 `apps/OpenComputerUseLinux`,以 Go 维护 CLI、`call --calls` sequence、MCP JSON-RPC、tool schema 和进程内 snapshot cache。- 构建入口是 `scripts/build-open-computer-use-linux.sh --arch arm64|amd64`,默认输出到 `dist/linux/<arch>/open-computer-use`;npm release package 会把两个 Linux artifact 内置到已有 root/alias packages,Node launcher 按 `process.platform/process.arch` 自动选择。- Go runtime 通过 `go:embed` 带上 `runtime.py`,执行 tool call 时临时落盘并调用 `python3`。Python bridge 使用 GNOME/GObject Introspection 暴露的 AT-SPI2 接口做 app/window discovery、accessibility tree rendering、semantic action、editable text、value set,以及 best-effort 的 key/mouse fallback;文本能力通过 `Accessible.get_interfaces()` 检测 `Text` / `EditableText`,不依赖不同 PyGObject 版本未必存在的便捷属性。-- Linux 上最接近 macOS AX 的是 AT-SPI2/D-Bus accessibility,而不是一套统一的后台键鼠输入模型。第一版优先使用元素暴露的 AT-SPI action、EditableText 和 Value 接口;coordinate `click` / `drag` 与 `press_key` 使用 AT-SPI event synthesis fallback,在 Wayland 下只能按 best-effort 处理。+- Linux 上最接近 macOS AX 的是 AT-SPI2/D-Bus accessibility,而不是一套统一的后台键鼠输入模型。第一版优先使用元素暴露的 AT-SPI action、EditableText 和 Value 接口;coordinate `click` / `drag` 与 `press_key` 使用 AT-SPI event synthesis fallback,在 Wayland 下只能按 best-effort 处理。`press_key` 的具名键以 keysym 配合 `KeySynthType.SYM` 发送,修饰键以 X modifier mask 配合 `LOCKMODIFIERS` / `UNLOCKMODIFIERS` 保持;`PRESS` / `RELEASE` / `PRESSRELEASE` 只接受硬件 keycode,不能传 keysym。- Linux runtime 需要运行在已登录桌面用户 session 里。缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,Go runtime 会在启动 Python AT-SPI bridge 前尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session bus、display / Wayland 值;纯 SSH tty 如果找不到已登录桌面 session,可以启动二进制,但不能直接 inspect 或操作 GUI session。- `get_app_state` 的 accessibility tree 在 GTK/GNOME app 上可能很深,Linux bridge 使用与 macOS / Windows 一致的 1200 节点、64 层默认 tree budget,并支持显式提高 `max_tree_nodes` / `max_tree_depth`。截图通过 GDK root window best-effort capture;GNOME Wayland 可能返回黑图,bridge 会检测全黑采样并省略 image block。- 这 9 个 tool 的协议面与 macOS / Windows 保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime path metadata,coordinate action 使用 screenshot/window-relative 坐标。diff --git a/docs/histories/2026-10/20261009-1739-fix-linux-press-key-keysym.md b/docs/histories/2026-10/20261009-1739-fix-linux-press-key-keysym.mdnew file mode 100644index 0000000..bf7ba2e--- /dev/null+++ b/docs/histories/2026-10/20261009-1739-fix-linux-press-key-keysym.md@@ -0,0 +1,26 @@+## [2026-10-09 17:39] | Task: Fix Linux press_key keysym synthesis++### 🤖 Execution Context+* **Agent ID**: `Claude Code`+* **Base Model**: `Claude Opus 5.5`+* **Runtime**: `T3 Code / Claude Code harness`++### 📥 User Query+> 在 Linux 上 `press_key` 发送的键不对(Enter 变成 `4`),确认原因并准备修复 Pull Request。++### 🛠 Changes Overview+**Scope:** Linux Computer Use runtime++**Key Actions:**+- **具名键改用 keysym 合成**: `send_key` 对 Enter、Tab、Escape、方向键等具名键使用 `KeySynthType.SYM`,不再把 keysym 当作 keycode 传给 `PRESSRELEASE`。+- **修饰键改用 modifier mask**: `ctrl` / `shift` / `alt` / `super` 通过 `LOCKMODIFIERS` / `UNLOCKMODIFIERS` 保持,组合键中的字符通过 `Gdk.unicode_to_keyval` 转为 keysym(`/`、`-`、`=` 等标点在 GDK 中没有同名 keysym 名称);未知修饰键和返回 `VoidSymbol` 的未知键名直接报错,不再静默发送。+- **回归覆盖**: 新增不依赖真实桌面的 `send_key` Python 测试,并同步 Linux 架构说明与功能发布记录。++### 🧠 Design Intent (Why)+`Atspi.generate_keyboard_event` 在 `PRESS` / `RELEASE` / `PRESSRELEASE` 模式下只接受硬件 keycode。原实现传入 `Gdk.keyval_from_name` 返回的 keysym,AT-SPI 取其低字节当作 keycode:Return(0xFF0D)变成 keycode 13,即美式键盘的 `4`;Control_L(0xFFE3)变成未映射的 keycode 227,因此快捷键缺少修饰键。`SYM` 模式接受 keysym 并完成按下与抬起,modifier mask 则由 AT-SPI 负责锁定与释放。在 Ubuntu 26.04、X11、Google Chrome 中用 keydown 记录验证:Return、Tab、Escape、Down、Page_Down、Left、BackSpace、`ctrl+a`、`shift+Left`、`shift+Home`、`alt+Left`、`ctrl+/`、`ctrl+-`、`ctrl+=`、`ctrl+,` 均送达正确按键。++### 📁 Files Modified+- `apps/OpenComputerUseLinux/runtime.py`+- `apps/OpenComputerUseLinux/runtime_test.py`+- `docs/ARCHITECTURE.md`+- `docs/releases/feature-release-notes.md`diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.mdindex 4a24607..d6383cd 100644--- a/docs/releases/feature-release-notes.md+++ b/docs/releases/feature-release-notes.md@@ -4,6 +4,7 @@| 日期 | 功能域 | 用户价值 | 变更摘要 || --- | --- | --- | --- |+| 2026-10-09 | Linux 按键 | Linux 上 `press_key` 的 Enter、Tab、Escape、方向键和 `ctrl+a` 等快捷键会送达正确的按键,Agent 可以提交表单、切换焦点和使用快捷键。 | Linux bridge 改用 `KeySynthType.SYM` 发送具名键 keysym,并以 `LOCKMODIFIERS` / `UNLOCKMODIFIERS` 保持修饰键;此前 keysym 被当作 keycode,Enter 会输入 `4`,修饰键不生效。 || 2026-10-09 | Windows 截图可靠性 | 极小、最小化、隐藏、cloaked 或离屏窗口不会再向模型返回不可用的 PNG;极端情况下仍可读取 accessibility tree,并明确限制依赖截图坐标的操作。 | Windows bridge 在截图前校验窗口状态、virtual screen 交集和最小尺寸,Go 层再次解码 PNG header 复核;无有效截图时省略 image block,coordinate click / drag fail closed,不做放大、补边或占位图。 |## 2026-09