Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

fix(linux): send press_key named keys and modifiers as keysyms

Aaron Hammer40c12decc1ee
diff --git a/apps/OpenComputerUseLinux/runtime.py b/apps/OpenComputerUseLinux/runtime.py
index 6c0f248..0759575 100644
--- a/apps/OpenComputerUseLinux/runtime.py
+++ b/apps/OpenComputerUseLinux/runtime.py
@@ -644,24 +644,33 @@ KEY_ALIASES = {
"next": "Page_Down",
}
-MODIFIER_KEYS = {
- "ctrl": "Control_L",
- "control": "Control_L",
- "shift": "Shift_L",
- "alt": "Alt_L",
- "super": "Super_L",
- "win": "Super_L",
- "cmd": "Super_L",
+# X11 modifier bits for LOCKMODIFIERS/UNLOCKMODIFIERS. Super is usually Mod4.
+MODIFIER_BITS = {
+ "ctrl": "CONTROL",
+ "control": "CONTROL",
+ "shift": "SHIFT",
+ "alt": "ALT",
+ "super": "META3",
+ "win": "META3",
+ "cmd": "META3",
}
+# GDK returns VoidSymbol for names it doesn't know.
+VOID_SYMBOL = 0xFFFFFF
+
+
def keyval(name):
- if Gdk is not None:
- value = Gdk.keyval_from_name(name)
- if value:
- return int(value)
+ # Characters have no name lookup in GDK ("/" is "slash"), so map them
+ # from Unicode instead.
if len(name) == 1:
+ if Gdk is not None:
+ return int(Gdk.unicode_to_keyval(ord(name)))
return ord(name)
+ if Gdk is not None:
+ value = int(Gdk.keyval_from_name(name))
+ if value and value != VOID_SYMBOL:
+ return value
raise RuntimeError("Unsupported key: " + name)
@@ -670,24 +679,29 @@ def send_key(key):
if not parts:
raise RuntimeError("Unsupported key: " + str(key))
main = parts[-1]
- modifiers = parts[:-1]
- pressed = []
- for modifier in modifiers:
- name = MODIFIER_KEYS.get(modifier.lower())
+ mask = 0
+ for modifier in parts[:-1]:
+ name = MODIFIER_BITS.get(modifier.lower())
if name is None:
- continue
- value = keyval(name)
- Atspi.generate_keyboard_event(value, None, Atspi.KeySynthType.PRESS)
- pressed.append(value)
+ raise RuntimeError("Unsupported modifier: " + modifier)
+ mask |= 1 << int(getattr(Atspi.ModifierType, name))
normalized = KEY_ALIASES.get(main.lower(), main)
- if len(normalized) == 1:
- Atspi.generate_keyboard_event(0, normalized, Atspi.KeySynthType.STRING)
- else:
- Atspi.generate_keyboard_event(
- keyval(normalized), None, Atspi.KeySynthType.PRESSRELEASE
- )
- for value in reversed(pressed):
- Atspi.generate_keyboard_event(value, None, Atspi.KeySynthType.RELEASE)
+ if mask:
+ Atspi.generate_keyboard_event(mask, None, Atspi.KeySynthType.LOCKMODIFIERS)
+ try:
+ if len(normalized) == 1 and not mask:
+ Atspi.generate_keyboard_event(0, normalized, Atspi.KeySynthType.STRING)
+ else:
+ # SYM takes a keysym. PRESS, RELEASE, and PRESSRELEASE take a
+ # hardware keycode, so passing a keysym there sends another key.
+ Atspi.generate_keyboard_event(
+ keyval(normalized), None, Atspi.KeySynthType.SYM
+ )
+ finally:
+ if mask:
+ Atspi.generate_keyboard_event(
+ mask, None, Atspi.KeySynthType.UNLOCKMODIFIERS
+ )
def send_text(text):
diff --git a/apps/OpenComputerUseLinux/runtime_test.py b/apps/OpenComputerUseLinux/runtime_test.py
index c42f41e..f52e8e3 100644
--- a/apps/OpenComputerUseLinux/runtime_test.py
+++ b/apps/OpenComputerUseLinux/runtime_test.py
@@ -86,5 +86,90 @@ class RuntimeInterfaceDetectionTests(unittest.TestCase):
self.assertTrue(self.runtime.set_element_value(node, "hello"))
+class FakeKeySynthType:
+ PRESS = "PRESS"
+ RELEASE = "RELEASE"
+ PRESSRELEASE = "PRESSRELEASE"
+ SYM = "SYM"
+ STRING = "STRING"
+ LOCKMODIFIERS = "LOCKMODIFIERS"
+ UNLOCKMODIFIERS = "UNLOCKMODIFIERS"
+
+
+class FakeModifierType:
+ SHIFT = 0
+ CONTROL = 2
+ ALT = 3
+ META3 = 6
+
+
+KEYSYMS = {"Return": 0xFF0D, "Tab": 0xFF09, "Down": 0xFF54}
+
+
+class RuntimeKeyTests(unittest.TestCase):
+ def setUp(self):
+ self.runtime = load_runtime()
+ self.events = []
+ self.runtime.Atspi = types.SimpleNamespace(
+ KeySynthType=FakeKeySynthType,
+ ModifierType=FakeModifierType,
+ generate_keyboard_event=lambda value, text, synth: self.events.append(
+ (value, text, synth)
+ ),
+ )
+ self.runtime.Gdk = types.SimpleNamespace(
+ # Like GDK, unknown names map to VoidSymbol rather than 0.
+ keyval_from_name=lambda name: KEYSYMS.get(name, 0xFFFFFF),
+ unicode_to_keyval=lambda codepoint: codepoint,
+ )
+
+ def test_named_key_is_sent_as_keysym(self):
+ self.runtime.send_key("Enter")
+
+ self.assertEqual(self.events, [(0xFF0D, None, "SYM")])
+
+ def test_modifiers_are_held_around_the_key(self):
+ self.runtime.send_key("ctrl+shift+Tab")
+
+ mask = (1 << FakeModifierType.CONTROL) | (1 << FakeModifierType.SHIFT)
+ self.assertEqual(
+ self.events,
+ [
+ (mask, None, "LOCKMODIFIERS"),
+ (0xFF09, None, "SYM"),
+ (mask, None, "UNLOCKMODIFIERS"),
+ ],
+ )
+
+ def test_modified_character_is_sent_as_keysym(self):
+ self.runtime.send_key("ctrl+a")
+
+ mask = 1 << FakeModifierType.CONTROL
+ self.assertEqual(self.events[1], (ord("a"), None, "SYM"))
+ self.assertEqual(self.events[0], (mask, None, "LOCKMODIFIERS"))
+
+ def test_modified_punctuation_is_sent_as_keysym(self):
+ self.runtime.send_key("ctrl+/")
+
+ self.assertEqual(self.events[1], (ord("/"), None, "SYM"))
+
+ def test_unknown_key_name_is_rejected(self):
+ with self.assertRaises(RuntimeError):
+ self.runtime.send_key("NoSuchKey")
+
+ self.assertEqual(self.events, [])
+
+ def test_plain_character_is_typed(self):
+ self.runtime.send_key("a")
+
+ self.assertEqual(self.events, [(0, "a", "STRING")])
+
+ def test_unknown_modifier_is_rejected(self):
+ with self.assertRaises(RuntimeError):
+ self.runtime.send_key("hyper+a")
+
+ self.assertEqual(self.events, [])
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 7042651..2ed833b 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -128,7 +128,7 @@
- Linux runtime 位于 `apps/OpenComputerUseLinux`,以 Go 维护 CLI、`call --calls` sequence、MCP JSON-RPC、tool schema 和进程内 snapshot cache。
- 构建入口是 `scripts/build-open-computer-use-linux.sh --arch arm64|amd64`,默认输出到 `dist/linux/<arch>/open-computer-use`;npm release package 会把两个 Linux artifact 内置到已有 root/alias packages,Node launcher 按 `process.platform/process.arch` 自动选择。
- Go runtime 通过 `go:embed` 带上 `runtime.py`,执行 tool call 时临时落盘并调用 `python3`。Python bridge 使用 GNOME/GObject Introspection 暴露的 AT-SPI2 接口做 app/window discovery、accessibility tree rendering、semantic action、editable text、value set,以及 best-effort 的 key/mouse fallback;文本能力通过 `Accessible.get_interfaces()` 检测 `Text` / `EditableText`,不依赖不同 PyGObject 版本未必存在的便捷属性。
-- Linux 上最接近 macOS AX 的是 AT-SPI2/D-Bus accessibility,而不是一套统一的后台键鼠输入模型。第一版优先使用元素暴露的 AT-SPI action、EditableText 和 Value 接口;coordinate `click` / `drag` 与 `press_key` 使用 AT-SPI event synthesis fallback,在 Wayland 下只能按 best-effort 处理。
+- Linux 上最接近 macOS AX 的是 AT-SPI2/D-Bus accessibility,而不是一套统一的后台键鼠输入模型。第一版优先使用元素暴露的 AT-SPI action、EditableText 和 Value 接口;coordinate `click` / `drag` 与 `press_key` 使用 AT-SPI event synthesis fallback,在 Wayland 下只能按 best-effort 处理。`press_key` 的具名键以 keysym 配合 `KeySynthType.SYM` 发送,修饰键以 X modifier mask 配合 `LOCKMODIFIERS` / `UNLOCKMODIFIERS` 保持;`PRESS` / `RELEASE` / `PRESSRELEASE` 只接受硬件 keycode,不能传 keysym。
- Linux runtime 需要运行在已登录桌面用户 session 里。缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,Go runtime 会在启动 Python AT-SPI bridge 前尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session bus、display / Wayland 值;纯 SSH tty 如果找不到已登录桌面 session,可以启动二进制,但不能直接 inspect 或操作 GUI session。
- `get_app_state` 的 accessibility tree 在 GTK/GNOME app 上可能很深,Linux bridge 使用与 macOS / Windows 一致的 1200 节点、64 层默认 tree budget,并支持显式提高 `max_tree_nodes` / `max_tree_depth`。截图通过 GDK root window best-effort capture;GNOME Wayland 可能返回黑图,bridge 会检测全黑采样并省略 image block。
- 这 9 个 tool 的协议面与 macOS / Windows 保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime path metadata,coordinate action 使用 screenshot/window-relative 坐标。
diff --git a/docs/histories/2026-10/20261009-1739-fix-linux-press-key-keysym.md b/docs/histories/2026-10/20261009-1739-fix-linux-press-key-keysym.md
new file mode 100644
index 0000000..bf7ba2e
--- /dev/null
+++ b/docs/histories/2026-10/20261009-1739-fix-linux-press-key-keysym.md
@@ -0,0 +1,26 @@
+## [2026-10-09 17:39] | Task: Fix Linux press_key keysym synthesis
+
+### 🤖 Execution Context
+* **Agent ID**: `Claude Code`
+* **Base Model**: `Claude Opus 5.5`
+* **Runtime**: `T3 Code / Claude Code harness`
+
+### 📥 User Query
+> 在 Linux 上 `press_key` 发送的键不对(Enter 变成 `4`),确认原因并准备修复 Pull Request。
+
+### 🛠 Changes Overview
+**Scope:** Linux Computer Use runtime
+
+**Key Actions:**
+- **具名键改用 keysym 合成**: `send_key` 对 Enter、Tab、Escape、方向键等具名键使用 `KeySynthType.SYM`,不再把 keysym 当作 keycode 传给 `PRESSRELEASE`。
+- **修饰键改用 modifier mask**: `ctrl` / `shift` / `alt` / `super` 通过 `LOCKMODIFIERS` / `UNLOCKMODIFIERS` 保持,组合键中的字符通过 `Gdk.unicode_to_keyval` 转为 keysym(`/`、`-`、`=` 等标点在 GDK 中没有同名 keysym 名称);未知修饰键和返回 `VoidSymbol` 的未知键名直接报错,不再静默发送。
+- **回归覆盖**: 新增不依赖真实桌面的 `send_key` Python 测试,并同步 Linux 架构说明与功能发布记录。
+
+### 🧠 Design Intent (Why)
+`Atspi.generate_keyboard_event` 在 `PRESS` / `RELEASE` / `PRESSRELEASE` 模式下只接受硬件 keycode。原实现传入 `Gdk.keyval_from_name` 返回的 keysym,AT-SPI 取其低字节当作 keycode:Return(0xFF0D)变成 keycode 13,即美式键盘的 `4`;Control_L(0xFFE3)变成未映射的 keycode 227,因此快捷键缺少修饰键。`SYM` 模式接受 keysym 并完成按下与抬起,modifier mask 则由 AT-SPI 负责锁定与释放。在 Ubuntu 26.04、X11、Google Chrome 中用 keydown 记录验证:Return、Tab、Escape、Down、Page_Down、Left、BackSpace、`ctrl+a`、`shift+Left`、`shift+Home`、`alt+Left`、`ctrl+/`、`ctrl+-`、`ctrl+=`、`ctrl+,` 均送达正确按键。
+
+### 📁 Files Modified
+- `apps/OpenComputerUseLinux/runtime.py`
+- `apps/OpenComputerUseLinux/runtime_test.py`
+- `docs/ARCHITECTURE.md`
+- `docs/releases/feature-release-notes.md`
diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.md
index 4a24607..d6383cd 100644
--- a/docs/releases/feature-release-notes.md
+++ b/docs/releases/feature-release-notes.md
@@ -4,6 +4,7 @@
| 日期 | 功能域 | 用户价值 | 变更摘要 |
| --- | --- | --- | --- |
+| 2026-10-09 | Linux 按键 | Linux 上 `press_key` 的 Enter、Tab、Escape、方向键和 `ctrl+a` 等快捷键会送达正确的按键,Agent 可以提交表单、切换焦点和使用快捷键。 | Linux bridge 改用 `KeySynthType.SYM` 发送具名键 keysym,并以 `LOCKMODIFIERS` / `UNLOCKMODIFIERS` 保持修饰键;此前 keysym 被当作 keycode,Enter 会输入 `4`,修饰键不生效。 |
| 2026-10-09 | Windows 截图可靠性 | 极小、最小化、隐藏、cloaked 或离屏窗口不会再向模型返回不可用的 PNG;极端情况下仍可读取 accessibility tree,并明确限制依赖截图坐标的操作。 | Windows bridge 在截图前校验窗口状态、virtual screen 交集和最小尺寸,Go 层再次解码 PNG header 复核;无有效截图时省略 image block,coordinate click / drag fail closed,不做放大、补边或占位图。 |
## 2026-09