Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

Auto-detect Linux desktop session env

Leo724fb2a8f7b5
diff --git a/README.md b/README.md
index 57cc7a6..dce6374 100644
--- a/README.md
+++ b/README.md
@@ -107,7 +107,7 @@ open-computer-use call list_apps
open-computer-use call --calls '[{"tool":"get_app_state","args":{"app":"gnome-text-editor"}},{"tool":"type_text","args":{"app":"gnome-text-editor","text":"hello"}}]'
```
-The process needs the desktop user's `XDG_RUNTIME_DIR`, `DBUS_SESSION_BUS_ADDRESS`, and display environment. A pure SSH tty without those variables can build and launch the binary, but it cannot inspect or operate the GUI session. Screenshot capture is best-effort on GNOME Wayland and may be omitted when the compositor returns a black frame.
+The runtime needs the desktop user's D-Bus and display session. When those variables are missing, it tries to discover the current user's signed-in desktop session at launch from `/run/user/<uid>` and common desktop processes, so the normal Codex setup remains `npm i -g open-computer-use`, `open-computer-use install-codex-mcp`, then restart Codex from the same desktop user. A pure SSH tty without a discoverable signed-in desktop session can build and launch the binary, but it cannot inspect or operate the GUI session. Screenshot capture is best-effort on GNOME Wayland and may be omitted when the compositor returns a black frame.
## Cursor Motion
diff --git a/README.zh-CN.md b/README.zh-CN.md
index 91c370f..cb9aa8b 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -117,7 +117,7 @@ open-computer-use call list_apps
open-computer-use call --calls '[{"tool":"get_app_state","args":{"app":"gnome-text-editor"}},{"tool":"type_text","args":{"app":"gnome-text-editor","text":"hello"}}]'
```
-这个进程需要桌面用户的 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 和 display 环境。纯 SSH tty 可以构建和启动二进制,但不能直接 inspect 或操作 GUI session。GNOME Wayland 下截图是 best-effort,如果 compositor 返回黑图,Linux bridge 会省略 image block。
+runtime 需要桌面用户的 D-Bus 和 display session。缺少这些变量时,它会在启动时尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户已登录的桌面 session,所以 Codex 的正常接入路径仍然是 `npm i -g open-computer-use`、`open-computer-use install-codex-mcp`,然后用同一个桌面用户重启 Codex。纯 SSH tty 如果找不到已登录桌面 session,可以构建和启动二进制,但不能直接 inspect 或操作 GUI session。GNOME Wayland 下截图是 best-effort,如果 compositor 返回黑图,Linux bridge 会省略 image block。
## Cursor Motion
diff --git a/apps/OpenComputerUseLinux/main.go b/apps/OpenComputerUseLinux/main.go
index 5b18bb9..982d228 100644
--- a/apps/OpenComputerUseLinux/main.go
+++ b/apps/OpenComputerUseLinux/main.go
@@ -12,12 +12,14 @@ import (
"os/exec"
"path/filepath"
"runtime"
+ "sort"
"strconv"
"strings"
+ "syscall"
"time"
)
-var version = "0.1.35"
+var version = "0.1.36"
//go:embed runtime.py
var linuxRuntimeScript string
@@ -459,6 +461,7 @@ func runPython(request linuxRequest) (*linuxResponse, error) {
defer cancel()
cmd := exec.CommandContext(ctx, "python3", scriptPath, operationPath)
+ cmd.Env = linuxRuntimeEnvironment(os.Environ())
var stderr bytes.Buffer
cmd.Stderr = &stderr
output, err := cmd.Output()
@@ -483,6 +486,414 @@ func runPython(request linuxRequest) (*linuxResponse, error) {
return &response, nil
}
+func linuxRuntimeEnvironment(base []string) []string {
+ uid := os.Getuid()
+ return linuxRuntimeEnvironmentFrom(base, uid, desktopProcessEnvironments(uid))
+}
+
+func linuxRuntimeEnvironmentFrom(base []string, uid int, processEnvs []map[string]string) []string {
+ env := envSliceToMap(base)
+ runtimeDir := chooseRuntimeDir(env, processEnvs, uid)
+ if runtimeDir != "" {
+ env["XDG_RUNTIME_DIR"] = runtimeDir
+ }
+
+ if value := sessionBusAddress(env["DBUS_SESSION_BUS_ADDRESS"], runtimeDir, processEnvs); value != "" {
+ env["DBUS_SESSION_BUS_ADDRESS"] = value
+ }
+ if value := waylandDisplay(env["WAYLAND_DISPLAY"], runtimeDir, processEnvs); value != "" {
+ env["WAYLAND_DISPLAY"] = value
+ }
+
+ for _, key := range []string{
+ "DISPLAY",
+ "XAUTHORITY",
+ "XDG_CURRENT_DESKTOP",
+ "XDG_SESSION_DESKTOP",
+ "XDG_SESSION_TYPE",
+ "DESKTOP_SESSION",
+ "GDK_BACKEND",
+ "QT_QPA_PLATFORMTHEME",
+ "AT_SPI_BUS_ADDRESS",
+ } {
+ if strings.TrimSpace(env[key]) == "" {
+ if value := firstSessionValue(processEnvs, key); value != "" {
+ env[key] = value
+ }
+ }
+ }
+
+ return envMapToSlice(base, env)
+}
+
+func chooseRuntimeDir(env map[string]string, processEnvs []map[string]string, uid int) string {
+ candidates := []string{env["XDG_RUNTIME_DIR"]}
+ for _, processEnv := range processEnvs {
+ candidates = append(candidates, processEnv["XDG_RUNTIME_DIR"])
+ }
+ candidates = append(candidates, fmt.Sprintf("/run/user/%d", uid))
+
+ seen := map[string]bool{}
+ for _, candidate := range candidates {
+ candidate = strings.TrimSpace(candidate)
+ if candidate == "" {
+ continue
+ }
+ candidate = filepath.Clean(candidate)
+ if seen[candidate] {
+ continue
+ }
+ seen[candidate] = true
+ if validRuntimeDir(candidate, uid) {
+ return candidate
+ }
+ }
+ return ""
+}
+
+func sessionBusAddress(current, runtimeDir string, processEnvs []map[string]string) string {
+ current = strings.TrimSpace(current)
+ if runtimeDir != "" {
+ busPath := filepath.Join(runtimeDir, "bus")
+ if isSocket(busPath) && shouldUseRuntimeBus(current, runtimeDir) {
+ return "unix:path=" + busPath
+ }
+ }
+ if current != "" {
+ return current
+ }
+ for _, processEnv := range processEnvs {
+ value := strings.TrimSpace(processEnv["DBUS_SESSION_BUS_ADDRESS"])
+ if value == "" {
+ continue
+ }
+ if runtimeDir != "" {
+ busPath := filepath.Join(runtimeDir, "bus")
+ if isSocket(busPath) && strings.Contains(value, busPath) {
+ return "unix:path=" + busPath
+ }
+ }
+ return value
+ }
+ if runtimeDir != "" {
+ busPath := filepath.Join(runtimeDir, "bus")
+ if isSocket(busPath) {
+ return "unix:path=" + busPath
+ }
+ }
+ return ""
+}
+
+func shouldUseRuntimeBus(current, runtimeDir string) bool {
+ current = strings.TrimSpace(current)
+ if current == "" {
+ return true
+ }
+ busPath := filepath.Join(runtimeDir, "bus")
+ if strings.Contains(current, busPath) {
+ return true
+ }
+ return strings.Contains(current, "/run/user/") && !strings.Contains(current, runtimeDir)
+}
+
+func waylandDisplay(current, runtimeDir string, processEnvs []map[string]string) string {
+ if value := normalizeWaylandDisplay(current, runtimeDir); value != "" {
+ return value
+ }
+ for _, processEnv := range processEnvs {
+ if value := normalizeWaylandDisplay(processEnv["WAYLAND_DISPLAY"], runtimeDir); value != "" {
+ return value
+ }
+ }
+ if runtimeDir == "" {
+ return ""
+ }
+ return firstWaylandSocket(runtimeDir)
+}
+
+func normalizeWaylandDisplay(value, runtimeDir string) string {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return ""
+ }
+ if runtimeDir == "" {
+ return value
+ }
+ if filepath.IsAbs(value) {
+ if isSocket(value) {
+ return value
+ }
+ return ""
+ }
+ if isSocket(filepath.Join(runtimeDir, value)) {
+ return value
+ }
+ return ""
+}
+
+func firstWaylandSocket(runtimeDir string) string {
+ for _, name := range []string{"wayland-0", "wayland-1"} {
+ if isSocket(filepath.Join(runtimeDir, name)) {
+ return name
+ }
+ }
+ matches, err := filepath.Glob(filepath.Join(runtimeDir, "wayland-*"))
+ if err != nil {
+ return ""
+ }
+ sort.Strings(matches)
+ for _, match := range matches {
+ if strings.HasSuffix(match, ".lock") {
+ continue
+ }
+ if isSocket(match) {
+ return filepath.Base(match)
+ }
+ }
+ return ""
+}
+
+func firstSessionValue(processEnvs []map[string]string, key string) string {
+ for _, processEnv := range processEnvs {
+ if value := strings.TrimSpace(processEnv[key]); value != "" {
+ return value
+ }
+ }
+ return ""
+}
+
+type rankedProcessEnv struct {
+ env map[string]string
+ rank int
+ pid int
+}
+
+func desktopProcessEnvironments(uid int) []map[string]string {
+ entries, err := os.ReadDir("/proc")
+ if err != nil {
+ return nil
+ }
+
+ var candidates []rankedProcessEnv
+ for _, entry := range entries {
+ if !entry.IsDir() {
+ continue
+ }
+ pid, err := strconv.Atoi(entry.Name())
+ if err != nil {
+ continue
+ }
+ procDir := filepath.Join("/proc", entry.Name())
+ if !pathOwnedByUID(procDir, uid) {
+ continue
+ }
+ rank := desktopProcessRank(processSearchText(procDir))
+ if rank == 0 {
+ continue
+ }
+ processEnv := readProcEnviron(procDir)
+ if !hasSessionEnvSignal(processEnv) {
+ continue
+ }
+ candidates = append(candidates, rankedProcessEnv{
+ env: processEnv,
+ rank: rank + sessionEnvRank(processEnv),
+ pid: pid,
+ })
+ }
+
+ sort.SliceStable(candidates, func(i, j int) bool {
+ if candidates[i].rank != candidates[j].rank {
+ return candidates[i].rank > candidates[j].rank
+ }
+ return candidates[i].pid < candidates[j].pid
+ })
+
+ results := make([]map[string]string, 0, len(candidates))
+ for _, candidate := range candidates {
+ results = append(results, candidate.env)
+ }
+ return results
+}
+
+func processSearchText(procDir string) string {
+ var parts []string
+ if data, err := os.ReadFile(filepath.Join(procDir, "comm")); err == nil {
+ parts = append(parts, string(bytes.TrimSpace(data)))
+ }
+ if data, err := os.ReadFile(filepath.Join(procDir, "cmdline")); err == nil {
+ data = bytes.Trim(data, "\x00")
+ parts = append(parts, strings.ReplaceAll(string(data), "\x00", " "))
+ }
+ return strings.ToLower(strings.Join(parts, " "))
+}
+
+func desktopProcessRank(text string) int {
+ patterns := []struct {
+ needle string
+ rank int
+ }{
+ {"gnome-session", 100},
+ {"gnome-shell", 95},
+ {"plasmashell", 95},
+ {"kwin_wayland", 95},
+ {"kwin_x11", 95},
+ {"startplasma", 95},
+ {"cinnamon-session", 95},
+ {"mate-session", 95},
+ {"xfce4-session", 95},
+ {"lxqt-session", 95},
+ {"sway", 95},
+ {"wayfire", 95},
+ {"xorg", 80},
+ {"xwayland", 75},
+ {"gnome-terminal-server", 65},
+ {"ptyxis", 65},
+ {"kgx", 65},
+ {"konsole", 65},
+ {"xfce4-terminal", 65},
+ {"alacritty", 65},
+ {"wezterm", 65},
+ {"kitty", 65},
+ {"tilix", 65},
+ {"codex", 50},
+ {"dbus-daemon", 45},
+ {"systemd --user", 40},
+ }
+
+ rank := 0
+ for _, pattern := range patterns {
+ if strings.Contains(text, pattern.needle) && pattern.rank > rank {
+ rank = pattern.rank
+ }
+ }
+ return rank
+}
+
+func sessionEnvRank(env map[string]string) int {
+ rank := 0
+ for _, key := range []string{"XDG_RUNTIME_DIR", "DBUS_SESSION_BUS_ADDRESS"} {
+ if strings.TrimSpace(env[key]) != "" {
+ rank += 20
+ }
+ }
+ for _, key := range []string{"DISPLAY", "WAYLAND_DISPLAY"} {
+ if strings.TrimSpace(env[key]) != "" {
+ rank += 10
+ }
+ }
+ if strings.TrimSpace(env["XAUTHORITY"]) != "" {
+ rank += 5
+ }
+ return rank
+}
+
+func hasSessionEnvSignal(env map[string]string) bool {
+ for _, key := range []string{
+ "XDG_RUNTIME_DIR",
+ "DBUS_SESSION_BUS_ADDRESS",
+ "DISPLAY",
+ "WAYLAND_DISPLAY",
+ "XAUTHORITY",
+ "AT_SPI_BUS_ADDRESS",
+ } {
+ if strings.TrimSpace(env[key]) != "" {
+ return true
+ }
+ }
+ return false
+}
+
+func readProcEnviron(procDir string) map[string]string {
+ data, err := os.ReadFile(filepath.Join(procDir, "environ"))
+ if err != nil {
+ return nil
+ }
+ return parseNullEnv(data)
+}
+
+func parseNullEnv(data []byte) map[string]string {
+ env := map[string]string{}
+ for _, entry := range bytes.Split(data, []byte{0}) {
+ if len(entry) == 0 {
+ continue
+ }
+ key, value, ok := strings.Cut(string(entry), "=")
+ if ok && key != "" {
+ env[key] = value
+ }
+ }
+ return env
+}
+
+func envSliceToMap(items []string) map[string]string {
+ env := map[string]string{}
+ for _, item := range items {
+ key, value, ok := strings.Cut(item, "=")
+ if ok && key != "" {
+ env[key] = value
+ }
+ }
+ return env
+}
+
+func envMapToSlice(base []string, env map[string]string) []string {
+ items := make([]string, 0, len(env))
+ seen := map[string]bool{}
+ for _, item := range base {
+ key, _, ok := strings.Cut(item, "=")
+ if !ok || key == "" {
+ items = append(items, item)
+ continue
+ }
+ if value, ok := env[key]; ok {
+ items = append(items, key+"="+value)
+ seen[key] = true
+ }
+ }
+
+ var added []string
+ for key := range env {
+ if !seen[key] {
+ added = append(added, key)
+ }
+ }
+ sort.Strings(added)
+ for _, key := range added {
+ items = append(items, key+"="+env[key])
+ }
+ return items
+}
+
+func validRuntimeDir(path string, uid int) bool {
+ info, err := os.Stat(path)
+ if err != nil || !info.IsDir() {
+ return false
+ }
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ if !ok {
+ return false
+ }
+ return int(stat.Uid) == uid
+}
+
+func pathOwnedByUID(path string, uid int) bool {
+ info, err := os.Stat(path)
+ if err != nil {
+ return false
+ }
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ if !ok {
+ return false
+ }
+ return int(stat.Uid) == uid
+}
+
+func isSocket(path string) bool {
+ info, err := os.Stat(path)
+ return err == nil && info.Mode()&os.ModeSocket != 0
+}
+
func requiredString(args map[string]any, key string) string {
value, _ := args[key].(string)
return strings.TrimSpace(value)
@@ -692,7 +1103,7 @@ func runCLI(args []string, stdout io.Writer) error {
case "mcp":
return runMCP(os.Stdin, stdout)
case "doctor":
- fmt.Fprintln(stdout, "Linux runtime: AT-SPI2 and GDK are available when this process runs in the signed-in desktop session with XDG_RUNTIME_DIR and DBUS_SESSION_BUS_ADDRESS set.")
+ fmt.Fprintln(stdout, "Linux runtime: AT-SPI2 and GDK run against the signed-in desktop user's accessibility session. When Codex starts without XDG_RUNTIME_DIR, DBUS_SESSION_BUS_ADDRESS, or display variables, open-computer-use tries to discover the same user's session from /run/user/<uid> and desktop processes.")
return nil
case "list-apps":
result := newService().callTool("list_apps", map[string]any{})
diff --git a/apps/OpenComputerUseLinux/main_test.go b/apps/OpenComputerUseLinux/main_test.go
index 516d635..f320013 100644
--- a/apps/OpenComputerUseLinux/main_test.go
+++ b/apps/OpenComputerUseLinux/main_test.go
@@ -3,6 +3,9 @@ package main
import (
"bytes"
"encoding/json"
+ "net"
+ "os"
+ "path/filepath"
"strings"
"testing"
)
@@ -85,3 +88,79 @@ func TestLinuxRuntimeDocumentsATSPIAndFallbackBoundary(t *testing.T) {
t.Fatal("MCP instructions must document the Linux background-input boundary")
}
}
+
+func TestLinuxRuntimeEnvironmentDiscoversDesktopSession(t *testing.T) {
+ runtimeDir := shortTempDir(t)
+ listenUnixSocket(t, filepath.Join(runtimeDir, "bus"))
+ listenUnixSocket(t, filepath.Join(runtimeDir, "wayland-0"))
+
+ env := envSliceToMap(linuxRuntimeEnvironmentFrom(
+ []string{"PATH=/usr/bin"},
+ os.Getuid(),
+ []map[string]string{{
+ "XDG_RUNTIME_DIR": runtimeDir,
+ "DISPLAY": ":1",
+ "XAUTHORITY": "/tmp/open-computer-use-xauth",
+ "XDG_SESSION_TYPE": "wayland",
+ "XDG_CURRENT_DESKTOP": "GNOME",
+ }},
+ ))
+
+ if got := env["XDG_RUNTIME_DIR"]; got != runtimeDir {
+ t.Fatalf("XDG_RUNTIME_DIR = %q, want %q", got, runtimeDir)
+ }
+ if got, want := env["DBUS_SESSION_BUS_ADDRESS"], "unix:path="+filepath.Join(runtimeDir, "bus"); got != want {
+ t.Fatalf("DBUS_SESSION_BUS_ADDRESS = %q, want %q", got, want)
+ }
+ if got := env["WAYLAND_DISPLAY"]; got != "wayland-0" {
+ t.Fatalf("WAYLAND_DISPLAY = %q, want wayland-0", got)
+ }
+ if got := env["DISPLAY"]; got != ":1" {
+ t.Fatalf("DISPLAY = %q, want :1", got)
+ }
+ if got := env["XDG_CURRENT_DESKTOP"]; got != "GNOME" {
+ t.Fatalf("XDG_CURRENT_DESKTOP = %q, want GNOME", got)
+ }
+}
+
+func TestLinuxRuntimeEnvironmentCanonicalizesRuntimeBus(t *testing.T) {
+ runtimeDir := shortTempDir(t)
+ listenUnixSocket(t, filepath.Join(runtimeDir, "bus"))
+
+ env := envSliceToMap(linuxRuntimeEnvironmentFrom(
+ []string{
+ "XDG_RUNTIME_DIR=" + runtimeDir,
+ "DBUS_SESSION_BUS_ADDRESS=unix:path=" + filepath.Join(runtimeDir, "bus") + ",guid=stale",
+ },
+ os.Getuid(),
+ nil,
+ ))
+
+ if got, want := env["DBUS_SESSION_BUS_ADDRESS"], "unix:path="+filepath.Join(runtimeDir, "bus"); got != want {
+ t.Fatalf("DBUS_SESSION_BUS_ADDRESS = %q, want %q", got, want)
+ }
+}
+
+func listenUnixSocket(t *testing.T, path string) {
+ t.Helper()
+ listener, err := net.Listen("unix", path)
+ if err != nil {
+ t.Fatalf("listen unix socket %s: %v", path, err)
+ }
+ t.Cleanup(func() {
+ _ = listener.Close()
+ _ = os.Remove(path)
+ })
+}
+
+func shortTempDir(t *testing.T) string {
+ t.Helper()
+ path, err := os.MkdirTemp("/tmp", "ocu-*")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ _ = os.RemoveAll(path)
+ })
+ return path
+}
diff --git a/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift b/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
index 3f383da..0932a0a 100644
--- a/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
+++ b/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
@@ -35,7 +35,7 @@ final class MCPClient {
_ = try request(method: "initialize", params: [
"clientInfo": [
"name": "OpenComputerUseSmokeSuite",
- "version": "0.1.35",
+ "version": "0.1.36",
],
"capabilities": [:],
"protocolVersion": "2025-03-26",
diff --git a/apps/OpenComputerUseWindows/main.go b/apps/OpenComputerUseWindows/main.go
index eb6440a..aa39b80 100644
--- a/apps/OpenComputerUseWindows/main.go
+++ b/apps/OpenComputerUseWindows/main.go
@@ -16,7 +16,7 @@ import (
"time"
)
-var version = "0.1.35"
+var version = "0.1.36"
//go:embed runtime.ps1
var windowsRuntimeScript string
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index c33184c..6867870 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -114,7 +114,7 @@
- 构建入口是 `scripts/build-open-computer-use-linux.sh --arch arm64|amd64`,默认输出到 `dist/linux/<arch>/open-computer-use`;npm release package 会把两个 Linux artifact 内置到已有 root/alias packages,Node launcher 按 `process.platform/process.arch` 自动选择。
- Go runtime 通过 `go:embed` 带上 `runtime.py`,执行 tool call 时临时落盘并调用 `python3`。Python bridge 使用 GNOME/GObject Introspection 暴露的 AT-SPI2 接口做 app/window discovery、accessibility tree rendering、semantic action、editable text、value set,以及 best-effort 的 key/mouse fallback。
- Linux 上最接近 macOS AX 的是 AT-SPI2/D-Bus accessibility,而不是一套统一的后台键鼠输入模型。第一版优先使用元素暴露的 AT-SPI action、EditableText 和 Value 接口;coordinate `click` / `drag` 与 `press_key` 使用 AT-SPI event synthesis fallback,在 Wayland 下只能按 best-effort 处理。
-- Linux runtime 需要运行在已登录桌面用户 session 里,并带有 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 和 display 环境。纯 SSH tty 可以启动二进制,但不能直接 inspect 或操作 GUI session;实机验证时通过 `runuser -u <desktop-user>` 挂到 `/run/user/<uid>/bus`。
+- Linux runtime 需要运行在已登录桌面用户 session 里。缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,Go runtime 会在启动 Python AT-SPI bridge 前尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session bus、display / Wayland 值;纯 SSH tty 如果找不到已登录桌面 session,可以启动二进制,但不能直接 inspect 或操作 GUI session。
- `get_app_state` 的 accessibility tree 在 GTK/GNOME app 上可能很深,Linux bridge 当前把 traversal depth 放宽到 64。截图通过 GDK root window best-effort capture;GNOME Wayland 可能返回黑图,bridge 会检测全黑采样并省略 image block。
- 这 9 个 tool 的协议面与 macOS / Windows 保持一致:`list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value`。其中 element-targeted action 会优先复用上一轮 `get_app_state` 的 runtime path metadata,coordinate action 使用 screenshot/window-relative 坐标。
- 当前 Linux 侧仍是功能性第一版:没有 visual cursor overlay、没有 installer/desktop entry,也没有独立 Linux fixture。后续 TODO 记录在 `docs/exec-plans/active/20260422-linux-computer-use-runtime.md`。
diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md
index c5eda97..bc49c0e 100644
--- a/docs/RELIABILITY.md
+++ b/docs/RELIABILITY.md
@@ -15,7 +15,7 @@
- 宿主必须有 `Accessibility` 与 `Screen Recording` 权限。
- smoke suite 依赖本地 GUI session,不能把它当成无头环境命令。
- 普通 app 的 `get_app_state` 结果依赖 AX tree 和窗口截图,复杂 app 上输出会有差异。
-- Linux runtime 依赖已登录桌面用户 session 的 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 和 display 环境;纯 SSH tty 默认不能直接访问 AT-SPI GUI tree。
+- Linux runtime 依赖已登录桌面用户 session;缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,会尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session env。纯 SSH tty 如果找不到桌面 session 仍不能直接访问 AT-SPI GUI tree。
- GNOME Wayland 截图可能被 compositor 限制,当前 Linux bridge 会把黑图视为无效截图并省略 image block。
## 当前故障排查顺序
@@ -24,7 +24,7 @@
2. 用 `.build/debug/OpenComputerUse list-apps` 确认目标 app 是否被发现。
3. 用 `.build/debug/OpenComputerUse snapshot <app>` 看是 transport 问题还是 snapshot / action 问题。
4. 如果只想验证仓库基线,直接跑 fixture + smoke,不要先在复杂第三方 app 上排查。
-5. 排查 Linux runtime 时,先确认目标命令是否运行在桌面用户 session 中,再用 `open-computer-use call list_apps` 和 `open-computer-use snapshot <app>` 区分 session/env 问题与 AT-SPI tree/action 问题。
+5. 排查 Linux runtime 时,先确认目标命令是否由桌面用户运行,再用 `open-computer-use call list_apps` 和 `open-computer-use snapshot <app>` 区分 session/env 问题与 AT-SPI tree/action 问题。如果是 Codex MCP,重新执行 `open-computer-use install-codex-mcp` 后重启 Codex,确认配置仍是 `open-computer-use mcp`。
## 后续补强方向
diff --git a/docs/exec-plans/active/20260422-linux-computer-use-runtime.md b/docs/exec-plans/active/20260422-linux-computer-use-runtime.md
index 39bd2cc..658ae41 100644
--- a/docs/exec-plans/active/20260422-linux-computer-use-runtime.md
+++ b/docs/exec-plans/active/20260422-linux-computer-use-runtime.md
@@ -35,7 +35,7 @@
- Linux 上最接近 macOS AX 的桌面接口是 AT-SPI2,经由 D-Bus 暴露 app/window/accessibility tree、actions、editable text 和 value 接口。
- Ubuntu GNOME 默认 Wayland session 下,任意后台坐标键鼠和截图都不是一套等价于 macOS AX 的通用模型。
- 第一版策略是 AT-SPI semantic action / editable text / value 优先,coordinate click / drag / key synthesis 仅作为 best-effort fallback。
- - SSH tty 默认没有 `XDG_RUNTIME_DIR` / `DBUS_SESSION_BUS_ADDRESS` / display 环境;真实 UI 测试需要跑在已登录桌面用户 session 里。
+ - SSH tty 默认没有 `XDG_RUNTIME_DIR` / `DBUS_SESSION_BUS_ADDRESS` / display 环境;runtime 会尝试为当前 Unix 用户自动发现已登录桌面 session,但跨用户 root 进程不应该被当作普通用户桌面的控制入口。
## 风险
@@ -67,7 +67,7 @@
- 运行 `get_app_state -> set_value -> type_text -> press_key -> perform_secondary_action -> click -> scroll -> drag` sequence。
- 确认每个 tool 返回 `isError=false`,并且 Text Editor 内容包含 marker。
- 观测检查:
- - 纯 SSH tty 没有桌面环境变量时应返回明确错误,而不是误判为 AT-SPI 逻辑失败。
+ - 当前桌面用户缺少桌面环境变量时,runtime 应先尝试自动发现同用户 session env;如果找不到已登录 session,再返回明确错误,而不是误判为 AT-SPI 逻辑失败。
## 进度记录
@@ -82,6 +82,7 @@
- [x] 在 Ubuntu VM 中验证 `call list_apps` 返回 `isError=false` 并包含 `gnome-text-editor`。
- [x] 在 Ubuntu VM 中验证 MCP `initialize` / `tools/list`,tool count 为 9。
- [x] 在 Ubuntu VM 中验证 8-tool sequence:`get_app_state`、`set_value`、`type_text`、`press_key`、`perform_secondary_action`、`click`、`scroll`、`drag` 均 `isError=false`。
+- [x] 在 Ubuntu VM 中验证 `0.1.36` 预发布二进制可在 `leo` 用户 `env -i` 下自动发现 session env,并跑通 MCP `tools/list`、`tools/call(list_apps)` 和 9-tool sequence。
- [ ] 增加 Linux fixture 和可重复 smoke runner。
- [ ] 评估 xdg-desktop-portal / compositor-specific screenshot 路径,补非黑图 capture。
- [x] 将 Linux artifact 接入 npm release packaging,作为既有 npm root/alias packages 的 bundled artifacts 分发。
@@ -95,3 +96,4 @@
- 2026-04-22:GNOME Text Editor 的 AT-SPI tree 深度超过 Windows runtime 沿用的 16 层,Linux bridge 单独把 traversal depth 放宽到 64。
- 2026-04-22:GNOME Wayland 下 GDK root capture 在 VM 上返回黑图;Linux bridge 检测全黑采样后省略 image block,后续再评估 portal/compositor-specific capture。
- 2026-04-23:Linux release artifact 接入 npm package bundled artifacts,不新增系统 installer;root `open-computer-use` package 通过 launcher 按 `linux-arm64` / `linux-x64` 自动选择 binary。
+- 2026-04-23:Linux runtime 不把 session env 写入 Codex config 或 shell profile;Go runtime 在每次启动 Python AT-SPI bridge 前为当前 Unix 用户动态发现 `/run/user/<uid>`、session bus、Wayland / X11 display 和 AT-SPI 相关环境。
diff --git a/docs/exec-plans/active/20260423-cross-platform-npm-distribution.md b/docs/exec-plans/active/20260423-cross-platform-npm-distribution.md
index a0a9ffd..7b61d0b 100644
--- a/docs/exec-plans/active/20260423-cross-platform-npm-distribution.md
+++ b/docs/exec-plans/active/20260423-cross-platform-npm-distribution.md
@@ -43,8 +43,8 @@
- 缓解方式:launcher 输出明确的缺失 bundled artifact 路径和重装命令。
- 风险:CI macOS runner 没有 Go toolchain,导致 Linux/Windows cross compile 失败。
- 缓解方式:release workflow 显式 setup Go。
-- 风险:Linux runtime 需要桌面 session,纯 SSH 不能验证 GUI 操作。
- - 缓解方式:本次 Linux 发布后验证目标限定为 npm install、版本、MCP initialize 和 `tools/list`。
+- 风险:Linux runtime 需要同一个桌面用户的已登录 session;跨用户 root 进程不能可靠控制普通用户桌面。
+ - 缓解方式:runtime 自动发现当前用户的 session env,发布后用 `leo` 桌面用户验证 npm install、MCP initialize、`tools/list` 和实际 `list_apps`。
## 里程碑
@@ -71,6 +71,7 @@
- GitHub Actions release workflow 成功:`24816330343`。
- npm registry `[email protected]`、`[email protected]`、`[email protected]` 可见。
- Linux VM `npm i -g [email protected]` 后 raw MCP `tools/list` 返回 9 个 tools。
+ - Linux VM `0.1.36` 预发布二进制在 `leo` 用户 `env -i` 下可自动发现 session env,并通过 raw MCP `tools/list` / `tools/call(list_apps)`。
## 进度记录
@@ -81,6 +82,7 @@
- [x] 完成本地验证:staging / release tarballs / dry-run publish / Swift tests / Linux Go tests / Windows Go tests / macOS npm prefix install / MCP tools list。
- [x] 完成 tag release、CI 跟踪、npm registry 验证。
- [x] 完成 Linux VM npm install 与 MCP tools/list 验证。
+- [x] 完成 Linux VM `0.1.36` 预发布二进制 env-less MCP tools/list 与 list_apps 验证。
## 决策记录
diff --git a/docs/histories/2026-04/20260423-1236-auto-linux-codex-mcp-session-env.md b/docs/histories/2026-04/20260423-1236-auto-linux-codex-mcp-session-env.md
new file mode 100644
index 0000000..ee2c4cb
--- /dev/null
+++ b/docs/histories/2026-04/20260423-1236-auto-linux-codex-mcp-session-env.md
@@ -0,0 +1,45 @@
+## [2026-04-23 12:36] | Task: Linux runtime should auto-detect desktop session env
+
+## 用户诉求
+
+希望 Linux 上也能直接按 `npm i -g open-computer-use`、`open-computer-use install-codex-mcp`、`codex` 的路径使用,不需要手动把 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 等桌面 session 环境变量写进 Codex 配置。
+
+## 主要改动
+
+- **[Linux Runtime]**: Go runtime 在调用 Python AT-SPI bridge 前会补齐缺失的 Linux 桌面 session env,从当前用户的 `/proc` 桌面进程和 `/run/user/<uid>` 自动发现 session bus、Wayland / X11 display、X authority 和 AT-SPI 相关环境。
+- **[Codex Install]**: `install-codex-mcp` 继续写入简单的 `open-computer-use mcp`,不把 session 相关变量固化到 `~/.codex/config.toml`。
+- **[Docs]**: README、中文 README、架构、可靠性文档和 release notes 同步说明 Linux runtime 的动态 session env 发现行为。
+- **[Version Bump]**: 将 Open Computer Use bump 到 `0.1.36`,用于发布包含 installer 修复的新 npm 版本。
+
+## 设计动机
+
+Linux AT-SPI 不是无环境的后台系统服务,它挂在已登录桌面用户的 D-Bus session 下。`tools/list` 可以只暴露 schema,但真实 `list_apps` / `get_app_state` 需要 bridge 进程带着桌面 session 环境启动。把探测逻辑放进 runtime,而不是写死到 Codex config,更适合 session 重启、不同 shell / terminal 和不同主流发行版的默认体验。
+
+## 受影响文件
+
+- `apps/OpenComputerUseLinux/main.go`
+- `apps/OpenComputerUseLinux/main_test.go`
+- `README.md`
+- `README.zh-CN.md`
+- `docs/ARCHITECTURE.md`
+- `docs/RELIABILITY.md`
+- `docs/releases/RELEASE_GUIDE.md`
+- `docs/releases/feature-release-notes.md`
+- 版本源相关文件
+
+## 验证
+
+- 通过:`bash -n scripts/install-codex-mcp.sh`
+- 通过:`node --check scripts/install-config-helper.mjs`
+- 通过:`node --check scripts/npm/build-packages.mjs`
+- 通过:`(cd apps/OpenComputerUseLinux && go test ./...)`
+- 通过:`(cd apps/OpenComputerUseWindows && go test ./...)`
+- 通过:`swift test`
+- 通过:`./scripts/build-open-computer-use-linux.sh --arch arm64`
+- 通过:`./scripts/build-open-computer-use-linux.sh --arch amd64`
+- 通过:`node ./scripts/npm/build-packages.mjs --out-dir dist/release/npm-staging-check`
+- 通过:`./scripts/release-package.sh`
+- 通过:临时 `CODEX_HOME` 安装配置检查,确认 Codex config 仍为 `command = "open-computer-use"` / `args = ["mcp"]`,没有写入 session env。
+- 通过:Linux VM 中按 `leo` 用户用 `env -i` 清空桌面环境后执行 `/tmp/open-computer-use-0.1.36-test call list_apps`,成功列出 `gnome-shell`、`gnome-text-editor` 和 `ptyxis`。
+- 通过:Linux VM 中按 `leo` 用户用 `env -i` 启动 MCP,`initialize`、`tools/list` 和 `tools/call(list_apps)` 均成功。
+- 通过:Linux VM 中按 `leo` 用户用 `env -i` 对 9 个工具执行 sequence smoke:`list_apps`、`get_app_state`、`click`、`set_value`、`type_text`、`press_key`、`scroll`、`drag`、`perform_secondary_action` 均返回 `isError=false`。
diff --git a/docs/releases/RELEASE_GUIDE.md b/docs/releases/RELEASE_GUIDE.md
index 57caa04..2bac200 100644
--- a/docs/releases/RELEASE_GUIDE.md
+++ b/docs/releases/RELEASE_GUIDE.md
@@ -31,7 +31,7 @@
也就是说:
- 只改 git tag,不改这个 manifest,不会得到新 npm 版本。
-- `scripts/npm/build-packages.mjs` 会从这个 manifest 读取版本,再生成三个 root/alias staging 包和六个 `open-computer-use-<os>-<arch>` platform staging 包。
+- `scripts/npm/build-packages.mjs` 会从这个 manifest 读取版本,再生成三个 root/alias staging 包;每个包内置 macOS、Linux 和 Windows runtime artifacts。
- 所以 release 前必须先把这份 manifest bump 到目标版本。
- 如果要让 `CursorMotion` 的 DMG 文件名和 release 页面资产名正确落到目标版本,也必须使用目标 tag 推送,或本地显式传入同样的 `--version`。
diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.md
index 301d746..8f34c68 100644
--- a/docs/releases/feature-release-notes.md
+++ b/docs/releases/feature-release-notes.md
@@ -4,6 +4,7 @@
| 日期 | 功能域 | 用户价值 | 变更摘要 |
| --- | --- | --- | --- |
+| 2026-04-23 | Linux Codex MCP 安装 | Linux 用户现在可以按 `npm i -g open-computer-use`、`open-computer-use install-codex-mcp`、`codex` 的路径接入,不再手动编辑桌面 session 的 D-Bus / display 环境变量。 | 发布 `0.1.36`,Linux runtime 在启动 AT-SPI bridge 前会动态发现当前用户的 `/run/user/<uid>` session bus 和 display / Wayland 环境,Codex 配置继续保持 `open-computer-use mcp`。 |
| 2026-04-23 | 三端 npm 安装 | `npm i -g open-computer-use` 现在会根据当前 `os-arch` 调用对应的 macOS `.app`、Linux binary 或 Windows `.exe`,不再把 npm 分发锁死在 macOS。 | 发布 `0.1.35`,三个既有 npm 包都会内置 macOS、Linux、Windows runtime,由跨平台 Node launcher 选择 native runtime;`0.1.34` 的新增 platform package 方案因 npm 包名权限失败后被 supersede。 |
| 2026-04-22 | Gemini / opencode MCP 安装 | Gemini CLI 和 opencode 用户现在也能像 Claude Code / Codex 一样用仓库内置命令完成 `open-computer-use` 的 MCP 接入,不需要再手动查各自配置格式。 | 发布 `0.1.33`,新增 `install-gemini-mcp` 和 `install-opencode-mcp`,将 npm launcher、README 与共享配置 helper 一并补齐,并让 Gemini 默认 project-scope 配置不会污染仓库 git 状态。 |
| 2026-04-22 | 发布版本重新对齐 | GitHub Release tag、插件 manifest 和仓库内版本常量重新回到同一个版本线;后续本地安装、staging 和诊断不会再出现 “release 已是新版本,但产物仍显示旧值” 的混乱状态。 | 发布 `0.1.32`,在远端 `v0.1.31` 已存在的前提下,将 `plugin.json`、Swift 版本常量、smoke/test 初始化请求和 CLI helper 文档统一 bump 到 `0.1.32`,为下一次 release 保持单一版本源。 |
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseVersion.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseVersion.swift
index 49249a6..3be1f45 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseVersion.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseVersion.swift
@@ -1,6 +1,6 @@
import Foundation
-public let openComputerUseVersion = "0.1.35"
+public let openComputerUseVersion = "0.1.36"
public func resolvedOpenComputerUseVersion(bundle: Bundle = .main) -> String {
if let version = bundle.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String,
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index f137d60..0edeb71 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -325,7 +325,7 @@ final class OpenComputerUseKitTests: XCTestCase {
func testInitializeResponseContainsToolsCapability() throws {
let server = StdioMCPServer(service: ComputerUseService())
- let response = server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","clientInfo":{"name":"test","version":"0.1.35"},"capabilities":{}}}"#)
+ let response = server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","clientInfo":{"name":"test","version":"0.1.36"},"capabilities":{}}}"#)
XCTAssertNotNil(response)
XCTAssertTrue(response!.contains(#""name":"open-computer-use""#))
XCTAssertTrue(response!.contains(#""tools":{"listChanged":false}"#))
@@ -334,7 +334,7 @@ final class OpenComputerUseKitTests: XCTestCase {
func testInitializeResponseContainsComputerUseInstructions() throws {
let server = StdioMCPServer(service: ComputerUseService())
let response = try XCTUnwrap(
- server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","clientInfo":{"name":"test","version":"0.1.35"},"capabilities":{}}}"#)
+ server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","clientInfo":{"name":"test","version":"0.1.36"},"capabilities":{}}}"#)
)
let data = try XCTUnwrap(response.data(using: .utf8))
let json = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any])
diff --git a/plugins/open-computer-use/.codex-plugin/plugin.json b/plugins/open-computer-use/.codex-plugin/plugin.json
index fe9d18e..5db968c 100644
--- a/plugins/open-computer-use/.codex-plugin/plugin.json
+++ b/plugins/open-computer-use/.codex-plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "open-computer-use",
- "version": "0.1.35",
+ "version": "0.1.36",
"description": "Open-source cross-platform Computer Use MCP server packaged as a Codex plugin.",
"author": {
"name": "Leo",
diff --git a/scripts/computer-use-cli/README.md b/scripts/computer-use-cli/README.md
index e1b7087..7b2c673 100644
--- a/scripts/computer-use-cli/README.md
+++ b/scripts/computer-use-cli/README.md
@@ -131,7 +131,7 @@ Example working invocation against `open-computer-use`:
```bash
go run . list-tools \
--transport direct \
- --server-bin ~/.codex/plugins/cache/open-computer-use-local/open-computer-use/0.1.35/scripts/launch-open-computer-use.sh
+ --server-bin ~/.codex/plugins/cache/open-computer-use-local/open-computer-use/0.1.36/scripts/launch-open-computer-use.sh
```
Example working comparison flow against a local repo build:
diff --git a/scripts/computer-use-cli/main.go b/scripts/computer-use-cli/main.go
index eadf6b9..b195ef7 100644
--- a/scripts/computer-use-cli/main.go
+++ b/scripts/computer-use-cli/main.go
@@ -21,7 +21,7 @@ import (
const (
cliName = "computer-use-cli"
- cliVersion = "0.1.35"
+ cliVersion = "0.1.36"
defaultTimeout = 60 * time.Second
pluginRootEnvVar = "COMPUTER_USE_PLUGIN_ROOT"
pluginVersionEnvVar = "COMPUTER_USE_PLUGIN_VERSION"