Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

Route macOS automation through app agent

xuyusong.0505fd603ba57a3c
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
new file mode 100644
index 0000000..c86fdce
--- /dev/null
+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
@@ -0,0 +1,485 @@
+import AppKit
+import Darwin
+import Foundation
+import OpenComputerUseKit
+
+private let appAgentCommand = "__open-computer-use-app-agent"
+private let appAgentDisableEnvironmentKey = "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY"
+
+enum MacOSAppAgentProxy {
+ static func isAgentInvocation(arguments: [String]) -> Bool {
+ arguments.first == appAgentCommand
+ }
+
+ @MainActor
+ static func runAgent(arguments: [String]) throws {
+ guard arguments.count == 2 else {
+ throw OpenComputerUseCLIError(message: "\(appAgentCommand) requires a socket path")
+ }
+
+ try MacOSAppAgentRuntime.run(socketPath: arguments[1])
+ }
+
+ static func shouldProxy(command: OpenComputerUseCLICommand) -> Bool {
+ shouldUseMacOSAppAgentProxy(
+ command: command,
+ proxyDisabled: proxyDisabled,
+ appBundleAvailable: PermissionSupport.currentAppBundleURL() != nil,
+ runningFromLaunchServicesAppInstance: isRunningFromLaunchServicesAppInstance
+ )
+ }
+
+ @MainActor
+ static func runProxy(command: OpenComputerUseCLICommand, arguments: [String]) throws -> Int32 {
+ let socketPath = defaultSocketPath()
+ let client = try connectOrLaunchAgent(socketPath: socketPath)
+
+ switch command {
+ case .mcp:
+ try proxyMCP(client: client)
+ return EXIT_SUCCESS
+ default:
+ let response = try sendCLIRequest(arguments: arguments, client: client)
+ if !response.stdout.isEmpty {
+ FileHandle.standardOutput.write(Data(response.stdout.utf8))
+ }
+ if !response.stderr.isEmpty {
+ FileHandle.standardError.write(Data(response.stderr.utf8))
+ }
+ return response.exitCode
+ }
+ }
+
+ private static var proxyDisabled: Bool {
+ let value = ProcessInfo.processInfo.environment[appAgentDisableEnvironmentKey]?.lowercased()
+ return value == "1" || value == "true" || value == "yes" || value == "on"
+ }
+
+ private static var isRunningFromOpenComputerUseAppBundle: Bool {
+ Bundle.main.bundleURL.standardizedFileURL.pathExtension == "app"
+ && PermissionSupport.isOpenComputerUseBundleIdentifier(Bundle.main.bundleIdentifier)
+ }
+
+ private static var isRunningFromLaunchServicesAppInstance: Bool {
+ isRunningFromOpenComputerUseAppBundle && getppid() == 1
+ }
+
+ private static func defaultSocketPath() -> String {
+ FileManager.default.temporaryDirectory
+ .appendingPathComponent("open-computer-use-agent.sock")
+ .standardizedFileURL
+ .path
+ }
+
+ @MainActor
+ private static func connectOrLaunchAgent(socketPath: String) throws -> AppAgentSocketClient {
+ if let client = AppAgentSocketClient.connect(path: socketPath) {
+ return client
+ }
+
+ unlink(socketPath)
+
+ guard let appURL = PermissionSupport.currentAppBundleURL() else {
+ throw OpenComputerUseCLIError(message: "Unable to locate Open Computer Use.app for app-scoped macOS permissions.")
+ }
+
+ let configuration = NSWorkspace.OpenConfiguration()
+ configuration.arguments = [appAgentCommand, socketPath]
+ configuration.activates = false
+ configuration.createsNewApplicationInstance = true
+
+ NSWorkspace.shared.openApplication(at: appURL, configuration: configuration) { _, _ in }
+
+ let deadline = Date().addingTimeInterval(10)
+ while Date() < deadline {
+ if let client = AppAgentSocketClient.connect(path: socketPath) {
+ return client
+ }
+ Thread.sleep(forTimeInterval: 0.05)
+ }
+
+ throw OpenComputerUseCLIError(message: "Timed out waiting for Open Computer Use.app agent to start.")
+ }
+
+ private static func proxyMCP(client: AppAgentSocketClient) throws {
+ while let line = readLine(strippingNewline: true) {
+ guard !line.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+ continue
+ }
+
+ let response = try client.request([
+ "kind": "mcp",
+ "line": line,
+ ])
+
+ if let responseLine = response["response"] as? String {
+ FileHandle.standardOutput.write(Data((responseLine + "\n").utf8))
+ }
+ }
+ }
+
+ private static func sendCLIRequest(arguments: [String], client: AppAgentSocketClient) throws -> CLIProxyResponse {
+ let response = try client.request([
+ "kind": "cli",
+ "arguments": arguments,
+ ])
+
+ return CLIProxyResponse(
+ stdout: response["stdout"] as? String ?? "",
+ stderr: response["stderr"] as? String ?? "",
+ exitCode: Int32(response["exitCode"] as? Int ?? 1)
+ )
+ }
+}
+
+private struct CLIProxyResponse {
+ let stdout: String
+ let stderr: String
+ let exitCode: Int32
+}
+
+@MainActor
+private final class MacOSAppAgentRuntime: NSObject, NSApplicationDelegate {
+ private let socketPath: String
+ private var listener: AppAgentSocketListener?
+ private var turnEndedObserver: NSObjectProtocol?
+
+ private init(socketPath: String) {
+ self.socketPath = socketPath
+ }
+
+ static func run(socketPath: String) throws {
+ let application = NSApplication.shared
+ application.setActivationPolicy(.accessory)
+
+ let delegate = MacOSAppAgentRuntime(socketPath: socketPath)
+ application.delegate = delegate
+ application.run()
+ }
+
+ func applicationDidFinishLaunching(_ notification: Notification) {
+ turnEndedObserver = DistributedNotificationCenter.default().addObserver(
+ forName: openComputerUseTurnEndedNotificationName,
+ object: nil,
+ queue: .main
+ ) { _ in
+ Task { @MainActor in
+ resetOpenComputerUseVisualCursor()
+ }
+ }
+
+ do {
+ let listener = try AppAgentSocketListener(path: socketPath)
+ self.listener = listener
+ listener.start()
+ } catch {
+ writeAgentError(error)
+ NSApp.terminate(nil)
+ }
+ }
+
+ func applicationWillTerminate(_ notification: Notification) {
+ if let turnEndedObserver {
+ DistributedNotificationCenter.default().removeObserver(turnEndedObserver)
+ }
+ listener?.stop()
+ }
+
+ func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool {
+ false
+ }
+
+ private func writeAgentError(_ error: Error) {
+ let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)
+ FileHandle.standardError.write(Data((message + "\n").utf8))
+ }
+}
+
+private final class AppAgentSocketListener: @unchecked Sendable {
+ private let path: String
+ private let socketFD: Int32
+ private var running = true
+
+ init(path: String) throws {
+ self.path = path
+ unlink(path)
+
+ socketFD = socket(AF_UNIX, SOCK_STREAM, 0)
+ guard socketFD >= 0 else {
+ throw POSIXError(.init(rawValue: errno) ?? .EIO)
+ }
+
+ var address = sockaddr_un()
+ address.sun_family = sa_family_t(AF_UNIX)
+ let pathCapacity = MemoryLayout.size(ofValue: address.sun_path)
+ try withUnsafeMutablePointer(to: &address.sun_path) { pointer in
+ try pointer.withMemoryRebound(to: CChar.self, capacity: pathCapacity) { buffer in
+ let bytes = Array(path.utf8)
+ guard bytes.count < pathCapacity else {
+ throw OpenComputerUseCLIError(message: "Socket path is too long: \(path)")
+ }
+ for index in 0..<bytes.count {
+ buffer[index] = CChar(bitPattern: bytes[index])
+ }
+ buffer[bytes.count] = 0
+ }
+ }
+
+ let bindResult = withUnsafePointer(to: &address) { pointer in
+ pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) {
+ bind(socketFD, $0, socklen_t(MemoryLayout<sockaddr_un>.size))
+ }
+ }
+ guard bindResult == 0 else {
+ close(socketFD)
+ throw POSIXError(.init(rawValue: errno) ?? .EIO)
+ }
+
+ guard listen(socketFD, 16) == 0 else {
+ close(socketFD)
+ throw POSIXError(.init(rawValue: errno) ?? .EIO)
+ }
+
+ guard chmod(path, mode_t(S_IRUSR | S_IWUSR)) == 0 else {
+ close(socketFD)
+ unlink(path)
+ throw POSIXError(.init(rawValue: errno) ?? .EIO)
+ }
+ }
+
+ func start() {
+ Thread.detachNewThread {
+ self.acceptLoop()
+ }
+ }
+
+ func stop() {
+ running = false
+ close(socketFD)
+ unlink(path)
+ }
+
+ private func acceptLoop() {
+ while running {
+ let clientFD = accept(socketFD, nil, nil)
+ guard clientFD >= 0 else {
+ if running {
+ Thread.sleep(forTimeInterval: 0.05)
+ }
+ continue
+ }
+
+ Thread.detachNewThread {
+ AppAgentConnection(fileDescriptor: clientFD).run()
+ }
+ }
+ }
+}
+
+private final class AppAgentConnection: @unchecked Sendable {
+ private let fileDescriptor: Int32
+ private let server = StdioMCPServer()
+
+ init(fileDescriptor: Int32) {
+ self.fileDescriptor = fileDescriptor
+ }
+
+ func run() {
+ guard let file = fdopen(fileDescriptor, "r+") else {
+ close(fileDescriptor)
+ return
+ }
+ defer { fclose(file) }
+
+ while let line = readAgentLine(file) {
+ let response = handle(requestLine: line)
+ writeAgentLine(response, to: file)
+ }
+ }
+
+ private func handle(requestLine: String) -> [String: Any] {
+ do {
+ guard let request = try JSONSerialization.jsonObject(with: Data(requestLine.utf8)) as? [String: Any],
+ let kind = request["kind"] as? String
+ else {
+ return ["error": "Invalid app-agent request"]
+ }
+
+ switch kind {
+ case "mcp":
+ let line = request["line"] as? String ?? ""
+ if let response = server.handle(line: line) {
+ return ["response": response]
+ }
+ return ["response": NSNull()]
+ case "cli":
+ let arguments = request["arguments"] as? [String] ?? []
+ let response = runCLI(arguments: arguments)
+ return [
+ "stdout": response.stdout,
+ "stderr": response.stderr,
+ "exitCode": Int(response.exitCode),
+ ]
+ default:
+ return ["error": "Unknown app-agent request kind: \(kind)"]
+ }
+ } catch {
+ let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)
+ return ["error": message]
+ }
+ }
+
+ private func runCLI(arguments: [String]) -> CLIProxyResponse {
+ do {
+ let command = try parseOpenComputerUseCLI(arguments: arguments)
+
+ switch command {
+ case .launchOnboarding:
+ let permissions = PermissionDiagnostics.current()
+ if !permissions.allGranted {
+ Task { @MainActor in
+ PermissionOnboardingApp.present()
+ }
+ }
+ return CLIProxyResponse(stdout: "", stderr: "", exitCode: EXIT_SUCCESS)
+
+ case .doctor:
+ let permissions = PermissionDiagnostics.current()
+ if !permissions.missingPermissions.isEmpty {
+ Task { @MainActor in
+ PermissionOnboardingApp.present()
+ }
+ }
+ return CLIProxyResponse(stdout: permissions.summary + "\n", stderr: "", exitCode: EXIT_SUCCESS)
+
+ case .listApps:
+ let service = ComputerUseService()
+ return CLIProxyResponse(stdout: (service.listApps().primaryText ?? "") + "\n", stderr: "", exitCode: EXIT_SUCCESS)
+
+ case let .snapshot(app):
+ let service = ComputerUseService()
+ let text = try service.getAppState(app: app).primaryText ?? ""
+ return CLIProxyResponse(stdout: text + "\n", stderr: "", exitCode: EXIT_SUCCESS)
+
+ case let .call(invocation):
+ let output = try runOpenComputerUseCall(invocation)
+ return CLIProxyResponse(
+ stdout: try output.jsonText() + "\n",
+ stderr: "",
+ exitCode: output.hasToolError ? EXIT_FAILURE : EXIT_SUCCESS
+ )
+
+ default:
+ return CLIProxyResponse(stdout: "", stderr: "Unsupported proxied command.\n", exitCode: EXIT_FAILURE)
+ }
+ } catch {
+ let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)
+ return CLIProxyResponse(stdout: "", stderr: message + "\n", exitCode: EXIT_FAILURE)
+ }
+ }
+}
+
+private final class AppAgentSocketClient: @unchecked Sendable {
+ private let file: UnsafeMutablePointer<FILE>
+
+ private init(file: UnsafeMutablePointer<FILE>) {
+ self.file = file
+ }
+
+ deinit {
+ fclose(file)
+ }
+
+ static func connect(path: String) -> AppAgentSocketClient? {
+ let fd = socket(AF_UNIX, SOCK_STREAM, 0)
+ guard fd >= 0 else {
+ return nil
+ }
+
+ var address = sockaddr_un()
+ address.sun_family = sa_family_t(AF_UNIX)
+ let pathCapacity = MemoryLayout.size(ofValue: address.sun_path)
+ let copied = withUnsafeMutablePointer(to: &address.sun_path) { pointer in
+ pointer.withMemoryRebound(to: CChar.self, capacity: pathCapacity) { buffer -> Bool in
+ let bytes = Array(path.utf8)
+ guard bytes.count < pathCapacity else {
+ return false
+ }
+ for index in 0..<bytes.count {
+ buffer[index] = CChar(bitPattern: bytes[index])
+ }
+ buffer[bytes.count] = 0
+ return true
+ }
+ }
+
+ guard copied else {
+ close(fd)
+ return nil
+ }
+
+ let result = withUnsafePointer(to: &address) { pointer in
+ pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) {
+ Darwin.connect(fd, $0, socklen_t(MemoryLayout<sockaddr_un>.size))
+ }
+ }
+ guard result == 0, let file = fdopen(fd, "r+") else {
+ close(fd)
+ return nil
+ }
+
+ return AppAgentSocketClient(file: file)
+ }
+
+ func request(_ object: [String: Any]) throws -> [String: Any] {
+ let data = try JSONSerialization.data(withJSONObject: object, options: [.withoutEscapingSlashes])
+ guard let line = String(data: data, encoding: .utf8) else {
+ throw ComputerUseError.message("Failed to encode app-agent request.")
+ }
+
+ writeAgentLine(line, to: file)
+
+ guard let responseLine = readAgentLine(file),
+ let response = try JSONSerialization.jsonObject(with: Data(responseLine.utf8)) as? [String: Any]
+ else {
+ throw ComputerUseError.message("Open Computer Use.app agent closed the connection.")
+ }
+
+ if let error = response["error"] as? String {
+ throw ComputerUseError.message(error)
+ }
+
+ return response
+ }
+}
+
+private func readAgentLine(_ file: UnsafeMutablePointer<FILE>) -> String? {
+ var bytes: [UInt8] = []
+
+ while true {
+ let character = fgetc(file)
+ if character == EOF {
+ return bytes.isEmpty ? nil : String(data: Data(bytes), encoding: .utf8)
+ }
+ if character == 10 {
+ return String(data: Data(bytes), encoding: .utf8)
+ }
+ bytes.append(UInt8(character))
+ }
+}
+
+private func writeAgentLine(_ object: [String: Any], to file: UnsafeMutablePointer<FILE>) {
+ if let data = try? JSONSerialization.data(withJSONObject: object, options: [.withoutEscapingSlashes]),
+ let line = String(data: data, encoding: .utf8)
+ {
+ writeAgentLine(line, to: file)
+ }
+}
+
+private func writeAgentLine(_ line: String, to file: UnsafeMutablePointer<FILE>) {
+ let output = line + "\n"
+ _ = output.withCString { pointer in
+ fputs(pointer, file)
+ }
+ fflush(file)
+}
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
index 7e764df..405633c 100644
--- a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
@@ -25,8 +25,18 @@ enum OpenComputerUseMain {
@MainActor
private static func run() throws {
let arguments = Array(CommandLine.arguments.dropFirst())
+
+ if MacOSAppAgentProxy.isAgentInvocation(arguments: arguments) {
+ try MacOSAppAgentProxy.runAgent(arguments: arguments)
+ return
+ }
+
let command = try parseOpenComputerUseCLI(arguments: arguments)
+ if MacOSAppAgentProxy.shouldProxy(command: command) {
+ exit(try MacOSAppAgentProxy.runProxy(command: command, arguments: arguments))
+ }
+
switch command {
case .mcp:
let service = ComputerUseService()
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift
index c01960c..f699592 100644
--- a/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift
+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift
@@ -5,6 +5,8 @@ import QuartzCore
@MainActor
enum PermissionOnboardingApp {
+ private static var presentedWindowController: PermissionWindowController?
+
static func launch() {
guard !PermissionDiagnostics.current().allGranted else {
return
@@ -18,6 +20,21 @@ enum PermissionOnboardingApp {
application.delegate = delegate
application.run()
}
+
+ static func present(terminateOnCompletion: Bool = true) {
+ guard !PermissionDiagnostics.current().allGranted else {
+ return
+ }
+
+ let application = NSApplication.shared
+ application.setActivationPolicy(.accessory)
+ application.applicationIconImage = Branding.makeAppIconImage(size: 256)
+
+ let controller = PermissionWindowController(terminateOnCompletion: terminateOnCompletion)
+ presentedWindowController = controller
+ controller.showWindow(nil)
+ NSApp.activate(ignoringOtherApps: true)
+ }
}
@MainActor
@@ -42,8 +59,11 @@ final class PermissionWindowController: NSWindowController {
private lazy var accessoryPanelController = PermissionAccessoryPanelController { [weak self] in
self?.handleAccessoryPanelBack()
}
+ private let terminateOnCompletion: Bool
+
+ init(terminateOnCompletion: Bool = true) {
+ self.terminateOnCompletion = terminateOnCompletion
- init() {
let window = NSWindow(
contentRect: NSRect(
x: 0,
@@ -97,7 +117,9 @@ extension PermissionWindowController: PermissionContentControllerDelegate {
func permissionContentControllerDidCompleteAllPermissions(_ controller: PermissionContentController) {
accessoryPanelController.hide()
close()
- NSApp.terminate(nil)
+ if terminateOnCompletion {
+ NSApp.terminate(nil)
+ }
}
private func handleAccessoryPanelBack() {
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 6867870..24cda41 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -37,13 +37,14 @@
- `OpenComputerUse` 默认 app 模式会拉起 `PermissionOnboardingApp`。
- app bundle 以 `LSUIElement` agent-style 形态运行,默认不在 Dock 暴露常驻图标,但仍可按需显示权限窗口。
+- 当用户从终端执行 macOS 版 `open-computer-use mcp`、`doctor`、`call`、`snapshot` 或 `list-apps` 时,CLI 会先通过 LaunchServices 启动同一个 `.app` bundle 的隐藏 app agent,并通过用户临时目录下的 Unix domain socket 转发请求;真正调用 Accessibility、ScreenCaptureKit 和动作 tools 的进程始终是 `Open Computer Use.app`,不是 iTerm / Terminal / Node launcher。
- 主窗口负责渲染 `Accessibility` / `Screen & System Audio Recording` 两类权限卡片、`Allow` / `Done` 状态和 relaunch 后的状态收敛;当两项权限都已完成时会自动关闭,不再要求用户手动退出。
- 辅助 drag panel 会跳转到对应的 `System Settings` 页面;点击 `Allow` 后,panel 会从主窗口里的按钮位置做一段 spring + curved frame 的入场,再落到 `System Settings` 内容区下沿。panel 默认保持在窗口右侧内容区下方居中并固定贴近窗口底边,不再依赖实时扫描权限页内部 `+ / -` 控件行;窗口层级上会显式排在当前 `System Settings` 窗口之上,避免被权限列表内容盖住,同时尽量减少对系统设置自身滚动区域的干扰。panel 内也补了显式返回按钮,允许用户中断当前 guidance、回到 onboarding 主窗口重新选择权限步骤。
- 权限状态优先基于 TCC 持久授权记录判断,避免 CLI 子进程与 GUI app 对授权状态看到不一致的结果;正式 release 仍以 CI 打出来的 `Open Computer Use.app` 为准,而本地 debug/dev 打包现在显式命名为 `Open Computer Use (Dev).app`,并在 dev bundle 运行时优先认当前 dev 副本,避免系统设置里出现两个完全同名的条目。
### 2. MCP 层
-- 当前只实现 `stdio` transport。
+- 面向 MCP host 的外部 transport 仍是 `stdio`;macOS 终端 CLI 到 `.app` app agent 之间额外有一层本地 Unix domain socket 代理,用来保证真实 automation 运行在 app bundle 权限身份下。
- 当 `OPEN_COMPUTER_USE_VISUAL_CURSOR` 未被显式关闭时,`mcp` 命令会切到一个最小 AppKit runtime:主线程保留 event loop 承载 overlay UI,stdio server 仍在后台线程串行读取与响应。
- 请求 framing 采用一行一个 JSON-RPC message。
- 当前支持的 method:
@@ -142,7 +143,7 @@
- Linux binary 构建:`./scripts/build-open-computer-use-linux.sh --arch arm64`
- 对比样本:`artifacts/tool-comparisons/20260417-focus-behavior/`
- 手工诊断:
- - `.build/debug/OpenComputerUse doctor`
- - `.build/debug/OpenComputerUse snapshot <app>`
- - `.build/debug/OpenComputerUse call list_apps`
- - `.build/debug/OpenComputerUse call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}}]'`
+ - `open-computer-use doctor`
+ - `open-computer-use snapshot <app>`
+ - `open-computer-use call list_apps`
+ - `open-computer-use call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}}]'`
diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md
index bc49c0e..1c85291 100644
--- a/docs/RELIABILITY.md
+++ b/docs/RELIABILITY.md
@@ -7,12 +7,12 @@
- 端到端 smoke:`./scripts/run-tool-smoke-tests.sh`
- Linux runtime:`(cd apps/OpenComputerUseLinux && go test ./...)`、`./scripts/build-open-computer-use-linux.sh --arch arm64`
- 本地诊断:
- - `.build/debug/OpenComputerUse doctor`
- - `.build/debug/OpenComputerUse snapshot <app>`
+ - `open-computer-use doctor`
+ - `open-computer-use snapshot <app>`
## 已知关键依赖
-- 宿主必须有 `Accessibility` 与 `Screen Recording` 权限。
+- macOS 上必须给 `Open Computer Use.app` 授权 `Accessibility` 与 `Screen Recording`;终端本身不应该再是必需授权对象。
- smoke suite 依赖本地 GUI session,不能把它当成无头环境命令。
- 普通 app 的 `get_app_state` 结果依赖 AX tree 和窗口截图,复杂 app 上输出会有差异。
- Linux runtime 依赖已登录桌面用户 session;缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,会尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session env。纯 SSH tty 如果找不到桌面 session 仍不能直接访问 AT-SPI GUI tree。
@@ -20,9 +20,9 @@
## 当前故障排查顺序
-1. 先跑 `.build/debug/OpenComputerUse doctor`,确认权限状态;如果缺权限,命令会直接拉起权限 onboarding 窗口,已全部授权则只打印状态并退出。
-2. 用 `.build/debug/OpenComputerUse list-apps` 确认目标 app 是否被发现。
-3. 用 `.build/debug/OpenComputerUse snapshot <app>` 看是 transport 问题还是 snapshot / action 问题。
+1. 先跑 `open-computer-use doctor`,确认权限状态;如果缺权限,命令会通过 `.app` app agent 拉起权限 onboarding 窗口,已全部授权则只打印状态并退出。
+2. 用 `open-computer-use list-apps` 确认目标 app 是否被发现。
+3. 用 `open-computer-use snapshot <app>` 看是 transport 问题还是 snapshot / action 问题。
4. 如果只想验证仓库基线,直接跑 fixture + smoke,不要先在复杂第三方 app 上排查。
5. 排查 Linux runtime 时,先确认目标命令是否由桌面用户运行,再用 `open-computer-use call list_apps` 和 `open-computer-use snapshot <app>` 区分 session/env 问题与 AT-SPI tree/action 问题。如果是 Codex MCP,重新执行 `open-computer-use install-codex-mcp` 后重启 Codex,确认配置仍是 `open-computer-use mcp`。
diff --git a/docs/SECURITY.md b/docs/SECURITY.md
index 744e587..eeaad4e 100644
--- a/docs/SECURITY.md
+++ b/docs/SECURITY.md
@@ -2,9 +2,9 @@
## 当前实现边界
-- 当前 server 只提供本地 `stdio` MCP,不对外监听 TCP/HTTP 端口。
+- 对 MCP host 暴露的接口仍是本地 `stdio`;macOS CLI 与 `.app` app agent 之间会使用用户临时目录下的 Unix domain socket,socket 创建后会收紧为当前用户读写,且不对外监听 TCP/HTTP 端口。
- 所有动作都必须显式带 `app` 参数;当前不会在后台自动扫描并控制任意 app。
-- 真实 app 路径依赖宿主进程已经获得 macOS `Accessibility` 与 `Screen Recording` 权限。
+- macOS 真实 app 路径依赖 `Open Computer Use.app` 已获得 `Accessibility` 与 `Screen Recording` 权限;终端里的 CLI / Node launcher 会把 `mcp`、`doctor`、`call`、`snapshot` 和 `list-apps` 转发给由 LaunchServices 启动的本地 app agent,避免把权限要求落到 iTerm / Terminal 身上。
- 实验性 Linux runtime 依赖已登录桌面用户的 AT-SPI2 / D-Bus session;coordinate mouse、drag、keyboard synthesis 只是 best-effort fallback,不应被视为跨 Wayland compositor 的通用后台输入授权。
## 数据处理
@@ -23,7 +23,7 @@
- 这意味着开源版当前的安全边界主要由:
- 明确的 tool 调用参数
- 内置高风险 denylist
- - 宿主进程权限
+ - `Open Computer Use.app` 的系统权限
- 本地使用场景
共同提供。
- 下一阶段应优先补:
diff --git a/docs/histories/2026-05/20260503-1731-app-scoped-permission-agent.md b/docs/histories/2026-05/20260503-1731-app-scoped-permission-agent.md
new file mode 100644
index 0000000..1d9c70a
--- /dev/null
+++ b/docs/histories/2026-05/20260503-1731-app-scoped-permission-agent.md
@@ -0,0 +1,53 @@
+## [2026-05-03 17:31] | Task: 修复 macOS 终端权限归属
+
+### Execution Context
+* **Agent ID**: `Codex`
+* **Base Model**: `GPT-5`
+* **Runtime**: `local macOS workspace`
+
+### User Query
+> `open-computer-use doctor` 能打开权限页面,但从终端执行时权限似乎要求的是 iTerm/Terminal,而不是 `Open Computer Use.app`;需要保证 iTerm 没有 Accessibility / Screen Recording 权限时,只给 Open Computer Use 授权也能工作。
+
+### Changes Overview
+**Scope:** `apps/OpenComputerUse`, `packages/OpenComputerUseKit`, `docs`
+
+**Key Actions:**
+- **[App agent proxy]**: 新增隐藏 app-agent 启动模式;终端 CLI 对 `mcp`、`doctor`、`call`、`snapshot` 和 `list-apps` 通过 Unix domain socket 转发到 LaunchServices 启动的 `.app` 进程。
+- **[Onboarding reuse]**: 将权限 onboarding 拆出可复用的 `present()` 路径,让 doctor 可以在 app agent 已运行的 NSApplication 内显示授权窗口,而不是在终端子进程内直接跑 UI。
+- **[Decision tests]**: 将 app-agent 代理选择规则下沉为 Kit 内纯函数,并补单测覆盖 automation 命令代理、非 automation 命令本地执行、LaunchServices 打开 app 不递归代理、禁用开关和缺少 bundle fallback。
+- **[Launch mode guard]**: 对无参数启动区分 LaunchServices 打开的 `.app` 和终端直接执行的 bundle executable,避免双击 app 时留下多余后台 agent,同时让终端入口仍走 app 身份代理。
+- **[Socket permissions]**: app-agent Unix socket 创建后收紧为当前用户读写,维持本地-only 权限边界。
+- **[Permission relaunch]**: app-agent 内完成授权后仍会终止当前 app 进程,确保下一次命令用重新启动后的授权进程执行 ScreenCaptureKit / AX 路径。
+- **[Permission wording]**: 更新权限错误和文档口径,明确 macOS 授权目标是 `Open Computer Use.app`,不是宿主终端。
+- **[Swift 6.2 build fix]**: 适配 Swift 6.2 并发检查,将 cursor reference `NSImage` 静态缓存显式标记为 `nonisolated(unsafe)`,保持当前 AppKit 主线程绘制路径可编译。
+
+### Design Intent (Why)
+macOS TCC 对 Accessibility 和 Screen Recording 的责任归属取决于真正调用系统 API 的进程。终端直接启动 native runtime 时,系统可能要求 iTerm/Terminal 获得权限;把真实 automation 放到通过 LaunchServices 启动的 app bundle 进程内,可以让用户只授权 `Open Computer Use.app`,终端负责 stdio/命令代理。
+
+### Files Modified
+- `apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift`
+- `apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift`
+- `apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorGlyphRenderer.swift`
+- `docs/ARCHITECTURE.md`
+- `docs/SECURITY.md`
+- `docs/RELIABILITY.md`
+
+### Verification
+- Passed: `git diff --check`
+- Passed: `./scripts/check-docs.sh`
+- Passed: `xcrun swiftc -parse apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift`
+- Passed: `xcrun swiftc -parse packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift`
+- Passed: `xcrun swiftc -parse packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
+- Passed: `PATH="$HOME/.swiftly/bin:$PATH" swift build --product OpenComputerUse` with Swift `6.2.4`.
+- Passed: `PATH="$HOME/.swiftly/bin:$PATH" ./scripts/build-open-computer-use-app.sh debug --arch native`.
+- Passed: `.build/arm64-apple-macosx/debug/OpenComputerUse doctor` returned missing permission status while launching `dist/Open Computer Use (Dev).app/Contents/MacOS/OpenComputerUse __open-computer-use-app-agent ...`, confirming the terminal command hands off to the app bundle process.
+- Passed: the app-agent socket was created as `srw-------`, current-user read/write only.
+- Passed: after granting the dev app both macOS permissions, `OpenComputerUse doctor` reported `accessibility=granted, screenRecording=granted`.
+- Passed: `OpenComputerUse call get_app_state --args '{"app":"Finder"}'` returned `isError=false` with both `text` and `image` content while iTerm did not hold the permissions.
+- Passed: Codex MCP integration via `codex exec` with the bundled official `computer-use` plugin disabled called `server="open-computer-use"` for `list_apps` and `get_app_state`.
+- Passed: Codex-driven action smoke covered `click`, `scroll`, `set_value`, `press_key`, `drag`, and `type_text`; Finder, Calendar, and the local fixture succeeded, while TextEdit `click` exposed an existing window-bounds edge case but TextEdit typing still succeeded.
+- Blocked locally: `PATH="$HOME/.swiftly/bin:$PATH" swift test` cannot import `XCTest` from the Swift.org toolchain in this CLT-only environment.
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index e591f3c..68591c9 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -79,7 +79,7 @@ enum SnapshotBuilder {
let permissions = PermissionDiagnostics.current()
guard permissions.accessibilityTrusted else {
- throw ComputerUseError.permissionDenied("Accessibility permission is required. Run `OpenComputerUse doctor` and grant access to the host terminal or app.")
+ throw ComputerUseError.permissionDenied("Accessibility permission is required. Run `open-computer-use doctor` and grant access to Open Computer Use.")
}
let appElement = AXUIElementCreateApplication(app.pid)
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
index 9db25f8..eb1307c 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
@@ -19,6 +19,26 @@ public enum OpenComputerUseCallInvocation: Equatable {
public let openComputerUseDefaultInterCallDelay: TimeInterval = 1
+public func shouldUseMacOSAppAgentProxy(
+ command: OpenComputerUseCLICommand,
+ proxyDisabled: Bool,
+ appBundleAvailable: Bool,
+ runningFromLaunchServicesAppInstance: Bool
+) -> Bool {
+ guard !proxyDisabled, appBundleAvailable else {
+ return false
+ }
+
+ switch command {
+ case .launchOnboarding:
+ return !runningFromLaunchServicesAppInstance
+ case .mcp, .doctor, .listApps, .snapshot, .call:
+ return true
+ case .turnEnded, .help, .version:
+ return false
+ }
+}
+
public struct OpenComputerUseCLIError: LocalizedError, Equatable {
public let message: String
public let helpCommand: String?
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorGlyphRenderer.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorGlyphRenderer.swift
index c616cb2..5df11d1 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorGlyphRenderer.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorGlyphRenderer.swift
@@ -56,7 +56,7 @@ private enum SoftwareCursorGlyphColors {
}
enum SoftwareCursorGlyphRenderer {
- private static let referenceImage = loadReferenceCursorWindowImage()
+ nonisolated(unsafe) private static let referenceImage = loadReferenceCursorWindowImage()
static func draw(
in bounds: CGRect,
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index 0edeb71..a08429e 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -204,6 +204,68 @@ final class OpenComputerUseKitTests: XCTestCase {
XCTAssertFalse(output.hasToolError)
}
+ func testMacOSAppAgentProxyDecisionRoutesAutomationCommandsThroughAppBundle() {
+ for command in [
+ OpenComputerUseCLICommand.mcp,
+ .doctor,
+ .listApps,
+ .snapshot(app: "TextEdit"),
+ .call(.single(toolName: "list_apps", argumentsJSON: nil, argumentsFile: nil)),
+ ] {
+ XCTAssertTrue(shouldUseMacOSAppAgentProxy(
+ command: command,
+ proxyDisabled: false,
+ appBundleAvailable: true,
+ runningFromLaunchServicesAppInstance: false
+ ))
+ }
+ }
+
+ func testMacOSAppAgentProxyDecisionKeepsNonAutomationCommandsLocal() {
+ for command in [
+ OpenComputerUseCLICommand.turnEnded(payload: nil),
+ .help(command: nil),
+ .version,
+ ] {
+ XCTAssertFalse(shouldUseMacOSAppAgentProxy(
+ command: command,
+ proxyDisabled: false,
+ appBundleAvailable: true,
+ runningFromLaunchServicesAppInstance: false
+ ))
+ }
+ }
+
+ func testMacOSAppAgentProxyDecisionDoesNotProxyLaunchServicesAppOpen() {
+ XCTAssertTrue(shouldUseMacOSAppAgentProxy(
+ command: .launchOnboarding,
+ proxyDisabled: false,
+ appBundleAvailable: true,
+ runningFromLaunchServicesAppInstance: false
+ ))
+ XCTAssertFalse(shouldUseMacOSAppAgentProxy(
+ command: .launchOnboarding,
+ proxyDisabled: false,
+ appBundleAvailable: true,
+ runningFromLaunchServicesAppInstance: true
+ ))
+ }
+
+ func testMacOSAppAgentProxyDecisionHonorsDisableAndMissingBundle() {
+ XCTAssertFalse(shouldUseMacOSAppAgentProxy(
+ command: .doctor,
+ proxyDisabled: true,
+ appBundleAvailable: true,
+ runningFromLaunchServicesAppInstance: false
+ ))
+ XCTAssertFalse(shouldUseMacOSAppAgentProxy(
+ command: .doctor,
+ proxyDisabled: false,
+ appBundleAvailable: false,
+ runningFromLaunchServicesAppInstance: false
+ ))
+ }
+
func testPermissionDiagnosticsListsMissingPermissionsInCanonicalOrder() {
let diagnostics = PermissionDiagnostics(
accessibilityTrusted: false,