👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
Route macOS automation through app agent
xuyusong.0505fd603ba57a3c
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swiftnew file mode 100644index 0000000..c86fdce--- /dev/null+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift@@ -0,0 +1,485 @@+import AppKit+import Darwin+import Foundation+import OpenComputerUseKit++private let appAgentCommand = "__open-computer-use-app-agent"+private let appAgentDisableEnvironmentKey = "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY"++enum MacOSAppAgentProxy {+ static func isAgentInvocation(arguments: [String]) -> Bool {+ arguments.first == appAgentCommand+ }++ @MainActor+ static func runAgent(arguments: [String]) throws {+ guard arguments.count == 2 else {+ throw OpenComputerUseCLIError(message: "\(appAgentCommand) requires a socket path")+ }++ try MacOSAppAgentRuntime.run(socketPath: arguments[1])+ }++ static func shouldProxy(command: OpenComputerUseCLICommand) -> Bool {+ shouldUseMacOSAppAgentProxy(+ command: command,+ proxyDisabled: proxyDisabled,+ appBundleAvailable: PermissionSupport.currentAppBundleURL() != nil,+ runningFromLaunchServicesAppInstance: isRunningFromLaunchServicesAppInstance+ )+ }++ @MainActor+ static func runProxy(command: OpenComputerUseCLICommand, arguments: [String]) throws -> Int32 {+ let socketPath = defaultSocketPath()+ let client = try connectOrLaunchAgent(socketPath: socketPath)++ switch command {+ case .mcp:+ try proxyMCP(client: client)+ return EXIT_SUCCESS+ default:+ let response = try sendCLIRequest(arguments: arguments, client: client)+ if !response.stdout.isEmpty {+ FileHandle.standardOutput.write(Data(response.stdout.utf8))+ }+ if !response.stderr.isEmpty {+ FileHandle.standardError.write(Data(response.stderr.utf8))+ }+ return response.exitCode+ }+ }++ private static var proxyDisabled: Bool {+ let value = ProcessInfo.processInfo.environment[appAgentDisableEnvironmentKey]?.lowercased()+ return value == "1" || value == "true" || value == "yes" || value == "on"+ }++ private static var isRunningFromOpenComputerUseAppBundle: Bool {+ Bundle.main.bundleURL.standardizedFileURL.pathExtension == "app"+ && PermissionSupport.isOpenComputerUseBundleIdentifier(Bundle.main.bundleIdentifier)+ }++ private static var isRunningFromLaunchServicesAppInstance: Bool {+ isRunningFromOpenComputerUseAppBundle && getppid() == 1+ }++ private static func defaultSocketPath() -> String {+ FileManager.default.temporaryDirectory+ .appendingPathComponent("open-computer-use-agent.sock")+ .standardizedFileURL+ .path+ }++ @MainActor+ private static func connectOrLaunchAgent(socketPath: String) throws -> AppAgentSocketClient {+ if let client = AppAgentSocketClient.connect(path: socketPath) {+ return client+ }++ unlink(socketPath)++ guard let appURL = PermissionSupport.currentAppBundleURL() else {+ throw OpenComputerUseCLIError(message: "Unable to locate Open Computer Use.app for app-scoped macOS permissions.")+ }++ let configuration = NSWorkspace.OpenConfiguration()+ configuration.arguments = [appAgentCommand, socketPath]+ configuration.activates = false+ configuration.createsNewApplicationInstance = true++ NSWorkspace.shared.openApplication(at: appURL, configuration: configuration) { _, _ in }++ let deadline = Date().addingTimeInterval(10)+ while Date() < deadline {+ if let client = AppAgentSocketClient.connect(path: socketPath) {+ return client+ }+ Thread.sleep(forTimeInterval: 0.05)+ }++ throw OpenComputerUseCLIError(message: "Timed out waiting for Open Computer Use.app agent to start.")+ }++ private static func proxyMCP(client: AppAgentSocketClient) throws {+ while let line = readLine(strippingNewline: true) {+ guard !line.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {+ continue+ }++ let response = try client.request([+ "kind": "mcp",+ "line": line,+ ])++ if let responseLine = response["response"] as? String {+ FileHandle.standardOutput.write(Data((responseLine + "\n").utf8))+ }+ }+ }++ private static func sendCLIRequest(arguments: [String], client: AppAgentSocketClient) throws -> CLIProxyResponse {+ let response = try client.request([+ "kind": "cli",+ "arguments": arguments,+ ])++ return CLIProxyResponse(+ stdout: response["stdout"] as? String ?? "",+ stderr: response["stderr"] as? String ?? "",+ exitCode: Int32(response["exitCode"] as? Int ?? 1)+ )+ }+}++private struct CLIProxyResponse {+ let stdout: String+ let stderr: String+ let exitCode: Int32+}++@MainActor+private final class MacOSAppAgentRuntime: NSObject, NSApplicationDelegate {+ private let socketPath: String+ private var listener: AppAgentSocketListener?+ private var turnEndedObserver: NSObjectProtocol?++ private init(socketPath: String) {+ self.socketPath = socketPath+ }++ static func run(socketPath: String) throws {+ let application = NSApplication.shared+ application.setActivationPolicy(.accessory)++ let delegate = MacOSAppAgentRuntime(socketPath: socketPath)+ application.delegate = delegate+ application.run()+ }++ func applicationDidFinishLaunching(_ notification: Notification) {+ turnEndedObserver = DistributedNotificationCenter.default().addObserver(+ forName: openComputerUseTurnEndedNotificationName,+ object: nil,+ queue: .main+ ) { _ in+ Task { @MainActor in+ resetOpenComputerUseVisualCursor()+ }+ }++ do {+ let listener = try AppAgentSocketListener(path: socketPath)+ self.listener = listener+ listener.start()+ } catch {+ writeAgentError(error)+ NSApp.terminate(nil)+ }+ }++ func applicationWillTerminate(_ notification: Notification) {+ if let turnEndedObserver {+ DistributedNotificationCenter.default().removeObserver(turnEndedObserver)+ }+ listener?.stop()+ }++ func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool {+ false+ }++ private func writeAgentError(_ error: Error) {+ let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)+ FileHandle.standardError.write(Data((message + "\n").utf8))+ }+}++private final class AppAgentSocketListener: @unchecked Sendable {+ private let path: String+ private let socketFD: Int32+ private var running = true++ init(path: String) throws {+ self.path = path+ unlink(path)++ socketFD = socket(AF_UNIX, SOCK_STREAM, 0)+ guard socketFD >= 0 else {+ throw POSIXError(.init(rawValue: errno) ?? .EIO)+ }++ var address = sockaddr_un()+ address.sun_family = sa_family_t(AF_UNIX)+ let pathCapacity = MemoryLayout.size(ofValue: address.sun_path)+ try withUnsafeMutablePointer(to: &address.sun_path) { pointer in+ try pointer.withMemoryRebound(to: CChar.self, capacity: pathCapacity) { buffer in+ let bytes = Array(path.utf8)+ guard bytes.count < pathCapacity else {+ throw OpenComputerUseCLIError(message: "Socket path is too long: \(path)")+ }+ for index in 0..<bytes.count {+ buffer[index] = CChar(bitPattern: bytes[index])+ }+ buffer[bytes.count] = 0+ }+ }++ let bindResult = withUnsafePointer(to: &address) { pointer in+ pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) {+ bind(socketFD, $0, socklen_t(MemoryLayout<sockaddr_un>.size))+ }+ }+ guard bindResult == 0 else {+ close(socketFD)+ throw POSIXError(.init(rawValue: errno) ?? .EIO)+ }++ guard listen(socketFD, 16) == 0 else {+ close(socketFD)+ throw POSIXError(.init(rawValue: errno) ?? .EIO)+ }++ guard chmod(path, mode_t(S_IRUSR | S_IWUSR)) == 0 else {+ close(socketFD)+ unlink(path)+ throw POSIXError(.init(rawValue: errno) ?? .EIO)+ }+ }++ func start() {+ Thread.detachNewThread {+ self.acceptLoop()+ }+ }++ func stop() {+ running = false+ close(socketFD)+ unlink(path)+ }++ private func acceptLoop() {+ while running {+ let clientFD = accept(socketFD, nil, nil)+ guard clientFD >= 0 else {+ if running {+ Thread.sleep(forTimeInterval: 0.05)+ }+ continue+ }++ Thread.detachNewThread {+ AppAgentConnection(fileDescriptor: clientFD).run()+ }+ }+ }+}++private final class AppAgentConnection: @unchecked Sendable {+ private let fileDescriptor: Int32+ private let server = StdioMCPServer()++ init(fileDescriptor: Int32) {+ self.fileDescriptor = fileDescriptor+ }++ func run() {+ guard let file = fdopen(fileDescriptor, "r+") else {+ close(fileDescriptor)+ return+ }+ defer { fclose(file) }++ while let line = readAgentLine(file) {+ let response = handle(requestLine: line)+ writeAgentLine(response, to: file)+ }+ }++ private func handle(requestLine: String) -> [String: Any] {+ do {+ guard let request = try JSONSerialization.jsonObject(with: Data(requestLine.utf8)) as? [String: Any],+ let kind = request["kind"] as? String+ else {+ return ["error": "Invalid app-agent request"]+ }++ switch kind {+ case "mcp":+ let line = request["line"] as? String ?? ""+ if let response = server.handle(line: line) {+ return ["response": response]+ }+ return ["response": NSNull()]+ case "cli":+ let arguments = request["arguments"] as? [String] ?? []+ let response = runCLI(arguments: arguments)+ return [+ "stdout": response.stdout,+ "stderr": response.stderr,+ "exitCode": Int(response.exitCode),+ ]+ default:+ return ["error": "Unknown app-agent request kind: \(kind)"]+ }+ } catch {+ let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)+ return ["error": message]+ }+ }++ private func runCLI(arguments: [String]) -> CLIProxyResponse {+ do {+ let command = try parseOpenComputerUseCLI(arguments: arguments)++ switch command {+ case .launchOnboarding:+ let permissions = PermissionDiagnostics.current()+ if !permissions.allGranted {+ Task { @MainActor in+ PermissionOnboardingApp.present()+ }+ }+ return CLIProxyResponse(stdout: "", stderr: "", exitCode: EXIT_SUCCESS)++ case .doctor:+ let permissions = PermissionDiagnostics.current()+ if !permissions.missingPermissions.isEmpty {+ Task { @MainActor in+ PermissionOnboardingApp.present()+ }+ }+ return CLIProxyResponse(stdout: permissions.summary + "\n", stderr: "", exitCode: EXIT_SUCCESS)++ case .listApps:+ let service = ComputerUseService()+ return CLIProxyResponse(stdout: (service.listApps().primaryText ?? "") + "\n", stderr: "", exitCode: EXIT_SUCCESS)++ case let .snapshot(app):+ let service = ComputerUseService()+ let text = try service.getAppState(app: app).primaryText ?? ""+ return CLIProxyResponse(stdout: text + "\n", stderr: "", exitCode: EXIT_SUCCESS)++ case let .call(invocation):+ let output = try runOpenComputerUseCall(invocation)+ return CLIProxyResponse(+ stdout: try output.jsonText() + "\n",+ stderr: "",+ exitCode: output.hasToolError ? EXIT_FAILURE : EXIT_SUCCESS+ )++ default:+ return CLIProxyResponse(stdout: "", stderr: "Unsupported proxied command.\n", exitCode: EXIT_FAILURE)+ }+ } catch {+ let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)+ return CLIProxyResponse(stdout: "", stderr: message + "\n", exitCode: EXIT_FAILURE)+ }+ }+}++private final class AppAgentSocketClient: @unchecked Sendable {+ private let file: UnsafeMutablePointer<FILE>++ private init(file: UnsafeMutablePointer<FILE>) {+ self.file = file+ }++ deinit {+ fclose(file)+ }++ static func connect(path: String) -> AppAgentSocketClient? {+ let fd = socket(AF_UNIX, SOCK_STREAM, 0)+ guard fd >= 0 else {+ return nil+ }++ var address = sockaddr_un()+ address.sun_family = sa_family_t(AF_UNIX)+ let pathCapacity = MemoryLayout.size(ofValue: address.sun_path)+ let copied = withUnsafeMutablePointer(to: &address.sun_path) { pointer in+ pointer.withMemoryRebound(to: CChar.self, capacity: pathCapacity) { buffer -> Bool in+ let bytes = Array(path.utf8)+ guard bytes.count < pathCapacity else {+ return false+ }+ for index in 0..<bytes.count {+ buffer[index] = CChar(bitPattern: bytes[index])+ }+ buffer[bytes.count] = 0+ return true+ }+ }++ guard copied else {+ close(fd)+ return nil+ }++ let result = withUnsafePointer(to: &address) { pointer in+ pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) {+ Darwin.connect(fd, $0, socklen_t(MemoryLayout<sockaddr_un>.size))+ }+ }+ guard result == 0, let file = fdopen(fd, "r+") else {+ close(fd)+ return nil+ }++ return AppAgentSocketClient(file: file)+ }++ func request(_ object: [String: Any]) throws -> [String: Any] {+ let data = try JSONSerialization.data(withJSONObject: object, options: [.withoutEscapingSlashes])+ guard let line = String(data: data, encoding: .utf8) else {+ throw ComputerUseError.message("Failed to encode app-agent request.")+ }++ writeAgentLine(line, to: file)++ guard let responseLine = readAgentLine(file),+ let response = try JSONSerialization.jsonObject(with: Data(responseLine.utf8)) as? [String: Any]+ else {+ throw ComputerUseError.message("Open Computer Use.app agent closed the connection.")+ }++ if let error = response["error"] as? String {+ throw ComputerUseError.message(error)+ }++ return response+ }+}++private func readAgentLine(_ file: UnsafeMutablePointer<FILE>) -> String? {+ var bytes: [UInt8] = []++ while true {+ let character = fgetc(file)+ if character == EOF {+ return bytes.isEmpty ? nil : String(data: Data(bytes), encoding: .utf8)+ }+ if character == 10 {+ return String(data: Data(bytes), encoding: .utf8)+ }+ bytes.append(UInt8(character))+ }+}++private func writeAgentLine(_ object: [String: Any], to file: UnsafeMutablePointer<FILE>) {+ if let data = try? JSONSerialization.data(withJSONObject: object, options: [.withoutEscapingSlashes]),+ let line = String(data: data, encoding: .utf8)+ {+ writeAgentLine(line, to: file)+ }+}++private func writeAgentLine(_ line: String, to file: UnsafeMutablePointer<FILE>) {+ let output = line + "\n"+ _ = output.withCString { pointer in+ fputs(pointer, file)+ }+ fflush(file)+}diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swiftindex 7e764df..405633c 100644--- a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift@@ -25,8 +25,18 @@ enum OpenComputerUseMain {@MainActorprivate static func run() throws {let arguments = Array(CommandLine.arguments.dropFirst())++ if MacOSAppAgentProxy.isAgentInvocation(arguments: arguments) {+ try MacOSAppAgentProxy.runAgent(arguments: arguments)+ return+ }+let command = try parseOpenComputerUseCLI(arguments: arguments)+ if MacOSAppAgentProxy.shouldProxy(command: command) {+ exit(try MacOSAppAgentProxy.runProxy(command: command, arguments: arguments))+ }+switch command {case .mcp:let service = ComputerUseService()diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swiftindex c01960c..f699592 100644--- a/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift@@ -5,6 +5,8 @@ import QuartzCore@MainActorenum PermissionOnboardingApp {+ private static var presentedWindowController: PermissionWindowController?+static func launch() {guard !PermissionDiagnostics.current().allGranted else {return@@ -18,6 +20,21 @@ enum PermissionOnboardingApp {application.delegate = delegateapplication.run()}++ static func present(terminateOnCompletion: Bool = true) {+ guard !PermissionDiagnostics.current().allGranted else {+ return+ }++ let application = NSApplication.shared+ application.setActivationPolicy(.accessory)+ application.applicationIconImage = Branding.makeAppIconImage(size: 256)++ let controller = PermissionWindowController(terminateOnCompletion: terminateOnCompletion)+ presentedWindowController = controller+ controller.showWindow(nil)+ NSApp.activate(ignoringOtherApps: true)+ }}@MainActor@@ -42,8 +59,11 @@ final class PermissionWindowController: NSWindowController {private lazy var accessoryPanelController = PermissionAccessoryPanelController { [weak self] inself?.handleAccessoryPanelBack()}+ private let terminateOnCompletion: Bool++ init(terminateOnCompletion: Bool = true) {+ self.terminateOnCompletion = terminateOnCompletion- init() {let window = NSWindow(contentRect: NSRect(x: 0,@@ -97,7 +117,9 @@ extension PermissionWindowController: PermissionContentControllerDelegate {func permissionContentControllerDidCompleteAllPermissions(_ controller: PermissionContentController) {accessoryPanelController.hide()close()- NSApp.terminate(nil)+ if terminateOnCompletion {+ NSApp.terminate(nil)+ }}private func handleAccessoryPanelBack() {diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 6867870..24cda41 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -37,13 +37,14 @@- `OpenComputerUse` 默认 app 模式会拉起 `PermissionOnboardingApp`。- app bundle 以 `LSUIElement` agent-style 形态运行,默认不在 Dock 暴露常驻图标,但仍可按需显示权限窗口。+- 当用户从终端执行 macOS 版 `open-computer-use mcp`、`doctor`、`call`、`snapshot` 或 `list-apps` 时,CLI 会先通过 LaunchServices 启动同一个 `.app` bundle 的隐藏 app agent,并通过用户临时目录下的 Unix domain socket 转发请求;真正调用 Accessibility、ScreenCaptureKit 和动作 tools 的进程始终是 `Open Computer Use.app`,不是 iTerm / Terminal / Node launcher。- 主窗口负责渲染 `Accessibility` / `Screen & System Audio Recording` 两类权限卡片、`Allow` / `Done` 状态和 relaunch 后的状态收敛;当两项权限都已完成时会自动关闭,不再要求用户手动退出。- 辅助 drag panel 会跳转到对应的 `System Settings` 页面;点击 `Allow` 后,panel 会从主窗口里的按钮位置做一段 spring + curved frame 的入场,再落到 `System Settings` 内容区下沿。panel 默认保持在窗口右侧内容区下方居中并固定贴近窗口底边,不再依赖实时扫描权限页内部 `+ / -` 控件行;窗口层级上会显式排在当前 `System Settings` 窗口之上,避免被权限列表内容盖住,同时尽量减少对系统设置自身滚动区域的干扰。panel 内也补了显式返回按钮,允许用户中断当前 guidance、回到 onboarding 主窗口重新选择权限步骤。- 权限状态优先基于 TCC 持久授权记录判断,避免 CLI 子进程与 GUI app 对授权状态看到不一致的结果;正式 release 仍以 CI 打出来的 `Open Computer Use.app` 为准,而本地 debug/dev 打包现在显式命名为 `Open Computer Use (Dev).app`,并在 dev bundle 运行时优先认当前 dev 副本,避免系统设置里出现两个完全同名的条目。### 2. MCP 层-- 当前只实现 `stdio` transport。+- 面向 MCP host 的外部 transport 仍是 `stdio`;macOS 终端 CLI 到 `.app` app agent 之间额外有一层本地 Unix domain socket 代理,用来保证真实 automation 运行在 app bundle 权限身份下。- 当 `OPEN_COMPUTER_USE_VISUAL_CURSOR` 未被显式关闭时,`mcp` 命令会切到一个最小 AppKit runtime:主线程保留 event loop 承载 overlay UI,stdio server 仍在后台线程串行读取与响应。- 请求 framing 采用一行一个 JSON-RPC message。- 当前支持的 method:@@ -142,7 +143,7 @@- Linux binary 构建:`./scripts/build-open-computer-use-linux.sh --arch arm64`- 对比样本:`artifacts/tool-comparisons/20260417-focus-behavior/`- 手工诊断:- - `.build/debug/OpenComputerUse doctor`- - `.build/debug/OpenComputerUse snapshot <app>`- - `.build/debug/OpenComputerUse call list_apps`- - `.build/debug/OpenComputerUse call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}}]'`+ - `open-computer-use doctor`+ - `open-computer-use snapshot <app>`+ - `open-computer-use call list_apps`+ - `open-computer-use call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}}]'`diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.mdindex bc49c0e..1c85291 100644--- a/docs/RELIABILITY.md+++ b/docs/RELIABILITY.md@@ -7,12 +7,12 @@- 端到端 smoke:`./scripts/run-tool-smoke-tests.sh`- Linux runtime:`(cd apps/OpenComputerUseLinux && go test ./...)`、`./scripts/build-open-computer-use-linux.sh --arch arm64`- 本地诊断:- - `.build/debug/OpenComputerUse doctor`- - `.build/debug/OpenComputerUse snapshot <app>`+ - `open-computer-use doctor`+ - `open-computer-use snapshot <app>`## 已知关键依赖-- 宿主必须有 `Accessibility` 与 `Screen Recording` 权限。+- macOS 上必须给 `Open Computer Use.app` 授权 `Accessibility` 与 `Screen Recording`;终端本身不应该再是必需授权对象。- smoke suite 依赖本地 GUI session,不能把它当成无头环境命令。- 普通 app 的 `get_app_state` 结果依赖 AX tree 和窗口截图,复杂 app 上输出会有差异。- Linux runtime 依赖已登录桌面用户 session;缺少 `XDG_RUNTIME_DIR`、`DBUS_SESSION_BUS_ADDRESS` 或 display 环境时,会尝试从 `/run/user/<uid>` 和常见桌面进程自动发现当前用户的 session env。纯 SSH tty 如果找不到桌面 session 仍不能直接访问 AT-SPI GUI tree。@@ -20,9 +20,9 @@## 当前故障排查顺序-1. 先跑 `.build/debug/OpenComputerUse doctor`,确认权限状态;如果缺权限,命令会直接拉起权限 onboarding 窗口,已全部授权则只打印状态并退出。-2. 用 `.build/debug/OpenComputerUse list-apps` 确认目标 app 是否被发现。-3. 用 `.build/debug/OpenComputerUse snapshot <app>` 看是 transport 问题还是 snapshot / action 问题。+1. 先跑 `open-computer-use doctor`,确认权限状态;如果缺权限,命令会通过 `.app` app agent 拉起权限 onboarding 窗口,已全部授权则只打印状态并退出。+2. 用 `open-computer-use list-apps` 确认目标 app 是否被发现。+3. 用 `open-computer-use snapshot <app>` 看是 transport 问题还是 snapshot / action 问题。4. 如果只想验证仓库基线,直接跑 fixture + smoke,不要先在复杂第三方 app 上排查。5. 排查 Linux runtime 时,先确认目标命令是否由桌面用户运行,再用 `open-computer-use call list_apps` 和 `open-computer-use snapshot <app>` 区分 session/env 问题与 AT-SPI tree/action 问题。如果是 Codex MCP,重新执行 `open-computer-use install-codex-mcp` 后重启 Codex,确认配置仍是 `open-computer-use mcp`。diff --git a/docs/SECURITY.md b/docs/SECURITY.mdindex 744e587..eeaad4e 100644--- a/docs/SECURITY.md+++ b/docs/SECURITY.md@@ -2,9 +2,9 @@## 当前实现边界-- 当前 server 只提供本地 `stdio` MCP,不对外监听 TCP/HTTP 端口。+- 对 MCP host 暴露的接口仍是本地 `stdio`;macOS CLI 与 `.app` app agent 之间会使用用户临时目录下的 Unix domain socket,socket 创建后会收紧为当前用户读写,且不对外监听 TCP/HTTP 端口。- 所有动作都必须显式带 `app` 参数;当前不会在后台自动扫描并控制任意 app。-- 真实 app 路径依赖宿主进程已经获得 macOS `Accessibility` 与 `Screen Recording` 权限。+- macOS 真实 app 路径依赖 `Open Computer Use.app` 已获得 `Accessibility` 与 `Screen Recording` 权限;终端里的 CLI / Node launcher 会把 `mcp`、`doctor`、`call`、`snapshot` 和 `list-apps` 转发给由 LaunchServices 启动的本地 app agent,避免把权限要求落到 iTerm / Terminal 身上。- 实验性 Linux runtime 依赖已登录桌面用户的 AT-SPI2 / D-Bus session;coordinate mouse、drag、keyboard synthesis 只是 best-effort fallback,不应被视为跨 Wayland compositor 的通用后台输入授权。## 数据处理@@ -23,7 +23,7 @@- 这意味着开源版当前的安全边界主要由:- 明确的 tool 调用参数- 内置高风险 denylist- - 宿主进程权限+ - `Open Computer Use.app` 的系统权限- 本地使用场景共同提供。- 下一阶段应优先补:diff --git a/docs/histories/2026-05/20260503-1731-app-scoped-permission-agent.md b/docs/histories/2026-05/20260503-1731-app-scoped-permission-agent.mdnew file mode 100644index 0000000..1d9c70a--- /dev/null+++ b/docs/histories/2026-05/20260503-1731-app-scoped-permission-agent.md@@ -0,0 +1,53 @@+## [2026-05-03 17:31] | Task: 修复 macOS 终端权限归属++### Execution Context+* **Agent ID**: `Codex`+* **Base Model**: `GPT-5`+* **Runtime**: `local macOS workspace`++### User Query+> `open-computer-use doctor` 能打开权限页面,但从终端执行时权限似乎要求的是 iTerm/Terminal,而不是 `Open Computer Use.app`;需要保证 iTerm 没有 Accessibility / Screen Recording 权限时,只给 Open Computer Use 授权也能工作。++### Changes Overview+**Scope:** `apps/OpenComputerUse`, `packages/OpenComputerUseKit`, `docs`++**Key Actions:**+- **[App agent proxy]**: 新增隐藏 app-agent 启动模式;终端 CLI 对 `mcp`、`doctor`、`call`、`snapshot` 和 `list-apps` 通过 Unix domain socket 转发到 LaunchServices 启动的 `.app` 进程。+- **[Onboarding reuse]**: 将权限 onboarding 拆出可复用的 `present()` 路径,让 doctor 可以在 app agent 已运行的 NSApplication 内显示授权窗口,而不是在终端子进程内直接跑 UI。+- **[Decision tests]**: 将 app-agent 代理选择规则下沉为 Kit 内纯函数,并补单测覆盖 automation 命令代理、非 automation 命令本地执行、LaunchServices 打开 app 不递归代理、禁用开关和缺少 bundle fallback。+- **[Launch mode guard]**: 对无参数启动区分 LaunchServices 打开的 `.app` 和终端直接执行的 bundle executable,避免双击 app 时留下多余后台 agent,同时让终端入口仍走 app 身份代理。+- **[Socket permissions]**: app-agent Unix socket 创建后收紧为当前用户读写,维持本地-only 权限边界。+- **[Permission relaunch]**: app-agent 内完成授权后仍会终止当前 app 进程,确保下一次命令用重新启动后的授权进程执行 ScreenCaptureKit / AX 路径。+- **[Permission wording]**: 更新权限错误和文档口径,明确 macOS 授权目标是 `Open Computer Use.app`,不是宿主终端。+- **[Swift 6.2 build fix]**: 适配 Swift 6.2 并发检查,将 cursor reference `NSImage` 静态缓存显式标记为 `nonisolated(unsafe)`,保持当前 AppKit 主线程绘制路径可编译。++### Design Intent (Why)+macOS TCC 对 Accessibility 和 Screen Recording 的责任归属取决于真正调用系统 API 的进程。终端直接启动 native runtime 时,系统可能要求 iTerm/Terminal 获得权限;把真实 automation 放到通过 LaunchServices 启动的 app bundle 进程内,可以让用户只授权 `Open Computer Use.app`,终端负责 stdio/命令代理。++### Files Modified+- `apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift`+- `apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift`+- `apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift`+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift`+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorGlyphRenderer.swift`+- `docs/ARCHITECTURE.md`+- `docs/SECURITY.md`+- `docs/RELIABILITY.md`++### Verification+- Passed: `git diff --check`+- Passed: `./scripts/check-docs.sh`+- Passed: `xcrun swiftc -parse apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift`+- Passed: `xcrun swiftc -parse packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift`+- Passed: `xcrun swiftc -parse packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`+- Passed: `PATH="$HOME/.swiftly/bin:$PATH" swift build --product OpenComputerUse` with Swift `6.2.4`.+- Passed: `PATH="$HOME/.swiftly/bin:$PATH" ./scripts/build-open-computer-use-app.sh debug --arch native`.+- Passed: `.build/arm64-apple-macosx/debug/OpenComputerUse doctor` returned missing permission status while launching `dist/Open Computer Use (Dev).app/Contents/MacOS/OpenComputerUse __open-computer-use-app-agent ...`, confirming the terminal command hands off to the app bundle process.+- Passed: the app-agent socket was created as `srw-------`, current-user read/write only.+- Passed: after granting the dev app both macOS permissions, `OpenComputerUse doctor` reported `accessibility=granted, screenRecording=granted`.+- Passed: `OpenComputerUse call get_app_state --args '{"app":"Finder"}'` returned `isError=false` with both `text` and `image` content while iTerm did not hold the permissions.+- Passed: Codex MCP integration via `codex exec` with the bundled official `computer-use` plugin disabled called `server="open-computer-use"` for `list_apps` and `get_app_state`.+- Passed: Codex-driven action smoke covered `click`, `scroll`, `set_value`, `press_key`, `drag`, and `type_text`; Finder, Calendar, and the local fixture succeeded, while TextEdit `click` exposed an existing window-bounds edge case but TextEdit typing still succeeded.+- Blocked locally: `PATH="$HOME/.swiftly/bin:$PATH" swift test` cannot import `XCTest` from the Swift.org toolchain in this CLT-only environment.diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swiftindex e591f3c..68591c9 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift@@ -79,7 +79,7 @@ enum SnapshotBuilder {let permissions = PermissionDiagnostics.current()guard permissions.accessibilityTrusted else {- throw ComputerUseError.permissionDenied("Accessibility permission is required. Run `OpenComputerUse doctor` and grant access to the host terminal or app.")+ throw ComputerUseError.permissionDenied("Accessibility permission is required. Run `open-computer-use doctor` and grant access to Open Computer Use.")}let appElement = AXUIElementCreateApplication(app.pid)diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swiftindex 9db25f8..eb1307c 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift@@ -19,6 +19,26 @@ public enum OpenComputerUseCallInvocation: Equatable {public let openComputerUseDefaultInterCallDelay: TimeInterval = 1+public func shouldUseMacOSAppAgentProxy(+ command: OpenComputerUseCLICommand,+ proxyDisabled: Bool,+ appBundleAvailable: Bool,+ runningFromLaunchServicesAppInstance: Bool+) -> Bool {+ guard !proxyDisabled, appBundleAvailable else {+ return false+ }++ switch command {+ case .launchOnboarding:+ return !runningFromLaunchServicesAppInstance+ case .mcp, .doctor, .listApps, .snapshot, .call:+ return true+ case .turnEnded, .help, .version:+ return false+ }+}+public struct OpenComputerUseCLIError: LocalizedError, Equatable {public let message: Stringpublic let helpCommand: String?diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorGlyphRenderer.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorGlyphRenderer.swiftindex c616cb2..5df11d1 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorGlyphRenderer.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorGlyphRenderer.swift@@ -56,7 +56,7 @@ private enum SoftwareCursorGlyphColors {}enum SoftwareCursorGlyphRenderer {- private static let referenceImage = loadReferenceCursorWindowImage()+ nonisolated(unsafe) private static let referenceImage = loadReferenceCursorWindowImage()static func draw(in bounds: CGRect,diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swiftindex 0edeb71..a08429e 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift@@ -204,6 +204,68 @@ final class OpenComputerUseKitTests: XCTestCase {XCTAssertFalse(output.hasToolError)}+ func testMacOSAppAgentProxyDecisionRoutesAutomationCommandsThroughAppBundle() {+ for command in [+ OpenComputerUseCLICommand.mcp,+ .doctor,+ .listApps,+ .snapshot(app: "TextEdit"),+ .call(.single(toolName: "list_apps", argumentsJSON: nil, argumentsFile: nil)),+ ] {+ XCTAssertTrue(shouldUseMacOSAppAgentProxy(+ command: command,+ proxyDisabled: false,+ appBundleAvailable: true,+ runningFromLaunchServicesAppInstance: false+ ))+ }+ }++ func testMacOSAppAgentProxyDecisionKeepsNonAutomationCommandsLocal() {+ for command in [+ OpenComputerUseCLICommand.turnEnded(payload: nil),+ .help(command: nil),+ .version,+ ] {+ XCTAssertFalse(shouldUseMacOSAppAgentProxy(+ command: command,+ proxyDisabled: false,+ appBundleAvailable: true,+ runningFromLaunchServicesAppInstance: false+ ))+ }+ }++ func testMacOSAppAgentProxyDecisionDoesNotProxyLaunchServicesAppOpen() {+ XCTAssertTrue(shouldUseMacOSAppAgentProxy(+ command: .launchOnboarding,+ proxyDisabled: false,+ appBundleAvailable: true,+ runningFromLaunchServicesAppInstance: false+ ))+ XCTAssertFalse(shouldUseMacOSAppAgentProxy(+ command: .launchOnboarding,+ proxyDisabled: false,+ appBundleAvailable: true,+ runningFromLaunchServicesAppInstance: true+ ))+ }++ func testMacOSAppAgentProxyDecisionHonorsDisableAndMissingBundle() {+ XCTAssertFalse(shouldUseMacOSAppAgentProxy(+ command: .doctor,+ proxyDisabled: true,+ appBundleAvailable: true,+ runningFromLaunchServicesAppInstance: false+ ))+ XCTAssertFalse(shouldUseMacOSAppAgentProxy(+ command: .doctor,+ proxyDisabled: false,+ appBundleAvailable: false,+ runningFromLaunchServicesAppInstance: false+ ))+ }+func testPermissionDiagnosticsListsMissingPermissionsInCanonicalOrder() {let diagnostics = PermissionDiagnostics(accessibilityTrusted: false,