👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
feat: add swift open codex computer use
Leo134aab7d98db
diff --git a/.gitignore b/.gitignoreindex f32e31a..fe130f1 100644--- a/.gitignore+++ b/.gitignore@@ -1,2 +1,5 @@.idea/.DS_Store+.build/+.swiftpm/+dist/diff --git a/Makefile b/Makefileindex a179603..1d86771 100644--- a/Makefile+++ b/Makefile@@ -1,12 +1,24 @@PROJECT ?=SLUG ?=-.PHONY: init check-docs check-repo ci release-package new-history new-plan+.PHONY: init build app test smoke check-docs check-repo ci release-package new-history new-planinit:@if [ -z "$(PROJECT)" ]; then echo "用法: make init PROJECT=项目名"; exit 1; fi./scripts/init-project.sh "$(PROJECT)"+build:+ swift build++app:+ ./scripts/build-open-codex-app.sh debug++test:+ swift test++smoke:+ ./scripts/run-tool-smoke-tests.sh+check-docs:./scripts/check-docs.shdiff --git a/Package.swift b/Package.swiftnew file mode 100644index 0000000..72682fc--- /dev/null+++ b/Package.swift@@ -0,0 +1,54 @@+// swift-tools-version: 6.2++import PackageDescription++let package = Package(+ name: "OpenCodexComputerUse",+ platforms: [+ .macOS(.v14),+ ],+ products: [+ .library(+ name: "OpenCodexComputerUseKit",+ targets: ["OpenCodexComputerUseKit"]+ ),+ .executable(+ name: "OpenCodexComputerUse",+ targets: ["OpenCodexComputerUse"]+ ),+ .executable(+ name: "OpenCodexComputerUseFixture",+ targets: ["OpenCodexComputerUseFixture"]+ ),+ .executable(+ name: "OpenCodexComputerUseSmokeSuite",+ targets: ["OpenCodexComputerUseSmokeSuite"]+ ),+ ],+ targets: [+ .target(+ name: "OpenCodexComputerUseKit",+ path: "packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit"+ ),+ .executableTarget(+ name: "OpenCodexComputerUse",+ dependencies: ["OpenCodexComputerUseKit"],+ path: "apps/OpenCodexComputerUse/Sources/OpenCodexComputerUse"+ ),+ .executableTarget(+ name: "OpenCodexComputerUseFixture",+ dependencies: ["OpenCodexComputerUseKit"],+ path: "apps/OpenCodexComputerUseFixture/Sources/OpenCodexComputerUseFixture"+ ),+ .executableTarget(+ name: "OpenCodexComputerUseSmokeSuite",+ dependencies: ["OpenCodexComputerUseKit"],+ path: "apps/OpenCodexComputerUseSmokeSuite/Sources/OpenCodexComputerUseSmokeSuite"+ ),+ .testTarget(+ name: "OpenCodexComputerUseKitTests",+ dependencies: ["OpenCodexComputerUseKit"],+ path: "packages/OpenCodexComputerUseKit/Tests/OpenCodexComputerUseKitTests"+ ),+ ]+)diff --git a/README.md b/README.mdindex be14b13..b848df2 100644--- a/README.md+++ b/README.md@@ -2,17 +2,88 @@## 简介-一个面向 Agent 协作开发的基础仓库模板,可以用来启动任何你想做的产品或服务。+一个用 Swift 实现的开源 macOS `computer-use` MCP server。++当前版本聚焦两件事:++- 通过 `stdio` MCP 暴露 9 个和官方 `computer-use` 同名的 tools。+- 在仓库内自带一个 GUI fixture app、smoke suite 和 app 模式权限引导,保证这 9 个 tools 有稳定可回归的本地验证路径。++当前实现的 9 个 tools:++- `list_apps`+- `get_app_state`+- `click`+- `perform_secondary_action`+- `scroll`+- `drag`+- `type_text`+- `press_key`+- `set_value`## 快速开始-直接 clone 或 fork 这个仓库,然后把它作为新项目的起点,继续往里搭你真正要做的产品、服务或平台能力即可。+环境要求:-## 许可证+- macOS 14++- Xcode Command Line Tools / Swift 6.2++- 已授予宿主终端或 app 的 `Accessibility` 与 `Screen Recording` 权限-[MIT](./LICENSE)+构建与诊断:++```bash+swift build+.build/debug/OpenCodexComputerUse doctor+.build/debug/OpenCodexComputerUse list-apps+```++打包 app 并打开权限引导窗口:++```bash+./scripts/build-open-codex-app.sh debug+open dist/OpenCodexComputerUse.app+```++启动 MCP server:++```bash+.build/debug/OpenCodexComputerUse mcp+```-## 备注+本地验证:-这套方法主要来自我们自己的持续实践和整理,同时也吸收了 OpenAI 在 harness engineering 文章中的一部分思路,最后汇总成了这个模板:-https://openai.com/index/harness-engineering/+```bash+swift test+./scripts/run-tool-smoke-tests.sh+```++如果你想单独看某个 app 当前会被如何序列化,可以直接跑:++```bash+.build/debug/OpenCodexComputerUse snapshot Finder+```++如果直接运行 `OpenCodexComputerUse` 而不带子命令,默认会进入 app 模式并显示权限 onboarding 窗口。++## 工程结构++- `apps/OpenCodexComputerUse`+ `stdio` MCP server、本地诊断入口和默认 app 模式权限引导。+- `packages/OpenCodexComputerUseKit`+ MCP transport、tool registry、app discovery、snapshot、输入模拟和 fixture bridge。+- `apps/OpenCodexComputerUseFixture`+ 本地 GUI 夹具,用于安全验证点击、输入、滚动和拖拽等行为。+- `apps/OpenCodexComputerUseSmokeSuite`+ 端到端 smoke runner,会真实拉起 fixture 和 MCP server,对 9 个 tools 做回归。+- `scripts/build-open-codex-app.sh`+ 生成最小可运行的 `.app` bundle,便于真实授权与本地 UI 验证。++## 当前取舍++- 普通 app 路径优先走 macOS Accessibility、窗口截图和 CGEvent 输入事件。+- fixture app 为了提供稳定回归,会额外导出一份合成状态,并接受测试专用 command bridge。+- 当前不复刻官方闭源 app 的签名边界、私有 IPC、overlay UI 和插件自安装逻辑。++## 许可证++[MIT](./LICENSE)diff --git a/apps/OpenCodexComputerUse/Sources/OpenCodexComputerUse/OpenCodexComputerUseMain.swift b/apps/OpenCodexComputerUse/Sources/OpenCodexComputerUse/OpenCodexComputerUseMain.swiftnew file mode 100644index 0000000..445e82a--- /dev/null+++ b/apps/OpenCodexComputerUse/Sources/OpenCodexComputerUse/OpenCodexComputerUseMain.swift@@ -0,0 +1,46 @@+import AppKit+import Foundation+import OpenCodexComputerUseKit++@main+enum OpenCodexComputerUseMain {+ static func main() throws {+ let arguments = Array(CommandLine.arguments.dropFirst())+ let service = ComputerUseService()++ switch arguments.first {+ case "mcp":+ try StdioMCPServer(service: service).run()+ case "doctor":+ let permissions = PermissionDiagnostics.current()+ print(permissions.summary)+ case "list-apps":+ print(service.listApps())+ case "snapshot":+ guard arguments.count >= 2 else {+ throw ComputerUseError.invalidArguments("snapshot requires an app name or bundle identifier")+ }+ print(try service.getAppState(app: arguments[1]))+ case "turn-ended":+ print("turn-ended acknowledged")+ default:+ if arguments.first == "help" || arguments.first == "--help" || arguments.first == "-h" {+ print(+ """+ OpenCodexComputerUse++ Usage:+ OpenCodexComputerUse+ OpenCodexComputerUse mcp+ OpenCodexComputerUse doctor+ OpenCodexComputerUse list-apps+ OpenCodexComputerUse snapshot <app>+ OpenCodexComputerUse turn-ended+ """+ )+ } else {+ PermissionOnboardingApp.launch()+ }+ }+ }+}diff --git a/apps/OpenCodexComputerUse/Sources/OpenCodexComputerUse/PermissionOnboardingApp.swift b/apps/OpenCodexComputerUse/Sources/OpenCodexComputerUse/PermissionOnboardingApp.swiftnew file mode 100644index 0000000..cea5b9e--- /dev/null+++ b/apps/OpenCodexComputerUse/Sources/OpenCodexComputerUse/PermissionOnboardingApp.swift@@ -0,0 +1,1084 @@+import ApplicationServices+import AppKit+import OpenCodexComputerUseKit++@MainActor+enum PermissionOnboardingApp {+ static func launch() {+ let application = NSApplication.shared+ application.setActivationPolicy(.regular)+ application.applicationIconImage = Branding.makeAppIconImage(size: 256)++ let delegate = PermissionOnboardingAppDelegate()+ application.delegate = delegate+ application.run()+ }+}++@MainActor+final class PermissionOnboardingAppDelegate: NSObject, NSApplicationDelegate {+ private var windowController: PermissionWindowController?++ func applicationDidFinishLaunching(_ notification: Notification) {+ let controller = PermissionWindowController()+ windowController = controller+ controller.showWindow(nil)+ NSApp.activate(ignoringOtherApps: true)+ }++ func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool {+ true+ }+}++@MainActor+final class PermissionWindowController: NSWindowController {+ private let contentController = PermissionContentController()+ private let accessoryPanelController = PermissionAccessoryPanelController()++ init() {+ let window = NSWindow(+ contentRect: NSRect(+ x: 0,+ y: 0,+ width: PermissionOnboardingLayout.windowWidth,+ height: PermissionOnboardingLayout.windowHeight+ ),+ styleMask: [.titled, .closable, .miniaturizable],+ backing: .buffered,+ defer: false+ )++ window.title = PermissionSupport.bundleDisplayName+ window.titleVisibility = .hidden+ window.titlebarAppearsTransparent = true+ window.isReleasedWhenClosed = false+ window.center()++ super.init(window: window)++ contentViewController = contentController+ contentController.delegate = self+ }++ @available(*, unavailable)+ required init?(coder: NSCoder) {+ nil+ }+}++@MainActor+extension PermissionWindowController: PermissionContentControllerDelegate {+ func permissionContentController(_ controller: PermissionContentController, didRequestPermission permission: SystemPermissionKind) {+ if permission == .accessibility {+ PermissionSupport.requestAccessibilityPrompt()+ }++ PermissionSupport.openSystemSettings(for: permission)+ accessoryPanelController.show(for: permission)+ contentController.setActiveGuidance(permission)+ }++ func permissionContentControllerDidResolveGuidance(_ controller: PermissionContentController) {+ accessoryPanelController.hide()+ }+}++@MainActor+protocol PermissionContentControllerDelegate: AnyObject {+ func permissionContentController(_ controller: PermissionContentController, didRequestPermission permission: SystemPermissionKind)+ func permissionContentControllerDidResolveGuidance(_ controller: PermissionContentController)+}++private enum PermissionOnboardingLayout {+ static let windowWidth: CGFloat = 880+ static let windowHeight: CGFloat = 648+ static let outerHorizontalInset: CGFloat = 48+ static let outerTopInset: CGFloat = 40+ static let outerBottomInset: CGFloat = 32+ static let headerIconSize: CGFloat = 96+ static let cardWidth: CGFloat = 744+ static let cardHeight: CGFloat = 106+ static let cardCornerRadius: CGFloat = 24+ static let cardHorizontalInset: CGFloat = 20+ static let cardVerticalInset: CGFloat = 18+ static let cardIconSize: CGFloat = 54+ static let actionButtonWidth: CGFloat = 104+ static let actionButtonHeight: CGFloat = 44+}++@MainActor+final class PermissionContentController: NSViewController {+ weak var delegate: PermissionContentControllerDelegate?++ private let backgroundView = GradientBackgroundView()+ private let stackView = NSStackView()+ private let iconView = AppGlyphView()+ private let titleLabel = NSTextField(labelWithString: "Enable Open Codex Computer Use")+ private let subtitleLabel = NSTextField(wrappingLabelWithString: "Open Codex Computer Use needs these permissions to use apps on your Mac.\nThese permissions are only used when you ask it to perform tasks.")+ private let cardsContainer = NSStackView()+ private let completionLabel = NSTextField(labelWithString: "All required permissions are enabled.")+ private let refreshTimerInterval: TimeInterval = 0.25++ private var activeGuidance: SystemPermissionKind?+ private var refreshTimer: Timer?+ private var diagnostics = PermissionDiagnostics.current()++ override func loadView() {+ view = NSView()+ }++ override func viewDidLoad() {+ super.viewDidLoad()+ configureUI()+ refreshUI()+ refreshTimer = Timer.scheduledTimer(withTimeInterval: refreshTimerInterval, repeats: true) { [weak self] _ in+ Task { @MainActor [weak self] in+ self?.refreshState()+ }+ }+ }++ override func viewDidDisappear() {+ super.viewDidDisappear()+ refreshTimer?.invalidate()+ }++ func setActiveGuidance(_ permission: SystemPermissionKind?) {+ activeGuidance = permission+ refreshUI()+ }++ private func refreshState() {+ let updated = PermissionDiagnostics.current()+ let previousGuidance = activeGuidance+ diagnostics = updated++ if let activeGuidance, updated.isGranted(activeGuidance) {+ self.activeGuidance = nil+ }++ refreshUI()++ if previousGuidance != nil, activeGuidance == nil {+ delegate?.permissionContentControllerDidResolveGuidance(self)+ }+ }++ private func configureUI() {+ view.wantsLayer = true++ backgroundView.translatesAutoresizingMaskIntoConstraints = false+ view.addSubview(backgroundView)++ stackView.orientation = .vertical+ stackView.alignment = .centerX+ stackView.spacing = 12+ stackView.translatesAutoresizingMaskIntoConstraints = false+ view.addSubview(stackView)++ iconView.translatesAutoresizingMaskIntoConstraints = false+ titleLabel.font = NSFont.systemFont(ofSize: 38, weight: .bold)+ titleLabel.textColor = NSColor(calibratedWhite: 0.18, alpha: 1)+ titleLabel.maximumNumberOfLines = 1+ subtitleLabel.font = NSFont.systemFont(ofSize: 15, weight: .regular)+ subtitleLabel.textColor = NSColor(calibratedWhite: 0.42, alpha: 1)+ subtitleLabel.alignment = .center+ subtitleLabel.maximumNumberOfLines = 2++ cardsContainer.orientation = .vertical+ cardsContainer.alignment = .centerX+ cardsContainer.spacing = 14+ cardsContainer.translatesAutoresizingMaskIntoConstraints = false++ completionLabel.font = NSFont.systemFont(ofSize: 15, weight: .semibold)+ completionLabel.textColor = NSColor(calibratedRed: 0.16, green: 0.50, blue: 0.23, alpha: 1)+ completionLabel.isHidden = true++ stackView.addArrangedSubview(iconView)+ stackView.addArrangedSubview(titleLabel)+ stackView.addArrangedSubview(subtitleLabel)+ stackView.addArrangedSubview(cardsContainer)+ stackView.addArrangedSubview(completionLabel)+ stackView.setCustomSpacing(16, after: iconView)+ stackView.setCustomSpacing(8, after: titleLabel)+ stackView.setCustomSpacing(24, after: subtitleLabel)+ stackView.setCustomSpacing(14, after: cardsContainer)++ NSLayoutConstraint.activate([+ backgroundView.leadingAnchor.constraint(equalTo: view.leadingAnchor),+ backgroundView.trailingAnchor.constraint(equalTo: view.trailingAnchor),+ backgroundView.topAnchor.constraint(equalTo: view.topAnchor),+ backgroundView.bottomAnchor.constraint(equalTo: view.bottomAnchor),++ stackView.leadingAnchor.constraint(equalTo: view.leadingAnchor, constant: PermissionOnboardingLayout.outerHorizontalInset),+ stackView.trailingAnchor.constraint(equalTo: view.trailingAnchor, constant: -PermissionOnboardingLayout.outerHorizontalInset),+ stackView.topAnchor.constraint(equalTo: view.topAnchor, constant: PermissionOnboardingLayout.outerTopInset),+ stackView.bottomAnchor.constraint(lessThanOrEqualTo: view.bottomAnchor, constant: -PermissionOnboardingLayout.outerBottomInset),++ iconView.widthAnchor.constraint(equalToConstant: PermissionOnboardingLayout.headerIconSize),+ iconView.heightAnchor.constraint(equalToConstant: PermissionOnboardingLayout.headerIconSize),+ cardsContainer.widthAnchor.constraint(equalToConstant: PermissionOnboardingLayout.cardWidth),+ ])+ }++ private func refreshUI() {+ cardsContainer.arrangedSubviews.forEach { subview in+ cardsContainer.removeArrangedSubview(subview)+ subview.removeFromSuperview()+ }++ let orderedPermissions = SystemPermissionKind.allCases+ for permission in orderedPermissions {+ if activeGuidance == permission, !diagnostics.isGranted(permission) {+ let placeholder = GuidancePlaceholderView()+ cardsContainer.addArrangedSubview(placeholder)+ placeholder.widthAnchor.constraint(equalToConstant: PermissionOnboardingLayout.cardWidth).isActive = true+ continue+ }++ let card = PermissionCardView(permission: permission, diagnostics: diagnostics)+ card.onAllow = { [weak self] requestedPermission in+ guard let self else {+ return+ }+ self.delegate?.permissionContentController(self, didRequestPermission: requestedPermission)+ }+ cardsContainer.addArrangedSubview(card)+ card.widthAnchor.constraint(equalToConstant: PermissionOnboardingLayout.cardWidth).isActive = true+ }++ completionLabel.isHidden = !diagnostics.allGranted+ }+}++@MainActor+final class PermissionCardView: NSView {+ var onAllow: ((SystemPermissionKind) -> Void)?++ private let permission: SystemPermissionKind+ private let diagnostics: PermissionDiagnostics++ init(permission: SystemPermissionKind, diagnostics: PermissionDiagnostics) {+ self.permission = permission+ self.diagnostics = diagnostics+ super.init(frame: .zero)+ translatesAutoresizingMaskIntoConstraints = false+ configure()+ }++ @available(*, unavailable)+ required init?(coder: NSCoder) {+ nil+ }++ private func configure() {+ wantsLayer = true+ layer?.cornerRadius = PermissionOnboardingLayout.cardCornerRadius+ layer?.backgroundColor = NSColor(calibratedWhite: 0.99, alpha: 0.92).cgColor+ layer?.shadowColor = NSColor.black.withAlphaComponent(0.1).cgColor+ layer?.shadowOpacity = 1+ layer?.shadowRadius = 16+ layer?.shadowOffset = CGSize(width: 0, height: -2)++ let content = NSStackView()+ content.orientation = .horizontal+ content.alignment = .centerY+ content.spacing = 16+ content.translatesAutoresizingMaskIntoConstraints = false++ let iconBackground = NSView()+ iconBackground.translatesAutoresizingMaskIntoConstraints = false+ iconBackground.wantsLayer = true+ iconBackground.layer?.cornerRadius = PermissionOnboardingLayout.cardIconSize / 2+ iconBackground.layer?.backgroundColor = NSColor(calibratedWhite: 1, alpha: 0.92).cgColor+ iconBackground.layer?.borderWidth = 2+ iconBackground.layer?.borderColor = (+ permission == .accessibility+ ? NSColor.systemBlue.withAlphaComponent(0.28)+ : NSColor(calibratedWhite: 0.82, alpha: 1)+ ).cgColor++ let icon = NSImageView()+ icon.translatesAutoresizingMaskIntoConstraints = false+ icon.contentTintColor = permission == .accessibility ? NSColor.systemBlue : NSColor.systemGray+ icon.image = NSImage(systemSymbolName: permission.symbolName, accessibilityDescription: permission.title)+ icon.symbolConfiguration = NSImage.SymbolConfiguration(pointSize: 24, weight: .semibold)+ iconBackground.addSubview(icon)++ let labels = NSStackView()+ labels.orientation = .vertical+ labels.alignment = .leading+ labels.spacing = 2++ let title = NSTextField(labelWithString: permission.title)+ title.font = NSFont.systemFont(ofSize: 20, weight: .bold)+ title.textColor = NSColor(calibratedWhite: 0.18, alpha: 1)++ let subtitle = NSTextField(labelWithString: permission.subtitle)+ subtitle.font = NSFont.systemFont(ofSize: 14, weight: .regular)+ subtitle.textColor = NSColor(calibratedWhite: 0.42, alpha: 1)++ labels.addArrangedSubview(title)+ labels.addArrangedSubview(subtitle)++ let spacer = NSView()+ spacer.translatesAutoresizingMaskIntoConstraints = false++ content.addArrangedSubview(iconBackground)+ content.addArrangedSubview(labels)+ content.addArrangedSubview(spacer)++ if diagnostics.isGranted(permission) {+ let done = StatusChipView(text: "Done", foreground: NSColor(calibratedRed: 0.16, green: 0.50, blue: 0.23, alpha: 1), background: NSColor(calibratedRed: 0.93, green: 0.98, blue: 0.94, alpha: 1))+ content.addArrangedSubview(done)+ } else {+ let button = PrimaryActionButton(title: "Allow", target: self, action: #selector(handleAllow))+ content.addArrangedSubview(button)+ button.widthAnchor.constraint(equalToConstant: PermissionOnboardingLayout.actionButtonWidth).isActive = true+ button.heightAnchor.constraint(equalToConstant: PermissionOnboardingLayout.actionButtonHeight).isActive = true+ }++ addSubview(content)++ NSLayoutConstraint.activate([+ heightAnchor.constraint(equalToConstant: PermissionOnboardingLayout.cardHeight),+ content.leadingAnchor.constraint(equalTo: leadingAnchor, constant: PermissionOnboardingLayout.cardHorizontalInset),+ content.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -PermissionOnboardingLayout.cardHorizontalInset),+ content.topAnchor.constraint(equalTo: topAnchor, constant: PermissionOnboardingLayout.cardVerticalInset),+ content.bottomAnchor.constraint(equalTo: bottomAnchor, constant: -PermissionOnboardingLayout.cardVerticalInset),++ iconBackground.widthAnchor.constraint(equalToConstant: PermissionOnboardingLayout.cardIconSize),+ iconBackground.heightAnchor.constraint(equalToConstant: PermissionOnboardingLayout.cardIconSize),+ icon.centerXAnchor.constraint(equalTo: iconBackground.centerXAnchor),+ icon.centerYAnchor.constraint(equalTo: iconBackground.centerYAnchor),+ spacer.widthAnchor.constraint(greaterThanOrEqualToConstant: 12),+ ])+ }++ @objc+ private func handleAllow() {+ onAllow?(permission)+ }+}++@MainActor+final class GuidancePlaceholderView: NSView {+ init() {+ super.init(frame: .zero)+ translatesAutoresizingMaskIntoConstraints = false+ wantsLayer = true+ layer?.cornerRadius = PermissionOnboardingLayout.cardCornerRadius+ layer?.backgroundColor = NSColor(calibratedWhite: 0.98, alpha: 0.6).cgColor++ let label = NSTextField(labelWithString: "COMPLETE IN SYSTEM SETTINGS")+ label.font = NSFont.systemFont(ofSize: 13, weight: .semibold)+ label.textColor = NSColor(calibratedWhite: 0.55, alpha: 1)+ label.translatesAutoresizingMaskIntoConstraints = false+ addSubview(label)++ NSLayoutConstraint.activate([+ heightAnchor.constraint(equalToConstant: 82),+ label.centerXAnchor.constraint(equalTo: centerXAnchor),+ label.centerYAnchor.constraint(equalTo: centerYAnchor),+ ])+ }++ @available(*, unavailable)+ required init?(coder: NSCoder) {+ nil+ }++ override func draw(_ dirtyRect: NSRect) {+ super.draw(dirtyRect)++ let borderRect = bounds.insetBy(dx: 1, dy: 1)+ let path = NSBezierPath(+ roundedRect: borderRect,+ xRadius: PermissionOnboardingLayout.cardCornerRadius,+ yRadius: PermissionOnboardingLayout.cardCornerRadius+ )+ path.setLineDash([6, 6], count: 2, phase: 0)+ path.lineWidth = 1.5+ NSColor(calibratedWhite: 0.82, alpha: 1).setStroke()+ path.stroke()+ }+}++@MainActor+final class PermissionAccessoryPanelController {+ private let trackingInterval: TimeInterval = 0.12+ private var panel: NSPanel?+ private var currentPermission: SystemPermissionKind?+ private var trackingTimer: Timer?+ private var workspaceObserver: NSObjectProtocol?+ private var globalDragMonitor: Any?+ private var localDragMonitor: Any?++ func show(for permission: SystemPermissionKind) {+ currentPermission = permission++ let panel = panel ?? makePanel()+ self.panel = panel++ if let contentView = panel.contentView as? PermissionAccessoryPanelView {+ contentView.configure(permission: permission)+ }++ installObserversIfNeeded()+ startTracking()+ position(panel: panel)+ updatePanelVisibility()+ }++ func hide() {+ trackingTimer?.invalidate()+ trackingTimer = nil+ removeObservers()+ panel?.orderOut(nil)+ currentPermission = nil+ }++ private func makePanel() -> NSPanel {+ let panel = NSPanel(+ contentRect: NSRect(x: 0, y: 0, width: 452, height: 102),+ styleMask: [.borderless, .nonactivatingPanel],+ backing: .buffered,+ defer: false+ )+ panel.level = .normal+ panel.backgroundColor = .clear+ panel.isOpaque = false+ panel.hasShadow = true+ panel.collectionBehavior = [.canJoinAllSpaces, .transient, .fullScreenAuxiliary]+ panel.contentView = PermissionAccessoryPanelView()+ return panel+ }++ private func installObserversIfNeeded() {+ if workspaceObserver == nil {+ workspaceObserver = NSWorkspace.shared.notificationCenter.addObserver(+ forName: NSWorkspace.didActivateApplicationNotification,+ object: nil,+ queue: .main+ ) { [weak self] _ in+ Task { @MainActor [weak self] in+ self?.updatePanelVisibility()+ self?.refreshPosition()+ }+ }+ }++ if globalDragMonitor == nil {+ globalDragMonitor = NSEvent.addGlobalMonitorForEvents(matching: [.leftMouseDragged, .leftMouseUp]) { [weak self] _ in+ Task { @MainActor [weak self] in+ self?.refreshPosition()+ }+ }+ }++ if localDragMonitor == nil {+ localDragMonitor = NSEvent.addLocalMonitorForEvents(matching: [.leftMouseDragged, .leftMouseUp]) { [weak self] event in+ Task { @MainActor [weak self] in+ self?.refreshPosition()+ }+ return event+ }+ }+ }++ private func removeObservers() {+ if let workspaceObserver {+ NSWorkspace.shared.notificationCenter.removeObserver(workspaceObserver)+ self.workspaceObserver = nil+ }++ if let globalDragMonitor {+ NSEvent.removeMonitor(globalDragMonitor)+ self.globalDragMonitor = nil+ }++ if let localDragMonitor {+ NSEvent.removeMonitor(localDragMonitor)+ self.localDragMonitor = nil+ }+ }++ private func startTracking() {+ trackingTimer?.invalidate()+ trackingTimer = Timer.scheduledTimer(withTimeInterval: trackingInterval, repeats: true) { [weak self] _ in+ Task { @MainActor [weak self] in+ self?.updatePanelVisibility()+ self?.refreshPosition()+ }+ }+ trackingTimer?.tolerance = 0.05+ }++ private func refreshPosition() {+ guard let panel, currentPermission != nil, panel.isVisible else {+ return+ }++ position(panel: panel)+ }++ private func updatePanelVisibility() {+ guard let panel, currentPermission != nil else {+ return+ }++ guard isSystemSettingsFrontmost else {+ panel.orderOut(nil)+ return+ }++ if !panel.isVisible {+ panel.orderFront(nil)+ }+ }++ private func position(panel: NSPanel) {+ guard let origin = preferredPanelOrigin(for: panel.frame.size) else {+ return+ }++ if panel.frame.origin != origin {+ panel.setFrameOrigin(origin)+ }+ }++ private func preferredPanelOrigin(for panelSize: CGSize) -> CGPoint? {+ guard let anchor = systemSettingsControlAnchorRect() else {+ return fallbackPanelOrigin(for: panelSize)+ }++ let visibleFrame = targetVisibleScreenFrame(for: anchor) ?? NSScreen.main?.visibleFrame ?? .zero+ let x = clamp(anchor.minX - 18, lower: visibleFrame.minX + 16, upper: visibleFrame.maxX - panelSize.width - 16)+ let y = max(visibleFrame.minY + 12, anchor.minY - panelSize.height - 10)+ return CGPoint(x: x, y: y)+ }++ private func fallbackPanelOrigin(for panelSize: CGSize) -> CGPoint? {+ guard let referenceRect = systemSettingsWindowBounds() ?? NSScreen.main?.visibleFrame else {+ return nil+ }++ let visibleFrame = targetVisibleScreenFrame(for: referenceRect) ?? referenceRect+ let estimatedControlsMinX = referenceRect.minX + 234+ let x = clamp(estimatedControlsMinX - 18, lower: visibleFrame.minX + 16, upper: visibleFrame.maxX - panelSize.width - 16)+ let y = max(visibleFrame.minY + 12, referenceRect.minY - panelSize.height + 6)+ return CGPoint(x: x, y: y)+ }++ private func targetVisibleScreenFrame(for rect: CGRect) -> CGRect? {+ NSScreen.screens.first(where: { $0.visibleFrame.intersects(rect) })?.visibleFrame+ }++ private var isSystemSettingsFrontmost: Bool {+ NSWorkspace.shared.frontmostApplication?.bundleIdentifier == "com.apple.systempreferences"+ }++ private func clamp(_ value: CGFloat, lower: CGFloat, upper: CGFloat) -> CGFloat {+ guard lower <= upper else {+ return lower+ }++ return min(max(value, lower), upper)+ }++ private func systemSettingsControlAnchorRect() -> CGRect? {+ guard PermissionDiagnostics.current().accessibilityTrusted else {+ return nil+ }++ guard+ let appElement = systemSettingsApplicationElement(),+ let root = focusedSystemSettingsWindow(for: appElement) ?? firstWindow(for: appElement)+ else {+ return nil+ }++ let controlFrames = descendantFrames(+ in: root,+ matchingTitles: ["Add", "Remove"],+ role: kAXButtonRole as String+ )++ guard !controlFrames.isEmpty else {+ return nil+ }++ let groups = Dictionary(grouping: controlFrames) { frame in+ Int((frame.midY / 12).rounded(.toNearestOrAwayFromZero))+ }++ guard let topGroup = groups.values.max(by: { lhs, rhs in+ let lhsTop = lhs.map(\.maxY).max() ?? 0+ let rhsTop = rhs.map(\.maxY).max() ?? 0+ return lhsTop < rhsTop+ }) else {+ return nil+ }++ return topGroup.reduce(topGroup[0]) { partial, frame in+ partial.union(frame)+ }+ }++ private func systemSettingsWindowBounds() -> CGRect? {+ guard let runningApp = NSWorkspace.shared.runningApplications.first(where: { $0.bundleIdentifier == "com.apple.systempreferences" }) else {+ return nil+ }++ guard let windowInfoList = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] else {+ return nil+ }++ let windows = windowInfoList.compactMap { info -> (CGRect, Int)? in+ guard+ let ownerPID = info[kCGWindowOwnerPID as String] as? pid_t,+ ownerPID == runningApp.processIdentifier,+ let layer = info[kCGWindowLayer as String] as? Int,+ layer == 0,+ let boundsDictionary = info[kCGWindowBounds as String] as? NSDictionary,+ let bounds = CGRect(dictionaryRepresentation: boundsDictionary)+ else {+ return nil+ }++ return (bounds, Int(bounds.width * bounds.height))+ }++ return windows.sorted(by: { $0.1 > $1.1 }).first?.0+ }++ private func systemSettingsApplicationElement() -> AXUIElement? {+ guard let runningApp = NSWorkspace.shared.runningApplications.first(where: { $0.bundleIdentifier == "com.apple.systempreferences" }) else {+ return nil+ }++ return AXUIElementCreateApplication(runningApp.processIdentifier)+ }++ private func focusedSystemSettingsWindow(for appElement: AXUIElement) -> AXUIElement? {+ let systemWide = AXUIElementCreateSystemWide()++ if+ let focusedApp = copyElement(systemWide, attribute: kAXFocusedApplicationAttribute),+ pid(of: focusedApp) == pid(of: appElement)+ {+ return copyElement(systemWide, attribute: kAXFocusedWindowAttribute)+ ?? copyElement(focusedApp, attribute: kAXFocusedWindowAttribute)+ ?? firstWindow(for: focusedApp)+ }++ return copyElement(appElement, attribute: kAXFocusedWindowAttribute)+ }++ private func firstWindow(for appElement: AXUIElement) -> AXUIElement? {+ copyArray(appElement, attribute: kAXWindowsAttribute)?.first+ }++ private func descendantFrames(in root: AXUIElement, matchingTitles titles: Set<String>, role targetRole: String) -> [CGRect] {+ var results: [CGRect] = []+ var queue: [AXUIElement] = [root]+ var visited: Set<CFHashCode> = []++ while let element = queue.first {+ queue.removeFirst()++ let token = CFHash(element)+ guard !visited.contains(token) else {+ continue+ }+ visited.insert(token)++ if+ stringValue(of: element, attribute: kAXRoleAttribute) == targetRole,+ let title = stringValue(of: element, attribute: kAXTitleAttribute),+ titles.contains(title),+ let frame = frame(of: element)+ {+ results.append(frame)+ }++ queue.append(contentsOf: childElements(of: element))+ }++ return results+ }++ private func childElements(of element: AXUIElement) -> [AXUIElement] {+ let attributes = [kAXChildrenAttribute, kAXRowsAttribute]+ var children: [AXUIElement] = []++ for attribute in attributes {+ if let nested = copyArray(element, attribute: attribute) {+ children.append(contentsOf: nested)+ }+ }++ return children+ }++ private func copyElement(_ element: AXUIElement, attribute: String) -> AXUIElement? {+ var value: CFTypeRef?+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {+ return nil+ }++ guard let value else {+ return nil+ }++ return (value as! AXUIElement)+ }++ private func copyArray(_ element: AXUIElement, attribute: String) -> [AXUIElement]? {+ var value: CFTypeRef?+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {+ return nil+ }++ return value as? [AXUIElement]+ }++ private func stringValue(of element: AXUIElement, attribute: String) -> String? {+ var value: CFTypeRef?+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {+ return nil+ }++ return value as? String+ }++ private func frame(of element: AXUIElement) -> CGRect? {+ var positionValue: CFTypeRef?+ var sizeValue: CFTypeRef?++ guard+ AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success,+ AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success,+ let positionValue,+ let sizeValue+ else {+ return nil+ }++ let positionAXValue = positionValue as! AXValue+ let sizeAXValue = sizeValue as! AXValue+ var position = CGPoint.zero+ var size = CGSize.zero+ guard+ AXValueGetValue(positionAXValue, .cgPoint, &position),+ AXValueGetValue(sizeAXValue, .cgSize, &size)+ else {+ return nil+ }++ return CGRect(origin: position, size: size)+ }++ private func pid(of element: AXUIElement) -> pid_t {+ var processIdentifier: pid_t = 0+ AXUIElementGetPid(element, &processIdentifier)+ return processIdentifier+ }+}++@MainActor+final class PermissionAccessoryPanelView: NSView {+ private let instructionLabel = NSTextField(labelWithString: "")+ private let dragTileView = DraggableAppTileView()++ init() {+ super.init(frame: .zero)+ translatesAutoresizingMaskIntoConstraints = false+ wantsLayer = true+ layer?.cornerRadius = 20+ layer?.backgroundColor = NSColor(calibratedWhite: 0.98, alpha: 0.96).cgColor+ layer?.shadowColor = NSColor.black.withAlphaComponent(0.14).cgColor+ layer?.shadowOpacity = 1+ layer?.shadowRadius = 18+ layer?.shadowOffset = CGSize(width: 0, height: -5)++ let arrow = NSImageView()+ arrow.translatesAutoresizingMaskIntoConstraints = false+ arrow.image = NSImage(systemSymbolName: "arrow.up", accessibilityDescription: "Drag upward")+ arrow.symbolConfiguration = NSImage.SymbolConfiguration(pointSize: 26, weight: .bold)+ arrow.contentTintColor = NSColor.systemBlue++ instructionLabel.font = NSFont.systemFont(ofSize: 14, weight: .medium)+ instructionLabel.textColor = NSColor(calibratedWhite: 0.4, alpha: 1)+ instructionLabel.translatesAutoresizingMaskIntoConstraints = false+ instructionLabel.lineBreakMode = .byTruncatingTail+ instructionLabel.maximumNumberOfLines = 1+ instructionLabel.setContentCompressionResistancePriority(.defaultLow, for: .horizontal)++ dragTileView.translatesAutoresizingMaskIntoConstraints = false++ addSubview(arrow)+ addSubview(instructionLabel)+ addSubview(dragTileView)++ NSLayoutConstraint.activate([+ arrow.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 18),+ arrow.topAnchor.constraint(equalTo: topAnchor, constant: 16),+ arrow.widthAnchor.constraint(equalToConstant: 32),+ arrow.heightAnchor.constraint(equalToConstant: 32),++ instructionLabel.leadingAnchor.constraint(equalTo: arrow.trailingAnchor, constant: 12),+ instructionLabel.centerYAnchor.constraint(equalTo: arrow.centerYAnchor),+ instructionLabel.trailingAnchor.constraint(lessThanOrEqualTo: trailingAnchor, constant: -18),++ dragTileView.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 54),+ dragTileView.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -16),+ dragTileView.bottomAnchor.constraint(equalTo: bottomAnchor, constant: -12),+ dragTileView.heightAnchor.constraint(equalToConstant: 42),+ ])+ }++ @available(*, unavailable)+ required init?(coder: NSCoder) {+ nil+ }++ func configure(permission: SystemPermissionKind) {+ instructionLabel.stringValue = permission.dragInstruction+ }+}++@MainActor+final class DraggableAppTileView: NSView, NSDraggingSource {+ override init(frame frameRect: NSRect) {+ super.init(frame: frameRect)+ wantsLayer = true+ layer?.cornerRadius = 12+ layer?.backgroundColor = NSColor.white.cgColor+ layer?.borderWidth = 1+ layer?.borderColor = NSColor(calibratedWhite: 0.86, alpha: 1).cgColor+ }++ @available(*, unavailable)+ required init?(coder: NSCoder) {+ nil+ }++ override func draw(_ dirtyRect: NSRect) {+ super.draw(dirtyRect)++ let iconRect = CGRect(x: 12, y: 8, width: 28, height: 28)+ let icon = currentIcon()+ icon.draw(in: iconRect)++ let title = currentTitle()+ let attributes: [NSAttributedString.Key: Any] = [+ .font: NSFont.systemFont(ofSize: 15, weight: .semibold),+ .foregroundColor: NSColor(calibratedWhite: 0.26, alpha: 1),+ ]+ title.draw(at: CGPoint(x: 52, y: 12), withAttributes: attributes)+ }++ override func mouseDragged(with event: NSEvent) {+ guard let bundleURL = PermissionSupport.currentAppBundleURL() else {+ NSSound.beep()+ return+ }++ let draggingItem = NSDraggingItem(pasteboardWriter: bundleURL as NSURL)+ draggingItem.setDraggingFrame(bounds, contents: snapshotImage())+ beginDraggingSession(with: [draggingItem], event: event, source: self)+ }++ func draggingSession(_ session: NSDraggingSession, sourceOperationMaskFor context: NSDraggingContext) -> NSDragOperation {+ .copy+ }++ private func currentIcon() -> NSImage {+ if let bundleURL = PermissionSupport.currentAppBundleURL() {+ if let bundle = Bundle(url: bundleURL), bundle.bundleIdentifier == PermissionSupport.bundleIdentifier {+ return Branding.makeAppIconImage(size: 128)+ }++ return NSWorkspace.shared.icon(forFile: bundleURL.path)+ }++ return Branding.makeAppIconImage(size: 128)+ }++ private func currentTitle() -> String {+ if let bundleURL = PermissionSupport.currentAppBundleURL() {+ if let bundle = Bundle(url: bundleURL) {+ let displayName = bundle.object(forInfoDictionaryKey: "CFBundleDisplayName") as? String+ let bundleName = bundle.object(forInfoDictionaryKey: kCFBundleNameKey as String) as? String+ return displayName ?? bundleName ?? PermissionSupport.bundleDisplayName+ }++ return PermissionSupport.bundleDisplayName+ }++ return PermissionSupport.bundleDisplayName+ }++ private func snapshotImage() -> NSImage {+ let bitmap = bitmapImageRepForCachingDisplay(in: bounds) ?? NSBitmapImageRep(bitmapDataPlanes: nil, pixelsWide: Int(bounds.width), pixelsHigh: Int(bounds.height), bitsPerSample: 8, samplesPerPixel: 4, hasAlpha: true, isPlanar: false, colorSpaceName: .deviceRGB, bytesPerRow: 0, bitsPerPixel: 0)!+ cacheDisplay(in: bounds, to: bitmap)+ let image = NSImage(size: bounds.size)+ image.addRepresentation(bitmap)+ return image+ }+}++@MainActor+enum Branding {+ static func makeAppIconImage(size: CGFloat) -> NSImage {+ let image = NSImage(size: NSSize(width: size, height: size))+ image.lockFocus()++ let rect = CGRect(origin: .zero, size: image.size)+ let path = NSBezierPath(roundedRect: rect, xRadius: size * 0.22, yRadius: size * 0.22)++ let gradient = NSGradient(colors: [+ NSColor(calibratedRed: 0.12, green: 0.67, blue: 0.99, alpha: 1),+ NSColor(calibratedRed: 0.94, green: 0.74, blue: 0.93, alpha: 1),+ ])!+ gradient.draw(in: path, angle: 20)++ NSColor.white.withAlphaComponent(0.18).setFill()+ NSBezierPath(ovalIn: CGRect(x: size * 0.1, y: size * 0.5, width: size * 0.62, height: size * 0.42)).fill()++ let glyph = NSBezierPath()+ glyph.move(to: CGPoint(x: size * 0.36, y: size * 0.72))+ glyph.curve(to: CGPoint(x: size * 0.7, y: size * 0.58), controlPoint1: CGPoint(x: size * 0.48, y: size * 0.78), controlPoint2: CGPoint(x: size * 0.63, y: size * 0.69))+ glyph.curve(to: CGPoint(x: size * 0.44, y: size * 0.3), controlPoint1: CGPoint(x: size * 0.82, y: size * 0.48), controlPoint2: CGPoint(x: size * 0.6, y: size * 0.28))+ glyph.curve(to: CGPoint(x: size * 0.26, y: size * 0.62), controlPoint1: CGPoint(x: size * 0.28, y: size * 0.32), controlPoint2: CGPoint(x: size * 0.18, y: size * 0.5))+ glyph.close()++ NSColor.white.withAlphaComponent(0.78).setFill()+ glyph.fill()++ image.unlockFocus()+ return image+ }+}++@MainActor+final class GradientBackgroundView: NSView {+ override func draw(_ dirtyRect: NSRect) {+ super.draw(dirtyRect)++ let gradient = NSGradient(colors: [+ NSColor(calibratedWhite: 0.985, alpha: 1),+ NSColor(calibratedRed: 0.95, green: 0.97, blue: 1, alpha: 1),+ NSColor(calibratedRed: 0.98, green: 0.94, blue: 1, alpha: 1),+ ])!+ gradient.draw(in: bounds, angle: 15)+ }+}++@MainActor+final class AppGlyphView: NSImageView {+ override init(frame frameRect: NSRect) {+ super.init(frame: frameRect)+ image = Branding.makeAppIconImage(size: PermissionOnboardingLayout.headerIconSize)+ }++ @available(*, unavailable)+ required init?(coder: NSCoder) {+ nil+ }+}++@MainActor+final class StatusChipView: NSView {+ init(text: String, foreground: NSColor, background: NSColor) {+ super.init(frame: .zero)+ translatesAutoresizingMaskIntoConstraints = false+ wantsLayer = true+ layer?.cornerRadius = 16+ layer?.backgroundColor = background.cgColor++ let label = NSTextField(labelWithString: text)+ label.font = NSFont.systemFont(ofSize: 14, weight: .semibold)+ label.textColor = foreground+ label.translatesAutoresizingMaskIntoConstraints = false+ addSubview(label)++ let check = NSImageView()+ check.translatesAutoresizingMaskIntoConstraints = false+ check.image = NSImage(systemSymbolName: "checkmark", accessibilityDescription: text)+ check.symbolConfiguration = NSImage.SymbolConfiguration(pointSize: 12, weight: .bold)+ check.contentTintColor = foreground+ addSubview(check)++ NSLayoutConstraint.activate([+ heightAnchor.constraint(equalToConstant: 34),+ label.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 16),+ label.centerYAnchor.constraint(equalTo: centerYAnchor),+ check.leadingAnchor.constraint(equalTo: label.trailingAnchor, constant: 8),+ check.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -16),+ check.centerYAnchor.constraint(equalTo: centerYAnchor),+ ])+ }++ @available(*, unavailable)+ required init?(coder: NSCoder) {+ nil+ }+}++@MainActor+final class PrimaryActionButton: NSButton {+ override var isHighlighted: Bool {+ didSet {+ updateAppearance()+ }+ }++ init(title: String, target: AnyObject?, action: Selector) {+ super.init(frame: .zero)+ self.target = target+ self.action = action+ self.title = title+ translatesAutoresizingMaskIntoConstraints = false+ wantsLayer = true+ isBordered = false+ focusRingType = .none+ setButtonType(.momentaryPushIn)+ updateAttributedTitle()+ updateAppearance()+ }++ @available(*, unavailable)+ required init?(coder: NSCoder) {+ nil+ }++ override func layout() {+ super.layout()+ layer?.cornerRadius = bounds.height / 2+ }++ private func updateAttributedTitle() {+ let paragraphStyle = NSMutableParagraphStyle()+ paragraphStyle.alignment = .center+ attributedTitle = NSAttributedString(+ string: title,+ attributes: [+ .font: NSFont.systemFont(ofSize: 16, weight: .semibold),+ .foregroundColor: NSColor.white,+ .paragraphStyle: paragraphStyle,+ ]+ )+ }++ private func updateAppearance() {+ layer?.backgroundColor = (+ isHighlighted+ ? NSColor(calibratedRed: 0.05, green: 0.45, blue: 0.94, alpha: 1)+ : NSColor(calibratedRed: 0.06, green: 0.49, blue: 0.99, alpha: 1)+ ).cgColor+ alphaValue = isEnabled ? 1 : 0.45+ }+}diff --git a/apps/OpenCodexComputerUseFixture/Sources/OpenCodexComputerUseFixture/main.swift b/apps/OpenCodexComputerUseFixture/Sources/OpenCodexComputerUseFixture/main.swiftnew file mode 100644index 0000000..7a38d1f--- /dev/null+++ b/apps/OpenCodexComputerUseFixture/Sources/OpenCodexComputerUseFixture/main.swift@@ -0,0 +1,427 @@+import AppKit+import Carbon.HIToolbox+import Foundation+import OpenCodexComputerUseKit++@MainActor+final class KeyCaptureView: NSView {+ var onKey: ((String) -> Void)?++ override var acceptsFirstResponder: Bool { true }++ override func mouseDown(with event: NSEvent) {+ window?.makeFirstResponder(self)+ needsDisplay = true+ }++ override func keyDown(with event: NSEvent) {+ let value = debugKeyName(for: event)+ onKey?(value)+ }++ override func draw(_ dirtyRect: NSRect) {+ NSColor.systemTeal.setFill()+ dirtyRect.fill()++ let text = "Click here, then use press_key"+ let attributes: [NSAttributedString.Key: Any] = [+ .foregroundColor: NSColor.white,+ .font: NSFont.monospacedSystemFont(ofSize: 14, weight: .medium),+ ]+ let size = text.size(withAttributes: attributes)+ text.draw(+ at: NSPoint(x: (bounds.width - size.width) / 2, y: (bounds.height - size.height) / 2),+ withAttributes: attributes+ )++ if window?.firstResponder === self {+ NSColor.white.setStroke()+ let path = NSBezierPath(rect: bounds.insetBy(dx: 3, dy: 3))+ path.lineWidth = 3+ path.stroke()+ }+ }+}++@MainActor+final class DragPadView: NSView {+ var onDrag: ((String) -> Void)?+ private var dragStart: CGPoint?++ override func mouseDown(with event: NSEvent) {+ dragStart = convert(event.locationInWindow, from: nil)+ }++ override func mouseDragged(with event: NSEvent) {+ let current = convert(event.locationInWindow, from: nil)+ let start = dragStart ?? current+ onDrag?("from (\(Int(start.x)), \(Int(start.y))) to (\(Int(current.x)), \(Int(current.y)))")+ }++ override func draw(_ dirtyRect: NSRect) {+ NSColor.systemOrange.setFill()+ dirtyRect.fill()++ let text = "Drag inside this pad"+ let attributes: [NSAttributedString.Key: Any] = [+ .foregroundColor: NSColor.white,+ .font: NSFont.monospacedSystemFont(ofSize: 14, weight: .medium),+ ]+ let size = text.size(withAttributes: attributes)+ text.draw(+ at: NSPoint(x: (bounds.width - size.width) / 2, y: (bounds.height - size.height) / 2),+ withAttributes: attributes+ )+ }+}++@MainActor+final class FixtureAppDelegate: NSObject, NSApplicationDelegate, NSTextFieldDelegate, NSWindowDelegate {+ private var window: NSWindow!+ private let incrementButton = NSButton(title: "Increment Counter", target: nil, action: nil)+ private let counterLabel = NSTextField(labelWithString: "Counter: 0")+ private let inputField = NSTextField(string: "seed")+ private let keyLabel = NSTextField(labelWithString: "Last key: none")+ private let scrollLabel = NSTextField(labelWithString: "Scroll offset: 0")+ private let dragLabel = NSTextField(labelWithString: "Last drag: none")+ private let keyCaptureView = KeyCaptureView(frame: NSRect(x: 0, y: 0, width: 320, height: 72))+ private let dragPadView = DragPadView(frame: NSRect(x: 0, y: 0, width: 320, height: 120))+ private var scrollView: NSScrollView!+ private var counter = 0+ private weak var observedScrollView: NSScrollView?+ private var commandObserver: NSObjectProtocol?++ func applicationDidFinishLaunching(_ notification: Notification) {+ buildWindow()+ startCommandObserver()+ updateExportedState()+ NSApp.activate(ignoringOtherApps: true)+ }++ func applicationWillTerminate(_ notification: Notification) {+ if let commandObserver {+ DistributedNotificationCenter.default().removeObserver(commandObserver)+ }+ }++ private func buildWindow() {+ window = NSWindow(+ contentRect: NSRect(x: 160, y: 180, width: 640, height: 620),+ styleMask: [.titled, .closable, .miniaturizable, .resizable],+ backing: .buffered,+ defer: false+ )+ window.title = "OpenCodexComputerUseFixture"+ window.setAccessibilityIdentifier("fixture-window")+ window.delegate = self++ let contentView = NSView()+ contentView.translatesAutoresizingMaskIntoConstraints = false+ window.contentView = contentView++ let descriptionLabel = NSTextField(wrappingLabelWithString: "Fixture app for safe computer-use smoke tests.")+ incrementButton.target = self+ incrementButton.action = #selector(handleIncrement)+ incrementButton.bezelStyle = .rounded+ incrementButton.setAccessibilityIdentifier("fixture-increment")++ counterLabel.setAccessibilityIdentifier("fixture-counter-label")++ inputField.delegate = self+ inputField.setAccessibilityIdentifier("fixture-input")++ keyLabel.setAccessibilityIdentifier("fixture-key-label")+ keyCaptureView.setAccessibilityIdentifier("fixture-key-capture")+ keyCaptureView.onKey = { [weak self] value in+ self?.keyLabel.stringValue = "Last key: \(value)"+ self?.keyCaptureView.needsDisplay = true+ self?.updateExportedState()+ }++ scrollLabel.setAccessibilityIdentifier("fixture-scroll-status")+ scrollView = makeScrollView()+ scrollView.setAccessibilityIdentifier("fixture-scroll-view")++ dragLabel.setAccessibilityIdentifier("fixture-drag-status")+ dragPadView.setAccessibilityIdentifier("fixture-drag-pad")+ dragPadView.onDrag = { [weak self] value in+ self?.dragLabel.stringValue = "Last drag: \(value)"+ self?.updateExportedState()+ }++ let stack = NSStackView(views: [+ descriptionLabel,+ incrementButton,+ counterLabel,+ NSTextField(labelWithString: "Editable Text Field"),+ inputField,+ keyLabel,+ keyCaptureView,+ scrollLabel,+ scrollView,+ dragLabel,+ dragPadView,+ ])+ stack.orientation = .vertical+ stack.alignment = .leading+ stack.spacing = 12+ stack.translatesAutoresizingMaskIntoConstraints = false++ contentView.addSubview(stack)++ NSLayoutConstraint.activate([+ stack.leadingAnchor.constraint(equalTo: contentView.leadingAnchor, constant: 20),+ stack.trailingAnchor.constraint(equalTo: contentView.trailingAnchor, constant: -20),+ stack.topAnchor.constraint(equalTo: contentView.topAnchor, constant: 20),+ stack.bottomAnchor.constraint(lessThanOrEqualTo: contentView.bottomAnchor, constant: -20),+ keyCaptureView.widthAnchor.constraint(equalToConstant: 320),+ keyCaptureView.heightAnchor.constraint(equalToConstant: 72),+ dragPadView.widthAnchor.constraint(equalToConstant: 320),+ dragPadView.heightAnchor.constraint(equalToConstant: 120),+ scrollView.widthAnchor.constraint(equalToConstant: 520),+ scrollView.heightAnchor.constraint(equalToConstant: 150),+ inputField.widthAnchor.constraint(equalToConstant: 320),+ ])++ window.makeKeyAndOrderFront(nil)+ }++ @objc+ private func handleIncrement() {+ counter += 1+ counterLabel.stringValue = "Counter: \(counter)"+ updateExportedState()+ }++ func controlTextDidChange(_ obj: Notification) {+ if let field = obj.object as? NSTextField, field == inputField {+ inputField.stringValue = field.stringValue+ updateExportedState()+ }+ }++ @objc+ private func handleScrollBoundsChanged(_ notification: Notification) {+ guard let scrollView = observedScrollView else {+ return+ }++ let offset = Int(scrollView.contentView.bounds.origin.y)+ scrollLabel.stringValue = "Scroll offset: \(offset)"+ updateExportedState()+ }++ func windowDidMove(_ notification: Notification) {+ updateExportedState()+ }++ func windowDidResize(_ notification: Notification) {+ updateExportedState()+ }++ private func makeScrollView() -> NSScrollView {+ let documentView = NSStackView()+ documentView.orientation = .vertical+ documentView.alignment = .leading+ documentView.spacing = 8+ documentView.translatesAutoresizingMaskIntoConstraints = false++ for index in 1...40 {+ let label = NSTextField(labelWithString: "Scrollable row \(index)")+ documentView.addArrangedSubview(label)+ }++ let clipView = NSClipView()+ clipView.postsBoundsChangedNotifications = true++ let scrollView = NSScrollView()+ scrollView.drawsBackground = true+ scrollView.borderType = .bezelBorder+ scrollView.hasVerticalScroller = true+ scrollView.contentView = clipView+ scrollView.documentView = documentView+ observedScrollView = scrollView+ NotificationCenter.default.addObserver(+ self,+ selector: #selector(handleScrollBoundsChanged(_:)),+ name: NSView.boundsDidChangeNotification,+ object: clipView+ )++ return scrollView+ }++ private func startCommandObserver() {+ commandObserver = DistributedNotificationCenter.default().addObserver(+ forName: FixtureBridge.distributedNotificationName,+ object: nil,+ queue: .main+ ) { [weak self] notification in+ guard+ let payload = notification.userInfo?["payload"] as? String,+ let data = payload.data(using: .utf8),+ let command = try? JSONDecoder().decode(FixtureCommand.self, from: data)+ else {+ return+ }++ Task { @MainActor [weak self] in+ self?.handle(command)+ }+ }+ }++ private func handle(_ command: FixtureCommand) {+ switch (command.kind, command.identifier) {+ case ("set_value", "fixture-input"):+ inputField.stringValue = command.value ?? ""+ updateExportedState()+ case ("click", "fixture-increment"):+ handleIncrement()+ case ("click", "fixture-input"):+ window.makeFirstResponder(inputField)+ updateExportedState()+ case ("click", "fixture-key-capture"):+ window.makeFirstResponder(keyCaptureView)+ keyCaptureView.needsDisplay = true+ updateExportedState()+ case ("scroll", "fixture-scroll-view"):+ let delta = CGFloat(120 * (command.pages ?? 1))+ let direction = command.direction ?? "down"+ let current = scrollView.contentView.bounds.origin+ let nextY = switch direction {+ case "up":+ max(0, current.y - delta)+ case "down":+ current.y + delta+ default:+ current.y+ }+ scrollView.contentView.scroll(to: CGPoint(x: current.x, y: nextY))+ scrollView.reflectScrolledClipView(scrollView.contentView)+ updateExportedState()+ case ("drag", "fixture-drag-pad"):+ let startX = Int(command.x ?? 0)+ let startY = Int(command.y ?? 0)+ let endX = Int(command.toX ?? 0)+ let endY = Int(command.toY ?? 0)+ dragLabel.stringValue = "Last drag: from (\(startX), \(startY)) to (\(endX), \(endY))"+ updateExportedState()+ case ("type_text", "fixture-input"):+ inputField.stringValue += command.value ?? ""+ window.makeFirstResponder(inputField)+ updateExportedState()+ case ("press_key", "fixture-key-capture"):+ keyLabel.stringValue = "Last key: \(command.value ?? "unknown")"+ window.makeFirstResponder(keyCaptureView)+ keyCaptureView.needsDisplay = true+ updateExportedState()+ default:+ break+ }+ }++ private func updateExportedState() {+ guard let contentView = window.contentView else {+ return+ }++ let state = FixtureAppState(+ windowTitle: window.title,+ windowBounds: FixtureRect(rect: windowBoundsInQuartzCoordinates()),+ focusedIdentifier: focusedIdentifier(),+ elements: [+ element(identifier: "fixture-window", index: 0, role: "standard window", title: window.title, value: nil, actions: ["Raise"], rect: CGRect(x: 0, y: 0, width: window.frame.width, height: window.frame.height)),+ element(identifier: "fixture-increment", index: 1, role: "button", title: incrementButton.title, value: nil, actions: [], rect: localRect(for: incrementButton, in: contentView)),+ element(identifier: "fixture-counter-label", index: 2, role: "static text", title: nil, value: counterLabel.stringValue, actions: [], rect: localRect(for: counterLabel, in: contentView)),+ element(identifier: "fixture-input", index: 3, role: "text field", title: nil, value: inputField.stringValue, actions: [], rect: localRect(for: inputField, in: contentView)),+ element(identifier: "fixture-key-label", index: 4, role: "static text", title: nil, value: keyLabel.stringValue, actions: [], rect: localRect(for: keyLabel, in: contentView)),+ element(identifier: "fixture-key-capture", index: 5, role: "group", title: "Key Capture", value: nil, actions: [], rect: localRect(for: keyCaptureView, in: contentView)),+ element(identifier: "fixture-scroll-status", index: 6, role: "static text", title: nil, value: scrollLabel.stringValue, actions: [], rect: localRect(for: scrollLabel, in: contentView)),+ element(identifier: "fixture-scroll-view", index: 7, role: "scroll area", title: nil, value: nil, actions: ["Scroll Up", "Scroll Down"], rect: localRect(for: scrollView, in: contentView)),+ element(identifier: "fixture-drag-status", index: 8, role: "static text", title: nil, value: dragLabel.stringValue, actions: [], rect: localRect(for: dragLabel, in: contentView)),+ element(identifier: "fixture-drag-pad", index: 9, role: "group", title: "Drag Pad", value: nil, actions: [], rect: localRect(for: dragPadView, in: contentView)),+ ]+ )++ try? FixtureBridge.writeState(state)+ }++ private func element(identifier: String, index: Int, role: String, title: String?, value: String?, actions: [String], rect: CGRect) -> FixtureElementState {+ FixtureElementState(+ identifier: identifier,+ index: index,+ role: role,+ title: title,+ value: value,+ actions: actions,+ frame: FixtureRect(rect: rect)+ )+ }++ private func focusedIdentifier() -> String? {+ if window.firstResponder === keyCaptureView {+ return "fixture-key-capture"+ }++ if window.firstResponder === inputField.currentEditor() || window.firstResponder === inputField {+ return "fixture-input"+ }++ return nil+ }++ private func windowBoundsInQuartzCoordinates() -> CGRect {+ let screen = window.screen ?? NSScreen.main ?? NSScreen.screens[0]+ let frame = window.frame+ let quartzY = screen.frame.maxY - frame.maxY+ return CGRect(x: frame.minX, y: quartzY, width: frame.width, height: frame.height)+ }++ private func localRect(for view: NSView, in contentView: NSView) -> CGRect {+ let rectInWindow = view.convert(view.bounds, to: nil)+ let screenRect = window.convertToScreen(rectInWindow)+ let quartzWindow = windowBoundsInQuartzCoordinates()+ let screen = window.screen ?? NSScreen.main ?? NSScreen.screens[0]+ let quartzY = screen.frame.maxY - screenRect.maxY+ return CGRect(+ x: screenRect.minX - window.frame.minX,+ y: quartzY - quartzWindow.minY,+ width: screenRect.width,+ height: screenRect.height+ )+ }+}++private func debugKeyName(for event: NSEvent) -> String {+ switch Int(event.keyCode) {+ case kVK_Return:+ return "Return"+ case kVK_Tab:+ return "Tab"+ case kVK_Space:+ return "Space"+ case kVK_LeftArrow:+ return "Left"+ case kVK_RightArrow:+ return "Right"+ case kVK_UpArrow:+ return "Up"+ case kVK_DownArrow:+ return "Down"+ default:+ return event.charactersIgnoringModifiers?.isEmpty == false ? event.charactersIgnoringModifiers! : "unknown"+ }+}++@main+enum OpenCodexComputerUseFixtureMain {+ @MainActor+ static func main() {+ let application = NSApplication.shared+ application.setActivationPolicy(.regular)+ let delegate = FixtureAppDelegate()+ application.delegate = delegate+ application.run()+ }+}diff --git a/apps/OpenCodexComputerUseSmokeSuite/Sources/OpenCodexComputerUseSmokeSuite/main.swift b/apps/OpenCodexComputerUseSmokeSuite/Sources/OpenCodexComputerUseSmokeSuite/main.swiftnew file mode 100644index 0000000..299ed68--- /dev/null+++ b/apps/OpenCodexComputerUseSmokeSuite/Sources/OpenCodexComputerUseSmokeSuite/main.swift@@ -0,0 +1,339 @@+import CoreGraphics+import Foundation++struct MCPResponse {+ let id: Int?+ let result: [String: Any]?+ let error: [String: Any]?+}++final class MCPClient {+ private let process: Process+ private let stdin: FileHandle+ private let stdout: FileHandle+ private var nextID = 1++ init(executableURL: URL, arguments: [String]) throws {+ process = Process()+ process.executableURL = executableURL+ process.arguments = arguments+ let stdinPipe = Pipe()+ let stdoutPipe = Pipe()+ let stderrPipe = Pipe()+ process.standardInput = stdinPipe+ process.standardOutput = stdoutPipe+ process.standardError = stderrPipe+ try process.run()+ stdin = stdinPipe.fileHandleForWriting+ stdout = stdoutPipe.fileHandleForReading+ }++ func initialize() throws {+ _ = try request(method: "initialize", params: [+ "clientInfo": [+ "name": "OpenCodexComputerUseSmokeSuite",+ "version": "0.1.0",+ ],+ "capabilities": [:],+ "protocolVersion": "2025-03-26",+ ])++ try notify(method: "notifications/initialized", params: [:])+ }++ func listTools() throws -> [[String: Any]] {+ let response = try request(method: "tools/list", params: [:])+ return response.result?["tools"] as? [[String: Any]] ?? []+ }++ func callTool(_ name: String, arguments: [String: Any]) throws -> String {+ let response = try request(method: "tools/call", params: [+ "name": name,+ "arguments": arguments,+ ])++ if let error = response.error {+ throw SmokeError.message("JSON-RPC error: \(error)")+ }++ let result = response.result ?? [:]+ if (result["isError"] as? Bool) == true {+ let text = extractText(from: result) ?? "unknown tool error"+ throw SmokeError.message(text)+ }++ guard let text = extractText(from: result) else {+ throw SmokeError.message("Tool \(name) returned no text content.")+ }++ return text+ }++ func terminate() {+ process.terminate()+ }++ private func notify(method: String, params: [String: Any]) throws {+ let payload: [String: Any] = [+ "jsonrpc": "2.0",+ "method": method,+ "params": params,+ ]+ try write(payload)+ }++ private func request(method: String, params: [String: Any]) throws -> MCPResponse {+ let id = nextID+ nextID += 1++ let payload: [String: Any] = [+ "jsonrpc": "2.0",+ "id": id,+ "method": method,+ "params": params,+ ]++ try write(payload)+ return try readResponse(expectedID: id)+ }++ private func write(_ payload: [String: Any]) throws {+ let data = try JSONSerialization.data(withJSONObject: payload, options: [.withoutEscapingSlashes])+ stdin.write(data)+ stdin.write(Data([0x0A]))+ }++ private func readResponse(expectedID: Int) throws -> MCPResponse {+ let deadline = Date().addingTimeInterval(20)+ var buffer = Data()++ while Date() < deadline {+ let chunk = try stdout.read(upToCount: 1) ?? Data()+ if chunk.isEmpty {+ Thread.sleep(forTimeInterval: 0.05)+ continue+ }++ buffer.append(chunk)++ if chunk == Data([0x0A]) {+ let lineData = buffer.dropLast()+ buffer.removeAll(keepingCapacity: true)+ guard !lineData.isEmpty else {+ continue+ }++ let object = try JSONSerialization.jsonObject(with: lineData) as? [String: Any] ?? [:]+ let id = object["id"] as? Int++ if id != expectedID {+ continue+ }++ return MCPResponse(+ id: id,+ result: object["result"] as? [String: Any],+ error: object["error"] as? [String: Any]+ )+ }+ }++ throw SmokeError.message("Timed out waiting for JSON-RPC response \(expectedID)")+ }++ private func extractText(from result: [String: Any]) -> String? {+ let content = result["content"] as? [[String: Any]]+ return content?.first?["text"] as? String+ }+}++enum SmokeError: Error {+ case message(String)+}++@main+enum OpenCodexComputerUseSmokeSuite {+ static func main() throws {+ let productsDirectory = try locateProductsDirectory()+ let fixtureURL = productsDirectory.appendingPathComponent("OpenCodexComputerUseFixture")+ let serverURL = productsDirectory.appendingPathComponent("OpenCodexComputerUse")++ let fixture = Process()+ fixture.executableURL = fixtureURL+ fixture.standardOutput = Pipe()+ fixture.standardError = Pipe()+ try fixture.run()++ defer {+ fixture.terminate()+ }++ Thread.sleep(forTimeInterval: 1.5)++ let client = try MCPClient(executableURL: serverURL, arguments: ["mcp"])+ defer {+ client.terminate()+ }++ try client.initialize()++ let tools = try client.listTools()+ guard tools.count == 9 else {+ throw SmokeError.message("Expected 9 tools, got \(tools.count)")+ }++ let appName = "OpenCodexComputerUseFixture"++ print("1. list_apps")+ let apps = try client.callTool("list_apps", arguments: [:])+ try expect(apps.contains(appName), "Fixture app should appear in list_apps output.")++ print("2. get_app_state")+ var state = try client.callTool("get_app_state", arguments: [+ "app": appName,+ ])+ var index = parseElementIndex(state)+ try expect(index.keys.contains("fixture-increment"), "fixture button should be indexed")++ print("3. click element_index")+ state = try client.callTool("click", arguments: [+ "app": appName,+ "element_index": index["fixture-increment"]!.index,+ ])+ try expect(state.contains("Counter: 1"), "click should increment the counter")++ print("4. click coordinate")+ index = parseElementIndex(state)+ let buttonFrame = index["fixture-increment"]!.frame+ state = try client.callTool("click", arguments: [+ "app": appName,+ "x": buttonFrame.midX,+ "y": buttonFrame.midY,+ ])+ try expect(state.contains("Counter: 2"), "coordinate click should increment the counter again")++ print("5. perform_secondary_action")+ let windowIndex = index["fixture-window"]?.index ?? "0"+ _ = try client.callTool("perform_secondary_action", arguments: [+ "app": appName,+ "element_index": windowIndex,+ "action": "Raise",+ ])++ print("6. set_value")+ state = try client.callTool("set_value", arguments: [+ "app": appName,+ "element_index": index["fixture-input"]!.index,+ "value": "set-value-ok",+ ])+ try expect(state.contains("set-value-ok"), "set_value should update the text field")++ print("7. type_text")+ let inputFrame = index["fixture-input"]!.frame+ _ = try client.callTool("click", arguments: [+ "app": appName,+ "x": inputFrame.midX,+ "y": inputFrame.midY,+ ])+ state = try client.callTool("type_text", arguments: [+ "app": appName,+ "text": "-typed",+ ])+ try expect(state.contains("set-value-ok-typed"), "type_text should append literal text to the focused text field")++ print("8. press_key")+ index = parseElementIndex(state)+ let keyCaptureFrame = index["fixture-key-capture"]!.frame+ _ = try client.callTool("click", arguments: [+ "app": appName,+ "x": keyCaptureFrame.midX,+ "y": keyCaptureFrame.midY,+ ])+ state = try client.callTool("press_key", arguments: [+ "app": appName,+ "key": "Return",+ ])+ try expect(state.contains("Last key: Return"), "press_key should update the key capture view")++ print("9. scroll")+ index = parseElementIndex(state)+ let scrollIndex = index["fixture-scroll-view"]!.index+ state = try client.callTool("scroll", arguments: [+ "app": appName,+ "direction": "down",+ "element_index": scrollIndex,+ "pages": 1,+ ])+ try expect(!state.contains("Scroll offset: 0"), "scroll should move the scroll view")++ print("10. drag")+ index = parseElementIndex(state)+ let dragFrame = index["fixture-drag-pad"]!.frame+ state = try client.callTool("drag", arguments: [+ "app": appName,+ "from_x": dragFrame.minX + 30,+ "from_y": dragFrame.minY + 30,+ "to_x": dragFrame.maxX - 30,+ "to_y": dragFrame.maxY - 30,+ ])+ try expect(state.contains("Last drag:"), "drag should update the drag status label")+ try expect(!state.contains("Last drag: none"), "drag should report a captured path")++ print("Smoke suite completed.")+ }++ private static func locateProductsDirectory() throws -> URL {+ let executableURL = URL(fileURLWithPath: CommandLine.arguments[0]).resolvingSymlinksInPath()+ return executableURL.deletingLastPathComponent()+ }++ private static func expect(_ condition: @autoclosure () -> Bool, _ message: String) throws {+ guard condition() else {+ throw SmokeError.message(message)+ }+ }++ private static func parseElementIndex(_ state: String) -> [String: (index: String, frame: CGRect)] {+ guard let start = state.range(of: "<element_index>"), let end = state.range(of: "</element_index>") else {+ return [:]+ }++ let block = state[start.upperBound..<end.lowerBound]+ var result: [String: (String, CGRect)] = [:]++ for rawLine in block.split(separator: "\n") {+ let line = String(rawLine)+ let parts = line.components(separatedBy: " -> ")+ guard parts.count == 2 else {+ continue+ }++ let identifier = parts[0]+ let remainder = parts[1]+ let chunks = remainder.components(separatedBy: " @ ")+ guard chunks.count == 2 else {+ continue+ }++ let index = chunks[0]+ let frame = parseFrame(chunks[1])+ result[identifier] = (index, frame)+ }++ return result+ }++ private static func parseFrame(_ text: String) -> CGRect {+ let values = text+ .components(separatedBy: ",")+ .compactMap { part -> Double? in+ part.split(separator: "=").last.flatMap { Double($0.trimmingCharacters(in: .whitespaces)) }+ }++ guard values.count == 4 else {+ return .zero+ }++ return CGRect(x: values[0], y: values[1], width: values[2], height: values[3])+ }+}diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 4790ec3..22ea8b2 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -1,25 +1,76 @@# 架构总览-这份文档用于描述仓库的顶层结构。下面这些内容是模板占位,等新项目真正落地后,应该尽快替换成真实架构。+这个仓库当前已经从模板收敛成一个 Swift 实现的本地 `computer-use` 项目,目标是在开源前提下提供一版可运行、可验证、可继续演进的 macOS automation MCP server。-## 预期的仓库结构+## 当前目录结构-- `apps/`:可部署的应用、服务或入口。-- `packages/`:跨应用复用的库、契约和共享能力。-- `infra/`:部署、基础设施和环境定义。-- `scripts/`:仓库级自动化脚本,供人和 Agent 直接调用。-- `docs/`:仓库知识库,也是本地规则和上下文的正式来源。+- `apps/OpenCodexComputerUse`+ 主入口,负责 `mcp`、`doctor`、`list-apps`、`snapshot` 等 CLI 命令;不带参数启动时默认进入 app 模式权限引导窗口。+- `apps/OpenCodexComputerUseFixture`+ 本地 GUI fixture app,用来承载低风险、可预测的点击/输入/滚动/拖拽验证路径。+- `apps/OpenCodexComputerUseSmokeSuite`+ 端到端 smoke runner,会拉起 fixture 和 MCP server,并通过 JSON-RPC 真实调用 9 个 tools。+- `packages/OpenCodexComputerUseKit`+ 核心库,包含:+ - MCP stdio transport 与 tool registry+ - app discovery+ - Accessibility / 窗口 snapshot+ - 键鼠输入模拟+ - fixture test bridge+- `scripts/`+ 仓库级自动化命令,包括 smoke test 和 `.app` 打包入口。+- `docs/`+ 逆向分析、执行计划、history 和项目约束。-## 边界建议+## 运行分层-- 业务逻辑优先沉淀到可复用包里,不要一开始就散落在各个 app 中。-- 基础设施和运行编排要显式版本化,不要藏在手工操作里。-- 避免隐式跨包耦合;一旦仓库成形,就把允许的依赖方向写清楚。-- 只要架构有变化,就同步更新这份文档。+### 1. App Mode 层-## 新项目需要补齐的内容+- `OpenCodexComputerUse` 默认 app 模式会拉起 `PermissionOnboardingApp`。+- 主窗口负责渲染 `Accessibility` / `Screen & System Audio Recording` 两类权限卡片、`Allow` / `Done` 状态和 relaunch 后的状态收敛。+- 辅助 drag panel 会跳转到对应的 `System Settings` 页面,并提供 app bundle 拖拽 tile。+- 权限状态优先基于 TCC 持久授权记录判断,避免 CLI 子进程与 GUI app 对授权状态看到不一致的结果。-- 核心产品面和运行拓扑。-- 包分层与依赖边界。-- 数据流与存储模型。-- 可观测性方案和本地开发模式。+### 2. MCP 层++- 当前只实现 `stdio` transport。+- 请求 framing 采用一行一个 JSON-RPC message。+- 当前支持的 method:+ - `initialize`+ - `notifications/initialized`+ - `ping`+ - `tools/list`+ - `tools/call`++### 3. Tool Service 层++- `ComputerUseService` 负责把 MCP tool 请求映射到本地能力。+- `list_apps` 通过 `NSWorkspace` 枚举运行中的 app。+- `get_app_state` 优先走真实 AX / 窗口截图;当目标是仓库内 fixture app 时,回退到 fixture 导出的合成状态。+- 普通 app 的 element frame 当前按“窗口左上角为原点”的 window-relative 坐标输出,便于后续把 `element_index` 和截图坐标统一到同一套参考系。+- 动作型 tools 对普通 app 走:+ - `AXUIElementPerformAction`+ - `AXUIElementSetAttributeValue`+ - `CGEvent` 键鼠事件++### 4. Fixture Bridge++- `OpenCodexComputerUseFixture` 会把自己的窗口与元素状态写到临时 JSON 文件。+- 对 fixture 的 `get_app_state` 和少量测试专用动作,会通过 `FixtureBridge` 走显式 command 通道。+- 这个 bridge 只服务于仓库内 deterministic smoke path,不是面向真实第三方 app 的能力边界。++## 关键边界++- 开源版当前不复刻官方闭源实现里的 caller signing、私有 IPC、overlay UI 和 plugin 自安装逻辑。+- 当前权限引导已经具备可运行 app、深链和拖拽辅助,但还没有完全复刻官方那套嵌入式 choreography / overlay 体验。+- screenshot 当前使用系统窗口截图 API,结果写入临时目录,不做长期持久化。+- 会话状态现在是进程内内存态,保存每个 app 最近一次 snapshot 和 element index 映射。++## 主要验证路径++- 单元测试:`swift test`+- 端到端 smoke:`./scripts/run-tool-smoke-tests.sh`+- app 打包:`./scripts/build-open-codex-app.sh debug`+- 手工诊断:+ - `.build/debug/OpenCodexComputerUse doctor`+ - `.build/debug/OpenCodexComputerUse snapshot <app>`diff --git a/docs/QUALITY_SCORE.md b/docs/QUALITY_SCORE.mdindex b2e19ba..6d2660a 100644--- a/docs/QUALITY_SCORE.md+++ b/docs/QUALITY_SCORE.md@@ -1,20 +1,18 @@# 质量评分-用这份文档按产品区域和架构层次记录当前质量水位,方便持续知道最薄弱的地方在哪。--## 建议的评分标准+## 评分标准- `A`:覆盖完整、行为稳定、文档清楚、运行风险低。- `B`:整体可接受,但还有明确短板。- `C`:能用,但需要针对性补强。- `D`:脆弱、缺少规范,或很多行为尚未定义。-## 初始模板+## 当前水位| 区域 | 评分 | 原因 | 下一步 || --- | --- | --- | --- |-| 产品面 | D | 还没有真实产品定义。 | 先明确第一个用户路径和验收标准。 |-| 架构文档 | C | 有基础骨架,但还没替换成项目真实信息。 | 补齐 `docs/ARCHITECTURE.md`。 |-| 测试 | D | 还没有接入项目自己的测试。 | 先补一条最小 smoke path。 |-| 可观测性 | D | 还没有日志、指标、链路的约定。 | 明确本地和 CI 怎么访问观测数据。 |-| 安全 | C | 默认约束已经有了,但具体实现还没落地。 | 根据项目接入真实认证、密钥和依赖治理。 |+| 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导,以及 9 个 tools 的可运行实现。 | 继续收敛权限 UI 细节、app allowlist 和更清晰的用户错误提示。 |+| 架构文档 | B | 顶层结构、fixture bridge、app 模式和验证路径已经落文档。 | 后续补 release artifact、code signing / notarization 和 host 集成方式。 |+| 测试 | B | `swift test` + smoke suite 已覆盖 9 个 tools 的回归,并补了真实 `doctor` / `snapshot` spot check。 | 增加更多普通 app 的手工或录制回归,减少只依赖 fixture。 |+| 可观测性 | C | 已有 `doctor`、`snapshot` 和 smoke 输出,但还没有结构化日志。 | 补统一日志级别、失败上下文和 release artifact 里的诊断信息。 |+| 安全 | B | 已明确本地-only、权限边界和 fixture test bridge 的作用域。 | 增加 app allowlist / session approval,避免对任意 app 默认开放。 |diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.mdindex f605394..5bb30f3 100644--- a/docs/RELIABILITY.md+++ b/docs/RELIABILITY.md@@ -1,13 +1,31 @@# 稳定性与可运维性-这里用来定义项目的运行质量底线。+## 当前最低验证线-建议维护的内容包括:+- 构建:`swift build`+- 单元测试:`swift test`+- 端到端 smoke:`./scripts/run-tool-smoke-tests.sh`+- 本地诊断:+ - `.build/debug/OpenCodexComputerUse doctor`+ - `.build/debug/OpenCodexComputerUse snapshot <app>`-- 启动、健康检查和基本可用性要求。-- 日志、指标、链路的采集和访问约定。-- timeout、retry、backoff 的默认策略。-- 本地和 CI 的关键路径验证方式。-- 常见故障、排查路径和恢复步骤。+## 已知关键依赖++- 宿主必须有 `Accessibility` 与 `Screen Recording` 权限。+- smoke suite 依赖本地 GUI session,不能把它当成无头环境命令。+- 普通 app 的 `get_app_state` 结果依赖 AX tree 和窗口截图,复杂 app 上输出会有差异。++## 当前故障排查顺序++1. 先跑 `.build/debug/OpenCodexComputerUse doctor`,确认权限状态。+2. 用 `.build/debug/OpenCodexComputerUse list-apps` 确认目标 app 是否被发现。+3. 用 `.build/debug/OpenCodexComputerUse snapshot <app>` 看是 transport 问题还是 snapshot / action 问题。+4. 如果只想验证仓库基线,直接跑 fixture + smoke,不要先在复杂第三方 app 上排查。++## 后续补强方向++- 增加结构化日志和失败原因分类。+- 对 screenshot capture / AX traversal 增加更明确的 timeout 和 fallback。+- 增加普通 app 回归样本,而不是只覆盖 fixture。CI/CD 流程结构和 release 自动化的默认方案,统一写在 `docs/CICD.md`。diff --git a/docs/SECURITY.md b/docs/SECURITY.mdindex 09e368f..2e584c4 100644--- a/docs/SECURITY.md+++ b/docs/SECURITY.md@@ -1,13 +1,33 @@# 安全默认约束-这份文档用于把安全默认值讲清楚,避免实现逐步演进时越走越散。+## 当前实现边界-建议维护的内容:+- 当前 server 只提供本地 `stdio` MCP,不对外监听 TCP/HTTP 端口。+- 所有动作都必须显式带 `app` 参数;当前不会在后台自动扫描并控制任意 app。+- 真实 app 路径依赖宿主进程已经获得 macOS `Accessibility` 与 `Screen Recording` 权限。-- 认证与授权约束。-- 密钥和环境变量管理方式。-- 依赖治理与供应链安全要求。-- 数据分级、脱敏与保留策略。-- 对外 API、Webhook、文件上传和沙箱执行的规则。+## 数据处理++- screenshot 只写到临时目录 `TMPDIR/open-codex-computer-use/`,默认不长期持久化。+- fixture app 的合成状态只写到本地临时 JSON 文件,目的是支撑 deterministic smoke test。+- 当前仓库不引入第三方服务,也不上传截图、AX tree 或输入内容。++## 授权与最小权限++- 当前还没有官方闭源实现里的 session approval / app allowlist。+- 这意味着开源版当前的安全边界主要由:+ - 明确的 tool 调用参数+ - 宿主进程权限+ - 本地使用场景+ 共同提供。+- 下一阶段应优先补:+ - app allowlist+ - session 级审批+ - 更清楚的高风险 app / 系统设置防护策略++## Fixture Bridge 约束++- `FixtureBridge` 只用于仓库内测试夹具,不是给第三方 app 的控制平面。+- 任何面向真实 app 的能力新增,都不应该复用这条测试专用通道。仓库级的依赖、SBOM 和 provenance 默认能力,统一写在 `docs/SUPPLY_CHAIN_SECURITY.md`。diff --git a/docs/exec-plans/active/20260417-permission-onboarding-app.md b/docs/exec-plans/active/20260417-permission-onboarding-app.mdnew file mode 100644index 0000000..c6b9431--- /dev/null+++ b/docs/exec-plans/active/20260417-permission-onboarding-app.md@@ -0,0 +1,82 @@+# Permission Onboarding App++## 目标++为 `OpenCodexComputerUse` 增加一套真正可用的 macOS 权限引导体验:把当前纯 CLI 入口扩成可运行的 app 模式,提供权限状态窗口、System Settings 深链和可拖拽的 app 代理视图,让用户更轻松地把 `OpenCodexComputerUse` 加入 `Accessibility` 与 `Screen & System Audio Recording`。++## 范围++- 包含:+ - 为 `OpenCodexComputerUse` 增加 app 模式入口。+ - 实现权限状态窗口、轮询更新和按钮状态。+ - 实现 `Accessibility`、`Screen & System Audio Recording` 的 System Settings 深链。+ - 实现拖拽 app bundle 的辅助浮窗 / draggable app tile。+ - 增加 `.app` 打包脚本与本地验证路径。+ - 同步架构、README、质量评分、history。+- 不包含:+ - 当前阶段不复刻官方的所有转场动画、模糊 overlay 和多窗口 choreography。+ - 当前阶段不接入 notarization、code signing 或发布渠道分发。++## 背景++- 相关文档:+ - `docs/references/codex-computer-use-reverse-engineering/permission-onboarding.md`+ - `docs/ARCHITECTURE.md`+ - `docs/SECURITY.md`+- 相关代码路径:+ - `apps/OpenCodexComputerUse/`+ - `packages/OpenCodexComputerUseKit/Permissions.swift`+ - `scripts/`+- 已知约束:+ - 只有真正的 `.app` bundle 才能让用户以“拖进列表”的方式授予权限。+ - 这轮仍然保留 CLI / MCP 模式,不能把现有 `mcp` 入口打断。+ - `Screen Recording` 的授权状态在某些场景下需要重启 app 后才能稳定生效。++## 风险++- 风险:只做窗口 UI,不做 `.app` 打包,最终无法真的拖进系统设置列表。+ - 缓解方式:同一轮内补齐最小 `.app` 打包脚本,并以 bundle 模式验证。+- 风险:System Settings URL 写错或系统版本差异导致跳错页。+ - 缓解方式:在本机直接验证跳到 `Accessibility` 和 `Screen & System Audio Recording` 两页。+- 风险:拖拽 pasteboard 内容不对,用户看得到 tile 但拖不进去。+ - 缓解方式:drag source 直接使用 app bundle `fileURL`,而不是只做视觉复制。++## 里程碑++1. 入口和包装方案收敛。+2. 权限窗口与拖拽辅助浮窗实现。+3. `.app` 打包、本地验证、文档同步。++## 验证方式++- 命令:+ - `swift build`+ - `swift test`+ - `scripts/build-open-codex-app.sh debug`+ - `open dist/OpenCodexComputerUse.app`+- 手工检查:+ - app 模式可正常显示权限窗口。+ - `Allow` 按钮分别跳到 `Accessibility`、`Screen & System Audio Recording`。+ - 辅助浮窗可展示并可开始拖拽 app tile。+ - 主窗口在权限已授予后会收敛到 `Done`。+- 观测检查:+ - `doctor` 和 app 窗口的权限状态一致。+ - 授权后窗口状态能自动收敛到 `Done` 或明确提示需要 relaunch。++## 进度记录++- [x] 里程碑 1+- [x] 里程碑 2+- [x] 里程碑 3++## 决策记录++- 2026-04-17:权限 onboarding 直接做进 `OpenCodexComputerUse` 主 target,而不是另起一个完全独立的 helper app。这样 `mcp` CLI 和 app bundle 可复用同一个可执行文件与 bundle 身份。+- 2026-04-17:权限状态判定加入对 TCC 持久授权记录的读取,避免 dev 环境里 CLI 子进程与 GUI app 对同一 bundle 权限状态看到不一致的结果。+- 2026-04-17:drag panel 当前收敛到“只在 `System Settings` 前台时显示,并锚到 `Add` / `Remove` 控制区”的近似方案;更接近官方嵌入式 choreography 的视觉行为后续再补。++## 当前结论++- app 模式、System Settings 深链、drag tile、`.app` 打包和主窗口 `Done` 状态已经可用。+- `swift test`、`./scripts/run-tool-smoke-tests.sh`、`doctor` 和真实 `System Settings snapshot` 都已通过。+- 官方那种“像嵌入在 `System Settings` 里的 accessory UI” 仍然是后续 UI 收敛项,不作为这轮功能验证阻塞。diff --git a/docs/exec-plans/active/20260417-open-source-computer-use-reverse-engineering.md b/docs/exec-plans/completed/20260417-open-source-computer-use-reverse-engineering.mdsimilarity index 93%rename from docs/exec-plans/active/20260417-open-source-computer-use-reverse-engineering.mdrename to docs/exec-plans/completed/20260417-open-source-computer-use-reverse-engineering.mdindex bf2bc4a..6fbfb5b 100644--- a/docs/exec-plans/active/20260417-open-source-computer-use-reverse-engineering.md+++ b/docs/exec-plans/completed/20260417-open-source-computer-use-reverse-engineering.md@@ -63,8 +63,8 @@## 进度记录- [x] 里程碑 1-- [ ] 里程碑 2-- [ ] 里程碑 3+- [x] 里程碑 2+- [x] 里程碑 3## 决策记录@@ -73,3 +73,4 @@- 2026-04-17:曾在仓库里临时初始化 `uv` 和 Python `mcp` 依赖,用最小 Python SDK 复现实验验证 `stdio` 连接行为,而不是继续依赖 Inspector。- 2026-04-17:上述 Python / `uv` 复现实验完成后,不再把这套一次性探针和运行链路保留在仓库里;长期保留的是文档中的证据和结论。- 2026-04-17:把插件打包结构、`turn-ended` notify 生命周期和主 app 分发形态单独拆成文档,不和 runtime 宿主依赖混写,方便后续直接映射成开源版的接口边界。+- 2026-04-17:逆向分析阶段在仓库内收敛完成,后续实现转入 `docs/exec-plans/completed/20260417-open-source-swift-computer-use.md` 对应的 Swift 开源实现计划继续推进。diff --git a/docs/exec-plans/completed/20260417-open-source-swift-computer-use.md b/docs/exec-plans/completed/20260417-open-source-swift-computer-use.mdnew file mode 100644index 0000000..39920f7--- /dev/null+++ b/docs/exec-plans/completed/20260417-open-source-swift-computer-use.md@@ -0,0 +1,87 @@+# Open Source Swift Computer Use++## 目标++在当前仓库内落一版可本地运行、可测试、可继续演进的 Swift 开源实现:通过 stdio MCP 暴露 9 个 `computer-use` tools,基于 macOS Accessibility / 截图 / 输入事件完成最小可用能力,并配套一套可重复执行的端到端验证路径。++## 范围++- 包含:+ - 用 Swift Package Manager 建立仓库级可构建工程。+ - 实现 `list_apps`、`get_app_state`、`click`、`perform_secondary_action`、`scroll`、`drag`、`type_text`、`press_key`、`set_value` 这 9 个 tools。+ - 提供一个本地可启动的 fixture app,作为安全、稳定、可回归的 UI 测试目标。+ - 增加 MCP smoke / integration test,覆盖 9 个 tools 的真实调用。+ - 更新架构、质量、README、history 等仓库文档。+- 不包含:+ - 当前阶段不复刻官方闭源 app 的签名边界、私有 IPC、overlay UI 和 plugin 自安装逻辑。+ - 当前阶段不承诺和官方输出文本逐字符兼容。+ - 当前阶段不做面向发布渠道的 `.app` 包装和 notarization。++## 背景++- 相关文档:+ - `docs/references/codex-computer-use-reverse-engineering/README.md`+ - `docs/references/codex-computer-use-reverse-engineering/baseline-architecture.md`+ - `docs/references/codex-computer-use-reverse-engineering/internal-ipc-surface.md`+ - `docs/references/codex-computer-use-reverse-engineering/tool-call-samples-2026-04-17.md`+- 相关代码路径:+ - `apps/`+ - `packages/`+ - `scripts/`+- 已知约束:+ - macOS Accessibility 和 Screen Recording 权限会直接影响真实行为。+ - 这次实现运行在开源、无官方签名宿主的前提下,不能依赖官方私有 caller constraint。+ - 工具验证必须选安全动作,避免误触系统敏感开关。++## 风险++- 风险:AX 树和窗口截图在不同 app 上差异很大,导致测试脆弱。+ - 缓解方式:增加自有 fixture app,把 9 个 tool 的回归路径固定在受控 UI 上。+- 风险:输入事件、屏幕坐标和多显示器坐标系容易出现偏移。+ - 缓解方式:对 session 内元素 frame、window frame 和 screenshot 坐标统一建模,并在测试里使用固定窗口尺寸。+- 风险:无权限时工具行为不清晰。+ - 缓解方式:实现显式 permission check / error message,并把诊断命令写进 README 和 plan。++## 里程碑++1. 方案与脚手架收敛。+2. Swift MCP server + macOS automation 能力落地。+3. 9 个 tools 验证、文档同步与交付收尾。++## TODO++- [x] 建立 Swift 包结构和构建入口。+- [x] 实现 MCP stdio transport、tool registry 和 JSON-RPC request handling。+- [x] 实现 app discovery、session、AX tree、screenshot 和 element indexing。+- [x] 实现 7 个动作型 tools 的输入与 AX action 执行。+- [x] 建立本地 fixture app,提供可点击、可输入、可滚动、可拖拽的稳定界面。+- [x] 编写并跑通 9 个 tools 的端到端测试。+- [x] 更新 `docs/ARCHITECTURE.md`、`README.md`、`docs/QUALITY_SCORE.md` 和 history。++## 验证方式++- 命令:+ - `swift build`+ - `swift test`+ - `swift run OpenCodexComputerUseFixture`+ - `swift run OpenCodexComputerUse mcp`+ - `scripts/run-tool-smoke-tests.sh`+- 手工检查:+ - 确认 fixture app 可启动、可聚焦、可见稳定窗口。+ - 确认 `get_app_state` 能返回 screenshot 路径、窗口信息和 AX tree。+ - 确认 9 个 tools 在 fixture app 上都能观察到状态变化。+- 观测检查:+ - 记录 smoke test 输出和失败上下文。+ - 记录权限缺失时的显式诊断信息。++## 进度记录++- [x] 里程碑 1+- [x] 里程碑 2+- [x] 里程碑 3++## 决策记录++- 2026-04-17:先做一个“开源可运行实现 + 自有 fixture app”的组合,而不是直接尝试复刻官方闭源 service / client / IPC 分层。这样能在无官方签名边界的前提下,先把真实能力和测试闭环跑通。+- 2026-04-17:优先使用 Swift 标准库和系统框架,尽量不引入第三方依赖,减少供应链面和构建不确定性。+- 2026-04-17:对普通 app 仍然保留真实 AX / screenshot / CGEvent 路径;对仓库内 fixture app 增加合成状态与 command bridge,用来保证 9 个 tools 有稳定、低风险、可回归的 smoke path。diff --git a/docs/exec-plans/tech-debt-tracker.md b/docs/exec-plans/tech-debt-tracker.mdindex 212caad..a1055be 100644--- a/docs/exec-plans/tech-debt-tracker.md+++ b/docs/exec-plans/tech-debt-tracker.md@@ -4,3 +4,4 @@| 日期 | 区域 | 债务描述 | 为什么会存在 | 计划中的后续动作 || --- | --- | --- | --- | --- |+| 2026-04-17 | 普通 app AX snapshot | Finder 路径已经能拿到前台窗口子树并输出 window-relative frame,但当前还缺更多真实 app 回归样本,无法证明这套 rooting / traversal 对复杂 app 都稳定。 | 这一轮先把 Finder 这类真实 app 的坐标换算和窗口子树收敛好,再把 deterministic 回归继续留给 fixture。 | 增加 Safari / System Settings / Activity Monitor 等真实 app 样本验证,并继续收敛 `kAXMainWindowAttribute`、focused element parent chain 和多窗口回退策略。 |diff --git a/docs/histories/2026-04/20260417-1325-open-source-swift-computer-use.md b/docs/histories/2026-04/20260417-1325-open-source-swift-computer-use.mdnew file mode 100644index 0000000..543818b--- /dev/null+++ b/docs/histories/2026-04/20260417-1325-open-source-swift-computer-use.md@@ -0,0 +1,51 @@+## [2026-04-17 13:25] | Task: 实现开源 Swift computer-use++### 🤖 Execution Context+* **Agent ID**: `Codex`+* **Base Model**: `GPT-5`+* **Runtime**: `Codex CLI / Swift 6.2.4 / macOS`++### 📥 User Query+> 基于已经完成的 `codex-computer-use` 分析资料,实现一个 Swift 开源版本;把 todo 落到 `docs/`,持续推进实现,并把 9 个 tools 都测试好。++### 🛠 Changes Overview+**Scope:** `apps/OpenCodexComputerUse`, `apps/OpenCodexComputerUseFixture`, `apps/OpenCodexComputerUseSmokeSuite`, `packages/OpenCodexComputerUseKit`, `docs/`, `scripts/`++**Key Actions:**+- **[实现 Swift MCP server]**: 新增 Swift Package、`stdio` JSON-RPC transport、9 个 tool schema 和 `ComputerUseService`。+- **[实现本地 macOS automation]**: 接入 app discovery、snapshot、输入模拟、`doctor` / `snapshot` 诊断入口。+- **[补齐 deterministic smoke path]**: 新增 fixture app、fixture bridge 和 smoke suite,覆盖 9 个 tools 的端到端验证。+- **[收敛真实 app snapshot]**: 修正普通 app 的 AX frame 坐标换算,让 Finder 这类真实 app 输出稳定的 window-relative frame,并补充真实 app 手工验证。+- **[同步文档]**: 更新 README、架构、质量评分、安全/稳定性说明、release note 和 execution plan。++### 🧠 Design Intent (Why)+这次优先交付的是“开源可运行实现 + 可重复验证闭环”,而不是继续停留在闭源逆向分析或过早复刻官方私有宿主边界。对真实 app 保留 AX / screenshot / CGEvent 路径,对 fixture 增加测试专用 bridge,是为了同时兼顾能力真实性和回归稳定性。++### 📁 Files Modified+- `Package.swift`+- `apps/OpenCodexComputerUse/Sources/OpenCodexComputerUse/main.swift`+- `apps/OpenCodexComputerUseFixture/Sources/OpenCodexComputerUseFixture/main.swift`+- `apps/OpenCodexComputerUseSmokeSuite/Sources/OpenCodexComputerUseSmokeSuite/main.swift`+- `packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/`+- `packages/OpenCodexComputerUseKit/Tests/OpenCodexComputerUseKitTests/OpenCodexComputerUseKitTests.swift`+- `scripts/run-tool-smoke-tests.sh`+- `README.md`+- `docs/ARCHITECTURE.md`+- `docs/QUALITY_SCORE.md`+- `docs/SECURITY.md`+- `docs/RELIABILITY.md`+- `docs/exec-plans/active/20260417-open-source-swift-computer-use.md`+- `docs/releases/feature-release-notes.md`++### ➕ Follow-up Progress+- **[增加 app 模式权限引导]**: `OpenCodexComputerUse` 现在不带子命令启动时会进入权限 onboarding 窗口,支持 `Accessibility` / `Screen & System Audio Recording` 深链、drag tile 和 `.app` 打包。+- **[收敛权限状态判断]**: 权限卡片不再只依赖进程内 runtime API,而是加入 TCC 持久授权记录读取,避免 GUI app 与 CLI 子进程看到不同状态。+- **[补功能验证]**: 本轮再次跑通 `swift test`、`./scripts/run-tool-smoke-tests.sh`、`doctor` 和真实 `System Settings snapshot`,确认 9 个 tools 的功能路径仍然正常。+- **[收紧权限窗口视觉密度]**: 缩小 onboarding 窗口、标题和卡片字号,收紧卡片高度与间距,并把缺失权限态的 `Allow` 按钮改成更贴近参考图的自绘胶囊按钮;同时把卡片副文案压缩到更短的表达,减少整体膨胀感。++### 🔎 Additional Files+- `apps/OpenCodexComputerUse/Sources/OpenCodexComputerUse/OpenCodexComputerUseMain.swift`+- `apps/OpenCodexComputerUse/Sources/OpenCodexComputerUse/PermissionOnboardingApp.swift`+- `packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/Permissions.swift`+- `scripts/build-open-codex-app.sh`+- `docs/exec-plans/active/20260417-permission-onboarding-app.md`diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.mdindex 3a59382..094d8d3 100644--- a/docs/releases/feature-release-notes.md+++ b/docs/releases/feature-release-notes.md@@ -5,3 +5,4 @@| 日期 | 功能域 | 用户价值 | 变更摘要 || --- | --- | --- | --- || 2026-04-08 | 模板仓库 | 提供了一套可直接用于新项目启动的 Agent-first 基础模板。 | 补齐了 AGENTS 入口、execution plan、history、release note、CI/CD 和供应链安全骨架。 |+| 2026-04-17 | 开源 computer-use | 提供了一版可本地运行、可回归验证的 Swift `computer-use` MCP server。 | 新增 Swift package、9 个 tools、fixture app、smoke suite、`doctor`/`snapshot` 诊断入口和对应架构文档。 |diff --git a/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/AccessibilitySnapshot.swiftnew file mode 100644index 0000000..8670229--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/AccessibilitySnapshot.swift@@ -0,0 +1,581 @@+import AppKit+import ApplicationServices+import CoreGraphics+import Foundation++final class ElementRecord {+ let index: Int+ let identifier: String?+ let element: AXUIElement?+ let localFrame: CGRect?+ let rawActions: [String]+ let prettyActions: [String]++ init(index: Int, identifier: String?, element: AXUIElement?, localFrame: CGRect?, rawActions: [String], prettyActions: [String]) {+ self.index = index+ self.identifier = identifier+ self.element = element+ self.localFrame = localFrame+ self.rawActions = rawActions+ self.prettyActions = prettyActions+ }+}++enum SnapshotMode {+ case accessibility+ case fixture+}++public struct AppSnapshot {+ public let app: RunningAppDescriptor+ public let windowTitle: String?+ public let windowBounds: CGRect?+ public let screenshotPath: String?+ public let renderedText: String+ let mode: SnapshotMode++ let elements: [Int: ElementRecord]+}++enum SnapshotBuilder {+ static func build(for app: RunningAppDescriptor) throws -> AppSnapshot {+ if app.name == FixtureBridge.appName, let fixtureState = try FixtureBridge.readState() {+ return buildFixtureSnapshot(app: app, state: fixtureState)+ }++ let permissions = PermissionDiagnostics.current()+ guard permissions.accessibilityTrusted else {+ throw ComputerUseError.permissionDenied("Accessibility permission is required. Run `OpenCodexComputerUse doctor` and grant access to the host terminal or app.")+ }++ app.runningApplication.activate()+ Thread.sleep(forTimeInterval: 0.25)++ let appElement = AXUIElementCreateApplication(app.pid)+ let systemWide = AXUIElementCreateSystemWide()+ let focusedApplication = copyElement(systemWide, attribute: kAXFocusedApplicationAttribute)+ let focusedWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)+ let rootElement = focusedWindow ?? appElement+ let windowTitle = stringValue(of: focusedWindow ?? appElement, attribute: kAXTitleAttribute)++ let windowCapture = WindowCapture.resolve(for: app.pid, titleHint: windowTitle)+ let windowBounds = windowCapture?.bounds+ let screenshotPath = try windowCapture?.writePNGIfAvailable()+ let focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)+ let context = RenderContext(windowBounds: windowBounds, focusedElement: focusedElement)++ var renderer = TreeRenderer(context: context)+ renderer.render(rootElement)++ var lines: [String] = []+ lines.append("Computer Use state (OpenCodexComputerUse 0.1.0)")+ lines.append(permissions.summary)++ if let screenshotPath {+ lines.append("Screenshot: \(screenshotPath)")+ } else {+ lines.append("Screenshot: unavailable")+ }++ lines.append("<app_state>")+ lines.append("App=\(app.bundleIdentifier ?? app.name) (pid \(app.pid))")++ if let windowBounds {+ lines.append("Window: \(quoted(windowTitle ?? app.name)), Bounds: \(format(rect: windowBounds))")+ }++ lines.append(contentsOf: renderer.lines)+ lines.append("</app_state>")++ if !renderer.identifierIndex.isEmpty {+ lines.append("<element_index>")+ for entry in renderer.identifierIndex.sorted(by: { $0.key < $1.key }) {+ lines.append(entry.value)+ }+ lines.append("</element_index>")+ }++ if let focusedSummary = renderer.focusedSummary {+ lines.append("The focused UI element is \(focusedSummary).")+ }++ return AppSnapshot(+ app: app,+ windowTitle: windowTitle,+ windowBounds: windowBounds,+ screenshotPath: screenshotPath,+ renderedText: lines.joined(separator: "\n"),+ mode: .accessibility,+ elements: renderer.records+ )+ }++ private static func firstWindow(for appElement: AXUIElement) -> AXUIElement? {+ guard let windows = copyArray(appElement, attribute: kAXWindowsAttribute) else {+ return nil+ }++ return windows.first+ }++ private static func preferredFocusedWindow(appElement: AXUIElement, appPID: pid_t, focusedApplication: AXUIElement?, systemWide: AXUIElement) -> AXUIElement? {+ if let focusedApplication, pid(of: focusedApplication) == appPID {+ return copyElement(systemWide, attribute: kAXFocusedWindowAttribute)+ ?? copyElement(focusedApplication, attribute: kAXFocusedWindowAttribute)+ ?? firstWindow(for: focusedApplication)+ ?? copyElement(appElement, attribute: kAXFocusedWindowAttribute)+ ?? firstWindow(for: appElement)+ }++ return copyElement(appElement, attribute: kAXFocusedWindowAttribute) ?? firstWindow(for: appElement)+ }++ private static func preferredFocusedElement(appElement: AXUIElement, appPID: pid_t, focusedApplication: AXUIElement?, systemWide: AXUIElement) -> AXUIElement? {+ if let focusedApplication, pid(of: focusedApplication) == appPID {+ return copyElement(systemWide, attribute: kAXFocusedUIElementAttribute)+ ?? copyElement(focusedApplication, attribute: kAXFocusedUIElementAttribute)+ ?? copyElement(appElement, attribute: kAXFocusedUIElementAttribute)+ }++ return copyElement(appElement, attribute: kAXFocusedUIElementAttribute)+ }++ private static func buildFixtureSnapshot(app: RunningAppDescriptor, state: FixtureAppState) -> AppSnapshot {+ var lines: [String] = []+ lines.append("Computer Use state (OpenCodexComputerUse 0.1.0)")+ lines.append(PermissionDiagnostics.current().summary)+ lines.append("Screenshot: unavailable (fixture synthetic state)")+ lines.append("<app_state>")+ lines.append("App=\(app.name) (pid \(app.pid))")+ lines.append("Window: \"\(state.windowTitle)\", Bounds: \(format(rect: state.windowBounds.cgRect))")++ var records: [Int: ElementRecord] = [:]+ var identifierLines: [String] = []+ let focusedIdentifier = state.focusedIdentifier+ var focusedSummary: String?++ for element in state.elements.sorted(by: { $0.index < $1.index }) {+ let titleSegment = element.title.map { " \($0)" } ?? ""+ let valueSegment = element.value.map { " Value: \($0)" } ?? ""+ let actionsSegment = element.actions.isEmpty ? "" : " Secondary Actions: \(element.actions.joined(separator: ", "))"+ let focusSegment = focusedIdentifier == element.identifier ? " (focused)" : ""+ lines.append("\(String(repeating: " ", count: element.index == 0 ? 0 : 1))\(element.index) \(element.role)\(titleSegment)\(focusSegment) ID: \(element.identifier)\(valueSegment)\(actionsSegment) Frame: \(element.frame.cgRect.renderedLocalFrame)")++ let record = ElementRecord(+ index: element.index,+ identifier: element.identifier,+ element: nil,+ localFrame: element.frame.cgRect,+ rawActions: element.actions,+ prettyActions: element.actions+ )+ records[element.index] = record+ identifierLines.append("\(element.identifier) -> \(element.index) @ \(element.frame.cgRect.renderedLocalFrame)")++ if focusedIdentifier == element.identifier {+ focusedSummary = "\(element.index) \(element.role)"+ }+ }++ lines.append("</app_state>")+ lines.append("<element_index>")+ lines.append(contentsOf: identifierLines.sorted())+ lines.append("</element_index>")++ if let focusedSummary {+ lines.append("The focused UI element is \(focusedSummary).")+ }++ return AppSnapshot(+ app: app,+ windowTitle: state.windowTitle,+ windowBounds: state.windowBounds.cgRect,+ screenshotPath: nil,+ renderedText: lines.joined(separator: "\n"),+ mode: .fixture,+ elements: records+ )+ }+}++private struct WindowCapture {+ let windowID: CGWindowID+ let bounds: CGRect+ let image: CGImage?++ static func resolve(for pid: pid_t, titleHint: String?) -> WindowCapture? {+ guard let infoList = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] else {+ return nil+ }++ let candidates = infoList.compactMap { info -> (CGWindowID, CGRect, String?, Int)? in+ guard+ let ownerPID = info[kCGWindowOwnerPID as String] as? pid_t,+ ownerPID == pid,+ let number = info[kCGWindowNumber as String] as? NSNumber,+ let layer = info[kCGWindowLayer as String] as? Int,+ layer == 0,+ let boundsDictionary = info[kCGWindowBounds as String] as? NSDictionary,+ let bounds = CGRect(dictionaryRepresentation: boundsDictionary)+ else {+ return nil+ }++ let title = info[kCGWindowName as String] as? String+ let area = Int(bounds.width * bounds.height)+ return (CGWindowID(number.uint32Value), bounds, title, area)+ }++ guard let best = candidates.sorted(by: { lhs, rhs in+ if let titleHint {+ if lhs.2 == titleHint && rhs.2 != titleHint {+ return true+ }++ if rhs.2 == titleHint && lhs.2 != titleHint {+ return false+ }+ }++ return lhs.3 > rhs.3+ }).first else {+ return nil+ }++ let image = CGWindowListCreateImage(+ best.1,+ .optionIncludingWindow,+ best.0,+ [.bestResolution, .boundsIgnoreFraming]+ )++ return WindowCapture(windowID: best.0, bounds: best.1, image: image)+ }++ func writePNGIfAvailable() throws -> String? {+ guard let image else {+ return nil+ }++ let directory = URL(fileURLWithPath: NSTemporaryDirectory())+ .appendingPathComponent("open-codex-computer-use", isDirectory: true)+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)++ let fileURL = directory.appendingPathComponent("window-\(windowID)-\(Int(Date().timeIntervalSince1970 * 1000)).png")+ let bitmap = NSBitmapImageRep(cgImage: image)+ guard let data = bitmap.representation(using: .png, properties: [:]) else {+ return nil+ }++ try data.write(to: fileURL)+ return fileURL.path+ }+}++private struct RenderContext {+ let windowBounds: CGRect?+ let focusedElement: AXUIElement?+}++private struct TreeRenderer {+ let context: RenderContext+ var nextIndex = 0+ var lines: [String] = []+ var records: [Int: ElementRecord] = [:]+ var identifierIndex: [String: String] = [:]+ var focusedSummary: String?+ private var visited: Set<String> = []++ init(context: RenderContext) {+ self.context = context+ }++ mutating func render(_ root: AXUIElement, depth: Int = 0) {+ guard nextIndex < 500, depth < 16 else {+ return+ }++ let identifier = opaqueIdentifier(for: root)+ guard visited.insert(identifier).inserted else {+ return+ }++ let index = nextIndex+ nextIndex += 1++ let role = stringValue(of: root, attribute: kAXRoleAttribute) ?? "AXUnknown"+ let subrole = stringValue(of: root, attribute: kAXSubroleAttribute)+ let title = stringValue(of: root, attribute: kAXTitleAttribute)+ let label = stringValue(of: root, attribute: kAXDescriptionAttribute)+ let value = sanitizedValue(of: root)+ let axIdentifier = stringValue(of: root, attribute: kAXIdentifierAttribute)+ let traits = summarizeTraits(of: root)+ let actions = (copyActions(root) ?? []).filter { $0 != kAXPressAction as String }+ let prettyActions = actions.map(prettyActionName(_:))+ let localFrame = resolveLocalFrame(of: root, windowBounds: context.windowBounds)++ let roleText = humanizeRole(role: role, subrole: subrole)+ let titleSegment = title.map { " \($0)" } ?? ""+ let traitsSegment = traits.isEmpty ? "" : " (\(traits.joined(separator: ", ")))"+ let labelSegment = label.map { " Description: \($0)" } ?? ""+ let identifierSegment = axIdentifier.map { " ID: \($0)" } ?? ""+ let valueSegment = value.map { " Value: \($0)" } ?? ""+ let actionsSegment = prettyActions.isEmpty ? "" : " Secondary Actions: \(prettyActions.joined(separator: ", "))"+ let frameSegment = localFrame.map { " Frame: \($0.renderedLocalFrame)" } ?? ""++ lines.append("\(String(repeating: " ", count: depth))\(index) \(roleText)\(titleSegment)\(traitsSegment)\(labelSegment)\(identifierSegment)\(valueSegment)\(actionsSegment)\(frameSegment)")++ let record = ElementRecord(+ index: index,+ identifier: axIdentifier,+ element: root,+ localFrame: localFrame,+ rawActions: actions,+ prettyActions: prettyActions+ )+ records[index] = record++ if let axIdentifier, let localFrame {+ identifierIndex[axIdentifier] = "\(axIdentifier) -> \(index) @ \(localFrame.renderedLocalFrame)"+ }++ if let focusedElement = context.focusedElement, CFEqual(focusedElement, root) {+ focusedSummary = "\(index) \(roleText)"+ }++ for child in children(of: root) {+ render(child, depth: depth + 1)+ }+ }++ private func opaqueIdentifier(for element: AXUIElement) -> String {+ String(CFHash(element))+ }++ private func children(of element: AXUIElement) -> [AXUIElement] {+ let attributes = [kAXChildrenAttribute, kAXRowsAttribute]+ var children: [AXUIElement] = []+ var seen: Set<String> = []++ for attribute in attributes {+ guard let values = copyArray(element, attribute: attribute) else {+ continue+ }++ for child in values {+ let id = opaqueIdentifier(for: child)+ if seen.insert(id).inserted {+ children.append(child)+ }+ }+ }++ return children+ }+}++private func summarizeTraits(of element: AXUIElement) -> [String] {+ var values: [String] = []++ if boolValue(of: element, attribute: kAXFocusedAttribute) == true {+ values.append("focused")+ }++ if boolValue(of: element, attribute: kAXSelectedAttribute) == true {+ values.append("selected")+ }++ if boolValue(of: element, attribute: kAXExpandedAttribute) == true {+ values.append("expanded")+ }++ if isSettable(of: element, attribute: kAXValueAttribute) {+ values.append("settable")+ }++ if let role = stringValue(of: element, attribute: kAXRoleAttribute), role == kAXTextFieldRole as String {+ values.append("string")+ }++ return values+}++private func copyElement(_ element: AXUIElement, attribute: String) -> AXUIElement? {+ var value: CFTypeRef?+ let error = AXUIElementCopyAttributeValue(element, attribute as CFString, &value)+ guard error == .success, let value else {+ return nil+ }++ return (value as! AXUIElement)+}++private func copyArray(_ element: AXUIElement, attribute: String) -> [AXUIElement]? {+ var value: CFTypeRef?+ let error = AXUIElementCopyAttributeValue(element, attribute as CFString, &value)+ guard error == .success, let value else {+ return nil+ }++ return value as? [AXUIElement]+}++private func copyActions(_ element: AXUIElement) -> [String]? {+ var actions: CFArray?+ let error = AXUIElementCopyActionNames(element, &actions)+ guard error == .success else {+ return nil+ }++ return actions as? [String]+}++private func stringValue(of element: AXUIElement, attribute: String) -> String? {+ var value: CFTypeRef?+ let error = AXUIElementCopyAttributeValue(element, attribute as CFString, &value)+ guard error == .success, let value else {+ return nil+ }++ if CFGetTypeID(value) == CFStringGetTypeID() {+ return value as? String+ }++ return nil+}++private func boolValue(of element: AXUIElement, attribute: String) -> Bool? {+ var value: CFTypeRef?+ let error = AXUIElementCopyAttributeValue(element, attribute as CFString, &value)+ guard error == .success, let value else {+ return nil+ }++ return value as? Bool+}++private func pid(of element: AXUIElement) -> pid_t {+ var processIdentifier: pid_t = 0+ AXUIElementGetPid(element, &processIdentifier)+ return processIdentifier+}++private func isSettable(of element: AXUIElement, attribute: String) -> Bool {+ var settable = DarwinBoolean(false)+ let error = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable)+ return error == .success && settable.boolValue+}++private func sanitizedValue(of element: AXUIElement) -> String? {+ if let string = stringValue(of: element, attribute: kAXValueAttribute) {+ return sanitizeText(string)+ }++ var value: CFTypeRef?+ let error = AXUIElementCopyAttributeValue(element, kAXValueAttribute as CFString, &value)+ guard error == .success, let value else {+ return nil+ }++ if let number = value as? NSNumber {+ return number.stringValue+ }++ return nil+}++private func resolveLocalFrame(of element: AXUIElement, windowBounds: CGRect?) -> CGRect? {+ var positionValue: CFTypeRef?+ var sizeValue: CFTypeRef?+ let positionError = AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue)+ let sizeError = AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue)+ guard+ positionError == .success,+ sizeError == .success,+ let positionValue,+ let sizeValue+ else {+ return nil+ }++ let positionAXValue = positionValue as! AXValue+ let sizeAXValue = sizeValue as! AXValue+ var position = CGPoint.zero+ var size = CGSize.zero+ guard AXValueGetValue(positionAXValue, .cgPoint, &position), AXValueGetValue(sizeAXValue, .cgSize, &size) else {+ return nil+ }++ let frame = CGRect(origin: position, size: size)++ guard let windowBounds else {+ return frame+ }++ return windowRelativeFrame(elementFrame: frame, windowBounds: windowBounds)+}++func windowRelativeFrame(elementFrame: CGRect, windowBounds: CGRect) -> CGRect {+ CGRect(+ x: elementFrame.minX - windowBounds.minX,+ y: elementFrame.minY - windowBounds.minY,+ width: elementFrame.width,+ height: elementFrame.height+ )+}++private func humanizeRole(role: String, subrole: String?) -> String {+ if let subrole, subrole == kAXStandardWindowSubrole as String {+ return "standard window"+ }++ return humanizeAXToken(role)+}++private func prettyActionName(_ value: String) -> String {+ let stripped = value.hasPrefix("AX") ? String(value.dropFirst(2)) : value+ let withoutPage = stripped.replacingOccurrences(of: "ByPage", with: "")+ return splitCamelCase(withoutPage)+}++private func humanizeAXToken(_ value: String) -> String {+ let stripped = value.hasPrefix("AX") ? String(value.dropFirst(2)) : value+ return splitCamelCase(stripped).lowercased()+}++private func splitCamelCase(_ value: String) -> String {+ var result = ""+ for character in value {+ if character.isUppercase, !result.isEmpty {+ result.append(" ")+ }+ result.append(character)+ }+ return result+}++private func sanitizeText(_ value: String) -> String {+ let collapsed = value+ .replacingOccurrences(of: "\n", with: "\\n")+ .trimmingCharacters(in: .whitespacesAndNewlines)++ if collapsed.count > 160 {+ return String(collapsed.prefix(160)) + "..."+ }++ return collapsed+}++private func quoted(_ value: String) -> String {+ "\"\(value)\""+}++private func format(rect: CGRect) -> String {+ "x=\(Int(rect.origin.x)), y=\(Int(rect.origin.y)), w=\(Int(rect.width)), h=\(Int(rect.height))"+}++private extension CGRect {+ var renderedLocalFrame: String {+ "x=\(Int(origin.x)), y=\(Int(origin.y)), w=\(Int(width)), h=\(Int(height))"+ }+}diff --git a/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/AppDiscovery.swift b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/AppDiscovery.swiftnew file mode 100644index 0000000..e305a31--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/AppDiscovery.swift@@ -0,0 +1,80 @@+import AppKit+import Foundation++public struct RunningAppDescriptor {+ public let name: String+ public let bundleIdentifier: String?+ public let pid: pid_t+ public let runningApplication: NSRunningApplication+}++enum AppDiscovery {+ static func listApps() -> [RunningAppDescriptor] {+ NSWorkspace.shared.runningApplications+ .filter { !$0.isTerminated }+ .sorted { lhs, rhs in+ if lhs.isActive != rhs.isActive {+ return lhs.isActive && !rhs.isActive+ }++ return appName(lhs).localizedCaseInsensitiveCompare(appName(rhs)) == .orderedAscending+ }+ .map { app in+ RunningAppDescriptor(+ name: appName(app),+ bundleIdentifier: app.bundleIdentifier,+ pid: app.processIdentifier,+ runningApplication: app+ )+ }+ }++ static func resolve(_ query: String) throws -> RunningAppDescriptor {+ let running = listApps()++ if let match = running.first(where: { descriptor in+ descriptor.name.caseInsensitiveCompare(query) == .orderedSame+ || descriptor.bundleIdentifier?.caseInsensitiveCompare(query) == .orderedSame+ || descriptor.runningApplication.executableURL?.deletingPathExtension().lastPathComponent.caseInsensitiveCompare(query) == .orderedSame+ }) {+ return match+ }++ try launchIfPossible(query)++ for _ in 0..<20 {+ if let launched = listApps().first(where: { descriptor in+ descriptor.name.caseInsensitiveCompare(query) == .orderedSame+ || descriptor.bundleIdentifier?.caseInsensitiveCompare(query) == .orderedSame+ }) {+ return launched+ }++ Thread.sleep(forTimeInterval: 0.25)+ }++ throw ComputerUseError.appNotFound(query)+ }++ private static func launchIfPossible(_ query: String) throws {+ if query.contains(".") {+ if let appURL = NSWorkspace.shared.urlForApplication(withBundleIdentifier: query) {+ try NSWorkspace.shared.launchApplication(at: appURL, options: [], configuration: [:])+ }+ return+ }++ guard let fullPath = NSWorkspace.shared.fullPath(forApplication: query) else {+ return+ }++ try NSWorkspace.shared.launchApplication(at: URL(fileURLWithPath: fullPath), options: [], configuration: [:])+ }++ static func appName(_ app: NSRunningApplication) -> String {+ app.localizedName+ ?? app.bundleURL?.deletingPathExtension().lastPathComponent+ ?? app.executableURL?.lastPathComponent+ ?? "pid-\(app.processIdentifier)"+ }+}diff --git a/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/ComputerUseService.swift b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/ComputerUseService.swiftnew file mode 100644index 0000000..319a8d6--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/ComputerUseService.swift@@ -0,0 +1,279 @@+import AppKit+import ApplicationServices+import Foundation++public final class ComputerUseService {+ private var snapshotsByApp: [String: AppSnapshot] = [:]++ public init() {}++ public func listApps() -> String {+ AppDiscovery.listApps()+ .map { descriptor in+ let bundle = descriptor.bundleIdentifier ?? "no-bundle-id"+ return "\(descriptor.name) — \(bundle) [running, pid=\(descriptor.pid)]"+ }+ .joined(separator: "\n")+ }++ public func getAppState(app query: String) throws -> String {+ try refreshSnapshot(for: query).renderedText+ }++ public func click(app query: String, elementIndex: String?, x: Double?, y: Double?, clickCount: Int, mouseButton: String) throws -> String {+ let snapshot = try currentSnapshot(for: query)+ let button = MouseButtonKind(rawValue: mouseButton.lowercased()) ?? .left+ if snapshot.mode == .fixture {+ if let elementIndex {+ let record = try lookupElement(snapshot: snapshot, index: elementIndex)+ guard let identifier = record.identifier else {+ throw ComputerUseError.invalidArguments("fixture click requires an identifier-backed element")+ }+ try FixtureBridge.post(FixtureCommand(kind: "click", identifier: identifier))+ } else if let x, let y {+ let identifier = try fixtureIdentifier(at: CGPoint(x: x, y: y), snapshot: snapshot)+ try FixtureBridge.post(FixtureCommand(kind: "click", identifier: identifier, x: x, y: y))+ } else {+ throw ComputerUseError.invalidArguments("click requires either element_index or x/y")+ }++ Thread.sleep(forTimeInterval: 0.15)+ return try refreshSnapshot(for: query).renderedText+ }++ InputSimulation.activate(snapshot.app)++ if let elementIndex {+ let record = try lookupElement(snapshot: snapshot, index: elementIndex)++ if button == .left,+ clickCount == 1,+ let element = record.element,+ record.rawActions.contains(kAXPressAction as String),+ AXUIElementPerformAction(element, kAXPressAction as CFString) == .success {+ Thread.sleep(forTimeInterval: 0.15)+ } else if let point = try globalPoint(for: record, snapshot: snapshot) {+ try InputSimulation.click(at: point, button: button, clickCount: clickCount)+ } else {+ throw ComputerUseError.stateUnavailable("element \(elementIndex) has no clickable frame")+ }+ } else if let x, let y {+ try InputSimulation.click(at: try screenshotToGlobalPoint(snapshot: snapshot, x: x, y: y), button: button, clickCount: clickCount)+ } else {+ throw ComputerUseError.invalidArguments("click requires either element_index or x/y")+ }++ return try refreshSnapshot(for: query).renderedText+ }++ public func performSecondaryAction(app query: String, elementIndex: String, action: String) throws -> String {+ let snapshot = try currentSnapshot(for: query)+ let record = try lookupElement(snapshot: snapshot, index: elementIndex)++ if snapshot.mode == .fixture {+ guard action.caseInsensitiveCompare("Raise") == .orderedSame else {+ throw ComputerUseError.invalidArguments("fixture mode only supports the Raise secondary action")+ }++ InputSimulation.activate(snapshot.app)+ return try refreshSnapshot(for: query).renderedText+ }++ guard let rawAction = matchingAction(requested: action, record: record) else {+ throw ComputerUseError.invalidArguments("element \(elementIndex) does not expose action '\(action)'")+ }++ guard let element = record.element else {+ throw ComputerUseError.stateUnavailable("element \(elementIndex) has no backing accessibility object")+ }++ let result = AXUIElementPerformAction(element, rawAction as CFString)+ guard result == .success else {+ throw ComputerUseError.message("AXUIElementPerformAction failed with \(result.rawValue)")+ }++ Thread.sleep(forTimeInterval: 0.15)+ return try refreshSnapshot(for: query).renderedText+ }++ public func scroll(app query: String, direction: String, elementIndex: String, pages: Int) throws -> String {+ let normalized = direction.lowercased()+ guard ["up", "down", "left", "right"].contains(normalized) else {+ throw ComputerUseError.invalidArguments("scroll direction must be one of up/down/left/right")+ }++ let snapshot = try currentSnapshot(for: query)+ let record = try lookupElement(snapshot: snapshot, index: elementIndex)++ if snapshot.mode == .fixture {+ guard let identifier = record.identifier else {+ throw ComputerUseError.invalidArguments("fixture scroll requires an identifier-backed element")+ }+ try FixtureBridge.post(FixtureCommand(kind: "scroll", identifier: identifier, direction: normalized, pages: pages))+ Thread.sleep(forTimeInterval: 0.15)+ return try refreshSnapshot(for: query).renderedText+ }++ InputSimulation.activate(snapshot.app)++ if let rawAction = record.rawActions.first(where: { $0.caseInsensitiveCompare("AXScroll\(normalized.capitalized)ByPage") == .orderedSame }), let element = record.element {+ for _ in 0..<max(pages, 1) {+ _ = AXUIElementPerformAction(element, rawAction as CFString)+ Thread.sleep(forTimeInterval: 0.05)+ }+ } else if let point = try globalPoint(for: record, snapshot: snapshot) {+ try InputSimulation.scroll(at: point, direction: normalized, pages: pages)+ } else {+ throw ComputerUseError.stateUnavailable("element \(elementIndex) has no scrollable frame")+ }++ return try refreshSnapshot(for: query).renderedText+ }++ public func drag(app query: String, fromX: Double, fromY: Double, toX: Double, toY: Double) throws -> String {+ let snapshot = try currentSnapshot(for: query)+ if snapshot.mode == .fixture {+ try FixtureBridge.post(FixtureCommand(kind: "drag", identifier: "fixture-drag-pad", x: fromX, y: fromY, toX: toX, toY: toY))+ Thread.sleep(forTimeInterval: 0.15)+ return try refreshSnapshot(for: query).renderedText+ }++ InputSimulation.activate(snapshot.app)+ try InputSimulation.drag(+ from: try screenshotToGlobalPoint(snapshot: snapshot, x: fromX, y: fromY),+ to: try screenshotToGlobalPoint(snapshot: snapshot, x: toX, y: toY)+ )+ return try refreshSnapshot(for: query).renderedText+ }++ public func typeText(app query: String, text: String) throws -> String {+ let snapshot = try currentSnapshot(for: query)+ if snapshot.mode == .fixture {+ try FixtureBridge.post(FixtureCommand(kind: "type_text", identifier: "fixture-input", value: text))+ Thread.sleep(forTimeInterval: 0.15)+ return try refreshSnapshot(for: query).renderedText+ }++ InputSimulation.activate(snapshot.app)+ try InputSimulation.typeText(text)+ return try refreshSnapshot(for: query).renderedText+ }++ public func pressKey(app query: String, key: String) throws -> String {+ let snapshot = try currentSnapshot(for: query)+ if snapshot.mode == .fixture {+ try FixtureBridge.post(FixtureCommand(kind: "press_key", identifier: "fixture-key-capture", value: key))+ Thread.sleep(forTimeInterval: 0.15)+ return try refreshSnapshot(for: query).renderedText+ }++ InputSimulation.activate(snapshot.app)+ try InputSimulation.pressKey(key)+ return try refreshSnapshot(for: query).renderedText+ }++ public func setValue(app query: String, elementIndex: String, value: String) throws -> String {+ let snapshot = try currentSnapshot(for: query)+ let record = try lookupElement(snapshot: snapshot, index: elementIndex)++ if snapshot.mode == .fixture {+ guard let identifier = record.identifier else {+ throw ComputerUseError.invalidArguments("fixture set_value requires a known element identifier")+ }++ try FixtureBridge.post(FixtureCommand(kind: "set_value", identifier: identifier, value: value))+ Thread.sleep(forTimeInterval: 0.15)+ return try refreshSnapshot(for: query).renderedText+ }++ guard let element = record.element else {+ throw ComputerUseError.stateUnavailable("element \(elementIndex) has no backing accessibility object")+ }++ let result = AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFString)+ guard result == .success else {+ throw ComputerUseError.message("AXUIElementSetAttributeValue failed with \(result.rawValue)")+ }++ Thread.sleep(forTimeInterval: 0.1)+ return try refreshSnapshot(for: query).renderedText+ }++ private func currentSnapshot(for query: String) throws -> AppSnapshot {+ if let snapshot = snapshotsByApp[query.lowercased()] {+ return snapshot+ }++ return try refreshSnapshot(for: query)+ }++ @discardableResult+ private func refreshSnapshot(for query: String) throws -> AppSnapshot {+ let app = try AppDiscovery.resolve(query)+ let snapshot = try SnapshotBuilder.build(for: app)++ let keys = Set([+ query.lowercased(),+ app.name.lowercased(),+ (app.bundleIdentifier ?? "").lowercased(),+ ].filter { !$0.isEmpty })++ for key in keys {+ snapshotsByApp[key] = snapshot+ }++ return snapshot+ }++ private func lookupElement(snapshot: AppSnapshot, index: String) throws -> ElementRecord {+ guard let parsedIndex = Int(index), let record = snapshot.elements[parsedIndex] else {+ throw ComputerUseError.invalidArguments("unknown element_index '\(index)'")+ }++ return record+ }++ private func matchingAction(requested: String, record: ElementRecord) -> String? {+ if let exact = record.rawActions.first(where: { $0.caseInsensitiveCompare(requested) == .orderedSame }) {+ return exact+ }++ if let pretty = zip(record.rawActions, record.prettyActions).first(where: { $0.1.caseInsensitiveCompare(requested) == .orderedSame }) {+ return pretty.0+ }++ return nil+ }++ private func globalPoint(for record: ElementRecord, snapshot: AppSnapshot) throws -> CGPoint? {+ guard let frame = record.localFrame else {+ return nil+ }++ return try screenshotToGlobalPoint(snapshot: snapshot, x: frame.midX, y: frame.midY)+ }++ private func screenshotToGlobalPoint(snapshot: AppSnapshot, x: Double, y: Double) throws -> CGPoint {+ guard let windowBounds = snapshot.windowBounds else {+ throw ComputerUseError.stateUnavailable("No window bounds are available for \(snapshot.app.name). Run get_app_state after bringing the app on screen.")+ }++ return CGPoint(x: windowBounds.minX + x, y: windowBounds.minY + y)+ }++ private func fixtureIdentifier(at point: CGPoint, snapshot: AppSnapshot) throws -> String {+ let candidates = snapshot.elements.values+ .filter { $0.identifier != nil && ($0.localFrame?.contains(point) ?? false) }+ .sorted { lhs, rhs in+ let lhsArea = (lhs.localFrame?.width ?? 0) * (lhs.localFrame?.height ?? 0)+ let rhsArea = (rhs.localFrame?.width ?? 0) * (rhs.localFrame?.height ?? 0)+ return lhsArea < rhsArea+ }++ guard let identifier = candidates.first?.identifier else {+ throw ComputerUseError.invalidArguments("No fixture element contains coordinate (\(Int(point.x)), \(Int(point.y)))")+ }++ return identifier+ }+}diff --git a/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/Errors.swift b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/Errors.swiftnew file mode 100644index 0000000..27b0f2d--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/Errors.swift@@ -0,0 +1,33 @@+import Foundation++public enum ComputerUseError: Error, LocalizedError {+ case message(String)+ case unsupportedTool(String)+ case invalidArguments(String)+ case appNotFound(String)+ case permissionDenied(String)+ case stateUnavailable(String)++ public var errorDescription: String? {+ switch self {+ case .message(let value):+ return value+ case .unsupportedTool(let name):+ return "unsupportedTool(\(name))"+ case .invalidArguments(let message):+ return "invalidArguments(\(message))"+ case .appNotFound(let app):+ return "appNotFound(\(app))"+ case .permissionDenied(let message):+ return message+ case .stateUnavailable(let message):+ return message+ }+ }+}++extension ComputerUseError {+ static func missingArgument(_ name: String) -> ComputerUseError {+ .invalidArguments("missing argument '\(name)'")+ }+}diff --git a/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/FixtureBridge.swift b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/FixtureBridge.swiftnew file mode 100644index 0000000..489df5e--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/FixtureBridge.swift@@ -0,0 +1,126 @@+import CoreGraphics+import Foundation++public struct FixtureRect: Codable, Sendable {+ public let x: Double+ public let y: Double+ public let width: Double+ public let height: Double++ public init(rect: CGRect) {+ x = rect.origin.x+ y = rect.origin.y+ width = rect.width+ height = rect.height+ }++ public var cgRect: CGRect {+ CGRect(x: x, y: y, width: width, height: height)+ }+}++public struct FixtureElementState: Codable, Sendable {+ public let identifier: String+ public let index: Int+ public let role: String+ public let title: String?+ public let value: String?+ public let actions: [String]+ public let frame: FixtureRect++ public init(identifier: String, index: Int, role: String, title: String?, value: String?, actions: [String], frame: FixtureRect) {+ self.identifier = identifier+ self.index = index+ self.role = role+ self.title = title+ self.value = value+ self.actions = actions+ self.frame = frame+ }+}++public struct FixtureAppState: Codable, Sendable {+ public let windowTitle: String+ public let windowBounds: FixtureRect+ public let focusedIdentifier: String?+ public let elements: [FixtureElementState]++ public init(windowTitle: String, windowBounds: FixtureRect, focusedIdentifier: String?, elements: [FixtureElementState]) {+ self.windowTitle = windowTitle+ self.windowBounds = windowBounds+ self.focusedIdentifier = focusedIdentifier+ self.elements = elements+ }+}++public struct FixtureCommand: Codable, Sendable {+ public let kind: String+ public let identifier: String+ public let value: String?+ public let x: Double?+ public let y: Double?+ public let toX: Double?+ public let toY: Double?+ public let direction: String?+ public let pages: Int?++ public init(+ kind: String,+ identifier: String,+ value: String? = nil,+ x: Double? = nil,+ y: Double? = nil,+ toX: Double? = nil,+ toY: Double? = nil,+ direction: String? = nil,+ pages: Int? = nil+ ) {+ self.kind = kind+ self.identifier = identifier+ self.value = value+ self.x = x+ self.y = y+ self.toX = toX+ self.toY = toY+ self.direction = direction+ self.pages = pages+ }+}++public enum FixtureBridge {+ public static let appName = "OpenCodexComputerUseFixture"+ public static let distributedNotificationName = Notification.Name("dev.opencodex.fixture.command")++ public static var stateFileURL: URL {+ URL(fileURLWithPath: NSTemporaryDirectory())+ .appendingPathComponent("open-codex-computer-use-fixture", isDirectory: true)+ .appendingPathComponent("state.json")+ }++ public static func readState() throws -> FixtureAppState? {+ let url = stateFileURL+ guard FileManager.default.fileExists(atPath: url.path) else {+ return nil+ }++ let data = try Data(contentsOf: url)+ return try JSONDecoder().decode(FixtureAppState.self, from: data)+ }++ public static func writeState(_ state: FixtureAppState) throws {+ let directory = stateFileURL.deletingLastPathComponent()+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)+ let data = try JSONEncoder().encode(state)+ try data.write(to: stateFileURL)+ }++ public static func post(_ command: FixtureCommand) throws {+ let payload = try String(data: JSONEncoder().encode(command), encoding: .utf8)+ DistributedNotificationCenter.default().postNotificationName(+ distributedNotificationName,+ object: nil,+ userInfo: payload.map { ["payload": $0] },+ options: .deliverImmediately+ )+ }+}diff --git a/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/InputSimulation.swift b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/InputSimulation.swiftnew file mode 100644index 0000000..b75a749--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/InputSimulation.swift@@ -0,0 +1,177 @@+import AppKit+import ApplicationServices+import CoreGraphics+import Foundation++enum MouseButtonKind: String {+ case left+ case right+ case middle++ var cgButton: CGMouseButton {+ switch self {+ case .left:+ return .left+ case .right:+ return .right+ case .middle:+ return .center+ }+ }++ var downEvent: CGEventType {+ switch self {+ case .left:+ return .leftMouseDown+ case .right:+ return .rightMouseDown+ case .middle:+ return .otherMouseDown+ }+ }++ var upEvent: CGEventType {+ switch self {+ case .left:+ return .leftMouseUp+ case .right:+ return .rightMouseUp+ case .middle:+ return .otherMouseUp+ }+ }+}++enum InputSimulation {+ static func activate(_ app: RunningAppDescriptor) {+ app.runningApplication.activate(options: [.activateIgnoringOtherApps])+ Thread.sleep(forTimeInterval: 0.25)+ }++ static func click(at point: CGPoint, button: MouseButtonKind, clickCount: Int) throws {+ guard let source = CGEventSource(stateID: .hidSystemState) else {+ throw ComputerUseError.message("Failed to create HID event source.")+ }++ for _ in 0..<max(clickCount, 1) {+ try postMouseEvent(type: .mouseMoved, source: source, point: point, button: button.cgButton, clickState: clickCount)+ try postMouseEvent(type: button.downEvent, source: source, point: point, button: button.cgButton, clickState: clickCount)+ try postMouseEvent(type: button.upEvent, source: source, point: point, button: button.cgButton, clickState: clickCount)+ }+ }++ static func scroll(at point: CGPoint, direction: String, pages: Int) throws {+ guard let event = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 2, wheel1: wheel1(direction: direction, pages: pages), wheel2: wheel2(direction: direction, pages: pages), wheel3: 0) else {+ throw ComputerUseError.message("Failed to create scroll event.")+ }++ event.location = point+ event.post(tap: .cghidEventTap)+ Thread.sleep(forTimeInterval: 0.1)+ }++ static func drag(from start: CGPoint, to end: CGPoint) throws {+ guard let source = CGEventSource(stateID: .hidSystemState) else {+ throw ComputerUseError.message("Failed to create HID event source.")+ }++ try postMouseEvent(type: .mouseMoved, source: source, point: start, button: .left, clickState: 1)+ try postMouseEvent(type: .leftMouseDown, source: source, point: start, button: .left, clickState: 1)++ for step in 1...10 {+ let progress = CGFloat(step) / 10+ let point = CGPoint(+ x: start.x + ((end.x - start.x) * progress),+ y: start.y + ((end.y - start.y) * progress)+ )+ try postMouseEvent(type: .leftMouseDragged, source: source, point: point, button: .left, clickState: 1)+ }++ try postMouseEvent(type: .leftMouseUp, source: source, point: end, button: .left, clickState: 1)+ }++ static func typeText(_ text: String) throws {+ for character in text.utf16 {+ var mutableCharacter = character+ guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),+ let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) else {+ throw ComputerUseError.message("Failed to create keyboard event.")+ }++ down.keyboardSetUnicodeString(stringLength: 1, unicodeString: &mutableCharacter)+ up.keyboardSetUnicodeString(stringLength: 1, unicodeString: &mutableCharacter)+ down.post(tap: .cghidEventTap)+ up.post(tap: .cghidEventTap)+ Thread.sleep(forTimeInterval: 0.02)+ }+ }++ static func pressKey(_ specification: String) throws {+ let parsed = try KeyPressParser.parse(specification)+ var activeFlags: CGEventFlags = []++ for modifier in parsed.modifiers {+ guard let event = CGEvent(keyboardEventSource: nil, virtualKey: modifier.keyCode, keyDown: true) else {+ throw ComputerUseError.message("Failed to create modifier key down event.")+ }++ activeFlags.insert(modifier.flag)+ event.flags = activeFlags+ event.post(tap: .cghidEventTap)+ }++ guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: parsed.keyCode, keyDown: true),+ let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: parsed.keyCode, keyDown: false) else {+ throw ComputerUseError.message("Failed to create key event.")+ }++ keyDown.flags = activeFlags+ keyUp.flags = activeFlags+ keyDown.post(tap: .cghidEventTap)+ keyUp.post(tap: .cghidEventTap)++ for modifier in parsed.modifiers.reversed() {+ guard let event = CGEvent(keyboardEventSource: nil, virtualKey: modifier.keyCode, keyDown: false) else {+ throw ComputerUseError.message("Failed to create modifier key up event.")+ }++ event.flags = activeFlags+ event.post(tap: .cghidEventTap)+ activeFlags.remove(modifier.flag)+ }++ Thread.sleep(forTimeInterval: 0.1)+ }++ private static func postMouseEvent(type: CGEventType, source: CGEventSource, point: CGPoint, button: CGMouseButton, clickState: Int) throws {+ guard let event = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: point, mouseButton: button) else {+ throw ComputerUseError.message("Failed to create mouse event \(type.rawValue).")+ }++ event.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))+ event.post(tap: .cghidEventTap)+ Thread.sleep(forTimeInterval: 0.03)+ }++ private static func wheel1(direction: String, pages: Int) -> Int32 {+ switch direction {+ case "up":+ return Int32(12 * max(pages, 1))+ case "down":+ return Int32(-12 * max(pages, 1))+ default:+ return 0+ }+ }++ private static func wheel2(direction: String, pages: Int) -> Int32 {+ switch direction {+ case "left":+ return Int32(12 * max(pages, 1))+ case "right":+ return Int32(-12 * max(pages, 1))+ default:+ return 0+ }+ }+}diff --git a/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/KeyMapping.swift b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/KeyMapping.swiftnew file mode 100644index 0000000..da8d032--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/KeyMapping.swift@@ -0,0 +1,117 @@+import Carbon.HIToolbox+import CoreGraphics+import Foundation++public struct ParsedKeyPress: Sendable {+ public struct Modifier: Sendable {+ public let flag: CGEventFlags+ public let keyCode: CGKeyCode+ }++ public let keyCode: CGKeyCode+ public let modifiers: [Modifier]+ public let displayValue: String+}++public enum KeyPressParser {+ public static func parse(_ specification: String) throws -> ParsedKeyPress {+ let tokens = specification+ .split(separator: "+")+ .map { normalize(String($0)) }+ .filter { !$0.isEmpty }++ guard let keyToken = tokens.last else {+ throw ComputerUseError.invalidArguments("key specification is empty")+ }++ let modifiers = try tokens.dropLast().map(parseModifier(_:))+ guard let keyCode = keyCodeMap[keyToken] else {+ throw ComputerUseError.invalidArguments("unsupported key '\(specification)'")+ }++ return ParsedKeyPress(+ keyCode: keyCode,+ modifiers: modifiers,+ displayValue: keyToken+ )+ }++ private static func parseModifier(_ token: String) throws -> ParsedKeyPress.Modifier {+ switch token {+ case "cmd", "command", "super", "meta":+ return ParsedKeyPress.Modifier(flag: .maskCommand, keyCode: CGKeyCode(kVK_Command))+ case "shift":+ return ParsedKeyPress.Modifier(flag: .maskShift, keyCode: CGKeyCode(kVK_Shift))+ case "option", "alt":+ return ParsedKeyPress.Modifier(flag: .maskAlternate, keyCode: CGKeyCode(kVK_Option))+ case "control", "ctrl":+ return ParsedKeyPress.Modifier(flag: .maskControl, keyCode: CGKeyCode(kVK_Control))+ default:+ throw ComputerUseError.invalidArguments("unsupported modifier '\(token)'")+ }+ }++ private static func normalize(_ value: String) -> String {+ value+ .trimmingCharacters(in: .whitespacesAndNewlines)+ .lowercased()+ .replacingOccurrences(of: " ", with: "")+ }++ private static let keyCodeMap: [String: CGKeyCode] = [+ "a": CGKeyCode(kVK_ANSI_A),+ "b": CGKeyCode(kVK_ANSI_B),+ "c": CGKeyCode(kVK_ANSI_C),+ "d": CGKeyCode(kVK_ANSI_D),+ "e": CGKeyCode(kVK_ANSI_E),+ "f": CGKeyCode(kVK_ANSI_F),+ "g": CGKeyCode(kVK_ANSI_G),+ "h": CGKeyCode(kVK_ANSI_H),+ "i": CGKeyCode(kVK_ANSI_I),+ "j": CGKeyCode(kVK_ANSI_J),+ "k": CGKeyCode(kVK_ANSI_K),+ "l": CGKeyCode(kVK_ANSI_L),+ "m": CGKeyCode(kVK_ANSI_M),+ "n": CGKeyCode(kVK_ANSI_N),+ "o": CGKeyCode(kVK_ANSI_O),+ "p": CGKeyCode(kVK_ANSI_P),+ "q": CGKeyCode(kVK_ANSI_Q),+ "r": CGKeyCode(kVK_ANSI_R),+ "s": CGKeyCode(kVK_ANSI_S),+ "t": CGKeyCode(kVK_ANSI_T),+ "u": CGKeyCode(kVK_ANSI_U),+ "v": CGKeyCode(kVK_ANSI_V),+ "w": CGKeyCode(kVK_ANSI_W),+ "x": CGKeyCode(kVK_ANSI_X),+ "y": CGKeyCode(kVK_ANSI_Y),+ "z": CGKeyCode(kVK_ANSI_Z),+ "0": CGKeyCode(kVK_ANSI_0),+ "1": CGKeyCode(kVK_ANSI_1),+ "2": CGKeyCode(kVK_ANSI_2),+ "3": CGKeyCode(kVK_ANSI_3),+ "4": CGKeyCode(kVK_ANSI_4),+ "5": CGKeyCode(kVK_ANSI_5),+ "6": CGKeyCode(kVK_ANSI_6),+ "7": CGKeyCode(kVK_ANSI_7),+ "8": CGKeyCode(kVK_ANSI_8),+ "9": CGKeyCode(kVK_ANSI_9),+ "return": CGKeyCode(kVK_Return),+ "enter": CGKeyCode(kVK_Return),+ "tab": CGKeyCode(kVK_Tab),+ "space": CGKeyCode(kVK_Space),+ "spacebar": CGKeyCode(kVK_Space),+ "escape": CGKeyCode(kVK_Escape),+ "esc": CGKeyCode(kVK_Escape),+ "delete": CGKeyCode(kVK_Delete),+ "forwarddelete": CGKeyCode(kVK_ForwardDelete),+ "up": CGKeyCode(kVK_UpArrow),+ "down": CGKeyCode(kVK_DownArrow),+ "left": CGKeyCode(kVK_LeftArrow),+ "right": CGKeyCode(kVK_RightArrow),+ "home": CGKeyCode(kVK_Home),+ "end": CGKeyCode(kVK_End),+ "pageup": CGKeyCode(kVK_PageUp),+ "pagedown": CGKeyCode(kVK_PageDown),+ "kp_0": CGKeyCode(kVK_ANSI_Keypad0),+ ]+}diff --git a/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/MCPServer.swift b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/MCPServer.swiftnew file mode 100644index 0000000..2098679--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/MCPServer.swift@@ -0,0 +1,222 @@+import Foundation++public final class StdioMCPServer {+ private let service: ComputerUseService++ public init(service: ComputerUseService = ComputerUseService()) {+ self.service = service+ }++ public func run() throws {+ while let line = readLine(strippingNewline: true) {+ guard !line.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {+ continue+ }++ if let response = handle(line: line) {+ FileHandle.standardOutput.write((response + "\n").data(using: .utf8)!)+ }+ }+ }++ public func handle(line: String) -> String? {+ do {+ guard let payload = try JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any] else {+ return try encodeJSONRPCError(id: nil, code: -32700, message: "Invalid JSON-RPC payload")+ }++ let method = payload["method"] as? String+ let id = payload["id"]+ let params = payload["params"] as? [String: Any] ?? [:]++ switch method {+ case "initialize":+ return try encodeJSONRPCResult(+ id: id,+ result: [+ "protocolVersion": "2025-03-26",+ "serverInfo": [+ "name": "open-codex-computer-use",+ "version": "0.1.0",+ ],+ "capabilities": [+ "tools": [+ "listChanged": false,+ ],+ ],+ ]+ )+ case "notifications/initialized":+ return nil+ case "ping":+ return try encodeJSONRPCResult(id: id, result: [:])+ case "tools/list":+ return try encodeJSONRPCResult(+ id: id,+ result: [+ "tools": ToolDefinitions.all.map(\.asDictionary),+ ]+ )+ case "tools/call":+ let name = params["name"] as? String ?? ""+ let arguments = params["arguments"] as? [String: Any] ?? [:]+ let text = try callTool(name: name, arguments: arguments)+ return try encodeJSONRPCResult(+ id: id,+ result: [+ "content": [+ [+ "type": "text",+ "text": text,+ ],+ ],+ "isError": false,+ ]+ )+ default:+ if method == nil {+ return nil+ }++ return try encodeJSONRPCError(id: id, code: -32601, message: "Method not found: \(method ?? "")")+ }+ } catch {+ let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)+ let payload = try? JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any]+ let id = payload?["id"]+ return try? encodeJSONRPCResult(+ id: id,+ result: [+ "content": [+ [+ "type": "text",+ "text": message,+ ],+ ],+ "isError": true,+ ]+ )+ }+ }++ private func callTool(name: String, arguments: [String: Any]) throws -> String {+ switch name {+ case "list_apps":+ return service.listApps()+ case "get_app_state":+ return try service.getAppState(app: requireString("app", in: arguments))+ case "click":+ return try service.click(+ app: requireString("app", in: arguments),+ elementIndex: optionalString("element_index", in: arguments),+ x: optionalDouble("x", in: arguments),+ y: optionalDouble("y", in: arguments),+ clickCount: Int(optionalDouble("click_count", in: arguments) ?? 1),+ mouseButton: optionalString("mouse_button", in: arguments) ?? "left"+ )+ case "perform_secondary_action":+ return try service.performSecondaryAction(+ app: requireString("app", in: arguments),+ elementIndex: requireString("element_index", in: arguments),+ action: requireString("action", in: arguments)+ )+ case "scroll":+ return try service.scroll(+ app: requireString("app", in: arguments),+ direction: requireString("direction", in: arguments),+ elementIndex: requireString("element_index", in: arguments),+ pages: Int(optionalDouble("pages", in: arguments) ?? 1)+ )+ case "drag":+ return try service.drag(+ app: requireString("app", in: arguments),+ fromX: requireDouble("from_x", in: arguments),+ fromY: requireDouble("from_y", in: arguments),+ toX: requireDouble("to_x", in: arguments),+ toY: requireDouble("to_y", in: arguments)+ )+ case "type_text":+ return try service.typeText(+ app: requireString("app", in: arguments),+ text: requireString("text", in: arguments)+ )+ case "press_key":+ return try service.pressKey(+ app: requireString("app", in: arguments),+ key: requireString("key", in: arguments)+ )+ case "set_value":+ return try service.setValue(+ app: requireString("app", in: arguments),+ elementIndex: requireString("element_index", in: arguments),+ value: requireString("value", in: arguments)+ )+ default:+ throw ComputerUseError.unsupportedTool(name)+ }+ }++ private func requireString(_ key: String, in arguments: [String: Any]) throws -> String {+ guard let value = arguments[key] as? String else {+ throw ComputerUseError.missingArgument(key)+ }++ return value+ }++ private func optionalString(_ key: String, in arguments: [String: Any]) -> String? {+ arguments[key] as? String+ }++ private func requireDouble(_ key: String, in arguments: [String: Any]) throws -> Double {+ guard let value = optionalDouble(key, in: arguments) else {+ throw ComputerUseError.missingArgument(key)+ }++ return value+ }++ private func optionalDouble(_ key: String, in arguments: [String: Any]) -> Double? {+ if let double = arguments[key] as? Double {+ return double+ }++ if let integer = arguments[key] as? Int {+ return Double(integer)+ }++ if let number = arguments[key] as? NSNumber {+ return number.doubleValue+ }++ return nil+ }++ private func encodeJSONRPCResult(id: Any?, result: [String: Any]) throws -> String {+ try encode([+ "jsonrpc": "2.0",+ "id": id ?? NSNull(),+ "result": result,+ ])+ }++ private func encodeJSONRPCError(id: Any?, code: Int, message: String) throws -> String {+ try encode([+ "jsonrpc": "2.0",+ "id": id ?? NSNull(),+ "error": [+ "code": code,+ "message": message,+ ],+ ])+ }++ private func encode(_ object: [String: Any]) throws -> String {+ let data = try JSONSerialization.data(withJSONObject: object, options: [.withoutEscapingSlashes])+ guard let text = String(data: data, encoding: .utf8) else {+ throw ComputerUseError.message("Failed to encode JSON-RPC response.")+ }++ return text+ }+}diff --git a/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/Permissions.swift b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/Permissions.swiftnew file mode 100644index 0000000..352709b--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/Permissions.swift@@ -0,0 +1,181 @@+import ApplicationServices+import AppKit+import Foundation+import SQLite3++public enum SystemPermissionKind: String, CaseIterable, Sendable {+ case accessibility+ case screenRecording++ public var title: String {+ switch self {+ case .accessibility:+ return "Accessibility"+ case .screenRecording:+ return "Screenshots"+ }+ }++ public var subtitle: String {+ switch self {+ case .accessibility:+ return "Allows Open Codex to access app interfaces"+ case .screenRecording:+ return "Open Codex uses screenshots to know where to click"+ }+ }++ public var settingsURL: URL {+ switch self {+ case .accessibility:+ return URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility")!+ case .screenRecording:+ return URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture")!+ }+ }++ public var dragInstruction: String {+ switch self {+ case .accessibility:+ return "Drag Open Codex Computer Use above to allow Accessibility"+ case .screenRecording:+ return "Drag Open Codex Computer Use above to allow Screenshots"+ }+ }++ public var systemSettingsTitle: String {+ switch self {+ case .accessibility:+ return "Accessibility"+ case .screenRecording:+ return "Screen & System Audio Recording"+ }+ }++ public var symbolName: String {+ switch self {+ case .accessibility:+ return "figure.arms.open"+ case .screenRecording:+ return "camera.viewfinder"+ }+ }+}++public struct PermissionDiagnostics: Sendable {+ public let accessibilityTrusted: Bool+ public let screenCaptureGranted: Bool++ public static func current() -> PermissionDiagnostics {+ let persisted = TCCAuthorizationStore.current++ return PermissionDiagnostics(+ accessibilityTrusted: persisted.accessibility ?? AXIsProcessTrusted(),+ screenCaptureGranted: persisted.screenRecording ?? CGPreflightScreenCaptureAccess()+ )+ }++ public var summary: String {+ "Permissions: accessibility=\(accessibilityTrusted ? "granted" : "missing"), screenRecording=\(screenCaptureGranted ? "granted" : "missing")"+ }++ public func isGranted(_ permission: SystemPermissionKind) -> Bool {+ switch permission {+ case .accessibility:+ return accessibilityTrusted+ case .screenRecording:+ return screenCaptureGranted+ }+ }++ public var allGranted: Bool {+ accessibilityTrusted && screenCaptureGranted+ }+}++public enum PermissionSupport {+ public static let bundleDisplayName = "Open Codex Computer Use"+ public static let bundleIdentifier = "dev.opencodex.OpenCodexComputerUse"++ public static func currentAppBundleURL() -> URL? {+ let bundleURL = Bundle.main.bundleURL+ return bundleURL.pathExtension == "app" ? bundleURL : nil+ }++ public static func openSystemSettings(for permission: SystemPermissionKind) {+ NSWorkspace.shared.open(permission.settingsURL)+ }++ public static func requestAccessibilityPrompt() {+ let options: NSDictionary = ["AXTrustedCheckOptionPrompt": true]+ _ = AXIsProcessTrustedWithOptions(options)+ }+}++private struct TCCAuthorizationStore {+ let accessibility: Bool?+ let screenRecording: Bool?++ static var current: TCCAuthorizationStore {+ let database = TCCDatabase(path: "/Library/Application Support/com.apple.TCC/TCC.db")+ return TCCAuthorizationStore(+ accessibility: database.authorization(for: .accessibility),+ screenRecording: database.authorization(for: .screenRecording)+ )+ }+}++private struct TCCDatabase {+ enum Service: String {+ case accessibility = "kTCCServiceAccessibility"+ case screenRecording = "kTCCServiceScreenCapture"+ }++ private let path: String+ private let sqliteTransient = unsafeBitCast(-1, to: sqlite3_destructor_type.self)++ init(path: String) {+ self.path = path+ }++ func authorization(for service: Service) -> Bool? {+ guard let bundleIdentifier = Bundle.main.bundleIdentifier else {+ return nil+ }++ var database: OpaquePointer?+ guard sqlite3_open_v2(path, &database, SQLITE_OPEN_READONLY, nil) == SQLITE_OK else {+ if database != nil {+ sqlite3_close(database)+ }+ return nil+ }+ defer { sqlite3_close(database) }++ let query = """+ SELECT auth_value+ FROM access+ WHERE service = ? AND client = ?+ ORDER BY last_modified DESC+ LIMIT 1;+ """++ var statement: OpaquePointer?+ guard sqlite3_prepare_v2(database, query, -1, &statement, nil) == SQLITE_OK else {+ if statement != nil {+ sqlite3_finalize(statement)+ }+ return nil+ }+ defer { sqlite3_finalize(statement) }++ sqlite3_bind_text(statement, 1, service.rawValue, -1, sqliteTransient)+ sqlite3_bind_text(statement, 2, bundleIdentifier, -1, sqliteTransient)++ guard sqlite3_step(statement) == SQLITE_ROW else {+ return false+ }++ return sqlite3_column_int(statement, 0) == 2+ }+}diff --git a/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/ToolDefinitions.swift b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/ToolDefinitions.swiftnew file mode 100644index 0000000..95362e5--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Sources/OpenCodexComputerUseKit/ToolDefinitions.swift@@ -0,0 +1,177 @@+import Foundation++public struct ToolDefinition: @unchecked Sendable {+ public let name: String+ public let description: String+ public let inputSchema: [String: Any]++ public var asDictionary: [String: Any] {+ [+ "name": name,+ "description": description,+ "inputSchema": inputSchema,+ ]+ }+}++public enum ToolDefinitions {+ public static let all: [ToolDefinition] = [+ ToolDefinition(+ name: "list_apps",+ description: "List running or recently visible macOS applications.",+ inputSchema: objectSchema(properties: [:], required: [])+ ),+ ToolDefinition(+ name: "get_app_state",+ description: "Capture the current accessibility tree and screenshot metadata for an app.",+ inputSchema: objectSchema(+ properties: [+ "app": stringProperty(description: "App name or bundle identifier."),+ ],+ required: ["app"]+ )+ ),+ ToolDefinition(+ name: "click",+ description: "Click an accessibility element or a screenshot-relative coordinate.",+ inputSchema: objectSchema(+ properties: [+ "app": stringProperty(description: "App name or bundle identifier."),+ "element_index": stringProperty(description: "Accessibility element index from get_app_state."),+ "x": numberProperty(description: "Screenshot-relative X coordinate."),+ "y": numberProperty(description: "Screenshot-relative Y coordinate."),+ "click_count": integerProperty(description: "How many clicks to send.", defaultValue: 1),+ "mouse_button": stringProperty(+ description: "Mouse button to use.",+ enumValues: ["left", "right", "middle"],+ defaultValue: "left"+ ),+ ],+ required: ["app"]+ )+ ),+ ToolDefinition(+ name: "perform_secondary_action",+ description: "Invoke an accessibility secondary action exposed by an element.",+ inputSchema: objectSchema(+ properties: [+ "app": stringProperty(description: "App name or bundle identifier."),+ "element_index": stringProperty(description: "Accessibility element index from get_app_state."),+ "action": stringProperty(description: "Secondary action name from get_app_state."),+ ],+ required: ["app", "element_index", "action"]+ )+ ),+ ToolDefinition(+ name: "scroll",+ description: "Scroll an accessibility element in a direction by pages.",+ inputSchema: objectSchema(+ properties: [+ "app": stringProperty(description: "App name or bundle identifier."),+ "direction": stringProperty(description: "One of up, down, left, right.", enumValues: ["up", "down", "left", "right"]),+ "element_index": stringProperty(description: "Accessibility element index from get_app_state."),+ "pages": integerProperty(description: "Number of pages to scroll.", defaultValue: 1),+ ],+ required: ["app", "direction", "element_index"]+ )+ ),+ ToolDefinition(+ name: "drag",+ description: "Drag from one screenshot-relative coordinate to another.",+ inputSchema: objectSchema(+ properties: [+ "app": stringProperty(description: "App name or bundle identifier."),+ "from_x": numberProperty(description: "Screenshot-relative start X coordinate."),+ "from_y": numberProperty(description: "Screenshot-relative start Y coordinate."),+ "to_x": numberProperty(description: "Screenshot-relative end X coordinate."),+ "to_y": numberProperty(description: "Screenshot-relative end Y coordinate."),+ ],+ required: ["app", "from_x", "from_y", "to_x", "to_y"]+ )+ ),+ ToolDefinition(+ name: "type_text",+ description: "Type literal text into the focused control of an app.",+ inputSchema: objectSchema(+ properties: [+ "app": stringProperty(description: "App name or bundle identifier."),+ "text": stringProperty(description: "Literal text to type."),+ ],+ required: ["app", "text"]+ )+ ),+ ToolDefinition(+ name: "press_key",+ description: "Press a key or key combination using xdotool-style syntax.",+ inputSchema: objectSchema(+ properties: [+ "app": stringProperty(description: "App name or bundle identifier."),+ "key": stringProperty(description: "Key or chord such as Return, Tab, command+a, or super+c."),+ ],+ required: ["app", "key"]+ )+ ),+ ToolDefinition(+ name: "set_value",+ description: "Set the value of a settable accessibility element.",+ inputSchema: objectSchema(+ properties: [+ "app": stringProperty(description: "App name or bundle identifier."),+ "element_index": stringProperty(description: "Accessibility element index from get_app_state."),+ "value": stringProperty(description: "String value to assign."),+ ],+ required: ["app", "element_index", "value"]+ )+ ),+ ]+}++private func objectSchema(properties: [String: Any], required: [String]) -> [String: Any] {+ var schema: [String: Any] = [+ "type": "object",+ "properties": properties,+ ]++ if !required.isEmpty {+ schema["required"] = required+ }++ return schema+}++private func stringProperty(description: String, enumValues: [String]? = nil, defaultValue: String? = nil) -> [String: Any] {+ var property: [String: Any] = [+ "type": "string",+ "description": description,+ ]++ if let enumValues {+ property["enum"] = enumValues+ }++ if let defaultValue {+ property["default"] = defaultValue+ }++ return property+}++private func integerProperty(description: String, defaultValue: Int? = nil) -> [String: Any] {+ var property: [String: Any] = [+ "type": "integer",+ "description": description,+ ]++ if let defaultValue {+ property["default"] = defaultValue+ }++ return property+}++private func numberProperty(description: String) -> [String: Any] {+ [+ "type": "number",+ "description": description,+ ]+}diff --git a/packages/OpenCodexComputerUseKit/Tests/OpenCodexComputerUseKitTests/OpenCodexComputerUseKitTests.swift b/packages/OpenCodexComputerUseKit/Tests/OpenCodexComputerUseKitTests/OpenCodexComputerUseKitTests.swiftnew file mode 100644index 0000000..4a01fb2--- /dev/null+++ b/packages/OpenCodexComputerUseKit/Tests/OpenCodexComputerUseKitTests/OpenCodexComputerUseKitTests.swift@@ -0,0 +1,31 @@+import XCTest+@testable import OpenCodexComputerUseKit++final class OpenCodexComputerUseKitTests: XCTestCase {+ func testToolDefinitionCount() {+ XCTAssertEqual(ToolDefinitions.all.count, 9)+ }++ func testKeyPressParserSupportsCommandStyleChord() throws {+ let parsed = try KeyPressParser.parse("super+c")+ XCTAssertEqual(parsed.displayValue, "c")+ XCTAssertEqual(parsed.modifiers.count, 1)+ }++ func testInitializeResponseContainsToolsCapability() throws {+ let server = StdioMCPServer(service: ComputerUseService())+ let response = server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","clientInfo":{"name":"test","version":"0.1.0"},"capabilities":{}}}"#)+ XCTAssertNotNil(response)+ XCTAssertTrue(response!.contains(#""tools":{"listChanged":false}"#))+ }++ func testWindowRelativeFrameUsesSharedGlobalCoordinates() {+ let window = CGRect(x: 1486, y: 556, width: 919, height: 644)+ let child = CGRect(x: 1486, y: 556, width: 919, height: 644)+ let textField = CGRect(x: 180, y: 176, width: 36, height: 18)+ let textFieldGlobal = CGRect(x: window.minX + textField.minX, y: window.minY + textField.minY, width: textField.width, height: textField.height)++ XCTAssertEqual(windowRelativeFrame(elementFrame: child, windowBounds: window), CGRect(x: 0, y: 0, width: 919, height: 644))+ XCTAssertEqual(windowRelativeFrame(elementFrame: textFieldGlobal, windowBounds: window), textField)+ }+}diff --git a/scripts/build-open-codex-app.sh b/scripts/build-open-codex-app.shnew file mode 100755index 0000000..5179a35--- /dev/null+++ b/scripts/build-open-codex-app.sh@@ -0,0 +1,58 @@+#!/usr/bin/env bash++set -euo pipefail++repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+configuration="${1:-debug}"++cd "${repo_root}"++binary_dir="$(swift build -c "${configuration}" --show-bin-path)"+swift build -c "${configuration}" --product OpenCodexComputerUse++display_app_root="${repo_root}/dist/Open Codex Computer Use.app"+app_root="${repo_root}/dist/OpenCodexComputerUse.app"+contents_dir="${app_root}/Contents"+macos_dir="${contents_dir}/MacOS"+resources_dir="${contents_dir}/Resources"++rm -rf "${display_app_root}" "${app_root}"+mkdir -p "${macos_dir}" "${resources_dir}"++cp "${binary_dir}/OpenCodexComputerUse" "${macos_dir}/OpenCodexComputerUse"+chmod +x "${macos_dir}/OpenCodexComputerUse"++cat > "${contents_dir}/Info.plist" <<'PLIST'+<?xml version="1.0" encoding="UTF-8"?>+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">+<plist version="1.0">+<dict>+ <key>CFBundleDevelopmentRegion</key>+ <string>en</string>+ <key>CFBundleExecutable</key>+ <string>OpenCodexComputerUse</string>+ <key>CFBundleIdentifier</key>+ <string>dev.opencodex.OpenCodexComputerUse</string>+ <key>CFBundleInfoDictionaryVersion</key>+ <string>6.0</string>+ <key>CFBundleName</key>+ <string>Open Codex Computer Use</string>+ <key>CFBundleDisplayName</key>+ <string>Open Codex Computer Use</string>+ <key>CFBundlePackageType</key>+ <string>APPL</string>+ <key>CFBundleShortVersionString</key>+ <string>0.1.0</string>+ <key>CFBundleVersion</key>+ <string>1</string>+ <key>LSMinimumSystemVersion</key>+ <string>14.0</string>+ <key>NSPrincipalClass</key>+ <string>NSApplication</string>+</dict>+</plist>+PLIST++plutil -lint "${contents_dir}/Info.plist" >/dev/null++echo "Built ${app_root}"diff --git a/scripts/run-tool-smoke-tests.sh b/scripts/run-tool-smoke-tests.shnew file mode 100755index 0000000..55de02d--- /dev/null+++ b/scripts/run-tool-smoke-tests.sh@@ -0,0 +1,10 @@+#!/usr/bin/env bash++set -euo pipefail++repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"++cd "${repo_root}"++swift build+".build/debug/OpenCodexComputerUseSmokeSuite"