👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
fix permission onboarding panel tracking
Leo42ef8fc08f68
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swiftindex a831785..20607da 100644--- a/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift@@ -1,4 +1,3 @@-import ApplicationServicesimport AppKitimport OpenComputerUseKit@@ -415,6 +414,17 @@ final class PermissionAccessoryPanelController {private var globalDragMonitor: Any?private var localDragMonitor: Any?+ private enum Layout {+ static let screenHorizontalInset: CGFloat = 16+ static let screenBottomInset: CGFloat = 12+ static let windowBottomOverlap: CGFloat = 6+ static let contentLeadingInset: CGFloat = 26+ static let contentTrailingInset: CGFloat = 28+ static let sidebarWidthRatio: CGFloat = 0.29+ static let sidebarWidthMin: CGFloat = 214+ static let sidebarWidthMax: CGFloat = 272+ }+func show(for permission: SystemPermissionKind) {currentPermission = permission@@ -549,28 +559,45 @@ final class PermissionAccessoryPanelController {}private func preferredPanelOrigin(for panelSize: CGSize) -> CGPoint? {- guard let anchor = systemSettingsControlAnchorRect() else {- return fallbackPanelOrigin(for: panelSize)- }-- let visibleFrame = targetVisibleScreenFrame(for: anchor) ?? NSScreen.main?.visibleFrame ?? .zero- let x = clamp(anchor.minX - 18, lower: visibleFrame.minX + 16, upper: visibleFrame.maxX - panelSize.width - 16)- let y = max(visibleFrame.minY + 12, anchor.minY - panelSize.height - 10)- return CGPoint(x: x, y: y)- }-- private func fallbackPanelOrigin(for panelSize: CGSize) -> CGPoint? {guard let referenceRect = systemSettingsWindowBounds() ?? NSScreen.main?.visibleFrame else {return nil}let visibleFrame = targetVisibleScreenFrame(for: referenceRect) ?? referenceRect- let estimatedControlsMinX = referenceRect.minX + 234- let x = clamp(estimatedControlsMinX - 18, lower: visibleFrame.minX + 16, upper: visibleFrame.maxX - panelSize.width - 16)- let y = max(visibleFrame.minY + 12, referenceRect.minY - panelSize.height + 6)+ let trackRect = systemSettingsContentTrackRect(in: referenceRect)+ let x = clamp(+ trackRect.midX - (panelSize.width / 2),+ lower: visibleFrame.minX + Layout.screenHorizontalInset,+ upper: visibleFrame.maxX - panelSize.width - Layout.screenHorizontalInset+ )+ let y = max(+ visibleFrame.minY + Layout.screenBottomInset,+ referenceRect.minY - panelSize.height + Layout.windowBottomOverlap+ )return CGPoint(x: x, y: y)}+ private func systemSettingsContentTrackRect(in windowBounds: CGRect) -> CGRect {+ // Track the System Settings window itself instead of the +/- controls so+ // the accessory panel stays stable while the page contents relayout.+ let sidebarWidth = clamp(+ windowBounds.width * Layout.sidebarWidthRatio,+ lower: Layout.sidebarWidthMin,+ upper: Layout.sidebarWidthMax+ )+ let contentMinX = min(+ windowBounds.maxX - Layout.contentTrailingInset - 1,+ windowBounds.minX + sidebarWidth + Layout.contentLeadingInset+ )+ let contentMaxX = max(contentMinX + 1, windowBounds.maxX - Layout.contentTrailingInset)+ return CGRect(+ x: contentMinX,+ y: windowBounds.minY,+ width: contentMaxX - contentMinX,+ height: windowBounds.height+ )+ }+private func targetVisibleScreenFrame(for rect: CGRect) -> CGRect? {NSScreen.screens.first(where: { $0.visibleFrame.intersects(rect) })?.visibleFrame}@@ -587,45 +614,6 @@ final class PermissionAccessoryPanelController {return min(max(value, lower), upper)}- private func systemSettingsControlAnchorRect() -> CGRect? {- guard PermissionDiagnostics.current().accessibilityTrusted else {- return nil- }-- guard- let appElement = systemSettingsApplicationElement(),- let root = focusedSystemSettingsWindow(for: appElement) ?? firstWindow(for: appElement)- else {- return nil- }-- let controlFrames = descendantFrames(- in: root,- matchingTitles: ["Add", "Remove"],- role: kAXButtonRole as String- )-- guard !controlFrames.isEmpty else {- return nil- }-- let groups = Dictionary(grouping: controlFrames) { frame in- Int((frame.midY / 12).rounded(.toNearestOrAwayFromZero))- }-- guard let topGroup = groups.values.max(by: { lhs, rhs in- let lhsTop = lhs.map(\.maxY).max() ?? 0- let rhsTop = rhs.map(\.maxY).max() ?? 0- return lhsTop < rhsTop- }) else {- return nil- }-- return topGroup.reduce(topGroup[0]) { partial, frame in- partial.union(frame)- }- }-private func systemSettingsWindowBounds() -> CGRect? {guard let runningApp = NSWorkspace.shared.runningApplications.first(where: { $0.bundleIdentifier == "com.apple.systempreferences" }) else {return nil@@ -652,139 +640,6 @@ final class PermissionAccessoryPanelController {return windows.sorted(by: { $0.1 > $1.1 }).first?.0}-- private func systemSettingsApplicationElement() -> AXUIElement? {- guard let runningApp = NSWorkspace.shared.runningApplications.first(where: { $0.bundleIdentifier == "com.apple.systempreferences" }) else {- return nil- }-- return AXUIElementCreateApplication(runningApp.processIdentifier)- }-- private func focusedSystemSettingsWindow(for appElement: AXUIElement) -> AXUIElement? {- let systemWide = AXUIElementCreateSystemWide()-- if- let focusedApp = copyElement(systemWide, attribute: kAXFocusedApplicationAttribute),- pid(of: focusedApp) == pid(of: appElement)- {- return copyElement(systemWide, attribute: kAXFocusedWindowAttribute)- ?? copyElement(focusedApp, attribute: kAXFocusedWindowAttribute)- ?? firstWindow(for: focusedApp)- }-- return copyElement(appElement, attribute: kAXFocusedWindowAttribute)- }-- private func firstWindow(for appElement: AXUIElement) -> AXUIElement? {- copyArray(appElement, attribute: kAXWindowsAttribute)?.first- }-- private func descendantFrames(in root: AXUIElement, matchingTitles titles: Set<String>, role targetRole: String) -> [CGRect] {- var results: [CGRect] = []- var queue: [AXUIElement] = [root]- var visited: Set<CFHashCode> = []-- while let element = queue.first {- queue.removeFirst()-- let token = CFHash(element)- guard !visited.contains(token) else {- continue- }- visited.insert(token)-- if- stringValue(of: element, attribute: kAXRoleAttribute) == targetRole,- let title = stringValue(of: element, attribute: kAXTitleAttribute),- titles.contains(title),- let frame = frame(of: element)- {- results.append(frame)- }-- queue.append(contentsOf: childElements(of: element))- }-- return results- }-- private func childElements(of element: AXUIElement) -> [AXUIElement] {- let attributes = [kAXChildrenAttribute, kAXRowsAttribute]- var children: [AXUIElement] = []-- for attribute in attributes {- if let nested = copyArray(element, attribute: attribute) {- children.append(contentsOf: nested)- }- }-- return children- }-- private func copyElement(_ element: AXUIElement, attribute: String) -> AXUIElement? {- var value: CFTypeRef?- guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {- return nil- }-- guard let value else {- return nil- }-- return (value as! AXUIElement)- }-- private func copyArray(_ element: AXUIElement, attribute: String) -> [AXUIElement]? {- var value: CFTypeRef?- guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {- return nil- }-- return value as? [AXUIElement]- }-- private func stringValue(of element: AXUIElement, attribute: String) -> String? {- var value: CFTypeRef?- guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {- return nil- }-- return value as? String- }-- private func frame(of element: AXUIElement) -> CGRect? {- var positionValue: CFTypeRef?- var sizeValue: CFTypeRef?-- guard- AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success,- AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success,- let positionValue,- let sizeValue- else {- return nil- }-- let positionAXValue = positionValue as! AXValue- let sizeAXValue = sizeValue as! AXValue- var position = CGPoint.zero- var size = CGSize.zero- guard- AXValueGetValue(positionAXValue, .cgPoint, &position),- AXValueGetValue(sizeAXValue, .cgSize, &size)- else {- return nil- }-- return CGRect(origin: position, size: size)- }-- private func pid(of element: AXUIElement) -> pid_t {- var processIdentifier: pid_t = 0- AXUIElementGetPid(element, &processIdentifier)- return processIdentifier- }}@MainActordiff --git a/docs/exec-plans/active/20260417-permission-onboarding-app.md b/docs/exec-plans/active/20260417-permission-onboarding-app.mdindex 7844ddd..b2fd311 100644--- a/docs/exec-plans/active/20260417-permission-onboarding-app.md+++ b/docs/exec-plans/active/20260417-permission-onboarding-app.md@@ -73,7 +73,7 @@- 2026-04-17:权限 onboarding 直接做进 `OpenComputerUse` 主 target,而不是另起一个完全独立的 helper app。这样 `mcp` CLI 和 app bundle 可复用同一个可执行文件与 bundle 身份。- 2026-04-17:权限状态判定加入对 TCC 持久授权记录的读取,避免 dev 环境里 CLI 子进程与 GUI app 对同一 bundle 权限状态看到不一致的结果。-- 2026-04-17:drag panel 当前收敛到“只在 `System Settings` 前台时显示,并锚到 `Add` / `Remove` 控制区”的近似方案;更接近官方嵌入式 choreography 的视觉行为后续再补。+- 2026-04-17:drag panel 仍然只在 `System Settings` 前台时显示,但定位策略已从 `Add` / `Remove` 控制区改成跟随主窗口右侧内容区的底边,避免 panel 被列表局部控件牵着跑。- 2026-04-17:app 模式改成 `LSUIElement` + `.accessory` agent-style 运行,保证权限窗口可见,但执行过程中不再额外在 Dock 暴露前台 app 图标。## 当前结论diff --git a/docs/histories/2026-04/20260417-2122-fix-permission-panel-tracking.md b/docs/histories/2026-04/20260417-2122-fix-permission-panel-tracking.mdnew file mode 100644index 0000000..c9898cf--- /dev/null+++ b/docs/histories/2026-04/20260417-2122-fix-permission-panel-tracking.md@@ -0,0 +1,24 @@+## [2026-04-17 21:22] | Task: 修正权限引导 panel 跟随逻辑++### 🤖 Execution Context+* **Agent ID**: `Codex`+* **Base Model**: `GPT-5.4`+* **Runtime**: `Codex CLI / Swift 6.2.4 / macOS`++### 📥 User Query+> 打开授权页面后,点 `Allow` 时辅助窗口的跟随有问题;它会一直跟在 `System Settings` 的 `Accessibility` 窗口里 `+ / -` 下面,需要修复。++### 🛠 Changes Overview+**Scope:** `apps/OpenComputerUse`, `docs/`++**Key Actions:**+- **[改掉控件级锚点]**: 删除基于 `Accessibility` 页 `Add` / `Remove` 按钮区域的定位逻辑,不再用局部控件驱动辅助 panel 的位置。+- **[改成窗口级跟随]**: 辅助 panel 现在改为跟随 `System Settings` 主窗口右侧内容区的底边,并继续按可见屏幕范围做 clamp。+- **[同步文档]**: 更新架构说明和权限 onboarding execution plan,移除“锚到 `Add` / `Remove`”的过时描述。++### 🧠 Design Intent (Why)+这次修复的重点不是简单把 panel 换个坐标,而是把定位源从“页面里最脆弱的局部控件”切回“稳定的窗口级几何信息”。这样即使 `Accessibility` 页列表滚动、布局细节变化,拖拽辅助 panel 也会继续贴着 `System Settings` 主窗口走,而不是显得像被 `+ / -` 区域绑住。++### 📁 Files Modified+- `apps/OpenComputerUse/Sources/OpenComputerUse/PermissionOnboardingApp.swift`+- `docs/exec-plans/active/20260417-permission-onboarding-app.md`