👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
Require AX window for snapshots
Leoe14c5c35de61
diff --git a/docs/histories/2026-05/20260507-1905-require-ax-window-for-snapshot.md b/docs/histories/2026-05/20260507-1905-require-ax-window-for-snapshot.mdnew file mode 100644index 0000000..e0b3745--- /dev/null+++ b/docs/histories/2026-05/20260507-1905-require-ax-window-for-snapshot.md@@ -0,0 +1,22 @@+## [2026-05-07 19:05] | Task: Align unavailable window state++### 🤖 Execution Context+* **Agent ID**: `Codex`+* **Base Model**: `GPT-5`+* **Runtime**: `macOS local shell`++### 📥 User Query+> 对比最新版 open-computer-use 和官方 computer-use 的工具返回,继续修正差异。++### 🛠 Changes Overview+**Scope:** `packages/OpenComputerUseKit`++**Key Actions:**+- **[Snapshot Guard]**: `get_app_state` now requires a real AX window before rendering an accessibility tree.+- **[Role Filtering]**: Focused-window and first-window candidates are filtered to `AXWindow`, avoiding misleading app-root-only trees when an app exposes no usable key window.++### 🧠 Design Intent (Why)+Official `computer-use` returns an unavailable-window error for the observed Lark state, while open-computer-use previously rendered only the application/menu-bar root and implied the app was actionable. Requiring a real accessibility window makes the failure explicit and prevents stale or non-actionable element indexes from reaching follow-up tools.++### 📁 Files Modified+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swiftindex 6ffac2c..e454c7a 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift@@ -91,11 +91,13 @@ enum SnapshotBuilder {enableBestEffortAccessibilityModes(appElement)let systemWide = AXUIElementCreateSystemWide()let focusedApplication = copyElement(systemWide, attribute: kAXFocusedApplicationAttribute)- let focusedWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)- let rootElement = focusedWindow ?? appElement- let windowTitle = stringValue(of: focusedWindow ?? appElement, attribute: kAXTitleAttribute)+ guard let focusedWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) else {+ throw ComputerUseError.stateUnavailable("No key window is available for \(app.bundleIdentifier ?? app.name). Bring the app window on screen and call get_app_state again.")+ }+ let windowTitle = stringValue(of: focusedWindow, attribute: kAXTitleAttribute)let windowCapture = WindowCapture.resolve(for: app.pid, titleHint: windowTitle)+ let rootElement = focusedWindowlet windowBounds = windowCapture?.boundslet screenshotPNGData = windowCapture?.pngDataIfAvailable()let focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)@@ -104,8 +106,7 @@ enum SnapshotBuilder {var renderer = TreeRenderer(context: context)renderer.render(rootElement)- if focusedWindow != nil,- let menuBar = copyElement(appElement, attribute: kAXMenuBarAttribute),+ if let menuBar = copyElement(appElement, attribute: kAXMenuBarAttribute),!CFEqual(menuBar, rootElement){renderer.render(menuBar)@@ -131,19 +132,31 @@ enum SnapshotBuilder {return nil}- return windows.first+ return windows.first(where: isWindowElement(_:))}private static func preferredFocusedWindow(appElement: AXUIElement, appPID: pid_t, focusedApplication: AXUIElement?, systemWide: AXUIElement) -> AXUIElement? {if let focusedApplication, pid(of: focusedApplication) == appPID {- return copyElement(systemWide, attribute: kAXFocusedWindowAttribute)- ?? copyElement(focusedApplication, attribute: kAXFocusedWindowAttribute)+ return windowElement(from: copyElement(systemWide, attribute: kAXFocusedWindowAttribute))+ ?? windowElement(from: copyElement(focusedApplication, attribute: kAXFocusedWindowAttribute))?? firstWindow(for: focusedApplication)- ?? copyElement(appElement, attribute: kAXFocusedWindowAttribute)+ ?? windowElement(from: copyElement(appElement, attribute: kAXFocusedWindowAttribute))?? firstWindow(for: appElement)}- return copyElement(appElement, attribute: kAXFocusedWindowAttribute) ?? firstWindow(for: appElement)+ return windowElement(from: copyElement(appElement, attribute: kAXFocusedWindowAttribute)) ?? firstWindow(for: appElement)+ }++ private static func windowElement(from element: AXUIElement?) -> AXUIElement? {+ guard let element, isWindowElement(element) else {+ return nil+ }++ return element+ }++ private static func isWindowElement(_ element: AXUIElement) -> Bool {+ stringValue(of: element, attribute: kAXRoleAttribute) == kAXWindowRole as String}private static func preferredFocusedElement(appElement: AXUIElement, appPID: pid_t, focusedApplication: AXUIElement?, systemWide: AXUIElement) -> AXUIElement? {