Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

Require AX window for snapshots

Leoe14c5c35de61
diff --git a/docs/histories/2026-05/20260507-1905-require-ax-window-for-snapshot.md b/docs/histories/2026-05/20260507-1905-require-ax-window-for-snapshot.md
new file mode 100644
index 0000000..e0b3745
--- /dev/null
+++ b/docs/histories/2026-05/20260507-1905-require-ax-window-for-snapshot.md
@@ -0,0 +1,22 @@
+## [2026-05-07 19:05] | Task: Align unavailable window state
+
+### 🤖 Execution Context
+* **Agent ID**: `Codex`
+* **Base Model**: `GPT-5`
+* **Runtime**: `macOS local shell`
+
+### 📥 User Query
+> 对比最新版 open-computer-use 和官方 computer-use 的工具返回,继续修正差异。
+
+### 🛠 Changes Overview
+**Scope:** `packages/OpenComputerUseKit`
+
+**Key Actions:**
+- **[Snapshot Guard]**: `get_app_state` now requires a real AX window before rendering an accessibility tree.
+- **[Role Filtering]**: Focused-window and first-window candidates are filtered to `AXWindow`, avoiding misleading app-root-only trees when an app exposes no usable key window.
+
+### 🧠 Design Intent (Why)
+Official `computer-use` returns an unavailable-window error for the observed Lark state, while open-computer-use previously rendered only the application/menu-bar root and implied the app was actionable. Requiring a real accessibility window makes the failure explicit and prevents stale or non-actionable element indexes from reaching follow-up tools.
+
+### 📁 Files Modified
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index 6ffac2c..e454c7a 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -91,11 +91,13 @@ enum SnapshotBuilder {
enableBestEffortAccessibilityModes(appElement)
let systemWide = AXUIElementCreateSystemWide()
let focusedApplication = copyElement(systemWide, attribute: kAXFocusedApplicationAttribute)
- let focusedWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)
- let rootElement = focusedWindow ?? appElement
- let windowTitle = stringValue(of: focusedWindow ?? appElement, attribute: kAXTitleAttribute)
+ guard let focusedWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) else {
+ throw ComputerUseError.stateUnavailable("No key window is available for \(app.bundleIdentifier ?? app.name). Bring the app window on screen and call get_app_state again.")
+ }
+ let windowTitle = stringValue(of: focusedWindow, attribute: kAXTitleAttribute)
let windowCapture = WindowCapture.resolve(for: app.pid, titleHint: windowTitle)
+ let rootElement = focusedWindow
let windowBounds = windowCapture?.bounds
let screenshotPNGData = windowCapture?.pngDataIfAvailable()
let focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)
@@ -104,8 +106,7 @@ enum SnapshotBuilder {
var renderer = TreeRenderer(context: context)
renderer.render(rootElement)
- if focusedWindow != nil,
- let menuBar = copyElement(appElement, attribute: kAXMenuBarAttribute),
+ if let menuBar = copyElement(appElement, attribute: kAXMenuBarAttribute),
!CFEqual(menuBar, rootElement)
{
renderer.render(menuBar)
@@ -131,19 +132,31 @@ enum SnapshotBuilder {
return nil
}
- return windows.first
+ return windows.first(where: isWindowElement(_:))
}
private static func preferredFocusedWindow(appElement: AXUIElement, appPID: pid_t, focusedApplication: AXUIElement?, systemWide: AXUIElement) -> AXUIElement? {
if let focusedApplication, pid(of: focusedApplication) == appPID {
- return copyElement(systemWide, attribute: kAXFocusedWindowAttribute)
- ?? copyElement(focusedApplication, attribute: kAXFocusedWindowAttribute)
+ return windowElement(from: copyElement(systemWide, attribute: kAXFocusedWindowAttribute))
+ ?? windowElement(from: copyElement(focusedApplication, attribute: kAXFocusedWindowAttribute))
?? firstWindow(for: focusedApplication)
- ?? copyElement(appElement, attribute: kAXFocusedWindowAttribute)
+ ?? windowElement(from: copyElement(appElement, attribute: kAXFocusedWindowAttribute))
?? firstWindow(for: appElement)
}
- return copyElement(appElement, attribute: kAXFocusedWindowAttribute) ?? firstWindow(for: appElement)
+ return windowElement(from: copyElement(appElement, attribute: kAXFocusedWindowAttribute)) ?? firstWindow(for: appElement)
+ }
+
+ private static func windowElement(from element: AXUIElement?) -> AXUIElement? {
+ guard let element, isWindowElement(element) else {
+ return nil
+ }
+
+ return element
+ }
+
+ private static func isWindowElement(_ element: AXUIElement) -> Bool {
+ stringValue(of: element, attribute: kAXRoleAttribute) == kAXWindowRole as String
}
private static func preferredFocusedElement(appElement: AXUIElement, appPID: pid_t, focusedApplication: AXUIElement?, systemWide: AXUIElement) -> AXUIElement? {