Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

Polish accessibility snapshot formatting

Leobede453c0920
diff --git a/docs/histories/2026-05/20260507-1825-polish-accessibility-formatting.md b/docs/histories/2026-05/20260507-1825-polish-accessibility-formatting.md
new file mode 100644
index 0000000..6f5d8ce
--- /dev/null
+++ b/docs/histories/2026-05/20260507-1825-polish-accessibility-formatting.md
@@ -0,0 +1,28 @@
+## [2026-05-07 18:25] | Task: Polish accessibility snapshot formatting
+
+### Execution Context
+* **Agent ID**: `Codex`
+* **Base Model**: `GPT-5.5`
+* **Runtime**: `Codex CLI, macOS`
+
+### User Query
+> Continue optimizing `open-computer-use` toward official `computer-use`, comparing Lark / Electron returns and using reverse-engineering where useful.
+
+### Changes Overview
+**Scope:** `OpenComputerUseKit` accessibility snapshot rendering.
+
+**Key Actions:**
+- Matched the official window button help/action format by rendering `Help:` without a leading comma and adding the comma before `Secondary Actions` when prior metadata exists.
+- Special-cased `AXZoomWindow` to render as `zoom the window`, matching the official full-screen button output.
+- Restored short text-summary wrapper nodes to `container` instead of `text`, which better matches official Electron rows where compact text usually lives under a container parent.
+
+### Verification
+- `OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY=1 swift run OpenComputerUse call get_app_state --args '{"app":"com.electron.lark"}'`
+ - Confirmed `full screen button Help: this button also has an action to zoom the window, Secondary Actions: zoom the window`.
+ - Confirmed short Lark message summaries render as `container ...`.
+ - Confirmed `text entry area`, `SideEdgeView`, and focused `HTML 内容 messenger-chat, URL: ...` are still present.
+ - Confirmed no `Scroll To Visible` or `selectable` noise in the sampled tree.
+
+### Files Modified
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index 4f048bb..5b0fa0c 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -452,7 +452,7 @@ private struct TreeRenderer {
if help == title || help == label {
return ""
}
- return ", Help: \(help)"
+ return " Help: \(help)"
}()
let urlSegment = formattedURLSegment(for: root, title: title, label: label)
let identifierSegment = displayIdentifierSegment(for: root, role: role, identifier: axIdentifier, title: title)
@@ -461,7 +461,12 @@ private struct TreeRenderer {
rawValueSegment,
precedingSegments: [labelSegment, helpSegment, urlSegment, identifierSegment]
)
- let actionsPrefix = (title != nil || inlineRowSummary != nil) ? ", Secondary Actions: " : " Secondary Actions: "
+ let actionsPrefix = shouldCommaSeparateActions(
+ title: title,
+ inlineRowSummary: inlineRowSummary,
+ genericTextSummary: genericTextSummary,
+ segments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment]
+ ) ? ", Secondary Actions: " : " Secondary Actions: "
let actionsSegment = prettyActions.isEmpty ? "" : "\(actionsPrefix)\(prettyActions.joined(separator: ", "))"
let linePrefix = roleText.isEmpty ? "\(index)" : "\(index) \(roleText)"
@@ -834,6 +839,18 @@ private func formattedValueSegmentWithSeparator(_ valueSegment: String, precedin
return ",\(valueSegment)"
}
+private func shouldCommaSeparateActions(
+ title: String?,
+ inlineRowSummary: String?,
+ genericTextSummary: String?,
+ segments: [String]
+) -> Bool {
+ title != nil
+ || inlineRowSummary != nil
+ || genericTextSummary != nil
+ || segments.contains(where: { !$0.isEmpty })
+}
+
private func formattedURLSegment(for element: AXUIElement, title: String?, label: String?) -> String {
guard stringValue(of: element, attribute: kAXRoleAttribute) == "AXWebArea" else {
return ""
@@ -1034,7 +1051,7 @@ private func displayRoleText(
}
if suppressChildren {
- return "text"
+ return "container"
}
if baseRoleText == "radio group", role == kAXRadioGroupRole as String, title == nil, label != nil {
@@ -1119,6 +1136,10 @@ func meaningfulActions(_ values: [String], role: String) -> [String] {
}
private func prettyActionName(_ value: String) -> String {
+ if value == "AXZoomWindow" {
+ return "zoom the window"
+ }
+
let stripped = value.hasPrefix("AX") ? String(value.dropFirst(2)) : value
let withoutPage = stripped.replacingOccurrences(of: "ByPage", with: "")
return splitCamelCase(withoutPage)
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index 21e8279..3eef160 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -628,6 +628,10 @@ final class OpenComputerUseKitTests: XCTestCase {
)
}
+ func testAccessibilityRendererUsesOfficialZoomWindowActionName() {
+ XCTAssertEqual(meaningfulActions(["AXZoomWindow"], role: kAXButtonRole as String), ["zoom the window"])
+ }
+
func testBlockingAsyncBridgeTimesOutScreenshotWork() {
XCTAssertThrowsError(
try BlockingAsyncBridge.run(timeout: 0.01) {