Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

Align WebArea container rendering

Leoc1115eeb4667
diff --git a/docs/histories/2026-05/20260507-1846-webarea-container-rendering.md b/docs/histories/2026-05/20260507-1846-webarea-container-rendering.md
new file mode 100644
index 0000000..deb540c
--- /dev/null
+++ b/docs/histories/2026-05/20260507-1846-webarea-container-rendering.md
@@ -0,0 +1,28 @@
+## [2026-05-07 18:46] | Task: Align WebArea container rendering
+
+### Execution Context
+* **Agent ID**: `Codex`
+* **Base Model**: `GPT-5.5`
+* **Runtime**: `Codex CLI, macOS`
+
+### User Query
+> Continue optimizing `open-computer-use` toward official `computer-use`, using Lark / Electron comparisons and reverse-engineering where useful.
+
+### Changes Overview
+**Scope:** `OpenComputerUseKit` accessibility snapshot rendering.
+
+**Key Actions:**
+- Compared current Feishu / Lark output from official bundled `computer-use` `1.0.770` with the local `open-computer-use` source build.
+- Used strings/symbol inspection on the official bundled app to confirm WebArea/Electron-specific accessibility paths are present in the current official client/service.
+- Adjusted generic wrapper elision so WebArea containers keep shallow layout wrappers and deeper branching containers, while still eliding deep single-child wrapper chains.
+- Added unit coverage for the WebArea-specific elision boundary.
+
+### Verification
+- `OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY=1 swift run OpenComputerUse call get_app_state --args '{"app":"com.electron.lark"}'`
+ - Confirmed the sampled tree remains free of `Scroll To Visible` and `selectable` noise.
+ - Confirmed `container -> text + image`, `text entry area`, `SideEdgeView`, `menu bar`, and `zoom the window` remain present.
+ - Confirmed the Lark node count moved close to the official sample in the same session: local source last index `389`, official `computer-use` sample last index `395`.
+
+### Files Modified
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index a26935f..6ffac2c 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -30,6 +30,7 @@ enum SnapshotMode {
let accessibilityTreeMaxNodeCount = 1200
let accessibilityTreeMaxDepth = 64
let screenshotCaptureTimeout: TimeInterval = 5
+private let axWebAreaRole = "AXWebArea"
public struct AppSnapshot {
public let app: RunningAppDescriptor
@@ -385,6 +386,7 @@ private struct TreeRenderer {
let traits = summarizeTraits(of: root)
let actions = copyActions(root) ?? []
let prettyActions = meaningfulActions(actions, role: role)
+ let webAreaDepth = webAreaDepth(role: role, ancestors: ancestors)
let localFrame = resolveLocalFrame(of: root, windowBounds: context.windowBounds)
let rowTexts = role == kAXRowRole as String ? flattenedRowTexts(of: root) : []
let childElements = children(of: root)
@@ -436,7 +438,8 @@ private struct TreeRenderer {
traits: traits,
actions: prettyActions,
childCount: childElements.count,
- genericTextSummary: genericTextSummary
+ genericTextSummary: genericTextSummary,
+ webAreaDepth: webAreaDepth
) {
for child in childElements {
render(child, depth: depth, ancestors: nextAncestors)
@@ -544,6 +547,20 @@ private struct TreeRenderer {
String(CFHash(element))
}
+ private func webAreaDepth(role: String, ancestors: [AXUIElement]) -> Int? {
+ if role == axWebAreaRole {
+ return 0
+ }
+
+ guard let webAreaIndex = ancestors.firstIndex(where: { ancestor in
+ stringValue(of: ancestor, attribute: kAXRoleAttribute) == axWebAreaRole
+ }) else {
+ return nil
+ }
+
+ return ancestors.count - webAreaIndex
+ }
+
private func children(of element: AXUIElement) -> [AXUIElement] {
let attributes = [kAXChildrenAttribute, kAXRowsAttribute]
var children: [AXUIElement] = []
@@ -970,7 +987,8 @@ func shouldElideNode(
traits: [String],
actions: [String],
childCount: Int,
- genericTextSummary: String? = nil
+ genericTextSummary: String? = nil,
+ webAreaDepth: Int? = nil
) -> Bool {
let genericRoles = [kAXGroupRole as String, kAXUnknownRole as String]
guard genericRoles.contains(role) else {
@@ -981,6 +999,10 @@ func shouldElideNode(
return false
}
+ if shouldPreserveWebAreaGenericContainer(childCount: childCount, webAreaDepth: webAreaDepth) {
+ return false
+ }
+
return title == nil
&& label == nil
&& value == nil
@@ -989,6 +1011,14 @@ func shouldElideNode(
&& actions.isEmpty
}
+func shouldPreserveWebAreaGenericContainer(childCount: Int, webAreaDepth: Int?) -> Bool {
+ guard childCount > 0, let webAreaDepth else {
+ return false
+ }
+
+ return childCount > 1 || webAreaDepth <= 4
+}
+
private func shouldSuppressChildren(
role: String,
title: String?,
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index 135d72f..00be210 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -590,6 +590,50 @@ final class OpenComputerUseKitTests: XCTestCase {
childCount: 3,
genericTextSummary: "AgentSphere 17:18 okay"
))
+ XCTAssertFalse(shouldElideNode(
+ role: kAXGroupRole as String,
+ title: nil,
+ label: nil,
+ value: nil,
+ identifier: nil,
+ traits: [],
+ actions: [],
+ childCount: 3,
+ webAreaDepth: 4
+ ))
+ XCTAssertTrue(shouldElideNode(
+ role: kAXGroupRole as String,
+ title: nil,
+ label: nil,
+ value: nil,
+ identifier: nil,
+ traits: [],
+ actions: [],
+ childCount: 0,
+ webAreaDepth: 4
+ ))
+ XCTAssertFalse(shouldElideNode(
+ role: kAXGroupRole as String,
+ title: nil,
+ label: nil,
+ value: nil,
+ identifier: nil,
+ traits: [],
+ actions: [],
+ childCount: 2,
+ webAreaDepth: 8
+ ))
+ XCTAssertTrue(shouldElideNode(
+ role: kAXGroupRole as String,
+ title: nil,
+ label: nil,
+ value: nil,
+ identifier: nil,
+ traits: [],
+ actions: [],
+ childCount: 1,
+ webAreaDepth: 8
+ ))
}
func testAccessibilityRendererOnlyMergesShortTextOnlySiblingRuns() {