👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
Align WebArea container rendering
Leoc1115eeb4667
diff --git a/docs/histories/2026-05/20260507-1846-webarea-container-rendering.md b/docs/histories/2026-05/20260507-1846-webarea-container-rendering.mdnew file mode 100644index 0000000..deb540c--- /dev/null+++ b/docs/histories/2026-05/20260507-1846-webarea-container-rendering.md@@ -0,0 +1,28 @@+## [2026-05-07 18:46] | Task: Align WebArea container rendering++### Execution Context+* **Agent ID**: `Codex`+* **Base Model**: `GPT-5.5`+* **Runtime**: `Codex CLI, macOS`++### User Query+> Continue optimizing `open-computer-use` toward official `computer-use`, using Lark / Electron comparisons and reverse-engineering where useful.++### Changes Overview+**Scope:** `OpenComputerUseKit` accessibility snapshot rendering.++**Key Actions:**+- Compared current Feishu / Lark output from official bundled `computer-use` `1.0.770` with the local `open-computer-use` source build.+- Used strings/symbol inspection on the official bundled app to confirm WebArea/Electron-specific accessibility paths are present in the current official client/service.+- Adjusted generic wrapper elision so WebArea containers keep shallow layout wrappers and deeper branching containers, while still eliding deep single-child wrapper chains.+- Added unit coverage for the WebArea-specific elision boundary.++### Verification+- `OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY=1 swift run OpenComputerUse call get_app_state --args '{"app":"com.electron.lark"}'`+ - Confirmed the sampled tree remains free of `Scroll To Visible` and `selectable` noise.+ - Confirmed `container -> text + image`, `text entry area`, `SideEdgeView`, `menu bar`, and `zoom the window` remain present.+ - Confirmed the Lark node count moved close to the official sample in the same session: local source last index `389`, official `computer-use` sample last index `395`.++### Files Modified+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swiftindex a26935f..6ffac2c 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift@@ -30,6 +30,7 @@ enum SnapshotMode {let accessibilityTreeMaxNodeCount = 1200let accessibilityTreeMaxDepth = 64let screenshotCaptureTimeout: TimeInterval = 5+private let axWebAreaRole = "AXWebArea"public struct AppSnapshot {public let app: RunningAppDescriptor@@ -385,6 +386,7 @@ private struct TreeRenderer {let traits = summarizeTraits(of: root)let actions = copyActions(root) ?? []let prettyActions = meaningfulActions(actions, role: role)+ let webAreaDepth = webAreaDepth(role: role, ancestors: ancestors)let localFrame = resolveLocalFrame(of: root, windowBounds: context.windowBounds)let rowTexts = role == kAXRowRole as String ? flattenedRowTexts(of: root) : []let childElements = children(of: root)@@ -436,7 +438,8 @@ private struct TreeRenderer {traits: traits,actions: prettyActions,childCount: childElements.count,- genericTextSummary: genericTextSummary+ genericTextSummary: genericTextSummary,+ webAreaDepth: webAreaDepth) {for child in childElements {render(child, depth: depth, ancestors: nextAncestors)@@ -544,6 +547,20 @@ private struct TreeRenderer {String(CFHash(element))}+ private func webAreaDepth(role: String, ancestors: [AXUIElement]) -> Int? {+ if role == axWebAreaRole {+ return 0+ }++ guard let webAreaIndex = ancestors.firstIndex(where: { ancestor in+ stringValue(of: ancestor, attribute: kAXRoleAttribute) == axWebAreaRole+ }) else {+ return nil+ }++ return ancestors.count - webAreaIndex+ }+private func children(of element: AXUIElement) -> [AXUIElement] {let attributes = [kAXChildrenAttribute, kAXRowsAttribute]var children: [AXUIElement] = []@@ -970,7 +987,8 @@ func shouldElideNode(traits: [String],actions: [String],childCount: Int,- genericTextSummary: String? = nil+ genericTextSummary: String? = nil,+ webAreaDepth: Int? = nil) -> Bool {let genericRoles = [kAXGroupRole as String, kAXUnknownRole as String]guard genericRoles.contains(role) else {@@ -981,6 +999,10 @@ func shouldElideNode(return false}+ if shouldPreserveWebAreaGenericContainer(childCount: childCount, webAreaDepth: webAreaDepth) {+ return false+ }+return title == nil&& label == nil&& value == nil@@ -989,6 +1011,14 @@ func shouldElideNode(&& actions.isEmpty}+func shouldPreserveWebAreaGenericContainer(childCount: Int, webAreaDepth: Int?) -> Bool {+ guard childCount > 0, let webAreaDepth else {+ return false+ }++ return childCount > 1 || webAreaDepth <= 4+}+private func shouldSuppressChildren(role: String,title: String?,diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swiftindex 135d72f..00be210 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift@@ -590,6 +590,50 @@ final class OpenComputerUseKitTests: XCTestCase {childCount: 3,genericTextSummary: "AgentSphere 17:18 okay"))+ XCTAssertFalse(shouldElideNode(+ role: kAXGroupRole as String,+ title: nil,+ label: nil,+ value: nil,+ identifier: nil,+ traits: [],+ actions: [],+ childCount: 3,+ webAreaDepth: 4+ ))+ XCTAssertTrue(shouldElideNode(+ role: kAXGroupRole as String,+ title: nil,+ label: nil,+ value: nil,+ identifier: nil,+ traits: [],+ actions: [],+ childCount: 0,+ webAreaDepth: 4+ ))+ XCTAssertFalse(shouldElideNode(+ role: kAXGroupRole as String,+ title: nil,+ label: nil,+ value: nil,+ identifier: nil,+ traits: [],+ actions: [],+ childCount: 2,+ webAreaDepth: 8+ ))+ XCTAssertTrue(shouldElideNode(+ role: kAXGroupRole as String,+ title: nil,+ label: nil,+ value: nil,+ identifier: nil,+ traits: [],+ actions: [],+ childCount: 1,+ webAreaDepth: 8+ ))}func testAccessibilityRendererOnlyMergesShortTextOnlySiblingRuns() {