👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
Render accessibility placeholder values
Leo1f4663daf90b
diff --git a/docs/histories/2026-05/20260507-2010-render-ax-placeholder.md b/docs/histories/2026-05/20260507-2010-render-ax-placeholder.mdnew file mode 100644index 0000000..983c374--- /dev/null+++ b/docs/histories/2026-05/20260507-2010-render-ax-placeholder.md@@ -0,0 +1,26 @@+# 渲染 AX placeholder 值++## 用户诉求++继续对齐开源版 `open-computer-use` 和官方 `computer-use` 的 app state 输出,尤其是 Electron / Browser 这类复杂 AX tree 的返回形状。++## 主要改动++- 在 AX renderer 中读取 `AXPlaceholderValue` / `AXPlaceholder`。+- 当 placeholder 不等于 title、description 或 value 时,在对应元素行追加 `Placeholder: ...`。+- 增加 placeholder segment 的单元测试,避免重复渲染 description 或已有 value。++## 设计动机++官方 Chrome 输出会在地址栏元素上保留 placeholder,例如 `Ask Google or type a URL`。开源版之前只输出 description 和 value,缺少这个语义字段,导致 browser / Electron app state 信息少于官方。++## 验证++- Chrome 本地回归确认地址栏行包含 `Placeholder: Ask Google or type a URL`。+- `swift test --filter AccessibilityRendererFormatsPlaceholderSegment`+- `./scripts/build-open-computer-use-app.sh debug`++## 受影响文件++- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swiftindex 6db0958..8adf9cd 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift@@ -508,6 +508,7 @@ private struct TreeRenderer {let traits = summarizeTraits(of: root)let actions = copyActions(root) ?? []let prettyActions = meaningfulActions(actions, role: role)+ let placeholder = placeholderValue(of: root)let webAreaDepth = webAreaDepth(role: role, ancestors: ancestors)let localFrame = resolveLocalFrame(of: root, windowBounds: context.windowBounds)let rowTexts = role == kAXRowRole as String ? flattenedRowTexts(of: root) : []@@ -592,16 +593,23 @@ private struct TreeRenderer {rawValueSegment,precedingSegments: [labelSegment, helpSegment, urlSegment, identifierSegment])+ let placeholderSegment = formattedPlaceholderSegment(+ placeholder,+ title: title,+ label: label,+ value: value,+ precedingSegments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment]+ )let actionsPrefix = shouldCommaSeparateActions(title: title,inlineRowSummary: inlineRowSummary,genericTextSummary: genericTextSummary,- segments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment]+ segments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment, placeholderSegment]) ? ", Secondary Actions: " : " Secondary Actions: "let actionsSegment = prettyActions.isEmpty ? "" : "\(actionsPrefix)\(prettyActions.joined(separator: ", "))"let linePrefix = roleText.isEmpty ? "\(index)" : "\(index) \(roleText)"- let lineBody = "\(linePrefix)\(traitsSegment)\(titleSegment)\(rowSummarySegment)\(labelSegment)\(helpSegment)\(urlSegment)\(identifierSegment)\(valueSegment)"+ let lineBody = "\(linePrefix)\(traitsSegment)\(titleSegment)\(rowSummarySegment)\(labelSegment)\(helpSegment)\(urlSegment)\(identifierSegment)\(valueSegment)\(placeholderSegment)"lines.append("\(String(repeating: "\t", count: depth))\(lineBody)\(actionsSegment)")let record = ElementRecord(@@ -899,6 +907,19 @@ private func sanitizedValue(of element: AXUIElement) -> String? {return nil}+private func placeholderValue(of element: AXUIElement) -> String? {+ for attribute in ["AXPlaceholderValue", "AXPlaceholder"] {+ if let string = stringValue(of: element, attribute: attribute) {+ let sanitized = sanitizeText(string)+ if !sanitized.isEmpty {+ return sanitized+ }+ }+ }++ return nil+}+private func numericValueRepresentsBoolean(for element: AXUIElement, value: CFTypeRef) -> Bool {guard let number = value as? NSNumber else {return false@@ -1011,6 +1032,19 @@ private func formattedValueSegmentWithSeparator(_ valueSegment: String, precedinreturn ",\(valueSegment)"}+func formattedPlaceholderSegment(_ placeholder: String?, title: String?, label: String?, value: String?, precedingSegments: [String]) -> String {+ guard let placeholder, !placeholder.isEmpty else {+ return ""+ }++ if placeholder == title || placeholder == label || placeholder == value {+ return ""+ }++ let prefix = precedingSegments.contains(where: { !$0.isEmpty }) || title != nil ? ", Placeholder: " : " Placeholder: "+ return "\(prefix)\(placeholder)"+}+private func shouldCommaSeparateActions(title: String?,inlineRowSummary: String?,diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swiftindex 3d503e6..0784f63 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift@@ -682,6 +682,29 @@ final class OpenComputerUseKitTests: XCTestCase {XCTAssertEqual(meaningfulActions(["AXZoomWindow"], role: kAXButtonRole as String), ["zoom the window"])}+ func testAccessibilityRendererFormatsPlaceholderSegment() {+ XCTAssertEqual(+ formattedPlaceholderSegment(+ "Ask Google or type a URL",+ title: nil,+ label: "Address and search bar",+ value: "example.com",+ precedingSegments: [" Description: Address and search bar", " Value: example.com"]+ ),+ ", Placeholder: Ask Google or type a URL"+ )+ XCTAssertEqual(+ formattedPlaceholderSegment(+ "Search mail",+ title: nil,+ label: "Search mail",+ value: nil,+ precedingSegments: []+ ),+ ""+ )+ }+func testBlockingAsyncBridgeTimesOutScreenshotWork() {XCTAssertThrowsError(try BlockingAsyncBridge.run(timeout: 0.01) {