Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

Render accessibility placeholder values

Leo1f4663daf90b
diff --git a/docs/histories/2026-05/20260507-2010-render-ax-placeholder.md b/docs/histories/2026-05/20260507-2010-render-ax-placeholder.md
new file mode 100644
index 0000000..983c374
--- /dev/null
+++ b/docs/histories/2026-05/20260507-2010-render-ax-placeholder.md
@@ -0,0 +1,26 @@
+# 渲染 AX placeholder 值
+
+## 用户诉求
+
+继续对齐开源版 `open-computer-use` 和官方 `computer-use` 的 app state 输出,尤其是 Electron / Browser 这类复杂 AX tree 的返回形状。
+
+## 主要改动
+
+- 在 AX renderer 中读取 `AXPlaceholderValue` / `AXPlaceholder`。
+- 当 placeholder 不等于 title、description 或 value 时,在对应元素行追加 `Placeholder: ...`。
+- 增加 placeholder segment 的单元测试,避免重复渲染 description 或已有 value。
+
+## 设计动机
+
+官方 Chrome 输出会在地址栏元素上保留 placeholder,例如 `Ask Google or type a URL`。开源版之前只输出 description 和 value,缺少这个语义字段,导致 browser / Electron app state 信息少于官方。
+
+## 验证
+
+- Chrome 本地回归确认地址栏行包含 `Placeholder: Ask Google or type a URL`。
+- `swift test --filter AccessibilityRendererFormatsPlaceholderSegment`
+- `./scripts/build-open-computer-use-app.sh debug`
+
+## 受影响文件
+
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index 6db0958..8adf9cd 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -508,6 +508,7 @@ private struct TreeRenderer {
let traits = summarizeTraits(of: root)
let actions = copyActions(root) ?? []
let prettyActions = meaningfulActions(actions, role: role)
+ let placeholder = placeholderValue(of: root)
let webAreaDepth = webAreaDepth(role: role, ancestors: ancestors)
let localFrame = resolveLocalFrame(of: root, windowBounds: context.windowBounds)
let rowTexts = role == kAXRowRole as String ? flattenedRowTexts(of: root) : []
@@ -592,16 +593,23 @@ private struct TreeRenderer {
rawValueSegment,
precedingSegments: [labelSegment, helpSegment, urlSegment, identifierSegment]
)
+ let placeholderSegment = formattedPlaceholderSegment(
+ placeholder,
+ title: title,
+ label: label,
+ value: value,
+ precedingSegments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment]
+ )
let actionsPrefix = shouldCommaSeparateActions(
title: title,
inlineRowSummary: inlineRowSummary,
genericTextSummary: genericTextSummary,
- segments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment]
+ segments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment, placeholderSegment]
) ? ", Secondary Actions: " : " Secondary Actions: "
let actionsSegment = prettyActions.isEmpty ? "" : "\(actionsPrefix)\(prettyActions.joined(separator: ", "))"
let linePrefix = roleText.isEmpty ? "\(index)" : "\(index) \(roleText)"
- let lineBody = "\(linePrefix)\(traitsSegment)\(titleSegment)\(rowSummarySegment)\(labelSegment)\(helpSegment)\(urlSegment)\(identifierSegment)\(valueSegment)"
+ let lineBody = "\(linePrefix)\(traitsSegment)\(titleSegment)\(rowSummarySegment)\(labelSegment)\(helpSegment)\(urlSegment)\(identifierSegment)\(valueSegment)\(placeholderSegment)"
lines.append("\(String(repeating: "\t", count: depth))\(lineBody)\(actionsSegment)")
let record = ElementRecord(
@@ -899,6 +907,19 @@ private func sanitizedValue(of element: AXUIElement) -> String? {
return nil
}
+private func placeholderValue(of element: AXUIElement) -> String? {
+ for attribute in ["AXPlaceholderValue", "AXPlaceholder"] {
+ if let string = stringValue(of: element, attribute: attribute) {
+ let sanitized = sanitizeText(string)
+ if !sanitized.isEmpty {
+ return sanitized
+ }
+ }
+ }
+
+ return nil
+}
+
private func numericValueRepresentsBoolean(for element: AXUIElement, value: CFTypeRef) -> Bool {
guard let number = value as? NSNumber else {
return false
@@ -1011,6 +1032,19 @@ private func formattedValueSegmentWithSeparator(_ valueSegment: String, precedin
return ",\(valueSegment)"
}
+func formattedPlaceholderSegment(_ placeholder: String?, title: String?, label: String?, value: String?, precedingSegments: [String]) -> String {
+ guard let placeholder, !placeholder.isEmpty else {
+ return ""
+ }
+
+ if placeholder == title || placeholder == label || placeholder == value {
+ return ""
+ }
+
+ let prefix = precedingSegments.contains(where: { !$0.isEmpty }) || title != nil ? ", Placeholder: " : " Placeholder: "
+ return "\(prefix)\(placeholder)"
+}
+
private func shouldCommaSeparateActions(
title: String?,
inlineRowSummary: String?,
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index 3d503e6..0784f63 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -682,6 +682,29 @@ final class OpenComputerUseKitTests: XCTestCase {
XCTAssertEqual(meaningfulActions(["AXZoomWindow"], role: kAXButtonRole as String), ["zoom the window"])
}
+ func testAccessibilityRendererFormatsPlaceholderSegment() {
+ XCTAssertEqual(
+ formattedPlaceholderSegment(
+ "Ask Google or type a URL",
+ title: nil,
+ label: "Address and search bar",
+ value: "example.com",
+ precedingSegments: [" Description: Address and search bar", " Value: example.com"]
+ ),
+ ", Placeholder: Ask Google or type a URL"
+ )
+ XCTAssertEqual(
+ formattedPlaceholderSegment(
+ "Search mail",
+ title: nil,
+ label: "Search mail",
+ value: nil,
+ precedingSegments: []
+ ),
+ ""
+ )
+ }
+
func testBlockingAsyncBridgeTimesOutScreenshotWork() {
XCTAssertThrowsError(
try BlockingAsyncBridge.run(timeout: 0.01) {