👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
Render AX links as markdown
Leofc4632c21957
diff --git a/docs/histories/2026-05/20260507-2023-render-links-as-markdown.md b/docs/histories/2026-05/20260507-2023-render-links-as-markdown.mdnew file mode 100644index 0000000..47fe90f--- /dev/null+++ b/docs/histories/2026-05/20260507-2023-render-links-as-markdown.md@@ -0,0 +1,27 @@+# 将 AXLink 渲染为 Markdown 链接++## 用户诉求++继续结合官方 `computer-use` 逆向线索优化复杂 app state,尤其是 Lark / Electron 和 browser 场景中的 AX tree 输出形状。++## 主要改动++- 对带 `AXURL` 的 `AXLink` 生成 Markdown 形态的 `[label](url)` 文本。+- 将 `AXLink` role 文案固定为英文 `link`,避免受系统语言影响输出为本地化 role。+- 对已转为 Markdown 的 link 抑制重复的 description / child text。+- 没有 URL 的 link 仍按普通 link 节点保留 description。++## 设计动机++官方 `SkyComputerUseService` 1.0.770 binary 中有 `flattenLinksIntoMarkdownText` transform。Lark / Chrome 实测也显示官方更倾向把链接语义并入可读文本,而不是输出本地化 `链接 Description: ...` 结构。这个改动让链接目标 URL 更直接出现在 state 里,同时保留元素记录用于后续点击。++## 验证++- Lark 本地回归确认链接从 `链接 Description: ...` 变为 Markdown 链接文本。+- Chrome 本地回归确认带 URL 的 link 被渲染为 Markdown,缺 URL 的 link 继续保留普通 `link Description`。+- `swift test --filter AccessibilityRenderer`+- `./scripts/build-open-computer-use-app.sh debug`++## 受影响文件++- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swiftindex 17167e9..581d6c1 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift@@ -531,10 +531,12 @@ private struct TreeRenderer {explicitValue: value,rowTexts: rowTexts)+ let linkText = role == "AXLink" ? markdownLinkText(for: root, title: title, label: label, value: value) : nil+ let displayTitle = linkText ?? titlelet inlineRowSummary = outlineRowSummary(for: root, role: role)let hidesChildren = shouldSuppressChildren(role: role,- title: title,+ title: displayTitle,label: label,help: help,value: value,@@ -547,14 +549,14 @@ private struct TreeRenderer {let roleText = displayRoleText(baseRoleText: baseRoleText,role: role,- title: title,+ title: displayTitle,label: label,suppressChildren: hidesChildren)if shouldElideNode(role: role,- title: title,+ title: displayTitle,label: label,value: value,identifier: axIdentifier,@@ -573,35 +575,35 @@ private struct TreeRenderer {nextIndex += 1let traitsSegment = traits.isEmpty ? "" : " (\(traits.joined(separator: ", ")))"- let titleSegment = title.map { " \($0)" } ?? ""+ let titleSegment = displayTitle.map { " \($0)" } ?? ""let rowSummary = inlineRowSummary ?? (rendersSummaryAsChildren ? nil : genericTextSummary)let rowSummarySegment = rowSummary.map { " \($0)" } ?? ""- let labelSegment = label != nil && label != title ? " Description: \(label!)" : ""+ let labelSegment = formattedLabelSegment(label, title: displayTitle, linkText: linkText)let helpSegment = {guard let help else {return ""}- if help == title || help == label {+ if help == displayTitle || help == label {return ""}return " Help: \(help)"}()- let urlSegment = formattedURLSegment(for: root, title: title, label: label)- let identifierSegment = displayIdentifierSegment(for: root, role: role, identifier: axIdentifier, title: title)- let rawValueSegment = formattedValueSegment(for: root, roleText: roleText, title: title, value: value)+ let urlSegment = formattedURLSegment(for: root, title: displayTitle, label: label)+ let identifierSegment = displayIdentifierSegment(for: root, role: role, identifier: axIdentifier, title: displayTitle)+ let rawValueSegment = formattedValueSegment(for: root, roleText: roleText, title: displayTitle, value: value)let valueSegment = formattedValueSegmentWithSeparator(rawValueSegment,precedingSegments: [labelSegment, helpSegment, urlSegment, identifierSegment])let placeholderSegment = formattedPlaceholderSegment(placeholder,- title: title,+ title: displayTitle,label: label,value: value,precedingSegments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment])let actionsPrefix = shouldCommaSeparateActions(- title: title,+ title: displayTitle,inlineRowSummary: inlineRowSummary,genericTextSummary: genericTextSummary,segments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment, placeholderSegment]@@ -983,6 +985,39 @@ private func preferredDisplayTitle(return explicitValue}+private func markdownLinkText(for element: AXUIElement, title: String?, label: String?, value: String?) -> String? {+ guard let url = urlValue(of: element, attribute: kAXURLAttribute), !url.isEmpty else {+ return nil+ }++ let text = [label, title, value]+ .compactMap { candidate -> String? in+ guard let candidate else {+ return nil+ }+ let sanitized = sanitizeText(candidate)+ return sanitized.isEmpty ? nil : sanitized+ }+ .first++ guard let text else {+ return nil+ }++ if text == url {+ return url+ }++ return "[\(markdownEscapedLinkText(text))](\(url))"+}++private func markdownEscapedLinkText(_ text: String) -> String {+ text+ .replacingOccurrences(of: "\\", with: "\\\\")+ .replacingOccurrences(of: "[", with: "\\[")+ .replacingOccurrences(of: "]", with: "\\]")+}+private func outlineRowSummary(for element: AXUIElement, role: String) -> String? {guard role == kAXOutlineRole as String || role == kAXListRole as String else {return nil@@ -1024,6 +1059,18 @@ private func formattedValueSegment(for element: AXUIElement, roleText: String, treturn " Value: \(value)"}+private func formattedLabelSegment(_ label: String?, title: String?, linkText: String?) -> String {+ guard let label, label != title else {+ return ""+ }++ if let linkText, linkText.hasPrefix("[\(markdownEscapedLinkText(label))](") {+ return ""+ }++ return " Description: \(label)"+}+private func formattedValueSegmentWithSeparator(_ valueSegment: String, precedingSegments: [String]) -> String {guard valueSegment.hasPrefix(" Value:"), precedingSegments.contains(where: { !$0.isEmpty }) else {return valueSegment@@ -1191,6 +1238,10 @@ private func shouldSuppressChildren(return true}+ if role == "AXLink", title?.hasPrefix("[") == true {+ return true+ }+return genericTextSummary != nil}@@ -1344,6 +1395,10 @@ private func roleDescription(of element: AXUIElement, role: String, subrole: Strreturn ""}+ if role == "AXLink" {+ return "link"+ }+if role == "AXWebArea" {return stringValue(of: element, attribute: kAXRoleDescriptionAttribute) ?? "HTML 内容"}