Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

Render AX links as markdown

Leofc4632c21957
diff --git a/docs/histories/2026-05/20260507-2023-render-links-as-markdown.md b/docs/histories/2026-05/20260507-2023-render-links-as-markdown.md
new file mode 100644
index 0000000..47fe90f
--- /dev/null
+++ b/docs/histories/2026-05/20260507-2023-render-links-as-markdown.md
@@ -0,0 +1,27 @@
+# 将 AXLink 渲染为 Markdown 链接
+
+## 用户诉求
+
+继续结合官方 `computer-use` 逆向线索优化复杂 app state,尤其是 Lark / Electron 和 browser 场景中的 AX tree 输出形状。
+
+## 主要改动
+
+- 对带 `AXURL` 的 `AXLink` 生成 Markdown 形态的 `[label](url)` 文本。
+- 将 `AXLink` role 文案固定为英文 `link`,避免受系统语言影响输出为本地化 role。
+- 对已转为 Markdown 的 link 抑制重复的 description / child text。
+- 没有 URL 的 link 仍按普通 link 节点保留 description。
+
+## 设计动机
+
+官方 `SkyComputerUseService` 1.0.770 binary 中有 `flattenLinksIntoMarkdownText` transform。Lark / Chrome 实测也显示官方更倾向把链接语义并入可读文本,而不是输出本地化 `链接 Description: ...` 结构。这个改动让链接目标 URL 更直接出现在 state 里,同时保留元素记录用于后续点击。
+
+## 验证
+
+- Lark 本地回归确认链接从 `链接 Description: ...` 变为 Markdown 链接文本。
+- Chrome 本地回归确认带 URL 的 link 被渲染为 Markdown,缺 URL 的 link 继续保留普通 `link Description`。
+- `swift test --filter AccessibilityRenderer`
+- `./scripts/build-open-computer-use-app.sh debug`
+
+## 受影响文件
+
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index 17167e9..581d6c1 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -531,10 +531,12 @@ private struct TreeRenderer {
explicitValue: value,
rowTexts: rowTexts
)
+ let linkText = role == "AXLink" ? markdownLinkText(for: root, title: title, label: label, value: value) : nil
+ let displayTitle = linkText ?? title
let inlineRowSummary = outlineRowSummary(for: root, role: role)
let hidesChildren = shouldSuppressChildren(
role: role,
- title: title,
+ title: displayTitle,
label: label,
help: help,
value: value,
@@ -547,14 +549,14 @@ private struct TreeRenderer {
let roleText = displayRoleText(
baseRoleText: baseRoleText,
role: role,
- title: title,
+ title: displayTitle,
label: label,
suppressChildren: hidesChildren
)
if shouldElideNode(
role: role,
- title: title,
+ title: displayTitle,
label: label,
value: value,
identifier: axIdentifier,
@@ -573,35 +575,35 @@ private struct TreeRenderer {
nextIndex += 1
let traitsSegment = traits.isEmpty ? "" : " (\(traits.joined(separator: ", ")))"
- let titleSegment = title.map { " \($0)" } ?? ""
+ let titleSegment = displayTitle.map { " \($0)" } ?? ""
let rowSummary = inlineRowSummary ?? (rendersSummaryAsChildren ? nil : genericTextSummary)
let rowSummarySegment = rowSummary.map { " \($0)" } ?? ""
- let labelSegment = label != nil && label != title ? " Description: \(label!)" : ""
+ let labelSegment = formattedLabelSegment(label, title: displayTitle, linkText: linkText)
let helpSegment = {
guard let help else {
return ""
}
- if help == title || help == label {
+ if help == displayTitle || help == label {
return ""
}
return " Help: \(help)"
}()
- let urlSegment = formattedURLSegment(for: root, title: title, label: label)
- let identifierSegment = displayIdentifierSegment(for: root, role: role, identifier: axIdentifier, title: title)
- let rawValueSegment = formattedValueSegment(for: root, roleText: roleText, title: title, value: value)
+ let urlSegment = formattedURLSegment(for: root, title: displayTitle, label: label)
+ let identifierSegment = displayIdentifierSegment(for: root, role: role, identifier: axIdentifier, title: displayTitle)
+ let rawValueSegment = formattedValueSegment(for: root, roleText: roleText, title: displayTitle, value: value)
let valueSegment = formattedValueSegmentWithSeparator(
rawValueSegment,
precedingSegments: [labelSegment, helpSegment, urlSegment, identifierSegment]
)
let placeholderSegment = formattedPlaceholderSegment(
placeholder,
- title: title,
+ title: displayTitle,
label: label,
value: value,
precedingSegments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment]
)
let actionsPrefix = shouldCommaSeparateActions(
- title: title,
+ title: displayTitle,
inlineRowSummary: inlineRowSummary,
genericTextSummary: genericTextSummary,
segments: [labelSegment, helpSegment, urlSegment, identifierSegment, valueSegment, placeholderSegment]
@@ -983,6 +985,39 @@ private func preferredDisplayTitle(
return explicitValue
}
+private func markdownLinkText(for element: AXUIElement, title: String?, label: String?, value: String?) -> String? {
+ guard let url = urlValue(of: element, attribute: kAXURLAttribute), !url.isEmpty else {
+ return nil
+ }
+
+ let text = [label, title, value]
+ .compactMap { candidate -> String? in
+ guard let candidate else {
+ return nil
+ }
+ let sanitized = sanitizeText(candidate)
+ return sanitized.isEmpty ? nil : sanitized
+ }
+ .first
+
+ guard let text else {
+ return nil
+ }
+
+ if text == url {
+ return url
+ }
+
+ return "[\(markdownEscapedLinkText(text))](\(url))"
+}
+
+private func markdownEscapedLinkText(_ text: String) -> String {
+ text
+ .replacingOccurrences(of: "\\", with: "\\\\")
+ .replacingOccurrences(of: "[", with: "\\[")
+ .replacingOccurrences(of: "]", with: "\\]")
+}
+
private func outlineRowSummary(for element: AXUIElement, role: String) -> String? {
guard role == kAXOutlineRole as String || role == kAXListRole as String else {
return nil
@@ -1024,6 +1059,18 @@ private func formattedValueSegment(for element: AXUIElement, roleText: String, t
return " Value: \(value)"
}
+private func formattedLabelSegment(_ label: String?, title: String?, linkText: String?) -> String {
+ guard let label, label != title else {
+ return ""
+ }
+
+ if let linkText, linkText.hasPrefix("[\(markdownEscapedLinkText(label))](") {
+ return ""
+ }
+
+ return " Description: \(label)"
+}
+
private func formattedValueSegmentWithSeparator(_ valueSegment: String, precedingSegments: [String]) -> String {
guard valueSegment.hasPrefix(" Value:"), precedingSegments.contains(where: { !$0.isEmpty }) else {
return valueSegment
@@ -1191,6 +1238,10 @@ private func shouldSuppressChildren(
return true
}
+ if role == "AXLink", title?.hasPrefix("[") == true {
+ return true
+ }
+
return genericTextSummary != nil
}
@@ -1344,6 +1395,10 @@ private func roleDescription(of element: AXUIElement, role: String, subrole: Str
return ""
}
+ if role == "AXLink" {
+ return "link"
+ }
+
if role == "AXWebArea" {
return stringValue(of: element, attribute: kAXRoleDescriptionAttribute) ?? "HTML 内容"
}