Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

Fix macOS text input and scoped Electron row clicks

Leo749110aa7b08
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
index 51ce21c..6790d64 100644
--- a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift
@@ -128,6 +128,7 @@ enum MacOSAppAgentProxy {
let response = try client.request([
"kind": "cli",
"arguments": arguments,
+ "environment": proxiedEnvironment(),
])
return CLIProxyResponse(
@@ -136,6 +137,12 @@ enum MacOSAppAgentProxy {
exitCode: Int32(response["exitCode"] as? Int ?? 1)
)
}
+
+ private static func proxiedEnvironment() -> [String: String] {
+ ProcessInfo.processInfo.environment.filter { key, _ in
+ key.hasPrefix("OPEN_COMPUTER_USE_")
+ }
+ }
}
private struct CLIProxyResponse {
@@ -332,7 +339,10 @@ private final class AppAgentConnection: @unchecked Sendable {
return ["response": NSNull()]
case "cli":
let arguments = request["arguments"] as? [String] ?? []
- let response = runCLI(arguments: arguments)
+ let environment = request["environment"] as? [String: String] ?? [:]
+ let response = AppAgentEnvironment.withOverrides(environment) {
+ runCLI(arguments: arguments)
+ }
return [
"stdout": response.stdout,
"stderr": response.stderr,
@@ -397,6 +407,40 @@ private final class AppAgentConnection: @unchecked Sendable {
}
}
+private enum AppAgentEnvironment {
+ private static let lock = NSLock()
+
+ static func withOverrides<T>(_ overrides: [String: String], _ body: () throws -> T) rethrows -> T {
+ guard !overrides.isEmpty else {
+ return try body()
+ }
+
+ lock.lock()
+ defer { lock.unlock() }
+
+ let previousValues = Dictionary(
+ uniqueKeysWithValues: overrides.keys.map { key in
+ (key, ProcessInfo.processInfo.environment[key])
+ }
+ )
+ for (key, value) in overrides {
+ setenv(key, value, 1)
+ }
+
+ defer {
+ for (key, previousValue) in previousValues {
+ if let previousValue {
+ setenv(key, previousValue, 1)
+ } else {
+ unsetenv(key)
+ }
+ }
+ }
+
+ return try body()
+ }
+}
+
private final class AppAgentSocketClient: @unchecked Sendable {
private let file: UnsafeMutablePointer<FILE>
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index c9e3d13..e68e14f 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -75,12 +75,13 @@
- 动作型 tools 对普通 app 采用“非侵入优先,物理指针路径显式 opt-in”策略:
- `perform_secondary_action` 只执行目标元素已经暴露出来的 AX action;无效 action 返回官方风格的 `... is not a valid secondary action for ...`,fixture 的 `Raise` 路径也不再为了测试去准备全局物理指针输入
- `set_value` 会先用 `AXUIElementIsAttributeSettable(kAXValueAttribute)` 判断目标是否真的是可设置值元素,只有 settable 时才调用 `AXUIElementSetAttributeValue`;不可设置时返回官方风格的 non-settable 错误,不退到键盘输入、剪贴板或未公开的文本替换接口
- - element-targeted `click` 的左键路径会先试原生列表的 `AXSelectedChildren` 选择,再试 `AXPress` / `AXConfirm` / `AXOpen` 这类真正语义化的激活动作;如果目标本身不可点,还会继续尝试其子孙 AX 元素(例如 Finder sidebar row 下面暴露 `AXOpen` 的 cell)和命中点附近的 AX hit-test 结果,只有这些都失败后才退到窗口/根元素常见的 `AXRaise`、`kAXMainAttribute`、`kAXFocusedAttribute`,最后才会退到 `postToPid` 定向鼠标事件与显式 opt-in 的全局物理指针 fallback。这里也不再把 `AXUIElementIsAttributeSettable` 的结果当成硬门槛,因为像 `TextEdit` window 这类元素在 `kAXFocusedAttribute` 上会出现“`isSettable=false` 但直接 set 成功”的官方同款场景;`click_count > 1` 也会优先重复可用的 AX action
+ - element-targeted `click` 的左键路径会先试原生列表的 `AXSelectedChildren` 选择,再试 `AXPress` / `AXConfirm` / `AXOpen` 这类真正语义化的激活动作;如果目标本身不可点,还会继续尝试其子孙 AX 元素(例如 Finder sidebar row 下面暴露 `AXOpen` 的 cell)和命中点附近的 AX hit-test 结果,最后才会退到 `postToPid` 定向鼠标事件。`AXRaise` / `kAXMainAttribute` / `kAXFocusedAttribute` 这类 activation-only fallback 只允许窗口级元素使用,避免普通静态文本或容器把“获得焦点”误报成“点击已处理”;`click_count > 1` 也会优先重复可用的 AX action
+ - 对 renderer 合成的 summary `text` 行,`click` 不再默认点击父容器中心;这类元素使用左侧安全锚点。对普通 row/container/text 的后代候选,也会过滤右侧紧凑 hover action(例如 Lark / Electron 会话列表里的“完成”勾),避免把主行点击误操作成 side action;Electron/Lark 这类 app 的 WebArea 合成文本会优先寻找紧邻的行级 `AXPress` 祖先并静默执行,避免用物理鼠标点行。浏览器 WebArea(例如 Chrome/GitHub)不走这条 Electron-scoped 行级祖先点击优化,仍保留通用 link/container 点击路径。命中点如果只反查到覆盖整页的 Electron/WebArea 级 AX 元素,不再继续扫描整个大容器的子孙候选,避免一次行点击被远处的可点击元素截走。
- `AXUIElementCopyElementAtPosition` 做坐标命中,尽量把 coordinate click 反解成可操作 AX 元素
- - `CGEvent.postToPid` 定向发送键盘事件,避免为了 `type_text` / `press_key` 抢前台;`press_key` 的 xdotool parser 覆盖官方 binary key table 里常见的 `BackSpace`、`Page_Up`、`Prior` / `Next`、`F1...F12` 和 `KP_*` alias
+ - `CGEvent.postToPid` 定向发送键盘事件,避免为了 `type_text` / `press_key` 抢前台;`type_text` 会把文本按 Unicode extended grapheme cluster 聚合成小批量 `keyboardSetUnicodeString` 事件,避免中文标点、emoji / 代理对和组合字符被逐个 UTF-16 code unit 拆开后在 Electron 富文本输入框里乱序或变形。如果当前 focused element 的 `AXValue` 可设置,`type_text` 会优先按可编辑内容追加并写回 `AXValue`,这覆盖 Feishu / Electron 富文本输入框不可靠接收后台键盘事件的场景,并会过滤已知占位提示,避免把 placeholder 拼进草稿;如果当前 focused element 不是可编辑文本目标,`type_text` 会报错要求先 click 文本输入区或使用 `set_value`,不再把无效果的后台键盘投递当成成功;`press_key` 的 xdotool parser 覆盖官方 binary key table 里常见的 `BackSpace`、`Page_Up`、`Prior` / `Next`、`F1...F12` 和 `KP_*` alias
- `scroll.pages` 对齐官方 `1.0.755` 的 `number` schema,支持小数页数;整数页且目标暴露 `AXScroll*ByPage` 时优先走 AX action,否则用 `CGEvent.postToPid` 向目标进程定向发送 scroll event
- `drag` 仍是 coordinate-only API,但默认不再使用全局 `.cghidEventTap` mouse event;默认改为 `CGEvent.postToPid` 定向发送 mouse move / down / dragged / up 事件,避免移动用户真实硬件光标;这些 coordinate tool 的 `x/y` 先按 screenshot pixel 坐标解释,再依据截图像素尺寸与目标 window bounds 的比例映射回 window point / Quartz global 坐标,避免 Retina 窗口上把 2x 像素误当成 1x point 导致点击落到错误位置
- - `click` / `scroll` / `drag` 只有设置 `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1` 时才允许全局 `CGEvent.post(tap: .cghidEventTap)` 物理指针兜底;其中 `click` 默认会先走 AX,再走 `postToPid` 定向鼠标事件,最后才是显式开启的全局物理指针 fallback;默认路径不再为了 fallback 调用 `NSRunningApplication.activate`
+ - `click` / `scroll` / `drag` 默认不会走全局 `.cghidEventTap`,因此不会移动或抢占用户真实鼠标;`OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1` 只作为本地诊断开关保留,不作为正常 OCU/OBU 验证路径。默认路径不再为了 fallback 调用 `NSRunningApplication.activate`
### 4. Fixture Bridge
diff --git a/docs/histories/2026-05/20260512-1817-fix-macos-unicode-type-text.md b/docs/histories/2026-05/20260512-1817-fix-macos-unicode-type-text.md
new file mode 100644
index 0000000..3e3278a
--- /dev/null
+++ b/docs/histories/2026-05/20260512-1817-fix-macos-unicode-type-text.md
@@ -0,0 +1,36 @@
+# Fix macOS Unicode type_text Ordering
+
+### Request
+
+排查 OCU 在飞书等输入框里用 `type_text` 输入中文时出现乱序或字符错误的问题;先定位原因,再修复并验证。
+
+### Changes
+
+- 将 macOS `type_text` 从逐个 UTF-16 code unit 发送键盘事件,改为按 Unicode extended grapheme cluster 聚合成小批量 `keyboardSetUnicodeString` 事件。
+- 为 accessibility snapshot 记录当前 focused element;`type_text` 在 focused element 的 `AXValue` 可设置时,优先追加并写回 `AXValue`,作为 Electron 富文本输入框不可靠接收后台键盘事件时的非前台兜底。
+- `AXValue` 兜底会从可编辑子文本推导已有草稿,并过滤 Feishu 输入框占位提示,避免把 placeholder 一起拼进消息内容。
+- 当前 focused element 不是可编辑文本目标时,`type_text` 现在会明确报错要求先 click 文本输入区或使用 `set_value`,不再把无效果的后台键盘投递当成成功。
+- 新增单元测试覆盖中文全角括号、emoji 代理对、ZWJ 序列、组合字符和 CJK 扩展字符,确保 chunking 后能无损还原且不拆分 grapheme cluster。
+- 同步更新架构文档,记录 `type_text` 的 Unicode 输入边界。
+
+### Motivation
+
+历史会话里 `type_text` 的参数是正确的,但飞书输入框最终出现中文括号方向、顺序和文本残留问题。代码排查显示一部分问题位于 macOS 输入注入层:原实现把文本拆成单个 UTF-16 code unit 逐次发送,复杂 Unicode 文本在 Electron 富文本控件中容易被异步处理成乱序或错误字符。批量发送完整 Unicode 文本块可以减少事件重排。
+
+真实飞书验证还发现,聚焦输入框后后台 `CGEvent.postToPid` 键盘事件仍可能不进入 Electron 富文本编辑器,而同一元素的 `set_value` 路径可以稳定写入完整 Unicode。因此 `type_text` 增加 focused settable `AXValue` 兜底,在不抢前台、不使用剪贴板的前提下复用同一条可设置值能力。后续测试还暴露了另一个假阳性:当前 focus 停在 WebArea 时 `type_text` 会返回成功但没有写入内容。现在这类状态会直接报错,要求先用 OCU click 聚焦输入框。
+
+### Files
+
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
+- `docs/ARCHITECTURE.md`
+
+### Validation
+
+- `swift test`
+- `./scripts/run-tool-smoke-tests.sh`
+- 本地 fixture 窗口手工验证:聚焦 `fixture-input` 后用 batched `CGEvent.keyboardSetUnicodeString` 投递 `(ocu发的)👩🏽‍💻é𠀀`,导出的 fixture state 精确显示同一字符串。
+- 使用 dev app 对飞书做真实验证:同一进程里先点击输入框,再执行 `type_text` 输入 `(ocu发的测试)👩🏽‍💻é𠀀`,后续 snapshot 中实际草稿子文本完整包含中文括号、ZWJ emoji、组合音和 CJK 扩展字符;测试后用 `Command+A` / `BackSpace` 清空草稿。
+- 真实发送链路中复测 `type_text`:未聚焦输入框时不再把 WebArea 当成可编辑目标;先用 OCU `click` 聚焦 text entry area 后,`type_text` + `press_key Return` 成功把完整中文测试消息发送到飞书会话。
diff --git a/docs/histories/2026-05/20260512-1852-avoid-synthetic-row-side-action-clicks.md b/docs/histories/2026-05/20260512-1852-avoid-synthetic-row-side-action-clicks.md
new file mode 100644
index 0000000..b32235d
--- /dev/null
+++ b/docs/histories/2026-05/20260512-1852-avoid-synthetic-row-side-action-clicks.md
@@ -0,0 +1,52 @@
+# Avoid Synthetic Row Side Action Clicks
+
+### Request
+
+排查并修复 OCU 点击飞书会话列表时,容易点到会话右侧 hover 出来的“完成”勾,导致消息会话被收掉的问题。
+
+### Changes
+
+- 为 accessibility renderer 合成出来的 summary `text` record 增加 `isSyntheticText` 标记。
+- element-targeted `click` 对 synthetic text 改用左侧安全锚点,不再使用父容器中心作为默认点击点。
+- synthetic text 点击时会过滤右侧紧凑 hover action(如“完成”/ done / archive),再允许剩余行内候选参与点击,避免 side action 被排序成首选候选,同时保留打开会话的主行点击能力。
+- 将右侧紧凑 hover action 过滤推广到普通 row/container/text 的后代候选;用户直接点到 side action 本身时仍可执行该按钮,但点 row 或 row text 时不再把 side action 当主候选。
+- 对静态文本点击优先使用更大的行级父容器 frame 计算点击锚点,降低 Electron text frame 偏移导致点到相邻行的风险。
+- 命中点反查到覆盖整页的 Electron/WebArea 级 AX 元素时,不再扫描整个大容器的子孙候选,避免远处可点击元素劫持当前行点击。
+- activation-only fallback 收窄到窗口级元素;普通静态文本或容器不能再靠 `AXFocused` / `AXMain` 成功就声明点击已处理,必须继续落到定向鼠标事件。
+- Electron/WebArea 里的合成会话行文本现在优先寻找紧邻的行级 `AXPress` 祖先并静默执行,避免依赖物理鼠标 fallback,也避免点到行内右侧 side action。
+- 复测发现这条 WebArea 行级祖先点击优化会误伤 Chrome/GitHub pinned repository card:正式版 OCU 可直接点击 profile pinned `container` 进入 repo,而 dev 版会把点击提前判定为已处理但没有导航。现在该优化收窄到 Electron/Lark 这类目标,浏览器 WebArea 回到通用 link/container 点击路径。
+- app-agent proxy 会透传 `OPEN_COMPUTER_USE_*` 环境变量,方便调试开关在 Dev `.app` agent 内生效;正常验证仍不启用全局物理 pointer fallback。
+- 新增单元测试覆盖 synthetic text 与普通元素的点击锚点策略、右侧 side action 过滤边界、宽泛 hit record 防护、行级 AX 祖先点击边界、Electron-scoped WebArea 优化边界,以及 activation-only fallback 的角色边界。
+- 同步更新架构文档,记录 synthetic text 的点击边界。
+
+### Motivation
+
+飞书会话列表中的可读文本经常是 renderer 为父 row/container 合成的 summary,而不是一个真实独立的可点击 text 元素。旧逻辑会用这个父容器的中心点,并继续扫描其可点击子孙;当 Lark 在 row 右侧 hover 出“完成”按钮时,这个按钮可能被选为首个 `AXPress` 候选,从而把“打开会话”误操作成“完成/收起会话”。新的策略把 synthetic text 视为只代表 row 的可读摘要:点击落在左侧安全区域,子孙候选只保留不像右侧 side action 的目标。
+
+真实验证还暴露另一个 false positive:不可点的静态文本会在 activation-only fallback 里因为 `AXFocused` / `AXMain` 返回成功而提前结束,导致实际会话没有切换。这个兜底现在只保留给窗口级元素;Electron/WebArea 行文本会优先使用行级 `AXPress` 祖先,保持静默 AX 操作,不把全局物理 pointer fallback 当作正常通过标准。
+
+Chrome/GitHub pinned card 复测暴露了优化作用域问题:GitHub profile 的 pinned repository card 在 AX 里也是带 URL 的 `container`,但它属于浏览器 WebArea,而不是 Electron 会话列表。对这类通用浏览器页面,应该保留原来的 target / hit-test / 定向鼠标 fallback 链路;Electron row 祖先点击优化只服务于 Lark/Feishu 这类目标,避免为了修会话列表而破坏原有 Chrome 行为。
+
+### Files
+
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift`
+- `apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
+- `docs/ARCHITECTURE.md`
+
+### Validation
+
+- `swift test`
+- `./scripts/run-tool-smoke-tests.sh`
+- `./scripts/build-open-computer-use-app.sh debug`
+- 使用 dev app 对飞书做真实验证:
+ - 旧补丁下点击 `AgentSphere 双周会群` / `AgentSphere 研发群` 这类会话行 text 不再触发“完成”,但也未切换会话。
+ - 旧补丁下点击 row container 仍可能把会话收掉,确认 side-action 过滤不能只作用于 synthetic text。
+ - 新补丁下点击 `AgentSphere 研发群` row text 后,会话行仍保留,未再被“完成”收掉;但当时仍未切换会话,后续定位为普通静态文本 activation-only fallback 抢先返回成功。
+ - 只用 OCU dev app 的 AX 路径复测:动态定位并打开 `司开星` 会话,右侧标题确认后发送 `OCU AX静默测试笑话:为什么程序员喜欢喝咖啡?因为没有咖啡因,线程就起不来。`,列表预览与消息区均出现该消息,输入框清空。
+ - 随后动态定位并打开 `徐昱嵩` 会话,右侧标题确认后先 AX click 聚焦 text entry area,再发送 `OCU AX静默测试笑话:为什么测试工程师进咖啡店先点空杯?因为要测边界条件。`,列表预览与消息区同样确认发送成功。
+ - 验证过程没有设置 `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS`,不依赖全局物理鼠标 fallback。
+ - 额外对 Chrome、Finder、Sublime Text 做非破坏性真实 app smoke:`get_app_state` 正常,点击当前已选 tab / 当前已选 Finder sidebar row / 当前已选 Sublime tab 均成功。
+ - 使用已安装正式版 OCU `0.1.49` 对 Chrome/GitHub profile 做对照:直接点击 6 个 pinned repository `container` 均可进入 repo 并读到 star。
+ - 修复后重建 dev app,终止旧 dev app agent,再用 dev OCU 对同一 Chrome/GitHub profile 做相同验证:动态解析 6 个 pinned repository `container`,逐个点击进入 repo、读取 star、Back 回 profile,6 个 URL 均匹配预期。
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index c5cbd70..51140ca 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -11,14 +11,24 @@ final class ElementRecord {
let localFrame: CGRect?
let rawActions: [String]
let prettyActions: [String]
-
- init(index: Int, identifier: String?, element: AXUIElement?, localFrame: CGRect?, rawActions: [String], prettyActions: [String]) {
+ let isSyntheticText: Bool
+
+ init(
+ index: Int,
+ identifier: String?,
+ element: AXUIElement?,
+ localFrame: CGRect?,
+ rawActions: [String],
+ prettyActions: [String],
+ isSyntheticText: Bool = false
+ ) {
self.index = index
self.identifier = identifier
self.element = element
self.localFrame = localFrame
self.rawActions = rawActions
self.prettyActions = prettyActions
+ self.isSyntheticText = isSyntheticText
}
}
@@ -48,6 +58,7 @@ public struct AppSnapshot {
let mode: SnapshotMode
let treeLines: [String]
let focusedSummary: String?
+ let focusedElement: AXUIElement?
let selectedText: String?
let elements: [Int: ElementRecord]
@@ -168,6 +179,7 @@ enum SnapshotBuilder {
mode: .accessibility,
treeLines: renderer.lines,
focusedSummary: renderer.focusedSummary,
+ focusedElement: focusedElement,
selectedText: selectedText,
elements: renderer.records
)
@@ -322,6 +334,7 @@ enum SnapshotBuilder {
mode: .fixture,
treeLines: lines,
focusedSummary: focusedSummary,
+ focusedElement: nil,
selectedText: nil,
elements: records
)
@@ -776,7 +789,8 @@ private struct TreeRenderer {
element: element,
localFrame: resolveLocalFrame(of: element, windowBounds: context.windowBounds),
rawActions: [],
- prettyActions: []
+ prettyActions: [],
+ isSyntheticText: true
)
}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
index 982e466..f72efa9 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
@@ -53,6 +53,13 @@ func visualCursorAppKitPoint(
)
}
+func inputEventPoint(
+ fromScreenStatePoint point: CGPoint,
+ screenMappings: [VisualCursorScreenMapping] = currentVisualCursorScreenMappings()
+) -> CGPoint {
+ point
+}
+
func makeVisualCursorTarget(
at point: CGPoint,
targetWindowID: CGWindowID?,
@@ -163,6 +170,183 @@ func invalidSecondaryActionErrorMessage(action: String, elementIndex: Int) -> St
"\(action) is not a valid secondary action for \(elementIndex)"
}
+func localClickActionPoints(frame: CGRect, isSyntheticText: Bool) -> [CGPoint] {
+ let center = CGPoint(x: frame.midX, y: frame.midY)
+ let leading = CGPoint(
+ x: frame.minX + min(max(frame.width * 0.3, 20), max(frame.width - 4, 20)),
+ y: frame.midY
+ )
+
+ if isSyntheticText {
+ return [leading]
+ }
+
+ if abs(leading.x - center.x) < 1 {
+ return [center]
+ }
+
+ return [center, leading]
+}
+
+func isLikelySyntheticSideActionCandidate(
+ parentFrame: CGRect?,
+ candidateFrame: CGRect?,
+ hasPrimaryAction: Bool,
+ labels: [String]
+) -> Bool {
+ let hasSideActionLabel = labels.contains { label in
+ let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
+ guard !normalized.isEmpty else {
+ return false
+ }
+
+ if normalized == "完成" || normalized == "done" || normalized == "complete" || normalized == "archive" {
+ return true
+ }
+
+ if normalized.count <= 24 {
+ if normalized.contains("完成") {
+ return true
+ }
+
+ if normalized.contains("mark") && (normalized.contains("done") || normalized.contains("complete")) {
+ return true
+ }
+ }
+
+ return false
+ }
+
+ guard let parentFrame, let candidateFrame else {
+ return false
+ }
+
+ let trailingBandWidth = min(max(parentFrame.width * 0.22, 56), 140)
+ let isTrailing = candidateFrame.midX >= parentFrame.maxX - trailingBandWidth
+ let compactWidth = candidateFrame.width <= max(88, parentFrame.width * 0.18)
+ let compactHeight = candidateFrame.height <= max(44, parentFrame.height * 1.2)
+ let isCompact = compactWidth && compactHeight
+
+ if hasSideActionLabel && hasPrimaryAction && isCompact {
+ return true
+ }
+
+ return isTrailing && isCompact && (hasPrimaryAction || hasSideActionLabel)
+}
+
+func shouldScanDescendantsOfHitRecord(originalFrame: CGRect?, hitFrame: CGRect?) -> Bool {
+ guard let originalFrame, let hitFrame else {
+ return true
+ }
+
+ let originalArea = max(originalFrame.width * originalFrame.height, 1)
+ let hitArea = hitFrame.width * hitFrame.height
+ if hitArea > max(originalArea * 12, 20_000) {
+ return false
+ }
+
+ if hitFrame.height > max(originalFrame.height * 4, 96),
+ hitFrame.width > max(originalFrame.width * 2, 240)
+ {
+ return false
+ }
+
+ return true
+}
+
+func isLikelyContainingRowActionFrame(
+ targetFrame: CGRect,
+ candidateFrame: CGRect?,
+ hasPrimaryAction: Bool
+) -> Bool {
+ let targetCenter = CGPoint(x: targetFrame.midX, y: targetFrame.midY)
+ guard
+ hasPrimaryAction,
+ let candidateFrame,
+ candidateFrame.insetBy(dx: -2, dy: -2).contains(targetCenter),
+ candidateFrame.width >= targetFrame.width,
+ candidateFrame.height >= targetFrame.height,
+ candidateFrame.height <= max(targetFrame.height + 32, targetFrame.height * 2)
+ else {
+ return false
+ }
+
+ return true
+}
+
+func canUseActivationOnlyClickFallback(role: String?) -> Bool {
+ guard let role else {
+ return false
+ }
+
+ return role == kAXWindowRole as String
+}
+
+func canUseKeyboardTextFallback(role: String?, roleDescription: String?, isValueSettable: Bool) -> Bool {
+ if isValueSettable {
+ return true
+ }
+
+ guard let role else {
+ return false
+ }
+
+ if role == kAXTextFieldRole as String || role == "AXTextArea" || role == "AXTextView" {
+ return true
+ }
+
+ guard let roleDescription = roleDescription?.lowercased() else {
+ return false
+ }
+
+ return roleDescription.contains("text field")
+ || roleDescription.contains("text area")
+ || roleDescription.contains("text entry")
+}
+
+func isElectronScopedWebRowClickOptimizationTarget(appName: String, bundleIdentifier: String?) -> Bool {
+ let normalizedBundleIdentifier = bundleIdentifier?
+ .trimmingCharacters(in: .whitespacesAndNewlines)
+ .lowercased()
+ let normalizedName = appName
+ .trimmingCharacters(in: .whitespacesAndNewlines)
+ .lowercased()
+
+ if let normalizedBundleIdentifier,
+ normalizedBundleIdentifier.hasPrefix("com.electron.")
+ || normalizedBundleIdentifier.contains(".electron.")
+ || normalizedBundleIdentifier.contains("lark")
+ || normalizedBundleIdentifier.contains("feishu")
+ {
+ return true
+ }
+
+ return normalizedName == "lark" || normalizedName == "feishu" || normalizedName == "飞书"
+}
+
+func shouldPreferContainingWebRowAXClickCandidate(
+ role: String?,
+ isSyntheticText: Bool,
+ hasWebAreaAncestor: Bool,
+ appName: String,
+ bundleIdentifier: String?
+) -> Bool {
+ guard hasWebAreaAncestor,
+ isElectronScopedWebRowClickOptimizationTarget(
+ appName: appName,
+ bundleIdentifier: bundleIdentifier
+ )
+ else {
+ return false
+ }
+
+ guard let role else {
+ return isSyntheticText
+ }
+
+ return role == kAXStaticTextRole as String || role == kAXGroupRole as String || isSyntheticText
+}
+
public final class ComputerUseService {
private var snapshotsByApp: [String: AppSnapshot] = [:]
@@ -209,7 +393,7 @@ public final class ComputerUseService {
if let elementIndex {
let record = try lookupElement(snapshot: snapshot, index: elementIndex)
- guard let targetPoint = try globalPoint(for: record, snapshot: snapshot) else {
+ guard let targetPoint = try globalClickPoint(for: record, snapshot: snapshot) else {
throw ComputerUseError.stateUnavailable("element \(elementIndex) has no clickable frame")
}
let cursorTarget = makeVisualCursorTarget(
@@ -393,6 +577,15 @@ public final class ComputerUseService {
return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
+ if try typeTextBySettingFocusedValueIfAvailable(text, in: snapshot) {
+ Thread.sleep(forTimeInterval: 0.1)
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
+ }
+
+ guard try canTypeTextUsingKeyboardFallback(in: snapshot) else {
+ throw ComputerUseError.stateUnavailable("type_text requires a focused editable text element. Click a text entry area first, or use set_value on a settable text element.")
+ }
+
try InputSimulation.typeText(text, pid: snapshot.app.pid)
return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
@@ -510,7 +703,10 @@ public final class ComputerUseService {
switch button {
case .left:
- if clickCount <= 1, try selectContainingListItem(for: element) {
+ if clickCount <= 1,
+ !hasAncestorRole("AXWebArea", of: element),
+ try selectContainingListItem(for: element)
+ {
return true
}
@@ -614,43 +810,77 @@ public final class ComputerUseService {
includeNearbyHitTesting: Bool,
allowActivationFallback: Bool
) throws -> Bool {
- if try performPreferredClick(on: record, button: button, clickCount: clickCount) {
+ let preferContainingWebRowAXClick = shouldPreferContainingWebRowAXClick(record, in: snapshot)
+ debugClickDecision("record=\(clickDebugDescription(record)) preferContainingWebRowAXClick=\(preferContainingWebRowAXClick)")
+
+ if preferContainingWebRowAXClick,
+ try performContainingWebRowClick(for: record, snapshot: snapshot, button: button, clickCount: clickCount)
+ {
Thread.sleep(forTimeInterval: 0.15)
return true
}
- for candidate in descendantClickCandidates(for: record, snapshot: snapshot) {
- if try performPreferredClick(on: candidate, button: button, clickCount: clickCount) {
+ if !preferContainingWebRowAXClick {
+ if try performPreferredClick(on: record, button: button, clickCount: clickCount) {
+ debugClickDecision("handled by preferred target \(clickDebugDescription(record))")
Thread.sleep(forTimeInterval: 0.15)
return true
}
- }
-
- if includeNearbyHitTesting {
- for localPoint in clickActionPoints(for: record) {
- guard let hitRecord = try hitTestElement(at: localPoint, in: snapshot) ?? bestElement(containing: localPoint, in: snapshot) else {
- continue
- }
- if try performPreferredClick(on: hitRecord, button: button, clickCount: clickCount) {
+ for candidate in descendantClickCandidates(for: record, snapshot: snapshot) {
+ if try performPreferredClick(on: candidate, button: button, clickCount: clickCount) {
+ debugClickDecision("handled by descendant \(clickDebugDescription(candidate))")
Thread.sleep(forTimeInterval: 0.15)
return true
}
+ }
+
+ if includeNearbyHitTesting {
+ for localPoint in clickActionPoints(for: record, snapshot: snapshot) {
+ guard let hitRecord = try hitTestElement(at: localPoint, in: snapshot) ?? bestElement(containing: localPoint, in: snapshot) else {
+ continue
+ }
- for candidate in descendantClickCandidates(for: hitRecord, snapshot: snapshot) {
- if try performPreferredClick(on: candidate, button: button, clickCount: clickCount) {
+ if !isLikelySyntheticSideAction(hitRecord, in: record),
+ try performPreferredClick(on: hitRecord, button: button, clickCount: clickCount)
+ {
+ debugClickDecision("handled by hit record \(clickDebugDescription(hitRecord))")
Thread.sleep(forTimeInterval: 0.15)
return true
}
+
+ if shouldScanDescendantsOfHitRecord(
+ originalFrame: clickFrame(for: record, snapshot: snapshot),
+ hitFrame: hitRecord.localFrame
+ ) {
+ for candidate in descendantClickCandidates(
+ for: hitRecord,
+ snapshot: snapshot,
+ sideActionScope: record
+ ) {
+ if try performPreferredClick(on: candidate, button: button, clickCount: clickCount) {
+ debugClickDecision("handled by hit descendant \(clickDebugDescription(candidate))")
+ Thread.sleep(forTimeInterval: 0.15)
+ return true
+ }
+ }
+ }
}
}
}
- guard allowActivationFallback, button == .left, let element = record.element else {
+ guard
+ allowActivationFallback,
+ !record.isSyntheticText,
+ button == .left,
+ let element = record.element,
+ canUseActivationOnlyClickFallback(role: stringValue(of: element, attribute: kAXRoleAttribute))
+ else {
return false
}
if try activateClickTarget(element: element, availableActions: record.rawActions) {
+ debugClickDecision("handled by activation fallback \(clickDebugDescription(record))")
Thread.sleep(forTimeInterval: 0.15)
return true
}
@@ -799,30 +1029,28 @@ public final class ComputerUseService {
return CGPoint(x: frame.midX, y: frame.midY)
}
- private func clickActionPoints(for record: ElementRecord) -> [CGPoint] {
- guard let frame = record.localFrame else {
+ private func clickActionPoints(for record: ElementRecord, snapshot: AppSnapshot) -> [CGPoint] {
+ guard let frame = clickFrame(for: record, snapshot: snapshot) else {
return []
}
- let center = CGPoint(x: frame.midX, y: frame.midY)
- let leading = CGPoint(
- x: frame.minX + min(max(frame.width * 0.3, 20), max(frame.width - 4, 20)),
- y: frame.midY
- )
-
- if abs(leading.x - center.x) < 1 {
- return [center]
- }
-
- return [center, leading]
+ return localClickActionPoints(frame: frame, isSyntheticText: record.isSyntheticText)
}
- private func descendantClickCandidates(for record: ElementRecord, snapshot: AppSnapshot) -> [ElementRecord] {
+ private func descendantClickCandidates(
+ for record: ElementRecord,
+ snapshot: AppSnapshot,
+ sideActionScope: ElementRecord? = nil
+ ) -> [ElementRecord] {
guard let element = record.element else {
return []
}
+ let sideActionParent = sideActionScope ?? record
return descendantClickCandidates(of: element, windowBounds: snapshot.windowBounds)
+ .filter { candidate in
+ !isLikelySyntheticSideAction(candidate, in: sideActionParent)
+ }
.sorted { lhs, rhs in
let lhsPriority = clickPriority(for: lhs)
let rhsPriority = clickPriority(for: rhs)
@@ -858,6 +1086,285 @@ public final class ComputerUseService {
return results
}
+ private func isLikelySyntheticSideAction(_ candidate: ElementRecord, in parent: ElementRecord) -> Bool {
+ isLikelySyntheticSideActionCandidate(
+ parentFrame: parent.localFrame,
+ candidateFrame: candidate.localFrame,
+ hasPrimaryAction: hasPrimaryClickAction(candidate),
+ labels: accessibilityLabels(for: candidate.element)
+ )
+ }
+
+ private func hasPrimaryClickAction(_ record: ElementRecord) -> Bool {
+ record.rawActions.contains { action in
+ action.caseInsensitiveCompare(kAXPressAction as String) == .orderedSame ||
+ action.caseInsensitiveCompare(kAXConfirmAction as String) == .orderedSame ||
+ action.caseInsensitiveCompare("AXOpen") == .orderedSame ||
+ action.caseInsensitiveCompare(kAXShowMenuAction as String) == .orderedSame
+ }
+ }
+
+ private func shouldPreferContainingWebRowAXClick(_ record: ElementRecord, in snapshot: AppSnapshot) -> Bool {
+ guard
+ let element = record.element
+ else {
+ return false
+ }
+
+ return shouldPreferContainingWebRowAXClickCandidate(
+ role: stringValue(of: element, attribute: kAXRoleAttribute),
+ isSyntheticText: record.isSyntheticText,
+ hasWebAreaAncestor: hasAncestorRole("AXWebArea", of: element),
+ appName: snapshot.app.name,
+ bundleIdentifier: snapshot.app.bundleIdentifier
+ )
+ }
+
+ private func performContainingWebRowClick(
+ for record: ElementRecord,
+ snapshot: AppSnapshot,
+ button: MouseButtonKind,
+ clickCount: Int
+ ) throws -> Bool {
+ guard
+ button == .left,
+ clickCount <= 1,
+ let element = record.element,
+ let targetFrame = record.localFrame
+ else {
+ return false
+ }
+
+ var current = element
+
+ for _ in 0..<6 {
+ guard let parent = copyParent(of: current) else {
+ return false
+ }
+
+ let rawActions = copyActions(for: parent) ?? []
+ let candidate = ElementRecord(
+ index: -1,
+ identifier: nil,
+ element: parent,
+ localFrame: localFrame(of: parent, windowBounds: snapshot.windowBounds),
+ rawActions: rawActions,
+ prettyActions: rawActions
+ )
+
+ if isLikelyContainingWebRowAction(targetFrame: targetFrame, candidate: candidate),
+ !isLikelySyntheticSideAction(candidate, in: record),
+ try performAction(named: kAXPressAction as String, on: parent, availableActions: rawActions)
+ {
+ debugClickDecision("handled by containing web row \(clickDebugDescription(candidate))")
+ return true
+ }
+
+ current = parent
+ }
+
+ return false
+ }
+
+ private func isLikelyContainingWebRowAction(
+ targetFrame: CGRect,
+ candidate: ElementRecord
+ ) -> Bool {
+ isLikelyContainingRowActionFrame(
+ targetFrame: targetFrame,
+ candidateFrame: candidate.localFrame,
+ hasPrimaryAction: hasPrimaryClickAction(candidate)
+ )
+ }
+
+ private func hasAncestorRole(_ role: String, of element: AXUIElement) -> Bool {
+ var current = element
+
+ for _ in 0..<12 {
+ guard let parent = copyParent(of: current) else {
+ return false
+ }
+
+ if stringValue(of: parent, attribute: kAXRoleAttribute) == role {
+ return true
+ }
+
+ current = parent
+ }
+
+ return false
+ }
+
+ private func accessibilityLabels(for element: AXUIElement?) -> [String] {
+ guard let element else {
+ return []
+ }
+
+ return [
+ kAXTitleAttribute as String,
+ kAXDescriptionAttribute as String,
+ kAXHelpAttribute as String,
+ kAXValueAttribute as String,
+ "AXIdentifier"
+ ].compactMap { attribute in
+ stringValue(of: element, attribute: attribute)
+ }
+ }
+
+ private func typeTextBySettingFocusedValueIfAvailable(_ text: String, in snapshot: AppSnapshot) throws -> Bool {
+ guard let element = snapshot.focusedElement else {
+ return false
+ }
+
+ guard try isSettableForSetValue(element: element, attribute: kAXValueAttribute) else {
+ return false
+ }
+
+ let baseValue = editableBaseValue(for: element)
+ let result = AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, (baseValue + text) as CFString)
+ switch result {
+ case .success:
+ return true
+ case .failure, .attributeUnsupported, .actionUnsupported, .cannotComplete, .noValue, .invalidUIElement, .illegalArgument:
+ return false
+ default:
+ throw ComputerUseError.message("AXUIElementSetAttributeValue failed with \(result.rawValue)")
+ }
+ }
+
+ private func canTypeTextUsingKeyboardFallback(in snapshot: AppSnapshot) throws -> Bool {
+ guard let element = snapshot.focusedElement else {
+ return false
+ }
+
+ let role = stringValue(of: element, attribute: kAXRoleAttribute)
+ let roleDescription = role.flatMap {
+ stringValue(of: element, attribute: kAXRoleDescriptionAttribute) ?? humanizedRoleDescription(for: $0)
+ }
+ return canUseKeyboardTextFallback(
+ role: role,
+ roleDescription: roleDescription,
+ isValueSettable: try isSettableForSetValue(element: element, attribute: kAXValueAttribute)
+ )
+ }
+
+ private func humanizedRoleDescription(for role: String) -> String {
+ if role == kAXTextFieldRole as String {
+ return "text field"
+ }
+
+ switch role {
+ case "AXTextArea", "AXTextView":
+ return "text entry area"
+ default:
+ return ""
+ }
+ }
+
+ private func editableBaseValue(for element: AXUIElement) -> String {
+ let childTextValues = editableDescendantTextValues(in: element)
+ .filter { !looksLikeEditablePlaceholder($0) }
+ if !childTextValues.isEmpty {
+ return childTextValues.joined()
+ }
+
+ guard let currentValue = stringValue(of: element, attribute: kAXValueAttribute) else {
+ return ""
+ }
+
+ let normalizedValue = normalizeEditablePlaceholderText(currentValue)
+ if normalizedValue.isEmpty || looksLikeEditablePlaceholder(normalizedValue) {
+ return ""
+ }
+
+ for attribute in ["AXPlaceholderValue", "AXPlaceholder"] {
+ guard let placeholder = stringValue(of: element, attribute: attribute) else {
+ continue
+ }
+
+ if normalizedValue == normalizeEditablePlaceholderText(placeholder) {
+ return ""
+ }
+ }
+
+ return currentValue
+ }
+
+ private func editableDescendantTextValues(in element: AXUIElement, depth: Int = 0) -> [String] {
+ guard depth < 4 else {
+ return []
+ }
+
+ var values: [String] = []
+ for child in copyChildren(of: element) {
+ if stringValue(of: child, attribute: kAXRoleAttribute) == kAXStaticTextRole as String,
+ let value = stringValue(of: child, attribute: kAXValueAttribute)
+ ?? stringValue(of: child, attribute: kAXTitleAttribute)
+ {
+ let normalized = normalizeEditablePlaceholderText(value)
+ if !normalized.isEmpty {
+ values.append(normalized)
+ }
+ }
+
+ values.append(contentsOf: editableDescendantTextValues(in: child, depth: depth + 1))
+ }
+
+ return values
+ }
+
+ private func looksLikeEditablePlaceholder(_ value: String) -> Bool {
+ let normalized = normalizeEditablePlaceholderText(value)
+ return normalized == "沟通时请保持“公开可接受”"
+ }
+
+ private func normalizeEditablePlaceholderText(_ value: String) -> String {
+ value
+ .replacingOccurrences(of: "\u{200B}", with: "")
+ .trimmingCharacters(in: .whitespacesAndNewlines)
+ }
+
+ private func clickFrame(for record: ElementRecord, snapshot: AppSnapshot) -> CGRect? {
+ guard let frame = record.localFrame else {
+ return nil
+ }
+
+ guard
+ !record.isSyntheticText,
+ let element = record.element,
+ stringValue(of: element, attribute: kAXRoleAttribute) == kAXStaticTextRole as String,
+ let rowFrame = containingRowFrame(for: element, textFrame: frame, windowBounds: snapshot.windowBounds)
+ else {
+ return frame
+ }
+
+ return rowFrame
+ }
+
+ private func containingRowFrame(for element: AXUIElement, textFrame: CGRect, windowBounds: CGRect?) -> CGRect? {
+ let textCenter = CGPoint(x: textFrame.midX, y: textFrame.midY)
+ var current = element
+
+ for _ in 0..<4 {
+ guard let parent = copyParent(of: current) else {
+ return nil
+ }
+
+ if let frame = localFrame(of: parent, windowBounds: windowBounds),
+ frame.insetBy(dx: -2, dy: -2).contains(textCenter),
+ frame.width >= textFrame.width + 40,
+ frame.height >= textFrame.height,
+ frame.height <= max(textFrame.height * 4, 96)
+ {
+ return frame
+ }
+
+ current = parent
+ }
+
+ return nil
+ }
+
private func copyActions(for element: AXUIElement) -> [String]? {
var actions: CFArray?
let result = AXUIElementCopyActionNames(element, &actions)
@@ -940,6 +1447,14 @@ public final class ComputerUseService {
)
}
+ private func globalClickPoint(for record: ElementRecord, snapshot: AppSnapshot) throws -> CGPoint? {
+ guard let point = clickActionPoints(for: record, snapshot: snapshot).first ?? localCenter(for: record) else {
+ return nil
+ }
+
+ return try windowPointToGlobalPoint(snapshot: snapshot, point: point)
+ }
+
private func screenshotToGlobalPoint(snapshot: AppSnapshot, x: Double, y: Double) throws -> CGPoint {
try windowPointToGlobalPoint(
snapshot: snapshot,
@@ -1060,6 +1575,21 @@ public final class ComputerUseService {
)
}
+ private func debugClickDecision(_ message: String) {
+ guard inputFallbackDebugEnabled(environment: ProcessInfo.processInfo.environment) else {
+ return
+ }
+
+ fputs("[open-computer-use] click decision \(message)\n", stderr)
+ }
+
+ private func clickDebugDescription(_ record: ElementRecord) -> String {
+ let role = record.element.flatMap { stringValue(of: $0, attribute: kAXRoleAttribute) } ?? "nil"
+ let actions = record.rawActions.joined(separator: ",")
+ let frame = record.localFrame.map { "x=\(Int($0.minX)) y=\(Int($0.minY)) w=\(Int($0.width)) h=\(Int($0.height))" } ?? "nil"
+ return "index=\(record.index) role=\(role) synthetic=\(record.isSyntheticText) actions=[\(actions)] frame=\(frame)"
+ }
+
private func integralScrollPageCount(_ pages: Double) -> Int? {
let rounded = pages.rounded(.toNearestOrAwayFromZero)
guard abs(pages - rounded) < 0.000001 else {
@@ -1075,6 +1605,8 @@ public final class ComputerUseService {
targetDescription: String,
snapshot: AppSnapshot
) throws {
+ let eventPoint = inputEventPoint(fromScreenStatePoint: point)
+
if globalPointerFallbacksEnabled(environment: ProcessInfo.processInfo.environment) {
debugInputFallback(
tool: "scroll",
@@ -1082,11 +1614,11 @@ public final class ComputerUseService {
snapshot: snapshot
)
InputSimulation.prepareAppForGlobalPointerInput(snapshot.app)
- try InputSimulation.scrollGlobally(at: point, direction: direction, pages: pages)
+ try InputSimulation.scrollGlobally(at: eventPoint, direction: direction, pages: pages)
return
}
- try InputSimulation.scrollTargeted(at: point, direction: direction, pages: pages, pid: snapshot.app.pid)
+ try InputSimulation.scrollTargeted(at: eventPoint, direction: direction, pages: pages, pid: snapshot.app.pid)
}
private func performDragEvent(
@@ -1095,6 +1627,9 @@ public final class ComputerUseService {
targetDescription: String,
snapshot: AppSnapshot
) throws {
+ let eventStart = inputEventPoint(fromScreenStatePoint: start)
+ let eventEnd = inputEventPoint(fromScreenStatePoint: end)
+
if globalPointerFallbacksEnabled(environment: ProcessInfo.processInfo.environment) {
debugInputFallback(
tool: "drag",
@@ -1102,11 +1637,11 @@ public final class ComputerUseService {
snapshot: snapshot
)
InputSimulation.prepareAppForGlobalPointerInput(snapshot.app)
- try InputSimulation.dragGlobally(from: start, to: end)
+ try InputSimulation.dragGlobally(from: eventStart, to: eventEnd)
return
}
- try InputSimulation.dragTargeted(from: start, to: end, pid: snapshot.app.pid)
+ try InputSimulation.dragTargeted(from: eventStart, to: eventEnd, pid: snapshot.app.pid)
}
private func performNonAXClickFallback(
@@ -1116,9 +1651,22 @@ public final class ComputerUseService {
targetDescription: String,
snapshot: AppSnapshot
) throws {
+ let eventPoint = inputEventPoint(fromScreenStatePoint: point)
+
+ if globalPointerFallbacksEnabled(environment: ProcessInfo.processInfo.environment) {
+ debugInputFallback(
+ tool: "click",
+ targetDescription: targetDescription,
+ snapshot: snapshot
+ )
+ InputSimulation.prepareAppForGlobalPointerInput(snapshot.app)
+ try InputSimulation.clickGlobally(at: eventPoint, button: button, clickCount: clickCount)
+ return
+ }
+
do {
try InputSimulation.clickTargeted(
- at: point,
+ at: eventPoint,
button: button,
clickCount: clickCount,
pid: snapshot.app.pid
@@ -1131,20 +1679,6 @@ public final class ComputerUseService {
)
}
}
-
- guard globalPointerFallbacksEnabled(environment: ProcessInfo.processInfo.environment) else {
- throw ComputerUseError.message(
- "click could not be handled through accessibility, and global pointer fallback is disabled. Set OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1 to allow physical-pointer fallback for this process."
- )
- }
-
- debugInputFallback(
- tool: "click",
- targetDescription: targetDescription,
- snapshot: snapshot
- )
- InputSimulation.prepareAppForGlobalPointerInput(snapshot.app)
- try InputSimulation.clickGlobally(at: point, button: button, clickCount: clickCount)
}
private func snapshotResult(for snapshot: AppSnapshot, style: SnapshotTextStyle) -> ToolCallResult {
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift
index 227af7b..fd41cfd 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift
@@ -43,6 +43,8 @@ enum MouseButtonKind: String {
}
enum InputSimulation {
+ static let maxKeyboardUnicodeChunkLength = 64
+
static func prepareAppForGlobalPointerInput(_ app: RunningAppDescriptor) {
if raiseAppWindowViaAccessibility(pid: app.pid) {
Thread.sleep(forTimeInterval: 0.12)
@@ -138,21 +140,50 @@ enum InputSimulation {
}
static func typeText(_ text: String, pid: pid_t) throws {
- for character in text.utf16 {
- var mutableCharacter = character
+ for chunk in keyboardUnicodeChunks(for: text) {
+ var mutableChunk = chunk
guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),
let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) else {
throw ComputerUseError.message("Failed to create keyboard event.")
}
- down.keyboardSetUnicodeString(stringLength: 1, unicodeString: &mutableCharacter)
- up.keyboardSetUnicodeString(stringLength: 1, unicodeString: &mutableCharacter)
+ mutableChunk.withUnsafeMutableBufferPointer { buffer in
+ guard let baseAddress = buffer.baseAddress else {
+ return
+ }
+
+ down.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: baseAddress)
+ up.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: baseAddress)
+ }
down.postToPid(pid)
up.postToPid(pid)
Thread.sleep(forTimeInterval: 0.02)
}
}
+ static func keyboardUnicodeChunks(for text: String, maxUTF16Units: Int = maxKeyboardUnicodeChunkLength) -> [[UniChar]] {
+ precondition(maxUTF16Units > 0, "maxUTF16Units must be positive")
+
+ var chunks: [[UniChar]] = []
+ var current: [UniChar] = []
+
+ for character in text {
+ let units = Array(String(character).utf16)
+ if !current.isEmpty, current.count + units.count > maxUTF16Units {
+ chunks.append(current)
+ current.removeAll(keepingCapacity: true)
+ }
+
+ current.append(contentsOf: units)
+ }
+
+ if !current.isEmpty {
+ chunks.append(current)
+ }
+
+ return chunks
+ }
+
static func pressKey(_ specification: String, pid: pid_t) throws {
let parsed = try KeyPressParser.parse(specification)
var activeFlags: CGEventFlags = []
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index c91db76..8e2cc48 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -539,6 +539,27 @@ final class OpenComputerUseKitTests: XCTestCase {
XCTAssertEqual(emptyResult.primaryText, "Missing required argument: text")
}
+ func testTypeTextUnicodeChunksPreserveGraphemeClusters() {
+ let text = "(ocu发的)👩🏽‍💻e\u{301}𠀀"
+ let chunks = InputSimulation.keyboardUnicodeChunks(for: text, maxUTF16Units: 8)
+ let decoded = chunks
+ .map { String(decoding: $0, as: UTF16.self) }
+ .joined()
+
+ XCTAssertEqual(decoded, text)
+ XCTAssertTrue(chunks.count > 1)
+ XCTAssertTrue(chunks.allSatisfy { chunk in
+ let decodedChunk = String(decoding: chunk, as: UTF16.self)
+ return decodedChunk.unicodeScalars.allSatisfy { $0.value != 0xFFFD }
+ })
+ for cluster in ["👩🏽‍💻", "e\u{301}", "𠀀"] {
+ XCTAssertEqual(
+ chunks.filter { String(decoding: $0, as: UTF16.self).contains(cluster) }.count,
+ 1
+ )
+ }
+ }
+
func testScrollRejectsInvalidDirectionWithOfficialMessage() {
let dispatcher = ComputerUseToolDispatcher()
let result = dispatcher.callToolAsResult(
@@ -568,6 +589,190 @@ final class OpenComputerUseKitTests: XCTestCase {
)
}
+ func testSyntheticTextClickUsesLeadingSafePointOnly() {
+ let frame = CGRect(x: 40, y: 20, width: 300, height: 48)
+ let points = localClickActionPoints(frame: frame, isSyntheticText: true)
+
+ XCTAssertEqual(points, [CGPoint(x: 130, y: 44)])
+ XCTAssertFalse(points.contains(CGPoint(x: 190, y: 44)))
+ }
+
+ func testNormalClickKeepsCenterThenLeadingFallback() {
+ let frame = CGRect(x: 40, y: 20, width: 300, height: 48)
+
+ XCTAssertEqual(
+ localClickActionPoints(frame: frame, isSyntheticText: false),
+ [CGPoint(x: 190, y: 44), CGPoint(x: 130, y: 44)]
+ )
+ }
+
+ func testSyntheticSideActionFilterRejectsTrailingDoneButton() {
+ XCTAssertTrue(
+ isLikelySyntheticSideActionCandidate(
+ parentFrame: CGRect(x: 40, y: 20, width: 300, height: 48),
+ candidateFrame: CGRect(x: 296, y: 24, width: 36, height: 36),
+ hasPrimaryAction: true,
+ labels: ["完成"]
+ )
+ )
+ }
+
+ func testSyntheticSideActionFilterKeepsMainRowPreviewContainingDone() {
+ XCTAssertFalse(
+ isLikelySyntheticSideActionCandidate(
+ parentFrame: CGRect(x: 40, y: 20, width: 300, height: 48),
+ candidateFrame: CGRect(x: 48, y: 22, width: 236, height: 44),
+ hasPrimaryAction: true,
+ labels: ["AK账号管控 @所有人 变更完成,如果有问题,请联系我"]
+ )
+ )
+ }
+
+ func testSyntheticSideActionFilterKeepsLargeRowNamedDone() {
+ XCTAssertFalse(
+ isLikelySyntheticSideActionCandidate(
+ parentFrame: CGRect(x: 40, y: 20, width: 300, height: 48),
+ candidateFrame: CGRect(x: 40, y: 20, width: 300, height: 48),
+ hasPrimaryAction: true,
+ labels: ["完成"]
+ )
+ )
+ }
+
+ func testHitRecordDescendantScanRejectsBroadWebAreaHit() {
+ XCTAssertFalse(
+ shouldScanDescendantsOfHitRecord(
+ originalFrame: CGRect(x: 40, y: 120, width: 300, height: 48),
+ hitFrame: CGRect(x: 0, y: 0, width: 1200, height: 800)
+ )
+ )
+ }
+
+ func testHitRecordDescendantScanKeepsNearbyRowHit() {
+ XCTAssertTrue(
+ shouldScanDescendantsOfHitRecord(
+ originalFrame: CGRect(x: 40, y: 120, width: 300, height: 48),
+ hitFrame: CGRect(x: 32, y: 112, width: 320, height: 56)
+ )
+ )
+ }
+
+ func testContainingRowActionAcceptsTightClickableAncestor() {
+ XCTAssertTrue(
+ isLikelyContainingRowActionFrame(
+ targetFrame: CGRect(x: 132, y: 381, width: 268, height: 44),
+ candidateFrame: CGRect(x: 124, y: 373, width: 284, height: 60),
+ hasPrimaryAction: true
+ )
+ )
+ }
+
+ func testContainingRowActionRejectsBroadWebArea() {
+ XCTAssertFalse(
+ isLikelyContainingRowActionFrame(
+ targetFrame: CGRect(x: 132, y: 381, width: 268, height: 44),
+ candidateFrame: CGRect(x: 0, y: 0, width: 1200, height: 800),
+ hasPrimaryAction: true
+ )
+ )
+ }
+
+ func testContainingRowActionRequiresPrimaryAction() {
+ XCTAssertFalse(
+ isLikelyContainingRowActionFrame(
+ targetFrame: CGRect(x: 132, y: 381, width: 268, height: 44),
+ candidateFrame: CGRect(x: 124, y: 373, width: 284, height: 60),
+ hasPrimaryAction: false
+ )
+ )
+ }
+
+ func testContainingWebRowOptimizationRejectsChromeWebGroups() {
+ XCTAssertFalse(
+ shouldPreferContainingWebRowAXClickCandidate(
+ role: kAXGroupRole as String,
+ isSyntheticText: false,
+ hasWebAreaAncestor: true,
+ appName: "Google Chrome",
+ bundleIdentifier: "com.google.Chrome"
+ )
+ )
+ }
+
+ func testContainingWebRowOptimizationRejectsChromeSyntheticText() {
+ XCTAssertFalse(
+ shouldPreferContainingWebRowAXClickCandidate(
+ role: kAXStaticTextRole as String,
+ isSyntheticText: true,
+ hasWebAreaAncestor: true,
+ appName: "Google Chrome",
+ bundleIdentifier: "com.google.Chrome"
+ )
+ )
+ }
+
+ func testContainingWebRowOptimizationKeepsLarkSyntheticRows() {
+ XCTAssertTrue(
+ shouldPreferContainingWebRowAXClickCandidate(
+ role: kAXStaticTextRole as String,
+ isSyntheticText: true,
+ hasWebAreaAncestor: true,
+ appName: "Lark",
+ bundleIdentifier: "com.electron.lark"
+ )
+ )
+ }
+
+ func testContainingWebRowOptimizationRequiresWebAreaAncestor() {
+ XCTAssertFalse(
+ shouldPreferContainingWebRowAXClickCandidate(
+ role: kAXGroupRole as String,
+ isSyntheticText: false,
+ hasWebAreaAncestor: false,
+ appName: "Lark",
+ bundleIdentifier: "com.electron.lark"
+ )
+ )
+ }
+
+ func testActivationOnlyClickFallbackRejectsPlainStaticText() {
+ XCTAssertFalse(canUseActivationOnlyClickFallback(role: kAXStaticTextRole as String))
+ }
+
+ func testActivationOnlyClickFallbackKeepsWindowRaisePath() {
+ XCTAssertTrue(canUseActivationOnlyClickFallback(role: kAXWindowRole as String))
+ }
+
+ func testKeyboardTextFallbackRejectsPlainWebArea() {
+ XCTAssertFalse(
+ canUseKeyboardTextFallback(
+ role: "AXWebArea",
+ roleDescription: "HTML content",
+ isValueSettable: false
+ )
+ )
+ }
+
+ func testKeyboardTextFallbackAcceptsEditableTextRole() {
+ XCTAssertTrue(
+ canUseKeyboardTextFallback(
+ role: kAXTextFieldRole as String,
+ roleDescription: "text field",
+ isValueSettable: false
+ )
+ )
+ }
+
+ func testKeyboardTextFallbackAcceptsSettableValueElement() {
+ XCTAssertTrue(
+ canUseKeyboardTextFallback(
+ role: kAXGroupRole as String,
+ roleDescription: "text entry area",
+ isValueSettable: true
+ )
+ )
+ }
+
func testSnapshotRenderedTextStartsDirectlyWithAppHeader() {
let snapshot = makeSnapshot(
treeLines: ["\t0 standard window Sample Chat"],
@@ -975,6 +1180,24 @@ final class OpenComputerUseKitTests: XCTestCase {
XCTAssertEqual(point, CGPoint(x: 2415, y: 1783))
}
+ func testInputEventPointKeepsCoreGraphicsScreenStateCoordinates() {
+ let point = inputEventPoint(
+ fromScreenStatePoint: CGPoint(x: -1311, y: 701),
+ screenMappings: [
+ VisualCursorScreenMapping(
+ screenStateFrame: CGRect(x: 0, y: 0, width: 2560, height: 1440),
+ appKitFrame: CGRect(x: 0, y: 0, width: 2560, height: 1440)
+ ),
+ VisualCursorScreenMapping(
+ screenStateFrame: CGRect(x: -1512, y: 458, width: 1512, height: 982),
+ appKitFrame: CGRect(x: -1512, y: 0, width: 1512, height: 982)
+ ),
+ ]
+ )
+
+ XCTAssertEqual(point, CGPoint(x: -1311, y: 701))
+ }
+
func testScreenshotPixelScaleUsesRetinaSizedImageAgainstWindowBounds() {
let scale = screenshotPixelScale(
screenshotPixelSize: CGSize(width: 2048, height: 1266),
@@ -1449,6 +1672,7 @@ final class OpenComputerUseKitTests: XCTestCase {
mode: .accessibility,
treeLines: treeLines,
focusedSummary: focusedSummary,
+ focusedElement: nil,
selectedText: selectedText,
elements: [:]
)