Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

feat(select_text): add the missing official select_text tool (#68)

Wicgee19ea51eb693f
diff --git a/apps/OpenComputerUseFixture/Sources/OpenComputerUseFixture/main.swift b/apps/OpenComputerUseFixture/Sources/OpenComputerUseFixture/main.swift
index 9b9cf61..c5d818c 100644
--- a/apps/OpenComputerUseFixture/Sources/OpenComputerUseFixture/main.swift
+++ b/apps/OpenComputerUseFixture/Sources/OpenComputerUseFixture/main.swift
@@ -307,6 +307,21 @@ final class FixtureAppDelegate: NSObject, NSApplicationDelegate, NSTextFieldDele
case ("set_value", "fixture-input"):
inputField.stringValue = command.value ?? ""
updateExportedState()
+ case ("select_text", "fixture-input"):
+ window.makeFirstResponder(inputField)
+ if let editor = inputField.currentEditor() {
+ let mode = command.selection.flatMap(TextSelectionMode.init(rawValue:)) ?? .text
+ if let target = try? TextSelectionResolver.resolve(
+ value: inputField.stringValue,
+ text: command.value ?? "",
+ prefix: command.prefix,
+ suffix: command.suffix,
+ selection: mode
+ ) {
+ editor.selectedRange = NSRange(location: target.location, length: target.length)
+ }
+ }
+ updateExportedState()
case ("click", "fixture-increment"):
handleIncrement()
case ("click", "fixture-input"):
@@ -352,6 +367,20 @@ final class FixtureAppDelegate: NSObject, NSApplicationDelegate, NSTextFieldDele
}
}
+ private var selectedTextInInputField: String? {
+ guard let editor = inputField.currentEditor(), editor.selectedRange.length > 0 else {
+ return nil
+ }
+
+ let value = inputField.stringValue as NSString
+ let range = editor.selectedRange
+ guard range.location >= 0, NSMaxRange(range) <= value.length else {
+ return nil
+ }
+
+ return value.substring(with: range)
+ }
+
private func updateExportedState() {
guard let window, let contentView = window.contentView else {
return
@@ -377,7 +406,10 @@ final class FixtureAppDelegate: NSObject, NSApplicationDelegate, NSTextFieldDele
isActive: NSApp.isActive,
isKeyWindow: window.isKeyWindow,
activationLossCount: activationLossCount,
- keyWindowLossCount: keyWindowLossCount
+ keyWindowLossCount: keyWindowLossCount,
+ selectedText: selectedTextInInputField,
+ selectedRangeLocation: inputField.currentEditor()?.selectedRange.location,
+ selectedRangeLength: inputField.currentEditor()?.selectedRange.length
)
try? FixtureBridge.writeState(state)
diff --git a/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift b/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
index 087c234..954116e 100644
--- a/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
+++ b/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
@@ -198,7 +198,7 @@ enum OpenComputerUseSmokeSuite {
let tools = try client.listTools()
let expectedTools: Set<String> = [
"list_apps", "get_app_state", "query", "click", "perform_secondary_action",
- "scroll", "drag", "type_text", "press_key", "set_value",
+ "scroll", "drag", "type_text", "press_key", "set_value", "select_text",
]
let toolNames = Set(tools.compactMap { $0["name"] as? String })
guard tools.count == expectedTools.count, toolNames == expectedTools else {
@@ -263,7 +263,16 @@ enum OpenComputerUseSmokeSuite {
])
try expect(state.contains("set-value-ok-typed"), "type_text should append literal text to the focused text field")
- print("8. press_key")
+ print("8. select_text")
+ index = parseElementIndex(state)
+ state = try client.callTool("select_text", arguments: [
+ "app": appName,
+ "element_index": index["fixture-input"]!.index,
+ "text": "value-ok",
+ ])
+ try expect(state.contains("Selected text: [value-ok]"), "select_text should select the requested text inside the text field")
+
+ print("9. press_key")
index = parseElementIndex(state)
let keyCaptureFrame = index["fixture-key-capture"]!.frame
_ = try client.callTool("click", arguments: [
@@ -277,7 +286,7 @@ enum OpenComputerUseSmokeSuite {
])
try expect(state.contains("Last key: Return"), "press_key should update the key capture view")
- print("9. scroll")
+ print("10. scroll")
index = parseElementIndex(state)
let scrollIndex = index["fixture-scroll-view"]!.index
state = try client.callTool("scroll", arguments: [
@@ -288,7 +297,7 @@ enum OpenComputerUseSmokeSuite {
])
try expect(!state.contains("Scroll offset: 0"), "scroll should move the scroll view")
- print("10. drag")
+ print("11. drag")
index = parseElementIndex(state)
let dragFrame = index["fixture-drag-pad"]!.frame
state = try client.callTool("drag", arguments: [
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 6fd2d50..5e0ba8a 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -74,6 +74,7 @@
- `open-computer-use call <tool> --args '{...}'` 会直接输出 MCP-style JSON result;`open-computer-use call --calls '[...]'` / `--calls-file <path>` 会在同一进程里顺序执行 JSON 数组里的 tool calls,并复用同一个 `ComputerUseService` 内存态,因此 `get_app_state` 之后的 action tool 可以继续使用同一轮 snapshot 的 `element_index`。序列执行默认会在成功的相邻操作之间 sleep 1 秒,也可以用 `--sleep <seconds>` 覆盖;遇到 `isError=true` 的 tool result 后停止。
- 对真实 app 的 `get_app_state` / action tool 入口,当前只保留一层密码管理器 bundle denylist:bundle-id 直传时直接返回 safety denial;名称匹配时默认不解析到这些 app。终端、Chrome / Atlas 和系统组件不再属于内置阻止目标。
- 普通 app 的 element frame 当前按“窗口左上角为原点”的 window-relative 坐标输出,便于后续把 `element_index` 和截图坐标统一到同一套参考系。
+- macOS 额外提供 `query` 和 `select_text`,当前 native surface 为 11 个工具;Linux / Windows 仍为 9 个核心工具。JS `app.selectText(elementIndex, text, {prefix?, suffix?, selection?})` 复用快照或 query 索引,支持选中文字及前后光标。按字面文本和 UTF-16 定位、拒绝歧义;写入前及返回前检查会话,验证文本未变且实际选区等于请求,即使关闭完整回读也不能省略验证。失败通过 native `isError` / JS 异常传出;结果刷新使用 read-only recovery,不通过激活、键盘或真实指针补救。
- `click` / `set_value` 在执行真实动作前后,会额外驱动一层透明 `SoftwareCursorOverlay` window:两者的移动阶段现在共用一条 heading-driven 的官方风格 motion 内核,显式把“当前 cursor 朝向”和“最终 resting pose”一起喂给选路器,优先生成需要时先掉头、再沿车头方向推进的 C 形/单侧大弧轨迹;首次显示时按官方 binary 的 fresh state 从 AppKit 全局 `(0,0)` window origin 生成起点,后续动作继续复用上一帧 visible tip。真正显示出来的 cursor 不再直接等于 path sample,而是经过一层独立的 visual dynamics 状态,把 visible tip、velocity、angle 和 fog/offset 持续推进。`click` 结尾会衔接 click pulse 和更明显但仍然很小的 rotate wobble,`set_value` 则只做 settle / idle,不给 pulse;两者收尾后会在目标点继续保持 idle 状态,等待下一次动作时 tip 保持 anchored、只保留可感知的小角度摆动;只有连续 30 秒没有新动作时才做 cleanup,这样连续 tool call 不会反复从 fresh `(0,0)` 起步;如果宿主在任务 / turn 结束时发出 `turn-ended`,cursor 会立即消失并清掉本轮位置状态。
- overlay 用目标点相对窗口的锚点跟随窗口移动;行进、settle、点击反馈和 idle 都读取最新缓存帧,发现位置变化时直接重定位可见 tip 并清掉旧速度。窗口帧在后台串行读取,最多 30 Hz、同时最多一个请求,避免 AX 阻塞渲染线程;优先读取快照持有的 AX 窗口,仅在没有 AX 元素时使用窗口列表,读取失败保留最后有效帧。只保留当前窗口的追踪状态,reset / idle 超时后释放;这层跟随只调整显示位置,输入投递仍走既有路径。
- overlay 的 visual style 不再自己从官方 app bundle 裁 `SoftwareCursor` 小图;主 MCP runtime 现在和 `CursorMotion` 一样优先渲染仓库里沉淀的 `official-software-cursor-window-252.png` baseline,只有资源缺失时才退回 `OpenComputerUseKit` 内部的程序化 pointer/fog fallback。命中点 anchor 仍固定在 `126x126` 画布里的同一组 tip-offset 上;glyph 自身的 neutral heading 继续沿用 `CursorMotion` / 官方 baseline 的 `-3π/4`。主 runtime overlay window 按 AppKit 全局坐标移动,因此在把 AX / `CGWindowList` 产出的 y-down screen-space 点击目标喂给 overlay 之前,会先转换成对应屏幕的 AppKit 全局坐标;路径选路用屏幕上实际可见的 AppKit forward heading,进入 visual dynamics / render state 前则把 velocity 的 y 轴翻回 CursorMotion 的 y-down screen state,再交给 AppKit 绘制层做角度和 `dy` 翻转。程序化 fallback 保留 neutral artwork correction,把它的天然轮廓轴对齐到 `CursorMotion` / 官方 baseline 的 `-3π/4` forward 方向,但不让实验线依赖 runtime 代码。
diff --git a/docs/QUALITY_SCORE.md b/docs/QUALITY_SCORE.md
index a6200bb..407622c 100644
--- a/docs/QUALITY_SCORE.md
+++ b/docs/QUALITY_SCORE.md
@@ -11,7 +11,7 @@
| 区域 | 评分 | 原因 | 下一步 |
| --- | --- | --- | --- |
-| 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导,以及一轮按官方 surface / result 行为收敛过的 9 个 tools。 | 继续收敛复杂 AX 场景下的 state rendering 细节、权限 UI 和更清晰的用户错误提示。 |
+| 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导,以及共享的 9 个核心 tools;macOS 另有 query、select_text,JS 可调用。 | 继续收敛复杂 AX 场景下的 state rendering 细节、权限 UI 和更清晰的用户错误提示。 |
| Windows runtime | C | 已新增独立 Go `.exe`,通过 Windows UI Automation + Win32 window message 暴露同样 9 个 tools、MCP server 和 `call --calls`;默认不再自动启动 app、执行 `SetFocus`,或让 `type_text` 走可能抢前台的 UIA text fallback;截图会过滤隐藏、最小化、cloaked、离屏和退化窗口,并在 Go 层复核 PNG,但整体仍是功能性第一版。 | 把当前实机 screenshot 边界用例收敛成仓库内 Windows fixture / 交互式 smoke,再补 installer/signing,以及更原生的 Go UIA 实现或更稳定的 bridge。 |
| Linux runtime | C | 已新增独立 Go binary,通过 Python GI / AT-SPI2 暴露同样 9 个 tools、MCP server 和 `call --calls`;Ubuntu GNOME VM 已跑通 `list_apps`、MCP tools list 和 Text Editor 8-tool sequence,并已接入 npm bundled artifact 分发,但截图在 GNOME Wayland 下仍只能 best-effort,coordinate input 也不是通用后台模型。 | 扩展已有的隔离 GTK 动作回读 smoke,补完整工具覆盖、portal/compositor screenshot 路径,以及更原生的 Go D-Bus/libatspi bridge。 |
| macOS Locked Use | D | 已有签名 Broker / 插件、双进程保护、原生租约与恢复入口,离线和遮罩 rehearsal 有回归;安装恢复和无许可双保护释放已实测,独立无遮罩 Return 解锁 / 重锁已实测,受保护自动解锁 / AX / SCK / 重锁及隔离 legacy Keychain 已有一轮实测通过,完整会话认证状态 / 故障链路尚未通过实测;正常启用改为管理员 enable 与签名校验,普通 enable / 正常 MCP 的完整 GUI 闭环已实测。 | 物理覆盖已确认,继续完成 Data Protection / 会话认证状态、Secure Input / 崩溃 / 显示器故障回归及正式发布制品检查。 |
diff --git a/docs/exec-plans/completed/20261017-pr68-select-text.md b/docs/exec-plans/completed/20261017-pr68-select-text.md
new file mode 100644
index 0000000..c464e76
--- /dev/null
+++ b/docs/exec-plans/completed/20261017-pr68-select-text.md
@@ -0,0 +1,34 @@
+# PR #68: select_text through the current JS entry
+
+## Objective
+
+Merge current main into #68, preserve the contributor's implementation, complete app-bound JavaScript selection, review native safety/targeting and Unicode behavior, add meaningful tests and push follow-up commits to the same PR. Do not merge.
+
+## Scope and constraints
+
+- macOS native select_text plus app.selectText; preserve clear unsupported behavior on Linux/Windows.
+- Use current queried-index resolution, Locked Use action checks, read-back mode and read-only snapshot recovery.
+- Missing/ambiguous text, unsupported attributes and unverified results must be visible to callers without keyboard/pointer fallback.
+- Test only disposable fixture/own AppKit widgets, not personal documents. Keep existing identity protection intact.
+
+## Progress
+
+- [x] Merge latest main and resolve tool-list, service, test and skill conflicts.
+- [x] Review resolver/native implementation, repair safety/verification and compatibility defects.
+- [x] Add validated JS API, forwarding tests and documentation.
+- [x] Run full Swift/Node, native/MCP/JS fixture smoke and real AX selection checks where possible.
+- [x] Update existing history, synchronize remotes, push and report merge readiness.
+
+## Validation
+
+Cover UTF-16 ranges, literal repeated/overlapping text and prefix/suffix disambiguation, both caret modes, unsupported/refused/unreadable selection, queried indices, no-read-back mode, session revalidation and JS error propagation. Distinguish synthetic fixture transport tests from real AX widget checks and document host restrictions.
+
+## Evidence and outcome
+
+- Integrated current main through `531c9d4`, including repository identity, trusted-publishing documentation and the upstream 1.1.2 release update. Preserve the original PR commit.
+- Added app.selectText for Persistent and Worker sessions. Native actions resolve query indexes, validate leases, require a settable range, detect changed UTF-16 text and require an exact applied range, even without full read-back. Results use read-only recovery.
+- Literal Unicode search fixes canonically equivalent spelling producing an incorrect range; exact overlapping counts use constant match-storage space.
+- Swift 374 tests, 8 skipped, 0 failures. Selection-specific 22/22 includes a real own NSTextView AX test. Node 42/42 passed; native tool/cursor smoke and Worker JS/native fixture selection smoke passed.
+- Follow-up real-service validation passed using the repository's normally packaged and signed release app, without changing system approvals or bypassing identity/session checks. The earlier Broken pipe came from using a development signing identifier absent from the installed client approvals.
+- Worker JS → native MCP → real AX query index → selectText passed against disposable AppKit text views in both editable and read-only modes. Independently exported widget state confirmed UTF-16 ranges 3/2, 6/0 and 8/0, unchanged text, ambiguity/missing-text errors and no-read-back acknowledgments. The window could not become key/main, avoiding keyboard interference. This closes the previously unverified real-service chain; platform/opt-in suite skips remain separately documented.
+- UUID-scoped fixture state and notifications isolate the new smoke. All temporary probe executables and windows were cleaned up. No version bump; Linux/Windows keep explicit unsupported behavior.
diff --git a/docs/histories/2026-09/20260911-2001-add-select-text-tool.md b/docs/histories/2026-09/20260911-2001-add-select-text-tool.md
new file mode 100644
index 0000000..055a64d
--- /dev/null
+++ b/docs/histories/2026-09/20260911-2001-add-select-text-tool.md
@@ -0,0 +1,76 @@
+## [2026-09-11 20:01] | Task: 补齐官方 `select_text` 工具
+
+### 🤖 Execution Context
+* **Agent ID**: DeepSeek Harness agent session (`dsh web`)
+* **Base Model**: deepseek-flash
+* **Runtime**: DSH `run_code` + bash;macOS 26.1 (arm64),Swift 6.2 (swiftlang-6.2.0.19.9)
+
+### 📥 User Query
+> 我把 ocu 的仓库 fork 下来了(`github.com/wicgee/open-codex-computer-use`),请你在这上面改,并补齐缺少的与官方 codex computer use。
+
+### 🛠 Changes Overview
+**Scope:** `packages/OpenComputerUseKit`、`apps/OpenComputerUseFixture`、`apps/OpenComputerUseSmokeSuite`、`docs/`、`skills/`
+
+**Key Actions:**
+- **[新增工具]**: 实现官方 10 个 Computer Use tool 中唯一缺失的 `select_text`,description / 参数名 / 参数说明 / `enum` 与官方 `computer-use` 逐字对齐。
+- **[解析层]**: 新增纯函数 `TextSelectionResolver`,把「目标文本 + prefix/suffix 消歧」解析成 UTF-16 偏移选区,歧义时 fail closed。
+- **[执行层]**: `ComputerUseService.selectText` 通过 `kAXSelectedTextRangeAttribute` 落地选区,并在写后读回真实 range。
+- **[验证能力]**: fixture 支持 `select_text` 并导出当前选区;smoke suite 增加第 8 步,断言 snapshot 中出现 `Selected text: [value-ok]`。
+- **[文档同步]**: `docs/ARCHITECTURE.md`、`docs/QUALITY_SCORE.md`、`skills/open-computer-use/SKILL.md`、`references/usage.md`、MCP server instructions 的工具面描述一并更新。
+
+### 🧠 Design Intent (Why)
+* 官方 surface 是 10 个 tool,本仓库此前只覆盖 9 个;`select_text` 是唯一的协议缺口,补上后 macOS 主线与官方完全对齐。
+* 选区单位必须是 UTF-16 偏移(`CFRange` 语义),不能用 Swift `Character` 计数,否则 CJK / emoji 会错位——单测专门锁住这点。
+* 文本命中不唯一时 fail closed 并提示用 `prefix` / `suffix` 消歧,而不是静默选第一个匹配;这与仓库既有的 `sky_click` / `drag`「失败即停、不静默 fallback」策略一致。
+* 设置选区后读回真实 range:目标 app 未接受时在 result 里附 verification note,不把未生效的选区伪装成成功。
+
+### 📁 Files Modified
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TextSelectionResolver.swift`(新增)
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/FixtureBridge.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TextSelectionResolverTests.swift`(新增)
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
+- `apps/OpenComputerUseFixture/Sources/OpenComputerUseFixture/main.swift`
+- `apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift`
+- `docs/ARCHITECTURE.md`、`docs/QUALITY_SCORE.md`
+- `skills/open-computer-use/SKILL.md`、`skills/open-computer-use/references/usage.md`
+
+### ✅ Verification
+- `swift build`:Build complete。
+- `swift test`:**179 tests / 0 failures**(1 skipped:需 `OPEN_COMPUTER_USE_RUN_SKY_CLICK_LIVE_TEST=1` 的 Chrome live 测试)。新增 `TextSelectionResolverTests` 11 个用例 + 官方 surface 文案/schema 断言 + 非法 `selection` 用例。
+- `make smoke`:通过,输出包含新增的 `8. select_text`。
+- 回归口径:改动前该 suite 的 `testToolDefinitionCount` 锁定 9 个 tool、smoke runner 锁定 `Expected 9 tools`,均随本次行为变化同步为 10(属预期更新,非新引入失败)。
+
+### 📌 Notes / 未覆盖
+- Windows(Go + PowerShell UIA)与 Linux(Go + AT-SPI2)runtime 仍是 9 个 tool,`select_text` 未实现;已在 `docs/ARCHITECTURE.md` / `docs/QUALITY_SCORE.md` 明确标注平台差异。
+- 未做版本 bump 与 release notes:本次是能力补齐,按仓库发版流程需要单独走 `docs/releases/RELEASE_GUIDE.md`。
+
+
+### 2026-10-17:基于当前 main 补齐 JS 入口
+
+用户要求同步最新 main、解决冲突、补齐 JS 调用、检查测试并推送原 PR。保留原贡献提交,合并最新主线并修正 query / select_text 的工具清单、测试及 skill 冲突。macOS 当前为 11 个 native tools,Linux / Windows 仍为 9 个。
+
+- 新增 `app.selectText(elementIndex, text, {prefix?, suffix?, selection?})`,校验索引、文本、选项及模式;支持快照和 query 索引,原生失败通过两种 JS 会话的异常返回。
+- 原 selectText 直接查旧 snapshot,无法使用 query 索引;改为现有 snapshotForAction 路径。定向索引解析前检查会话,原生读取、实际写入前及验证后继续检查会话 / 租约,取消不能继续写入或报告成功。
+- 写入前检查 AX range 可设置,使用只读 recovery 刷新结果;不激活 app、不增加键盘或真实指针 fallback。
+- 修复不可验证结果被当成成功:缺失或不匹配的实际 range 明确返回错误。即使关闭完整回读也验证文本未变和实际 range;JS 不再吞掉非错误的提示。测试覆盖写前 / 写后文本变化、租约撤销、缺失及拒绝选区。
+- 原 NSString 搜索会按规范等价字符匹配,但按请求字符串的 UTF-16 长度计算范围。实测原实现把两单元的分解字符匹配成长度 1;改为字面匹配,拒绝不同 Unicode 拼写。重叠匹配仍完整计数,扫描不再保存所有匹配范围。
+- fixture 导出实际 range,验证选区 / caret 后才返回;JS/native smoke 使用 UUID 隔离状态及通知,支持并行测试清理。同步 API、平台范围和功能说明。
+
+验证:
+
+- `swift test`:374 项、8 项跳过、0 失败;新增选择相关 22 项均通过,包含独立 AppKit NSTextView 的真实 AX UTF-16 选区及两种 caret 模式。
+- Node REPL / CLI / 配置:42 项通过,包含 Persistent 和 Worker 两种会话的参数转发、错误传播和无快照调用。
+- 原有工具 smoke(含 select_text)及 cursor idle smoke 通过。
+- Worker JS → native MCP → 独立 fixture 全链通过:重复 / 不存在文本报错、前缀消歧、中文选区、两种 caret 模式、只返回 ok 的 no-read-back,以及显式读取选区。
+- 首次临时 native probe 返回 Locked Use 观察连接的 Broken pipe;后续只读核对准入配置,确认开发版 signing identifier 未登记。改用仓库正常打包、签名的正式版后,真实服务链路通过,未修改系统准入配置或绕过保护。
+- 补测 Worker JS → native MCP → 真实 AX query 索引 → selectText,分别覆盖独立 AppKit 文本框的可编辑与只读模式。独立读取控件导出状态,确认中文选区 UTF-16 location/length 为 3/2,前后 caret 为 6/0、8/0;文本不变、歧义 / 缺失报错、no-read-back 只返回 ok。测试窗口禁止成为 key/main window,避免键盘输入干扰。此前完整真实服务链未验证的限制已消除,未发现新增功能问题。
+- 文档与 diff whitespace 检查通过。无版本 bump。
+
+交付前再次合入 `0daece0` 的 Trusted Publishing 文档更新;该同步仅涉及文档,文档与 whitespace 检查通过。
+
+补测交付时同步主线 `531c9d4`(上游 1.1.2 版本及发布文档更新),初始化协议相关 2 项测试通过;本任务未额外 bump 版本。
diff --git a/docs/references/js-repl.md b/docs/references/js-repl.md
index 8323545..a4259c9 100644
--- a/docs/references/js-repl.md
+++ b/docs/references/js-repl.md
@@ -2,7 +2,7 @@
Open Computer Use ships a Node.js adapter for hosts that want a code-first
Computer Use surface. The adapter exposes two model-facing MCP tools, `js` and
-`js_reset`, and keeps the existing native 9-tool MCP server behind it.
+`js_reset`, and keeps the native MCP server behind it (11 tools on macOS, 9 on Linux/Windows).
## Why this shape
@@ -49,10 +49,29 @@ drag([fromX, fromY], [toX, toY])
pressKey(key, options?)
scroll(elementIndex, direction, pages?)
setValue(elementIndex, value)
+selectText(elementIndex, text, { prefix?, suffix?, selection? }) // macOS
typeText(text, options?)
performSecondaryAction(elementIndex, action)
```
+On macOS, `selectText` selects exact text inside a text element. `prefix` and `suffix`
+match the immediately adjacent text to disambiguate repeated occurrences. `selection`
+is `text` (default), `cursor_before` or `cursor_after`. Snapshot and `query` indexes
+both work in the same session. For example:
+
+```js
+var editor = await cua.getApp("TextEdit", { initialState: false });
+var fields = await editor.query({ role: "AXTextArea" });
+if (fields.matches.length !== 1) throw new Error("Choose the intended text area first");
+await editor.selectText(fields.matches[0].index, "target", { selection: "cursor_after" });
+await editor.getAXState();
+```
+
+Missing/ambiguous text, unsupported range selection, changed text and failed range
+verification throw through `js`. Verification still reads the applied range when full
+action read-back is disabled. The tool does not use keyboard/pointer fallback or
+activate the application. Linux/Windows report the native unsupported-tool error.
+
Discovery is available through `await cua.getState()` and
`await cua.listApps()`. `listApps()` normalizes the native human-readable app
catalog into an array of `{ id, displayName, lastUsedDate?, useCount?,
@@ -98,15 +117,16 @@ explicitly, so it launches the native runtime with
`OPEN_COMPUTER_USE_ACTION_READ_BACK=0`: actions return a short status with no
settle and no snapshot, and one `getAXState()` at the end of a batch reads the
outcome. On macOS a ten-click batch drops from several seconds to roughly one
-state read. Linux and Windows runtimes ignore the variable for now and keep
-reading back after each action.
+state read. Linux also honors this switch; Windows keeps reading back after each
+action. `selectText` still verifies the value and applied range, without taking a
+full snapshot.
## Entrypoints
- The Codex plugin manifest uses the REPL launcher by default, so its advertised
model-facing surface is `js` plus `js_reset`.
-- `open-computer-use mcp` remains the native 9-tool compatibility surface for
- other MCP clients.
+- `open-computer-use mcp` remains the native compatibility surface for other MCP
+ clients (11 tools on macOS, 9 on Linux/Windows).
- The npm CLI also exposes the same runtime directly:
```sh
diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.md
index 021003d..cf201c5 100644
--- a/docs/releases/feature-release-notes.md
+++ b/docs/releases/feature-release-notes.md
@@ -6,6 +6,7 @@
| --- | --- | --- | --- |
| 2026-10-17 | 1.1.2 修复发布 | 软件光标持续跟随移动窗口,Linux 批量动作可减少回读开销,安装链接统一到新仓库。 | 同步所有版本源至 1.1.2;发布窗口锚点跟踪、Linux 动作回读配置与仓库改名后的 npm / 插件元信息,沿用既有命令和 App 签名身份。 |
| 2026-10-17 | 仓库名称统一 | 项目地址与现有 npm 包、CLI 和 App 名称保持一致,方便多个 Agent 使用。 | 仓库迁移到 `iFurySt/open-computer-use`,同步当前安装说明、npm / plugin 元信息与发布模板;保留原有包名、命令、App 签名身份和历史发布链接兼容。 |
+| 2026-10-17 | macOS 文本选择 | Agent 可选中文字或放置光标,不必模拟拖拽。 | 新增 native select_text 和 JS app.selectText,支持快照 / query 索引、前后文消歧和 UTF-16 选区;重新检查会话与文本,验证失败明确报错,关闭完整回读时仍验证实际选区。 |
| 2026-10-17 | macOS 软件光标窗口跟随 | 目标窗口移动后,行进、点击反馈和等待中的软件光标继续跟随,避免停在旧位置。 | 接入窗口相对锚点,修复 idle / pulse 的旧落点、行进中止后的可见位置及重复计时器;AX 帧读取改为后台串行缓存,限制为最多 30 Hz / 一个请求,并在清理时释放追踪。 |
| 2026-10-15 | 1.1.1 修复发布 | Locked Use 安装更新更可靠,降低认证与后台 IPC 开销。 | 统一版本为 1.1.1,发布授权宿主、安装加载检查、旧组件卸载与连接生命周期修复;完整双屏遮罩、自动解锁、点击验证和重锁实测通过。 |
| 2026-10-15 | Locked Use 安装与性能修复 | 减少认证和 IPC 开销,旧系统组件也能正常移除并重装。 | 非 UI 插件使用 privileged host,安装前验证真实加载;修复系统缓存卸载与转交 socket 的生命周期,降低空闲唤醒,status 提示组件版本差异。 |
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index 96fd6bc..3a3db1e 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -485,7 +485,7 @@ enum SnapshotBuilder {
treeLines: lines,
focusedSummary: focusedSummary,
focusedElement: nil,
- selectedText: nil,
+ selectedText: state.selectedText,
windowElement: nil,
elements: records
)
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilityTextSelection.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilityTextSelection.swift
new file mode 100644
index 0000000..b70db22
--- /dev/null
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilityTextSelection.swift
@@ -0,0 +1,51 @@
+import ApplicationServices
+import Foundation
+
+/// AX range access shared by the service and isolated real-widget tests.
+enum AccessibilityTextSelection {
+ static func readRange(of element: AXUIElement) -> CFRange? {
+ var value: CFTypeRef?
+ guard AXUIElementCopyAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, &value) == .success,
+ let value, CFGetTypeID(value) == AXValueGetTypeID(),
+ AXValueGetType(value as! AXValue) == .cfRange else { return nil }
+ var range = CFRange()
+ return AXValueGetValue(value as! AXValue, .cfRange, &range) ? range : nil
+ }
+
+ static func writeRange(_ range: CFRange, to element: AXUIElement) throws {
+ var range = range
+ guard let value = AXValueCreate(.cfRange, &range) else {
+ throw ComputerUseError.message("select_text could not create an accessibility range")
+ }
+ let result = AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, value)
+ guard result == .success else {
+ throw ComputerUseError.message("select_text range write failed with \(result.rawValue)")
+ }
+ }
+}
+
+/// Verify the requested effect even when full action read-back is disabled.
+/// Closures allow race/refusal tests without modifying a real application's state.
+func applyVerifiedTextSelection(
+ target: TextSelectionTarget,
+ originalValue: String,
+ validate: () throws -> Void,
+ readValue: () -> String?,
+ writeRange: (CFRange) throws -> Void,
+ readRange: () -> CFRange?
+) throws {
+ func requireUnchangedText() throws {
+ guard let current = readValue(), current.utf16.elementsEqual(originalValue.utf16) else {
+ throw ComputerUseError.stateUnavailable("select_text text changed or became unreadable; refresh app state and retry")
+ }
+ }
+ try validate()
+ try requireUnchangedText()
+ try writeRange(CFRange(location: target.location, length: target.length))
+ try validate()
+ guard let applied = readRange(), applied.location == target.location, applied.length == target.length else {
+ throw ComputerUseError.message("select_text could not verify the requested range; the app may not support accessibility selection")
+ }
+ try requireUnchangedText()
+ try validate()
+}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
index cf1060a..76485aa 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
@@ -295,6 +295,23 @@ func appendingDragDeliveryNote(to result: ToolCallResult, path: DragDeliveryPath
return ToolCallResult(content: content, isError: result.isError)
}
+func selectTextErrorMessage(_ error: TextSelectionResolverError, elementIndex: String) -> String {
+ switch error {
+ case .emptyText:
+ return "select_text requires non-empty text"
+ case .notFound:
+ return "select_text could not find the requested text in element \(elementIndex); pass text exactly as it appears in the accessibility tree"
+ case .ambiguous(let occurrenceCount):
+ return "select_text found \(occurrenceCount) matches in element \(elementIndex); provide prefix or suffix to disambiguate the target"
+ }
+}
+
+func selectTextUnreadableValueErrorMessage(elementIndex: String) -> String {
+ return "element \(elementIndex) has no readable accessibility text value, so select_text cannot locate the requested text"
+}
+
+/// Inserts a verification note after the snapshot text and before any screenshot, matching
+/// `appendingDragDeliveryNote` so `primaryText` stays the snapshot for existing consumers.
func screenshotPixelScale(
screenshotPixelSize: CGSize?,
windowBounds: CGRect?
@@ -997,10 +1014,83 @@ public final class ComputerUseService {
return try actionResult(for: query)
}
+ public func selectText(
+ app query: String,
+ elementIndex: String,
+ text: String,
+ prefix: String?,
+ suffix: String?,
+ selection: TextSelectionMode
+ ) throws -> ToolCallResult {
+ let snapshot = try snapshotForAction(app: query, elementIndex: elementIndex)
+ let record = try lookupElement(snapshot: snapshot, index: elementIndex)
+
+ if snapshot.mode == .fixture {
+ guard let identifier = record.identifier, identifier == "fixture-input",
+ let state = try FixtureBridge.readState(),
+ let value = state.elements.first(where: { $0.identifier == identifier })?.value else {
+ throw ComputerUseError.invalidArguments("fixture select_text requires a readable input element")
+ }
+ let target = try resolveTextSelection(value: value, text: text, prefix: prefix, suffix: suffix,
+ selection: selection, elementIndex: elementIndex)
+ let cursorTarget = visualCursorTarget(for: record, snapshot: snapshot)
+ moveVisualCursor(to: cursorTarget)
+ defer { settleVisualCursor(at: cursorTarget) }
+ try FixtureBridge.post(FixtureCommand(kind: "select_text", identifier: identifier, value: text,
+ selection: selection.rawValue, prefix: prefix, suffix: suffix))
+ let deadline = Date().addingTimeInterval(1)
+ while Date() < deadline {
+ if let applied = try FixtureBridge.readState(), applied.focusedIdentifier == identifier,
+ applied.selectedRangeLocation == target.location, applied.selectedRangeLength == target.length,
+ let current = applied.elements.first(where: { $0.identifier == identifier })?.value,
+ current.utf16.elementsEqual(value.utf16) {
+ return try actionResult(for: query, recoveryPolicy: .readOnly)
+ }
+ Thread.sleep(forTimeInterval: 0.01)
+ }
+ throw ComputerUseError.message("select_text could not verify fixture selection")
+ }
+
+ try requireUsableComputerUseSession()
+ guard let element = record.element else {
+ throw ComputerUseError.stateUnavailable("element \(elementIndex) has no backing accessibility object")
+ }
+ guard let value = stringValue(of: element, attribute: kAXValueAttribute) else {
+ throw ComputerUseError.message(selectTextUnreadableValueErrorMessage(elementIndex: elementIndex))
+ }
+ let target = try resolveTextSelection(value: value, text: text, prefix: prefix, suffix: suffix,
+ selection: selection, elementIndex: elementIndex)
+ var settable = DarwinBoolean(false)
+ guard AXUIElementIsAttributeSettable(element, kAXSelectedTextRangeAttribute as CFString, &settable) == .success,
+ settable.boolValue else {
+ throw ComputerUseError.message("element \(elementIndex) does not support setting an accessibility text range")
+ }
+ let cursorTarget = visualCursorTarget(for: record, snapshot: snapshot)
+ moveVisualCursor(to: cursorTarget)
+ defer { settleVisualCursor(at: cursorTarget) }
+ try applyVerifiedTextSelection(
+ target: target, originalValue: value,
+ validate: { try requireUsableComputerUseSession() },
+ readValue: { stringValue(of: element, attribute: kAXValueAttribute) },
+ writeRange: { try AccessibilityTextSelection.writeRange($0, to: element) },
+ readRange: { AccessibilityTextSelection.readRange(of: element) }
+ )
+ return try actionResult(for: query, recoveryPolicy: .readOnly)
+ }
+
+ private func resolveTextSelection(value: String, text: String, prefix: String?, suffix: String?,
+ selection: TextSelectionMode, elementIndex: String) throws -> TextSelectionTarget {
+ do {
+ return try TextSelectionResolver.resolve(value: value, text: text, prefix: prefix, suffix: suffix, selection: selection)
+ } catch let error as TextSelectionResolverError {
+ throw ComputerUseError.message(selectTextErrorMessage(error, elementIndex: elementIndex))
+ }
+ }
+
/// Find controls in the app's current window by text and/or role, using the
/// app's own accessibility search where it offers one. No tree render, no
/// screenshot. Each result carries an `index` the existing element actions
- /// (click, set_value, scroll, perform_secondary_action) accept.
+ /// (click, set_value, select_text, scroll, perform_secondary_action) accept.
public func query(
app query: String,
text: String? = nil,
@@ -1056,6 +1146,7 @@ public final class ComputerUseService {
/// the normal current snapshot.
private func snapshotForAction(app query: String, elementIndex: String?) throws -> AppSnapshot {
if let elementIndex, let index = Int(elementIndex), index > TargetedElementRegistry<TargetedElement>.indexBase {
+ try requireUsableComputerUseSession()
let app = try AppDiscovery.resolveRunning(query)
let target = try targetedElements.resolve(index, app: TargetedAppIdentity(app: app))
do {
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
index 6afbc8a..e104de9 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
@@ -118,6 +118,15 @@ public final class ComputerUseToolDispatcher {
elementIndex: requireElementIndex(in: arguments),
value: requireString("value", in: arguments)
)
+ case "select_text":
+ return try service.selectText(
+ app: requireString("app", in: arguments),
+ elementIndex: requireElementIndex(in: arguments),
+ text: requireString("text", in: arguments),
+ prefix: try selectionString("prefix", in: arguments),
+ suffix: try selectionString("suffix", in: arguments),
+ selection: try parseSelectionMode(try selectionString("selection", in: arguments))
+ )
default:
throw ComputerUseError.unsupportedTool(name)
}
@@ -167,6 +176,26 @@ public final class ComputerUseToolDispatcher {
arguments[key] as? String
}
+ private func selectionString(_ key: String, in arguments: [String: Any]) throws -> String? {
+ guard let value = arguments[key] else { return nil }
+ guard let string = value as? String else {
+ throw ComputerUseError.invalidArguments("\(key) must be a string")
+ }
+ return string
+ }
+
+ private func parseSelectionMode(_ raw: String?) throws -> TextSelectionMode {
+ guard let raw else { return .text }
+
+ guard let mode = TextSelectionMode(rawValue: raw) else {
+ throw ComputerUseError.message(
+ "selection must be one of \(TextSelectionMode.allCases.map(\.rawValue).joined(separator: ", "))"
+ )
+ }
+
+ return mode
+ }
+
private func optionalTextLimit(_ key: String, in arguments: [String: Any]) throws -> SnapshotTextLimit? {
guard let value = arguments[key] else {
return nil
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/FixtureBridge.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/FixtureBridge.swift
index d9723f9..3ff06d4 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/FixtureBridge.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/FixtureBridge.swift
@@ -49,6 +49,9 @@ public struct FixtureAppState: Codable, Sendable {
public let isKeyWindow: Bool?
public let activationLossCount: Int?
public let keyWindowLossCount: Int?
+ public let selectedText: String?
+ public let selectedRangeLocation: Int?
+ public let selectedRangeLength: Int?
public init(
processIdentifier: pid_t? = nil,
@@ -59,7 +62,10 @@ public struct FixtureAppState: Codable, Sendable {
isActive: Bool? = nil,
isKeyWindow: Bool? = nil,
activationLossCount: Int? = nil,
- keyWindowLossCount: Int? = nil
+ keyWindowLossCount: Int? = nil,
+ selectedText: String? = nil,
+ selectedRangeLocation: Int? = nil,
+ selectedRangeLength: Int? = nil
) {
self.processIdentifier = processIdentifier
self.windowTitle = windowTitle
@@ -70,6 +76,9 @@ public struct FixtureAppState: Codable, Sendable {
self.isKeyWindow = isKeyWindow
self.activationLossCount = activationLossCount
self.keyWindowLossCount = keyWindowLossCount
+ self.selectedText = selectedText
+ self.selectedRangeLocation = selectedRangeLocation
+ self.selectedRangeLength = selectedRangeLength
}
}
@@ -83,6 +92,9 @@ public struct FixtureCommand: Codable, Sendable {
public let toY: Double?
public let direction: String?
public let pages: Double?
+ public let selection: String?
+ public let prefix: String?
+ public let suffix: String?
public init(
kind: String,
@@ -93,7 +105,10 @@ public struct FixtureCommand: Codable, Sendable {
toX: Double? = nil,
toY: Double? = nil,
direction: String? = nil,
- pages: Double? = nil
+ pages: Double? = nil,
+ selection: String? = nil,
+ prefix: String? = nil,
+ suffix: String? = nil
) {
self.kind = kind
self.identifier = identifier
@@ -104,16 +119,27 @@ public struct FixtureCommand: Codable, Sendable {
self.toY = toY
self.direction = direction
self.pages = pages
+ self.selection = selection
+ self.prefix = prefix
+ self.suffix = suffix
}
}
public enum FixtureBridge {
public static let appName = "OpenComputerUseFixture"
- public static let distributedNotificationName = Notification.Name("dev.opencodex.opencomputeruse.fixture.command")
+ // Optional UUID isolates concurrent smoke sessions; ordinary fixture behavior is unchanged.
+ private static var testNamespace: String? {
+ ProcessInfo.processInfo.environment["OPEN_COMPUTER_USE_FIXTURE_NAMESPACE"]
+ .flatMap(UUID.init(uuidString:))?.uuidString
+ }
+
+ public static var distributedNotificationName: Notification.Name {
+ Notification.Name("dev.opencodex.opencomputeruse.fixture.command" + (testNamespace.map { "." + $0 } ?? ""))
+ }
public static var stateFileURL: URL {
URL(fileURLWithPath: NSTemporaryDirectory())
- .appendingPathComponent("open-computer-use-fixture", isDirectory: true)
+ .appendingPathComponent("open-computer-use-fixture" + (testNamespace.map { "-" + $0 } ?? ""), isDirectory: true)
.appendingPathComponent("state.json")
}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift
index e592e41..d1ab435 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift
@@ -7,7 +7,7 @@ Some apps might have a separate dedicated plugin or skill. You may want to use t
Begin by calling `get_app_state` every turn you want to use Computer Use to get the latest state before acting. Codex will automatically stop the session after each assistant turn, so this step is required before interacting with apps in a new assistant turn.
-The available tools are list_apps, get_app_state, click, perform_secondary_action, scroll, drag, type_text, press_key, and set_value. If any of these are not available in your environment, use tool_search to surface one before calling any Computer Use action tools.
+The available tools are list_apps, get_app_state, click, perform_secondary_action, scroll, drag, type_text, press_key, set_value, select_text, and query. If any of these are not available in your environment, use tool_search to surface one before calling any Computer Use action tools.
Computer Use tools allow you to use the user's apps in the background, so while you're using an app, the user can continue to use other apps on their computer. Avoid doing anything that would disrupt the user's active session, such as overwriting the contents of their clipboard, unless they asked you to!
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TextSelectionResolver.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TextSelectionResolver.swift
new file mode 100644
index 0000000..62e12fe
--- /dev/null
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/TextSelectionResolver.swift
@@ -0,0 +1,115 @@
+import Foundation
+
+/// How `select_text` maps a matched text range onto the accessibility selection range.
+public enum TextSelectionMode: String, CaseIterable, Sendable {
+ case text
+ case cursorBefore = "cursor_before"
+ case cursorAfter = "cursor_after"
+}
+
+/// A resolved accessibility selection target.
+///
+/// Offsets are UTF-16 code units because `kAXSelectedTextRangeAttribute` carries a `CFRange`
+/// whose units follow the backing `CFString`, not Swift `Character` counts.
+public struct TextSelectionTarget: Equatable, Sendable {
+ public let location: Int
+ public let length: Int
+ public let matchedLocation: Int
+ public let matchedLength: Int
+ public let selectionMode: TextSelectionMode
+}
+
+public enum TextSelectionResolverError: Error, Equatable {
+ case emptyText
+ case notFound
+ case ambiguous(occurrenceCount: Int)
+}
+
+public enum TextSelectionResolver {
+ /// Resolves `text` inside `value`, optionally disambiguated by the text immediately before
+ /// (`prefix`) and after (`suffix`) it, into the range `select_text` should select.
+ ///
+ /// Every possible placement counts as an occurrence, including overlapping ones, so an
+ /// ambiguous target fails closed instead of silently selecting the first match.
+ public static func resolve(
+ value: String,
+ text: String,
+ prefix: String?,
+ suffix: String?,
+ selection mode: TextSelectionMode
+ ) throws -> TextSelectionTarget {
+ guard !text.isEmpty else {
+ throw TextSelectionResolverError.emptyText
+ }
+
+ let leading = normalized(prefix)
+ let trailing = normalized(suffix)
+ let needle = leading + text + trailing
+ let haystack = value as NSString
+ let (firstOccurrence, occurrenceCount) = occurrences(of: needle, in: haystack)
+
+ guard let first = firstOccurrence else {
+ throw TextSelectionResolverError.notFound
+ }
+
+ guard occurrenceCount == 1 else {
+ throw TextSelectionResolverError.ambiguous(occurrenceCount: occurrenceCount)
+ }
+
+ let matchedLocation = first.location + (leading as NSString).length
+ let matchedLength = (text as NSString).length
+
+ switch mode {
+ case .text:
+ return TextSelectionTarget(
+ location: matchedLocation,
+ length: matchedLength,
+ matchedLocation: matchedLocation,
+ matchedLength: matchedLength,
+ selectionMode: mode
+ )
+ case .cursorBefore:
+ return TextSelectionTarget(
+ location: matchedLocation,
+ length: 0,
+ matchedLocation: matchedLocation,
+ matchedLength: matchedLength,
+ selectionMode: mode
+ )
+ case .cursorAfter:
+ return TextSelectionTarget(
+ location: matchedLocation + matchedLength,
+ length: 0,
+ matchedLocation: matchedLocation,
+ matchedLength: matchedLength,
+ selectionMode: mode
+ )
+ }
+ }
+
+ private static func normalized(_ value: String?) -> String {
+ guard let value, !value.isEmpty else {
+ return ""
+ }
+
+ return value
+ }
+
+ private static func occurrences(of needle: String, in haystack: NSString) -> (NSRange?, Int) {
+ guard haystack.length > 0, !needle.isEmpty else { return (nil, 0) }
+ var first: NSRange?
+ var count = 0
+ var searchLocation = 0
+ while searchLocation < haystack.length {
+ let searchRange = NSRange(location: searchLocation, length: haystack.length - searchLocation)
+ // Exact code units keep prefix and target lengths aligned with AX.
+ // Canonically equivalent spellings can have different UTF-16 lengths.
+ let found = haystack.range(of: needle, options: .literal, range: searchRange)
+ guard found.location != NSNotFound else { break }
+ if first == nil { first = found }
+ count += 1
+ searchLocation = found.location + 1
+ }
+ return (first, count)
+ }
+}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
index e8d0c51..e53da12 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
@@ -165,6 +165,25 @@ public enum ToolDefinitions {
required: ["app", "element_index", "value"]
)
),
+ ToolDefinition(
+ name: "select_text",
+ description: "Select text inside a text element, or place the text cursor before or after it. Provide text exactly as it appears in the accessibility tree, including any Markdown formatting. If the text is not unique, provide surrounding prefix or suffix text to disambiguate it. This tool is part of plugin `Computer Use`.",
+ annotations: defaultAnnotations(),
+ inputSchema: objectSchema(
+ properties: [
+ "app": stringProperty(description: "App name or bundle identifier"),
+ "element_index": stringProperty(description: "Text element identifier"),
+ "text": stringProperty(description: "Target text as shown in the accessibility tree"),
+ "prefix": stringProperty(description: "Optional text immediately before the target, used to disambiguate repeated matches"),
+ "suffix": stringProperty(description: "Optional text immediately after the target, used to disambiguate repeated matches"),
+ "selection": stringProperty(
+ description: "Whether to select the text or place the cursor before or after it. Defaults to text.",
+ enumValues: TextSelectionMode.allCases.map(\.rawValue)
+ ),
+ ],
+ required: ["app", "element_index", "text"]
+ )
+ ),
ToolDefinition(
name: "type_text",
description: "Type literal text using keyboard input. This tool is part of plugin `Computer Use`.",
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index b667225..1bf4c58 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -279,7 +279,23 @@ final class OpenComputerUseKitTests: XCTestCase {
}
func testToolDefinitionCount() {
- XCTAssertEqual(ToolDefinitions.all.count, 10)
+ XCTAssertEqual(ToolDefinitions.all.count, 11)
+ XCTAssertEqual(
+ ToolDefinitions.all.map(\.name).sorted(),
+ [
+ "click",
+ "drag",
+ "get_app_state",
+ "list_apps",
+ "perform_secondary_action",
+ "press_key",
+ "query",
+ "scroll",
+ "select_text",
+ "set_value",
+ "type_text",
+ ]
+ )
}
func testReadToolArgumentsAcceptsJSONObject() throws {
@@ -722,6 +738,46 @@ final class OpenComputerUseKitTests: XCTestCase {
scrollPages?["description"] as? String,
"Number of pages to scroll. Fractional values are supported. Defaults to 1"
)
+ XCTAssertEqual(
+ tools["select_text"]?.description,
+ "Select text inside a text element, or place the text cursor before or after it. Provide text exactly as it appears in the accessibility tree, including any Markdown formatting. If the text is not unique, provide surrounding prefix or suffix text to disambiguate it. This tool is part of plugin `Computer Use`."
+ )
+ let selectTextSchema = tools["select_text"]?.inputSchema
+ let selectTextProperties = selectTextSchema?["properties"] as? [String: [String: Any]]
+ XCTAssertEqual(selectTextSchema?["required"] as? [String], ["app", "element_index", "text"])
+ XCTAssertEqual(selectTextSchema?["additionalProperties"] as? Bool, false)
+ XCTAssertEqual(
+ selectTextProperties?["selection"]?["enum"] as? [String],
+ ["text", "cursor_before", "cursor_after"]
+ )
+ XCTAssertEqual(
+ selectTextProperties?["selection"]?["description"] as? String,
+ "Whether to select the text or place the cursor before or after it. Defaults to text."
+ )
+ XCTAssertEqual(selectTextProperties?["element_index"]?["description"] as? String, "Text element identifier")
+ XCTAssertEqual(
+ selectTextProperties?["prefix"]?["description"] as? String,
+ "Optional text immediately before the target, used to disambiguate repeated matches"
+ )
+ XCTAssertEqual(
+ selectTextProperties?["suffix"]?["description"] as? String,
+ "Optional text immediately after the target, used to disambiguate repeated matches"
+ )
+ XCTAssertEqual(
+ selectTextProperties?["text"]?["description"] as? String,
+ "Target text as shown in the accessibility tree"
+ )
+ }
+
+ func testSelectTextRejectsInvalidSelectionMode() {
+ let dispatcher = ComputerUseToolDispatcher()
+ let result = dispatcher.callToolAsResult(
+ name: "select_text",
+ arguments: ["app": "Sublime Text", "element_index": "14", "text": "abc", "selection": "sideways"]
+ )
+
+ XCTAssertTrue(result.isError)
+ XCTAssertEqual(result.primaryText, "selection must be one of text, cursor_before, cursor_after")
}
func testDispatcherMissingArgumentsMatchOfficialToolText() {
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TextSelectionOperationTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TextSelectionOperationTests.swift
new file mode 100644
index 0000000..46681ee
--- /dev/null
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TextSelectionOperationTests.swift
@@ -0,0 +1,118 @@
+import AppKit
+import XCTest
+@testable import OpenComputerUseKit
+
+final class TextSelectionOperationTests: XCTestCase {
+ private func target(_ mode: TextSelectionMode = .text) throws -> TextSelectionTarget {
+ try TextSelectionResolver.resolve(value: "a👍中文", text: "中文", prefix: nil, suffix: nil, selection: mode)
+ }
+
+ func testAllModesAreVerified() throws {
+ for mode in TextSelectionMode.allCases {
+ let target = try target(mode)
+ var applied: CFRange?
+ try applyVerifiedTextSelection(target: target, originalValue: "a👍中文", validate: {},
+ readValue: { "a👍中文" }, writeRange: { applied = $0 }, readRange: { applied })
+ XCTAssertEqual(applied?.location, target.location)
+ XCTAssertEqual(applied?.length, target.length)
+ }
+ }
+
+ func testMissingOrRefusedRangeIsAnError() throws {
+ for range in [nil, CFRange(location: 0, length: 0)] {
+ XCTAssertThrowsError(try applyVerifiedTextSelection(target: target(), originalValue: "a👍中文", validate: {},
+ readValue: { "a👍中文" }, writeRange: { _ in }, readRange: { range })) {
+ XCTAssertTrue(String(describing: $0).contains("could not verify"))
+ }
+ }
+ }
+
+ func testRevokedSessionCannotWrite() throws {
+ var wrote = false
+ XCTAssertThrowsError(try applyVerifiedTextSelection(target: target(), originalValue: "a👍中文",
+ validate: { throw ComputerUseError.message("revoked") }, readValue: { "a👍中文" },
+ writeRange: { _ in wrote = true }, readRange: { nil }))
+ XCTAssertFalse(wrote)
+ }
+
+ func testRevocationAfterWritingCannotReportSuccess() throws {
+ var range: CFRange?
+ XCTAssertThrowsError(try applyVerifiedTextSelection(target: target(), originalValue: "a👍中文",
+ validate: { if range != nil { throw ComputerUseError.message("revoked") } }, readValue: { "a👍中文" },
+ writeRange: { range = $0 }, readRange: { range }))
+ }
+
+ func testCanonicallyEqualButDifferentValueCannotUseTheOldRange() throws {
+ var wrote = false
+ XCTAssertThrowsError(try applyVerifiedTextSelection(target: target(), originalValue: "é",
+ validate: {}, readValue: { "e\u{301}" }, writeRange: { _ in wrote = true }, readRange: { nil }))
+ XCTAssertFalse(wrote)
+ }
+
+ func testChangedTextAfterWritingCannotReportSuccess() throws {
+ var value = "a👍中文"
+ let target = try target()
+ XCTAssertThrowsError(try applyVerifiedTextSelection(target: target, originalValue: value,
+ validate: {}, readValue: { value }, writeRange: { _ in value = "changed" },
+ readRange: { CFRange(location: target.location, length: target.length) }))
+ }
+
+ func testQueryIndexesValidateSessionBeforeLookingUpTheApp() {
+ let service = ComputerUseService()
+ XCTAssertThrowsError(try LockedUseActionScope.withValidator({ throw ComputerUseError.message("revoked") }) {
+ try service.selectText(app: "nonexistent-selection-test-app", elementIndex: "1000001", text: "x",
+ prefix: nil, suffix: nil, selection: .text)
+ }) { XCTAssertTrue(String(describing: $0).contains("revoked")) }
+ }
+
+ func testNativeDispatcherRejectsMalformedOptions() {
+ let dispatcher = ComputerUseToolDispatcher()
+ for (key, value) in [("prefix", 2 as Any), ("suffix", false as Any), ("selection", 3 as Any), ("selection", "" as Any)] {
+ let result = dispatcher.callToolAsResult(name: "select_text", arguments: [
+ "app": "nonexistent-selection-test-app", "element_index": "1", "text": "x", key: value,
+ ])
+ XCTAssertTrue(result.isError)
+ XCTAssertTrue(result.primaryText?.contains(key) == true)
+ }
+ }
+
+ @MainActor
+ func testRealAXTextViewSupportsUTF16SelectionAndBothCaretModes() throws {
+ guard AXIsProcessTrusted(), NSScreen.main != nil else { throw XCTSkip("Real AX selection requires Accessibility and a display") }
+ NSApplication.shared.finishLaunching()
+ let window = NSWindow(contentRect: NSRect(x: 120, y: 120, width: 350, height: 150),
+ styleMask: [.titled], backing: .buffered, defer: false)
+ window.isReleasedWhenClosed = false
+ window.title = "OCU selection fixture \(UUID().uuidString)"
+ let textView = NSTextView(frame: NSRect(x: 0, y: 0, width: 350, height: 150))
+ textView.string = "a👍中文"
+ window.contentView = textView
+ window.orderFront(nil)
+ defer { window.close() }
+ func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
+ var value: CFTypeRef?
+ return AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success ? value : nil
+ }
+ func textArea(_ element: AXUIElement, depth: Int = 0) -> AXUIElement? {
+ if attribute(element, kAXRoleAttribute) as? String == kAXTextAreaRole { return element }
+ guard depth < 8 else { return nil }
+ for child in attribute(element, kAXChildrenAttribute) as? [AXUIElement] ?? [] {
+ if let found = textArea(child, depth: depth + 1) { return found }
+ }
+ return nil
+ }
+ RunLoop.main.run(until: Date().addingTimeInterval(0.03))
+ let application = AXUIElementCreateApplication(getpid())
+ let windows = try XCTUnwrap(attribute(application, kAXWindowsAttribute) as? [AXUIElement])
+ let axWindow = try XCTUnwrap(windows.first { attribute($0, kAXTitleAttribute) as? String == window.title })
+ let element = try XCTUnwrap(textArea(axWindow))
+ for mode in TextSelectionMode.allCases {
+ let target = try target(mode)
+ try AccessibilityTextSelection.writeRange(CFRange(location: target.location, length: target.length), to: element)
+ let range = try XCTUnwrap(AccessibilityTextSelection.readRange(of: element))
+ XCTAssertEqual(range.location, target.location)
+ XCTAssertEqual(range.length, target.length)
+ XCTAssertEqual(textView.selectedRange(), NSRange(location: target.location, length: target.length))
+ }
+ }
+}
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TextSelectionResolverTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TextSelectionResolverTests.swift
new file mode 100644
index 0000000..c0f6aa9
--- /dev/null
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/TextSelectionResolverTests.swift
@@ -0,0 +1,104 @@
+import XCTest
+@testable import OpenComputerUseKit
+
+final class TextSelectionResolverTests: XCTestCase {
+ private func resolve(
+ value: String,
+ text: String,
+ prefix: String? = nil,
+ suffix: String? = nil,
+ selection: TextSelectionMode = .text
+ ) throws -> TextSelectionTarget {
+ try TextSelectionResolver.resolve(value: value, text: text, prefix: prefix, suffix: suffix, selection: selection)
+ }
+
+ func testResolvesUniqueMatch() throws {
+ let target = try resolve(value: "hello world", text: "world")
+
+ XCTAssertEqual(target.location, 6)
+ XCTAssertEqual(target.length, 5)
+ XCTAssertEqual(target.matchedLocation, 6)
+ XCTAssertEqual(target.matchedLength, 5)
+ XCTAssertEqual(target.selectionMode, .text)
+ }
+
+ func testCursorModesCollapseToZeroLengthRanges() throws {
+ let before = try resolve(value: "hello world", text: "world", selection: .cursorBefore)
+ let after = try resolve(value: "hello world", text: "world", selection: .cursorAfter)
+
+ XCTAssertEqual(before.location, 6)
+ XCTAssertEqual(before.length, 0)
+ XCTAssertEqual(after.location, 11)
+ XCTAssertEqual(after.length, 0)
+ }
+
+ func testMissingTextFailsClosed() {
+ XCTAssertThrowsError(try resolve(value: "hello world", text: "missing")) { error in
+ XCTAssertEqual(error as? TextSelectionResolverError, .notFound)
+ }
+ }
+
+ func testEmptyTextFailsClosed() {
+ XCTAssertThrowsError(try resolve(value: "hello", text: "")) { error in
+ XCTAssertEqual(error as? TextSelectionResolverError, .emptyText)
+ }
+ }
+
+ func testAmbiguousTextRequiresDisambiguation() {
+ XCTAssertThrowsError(try resolve(value: "ab ab", text: "ab")) { error in
+ XCTAssertEqual(error as? TextSelectionResolverError, .ambiguous(occurrenceCount: 2))
+ }
+ }
+
+ func testOverlappingPlacementsAlsoCountAsAmbiguous() {
+ XCTAssertThrowsError(try resolve(value: "aaa", text: "aa")) { error in
+ XCTAssertEqual(error as? TextSelectionResolverError, .ambiguous(occurrenceCount: 2))
+ }
+ }
+
+ func testPrefixDisambiguatesToTheLaterMatch() throws {
+ let target = try resolve(value: "ab ab", text: "ab", prefix: "ab ")
+
+ XCTAssertEqual(target.location, 3)
+ XCTAssertEqual(target.length, 2)
+ }
+
+ func testSuffixDisambiguatesToTheEarlierMatch() throws {
+ let target = try resolve(value: "ab cd ab ef", text: "ab", suffix: " cd")
+
+ XCTAssertEqual(target.location, 0)
+ XCTAssertEqual(target.length, 2)
+ }
+
+ func testEmptyPrefixAndSuffixBehaveLikeNil() throws {
+ let target = try resolve(value: "hello world", text: "world", prefix: "", suffix: "")
+
+ XCTAssertEqual(target.location, 6)
+ }
+
+ func testOffsetsFollowUTF16UnitsForEmojiAndCJK() throws {
+ let target = try resolve(value: "a\u{1F44D}中文b\u{1F44D}", text: "中文")
+
+ XCTAssertEqual(target.location, 3)
+ XCTAssertEqual(target.length, 2)
+ }
+
+ func testCanonicalEquivalenceDoesNotProduceAnIncorrectUTF16Range() throws {
+ XCTAssertThrowsError(try resolve(value: "e\u{301} target", text: "é")) {
+ XCTAssertEqual($0 as? TextSelectionResolverError, .notFound)
+ }
+ let target = try resolve(value: "é target; e\u{301} target", text: "target", prefix: "e\u{301} ")
+ XCTAssertEqual(target.location, 13)
+ XCTAssertEqual(target.length, 6)
+ }
+
+ func testManyOverlappingMatchesReportTheFullCount() {
+ XCTAssertThrowsError(try resolve(value: String(repeating: "a", count: 1000), text: "aa")) {
+ XCTAssertEqual($0 as? TextSelectionResolverError, .ambiguous(occurrenceCount: 999))
+ }
+ }
+
+ func testSelectionModeRawValuesMatchTheToolSchema() {
+ XCTAssertEqual(TextSelectionMode.allCases.map(\.rawValue), ["text", "cursor_before", "cursor_after"])
+ }
+}
diff --git a/scripts/node-repl/open-computer-use-repl.mjs b/scripts/node-repl/open-computer-use-repl.mjs
index 6fb1dfa..89b798c 100755
--- a/scripts/node-repl/open-computer-use-repl.mjs
+++ b/scripts/node-repl/open-computer-use-repl.mjs
@@ -47,6 +47,7 @@ The runtime exposes an asynchronous app-bound API:
- \`await app.typeText(text, { keyMethod? })\`
- \`await app.pressKey(key, { keyMethod? })\`
- \`await app.setValue(elementIndex, value)\`
+- \`await app.selectText(elementIndex, text, { prefix?, suffix?, selection? })\`: macOS-only; selection is text (default), cursor_before or cursor_after. Ambiguous text and unverified selections throw.
- \`await app.performSecondaryAction(elementIndex, action)\`
After actions, call \`getAXState()\` in the same js invocation when the next decision depends on the updated UI. Re-derive element indexes from fresh state after navigation or layout changes. Prefer element indexes over coordinates. Open Computer Use keeps its existing local safety gates, including password-manager denial and explicit authorization for global pointer fallbacks.`;
@@ -348,6 +349,23 @@ export function createCuaApi(native, activeOutput) {
await call("scroll", { app, element_index: target, direction, pages });
},
async setValue(elementIndex, value) { await call("set_value", { app, element_index: elementIndex, value }); },
+ async selectText(elementIndex, text, options = {}) {
+ const index = typeof elementIndex === "string" && /^(0|[1-9]\d*)$/u.test(elementIndex) ? Number(elementIndex) : elementIndex;
+ if (!Number.isSafeInteger(index) || index < 0) throw new Error("selectText requires a non-negative integer element index");
+ if (typeof text !== "string" || !text.length) throw new Error("selectText requires non-empty text");
+ if (!isPlainObject(options)) throw new Error("selectText expects an options object");
+ for (const key of Object.keys(options)) {
+ if (!["prefix", "suffix", "selection"].includes(key)) throw new Error(`Unknown selectText option: ${key}`);
+ }
+ for (const key of ["prefix", "suffix"]) {
+ if (options[key] !== undefined && typeof options[key] !== "string") throw new Error(`${key} must be a string`);
+ }
+ if (options.selection !== undefined && !["text", "cursor_before", "cursor_after"].includes(options.selection)) {
+ throw new Error("selection must be text, cursor_before or cursor_after");
+ }
+ await call("select_text", { app, element_index: elementIndex, text,
+ prefix: options.prefix, suffix: options.suffix, selection: options.selection });
+ },
async typeText(text, options = {}) { await call("type_text", { app, text, ...optionsToKeyArgs(options) }); },
async performSecondaryAction(elementIndex, action) { await call("perform_secondary_action", { app, element_index: elementIndex, action }); },
});
diff --git a/scripts/node-repl/open-computer-use-repl.test.mjs b/scripts/node-repl/open-computer-use-repl.test.mjs
index fcea8a6..f343bdc 100644
--- a/scripts/node-repl/open-computer-use-repl.test.mjs
+++ b/scripts/node-repl/open-computer-use-repl.test.mjs
@@ -352,3 +352,43 @@ test("query returns truncation metadata and rejects malformed native responses",
assert.equal(bad.isError, true);
assert.match(bad.content.at(-1).text, /Invalid query response/);
});
+
+
+for (const Session of [PersistentJavaScriptSession, WorkerJavaScriptSession]) {
+ test(`${Session.name}: selectText forwards all selection modes without a snapshot`, async t => {
+ const native = mockNative(), session = new Session({ native });
+ t.after(() => session.close?.());
+ let result = await session.run(`var selectionApp = await cua.getApp("Text", { initialState: false }); await selectionApp.selectText(1000001, "中文🙂", {prefix: "before ", suffix: " after"});`);
+ assert.equal(result.isError, false, result.content.at(-1).text);
+ assert.deepEqual(native.calls[0], { name: "select_text", arguments: { app: "Text", element_index: 1000001, text: "中文🙂", prefix: "before ", suffix: " after", selection: undefined } });
+ for (const mode of ["text", "cursor_before", "cursor_after"]) {
+ result = await session.run(`await selectionApp.selectText("5", "target", {selection: "${mode}"});`);
+ assert.equal(result.isError, false, result.content.at(-1).text);
+ assert.equal(native.calls.at(-1).arguments.selection, mode);
+ }
+ assert.equal(native.calls.some(call => call.name === "get_app_state"), false);
+ });
+
+ test(`${Session.name}: selectText rejects invalid inputs before native calls`, async t => {
+ const native = mockNative(), session = new Session({ native });
+ t.after(() => session.close?.());
+ await session.run(`var selectionApp = await cua.getApp("Text", { initialState: false });`);
+ for (const args of [`-1,"x"`, `1.5,"x"`, `true,"x"`, `"bad","x"`, `[],"x"`, `0,""`, `0,3`, `0,"x",null`, `0,"x",[]`, `0,"x",{prefix:2}`, `0,"x",{suffix:null}`, `0,"x",{selection:"after"}`, `0,"x",{selection:""}`, `0,"x",{typo:true}`]) {
+ const result = await session.run(`await selectionApp.selectText(${args});`);
+ assert.equal(result.isError, true, args);
+ }
+ assert.equal(native.calls.length, 0);
+ });
+
+ test(`${Session.name}: selectText preserves native verification and platform failures`, async t => {
+ const native = mockNative(), session = new Session({ native });
+ t.after(() => session.close?.());
+ await session.run(`var selectionApp = await cua.getApp("Text", { initialState: false });`);
+ for (const message of ["select_text could not verify the requested range", 'unsupportedTool("select_text")']) {
+ native.request = async () => textResult(message, true);
+ const result = await session.run(`await selectionApp.selectText(0,"target");`);
+ assert.equal(result.isError, true);
+ assert.ok(result.content.at(-1).text.includes(message), result.content.at(-1).text);
+ }
+ });
+}
diff --git a/scripts/tests/select-text-smoke.mjs b/scripts/tests/select-text-smoke.mjs
new file mode 100644
index 0000000..a85427d
--- /dev/null
+++ b/scripts/tests/select-text-smoke.mjs
@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { existsSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { JsonLinePeer, WorkerJavaScriptSession } from "../node-repl/open-computer-use-repl.mjs";
+
+assert.equal(process.platform, "darwin", "select-text smoke requires macOS");
+const root = fileURLToPath(new URL("../../", import.meta.url));
+const nativePath = path.resolve(process.argv[2] ?? path.join(root, ".build/debug/OpenComputerUse"));
+const fixturePath = path.join(path.dirname(nativePath), "OpenComputerUseFixture");
+assert.ok(existsSync(nativePath) && existsSync(fixturePath), "Run swift build first");
+const namespace = randomUUID().toUpperCase();
+const directory = path.join(tmpdir(), `open-computer-use-fixture-${namespace}`);
+const env = { ...process.env, OPEN_COMPUTER_USE_FIXTURE_NAMESPACE: namespace, OPEN_COMPUTER_USE_FIXTURE_HEADLESS: "1",
+ OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY: "1", OPEN_COMPUTER_USE_VISUAL_CURSOR: "0",
+ OPEN_COMPUTER_USE_ACTION_READ_BACK: "0" };
+const fixture = spawn(fixturePath, [], { env, stdio: ["ignore", "ignore", "pipe"] });
+let fixtureError = "";
+fixture.stderr.on("data", data => { fixtureError = (fixtureError + data).slice(-4096); });
+const stateFile = path.join(directory, "state.json");
+let native, session;
+async function waitFor(test) {
+ const deadline = Date.now() + 5000;
+ while (Date.now() < deadline) {
+ if (test()) return;
+ if (fixture.exitCode !== null) throw new Error(`Fixture exited: ${fixtureError}`);
+ await new Promise(resolve => setTimeout(resolve, 30));
+ }
+ throw new Error(`Fixture timeout: ${fixtureError}`);
+}
+function state() { return JSON.parse(readFileSync(stateFile, "utf8")); }
+async function run(code, expectedError) {
+ const result = await session.run(code);
+ const output = result.content.filter(item => item.type === "text").map(item => item.text).join("\n");
+ if (expectedError) { assert.equal(result.isError, true, output); assert.match(output, expectedError); }
+ else assert.equal(result.isError, false, output);
+ return output;
+}
+try {
+ await waitFor(() => existsSync(stateFile));
+ native = new JsonLinePeer({ command: nativePath, args: ["mcp"], env });
+ await native.initialize();
+ const request = native.request.bind(native);
+ native.request = async (method, params, timeout) => {
+ const result = await request(method, params, timeout);
+ if (method === "tools/call" && params.name === "select_text" && !result.isError) {
+ assert.deepEqual(result.content, [{ type: "text", text: "ok" }], "Selection must honor no-read-back mode");
+ }
+ return result;
+ };
+ session = new WorkerJavaScriptSession({ native });
+ await run(`var app = await cua.getApp("OpenComputerUseFixture");
+ var ax = await app.getAXState({emit:false});
+ var index = Number(ax.match(/^\\s*(\\d+).*ID: fixture-input(?:\\s|$)/m)[1]);
+ await app.setValue(index, "a👍中文|中文");`);
+ await run(`await app.selectText(index, "中文");`, /matches.*prefix or suffix/);
+ await run(`await app.selectText(index, "missing");`, /could not find/);
+ await run(`await app.selectText(index, "中文", {prefix:"a👍"});`);
+ assert.equal(state().selectedRangeLocation, 3);
+ assert.equal(state().selectedRangeLength, 2);
+ assert.equal(state().selectedText, "中文");
+ const observed = await run(`await app.getAXState();`);
+ assert.match(observed, /Selected text: \[中文\]/);
+ await run(`await app.selectText(index, "中文", {prefix:"|", selection:"cursor_before"});`);
+ assert.equal(state().selectedRangeLocation, 6);
+ assert.equal(state().selectedRangeLength, 0);
+ await run(`await app.selectText(index, "中文", {prefix:"|", selection:"cursor_after"});`);
+ assert.equal(state().selectedRangeLocation, 8);
+ assert.equal(state().selectedRangeLength, 0);
+ assert.equal(state().elements.find(item => item.identifier === "fixture-input").value, "a👍中文|中文");
+ console.log("Worker JS → native MCP → fixture: ambiguity, missing text, Unicode selection, both caret modes and no-read-back passed.");
+} finally {
+ await session?.close();
+ native?.close();
+ fixture.kill("SIGTERM");
+ if (fixture.exitCode === null && fixture.signalCode === null) {
+ await new Promise(resolve => {
+ const timer = setTimeout(() => { fixture.kill("SIGKILL"); resolve(); }, 3000);
+ fixture.once("exit", () => { clearTimeout(timer); resolve(); });
+ });
+ }
+ rmSync(directory, { recursive: true, force: true });
+}
diff --git a/skills/open-computer-use/SKILL.md b/skills/open-computer-use/SKILL.md
index b4f3952..c200440 100644
--- a/skills/open-computer-use/SKILL.md
+++ b/skills/open-computer-use/SKILL.md
@@ -15,6 +15,12 @@ It supports the same core tool surface across macOS, Linux, and Windows:
`list_apps`, `get_app_state`, `click`, `perform_secondary_action`, `scroll`,
`drag`, `type_text`, `press_key`, and `set_value`.
+The macOS runtime additionally implements `select_text`, which selects the given text inside a text
+element or places the text cursor before / after it (`selection`: `text`, `cursor_before`,
+`cursor_after`). Pass `text` exactly as it appears in the accessibility tree, plus `prefix` /
+`suffix` when it is not unique; an ambiguous target fails closed instead of silently selecting the
+first match. JavaScript exposes `app.selectText(elementIndex, text, options?)`. Unreadable or mismatched range verification throws, including when full action read-back is off. Windows and Linux runtimes do not implement this tool yet.
+
macOS additionally exposes `query`: a targeted accessibility lookup that finds
controls by text and/or role without rendering a tree or taking a screenshot.
@@ -38,7 +44,7 @@ controls by text and/or role without rendering a tree or taking a screenshot.
- Treat the target desktop as the user's real session. Do not inspect password managers, unrelated private content, or sensitive apps unless the user explicitly asked for that task.
- Ask before sending, deleting, purchasing, approving, uploading, or making other externally visible changes.
- Do not assume Codex.app plugin helpers are available. Use the installed `open-computer-use` / `ocu` CLI or an explicit MCP config.
-- Always run `get_app_state` before using `element_index`; do not guess indexes across sessions or after large UI changes.
+- Use indexes returned by `get_app_state` or macOS `query` in the same native session; do not guess indexes across sessions or after large UI changes.
- Prefer semantic actions and `set_value` for editable controls. Use coordinate `click`, `scroll`, and `drag` only when the element tree does not expose a safer target.
- On macOS, do not enable `OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS=1` unless the user explicitly requested `click_method: "global"`, a `drag` that must drive a window-server drag session (window move, drag-select text, Finder drag-and-drop), or other diagnostic behavior that may move the real pointer. Without it `drag` reports `Drag delivered via app_post` and those operations have no effect; see `references/usage.md` for alternatives.
- On Windows and Linux, confirm the command is running inside the logged-in desktop session before assuming GUI automation is available.
diff --git a/skills/open-computer-use/references/usage.md b/skills/open-computer-use/references/usage.md
index 9a710b5..84048c7 100644
--- a/skills/open-computer-use/references/usage.md
+++ b/skills/open-computer-use/references/usage.md
@@ -39,6 +39,7 @@ drag
type_text
press_key
set_value
+select_text (macOS runtime only)
```
macOS also exposes `query` for targeted AX lookup. These nine core tools remain the cross-platform compatibility surface. The bundled Codex plugin uses a
@@ -228,6 +229,24 @@ Every non-fixture `drag` result includes a text item that begins `Drag delivered
When the gate is not enabled, treat window-server drags as unavailable and reach the same outcome another way: copy or move files with a shell command instead of a Finder drag, use `set_value` or keyboard selection instead of drag-selecting text, and use the app's own window controls instead of dragging a title bar.
+## Text Selection (macOS)
+
+Use `await app.selectText(elementIndex, text, options?)` with a snapshot or `query`
+index from the current session. The native CLI/MCP tool is `select_text`.
+
+| Argument | Meaning |
+| --- | --- |
+| `elementIndex` | The text element's current index. |
+| `text` | Exact target text, including whitespace and Unicode spelling. |
+| `options.prefix` / `options.suffix` | Text immediately before / after the target, for repeated matches. |
+| `options.selection` | `text` (default), `cursor_before` or `cursor_after`. |
+
+Ambiguous or missing text, unsupported selection, text changes and unverified ranges
+return errors. The action verifies the applied UTF-16 range even when full action
+read-back is off; use `getAXState()` for the next UI observation. It does not activate
+the app or fall back to keyboard/pointer input. Linux and Windows do not implement
+`select_text` yet.
+
## Software Cursor
The software cursor keeps a window-relative anchor for its target point. During travel, settle, click