Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

Add native computer-use call command

Leocf087dc66da5
diff --git a/README.md b/README.md
index b4281d2..7d26dff 100644
--- a/README.md
+++ b/README.md
@@ -50,6 +50,11 @@ open-computer-use install-codex-mcp
open-computer-use install-codex-plugin
# Start the MCP server directly
open-computer-use mcp
+# Call a single Computer Use tool and print the MCP-style JSON result
+open-computer-use call list_apps
+open-computer-use call get_app_state --args '{"app":"TextEdit"}'
+# Run a sequence in one process so element_index state can be reused
+open-computer-use call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}},{"tool":"press_key","args":{"app":"TextEdit","key":"Return"}}]'
# Check permissions; onboarding only opens when something is missing
open-computer-use doctor
# Show help
diff --git a/README.zh-CN.md b/README.zh-CN.md
index b0cb29c..9d6bad3 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -54,6 +54,13 @@ open-computer-use install-codex-plugin
# 直接启动 MCP server
open-computer-use mcp
+# 直接调用单个 Computer Use tool,输出 MCP 风格的 JSON result
+open-computer-use call list_apps
+open-computer-use call get_app_state --args '{"app":"TextEdit"}'
+
+# 在同一个进程里编排连续动作,复用 get_app_state 拿到的 element_index
+open-computer-use call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}},{"tool":"press_key","args":{"app":"TextEdit","key":"Return"}}]'
+
# 检查权限;只有缺失时才会拉起引导,已全部授权则只打印状态并退出
open-computer-use doctor
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
index f66d167..a4d70ff 100644
--- a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
@@ -5,16 +5,27 @@ import OpenComputerUseKit
@main
enum OpenComputerUseMain {
- static func main() throws {
- let arguments = Array(CommandLine.arguments.dropFirst())
- let command: OpenComputerUseCLICommand
-
+ @MainActor
+ static func main() {
do {
- command = try parseOpenComputerUseCLI(arguments: arguments)
+ try run()
} catch let error as OpenComputerUseCLIError {
writeToStandardError(error.errorDescription ?? error.message)
exit(EXIT_FAILURE)
+ } catch let error as ComputerUseError {
+ writeToStandardError(error.errorDescription ?? String(describing: error))
+ exit(EXIT_FAILURE)
+ } catch {
+ let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)
+ writeToStandardError(message)
+ exit(EXIT_FAILURE)
}
+ }
+
+ @MainActor
+ private static func run() throws {
+ let arguments = Array(CommandLine.arguments.dropFirst())
+ let command = try parseOpenComputerUseCLI(arguments: arguments)
switch command {
case .mcp:
@@ -39,6 +50,15 @@ enum OpenComputerUseMain {
case let .snapshot(app):
let service = ComputerUseService()
print(try service.getAppState(app: app).primaryText ?? "")
+ case let .call(invocation):
+ if VisualCursorSupport.isEnabled {
+ _ = NSApplication.shared.setActivationPolicy(.accessory)
+ }
+ let output = try runOpenComputerUseCall(invocation)
+ print(try output.jsonText())
+ if output.hasToolError {
+ exit(EXIT_FAILURE)
+ }
case .turnEnded:
print("turn-ended acknowledged")
case let .help(command):
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index d58ea1f..faf811d 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -5,7 +5,7 @@
## 当前目录结构
- `apps/OpenComputerUse`
- 主入口,负责 `mcp`、`doctor`、`list-apps`、`snapshot`、`turn-ended` 等 CLI 命令,以及 `-h` / `--help` / `-v` / `--version` 这类全局参数;不带参数启动时会先检查权限,只有缺失时才进入无 Dock 图标的 app 模式权限引导窗口,`doctor` 也只会在检测到缺失权限时拉起这套 onboarding UI。
+ 主入口,负责 `mcp`、`doctor`、`list-apps`、`snapshot`、`call`、`turn-ended` 等 CLI 命令,以及 `-h` / `--help` / `-v` / `--version` 这类全局参数;不带参数启动时会先检查权限,只有缺失时才进入无 Dock 图标的 app 模式权限引导窗口,`doctor` 也只会在检测到缺失权限时拉起这套 onboarding UI。
- `apps/OpenComputerUseFixture`
本地 GUI fixture app,用来承载低风险、可预测的点击/输入/滚动/拖拽验证路径。
- `apps/OpenComputerUseSmokeSuite`
@@ -51,10 +51,11 @@
### 3. Tool Service 层
-- `ComputerUseService` 负责把 MCP tool 请求映射到本地能力。
+- `ComputerUseService` 负责把 Computer Use tool 请求映射到本地能力,`ComputerUseToolDispatcher` 则把 9 个 tool 的参数解析与 service 方法分发收敛成 MCP server 和 `open-computer-use call` 共用的一层。
- `list_apps` 通过 Spotlight metadata query 拉取标准 application 目录里的 app bundle,并读取 `kMDItemUseCount` / `kMDItemLastUsedDate_Ranking` 这类系统元数据;再与 `NSWorkspace` 的运行态 app 合并,输出“当前运行中 + 近 14 天用过”的视图。
- `get_app_state` 优先走真实 AX / 窗口截图,但不再为了读状态而显式 `activate` 目标 app;当目标是仓库内 fixture app 时,回退到 fixture 导出的合成状态。
- MCP `tools/list` 的 description / input schema 当前按官方 `computer-use` 的 9 个 tools 文案和参数面收敛,尽量减少 host 侧提示词和 tool surface 偏差。
+- `open-computer-use call <tool> --args '{...}'` 会直接输出 MCP-style JSON result;`open-computer-use call --calls '[...]'` 会在同一进程里顺序执行 JSON 数组里的 tool calls,并复用同一个 `ComputerUseService` 内存态,因此 `get_app_state` 之后的 action tool 可以继续使用同一轮 snapshot 的 `element_index`。序列执行遇到 `isError=true` 的 tool result 后停止。
- 对真实 app 的 `get_app_state` / action tool 入口,当前新增了一层官方风格的高风险 bundle denylist:bundle-id 直传时直接返回 safety denial;名称匹配时默认不解析到这些 app,尽量贴近官方对终端、密码管理器、Chrome 与少量系统敏感组件的防护行为。
- 普通 app 的 element frame 当前按“窗口左上角为原点”的 window-relative 坐标输出,便于后续把 `element_index` 和截图坐标统一到同一套参考系。
- `click` / `set_value` 在执行真实动作前后,会额外驱动一层透明 `SoftwareCursorOverlay` window:两者的移动阶段现在共用一条 heading-driven 的官方风格 motion 内核,显式把“当前 cursor 朝向”和“最终 resting pose”一起喂给选路器,优先生成需要时先掉头、再沿车头方向推进的 C 形/单侧大弧轨迹;首次显示时按官方 binary 的 fresh state 从 AppKit 全局 `(0,0)` window origin 生成起点,后续动作继续复用上一帧 visible tip。真正显示出来的 cursor 不再直接等于 path sample,而是经过一层独立的 visual dynamics 状态,把 visible tip、velocity、angle 和 fog/offset 持续推进。`click` 结尾会衔接 click pulse、idle sway 和自动淡出,`set_value` 则只做 settle / idle / hide,不给 pulse。
@@ -110,3 +111,5 @@
- 手工诊断:
- `.build/debug/OpenComputerUse doctor`
- `.build/debug/OpenComputerUse snapshot <app>`
+ - `.build/debug/OpenComputerUse call list_apps`
+ - `.build/debug/OpenComputerUse call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}}]'`
diff --git a/docs/histories/2026-04/20260421-2058-add-native-call-command.md b/docs/histories/2026-04/20260421-2058-add-native-call-command.md
new file mode 100644
index 0000000..c62e9a4
--- /dev/null
+++ b/docs/histories/2026-04/20260421-2058-add-native-call-command.md
@@ -0,0 +1,31 @@
+## [2026-04-21 20:58] | Task: Add native call command
+
+### Execution Context
+* **Agent ID**: `Codex`
+* **Base Model**: `GPT-5`
+* **Runtime**: `Codex CLI`
+
+### User Query
+> 统一用 `open-computer-use call` 调用 9 个 tool,并支持 JSON 数组编排连续动作。
+
+### Changes Overview
+**Scope:** Swift CLI, MCP tool dispatch, docs, npm package help
+
+**Key Actions:**
+- **Shared dispatcher**: Added a reusable `ComputerUseToolDispatcher` so MCP `tools/call` and CLI `call` share the same 9-tool argument mapping.
+- **Native CLI call**: Added `open-computer-use call <tool> --args ...` plus `open-computer-use call --calls ...` / `--calls-file` for same-process sequences that preserve app snapshot state.
+- **Docs and package help**: Documented the new command in README, architecture notes, and npm launcher help.
+
+### Design Intent (Why)
+The JSON-array form keeps a single `ComputerUseService` instance alive across the sequence, so actions after `get_app_state` can reuse the captured `element_index` map instead of spawning fresh processes with empty state.
+
+### Files Modified
+- `apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
+- `docs/ARCHITECTURE.md`
+- `README.md`
+- `README.zh-CN.md`
+- `scripts/npm/build-packages.mjs`
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
new file mode 100644
index 0000000..9f8ff5f
--- /dev/null
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseToolDispatcher.swift
@@ -0,0 +1,281 @@
+import Foundation
+
+public final class ComputerUseToolDispatcher {
+ private let service: ComputerUseService
+
+ public init(service: ComputerUseService = ComputerUseService()) {
+ self.service = service
+ }
+
+ public func callTool(name: String, arguments: [String: Any]) throws -> ToolCallResult {
+ switch name {
+ case "list_apps":
+ return service.listApps()
+ case "get_app_state":
+ return try service.getAppState(app: requireString("app", in: arguments))
+ case "click":
+ return try service.click(
+ app: requireString("app", in: arguments),
+ elementIndex: optionalString("element_index", in: arguments),
+ x: optionalDouble("x", in: arguments),
+ y: optionalDouble("y", in: arguments),
+ clickCount: Int(optionalDouble("click_count", in: arguments) ?? 1),
+ mouseButton: optionalString("mouse_button", in: arguments) ?? "left"
+ )
+ case "perform_secondary_action":
+ return try service.performSecondaryAction(
+ app: requireString("app", in: arguments),
+ elementIndex: requireString("element_index", in: arguments),
+ action: requireString("action", in: arguments)
+ )
+ case "scroll":
+ return try service.scroll(
+ app: requireString("app", in: arguments),
+ direction: requireString("direction", in: arguments),
+ elementIndex: requireString("element_index", in: arguments),
+ pages: Int(optionalDouble("pages", in: arguments) ?? 1)
+ )
+ case "drag":
+ return try service.drag(
+ app: requireString("app", in: arguments),
+ fromX: requireDouble("from_x", in: arguments),
+ fromY: requireDouble("from_y", in: arguments),
+ toX: requireDouble("to_x", in: arguments),
+ toY: requireDouble("to_y", in: arguments)
+ )
+ case "type_text":
+ return try service.typeText(
+ app: requireString("app", in: arguments),
+ text: requireString("text", in: arguments)
+ )
+ case "press_key":
+ return try service.pressKey(
+ app: requireString("app", in: arguments),
+ key: requireString("key", in: arguments)
+ )
+ case "set_value":
+ return try service.setValue(
+ app: requireString("app", in: arguments),
+ elementIndex: requireString("element_index", in: arguments),
+ value: requireString("value", in: arguments)
+ )
+ default:
+ throw ComputerUseError.unsupportedTool(name)
+ }
+ }
+
+ public func callToolAsResult(name: String, arguments: [String: Any]) -> ToolCallResult {
+ do {
+ return try callTool(name: name, arguments: arguments)
+ } catch let error as ComputerUseError {
+ return ToolCallResult.text(
+ error.errorDescription ?? String(describing: error),
+ isError: error.toolResultIsError
+ )
+ } catch {
+ let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)
+ return ToolCallResult.text(message, isError: true)
+ }
+ }
+
+ private func requireString(_ key: String, in arguments: [String: Any]) throws -> String {
+ guard let value = arguments[key] as? String else {
+ throw ComputerUseError.missingArgument(key)
+ }
+
+ return value
+ }
+
+ private func optionalString(_ key: String, in arguments: [String: Any]) -> String? {
+ arguments[key] as? String
+ }
+
+ private func requireDouble(_ key: String, in arguments: [String: Any]) throws -> Double {
+ guard let value = optionalDouble(key, in: arguments) else {
+ throw ComputerUseError.missingArgument(key)
+ }
+
+ return value
+ }
+
+ private func optionalDouble(_ key: String, in arguments: [String: Any]) -> Double? {
+ if let double = arguments[key] as? Double {
+ return double
+ }
+
+ if let integer = arguments[key] as? Int {
+ return Double(integer)
+ }
+
+ if let number = arguments[key] as? NSNumber {
+ return number.doubleValue
+ }
+
+ return nil
+ }
+}
+
+public struct OpenComputerUseCallSpec {
+ public let tool: String
+ public let arguments: [String: Any]
+
+ public init(tool: String, arguments: [String: Any]) {
+ self.tool = tool
+ self.arguments = arguments
+ }
+}
+
+public struct OpenComputerUseCallOutput {
+ public let jsonObject: Any
+ public let hasToolError: Bool
+
+ public init(jsonObject: Any, hasToolError: Bool) {
+ self.jsonObject = jsonObject
+ self.hasToolError = hasToolError
+ }
+
+ public func jsonText() throws -> String {
+ let data = try JSONSerialization.data(
+ withJSONObject: jsonObject,
+ options: [.prettyPrinted, .withoutEscapingSlashes]
+ )
+ guard let text = String(data: data, encoding: .utf8) else {
+ throw ComputerUseError.message("Failed to encode call output as JSON.")
+ }
+ return text
+ }
+}
+
+public func runOpenComputerUseCall(
+ _ invocation: OpenComputerUseCallInvocation,
+ service: ComputerUseService = ComputerUseService()
+) throws -> OpenComputerUseCallOutput {
+ let dispatcher = ComputerUseToolDispatcher(service: service)
+
+ switch invocation {
+ case let .single(toolName, argumentsJSON, argumentsFile):
+ let arguments = try readOpenComputerUseToolArguments(
+ json: argumentsJSON,
+ file: argumentsFile
+ )
+ let result = dispatcher.callToolAsResult(name: toolName, arguments: arguments)
+ return OpenComputerUseCallOutput(
+ jsonObject: result.asDictionary,
+ hasToolError: result.isError
+ )
+
+ case let .sequence(callsJSON, callsFile):
+ let calls = try readOpenComputerUseCallSequence(json: callsJSON, file: callsFile)
+ var outputs: [[String: Any]] = []
+ var hasToolError = false
+
+ for call in calls {
+ let result = dispatcher.callToolAsResult(name: call.tool, arguments: call.arguments)
+ outputs.append([
+ "tool": call.tool,
+ "result": result.asDictionary,
+ ])
+
+ if result.isError {
+ hasToolError = true
+ break
+ }
+ }
+
+ return OpenComputerUseCallOutput(jsonObject: outputs, hasToolError: hasToolError)
+ }
+}
+
+public func readOpenComputerUseToolArguments(
+ json: String?,
+ file: String?
+) throws -> [String: Any] {
+ guard let source = try readOpenComputerUseJSONSource(json: json, file: file) else {
+ return [:]
+ }
+
+ let object = try decodeOpenComputerUseJSONObject(source)
+ guard let arguments = object as? [String: Any] else {
+ throw OpenComputerUseCLIError(message: "--args must be a JSON object", helpCommand: "call")
+ }
+
+ return arguments
+}
+
+public func readOpenComputerUseCallSequence(
+ json: String?,
+ file: String?
+) throws -> [OpenComputerUseCallSpec] {
+ guard let source = try readOpenComputerUseJSONSource(json: json, file: file) else {
+ throw OpenComputerUseCLIError(message: "call sequence requires --calls or --calls-file", helpCommand: "call")
+ }
+
+ let object = try decodeOpenComputerUseJSONObject(source)
+ guard let array = object as? [Any] else {
+ throw OpenComputerUseCLIError(message: "--calls must be a JSON array", helpCommand: "call")
+ }
+
+ return try array.enumerated().map { index, item in
+ guard let dictionary = item as? [String: Any] else {
+ throw OpenComputerUseCLIError(
+ message: "call sequence item #\(index + 1) must be a JSON object",
+ helpCommand: "call"
+ )
+ }
+
+ guard let tool = (dictionary["tool"] ?? dictionary["name"]) as? String, !tool.isEmpty else {
+ throw OpenComputerUseCLIError(
+ message: "call sequence item #\(index + 1) requires a non-empty tool",
+ helpCommand: "call"
+ )
+ }
+
+ let rawArguments = dictionary["args"] ?? dictionary["arguments"] ?? [:]
+ guard let arguments = rawArguments as? [String: Any] else {
+ throw OpenComputerUseCLIError(
+ message: "call sequence item #\(index + 1) args must be a JSON object",
+ helpCommand: "call"
+ )
+ }
+
+ return OpenComputerUseCallSpec(tool: tool, arguments: arguments)
+ }
+}
+
+private func readOpenComputerUseJSONSource(json: String?, file: String?) throws -> String? {
+ if json != nil, file != nil {
+ throw OpenComputerUseCLIError(message: "Use either inline JSON or a JSON file, not both", helpCommand: "call")
+ }
+
+ if let json {
+ return json
+ }
+
+ guard let file else {
+ return nil
+ }
+
+ do {
+ return try String(contentsOfFile: file, encoding: .utf8)
+ } catch {
+ throw OpenComputerUseCLIError(
+ message: "Unable to read JSON file \(file): \(error.localizedDescription)",
+ helpCommand: "call"
+ )
+ }
+}
+
+private func decodeOpenComputerUseJSONObject(_ source: String) throws -> Any {
+ guard let data = source.data(using: .utf8) else {
+ throw OpenComputerUseCLIError(message: "JSON input must be UTF-8 text", helpCommand: "call")
+ }
+
+ do {
+ return try JSONSerialization.jsonObject(with: data)
+ } catch {
+ throw OpenComputerUseCLIError(
+ message: "Invalid JSON input: \(error.localizedDescription)",
+ helpCommand: "call"
+ )
+ }
+}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift
index e68651b..ccec168 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift
@@ -18,10 +18,10 @@ Ask the user before taking destructive or externally visible actions such as sen
"""
public final class StdioMCPServer {
- private let service: ComputerUseService
+ private let dispatcher: ComputerUseToolDispatcher
public init(service: ComputerUseService = ComputerUseService()) {
- self.service = service
+ self.dispatcher = ComputerUseToolDispatcher(service: service)
}
public func run() throws {
@@ -78,7 +78,7 @@ public final class StdioMCPServer {
case "tools/call":
let name = params["name"] as? String ?? ""
let arguments = params["arguments"] as? [String: Any] ?? [:]
- let result = try callTool(name: name, arguments: arguments)
+ let result = try dispatcher.callTool(name: name, arguments: arguments)
return try encodeJSONRPCResult(
id: id,
result: result.asDictionary
@@ -114,99 +114,6 @@ public final class StdioMCPServer {
}
}
- private func callTool(name: String, arguments: [String: Any]) throws -> ToolCallResult {
- switch name {
- case "list_apps":
- return service.listApps()
- case "get_app_state":
- return try service.getAppState(app: requireString("app", in: arguments))
- case "click":
- return try service.click(
- app: requireString("app", in: arguments),
- elementIndex: optionalString("element_index", in: arguments),
- x: optionalDouble("x", in: arguments),
- y: optionalDouble("y", in: arguments),
- clickCount: Int(optionalDouble("click_count", in: arguments) ?? 1),
- mouseButton: optionalString("mouse_button", in: arguments) ?? "left"
- )
- case "perform_secondary_action":
- return try service.performSecondaryAction(
- app: requireString("app", in: arguments),
- elementIndex: requireString("element_index", in: arguments),
- action: requireString("action", in: arguments)
- )
- case "scroll":
- return try service.scroll(
- app: requireString("app", in: arguments),
- direction: requireString("direction", in: arguments),
- elementIndex: requireString("element_index", in: arguments),
- pages: Int(optionalDouble("pages", in: arguments) ?? 1)
- )
- case "drag":
- return try service.drag(
- app: requireString("app", in: arguments),
- fromX: requireDouble("from_x", in: arguments),
- fromY: requireDouble("from_y", in: arguments),
- toX: requireDouble("to_x", in: arguments),
- toY: requireDouble("to_y", in: arguments)
- )
- case "type_text":
- return try service.typeText(
- app: requireString("app", in: arguments),
- text: requireString("text", in: arguments)
- )
- case "press_key":
- return try service.pressKey(
- app: requireString("app", in: arguments),
- key: requireString("key", in: arguments)
- )
- case "set_value":
- return try service.setValue(
- app: requireString("app", in: arguments),
- elementIndex: requireString("element_index", in: arguments),
- value: requireString("value", in: arguments)
- )
- default:
- throw ComputerUseError.unsupportedTool(name)
- }
- }
-
- private func requireString(_ key: String, in arguments: [String: Any]) throws -> String {
- guard let value = arguments[key] as? String else {
- throw ComputerUseError.missingArgument(key)
- }
-
- return value
- }
-
- private func optionalString(_ key: String, in arguments: [String: Any]) -> String? {
- arguments[key] as? String
- }
-
- private func requireDouble(_ key: String, in arguments: [String: Any]) throws -> Double {
- guard let value = optionalDouble(key, in: arguments) else {
- throw ComputerUseError.missingArgument(key)
- }
-
- return value
- }
-
- private func optionalDouble(_ key: String, in arguments: [String: Any]) -> Double? {
- if let double = arguments[key] as? Double {
- return double
- }
-
- if let integer = arguments[key] as? Int {
- return Double(integer)
- }
-
- if let number = arguments[key] as? NSNumber {
- return number.doubleValue
- }
-
- return nil
- }
-
private func encodeJSONRPCResult(id: Any?, result: [String: Any]) throws -> String {
try encode([
"jsonrpc": "2.0",
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
index 25de181..428e30d 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift
@@ -6,11 +6,17 @@ public enum OpenComputerUseCLICommand: Equatable {
case doctor
case listApps
case snapshot(app: String)
+ case call(OpenComputerUseCallInvocation)
case turnEnded
case help(command: String?)
case version
}
+public enum OpenComputerUseCallInvocation: Equatable {
+ case single(toolName: String, argumentsJSON: String?, argumentsFile: String?)
+ case sequence(callsJSON: String?, callsFile: String?)
+}
+
public struct OpenComputerUseCLIError: LocalizedError, Equatable {
public let message: String
public let helpCommand: String?
@@ -52,6 +58,8 @@ public func parseOpenComputerUseCLI(arguments: [String]) throws -> OpenComputerU
return try parseSimpleCommand(name: "doctor", arguments: Array(arguments.dropFirst()), result: .doctor)
case "list-apps":
return try parseSimpleCommand(name: "list-apps", arguments: Array(arguments.dropFirst()), result: .listApps)
+ case "call":
+ return try parseCall(arguments: Array(arguments.dropFirst()))
case "turn-ended":
return try parseSimpleCommand(name: "turn-ended", arguments: Array(arguments.dropFirst()), result: .turnEnded)
case "snapshot":
@@ -80,6 +88,7 @@ public func openComputerUseHelpText(command: String? = nil) -> String {
doctor Print permission status and launch onboarding if needed.
list-apps Print running or recently used apps.
snapshot <app> Print the current accessibility snapshot for an app.
+ call <tool> Call one tool, or run a JSON array of tool calls.
turn-ended Acknowledge the host turn boundary.
help [command] Show general or command-specific help.
version Print the CLI version.
@@ -124,6 +133,23 @@ public func openComputerUseHelpText(command: String? = nil) -> String {
Print the current accessibility snapshot for the target app.
"""
+ case "call":
+ return """
+ Usage:
+ open-computer-use call <tool> [--args '<json-object>']
+ open-computer-use call <tool> [--args-file <path>]
+ open-computer-use call --calls '<json-array>'
+ open-computer-use call --calls-file <path>
+
+ Examples:
+ open-computer-use call list_apps
+ open-computer-use call get_app_state --args '{"app":"TextEdit"}'
+ open-computer-use call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}},{"tool":"press_key","args":{"app":"TextEdit","key":"Return"}}]'
+
+ The JSON array form keeps all calls in one process so follow-up actions
+ can reuse the app state and element indices captured by get_app_state.
+ Sequence execution stops after the first tool result with isError=true.
+ """
case "turn-ended":
return """
Usage:
@@ -188,3 +214,85 @@ private func parseSnapshot(arguments: [String]) throws -> OpenComputerUseCLIComm
throw OpenComputerUseCLIError(message: "snapshot accepts exactly one <app> argument", helpCommand: "snapshot")
}
+
+private func parseCall(arguments: [String]) throws -> OpenComputerUseCLICommand {
+ if arguments.count == 1, let option = arguments.first, option == "-h" || option == "--help" {
+ return .help(command: "call")
+ }
+
+ var toolName: String?
+ var argumentsJSON: String?
+ var argumentsFile: String?
+ var callsJSON: String?
+ var callsFile: String?
+
+ var index = 0
+ while index < arguments.count {
+ let argument = arguments[index]
+
+ switch argument {
+ case "--args":
+ argumentsJSON = try parseOptionValue("--args", arguments: arguments, index: &index)
+ case "--args-file":
+ argumentsFile = try parseOptionValue("--args-file", arguments: arguments, index: &index)
+ case "--calls":
+ callsJSON = try parseOptionValue("--calls", arguments: arguments, index: &index)
+ case "--calls-file":
+ callsFile = try parseOptionValue("--calls-file", arguments: arguments, index: &index)
+ case "-h", "--help":
+ throw OpenComputerUseCLIError(message: "call help must be requested as `open-computer-use call --help`", helpCommand: "call")
+ default:
+ if argument.hasPrefix("-") {
+ throw OpenComputerUseCLIError(message: "Unknown call option: \(argument)", helpCommand: "call")
+ }
+
+ guard toolName == nil else {
+ throw OpenComputerUseCLIError(message: "call accepts at most one tool name", helpCommand: "call")
+ }
+
+ toolName = argument
+ }
+
+ index += 1
+ }
+
+ let hasSequenceInput = callsJSON != nil || callsFile != nil
+ if hasSequenceInput {
+ if callsJSON != nil, callsFile != nil {
+ throw OpenComputerUseCLIError(message: "Use either --calls or --calls-file, not both", helpCommand: "call")
+ }
+
+ if toolName != nil || argumentsJSON != nil || argumentsFile != nil {
+ throw OpenComputerUseCLIError(
+ message: "call sequence does not accept a tool name, --args, or --args-file",
+ helpCommand: "call"
+ )
+ }
+
+ return .call(.sequence(callsJSON: callsJSON, callsFile: callsFile))
+ }
+
+ if argumentsJSON != nil, argumentsFile != nil {
+ throw OpenComputerUseCLIError(message: "Use either --args or --args-file, not both", helpCommand: "call")
+ }
+
+ guard let toolName else {
+ throw OpenComputerUseCLIError(message: "call requires a tool name or --calls/--calls-file", helpCommand: "call")
+ }
+
+ return .call(.single(toolName: toolName, argumentsJSON: argumentsJSON, argumentsFile: argumentsFile))
+}
+
+private func parseOptionValue(
+ _ option: String,
+ arguments: [String],
+ index: inout Int
+) throws -> String {
+ let valueIndex = index + 1
+ guard valueIndex < arguments.count else {
+ throw OpenComputerUseCLIError(message: "\(option) requires a value", helpCommand: "call")
+ }
+
+ index = valueIndex
+ return arguments[valueIndex]
+}
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index 0555468..09e143e 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -14,6 +14,28 @@ final class OpenComputerUseKitTests: XCTestCase {
XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["help", "snapshot"]), .help(command: "snapshot"))
XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "--help"]), .help(command: "snapshot"))
XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["doctor", "-h"]), .help(command: "doctor"))
+ XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["call", "--help"]), .help(command: "call"))
+ }
+
+ func testCLIRecognizesSingleToolCallCommand() throws {
+ XCTAssertEqual(
+ try parseOpenComputerUseCLI(arguments: ["call", "list_apps"]),
+ .call(.single(toolName: "list_apps", argumentsJSON: nil, argumentsFile: nil))
+ )
+
+ XCTAssertEqual(
+ try parseOpenComputerUseCLI(arguments: ["call", "get_app_state", "--args", #"{"app":"TextEdit"}"#]),
+ .call(.single(toolName: "get_app_state", argumentsJSON: #"{"app":"TextEdit"}"#, argumentsFile: nil))
+ )
+ }
+
+ func testCLIRecognizesJSONSequenceCallCommand() throws {
+ let calls = #"[{"tool":"get_app_state","args":{"app":"TextEdit"}},{"tool":"press_key","args":{"app":"TextEdit","key":"Return"}}]"#
+
+ XCTAssertEqual(
+ try parseOpenComputerUseCLI(arguments: ["call", "--calls", calls]),
+ .call(.sequence(callsJSON: calls, callsFile: nil))
+ )
}
func testCLIRequiresSnapshotArgument() {
@@ -28,6 +50,18 @@ final class OpenComputerUseKitTests: XCTestCase {
}
}
+ func testCLIRejectsMixedCallSequenceInputs() {
+ XCTAssertThrowsError(try parseOpenComputerUseCLI(arguments: ["call", "list_apps", "--calls", "[]"])) { error in
+ XCTAssertEqual(
+ error as? OpenComputerUseCLIError,
+ OpenComputerUseCLIError(
+ message: "call sequence does not accept a tool name, --args, or --args-file",
+ helpCommand: "call"
+ )
+ )
+ }
+ }
+
func testCLIRejectsUnknownOption() {
XCTAssertThrowsError(try parseOpenComputerUseCLI(arguments: ["--verbose"])) { error in
XCTAssertEqual(
@@ -45,6 +79,7 @@ final class OpenComputerUseKitTests: XCTestCase {
XCTAssertTrue(help.contains("open-computer-use [command] [options]"))
XCTAssertTrue(help.contains("snapshot <app>"))
+ XCTAssertTrue(help.contains("call <tool>"))
XCTAssertTrue(help.contains("-h, --help"))
XCTAssertTrue(help.contains("-v, --version"))
}
@@ -57,6 +92,51 @@ final class OpenComputerUseKitTests: XCTestCase {
XCTAssertEqual(ToolDefinitions.all.count, 9)
}
+ func testReadToolArgumentsAcceptsJSONObject() throws {
+ let arguments = try readOpenComputerUseToolArguments(
+ json: #"{"app":"TextEdit","pages":2}"#,
+ file: nil
+ )
+
+ XCTAssertEqual(arguments["app"] as? String, "TextEdit")
+ XCTAssertEqual((arguments["pages"] as? NSNumber)?.intValue, 2)
+ }
+
+ func testReadToolArgumentsRejectsNonObject() {
+ XCTAssertThrowsError(try readOpenComputerUseToolArguments(json: #"["TextEdit"]"#, file: nil)) { error in
+ XCTAssertEqual(
+ error as? OpenComputerUseCLIError,
+ OpenComputerUseCLIError(message: "--args must be a JSON object", helpCommand: "call")
+ )
+ }
+ }
+
+ func testReadCallSequenceAcceptsJSONArrays() throws {
+ let calls = try readOpenComputerUseCallSequence(
+ json: #"[{"tool":"get_app_state","args":{"app":"TextEdit"}},{"name":"press_key","arguments":{"app":"TextEdit","key":"Return"}}]"#,
+ file: nil
+ )
+
+ XCTAssertEqual(calls.count, 2)
+ XCTAssertEqual(calls[0].tool, "get_app_state")
+ XCTAssertEqual(calls[0].arguments["app"] as? String, "TextEdit")
+ XCTAssertEqual(calls[1].tool, "press_key")
+ XCTAssertEqual(calls[1].arguments["key"] as? String, "Return")
+ }
+
+ func testRunCallSequenceStopsAfterFirstToolError() throws {
+ let output = try runOpenComputerUseCall(
+ .sequence(
+ callsJSON: #"[{"tool":"not_a_tool"},{"tool":"list_apps"}]"#,
+ callsFile: nil
+ )
+ )
+
+ let outputs = try XCTUnwrap(output.jsonObject as? [[String: Any]])
+ XCTAssertEqual(outputs.count, 1)
+ XCTAssertTrue(output.hasToolError)
+ }
+
func testPermissionDiagnosticsListsMissingPermissionsInCanonicalOrder() {
let diagnostics = PermissionDiagnostics(
accessibilityTrusted: false,
diff --git a/scripts/npm/build-packages.mjs b/scripts/npm/build-packages.mjs
index 3edbd18..2ba4faf 100644
--- a/scripts/npm/build-packages.mjs
+++ b/scripts/npm/build-packages.mjs
@@ -154,6 +154,7 @@ Commands:
doctor Print permission status and launch onboarding if needed.
list-apps Print running or recently used apps.
snapshot <app> Print the current accessibility snapshot for an app.
+ call <tool> Call one tool, or run a JSON array of tool calls.
turn-ended Acknowledge the host turn boundary.
install-claude-mcp Install the MCP server into ~/.claude.json for this project.
install-codex-mcp Install the MCP server into ~/.codex/config.toml.
@@ -340,6 +341,11 @@ open-computer-use doctor
# Start the stdio MCP server for Claude Desktop, Cursor, Cline, or another MCP client
open-computer-use mcp
+# Call tools directly; the JSON-array form keeps state in one process for follow-up actions
+open-computer-use call list_apps
+open-computer-use call get_app_state --args '{"app":"TextEdit"}'
+open-computer-use call --calls '[{"tool":"get_app_state","args":{"app":"TextEdit"}},{"tool":"press_key","args":{"app":"TextEdit","key":"Return"}}]'
+
# Install this package into the local Codex plugin marketplace/cache
open-computer-use install-codex-plugin
\`\`\`