Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

Align computer-use tool results with official MCP surface

Leof3f0d04de938
diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
index fceb38d..069ce0f 100644
--- a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
+++ b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift
@@ -15,12 +15,12 @@ enum OpenComputerUseMain {
let permissions = PermissionDiagnostics.current()
print(permissions.summary)
case "list-apps":
- print(service.listApps())
+ print(service.listApps().primaryText ?? "")
case "snapshot":
guard arguments.count >= 2 else {
throw ComputerUseError.invalidArguments("snapshot requires an app name or bundle identifier")
}
- print(try service.getAppState(app: arguments[1]))
+ print(try service.getAppState(app: arguments[1]).primaryText ?? "")
case "turn-ended":
print("turn-ended acknowledged")
default:
diff --git a/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift b/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
index 4ecb6ee..5c6f05d 100644
--- a/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
+++ b/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
@@ -1,5 +1,7 @@
+import AppKit
import CoreGraphics
import Foundation
+import OpenComputerUseKit
struct MCPResponse {
let id: Int?
@@ -157,6 +159,10 @@ enum OpenComputerUseSmokeSuite {
let productsDirectory = try locateProductsDirectory()
let fixtureURL = productsDirectory.appendingPathComponent("OpenComputerUseFixture")
let serverURL = productsDirectory.appendingPathComponent("OpenComputerUse")
+ let appName = "OpenComputerUseFixture"
+
+ terminateExistingFixtures(named: appName)
+ try? FileManager.default.removeItem(at: FixtureBridge.stateFileURL)
let fixture = Process()
fixture.executableURL = fixtureURL
@@ -182,8 +188,6 @@ enum OpenComputerUseSmokeSuite {
throw SmokeError.message("Expected 9 tools, got \(tools.count)")
}
- let appName = "OpenComputerUseFixture"
-
print("1. list_apps")
let apps = try client.callTool("list_apps", arguments: [:])
try expect(apps.contains(appName), "Fixture app should appear in list_apps output.")
@@ -194,13 +198,14 @@ enum OpenComputerUseSmokeSuite {
])
var index = parseElementIndex(state)
try expect(index.keys.contains("fixture-increment"), "fixture button should be indexed")
+ let initialCounter = parseCounterValue(state)
print("3. click element_index")
state = try client.callTool("click", arguments: [
"app": appName,
"element_index": index["fixture-increment"]!.index,
])
- try expect(state.contains("Counter: 1"), "click should increment the counter")
+ try expect(parseCounterValue(state) == initialCounter + 1, "click should increment the counter")
print("4. click coordinate")
index = parseElementIndex(state)
@@ -210,7 +215,7 @@ enum OpenComputerUseSmokeSuite {
"x": buttonFrame.midX,
"y": buttonFrame.midY,
])
- try expect(state.contains("Counter: 2"), "coordinate click should increment the counter again")
+ try expect(parseCounterValue(state) == initialCounter + 2, "coordinate click should increment the counter again")
print("5. perform_secondary_action")
let windowIndex = index["fixture-window"]?.index ?? "0"
@@ -293,30 +298,53 @@ enum OpenComputerUseSmokeSuite {
}
}
- private static func parseElementIndex(_ state: String) -> [String: (index: String, frame: CGRect)] {
- guard let start = state.range(of: "<element_index>"), let end = state.range(of: "</element_index>") else {
- return [:]
+ private static func terminateExistingFixtures(named name: String) {
+ let candidates = NSWorkspace.shared.runningApplications.filter { app in
+ if app.localizedName == name {
+ return true
+ }
+
+ return app.executableURL?.deletingPathExtension().lastPathComponent == name
}
- let block = state[start.upperBound..<end.lowerBound]
+ for app in candidates {
+ _ = app.terminate()
+ }
+
+ for app in candidates where !app.isTerminated {
+ _ = app.forceTerminate()
+ }
+
+ if !candidates.isEmpty {
+ Thread.sleep(forTimeInterval: 0.5)
+ }
+ }
+
+ private static func parseElementIndex(_ state: String) -> [String: (index: String, frame: CGRect)] {
var result: [String: (String, CGRect)] = [:]
- for rawLine in block.split(separator: "\n") {
+ for rawLine in state.split(separator: "\n") {
let line = String(rawLine)
- let parts = line.components(separatedBy: " -> ")
- guard parts.count == 2 else {
+ guard
+ let identifierRange = line.range(of: " ID: "),
+ let frameRange = line.range(of: " Frame: ")
+ else {
continue
}
- let identifier = parts[0]
- let remainder = parts[1]
- let chunks = remainder.components(separatedBy: " @ ")
- guard chunks.count == 2 else {
+ let prefix = line[..<identifierRange.lowerBound].trimmingCharacters(in: .whitespaces)
+ guard let index = prefix.split(separator: " ").first.map(String.init) else {
continue
}
- let index = chunks[0]
- let frame = parseFrame(chunks[1])
+ let identifierSlice = line[identifierRange.upperBound..<frameRange.lowerBound]
+ let identifier = identifierSlice
+ .components(separatedBy: " Value:")
+ .first?
+ .components(separatedBy: " Secondary Actions:")
+ .first?
+ .trimmingCharacters(in: .whitespaces) ?? ""
+ let frame = parseFrame(String(line[frameRange.upperBound...]))
result[identifier] = (index, frame)
}
@@ -336,4 +364,16 @@ enum OpenComputerUseSmokeSuite {
return CGRect(x: values[0], y: values[1], width: values[2], height: values[3])
}
+
+ private static func parseCounterValue(_ state: String) -> Int {
+ guard
+ let range = state.range(of: "Counter: "),
+ let value = state[range.upperBound...].split(whereSeparator: { !$0.isNumber }).first,
+ let counter = Int(value)
+ else {
+ return -1
+ }
+
+ return counter
+ }
}
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index fbd2554..f9595a8 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -48,6 +48,7 @@
- `ComputerUseService` 负责把 MCP tool 请求映射到本地能力。
- `list_apps` 通过 `NSWorkspace` 枚举运行中的 app。
- `get_app_state` 优先走真实 AX / 窗口截图,但不再为了读状态而显式 `activate` 目标 app;当目标是仓库内 fixture app 时,回退到 fixture 导出的合成状态。
+- MCP `tools/list` 的 description / input schema 当前按官方 `computer-use` 的 9 个 tools 文案和参数面收敛,尽量减少 host 侧提示词和 tool surface 偏差。
- 普通 app 的 element frame 当前按“窗口左上角为原点”的 window-relative 坐标输出,便于后续把 `element_index` 和截图坐标统一到同一套参考系。
- 动作型 tools 对普通 app 采用“非侵入优先,HID 兜底”策略:
- `AXUIElementPerformAction`
@@ -68,7 +69,7 @@
- 开源版当前不复刻官方闭源实现里的 caller signing、私有 IPC、overlay UI 和 plugin 自安装逻辑。
- 因为官方 `SkyComputerUseClient` 带有宿主侧 launch constraints,普通 stdio MCP client 在本机上可能被系统直接杀掉;如果要探测官方 bundled `computer-use`,默认应通过 `scripts/computer-use-cli` 的 app-server 模式走已签名的 Codex 宿主。
- 当前权限引导已经具备可运行 app、深链和拖拽辅助,但还没有完全复刻官方那套嵌入式 choreography / overlay 体验。
-- screenshot 当前使用系统窗口截图 API,结果写入临时目录,不做长期持久化。
+- screenshot 当前使用系统窗口截图 API,但默认直接以 MCP `image` content block 的 base64 PNG 返回,不再把普通 app 截图落盘到仓库或临时目录。
- 会话状态现在是进程内内存态,保存每个 app 最近一次 snapshot 和 element index 映射。
## 主要验证路径
diff --git a/docs/SECURITY.md b/docs/SECURITY.md
index e0aa360..2530aaa 100644
--- a/docs/SECURITY.md
+++ b/docs/SECURITY.md
@@ -8,8 +8,8 @@
## 数据处理
-- screenshot 只写到临时目录 `TMPDIR/open-computer-use/`,默认不长期持久化。
-- fixture app 的合成状态只写到本地临时 JSON 文件,目的是支撑 deterministic smoke test。
+- 普通 app 的 screenshot 默认只在内存中编码成 PNG,并通过 MCP `image` content block 直接回传;默认不长期持久化。
+- fixture app 的合成状态只写到本地临时 JSON 文件,目的是支撑 deterministic smoke test;当前写入走原子替换,减少测试期间的读写竞争。
- 当前仓库不引入第三方服务,也不上传截图、AX tree 或输入内容。
## 授权与最小权限
diff --git a/docs/histories/2026-04/20260417-1816-align-computer-use-tool-results.md b/docs/histories/2026-04/20260417-1816-align-computer-use-tool-results.md
new file mode 100644
index 0000000..07bcc24
--- /dev/null
+++ b/docs/histories/2026-04/20260417-1816-align-computer-use-tool-results.md
@@ -0,0 +1,39 @@
+## [2026-04-17 18:16] | Task: 对齐 computer-use 的 tool schema 与结果载荷
+
+### 🤖 Execution Context
+* **Agent ID**: `primary`
+* **Base Model**: `gpt-5`
+* **Runtime**: `Codex CLI + SwiftPM`
+
+### 📥 User Query
+> 基于前面抓到的官方 `computer-use` / `open-computer-use` dump,对照两边的 MCP 调用参数和返回做优化。明确要求包括:
+> 1. action 后的截图不要写磁盘,而是直接通过 BASE64 返回。
+> 2. tools 的描述和参数要和官方严格对齐。
+> 其他差异也根据实际 dump 继续优化。
+
+### 🛠 Changes Overview
+**Scope:** `packages/OpenComputerUseKit`, `apps/OpenComputerUse`, `apps/OpenComputerUseSmokeSuite`, `docs`
+
+**Key Actions:**
+- **[Schema 对齐]**: 把 9 个 tools 的 description 和 input schema 文案收敛到当前官方 `computer-use` 暴露给模型的 surface。
+- **[结果载荷对齐]**: 新增 MCP tool result content 封装,让 `get_app_state` 和动作类 tools 返回 `text + image/png(base64)`,不再把普通 app 截图路径塞进文本里。
+- **[状态文本收口]**: 去掉开源实现自己的 `Screenshot:` 路径和 `<element_index>` 附加块,让 state 文本更接近官方 `computer-use` 的 `App=/Window=/tree` 结构。
+- **[错误语义调整]**: 将 `appNotFound("...")` 这类官方常见恢复型结果按普通 tool text 返回,而不是一律作为 MCP error。
+- **[Smoke 稳定性]**: 修复 fixture state 文件的非原子写入竞争,并让 smoke suite 启动前主动清理旧 fixture 进程与旧状态文件,避免残留进程互相污染。
+
+### 🧠 Design Intent (Why)
+这次优化的目标不是“做一个功能相近的开源版”,而是尽量把 host 实际看到的 tool shape、tool result shape 和恢复语义都收敛到官方 `computer-use` 当前的调用习惯上。这样后续无论是抓包对比、做 eval,还是继续优化焦点策略,都能建立在更接近真实 host 行为的兼容面上,而不是被自定义 schema、磁盘截图路径或测试夹具竞态这些非本质差异干扰。
+
+### 📁 Files Modified
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/Errors.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/FixtureBridge.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
+- `apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift`
+- `apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift`
+- `docs/ARCHITECTURE.md`
+- `docs/SECURITY.md`
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
index 9543f27..b92e8a0 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift
@@ -30,11 +30,44 @@ public struct AppSnapshot {
public let app: RunningAppDescriptor
public let windowTitle: String?
public let windowBounds: CGRect?
- public let screenshotPath: String?
- public let renderedText: String
+ public let screenshotPNGData: Data?
let mode: SnapshotMode
+ let treeLines: [String]
+ let focusedSummary: String?
let elements: [Int: ElementRecord]
+
+ public var renderedText: String {
+ renderedText(style: .fullState)
+ }
+
+ public func renderedText(style: SnapshotTextStyle) -> String {
+ var lines: [String] = []
+
+ if style == .fullState {
+ lines.append("Computer Use state (Open Computer Use 0.1.0)")
+ lines.append("<app_state>")
+ }
+
+ lines.append("App=\(app.bundleIdentifier ?? app.name) (pid \(app.pid))")
+ lines.append("Window: \(quoted(windowTitle ?? "")), App: \(app.name).")
+ lines.append(contentsOf: treeLines)
+
+ if let focusedSummary {
+ lines.append("The focused UI element is \(focusedSummary).")
+ }
+
+ if style == .fullState {
+ lines.append("</app_state>")
+ }
+
+ return lines.joined(separator: "\n")
+ }
+}
+
+public enum SnapshotTextStyle {
+ case fullState
+ case actionResult
}
enum SnapshotBuilder {
@@ -57,52 +90,21 @@ enum SnapshotBuilder {
let windowCapture = WindowCapture.resolve(for: app.pid, titleHint: windowTitle)
let windowBounds = windowCapture?.bounds
- let screenshotPath = try windowCapture?.writePNGIfAvailable()
+ let screenshotPNGData = windowCapture?.pngDataIfAvailable()
let focusedElement = preferredFocusedElement(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide)
let context = RenderContext(windowBounds: windowBounds, focusedElement: focusedElement)
var renderer = TreeRenderer(context: context)
renderer.render(rootElement)
- var lines: [String] = []
- lines.append("Computer Use state (OpenComputerUse 0.1.0)")
- lines.append(permissions.summary)
-
- if let screenshotPath {
- lines.append("Screenshot: \(screenshotPath)")
- } else {
- lines.append("Screenshot: unavailable")
- }
-
- lines.append("<app_state>")
- lines.append("App=\(app.bundleIdentifier ?? app.name) (pid \(app.pid))")
-
- if let windowBounds {
- lines.append("Window: \(quoted(windowTitle ?? app.name)), Bounds: \(format(rect: windowBounds))")
- }
-
- lines.append(contentsOf: renderer.lines)
- lines.append("</app_state>")
-
- if !renderer.identifierIndex.isEmpty {
- lines.append("<element_index>")
- for entry in renderer.identifierIndex.sorted(by: { $0.key < $1.key }) {
- lines.append(entry.value)
- }
- lines.append("</element_index>")
- }
-
- if let focusedSummary = renderer.focusedSummary {
- lines.append("The focused UI element is \(focusedSummary).")
- }
-
return AppSnapshot(
app: app,
windowTitle: windowTitle,
windowBounds: windowBounds,
- screenshotPath: screenshotPath,
- renderedText: lines.joined(separator: "\n"),
+ screenshotPNGData: screenshotPNGData,
mode: .accessibility,
+ treeLines: renderer.lines,
+ focusedSummary: renderer.focusedSummary,
elements: renderer.records
)
}
@@ -139,15 +141,8 @@ enum SnapshotBuilder {
private static func buildFixtureSnapshot(app: RunningAppDescriptor, state: FixtureAppState) -> AppSnapshot {
var lines: [String] = []
- lines.append("Computer Use state (OpenComputerUse 0.1.0)")
- lines.append(PermissionDiagnostics.current().summary)
- lines.append("Screenshot: unavailable (fixture synthetic state)")
- lines.append("<app_state>")
- lines.append("App=\(app.name) (pid \(app.pid))")
- lines.append("Window: \"\(state.windowTitle)\", Bounds: \(format(rect: state.windowBounds.cgRect))")
var records: [Int: ElementRecord] = [:]
- var identifierLines: [String] = []
let focusedIdentifier = state.focusedIdentifier
var focusedSummary: String?
@@ -167,36 +162,26 @@ enum SnapshotBuilder {
prettyActions: element.actions
)
records[element.index] = record
- identifierLines.append("\(element.identifier) -> \(element.index) @ \(element.frame.cgRect.renderedLocalFrame)")
if focusedIdentifier == element.identifier {
focusedSummary = "\(element.index) \(element.role)"
}
}
- lines.append("</app_state>")
- lines.append("<element_index>")
- lines.append(contentsOf: identifierLines.sorted())
- lines.append("</element_index>")
-
- if let focusedSummary {
- lines.append("The focused UI element is \(focusedSummary).")
- }
-
return AppSnapshot(
app: app,
windowTitle: state.windowTitle,
windowBounds: state.windowBounds.cgRect,
- screenshotPath: nil,
- renderedText: lines.joined(separator: "\n"),
+ screenshotPNGData: nil,
mode: .fixture,
+ treeLines: lines,
+ focusedSummary: focusedSummary,
elements: records
)
}
}
private struct WindowCapture {
- let windowID: CGWindowID
let bounds: CGRect
let image: CGImage?
@@ -246,26 +231,16 @@ private struct WindowCapture {
[.bestResolution, .boundsIgnoreFraming]
)
- return WindowCapture(windowID: best.0, bounds: best.1, image: image)
+ return WindowCapture(bounds: best.1, image: image)
}
- func writePNGIfAvailable() throws -> String? {
+ func pngDataIfAvailable() -> Data? {
guard let image else {
return nil
}
- let directory = URL(fileURLWithPath: NSTemporaryDirectory())
- .appendingPathComponent("open-computer-use", isDirectory: true)
- try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
-
- let fileURL = directory.appendingPathComponent("window-\(windowID)-\(Int(Date().timeIntervalSince1970 * 1000)).png")
let bitmap = NSBitmapImageRep(cgImage: image)
- guard let data = bitmap.representation(using: .png, properties: [:]) else {
- return nil
- }
-
- try data.write(to: fileURL)
- return fileURL.path
+ return bitmap.representation(using: .png, properties: [:])
}
}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
index 54cc739..f5a387d 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
@@ -7,20 +7,22 @@ public final class ComputerUseService {
public init() {}
- public func listApps() -> String {
- AppDiscovery.listApps()
- .map { descriptor in
- let bundle = descriptor.bundleIdentifier ?? "no-bundle-id"
- return "\(descriptor.name) — \(bundle) [running, pid=\(descriptor.pid)]"
- }
- .joined(separator: "\n")
+ public func listApps() -> ToolCallResult {
+ ToolCallResult.text(
+ AppDiscovery.listApps()
+ .map { descriptor in
+ let bundle = descriptor.bundleIdentifier ?? "no-bundle-id"
+ return "\(descriptor.name) — \(bundle) [running, pid=\(descriptor.pid)]"
+ }
+ .joined(separator: "\n")
+ )
}
- public func getAppState(app query: String) throws -> String {
- try refreshSnapshot(for: query).renderedText
+ public func getAppState(app query: String) throws -> ToolCallResult {
+ snapshotResult(for: try refreshSnapshot(for: query), style: .fullState)
}
- public func click(app query: String, elementIndex: String?, x: Double?, y: Double?, clickCount: Int, mouseButton: String) throws -> String {
+ public func click(app query: String, elementIndex: String?, x: Double?, y: Double?, clickCount: Int, mouseButton: String) throws -> ToolCallResult {
let snapshot = try currentSnapshot(for: query)
let button = MouseButtonKind(rawValue: mouseButton.lowercased()) ?? .left
if snapshot.mode == .fixture {
@@ -38,7 +40,7 @@ public final class ComputerUseService {
}
Thread.sleep(forTimeInterval: 0.15)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
if let elementIndex {
@@ -66,10 +68,10 @@ public final class ComputerUseService {
throw ComputerUseError.invalidArguments("click requires either element_index or x/y")
}
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
- public func performSecondaryAction(app query: String, elementIndex: String, action: String) throws -> String {
+ public func performSecondaryAction(app query: String, elementIndex: String, action: String) throws -> ToolCallResult {
let snapshot = try currentSnapshot(for: query)
let record = try lookupElement(snapshot: snapshot, index: elementIndex)
@@ -79,7 +81,7 @@ public final class ComputerUseService {
}
InputSimulation.prepareAppForGlobalPointerInput(snapshot.app)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
guard let rawAction = matchingAction(requested: action, record: record) else {
@@ -96,10 +98,10 @@ public final class ComputerUseService {
}
Thread.sleep(forTimeInterval: 0.15)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
- public func scroll(app query: String, direction: String, elementIndex: String, pages: Int) throws -> String {
+ public func scroll(app query: String, direction: String, elementIndex: String, pages: Int) throws -> ToolCallResult {
let normalized = direction.lowercased()
guard ["up", "down", "left", "right"].contains(normalized) else {
throw ComputerUseError.invalidArguments("scroll direction must be one of up/down/left/right")
@@ -114,7 +116,7 @@ public final class ComputerUseService {
}
try FixtureBridge.post(FixtureCommand(kind: "scroll", identifier: identifier, direction: normalized, pages: pages))
Thread.sleep(forTimeInterval: 0.15)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
if let rawAction = record.rawActions.first(where: { $0.caseInsensitiveCompare("AXScroll\(normalized.capitalized)ByPage") == .orderedSame }), let element = record.element {
@@ -129,15 +131,15 @@ public final class ComputerUseService {
throw ComputerUseError.stateUnavailable("element \(elementIndex) has no scrollable frame")
}
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
- public func drag(app query: String, fromX: Double, fromY: Double, toX: Double, toY: Double) throws -> String {
+ public func drag(app query: String, fromX: Double, fromY: Double, toX: Double, toY: Double) throws -> ToolCallResult {
let snapshot = try currentSnapshot(for: query)
if snapshot.mode == .fixture {
try FixtureBridge.post(FixtureCommand(kind: "drag", identifier: "fixture-drag-pad", x: fromX, y: fromY, toX: toX, toY: toY))
Thread.sleep(forTimeInterval: 0.15)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
InputSimulation.prepareAppForGlobalPointerInput(snapshot.app)
@@ -145,34 +147,34 @@ public final class ComputerUseService {
from: try screenshotToGlobalPoint(snapshot: snapshot, x: fromX, y: fromY),
to: try screenshotToGlobalPoint(snapshot: snapshot, x: toX, y: toY)
)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
- public func typeText(app query: String, text: String) throws -> String {
+ public func typeText(app query: String, text: String) throws -> ToolCallResult {
let snapshot = try currentSnapshot(for: query)
if snapshot.mode == .fixture {
try FixtureBridge.post(FixtureCommand(kind: "type_text", identifier: "fixture-input", value: text))
Thread.sleep(forTimeInterval: 0.15)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
try InputSimulation.typeText(text, pid: snapshot.app.pid)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
- public func pressKey(app query: String, key: String) throws -> String {
+ public func pressKey(app query: String, key: String) throws -> ToolCallResult {
let snapshot = try currentSnapshot(for: query)
if snapshot.mode == .fixture {
try FixtureBridge.post(FixtureCommand(kind: "press_key", identifier: "fixture-key-capture", value: key))
Thread.sleep(forTimeInterval: 0.15)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
try InputSimulation.pressKey(key, pid: snapshot.app.pid)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
- public func setValue(app query: String, elementIndex: String, value: String) throws -> String {
+ public func setValue(app query: String, elementIndex: String, value: String) throws -> ToolCallResult {
let snapshot = try currentSnapshot(for: query)
let record = try lookupElement(snapshot: snapshot, index: elementIndex)
@@ -183,7 +185,7 @@ public final class ComputerUseService {
try FixtureBridge.post(FixtureCommand(kind: "set_value", identifier: identifier, value: value))
Thread.sleep(forTimeInterval: 0.15)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
guard let element = record.element else {
@@ -196,7 +198,7 @@ public final class ComputerUseService {
}
Thread.sleep(forTimeInterval: 0.1)
- return try refreshSnapshot(for: query).renderedText
+ return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
private func currentSnapshot(for query: String) throws -> AppSnapshot {
@@ -443,4 +445,12 @@ public final class ComputerUseService {
return identifier
}
+
+ private func snapshotResult(for snapshot: AppSnapshot, style: SnapshotTextStyle) -> ToolCallResult {
+ var content = [ToolResultContentItem.text(snapshot.renderedText(style: style))]
+ if let screenshotPNGData = snapshot.screenshotPNGData {
+ content.append(.pngImage(screenshotPNGData))
+ }
+ return ToolCallResult(content: content)
+ }
}
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/Errors.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/Errors.swift
index 27b0f2d..20bfb9f 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/Errors.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/Errors.swift
@@ -13,17 +13,26 @@ public enum ComputerUseError: Error, LocalizedError {
case .message(let value):
return value
case .unsupportedTool(let name):
- return "unsupportedTool(\(name))"
+ return "unsupportedTool(\"\(name)\")"
case .invalidArguments(let message):
- return "invalidArguments(\(message))"
+ return "invalidArguments(\"\(message)\")"
case .appNotFound(let app):
- return "appNotFound(\(app))"
+ return "appNotFound(\"\(app)\")"
case .permissionDenied(let message):
return message
case .stateUnavailable(let message):
return message
}
}
+
+ var toolResultIsError: Bool {
+ switch self {
+ case .unsupportedTool, .invalidArguments:
+ return true
+ case .message, .appNotFound, .permissionDenied, .stateUnavailable:
+ return false
+ }
+ }
}
extension ComputerUseError {
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/FixtureBridge.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/FixtureBridge.swift
index 94f6660..c99f549 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/FixtureBridge.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/FixtureBridge.swift
@@ -103,15 +103,28 @@ public enum FixtureBridge {
return nil
}
- let data = try Data(contentsOf: url)
- return try JSONDecoder().decode(FixtureAppState.self, from: data)
+ var lastError: Error?
+
+ for attempt in 0..<5 {
+ do {
+ let data = try Data(contentsOf: url)
+ return try JSONDecoder().decode(FixtureAppState.self, from: data)
+ } catch {
+ lastError = error
+ if attempt < 4 {
+ Thread.sleep(forTimeInterval: 0.05)
+ }
+ }
+ }
+
+ throw lastError ?? ComputerUseError.message("Failed to read fixture state")
}
public static func writeState(_ state: FixtureAppState) throws {
let directory = stateFileURL.deletingLastPathComponent()
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
let data = try JSONEncoder().encode(state)
- try data.write(to: stateFileURL)
+ try data.write(to: stateFileURL, options: .atomic)
}
public static func post(_ command: FixtureCommand) throws {
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift
index f97eafa..ceb4207 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift
@@ -60,18 +60,10 @@ public final class StdioMCPServer {
case "tools/call":
let name = params["name"] as? String ?? ""
let arguments = params["arguments"] as? [String: Any] ?? [:]
- let text = try callTool(name: name, arguments: arguments)
+ let result = try callTool(name: name, arguments: arguments)
return try encodeJSONRPCResult(
id: id,
- result: [
- "content": [
- [
- "type": "text",
- "text": text,
- ],
- ],
- "isError": false,
- ]
+ result: result.asDictionary
)
default:
if method == nil {
@@ -80,6 +72,11 @@ public final class StdioMCPServer {
return try encodeJSONRPCError(id: id, code: -32601, message: "Method not found: \(method ?? "")")
}
+ } catch let error as ComputerUseError {
+ let payload = try? JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any]
+ let id = payload?["id"]
+ let result = ToolCallResult.text(error.errorDescription ?? String(describing: error), isError: error.toolResultIsError)
+ return try? encodeJSONRPCResult(id: id, result: result.asDictionary)
} catch {
let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)
let payload = try? JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any]
@@ -99,7 +96,7 @@ public final class StdioMCPServer {
}
}
- private func callTool(name: String, arguments: [String: Any]) throws -> String {
+ private func callTool(name: String, arguments: [String: Any]) throws -> ToolCallResult {
switch name {
case "list_apps":
return service.listApps()
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
index 4190746..eb23d1b 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolDefinitions.swift
@@ -18,31 +18,31 @@ public enum ToolDefinitions {
public static let all: [ToolDefinition] = [
ToolDefinition(
name: "list_apps",
- description: "List running or recently visible macOS applications. Read-only and low intrusion.",
+ description: "List the apps on this computer. Returns the set of apps that are currently running, as well as any that have been used in the last 14 days, including details on usage frequency.",
inputSchema: objectSchema(properties: [:], required: [])
),
ToolDefinition(
name: "get_app_state",
- description: "Capture the current accessibility tree and screenshot metadata for an app without activating it when possible. Usually call this first before deciding on an action tool.",
+ description: "Start an app use session if needed, then get the state of the app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app.",
inputSchema: objectSchema(
properties: [
- "app": stringProperty(description: "App name or bundle identifier."),
+ "app": stringProperty(description: "App name or bundle identifier"),
],
required: ["app"]
)
),
ToolDefinition(
name: "click",
- description: "Click an accessibility element or a screenshot-relative coordinate. Prefer this only when a direct AX action is required; coordinate fallback may bring the app forward and use the real pointer.",
+ description: "Click an element by index or pixel coordinates from screenshot.",
inputSchema: objectSchema(
properties: [
- "app": stringProperty(description: "App name or bundle identifier."),
- "element_index": stringProperty(description: "Accessibility element index from get_app_state."),
- "x": numberProperty(description: "Screenshot-relative X coordinate."),
- "y": numberProperty(description: "Screenshot-relative Y coordinate."),
- "click_count": integerProperty(description: "How many clicks to send.", defaultValue: 1),
+ "app": stringProperty(description: "App name or bundle identifier"),
+ "element_index": stringProperty(description: "Element index to click"),
+ "x": numberProperty(description: "X coordinate in screenshot pixel coordinates"),
+ "y": numberProperty(description: "Y coordinate in screenshot pixel coordinates"),
+ "click_count": integerProperty(description: "Number of clicks. Defaults to 1", defaultValue: 1),
"mouse_button": stringProperty(
- description: "Mouse button to use.",
+ description: "Mouse button to click. Defaults to left.",
enumValues: ["left", "right", "middle"],
defaultValue: "left"
),
@@ -52,73 +52,73 @@ public enum ToolDefinitions {
),
ToolDefinition(
name: "perform_secondary_action",
- description: "Invoke a listed accessibility secondary action such as Raise or Show menu. Prefer this over coordinate clicks when get_app_state already exposed the desired AX action.",
+ description: "Invoke a secondary accessibility action exposed by an element.",
inputSchema: objectSchema(
properties: [
- "app": stringProperty(description: "App name or bundle identifier."),
- "element_index": stringProperty(description: "Accessibility element index from get_app_state."),
- "action": stringProperty(description: "Secondary action name from get_app_state."),
+ "app": stringProperty(description: "App name or bundle identifier"),
+ "element_index": stringProperty(description: "Element identifier"),
+ "action": stringProperty(description: "Secondary accessibility action name"),
],
required: ["app", "element_index", "action"]
)
),
ToolDefinition(
name: "scroll",
- description: "Scroll an accessibility element in a direction by pages. Prefers AX page-scroll actions and only falls back to global wheel events when AX scrolling is unavailable.",
+ description: "Scroll an element in a direction by a number of pages.",
inputSchema: objectSchema(
properties: [
- "app": stringProperty(description: "App name or bundle identifier."),
- "direction": stringProperty(description: "One of up, down, left, right.", enumValues: ["up", "down", "left", "right"]),
- "element_index": stringProperty(description: "Accessibility element index from get_app_state."),
- "pages": integerProperty(description: "Number of pages to scroll.", defaultValue: 1),
+ "app": stringProperty(description: "App name or bundle identifier"),
+ "direction": stringProperty(description: "Scroll direction: up, down, left, or right", enumValues: ["up", "down", "left", "right"]),
+ "element_index": stringProperty(description: "Element identifier"),
+ "pages": integerProperty(description: "Number of page scroll actions. Defaults to 1", defaultValue: 1),
],
required: ["app", "direction", "element_index"]
)
),
ToolDefinition(
name: "drag",
- description: "Drag from one screenshot-relative coordinate to another. This uses global mouse events, moves the real pointer, and may need to bring the target app forward.",
+ description: "Drag from one point to another using pixel coordinates.",
inputSchema: objectSchema(
properties: [
- "app": stringProperty(description: "App name or bundle identifier."),
- "from_x": numberProperty(description: "Screenshot-relative start X coordinate."),
- "from_y": numberProperty(description: "Screenshot-relative start Y coordinate."),
- "to_x": numberProperty(description: "Screenshot-relative end X coordinate."),
- "to_y": numberProperty(description: "Screenshot-relative end Y coordinate."),
+ "app": stringProperty(description: "App name or bundle identifier"),
+ "from_x": numberProperty(description: "Start X coordinate"),
+ "from_y": numberProperty(description: "Start Y coordinate"),
+ "to_x": numberProperty(description: "End X coordinate"),
+ "to_y": numberProperty(description: "End Y coordinate"),
],
required: ["app", "from_x", "from_y", "to_x", "to_y"]
)
),
ToolDefinition(
name: "type_text",
- description: "Type literal text into the focused control of an app. Sends keyboard events directly to the target app PID, which usually avoids moving the real pointer or activating the app first.",
+ description: "Type literal text using keyboard input.",
inputSchema: objectSchema(
properties: [
- "app": stringProperty(description: "App name or bundle identifier."),
- "text": stringProperty(description: "Literal text to type."),
+ "app": stringProperty(description: "App name or bundle identifier"),
+ "text": stringProperty(description: "Literal text to type"),
],
required: ["app", "text"]
)
),
ToolDefinition(
name: "press_key",
- description: "Press a key or key combination using xdotool-style syntax. Prefer keyboard shortcuts like super+n when available; events are sent directly to the target app PID.",
+ description: "Press a key or key-combination on the keyboard, including modifier and navigation keys.\n- This supports xdotool's `key` syntax.\n- Examples: \"a\", \"Return\", \"Tab\", \"super+c\", \"Up\", \"KP_0\" (for the numpad 0 key).",
inputSchema: objectSchema(
properties: [
- "app": stringProperty(description: "App name or bundle identifier."),
- "key": stringProperty(description: "Key or chord such as Return, Tab, command+a, or super+c."),
+ "app": stringProperty(description: "App name or bundle identifier"),
+ "key": stringProperty(description: "Key or key combination to press"),
],
required: ["app", "key"]
)
),
ToolDefinition(
name: "set_value",
- description: "Set the value of a settable accessibility element. Prefer this over click plus type_text when the target control already exposes an AX value.",
+ description: "Set the value of a settable accessibility element.",
inputSchema: objectSchema(
properties: [
- "app": stringProperty(description: "App name or bundle identifier."),
- "element_index": stringProperty(description: "Accessibility element index from get_app_state."),
- "value": stringProperty(description: "String value to assign."),
+ "app": stringProperty(description: "App name or bundle identifier"),
+ "element_index": stringProperty(description: "Element identifier"),
+ "value": stringProperty(description: "Value to assign"),
],
required: ["app", "element_index", "value"]
)
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift
new file mode 100644
index 0000000..0072cb8
--- /dev/null
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ToolResult.swift
@@ -0,0 +1,49 @@
+import Foundation
+
+public struct ToolResultContentItem: @unchecked Sendable {
+ let dictionary: [String: Any]
+
+ public static func text(_ text: String) -> ToolResultContentItem {
+ ToolResultContentItem(
+ dictionary: [
+ "type": "text",
+ "text": text,
+ ]
+ )
+ }
+
+ public static func pngImage(_ data: Data) -> ToolResultContentItem {
+ ToolResultContentItem(
+ dictionary: [
+ "type": "image",
+ "data": data.base64EncodedString(),
+ "mimeType": "image/png",
+ ]
+ )
+ }
+}
+
+public struct ToolCallResult: @unchecked Sendable {
+ public let content: [ToolResultContentItem]
+ public let isError: Bool
+
+ public init(content: [ToolResultContentItem], isError: Bool = false) {
+ self.content = content
+ self.isError = isError
+ }
+
+ public var primaryText: String? {
+ content.first(where: { $0.dictionary["type"] as? String == "text" })?.dictionary["text"] as? String
+ }
+
+ public var asDictionary: [String: Any] {
+ [
+ "content": content.map(\.dictionary),
+ "isError": isError,
+ ]
+ }
+
+ public static func text(_ text: String, isError: Bool = false) -> ToolCallResult {
+ ToolCallResult(content: [.text(text)], isError: isError)
+ }
+}
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index 8499c6b..23efa51 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -30,12 +30,27 @@ final class OpenComputerUseKitTests: XCTestCase {
XCTAssertEqual(windowRelativeFrame(elementFrame: textFieldGlobal, windowBounds: window), textField)
}
- func testToolDescriptionsExposeIntrusionHints() {
- let tools = Dictionary(uniqueKeysWithValues: ToolDefinitions.all.map { ($0.name, $0.description) })
+ func testToolDescriptionsMatchOfficialComputerUseSurface() {
+ let tools = Dictionary(uniqueKeysWithValues: ToolDefinitions.all.map { ($0.name, $0) })
- XCTAssertTrue(tools["get_app_state"]?.contains("without activating") == true)
- XCTAssertTrue(tools["press_key"]?.contains("super+n") == true)
- XCTAssertTrue(tools["type_text"]?.contains("target app PID") == true)
- XCTAssertTrue(tools["drag"]?.contains("moves the real pointer") == true)
+ XCTAssertEqual(
+ tools["get_app_state"]?.description,
+ "Start an app use session if needed, then get the state of the app's key window and return a screenshot and accessibility tree. This must be called once per assistant turn before interacting with the app."
+ )
+ XCTAssertTrue(tools["press_key"]?.description.contains("xdotool") == true)
+ XCTAssertEqual(
+ (tools["click"]?.inputSchema["properties"] as? [String: [String: Any]])?["mouse_button"]?["default"] as? String,
+ "left"
+ )
+ XCTAssertEqual(
+ (tools["click"]?.inputSchema["properties"] as? [String: [String: Any]])?["click_count"]?["default"] as? Int,
+ 1
+ )
+ }
+
+ func testComputerUseErrorsFormatLikeToolText() {
+ XCTAssertEqual(ComputerUseError.appNotFound("Sublime Text").errorDescription, #"appNotFound("Sublime Text")"#)
+ XCTAssertFalse(ComputerUseError.appNotFound("Sublime Text").toolResultIsError)
+ XCTAssertTrue(ComputerUseError.invalidArguments("bad").toolResultIsError)
}
}