Skip to content

👾 Open Computer Use – Open-Source Alternative to Codex Computer Use

feat: strengthen visual cursor idle wobble

usoon.xu80868ccf179c
diff --git a/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift b/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
index 68385fc..31179f0 100644
--- a/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
+++ b/apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift
@@ -15,10 +15,11 @@ final class MCPClient {
private let stdout: FileHandle
private var nextID = 1
- init(executableURL: URL, arguments: [String]) throws {
+ init(executableURL: URL, arguments: [String], environment: [String: String]? = nil) throws {
process = Process()
process.executableURL = executableURL
process.arguments = arguments
+ process.environment = environment
let stdinPipe = Pipe()
let stdoutPipe = Pipe()
let stderrPipe = Pipe()
@@ -160,6 +161,7 @@ enum OpenComputerUseSmokeSuite {
let fixtureURL = productsDirectory.appendingPathComponent("OpenComputerUseFixture")
let serverURL = productsDirectory.appendingPathComponent("OpenComputerUse")
let appName = "OpenComputerUseFixture"
+ let mode = SmokeMode(arguments: CommandLine.arguments)
terminateExistingFixtures(named: appName)
try? FileManager.default.removeItem(at: FixtureBridge.stateFileURL)
@@ -176,6 +178,15 @@ enum OpenComputerUseSmokeSuite {
Thread.sleep(forTimeInterval: 1.5)
+ switch mode {
+ case .full:
+ try runFullSmoke(serverURL: serverURL, appName: appName)
+ case .cursorIdleOnly:
+ try runCursorIdleSmoke(serverURL: serverURL, appName: appName)
+ }
+ }
+
+ private static func runFullSmoke(serverURL: URL, appName: String) throws {
let client = try MCPClient(executableURL: serverURL, arguments: ["mcp"])
defer {
client.terminate()
@@ -287,6 +298,52 @@ enum OpenComputerUseSmokeSuite {
print("Smoke suite completed.")
}
+ private static func runCursorIdleSmoke(serverURL: URL, appName: String) throws {
+ print("1. cursor idle observation setup")
+ let observationURL = cursorObservationFileURL()
+ try? FileManager.default.removeItem(at: observationURL)
+ var environment = ProcessInfo.processInfo.environment
+ environment["OPEN_COMPUTER_USE_VISUAL_CURSOR"] = "1"
+ environment["OPEN_COMPUTER_USE_VISUAL_CURSOR_OBSERVATION_FILE"] = observationURL.path
+
+ let client = try MCPClient(executableURL: serverURL, arguments: ["mcp"], environment: environment)
+ defer {
+ client.terminate()
+ }
+
+ try client.initialize()
+ _ = try client.callTool("get_app_state", arguments: [
+ "app": appName,
+ ])
+
+ print("2. trigger click and wait for idle overlay")
+ let state = try client.callTool("click", arguments: [
+ "app": appName,
+ "element_index": "1",
+ ])
+ try expect(state.contains("Counter:"), "cursor smoke click should still return fixture state")
+
+ let firstIdleSnapshot = try waitForCursorObservation(at: observationURL, phase: "idle")
+ Thread.sleep(forTimeInterval: 0.25)
+ let secondIdleSnapshot = try waitForCursorObservation(at: observationURL, phase: "idle")
+
+ print("3. assert anchored tip plus changing rotation")
+ let firstTip = try expectPoint(firstIdleSnapshot.tipPosition, name: "first tip position")
+ let secondTip = try expectPoint(secondIdleSnapshot.tipPosition, name: "second tip position")
+ let firstResting = try expectPoint(firstIdleSnapshot.restingTipPosition, name: "first resting tip position")
+ let secondResting = try expectPoint(secondIdleSnapshot.restingTipPosition, name: "second resting tip position")
+ let firstRotation = try expectValue(firstIdleSnapshot.rotation, name: "first rotation")
+ let secondRotation = try expectValue(secondIdleSnapshot.rotation, name: "second rotation")
+
+ try expect(abs(firstTip.x - secondTip.x) < 0.25, "idle cursor should stay anchored horizontally instead of shaking")
+ try expect(abs(firstTip.y - secondTip.y) < 0.25, "idle cursor should stay anchored vertically")
+ try expect(distanceBetween(firstTip, firstResting) < 0.35, "idle cursor tip should stay near resting position")
+ try expect(distanceBetween(secondTip, secondResting) < 0.35, "idle cursor tip should keep resting anchor")
+ try expect(abs(secondRotation - firstRotation) > 0.01, "idle cursor should keep a clearly visible tiny rotation wobble")
+
+ print("Cursor idle smoke completed.")
+ }
+
private static func locateProductsDirectory() throws -> URL {
let executableURL = URL(fileURLWithPath: CommandLine.arguments[0]).resolvingSymlinksInPath()
return executableURL.deletingLastPathComponent()
@@ -376,4 +433,64 @@ enum OpenComputerUseSmokeSuite {
return counter
}
+
+ private static func cursorObservationFileURL() -> URL {
+ URL(fileURLWithPath: NSTemporaryDirectory())
+ .appendingPathComponent("open-computer-use-smoke", isDirectory: true)
+ .appendingPathComponent("visual-cursor-observation.json")
+ }
+
+ private static func waitForCursorObservation(at url: URL, phase: String, timeout: TimeInterval = 6) throws -> VisualCursorObservationSnapshot {
+ let deadline = Date().addingTimeInterval(timeout)
+ var lastSnapshot: VisualCursorObservationSnapshot?
+
+ while Date() < deadline {
+ if
+ let data = try? Data(contentsOf: url),
+ let snapshot = try? JSONDecoder().decode(VisualCursorObservationSnapshot.self, from: data)
+ {
+ lastSnapshot = snapshot
+ if snapshot.phase == phase {
+ return snapshot
+ }
+ }
+
+ Thread.sleep(forTimeInterval: 0.05)
+ }
+
+ throw SmokeError.message("Timed out waiting for cursor observation phase \(phase). Last snapshot: \(String(describing: lastSnapshot))")
+ }
+
+ private static func expectPoint(_ point: VisualCursorObservationPoint?, name: String) throws -> CGPoint {
+ guard let point else {
+ throw SmokeError.message("Missing \(name)")
+ }
+
+ return CGPoint(x: point.x, y: point.y)
+ }
+
+ private static func expectValue(_ value: Double?, name: String) throws -> Double {
+ guard let value else {
+ throw SmokeError.message("Missing \(name)")
+ }
+
+ return value
+ }
+
+ private static func distanceBetween(_ lhs: CGPoint, _ rhs: CGPoint) -> CGFloat {
+ hypot(lhs.x - rhs.x, lhs.y - rhs.y)
+ }
+}
+
+private enum SmokeMode {
+ case full
+ case cursorIdleOnly
+
+ init(arguments: [String]) {
+ if arguments.contains("--cursor-idle-only") {
+ self = .cursorIdleOnly
+ } else {
+ self = .full
+ }
+ }
}
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 04c612b..4513244 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -9,7 +9,7 @@
- `apps/OpenComputerUseFixture`
本地 GUI fixture app,用来承载低风险、可预测的点击/输入/滚动/拖拽验证路径。
- `apps/OpenComputerUseSmokeSuite`
- 端到端 smoke runner,会拉起 fixture 和 MCP server,并通过 JSON-RPC 真实调用 9 个 tools。
+ 端到端 smoke runner,会拉起 fixture 和 MCP server,并通过 JSON-RPC 真实调用 9 个 tools;同时也支持单独的 visual cursor idle smoke,用跨进程 observation file 断言等待下一次 move 时是 anchored tip + tiny rotate wobble,而不是横向漂移。
- `apps/OpenComputerUseWindows`
实验性 Windows runtime。它不依赖 Swift 或 `.app` bundle,Go CLI/MCP 入口会嵌入 PowerShell UI Automation bridge,构建产物是 `open-computer-use.exe`。
- `packages/OpenComputerUseKit`
@@ -62,7 +62,7 @@
- `open-computer-use call <tool> --args '{...}'` 会直接输出 MCP-style JSON result;`open-computer-use call --calls '[...]'` / `--calls-file <path>` 会在同一进程里顺序执行 JSON 数组里的 tool calls,并复用同一个 `ComputerUseService` 内存态,因此 `get_app_state` 之后的 action tool 可以继续使用同一轮 snapshot 的 `element_index`。序列执行默认会在成功的相邻操作之间 sleep 1 秒,也可以用 `--sleep <seconds>` 覆盖;遇到 `isError=true` 的 tool result 后停止。
- 对真实 app 的 `get_app_state` / action tool 入口,当前新增了一层官方风格的高风险 bundle denylist:bundle-id 直传时直接返回 safety denial;名称匹配时默认不解析到这些 app,尽量贴近官方对终端、密码管理器、Chrome 与少量系统敏感组件的防护行为。
- 普通 app 的 element frame 当前按“窗口左上角为原点”的 window-relative 坐标输出,便于后续把 `element_index` 和截图坐标统一到同一套参考系。
-- `click` / `set_value` 在执行真实动作前后,会额外驱动一层透明 `SoftwareCursorOverlay` window:两者的移动阶段现在共用一条 heading-driven 的官方风格 motion 内核,显式把“当前 cursor 朝向”和“最终 resting pose”一起喂给选路器,优先生成需要时先掉头、再沿车头方向推进的 C 形/单侧大弧轨迹;首次显示时按官方 binary 的 fresh state 从 AppKit 全局 `(0,0)` window origin 生成起点,后续动作继续复用上一帧 visible tip。真正显示出来的 cursor 不再直接等于 path sample,而是经过一层独立的 visual dynamics 状态,把 visible tip、velocity、angle 和 fog/offset 持续推进。`click` 结尾会衔接 click pulse 和 idle sway,`set_value` 则只做 settle / idle,不给 pulse;两者收尾后会在目标点继续保持 idle 状态,只有连续 30 秒没有新动作时才做 cleanup,这样连续 tool call 不会反复从 fresh `(0,0)` 起步;如果宿主在任务 / turn 结束时发出 `turn-ended`,cursor 会立即消失并清掉本轮位置状态。
+- `click` / `set_value` 在执行真实动作前后,会额外驱动一层透明 `SoftwareCursorOverlay` window:两者的移动阶段现在共用一条 heading-driven 的官方风格 motion 内核,显式把“当前 cursor 朝向”和“最终 resting pose”一起喂给选路器,优先生成需要时先掉头、再沿车头方向推进的 C 形/单侧大弧轨迹;首次显示时按官方 binary 的 fresh state 从 AppKit 全局 `(0,0)` window origin 生成起点,后续动作继续复用上一帧 visible tip。真正显示出来的 cursor 不再直接等于 path sample,而是经过一层独立的 visual dynamics 状态,把 visible tip、velocity、angle 和 fog/offset 持续推进。`click` 结尾会衔接 click pulse 和更明显但仍然很小的 rotate wobble,`set_value` 则只做 settle / idle,不给 pulse;两者收尾后会在目标点继续保持 idle 状态,等待下一次动作时 tip 保持 anchored、只保留可感知的小角度摆动;只有连续 30 秒没有新动作时才做 cleanup,这样连续 tool call 不会反复从 fresh `(0,0)` 起步;如果宿主在任务 / turn 结束时发出 `turn-ended`,cursor 会立即消失并清掉本轮位置状态。
- overlay 的 visual style 不再自己从官方 app bundle 裁 `SoftwareCursor` 小图;主 MCP runtime 现在和 `CursorMotion` 一样优先渲染仓库里沉淀的 `official-software-cursor-window-252.png` baseline,只有资源缺失时才退回 `OpenComputerUseKit` 内部的程序化 pointer/fog fallback。命中点 anchor 仍固定在 `126x126` 画布里的同一组 tip-offset 上;glyph 自身的 neutral heading 继续沿用 `CursorMotion` / 官方 baseline 的 `-3π/4`。主 runtime overlay window 按 AppKit 全局坐标移动,因此在把 AX / `CGWindowList` 产出的 y-down screen-space 点击目标喂给 overlay 之前,会先转换成对应屏幕的 AppKit 全局坐标;路径选路用屏幕上实际可见的 AppKit forward heading,进入 visual dynamics / render state 前则把 velocity 的 y 轴翻回 CursorMotion 的 y-down screen state,再交给 AppKit 绘制层做角度和 `dy` 翻转。程序化 fallback 保留 neutral artwork correction,把它的天然轮廓轴对齐到 `CursorMotion` / 官方 baseline 的 `-3π/4` forward 方向,但不让实验线依赖 runtime 代码。
- overlay 的层级不再固定 `.floating`;现在会跟随 snapshot 命中的目标 window id / layer,把自己排到该目标 window 之上,而不是粗暴压到所有前台 app 最上层。
- overlay 的曲线路径不再只按固定 Bezier 模板生成;当前主线采用 reverse-engineering 约束下的 heading-driven candidate 族,候选只保留 `direct` / `turn` / `brake` / `orbit` 这些能稳定产出单侧主弧的 family,并继续保留 target-window 命中策略作为同类候选间的 tie-break。原始 binary lift 恢复出来的 `20` 条路径和 score 仍然保留在独立的 `StandaloneCursor` viewer / Python 重建脚本里,用于对照分析,不再直接作为 runtime 默认 chooser。
@@ -119,7 +119,7 @@
- 单元测试:`swift test`
- standalone cursor 构建:`swift build --product StandaloneCursor`
- cursor lab 构建:`swift build --product CursorMotion`
-- 端到端 smoke:`./scripts/run-tool-smoke-tests.sh`
+- 端到端 smoke:`./scripts/run-tool-smoke-tests.sh`(标准 9-tool smoke + visual cursor idle smoke)
- app 打包:`./scripts/build-open-computer-use-app.sh debug`
- npm staging:`node ./scripts/npm/build-packages.mjs`
- release tgz:`./scripts/release-package.sh`
diff --git a/docs/histories/2026-04/20260422-1720-switch-visual-cursor-idle-to-rotate-shake.md b/docs/histories/2026-04/20260422-1720-switch-visual-cursor-idle-to-rotate-shake.md
new file mode 100644
index 0000000..52050b9
--- /dev/null
+++ b/docs/histories/2026-04/20260422-1720-switch-visual-cursor-idle-to-rotate-shake.md
@@ -0,0 +1,29 @@
+## [2026-04-22 17:20] | Task: 调整 visual cursor 等待态为轻微旋转抖动
+
+### Execution Context
+* **Agent ID**: `Codex`
+* **Base Model**: `GPT-5.4`
+* **Runtime**: `Codex CLI on macOS + SwiftPM`
+
+### User Query
+> when cursor is waiting for the new move, the animation is left and right horizontally shake, actually, I wanna it makes a tiny rotate shake
+
+### Changes Overview
+**Scope:** `OpenComputerUseKit` visual cursor runtime、端到端 smoke、测试、架构文档
+
+**Key Actions:**
+- **[Idle pose 收紧]**: 把 visual cursor 在等待下一次 move 时的 idle target 固定回 resting tip,不再做左右/上下位移抖动。
+- **[Rotate wobble 保留]**: idle 态只保留一个很小的 angle offset,让等待态更接近“原地轻微转动”而不是水平摇摆。
+- **[回归覆盖]**: 增加针对 idle pose 的单测,并为 `OpenComputerUseSmokeSuite` 新增 visual cursor idle smoke,通过 observation file 跨进程验证“tip anchored + rotation changes”。
+- **[振幅上调]**: 后续根据实机反馈把 idle rotation 振幅从几乎不可感知的档位提高到仍属 tiny、但肉眼能明显察觉的档位。
+
+### Design Intent
+用户想修的不是 move path,而是 cursor 停在目标点等待下一次动作时的观感。当前 runtime 在 idle 阶段仍然给 tip 位置叠加了横向为主的细小漂移,所以更像左右抖。这里把等待态收紧成“位置固定 + 小幅旋转 wobble”,让反馈更稳定,也和仓库里对 lab/runtime 的目标描述保持一致。后续又根据实机反馈把振幅从过小的近不可见档位上调到更容易感知的范围,避免用户几乎察觉不到 rotation。
+
+### Files Modified
+- `apps/OpenComputerUseSmokeSuite/Sources/OpenComputerUseSmokeSuite/main.swift`
+- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorOverlay.swift`
+- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift`
+- `scripts/run-tool-smoke-tests.sh`
+- `docs/ARCHITECTURE.md`
+- `docs/histories/2026-04/20260422-1720-switch-visual-cursor-idle-to-rotate-shake.md`
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
index db713d0..f9d3f24 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/ComputerUseService.swift
@@ -147,20 +147,26 @@ public final class ComputerUseService {
let snapshot = try currentSnapshot(for: query)
let button = MouseButtonKind(rawValue: mouseButton.lowercased()) ?? .left
if snapshot.mode == .fixture {
+ let cursorTarget: VisualCursorTarget?
if let elementIndex {
let record = try lookupElement(snapshot: snapshot, index: elementIndex)
guard let identifier = record.identifier else {
throw ComputerUseError.invalidArguments("fixture click requires an identifier-backed element")
}
+ cursorTarget = visualCursorTarget(for: record, snapshot: snapshot)
+ moveVisualCursor(to: cursorTarget)
try FixtureBridge.post(FixtureCommand(kind: "click", identifier: identifier))
} else if let x, let y {
let identifier = try fixtureIdentifier(at: CGPoint(x: x, y: y), snapshot: snapshot)
+ cursorTarget = fixtureVisualCursorTarget(identifier: identifier, snapshot: snapshot)
+ moveVisualCursor(to: cursorTarget)
try FixtureBridge.post(FixtureCommand(kind: "click", identifier: identifier, x: x, y: y))
} else {
throw ComputerUseError.invalidArguments("click requires either element_index or x/y")
}
Thread.sleep(forTimeInterval: 0.15)
+ pulseVisualCursor(at: cursorTarget, clickCount: clickCount, mouseButton: button)
return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
@@ -366,8 +372,11 @@ public final class ComputerUseService {
throw ComputerUseError.invalidArguments("fixture set_value requires a known element identifier")
}
+ let cursorTarget = visualCursorTarget(for: record, snapshot: snapshot)
+ moveVisualCursor(to: cursorTarget)
try FixtureBridge.post(FixtureCommand(kind: "set_value", identifier: identifier, value: value))
Thread.sleep(forTimeInterval: 0.15)
+ settleVisualCursor(at: cursorTarget)
return snapshotResult(for: try refreshSnapshot(for: query), style: .actionResult)
}
@@ -821,6 +830,11 @@ public final class ComputerUseService {
)
}
+ private func fixtureVisualCursorTarget(identifier: String, snapshot: AppSnapshot) -> VisualCursorTarget? {
+ let record = snapshot.elements.values.first { $0.identifier == identifier }
+ return record.flatMap { visualCursorTarget(for: $0, snapshot: snapshot) }
+ }
+
private func moveVisualCursor(to target: VisualCursorTarget?) {
guard let target else {
return
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorOverlay.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorOverlay.swift
index 42b4bd3..2dfe3ad 100644
--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorOverlay.swift
+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/SoftwareCursorOverlay.swift
@@ -72,6 +72,66 @@ func visualCursorPostInteractionIdleTimeout() -> TimeInterval {
30
}
+func visualCursorIdleRotationAmplitude() -> CGFloat {
+ 0.09
+}
+
+public struct VisualCursorObservationPoint: Codable, Sendable {
+ public let x: Double
+ public let y: Double
+
+ public init(point: CGPoint) {
+ x = point.x
+ y = point.y
+ }
+}
+
+public struct VisualCursorObservationSnapshot: Codable, Sendable {
+ public let phase: String
+ public let tipPosition: VisualCursorObservationPoint?
+ public let restingTipPosition: VisualCursorObservationPoint?
+ public let rotation: Double?
+ public let timestamp: Double
+
+ public init(
+ phase: String,
+ tipPosition: CGPoint?,
+ restingTipPosition: CGPoint?,
+ rotation: CGFloat?,
+ timestamp: CFTimeInterval
+ ) {
+ self.phase = phase
+ self.tipPosition = tipPosition.map(VisualCursorObservationPoint.init(point:))
+ self.restingTipPosition = restingTipPosition.map(VisualCursorObservationPoint.init(point:))
+ self.rotation = rotation.map(Double.init)
+ self.timestamp = timestamp
+ }
+}
+
+struct VisualCursorIdlePose {
+ let tipPosition: CGPoint
+ let angleOffset: CGFloat
+}
+
+func visualCursorIdlePose(restingTipPosition: CGPoint, phase: CGFloat) -> VisualCursorIdlePose {
+ VisualCursorIdlePose(
+ tipPosition: restingTipPosition,
+ angleOffset: sin(phase * 0.8) * visualCursorIdleRotationAmplitude()
+ )
+}
+
+public func visualCursorObservationFileURL(environment: [String: String]) -> URL? {
+ guard
+ let rawPath = environment["OPEN_COMPUTER_USE_VISUAL_CURSOR_OBSERVATION_FILE"]?
+ .trimmingCharacters(in: .whitespacesAndNewlines),
+ !rawPath.isEmpty
+ else {
+ return nil
+ }
+
+ return URL(fileURLWithPath: rawPath)
+}
+
public let openComputerUseTurnEndedNotificationName = Notification.Name("com.ifuryst.opencomputeruse.turn-ended")
public func postOpenComputerUseTurnEndedNotification() {
@@ -136,6 +196,7 @@ enum SoftwareCursorOverlay {
private static var idleTimer: Timer?
private static var hideTimer: Timer?
private static var idlePhase: CGFloat = 0
+ private static var observationPhase = "hidden"
static func moveCursor(to targetPoint: CGPoint, in targetWindow: CursorTargetWindow?) {
guard VisualCursorSupport.isEnabled, canPresentOverlay else {
@@ -152,6 +213,7 @@ enum SoftwareCursorOverlay {
let startPoint = displayedTipPosition ?? defaultInitialTipPosition()
let now = CACurrentMediaTime()
+ observationPhase = "moving"
panel?.alphaValue = 1
if isFreshStart {
visualDynamicsState = CursorVisualDynamicsAnimator.state(at: startPoint, time: CGFloat(now))
@@ -182,6 +244,7 @@ enum SoftwareCursorOverlay {
let now = CACurrentMediaTime()
seedVisualDynamicsIfNeeded(at: constrainedTarget, time: now)
restingTipPosition = constrainedTarget
+ observationPhase = "pulse"
animateClickPulse(at: constrainedTarget, clickCount: max(clickCount, 1), mouseButton: mouseButton)
startIdleAnimation()
scheduleHide(after: visualCursorPostInteractionIdleTimeout())
@@ -195,6 +258,7 @@ enum SoftwareCursorOverlay {
configureOrdering(relativeTo: targetWindow)
let constrainedTarget = clampTipPosition(targetPoint)
restingTipPosition = constrainedTarget
+ observationPhase = "settling"
placeCursor(
using: advanceVisualDynamics(
toward: constrainedTarget,
@@ -213,6 +277,8 @@ enum SoftwareCursorOverlay {
restingTipPosition = nil
activeTargetWindow = nil
visualDynamicsState = nil
+ observationPhase = "hidden"
+ writeObservationSnapshot(tipPosition: nil, rotation: nil)
panel?.orderOut(nil)
}
@@ -524,6 +590,7 @@ enum SoftwareCursorOverlay {
return
}
+ observationPhase = "idle"
idlePhase = 0
let timer = Timer(timeInterval: 1 / 60, repeats: true) { _ in
MainActor.assumeIsolated {
@@ -533,17 +600,17 @@ enum SoftwareCursorOverlay {
refreshActiveOrderingIfNeeded()
+ observationPhase = "idle"
idlePhase += 0.05
- let targetTipPosition = CGPoint(
- x: restingTipPosition.x + (sin(idlePhase) * 1.6),
- y: restingTipPosition.y + (cos(idlePhase * 0.47) * 0.7)
+ let idlePose = visualCursorIdlePose(
+ restingTipPosition: restingTipPosition,
+ phase: idlePhase
)
- let idleAngleOffset = sin(idlePhase * 0.8) * 0.03
placeCursor(
using: advanceVisualDynamics(
- toward: targetTipPosition,
- idleAngleOffset: idleAngleOffset,
+ toward: idlePose.tipPosition,
+ idleAngleOffset: idlePose.angleOffset,
at: CACurrentMediaTime()
),
clickProgress: 0
@@ -604,6 +671,8 @@ enum SoftwareCursorOverlay {
restingTipPosition = nil
activeTargetWindow = nil
visualDynamicsState = nil
+ observationPhase = "hidden"
+ writeObservationSnapshot(tipPosition: nil, rotation: nil)
}
}
}
@@ -671,6 +740,35 @@ enum SoftwareCursorOverlay {
cursorView.clickProgress = clickProgress
cursorView.needsDisplay = true
displayedTipPosition = renderState.tipPosition
+ writeObservationSnapshot(
+ tipPosition: renderState.tipPosition,
+ rotation: renderState.rotation
+ )
+ }
+
+ private static func writeObservationSnapshot(tipPosition: CGPoint?, rotation: CGFloat?) {
+ guard
+ let url = visualCursorObservationFileURL(environment: ProcessInfo.processInfo.environment)
+ else {
+ return
+ }
+
+ let snapshot = VisualCursorObservationSnapshot(
+ phase: observationPhase,
+ tipPosition: tipPosition,
+ restingTipPosition: restingTipPosition,
+ rotation: rotation,
+ timestamp: CACurrentMediaTime()
+ )
+
+ do {
+ let directory = url.deletingLastPathComponent()
+ try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
+ let data = try JSONEncoder().encode(snapshot)
+ try data.write(to: url, options: .atomic)
+ } catch {
+ // Observation is debug-only and must not affect tool execution.
+ }
}
private static func clampTipPosition(_ tipPosition: CGPoint) -> CGPoint {
diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
index deda869..44458bb 100644
--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift
@@ -916,6 +916,27 @@ final class OpenComputerUseKitTests: XCTestCase {
XCTAssertGreaterThan(peakRotation, 1.5)
}
+ func testVisualCursorIdlePoseKeepsTipAnchoredAndOnlyRotates() {
+ let restingTipPosition = CGPoint(x: 184, y: 92)
+ let positivePose = visualCursorIdlePose(restingTipPosition: restingTipPosition, phase: .pi / 2)
+ let negativePose = visualCursorIdlePose(
+ restingTipPosition: restingTipPosition,
+ phase: (.pi / 2) + (.pi / CGFloat(0.8))
+ )
+
+ XCTAssertEqual(positivePose.tipPosition.x, restingTipPosition.x, accuracy: 0.0001)
+ XCTAssertEqual(positivePose.tipPosition.y, restingTipPosition.y, accuracy: 0.0001)
+ XCTAssertGreaterThan(positivePose.angleOffset, 0)
+ XCTAssertLessThanOrEqual(abs(positivePose.angleOffset), visualCursorIdleRotationAmplitude() + 0.0001)
+ XCTAssertGreaterThan(abs(positivePose.angleOffset), 0.08)
+
+ XCTAssertEqual(negativePose.tipPosition.x, restingTipPosition.x, accuracy: 0.0001)
+ XCTAssertEqual(negativePose.tipPosition.y, restingTipPosition.y, accuracy: 0.0001)
+ XCTAssertLessThan(negativePose.angleOffset, 0)
+ XCTAssertLessThanOrEqual(abs(negativePose.angleOffset), visualCursorIdleRotationAmplitude() + 0.0001)
+ XCTAssertGreaterThan(abs(negativePose.angleOffset), 0.08)
+ }
+
private func makeSnapshot(treeLines: [String], focusedSummary: String?, selectedText: String? = nil) -> AppSnapshot {
AppSnapshot(
app: RunningAppDescriptor(
diff --git a/scripts/run-tool-smoke-tests.sh b/scripts/run-tool-smoke-tests.sh
index 45fb18a..26ada44 100755
--- a/scripts/run-tool-smoke-tests.sh
+++ b/scripts/run-tool-smoke-tests.sh
@@ -8,3 +8,4 @@ cd "${repo_root}"
swift build
OPEN_COMPUTER_USE_VISUAL_CURSOR=0 ".build/debug/OpenComputerUseSmokeSuite"
+".build/debug/OpenComputerUseSmokeSuite" --cursor-idle-only