Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""Cua Cursor Motion through the generated Python bindings.
Replays the golden trajectories that the Rust crate and the TypeScript port
share (libs/cua-driver/rust/crates/cua-cursor-motion/fixtures/golden.json)
through `plan_cursor_move` / `plan_cursor_spec` in the staged native library.
"""
from __future__ import annotations
import json
import math
import os
import platform
import unittest
from pathlib import Path
def _library_name() -> str:
if os.name == "nt":
return "cua_driver_sdk.dll"
if platform.system() == "Darwin":
return "libcua_driver_sdk.dylib"
return "libcua_driver_sdk.so"
ROOT = Path(__file__).parents[1]
LIBRARY = ROOT / "src" / "cua_driver" / _library_name()
GOLDEN = ROOT.parent / "rust/crates/cua-cursor-motion/fixtures/golden.json"
# The fixture rounds to 1e-9; the bindings run the same Rust code.
TOL = 1e-8
FRAME_FRACTIONS = (0.2, 0.5, 0.8, 1.0)
if os.environ.get("CUA_DRIVER_REQUIRE_UNIFFI") == "1" and not LIBRARY.exists():
raise RuntimeError(f"required staged UniFFI library is missing: {LIBRARY}")
@unittest.skipUnless(LIBRARY.exists(), "host-native UniFFI library is not staged")
class CursorMotionGoldenTests(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
from cua_driver import _native as n
from cua_driver import _native_contract as c
cls.n = n
cls.c = c
cls.golden = json.loads(GOLDEN.read_text())
# Fixture JSON -> generated records -------------------------------------
def params(self, p: dict):
n, c = self.n, self.c
return n.CursorMotionParams(
style=getattr(c.CursorMotionStyle, p["style"].upper()),
timing=getattr(c.CursorMotionTiming, p["timing"].upper()),
effects=c.CursorMotionEffects(
**{
k: None
if p["effects"][k] is None
else c.CursorEffectSetting.ON
if p["effects"][k]
else c.CursorEffectSetting.OFF
for k in self.effect_names
}
),
start_handle=p["start_handle"],
end_handle=p["end_handle"],
arc_size=p["arc_size"],
arc_flow=p["arc_flow"],
spring=p["spring"],
glide_duration_ms=p["glide_duration_ms"],
peak_speed=p["peak_speed"],
min_start_speed=p["min_start_speed"],
min_end_speed=p["min_end_speed"],
turn_radius=p["turn_radius"],
)
effect_names = ("trail", "glow", "magnet", "ripple", "squish")
def request(self, r: dict):
n = self.n
target = r["target"]
return n.CursorMoveRequest(
from_point=n.CursorMotionPoint(x=r["from"][0], y=r["from"][1]),
to_point=n.CursorMotionPoint(x=r["to"][0], y=r["to"][1]),
from_heading=r["from_heading"],
end_heading=r["end_heading"],
target=None
if target is None
else n.CursorMotionRect(x=target[0], y=target[1], width=target[2], height=target[3]),
seed=r["seed"],
reduced_motion=r["reduced_motion"],
)
@staticmethod
def variant(enum_type, value: dict):
fields = {k: v for k, v in value.items() if k != "type"}
return getattr(enum_type, value["type"].upper())(**fields)
def spec(self, s: dict):
n, c = self.n, self.c
return n.CursorMotionSpec(
path=self.variant(n.CursorPathShape, s["path"]),
ease=self.variant(n.CursorEase, s["ease"]),
settle=self.variant(n.CursorSettle, s["settle"]),
duration=self.variant(n.CursorMotionDuration, s["duration"]),
heading=getattr(n.CursorHeading, s["heading"].upper()),
effects=c.CursorMotionEffectsOutput(**s["effects"]),
trail=n.CursorTrailSpec(**s["trail"]),
)
# Comparison --------------------------------------------------------------
def near(self, name: str, got: float, want: float) -> None:
self.assertLessEqual(abs(got - want), TOL, f"{name}: {got} vs {want}")
def check_frame(self, name: str, frame, click, want: dict) -> None:
def present(value, expected, fields):
self.assertEqual(value is None, expected is None, f"{name} presence")
if value is not None:
for i, field in enumerate(fields):
self.near(f"{name}.{field}", field(value), expected[i])
present(frame.glow, want["glow"], [lambda g: g.x, lambda g: g.y, lambda g: g.r, lambda g: g.alpha])
present(
frame.magnet,
want["magnet"],
[lambda m: m.rect.x, lambda m: m.rect.y, lambda m: m.rect.width, lambda m: m.rect.height, lambda m: m.glow],
)
present(
click.ripple,
want["ripple"],
[lambda r: r.x, lambda r: r.y, lambda r: r.r, lambda r: r.width, lambda r: r.alpha],
)
self.near(f"{name}.squish", click.squish, want["squish"])
self.assertEqual(len(frame.trail), len(want["trail"]), f"{name} trail length")
for seg, w in zip(frame.trail, want["trail"]):
for got, expected in zip((seg.ax, seg.ay, seg.bx, seg.by, seg.width, seg.alpha), w):
self.near(f"{name}.trail", got, expected)
def check(self, name: str, traj, out: dict) -> None:
n, c = self.n, self.c
samples = traj.samples()
self.assertEqual(len(samples), out["samples"], f"{name}: sample count")
self.near(f"{name}.duration", traj.duration(), out["duration"])
self.near(f"{name}.arrival", traj.arrival_t(), out["arrival_t"])
snap = traj.snap_t()
self.assertEqual(snap is None, out["snap_t"] is None, f"{name}: snap")
if snap is not None:
self.near(f"{name}.snap", snap, out["snap_t"])
t = traj.target()
for got, want in zip((t.x, t.y, t.width, t.height), out["target"]):
self.near(f"{name}.target", got, want)
self.assertEqual(traj.target_known(), out["target_known"])
fx = traj.effects()
self.assertEqual({k: getattr(fx, k) for k in self.effect_names}, out["effects"])
d = traj.duration()
grid = self.golden["grid"]
for i, want in enumerate(out["grid"]):
s = traj.sample_at(d * i / grid)
for field, got, expected in zip("txyh", (s.t, s.x, s.y, s.heading), want):
self.near(f"{name}.grid[{i}].{field}", got, expected)
if "frames" in out:
self.near(f"{name}.linger", traj.linger(), out["linger"])
clicks = c.CursorMotionEffectsOutput(trail=False, glow=False, magnet=False, ripple=True, squish=True)
for k, f in zip(FRAME_FRACTIONS, out["frames"]):
at = d * k
s = traj.sample_at(at)
frame = traj.effect_frame(at, True)
click = n.cursor_click_effects(clicks, 0.03 + 0.4 * k, s.x, s.y)
self.check_frame(f"{name}@{k}", frame, click, f["frame"])
# Tests -------------------------------------------------------------------
def test_built_in_styles_match_the_golden_trajectories(self) -> None:
cases = self.golden["cases"]
self.assertGreater(len(cases), 100)
for case in cases:
traj = self.n.plan_cursor_move(self.params(case["params"]), self.request(case["request"]))
self.check(case["name"], traj, case["out"])
def test_custom_specs_match_the_golden_trajectories(self) -> None:
for case in self.golden["spec_cases"]:
traj = self.n.plan_cursor_spec(self.spec(case["spec"]), self.request(case["request"]))
self.check(case["name"], traj, case["out"])
def test_legacy_boolean_effects_reach_native_planner(self) -> None:
n, c = self.n, self.c
request = self.request(self.golden["cases"][0]["request"])
for name in self.effect_names:
for legacy, canonical in ((True, c.CursorEffectSetting.ON),
(False, c.CursorEffectSetting.OFF),
(None, None),
(c.CursorEffectSetting.DEFAULT, c.CursorEffectSetting.DEFAULT)):
with self.subTest(effect=name, value=legacy):
params = n.default_cursor_motion_params()
fields = dict.fromkeys(self.effect_names)
fields[name] = legacy
params.effects = c.CursorMotionEffects(**fields)
actual = n.plan_cursor_move(params, request).effects()
fields[name] = canonical
params.effects = c.CursorMotionEffects(**fields)
expected = n.plan_cursor_move(params, request).effects()
self.assertEqual(actual, expected)
if type(legacy) is bool:
self.assertEqual(getattr(actual, name), legacy)
def test_invalid_effect_is_refused_before_native_dispatch(self) -> None:
params = self.n.default_cursor_motion_params()
request = self.request(self.golden["cases"][0]["request"])
for invalid in ("true", "on", 1, 0, [], {}):
with self.subTest(value=invalid):
params.effects.trail = invalid
with self.assertRaises(ValueError):
self.n.plan_cursor_move(params, request)
def test_the_arc_styles_are_specs(self) -> None:
n, c = self.n, self.c
params = n.default_cursor_motion_params()
request = self.request(self.golden["cases"][0]["request"])
spec = n.cursor_motion_spec_for_style(c.CursorMotionStyle.COMET_SWOOP, params)
self.assertIsNotNone(spec)
params.style = c.CursorMotionStyle.COMET_SWOOP
# A raw spec matches the planned move under native timing; the default (steady) adds a floor.
params.timing = c.CursorMotionTiming.NATIVE
a = n.plan_cursor_move(params, request).samples()
b = n.plan_cursor_spec(spec, request).samples()
self.assertEqual(a, b)
self.assertIsNone(n.cursor_motion_spec_for_style(c.CursorMotionStyle.MAGNETIC, params))
self.assertTrue(math.isclose(params.arc_size, 0.25))
if __name__ == "__main__":
unittest.main()