Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
bench: park the real pointer before every trial (Amendment 24) (#6660)
Francesco Bonacci457c32e55ee3
diff --git a/libs/cua-driver-bench/METHOD.md b/libs/cua-driver-bench/METHOD.mdindex d61ad895ec..a84bbe260f 100644--- a/libs/cua-driver-bench/METHOD.md+++ b/libs/cua-driver-bench/METHOD.md@@ -304,3 +304,10 @@ differences are directional only.the foreground run). Windows: the Cua Driver 0.38.0 Windows build,agent-desktop 1.2.0, the Browser Use Windows harness 0.2.2 and Codex computeruse from the ChatGPT app 26.1029.9216.0 (Claude Code 2.1.316, Claude Sonnet 5.5).++## Amendment 24 (6 November 2026): pointer park before each trial++Before every trial the harness moves the real pointer to a fixed point (1000, 12) outside every registered+window, reads the position back and records it in the trial row, and a failed park fails the trial setup,+because a pointer left over the test app by an earlier task made macOS activate the app when it showed a+background sheet, and that changed the interruption results under every tool.diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py b/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.pyindex 2371ba411f..1391fbeb60 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py@@ -501,6 +501,51 @@ CONDITIONS = ("foreground", "background")# The user's window in the background condition, top-left screen coordinates (x, y, w, h). It partly covers# BenchLab's lower right and the middle of the CDB task windows, and stays clear of BenchLab's top strip.USER_WINDOW_FRAME = (420, 340, 900, 520)+# Amendment 24: before every trial the harness parks the real pointer at PARK_POINT (top-left screen+# coordinates), reads it back and records it in the trial row. A pointer left over a task window by an earlier+# task changes how the OS treats a background sheet, so the starting state is fixed and reported.+PARK_POINT = (+ 1000,+ 12,+) # an empty stretch of the top edge: above every window of the registered layout+PARK_TOLERANCE_PX = 3.0+# Registered window frames (x, y, w, h): the user window, BenchLab on macOS, BenchLab on Linux and Windows,+# and the foreground witness window.+REGISTERED_FRAMES = {+ "user": USER_WINDOW_FRAME,+ "benchlab_macos": (60, 80, 900, 560),+ "benchlab_linux_windows": (40, 40, 760, 540),+ "witness": (1000, 640, 600, 320),+}+++def park_point_clear(+ point: tuple[float, float], frames: dict[str, tuple[int, int, int, int]] | None = None+) -> list[str]:+ """Names of the registered frames that contain `point`; empty when the point is clear of all of them."""+ px, py = point+ return [+ name+ for name, (x, y, w, h) in (frames or REGISTERED_FRAMES).items()+ if x <= px < x + w and y <= py < y + h+ ]+++def park_record(read_back: tuple[float, float] | None, method: str) -> dict[str, Any]:+ """The trial-row entry for a park: target, position read back, whether it landed, and the method."""+ ok = (+ read_back is not None+ and abs(read_back[0] - PARK_POINT[0]) <= PARK_TOLERANCE_PX+ and abs(read_back[1] - PARK_POINT[1]) <= PARK_TOLERANCE_PX+ )+ return {+ "target": list(PARK_POINT),+ "read_back": list(read_back) if read_back is not None else None,+ "ok": ok,+ "method": method,+ }++BACKGROUND_BRIEF = ('The user is working in another window, titled "User activity", and is typing in it. Leave that window '"in front and in focus the whole time: do not activate, raise or bring any other app or window to the "diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/launch/hyprland_session.py b/libs/cua-driver-bench/automated-eval/macos_bench/launch/hyprland_session.pyindex 937343f51e..10c539ccfd 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/launch/hyprland_session.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/launch/hyprland_session.py@@ -123,6 +123,22 @@ def sentinel_lines(log: Path) -> list[dict]:return out+def park_pointer(H) -> dict:+ """Amendment 24: move the real pointer to the fixed park point and read it back. Hyprland 0.55+ takes the+ Lua dispatcher; the legacy `movecursor` is the fallback. The record says ok=False when it did not land."""+ x, y = v2.PARK_POINT+ record = v2.park_record(None, "hyprland dispatch")+ for expression in (f"hl.dsp.cursor.move({{ x = {x}, y = {y} }})", f"movecursor {x} {y}"):+ try:+ H.dispatch(expression)+ record = v2.park_record(tuple(H.cursor()), "hyprland dispatch")+ except OSError:+ continue+ if record["ok"]:+ break+ return record++def run(args: argparse.Namespace, command: list[str]) -> dict:G = hyprland_module("guard")H = hyprland_module("hypr")@@ -156,6 +172,11 @@ def run(args: argparse.Namespace, command: list[str]) -> dict:row["harness_error"] = "screen smaller than 1320x860 (load hyprland/bench.lua)"(out / "row.json").write_text(json.dumps(row, indent=1), encoding="utf-8")return row+ row["pointer_parked"] = park_pointer(H)+ if not row["pointer_parked"]["ok"]:+ row["harness_error"] = f"pointer park failed: {row['pointer_parked']}"+ (out / "row.json").write_text(json.dumps(row, indent=1), encoding="utf-8")+ return rowenv = {**os.environ, "GDK_BACKEND": "wayland", "NO_AT_BRIDGE": "0"}lab = subprocess.Popen([sys.executable, str(BENCH / "linux" / "benchlab.py"), "--mode", mode, "--seed", str(seed),diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/launch/windows_session.py b/libs/cua-driver-bench/automated-eval/macos_bench/launch/windows_session.pyindex 53e5c767c8..f680754b27 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/launch/windows_session.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/launch/windows_session.py@@ -97,6 +97,13 @@ def sentinel_lines(log: Path) -> list[dict]:return out+def park_pointer(W) -> dict:+ """Amendment 24: move the real pointer to the fixed park point and read it back (SetCursorPos/GetCursorPos)."""+ x, y = v2.PARK_POINT+ W.user32.SetCursorPos(int(x), int(y))+ return v2.park_record(tuple(float(v) for v in W.cursor()), "SetCursorPos")++def run(args: argparse.Namespace, command: list[str]) -> dict:import guard as Gimport winenv as W@@ -129,6 +136,10 @@ def run(args: argparse.Namespace, command: list[str]) -> dict:control.write_text("", encoding="utf-8")procs = []try:+ row["pointer_parked"] = park_pointer(W)+ if not row["pointer_parked"]["ok"]:+ row["harness_error"] = f"pointer park failed: {row['pointer_parked']}"+ return rowif cdb is not None:row["workspace"] = str(cdb.workspace)try:diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py b/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.pyindex 0f608071cb..120f4b0553 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py@@ -994,6 +994,9 @@ def run_attempt(ctx: Ctx, entry: dict[str, Any], attempt: int) -> dict[str, Any]): # A23: Cua Driver under Codex uses that build's daemonensure_agent_daemon(ctx, SOURCE_ARM[arm])if getattr(args, "park_pointer", False):+ row["pointer_parked"] = (+ None # set even when park_pointer raises, so the row shows the attempt+ )row["pointer_parked"] = park_pointer(ctx)lab_app = Path(args.build_dir or HERE / "linux") / "BenchLab.app"if cdb is not None:@@ -1601,37 +1604,78 @@ def ensure_lab_unoccluded(ctx: "Ctx") -> dict[str, Any]:return {"error": f"{type(error).__name__}: {error}", "unoccluded": None}-PARK_POINT = (- 1000,- 12,-) # Amendment 3, A3.9: an empty stretch of the menu bar, away from every task window+PARK_POINT = v2.PARK_POINT # Amendment 24: fixed, outside every registered window frame-def park_pointer(ctx: "Ctx") -> Any:- """Move the real pointer to PARK_POINT through the recorder daemon before each trial, so no trial starts- with the pointer left over a hover target by the previous one. Returns the pointer position read back."""- rec_home = ca.RECORDER_STATE / "home"- args = {"x": PARK_POINT[0], "y": PARK_POINT[1], "scope": "desktop"}+class ParkFailed(RuntimeError):+ """The pointer could not be parked and verified; the trial setup fails (a harness error, never scored)."""+++def _xy_of(node: Any) -> tuple[float, float] | None:+ for item in _walk_dicts(node):+ x, y = item.get("x"), item.get("y")+ if isinstance(x, (int, float)) and isinstance(y, (int, float)):+ return float(x), float(y)+ return None+++def park_pointer(ctx: "Ctx") -> dict[str, Any]:+ """Move the real pointer to PARK_POINT before each trial, read the position back and return the record for+ the trial row. macOS: through the recorder daemon. Linux X11: xdotool. Raises ParkFailed when the pointer+ does not land within tolerance, or when the platform has no supported way to do it (no faking)."""+ clash = v2.park_point_clear(PARK_POINT)+ if clash:+ raise ParkFailed(f"park point {PARK_POINT} lies inside registered frame(s) {clash}")+ platform_name = desktop.detect_platform()try:- ca.cua_cli(- "call",- "move_cursor",- json.dumps(args),- socket=ca.RECORDER_SOCKET,- home=rec_home,- timeout=20,- )- done = ca.cua_cli(- "call",- "get_cursor_position",- "{}",- socket=ca.RECORDER_SOCKET,- home=rec_home,- timeout=20,- )- return json.loads(done.stdout[done.stdout.index("{") :])- except (OSError, ValueError, subprocess.TimeoutExpired) as error:- return {"error": f"{type(error).__name__}: {error}"}+ if platform_name == "macos":+ rec_home = ca.RECORDER_STATE / "home"+ args = {"x": PARK_POINT[0], "y": PARK_POINT[1], "scope": "desktop"}+ ca.cua_cli(+ "call",+ "move_cursor",+ json.dumps(args),+ socket=ca.RECORDER_SOCKET,+ home=rec_home,+ timeout=20,+ )+ done = ca.cua_cli(+ "call",+ "get_cursor_position",+ "{}",+ socket=ca.RECORDER_SOCKET,+ home=rec_home,+ timeout=20,+ )+ read_back = _xy_of(json.loads(done.stdout[done.stdout.index("{") :]))+ record = v2.park_record(read_back, "recorder-daemon move_cursor")+ elif platform_name == "linux" and shutil.which("xdotool") and os.environ.get("DISPLAY"):+ subprocess.run(+ ["xdotool", "mousemove", str(PARK_POINT[0]), str(PARK_POINT[1])],+ check=True,+ timeout=20,+ )+ out = subprocess.run(+ ["xdotool", "getmouselocation", "--shell"],+ check=True,+ capture_output=True,+ text=True,+ timeout=20,+ ).stdout+ found = dict(line.split("=", 1) for line in out.split() if "=" in line)+ read_back = (float(found["X"]), float(found["Y"]))+ record = v2.park_record(read_back, "xdotool mousemove")+ else:+ raise ParkFailed(+ f"no supported way to park the pointer on {platform_name} without X11 xdotool"+ )+ except ParkFailed:+ raise+ except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error:+ raise ParkFailed(f"{type(error).__name__}: {error}") from error+ if not record["ok"]:+ raise ParkFailed(f"pointer read back at {record['read_back']}, expected {list(PARK_POINT)}")+ return recorddef _walk_dicts(node: Any):@@ -2610,8 +2654,10 @@ def build_parser() -> argparse.ArgumentParser:)p.add_argument("--park-pointer",- action="store_true",- help="move the real pointer to an empty menu-bar point before each trial (Amendment 3, A3.9)",+ action=argparse.BooleanOptionalAction,+ default=True,+ help="park the real pointer at a fixed point outside every registered window, verify it and record "+ "it in the trial row, before each trial; a failed park fails the trial setup (Amendment 24)",)p.add_argument("--no-recorder", action="store_true")p.add_argument(diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.pyindex a198ef333f..af439b9cec 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py@@ -568,3 +568,47 @@ class MultiplayerTest(unittest.TestCase):if __name__ == "__main__":unittest.main()+++class ParkPointerTests(unittest.TestCase):+ """Amendment 24: the pointer is parked at a fixed point outside every registered window, verified, recorded."""++ def test_park_point_clear_of_registered_frames(self):+ self.assertEqual(v2.park_point_clear(v2.PARK_POINT), [])++ def test_park_point_clear_of_every_task_window(self):+ pack = Path(__file__).resolve().parents[3] / "taskpack"+ frames = {}+ for path in pack.rglob("launch.*.json"):+ for node in _bounds_nodes(json.loads(path.read_text("utf-8"))):+ frames[f"{path.name}:{len(frames)}"] = (+ node["x"],+ node["y"],+ node["width"],+ node["height"],+ )+ self.assertTrue(frames)+ self.assertEqual(v2.park_point_clear(v2.PARK_POINT, frames), [])++ def test_clear_detects_overlap(self):+ self.assertIn("user", v2.park_point_clear((500, 400)))+ self.assertIn("benchlab_macos", v2.park_point_clear((162, 158)))++ def test_record_ok_within_tolerance_and_fails_outside(self):+ near = (v2.PARK_POINT[0] + 1.0, v2.PARK_POINT[1] - 1.0)+ self.assertTrue(v2.park_record(near, "m")["ok"])+ self.assertFalse(v2.park_record((162.0, 158.0), "m")["ok"])+ self.assertFalse(v2.park_record(None, "m")["ok"])+ self.assertEqual(v2.park_record(near, "m")["target"], list(v2.PARK_POINT))+++def _bounds_nodes(node):+ if isinstance(node, dict):+ bounds = node.get("bounds")+ if isinstance(bounds, dict) and {"x", "y", "width", "height"} <= set(bounds):+ yield bounds+ for value in node.values():+ yield from _bounds_nodes(value)+ elif isinstance(node, list):+ for value in node:+ yield from _bounds_nodes(value)diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.pyindex 4f37508539..57838f4f75 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py@@ -1110,3 +1110,31 @@ class MiscTest(unittest.TestCase):if __name__ == "__main__":unittest.main()+++class ParkPointerFlowTests(unittest.TestCase):+ def test_failed_park_raises_and_success_returns_record(self):+ point = rb.PARK_POINT+ ok = mock.Mock(stdout=json.dumps({"x": point[0], "y": point[1]}))+ with (+ mock.patch.object(rb.desktop, "detect_platform", return_value="macos"),+ mock.patch.object(rb.ca, "cua_cli", return_value=ok),+ ):+ record = rb.park_pointer(mock.Mock())+ self.assertTrue(record["ok"])+ self.assertEqual(record["read_back"], [float(point[0]), float(point[1])])+ off = mock.Mock(stdout=json.dumps({"x": 162, "y": 158}))+ with (+ mock.patch.object(rb.desktop, "detect_platform", return_value="macos"),+ mock.patch.object(rb.ca, "cua_cli", return_value=off),+ ):+ with self.assertRaises(rb.ParkFailed):+ rb.park_pointer(mock.Mock())++ def test_unsupported_platform_fails_instead_of_faking(self):+ with (+ mock.patch.object(rb.desktop, "detect_platform", return_value="linux"),+ mock.patch.object(rb.shutil, "which", return_value=None),+ ):+ with self.assertRaises(rb.ParkFailed):+ rb.park_pointer(mock.Mock())