Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

bench: park the real pointer before every trial (Amendment 24) (#6660)

Francesco Bonacci457c32e55ee3
diff --git a/libs/cua-driver-bench/METHOD.md b/libs/cua-driver-bench/METHOD.md
index d61ad895ec..a84bbe260f 100644
--- a/libs/cua-driver-bench/METHOD.md
+++ b/libs/cua-driver-bench/METHOD.md
@@ -304,3 +304,10 @@ differences are directional only.
the foreground run). Windows: the Cua Driver 0.38.0 Windows build,
agent-desktop 1.2.0, the Browser Use Windows harness 0.2.2 and Codex computer
use from the ChatGPT app 26.1029.9216.0 (Claude Code 2.1.316, Claude Sonnet 5.5).
+
+## Amendment 24 (6 November 2026): pointer park before each trial
+
+Before every trial the harness moves the real pointer to a fixed point (1000, 12) outside every registered
+window, reads the position back and records it in the trial row, and a failed park fails the trial setup,
+because a pointer left over the test app by an earlier task made macOS activate the app when it showed a
+background sheet, and that changed the interruption results under every tool.
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py b/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py
index 2371ba411f..1391fbeb60 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/bench_v2.py
@@ -501,6 +501,51 @@ CONDITIONS = ("foreground", "background")
# The user's window in the background condition, top-left screen coordinates (x, y, w, h). It partly covers
# BenchLab's lower right and the middle of the CDB task windows, and stays clear of BenchLab's top strip.
USER_WINDOW_FRAME = (420, 340, 900, 520)
+# Amendment 24: before every trial the harness parks the real pointer at PARK_POINT (top-left screen
+# coordinates), reads it back and records it in the trial row. A pointer left over a task window by an earlier
+# task changes how the OS treats a background sheet, so the starting state is fixed and reported.
+PARK_POINT = (
+ 1000,
+ 12,
+) # an empty stretch of the top edge: above every window of the registered layout
+PARK_TOLERANCE_PX = 3.0
+# Registered window frames (x, y, w, h): the user window, BenchLab on macOS, BenchLab on Linux and Windows,
+# and the foreground witness window.
+REGISTERED_FRAMES = {
+ "user": USER_WINDOW_FRAME,
+ "benchlab_macos": (60, 80, 900, 560),
+ "benchlab_linux_windows": (40, 40, 760, 540),
+ "witness": (1000, 640, 600, 320),
+}
+
+
+def park_point_clear(
+ point: tuple[float, float], frames: dict[str, tuple[int, int, int, int]] | None = None
+) -> list[str]:
+ """Names of the registered frames that contain `point`; empty when the point is clear of all of them."""
+ px, py = point
+ return [
+ name
+ for name, (x, y, w, h) in (frames or REGISTERED_FRAMES).items()
+ if x <= px < x + w and y <= py < y + h
+ ]
+
+
+def park_record(read_back: tuple[float, float] | None, method: str) -> dict[str, Any]:
+ """The trial-row entry for a park: target, position read back, whether it landed, and the method."""
+ ok = (
+ read_back is not None
+ and abs(read_back[0] - PARK_POINT[0]) <= PARK_TOLERANCE_PX
+ and abs(read_back[1] - PARK_POINT[1]) <= PARK_TOLERANCE_PX
+ )
+ return {
+ "target": list(PARK_POINT),
+ "read_back": list(read_back) if read_back is not None else None,
+ "ok": ok,
+ "method": method,
+ }
+
+
BACKGROUND_BRIEF = (
'The user is working in another window, titled "User activity", and is typing in it. Leave that window '
"in front and in focus the whole time: do not activate, raise or bring any other app or window to the "
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/launch/hyprland_session.py b/libs/cua-driver-bench/automated-eval/macos_bench/launch/hyprland_session.py
index 937343f51e..10c539ccfd 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/launch/hyprland_session.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/launch/hyprland_session.py
@@ -123,6 +123,22 @@ def sentinel_lines(log: Path) -> list[dict]:
return out
+def park_pointer(H) -> dict:
+ """Amendment 24: move the real pointer to the fixed park point and read it back. Hyprland 0.55+ takes the
+ Lua dispatcher; the legacy `movecursor` is the fallback. The record says ok=False when it did not land."""
+ x, y = v2.PARK_POINT
+ record = v2.park_record(None, "hyprland dispatch")
+ for expression in (f"hl.dsp.cursor.move({{ x = {x}, y = {y} }})", f"movecursor {x} {y}"):
+ try:
+ H.dispatch(expression)
+ record = v2.park_record(tuple(H.cursor()), "hyprland dispatch")
+ except OSError:
+ continue
+ if record["ok"]:
+ break
+ return record
+
+
def run(args: argparse.Namespace, command: list[str]) -> dict:
G = hyprland_module("guard")
H = hyprland_module("hypr")
@@ -156,6 +172,11 @@ def run(args: argparse.Namespace, command: list[str]) -> dict:
row["harness_error"] = "screen smaller than 1320x860 (load hyprland/bench.lua)"
(out / "row.json").write_text(json.dumps(row, indent=1), encoding="utf-8")
return row
+ row["pointer_parked"] = park_pointer(H)
+ if not row["pointer_parked"]["ok"]:
+ row["harness_error"] = f"pointer park failed: {row['pointer_parked']}"
+ (out / "row.json").write_text(json.dumps(row, indent=1), encoding="utf-8")
+ return row
env = {**os.environ, "GDK_BACKEND": "wayland", "NO_AT_BRIDGE": "0"}
lab = subprocess.Popen(
[sys.executable, str(BENCH / "linux" / "benchlab.py"), "--mode", mode, "--seed", str(seed),
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/launch/windows_session.py b/libs/cua-driver-bench/automated-eval/macos_bench/launch/windows_session.py
index 53e5c767c8..f680754b27 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/launch/windows_session.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/launch/windows_session.py
@@ -97,6 +97,13 @@ def sentinel_lines(log: Path) -> list[dict]:
return out
+def park_pointer(W) -> dict:
+ """Amendment 24: move the real pointer to the fixed park point and read it back (SetCursorPos/GetCursorPos)."""
+ x, y = v2.PARK_POINT
+ W.user32.SetCursorPos(int(x), int(y))
+ return v2.park_record(tuple(float(v) for v in W.cursor()), "SetCursorPos")
+
+
def run(args: argparse.Namespace, command: list[str]) -> dict:
import guard as G
import winenv as W
@@ -129,6 +136,10 @@ def run(args: argparse.Namespace, command: list[str]) -> dict:
control.write_text("", encoding="utf-8")
procs = []
try:
+ row["pointer_parked"] = park_pointer(W)
+ if not row["pointer_parked"]["ok"]:
+ row["harness_error"] = f"pointer park failed: {row['pointer_parked']}"
+ return row
if cdb is not None:
row["workspace"] = str(cdb.workspace)
try:
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py b/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py
index 0f608071cb..120f4b0553 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/run_bench.py
@@ -994,6 +994,9 @@ def run_attempt(ctx: Ctx, entry: dict[str, Any], attempt: int) -> dict[str, Any]
): # A23: Cua Driver under Codex uses that build's daemon
ensure_agent_daemon(ctx, SOURCE_ARM[arm])
if getattr(args, "park_pointer", False):
+ row["pointer_parked"] = (
+ None # set even when park_pointer raises, so the row shows the attempt
+ )
row["pointer_parked"] = park_pointer(ctx)
lab_app = Path(args.build_dir or HERE / "linux") / "BenchLab.app"
if cdb is not None:
@@ -1601,37 +1604,78 @@ def ensure_lab_unoccluded(ctx: "Ctx") -> dict[str, Any]:
return {"error": f"{type(error).__name__}: {error}", "unoccluded": None}
-PARK_POINT = (
- 1000,
- 12,
-) # Amendment 3, A3.9: an empty stretch of the menu bar, away from every task window
+PARK_POINT = v2.PARK_POINT # Amendment 24: fixed, outside every registered window frame
-def park_pointer(ctx: "Ctx") -> Any:
- """Move the real pointer to PARK_POINT through the recorder daemon before each trial, so no trial starts
- with the pointer left over a hover target by the previous one. Returns the pointer position read back."""
- rec_home = ca.RECORDER_STATE / "home"
- args = {"x": PARK_POINT[0], "y": PARK_POINT[1], "scope": "desktop"}
+class ParkFailed(RuntimeError):
+ """The pointer could not be parked and verified; the trial setup fails (a harness error, never scored)."""
+
+
+def _xy_of(node: Any) -> tuple[float, float] | None:
+ for item in _walk_dicts(node):
+ x, y = item.get("x"), item.get("y")
+ if isinstance(x, (int, float)) and isinstance(y, (int, float)):
+ return float(x), float(y)
+ return None
+
+
+def park_pointer(ctx: "Ctx") -> dict[str, Any]:
+ """Move the real pointer to PARK_POINT before each trial, read the position back and return the record for
+ the trial row. macOS: through the recorder daemon. Linux X11: xdotool. Raises ParkFailed when the pointer
+ does not land within tolerance, or when the platform has no supported way to do it (no faking)."""
+ clash = v2.park_point_clear(PARK_POINT)
+ if clash:
+ raise ParkFailed(f"park point {PARK_POINT} lies inside registered frame(s) {clash}")
+ platform_name = desktop.detect_platform()
try:
- ca.cua_cli(
- "call",
- "move_cursor",
- json.dumps(args),
- socket=ca.RECORDER_SOCKET,
- home=rec_home,
- timeout=20,
- )
- done = ca.cua_cli(
- "call",
- "get_cursor_position",
- "{}",
- socket=ca.RECORDER_SOCKET,
- home=rec_home,
- timeout=20,
- )
- return json.loads(done.stdout[done.stdout.index("{") :])
- except (OSError, ValueError, subprocess.TimeoutExpired) as error:
- return {"error": f"{type(error).__name__}: {error}"}
+ if platform_name == "macos":
+ rec_home = ca.RECORDER_STATE / "home"
+ args = {"x": PARK_POINT[0], "y": PARK_POINT[1], "scope": "desktop"}
+ ca.cua_cli(
+ "call",
+ "move_cursor",
+ json.dumps(args),
+ socket=ca.RECORDER_SOCKET,
+ home=rec_home,
+ timeout=20,
+ )
+ done = ca.cua_cli(
+ "call",
+ "get_cursor_position",
+ "{}",
+ socket=ca.RECORDER_SOCKET,
+ home=rec_home,
+ timeout=20,
+ )
+ read_back = _xy_of(json.loads(done.stdout[done.stdout.index("{") :]))
+ record = v2.park_record(read_back, "recorder-daemon move_cursor")
+ elif platform_name == "linux" and shutil.which("xdotool") and os.environ.get("DISPLAY"):
+ subprocess.run(
+ ["xdotool", "mousemove", str(PARK_POINT[0]), str(PARK_POINT[1])],
+ check=True,
+ timeout=20,
+ )
+ out = subprocess.run(
+ ["xdotool", "getmouselocation", "--shell"],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=20,
+ ).stdout
+ found = dict(line.split("=", 1) for line in out.split() if "=" in line)
+ read_back = (float(found["X"]), float(found["Y"]))
+ record = v2.park_record(read_back, "xdotool mousemove")
+ else:
+ raise ParkFailed(
+ f"no supported way to park the pointer on {platform_name} without X11 xdotool"
+ )
+ except ParkFailed:
+ raise
+ except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error:
+ raise ParkFailed(f"{type(error).__name__}: {error}") from error
+ if not record["ok"]:
+ raise ParkFailed(f"pointer read back at {record['read_back']}, expected {list(PARK_POINT)}")
+ return record
def _walk_dicts(node: Any):
@@ -2610,8 +2654,10 @@ def build_parser() -> argparse.ArgumentParser:
)
p.add_argument(
"--park-pointer",
- action="store_true",
- help="move the real pointer to an empty menu-bar point before each trial (Amendment 3, A3.9)",
+ action=argparse.BooleanOptionalAction,
+ default=True,
+ help="park the real pointer at a fixed point outside every registered window, verify it and record "
+ "it in the trial row, before each trial; a failed park fails the trial setup (Amendment 24)",
)
p.add_argument("--no-recorder", action="store_true")
p.add_argument(
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py
index a198ef333f..af439b9cec 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench_v2.py
@@ -568,3 +568,47 @@ class MultiplayerTest(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+class ParkPointerTests(unittest.TestCase):
+ """Amendment 24: the pointer is parked at a fixed point outside every registered window, verified, recorded."""
+
+ def test_park_point_clear_of_registered_frames(self):
+ self.assertEqual(v2.park_point_clear(v2.PARK_POINT), [])
+
+ def test_park_point_clear_of_every_task_window(self):
+ pack = Path(__file__).resolve().parents[3] / "taskpack"
+ frames = {}
+ for path in pack.rglob("launch.*.json"):
+ for node in _bounds_nodes(json.loads(path.read_text("utf-8"))):
+ frames[f"{path.name}:{len(frames)}"] = (
+ node["x"],
+ node["y"],
+ node["width"],
+ node["height"],
+ )
+ self.assertTrue(frames)
+ self.assertEqual(v2.park_point_clear(v2.PARK_POINT, frames), [])
+
+ def test_clear_detects_overlap(self):
+ self.assertIn("user", v2.park_point_clear((500, 400)))
+ self.assertIn("benchlab_macos", v2.park_point_clear((162, 158)))
+
+ def test_record_ok_within_tolerance_and_fails_outside(self):
+ near = (v2.PARK_POINT[0] + 1.0, v2.PARK_POINT[1] - 1.0)
+ self.assertTrue(v2.park_record(near, "m")["ok"])
+ self.assertFalse(v2.park_record((162.0, 158.0), "m")["ok"])
+ self.assertFalse(v2.park_record(None, "m")["ok"])
+ self.assertEqual(v2.park_record(near, "m")["target"], list(v2.PARK_POINT))
+
+
+def _bounds_nodes(node):
+ if isinstance(node, dict):
+ bounds = node.get("bounds")
+ if isinstance(bounds, dict) and {"x", "y", "width", "height"} <= set(bounds):
+ yield bounds
+ for value in node.values():
+ yield from _bounds_nodes(value)
+ elif isinstance(node, list):
+ for value in node:
+ yield from _bounds_nodes(value)
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py
index 4f37508539..57838f4f75 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_runner_flow.py
@@ -1110,3 +1110,31 @@ class MiscTest(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+class ParkPointerFlowTests(unittest.TestCase):
+ def test_failed_park_raises_and_success_returns_record(self):
+ point = rb.PARK_POINT
+ ok = mock.Mock(stdout=json.dumps({"x": point[0], "y": point[1]}))
+ with (
+ mock.patch.object(rb.desktop, "detect_platform", return_value="macos"),
+ mock.patch.object(rb.ca, "cua_cli", return_value=ok),
+ ):
+ record = rb.park_pointer(mock.Mock())
+ self.assertTrue(record["ok"])
+ self.assertEqual(record["read_back"], [float(point[0]), float(point[1])])
+ off = mock.Mock(stdout=json.dumps({"x": 162, "y": 158}))
+ with (
+ mock.patch.object(rb.desktop, "detect_platform", return_value="macos"),
+ mock.patch.object(rb.ca, "cua_cli", return_value=off),
+ ):
+ with self.assertRaises(rb.ParkFailed):
+ rb.park_pointer(mock.Mock())
+
+ def test_unsupported_platform_fails_instead_of_faking(self):
+ with (
+ mock.patch.object(rb.desktop, "detect_platform", return_value="linux"),
+ mock.patch.object(rb.shutil, "which", return_value=None),
+ ):
+ with self.assertRaises(rb.ParkFailed):
+ rb.park_pointer(mock.Mock())