Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Update main build pin and derive app versions from workspace (#5746)

injaneity7aae0d919f75
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/pins.json b/libs/cua-driver-bench/automated-eval/macos_bench/pins.json
index 830679f39b..4e94e81160 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/pins.json
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/pins.json
@@ -38,14 +38,14 @@
}
},
"cua_main": {
- "note": "Bench v2 runs: main at the commit of the latest registered run (A17.3 rerun: 984e5f9c5). Built in the VM with tools/vm_main_build. Earlier registered runs used other main commits; see the dated amendment commits.",
- "git_sha": "984e5f9c5c5a39185e7d0e03f3d6937e44503c31",
- "version": "0.34.0",
- "version_string": "cua-driver 0.34.0",
- "binary_sha256": "3657e899b14f734f71b4aa52af3f9bbf44c2a97a62947ea9c40cbf0e0a5c1098",
- "skills_tree_sha256": "7f1c0fd0f22cb4a3dd0ae04256a3e78c9f4d51194e92284765cc554381249504",
+ "note": "Bench v2 uses main at 1fa2247ae built in the VM with tools/vm_main_build.",
+ "git_sha": "1fa2247ae745f5029211573ad4f5c3087efcf641",
+ "version": "0.38.0",
+ "version_string": "cua-driver 0.38.0",
+ "binary_sha256": "f53304e429ca0c2a3fc3f0fd5699d273251b2ef847eaa728949c3260b6da8969",
+ "skills_tree_sha256": "e1ea6ffdabfef1cf1757283ae281dd75ab0a7a3b179b14bb50d9869b9f8e6eb7",
"skills_source": "libs/cua-driver/rust/Skills/cua-driver at git_sha",
- "script_binary_sha256": "8d5dc83f23e7ff0f95bcade4fff67e25cd7309bf02d67a5ca31e86ca5059cce4"
+ "script_binary_sha256": "3398da9e53a2fc7980c58ae9d45060c7befecb9eebd7e602f97f5d9b4f8a2f56"
},
"codex_1007": {
"_note": "Arm cc-codex-cu-1007-browser (Amendment 10): the ChatGPT/Codex app 26.1007.21159 (build 20052) from the public appcast, installed in a clone of the bench VM only and never launched. The Computer Use service is not in the app bundle (the app installs it at run time), so it stays 26.929.1001365.",
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench.py
index c297870c7d..4ff6ccac1c 100644
--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench.py
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench.py
@@ -10,6 +10,8 @@ from __future__ import annotations
import dataclasses
import json
import os
+import plistlib
+import subprocess
import sys
import tempfile
import unittest
@@ -213,6 +215,81 @@ class LedgerTest(unittest.TestCase):
self.assertFalse(core.budget_allows(12.0, 4.0, 15.0))
[email protected](sys.platform == "darwin", "assembly requires macOS plutil and zsh")
+class MainAssemblyTest(unittest.TestCase):
+ def setUp(self) -> None:
+ tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(tmp.cleanup)
+ self.home = Path(tmp.name)
+ self.main = self.home / "bench-work/cua-main"
+ self.rust = self.main / "src/libs/cua-driver/rust"
+ release = self.rust / "target/release"
+ release.mkdir(parents=True)
+ for name in ("cua-driver", "cua-cursor-theme"):
+ binary = release / name
+ binary.write_text("#!/bin/sh\necho fixture-driver\n")
+ binary.chmod(0o755)
+ skills = self.rust / "Skills/cua-driver"
+ skills.mkdir(parents=True)
+ (skills / "SKILL.md").write_text("fixture skill\n")
+ self.sha = "123456789abcdef0123456789abcdef0123456789"
+ (self.main / "SOURCE_SHA").write_text(self.sha + "\n")
+ donor = self.home / "bench-work/cua-0.34.0/CuaDriver-0.34.0.app/Contents"
+ (donor / "Resources").mkdir(parents=True)
+ (donor / "Resources/AppIcon.icns").write_bytes(b"fixture icon")
+ with (donor / "Info.plist").open("wb") as stream:
+ plistlib.dump(
+ {"CFBundleShortVersionString": "0.34.0", "CFBundleVersion": "0.34.0"}, stream
+ )
+ shim = self.home / "bin"
+ shim.mkdir()
+ codesign = shim / "codesign"
+ codesign.write_text("#!/bin/sh\nexit 0\n")
+ codesign.chmod(0o755)
+ self.env = {**os.environ, "HOME": str(self.home), "PATH": f"{shim}:{os.environ['PATH']}"}
+
+ def test_bundle_versions_follow_workspace_package(self) -> None:
+ (self.rust / "Cargo.toml").write_text(
+ '[package]\nversion = "0.0.1"\n'
+ '[workspace.package]\nversion = "9.8.7"\n'
+ '[workspace.dependencies]\nversion = "0.0.2"\n'
+ )
+ result = subprocess.run(
+ ["/bin/zsh", str(HERE.parent / "tools/vm_main_build/assemble_main.sh")],
+ env=self.env,
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ with (self.main / "CuaDriverBenchMain.app/Contents/Info.plist").open("rb") as stream:
+ info = plistlib.load(stream)
+ self.assertEqual(info["CFBundleShortVersionString"], "9.8.7")
+ self.assertEqual(info["CFBundleVersion"], "9.8.7." + self.sha[:9])
+
+ def test_missing_workspace_version_fails(self) -> None:
+ for workspace in (
+ "",
+ '[workspace.package]\nedition = "2021"\n',
+ '[workspace.package]\nversion = ""\n',
+ ):
+ with self.subTest(workspace=workspace):
+ (self.rust / "Cargo.toml").write_text(
+ '[package]\nversion = "0.0.1"\n'
+ + workspace
+ + '[workspace.dependencies]\nversion = "0.0.2"\n'
+ )
+ result = subprocess.run(
+ ["/bin/zsh", str(HERE.parent / "tools/vm_main_build/assemble_main.sh")],
+ env=self.env,
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("no [workspace.package] version", result.stderr)
+
+
class PinsAndGateTest(unittest.TestCase):
def test_compare_pins_flags_every_difference_and_missing_values(self) -> None:
expected = {"a": "1", "b": "2", "c": "3", "release": "url", "cua_driver_git_sha": "x"}
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tools/vm_main_build/assemble_main.sh b/libs/cua-driver-bench/automated-eval/macos_bench/tools/vm_main_build/assemble_main.sh
index 3be6619007..c34e1665b3 100755
--- a/libs/cua-driver-bench/automated-eval/macos_bench/tools/vm_main_build/assemble_main.sh
+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tools/vm_main_build/assemble_main.sh
@@ -9,6 +9,8 @@ R=$M/src/libs/cua-driver/rust
A=$M/CuaDriverBenchMain.app
OLD=~/bench-work/cua-0.34.0/CuaDriver-0.34.0.app
SHA=$(cat $M/SOURCE_SHA)
+VERSION=$(awk -F '"' '/^\[workspace\.package\][[:space:]]*$/ { package=1; next } /^\[/ { package=0 } package && /^[[:space:]]*version[[:space:]]*=/ { print $2; exit }' "$R/Cargo.toml")
+[[ -n $VERSION ]] || { echo "no [workspace.package] version in $R/Cargo.toml" >&2; exit 1; }
rm -rf $A; mkdir -p $A/Contents/MacOS $A/Contents/Resources
cp $R/target/release/cua-driver $A/Contents/MacOS/cua-driver
cp $R/target/release/cua-cursor-theme $A/Contents/MacOS/cua-cursor-theme
@@ -17,8 +19,8 @@ cp $OLD/Contents/Info.plist $A/Contents/Info.plist
plutil -replace CFBundleIdentifier -string com.trycua.driver.benchmain $A/Contents/Info.plist
plutil -replace CFBundleName -string "Cua Driver Bench Main" $A/Contents/Info.plist
plutil -replace CFBundleDisplayName -string "Cua Driver Bench Main" $A/Contents/Info.plist
-plutil -replace CFBundleShortVersionString -string "0.34.0" $A/Contents/Info.plist
-plutil -replace CFBundleVersion -string "0.34.0.${SHA:0:9}" $A/Contents/Info.plist
+plutil -replace CFBundleShortVersionString -string "$VERSION" $A/Contents/Info.plist
+plutil -replace CFBundleVersion -string "${VERSION}.${SHA:0:9}" $A/Contents/Info.plist
cat > $M/ent.plist <<PL
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">