Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Update main build pin and derive app versions from workspace (#5746)
injaneity7aae0d919f75
diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/pins.json b/libs/cua-driver-bench/automated-eval/macos_bench/pins.jsonindex 830679f39b..4e94e81160 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/pins.json+++ b/libs/cua-driver-bench/automated-eval/macos_bench/pins.json@@ -38,14 +38,14 @@}},"cua_main": {- "note": "Bench v2 runs: main at the commit of the latest registered run (A17.3 rerun: 984e5f9c5). Built in the VM with tools/vm_main_build. Earlier registered runs used other main commits; see the dated amendment commits.",- "git_sha": "984e5f9c5c5a39185e7d0e03f3d6937e44503c31",- "version": "0.34.0",- "version_string": "cua-driver 0.34.0",- "binary_sha256": "3657e899b14f734f71b4aa52af3f9bbf44c2a97a62947ea9c40cbf0e0a5c1098",- "skills_tree_sha256": "7f1c0fd0f22cb4a3dd0ae04256a3e78c9f4d51194e92284765cc554381249504",+ "note": "Bench v2 uses main at 1fa2247ae built in the VM with tools/vm_main_build.",+ "git_sha": "1fa2247ae745f5029211573ad4f5c3087efcf641",+ "version": "0.38.0",+ "version_string": "cua-driver 0.38.0",+ "binary_sha256": "f53304e429ca0c2a3fc3f0fd5699d273251b2ef847eaa728949c3260b6da8969",+ "skills_tree_sha256": "e1ea6ffdabfef1cf1757283ae281dd75ab0a7a3b179b14bb50d9869b9f8e6eb7","skills_source": "libs/cua-driver/rust/Skills/cua-driver at git_sha",- "script_binary_sha256": "8d5dc83f23e7ff0f95bcade4fff67e25cd7309bf02d67a5ca31e86ca5059cce4"+ "script_binary_sha256": "3398da9e53a2fc7980c58ae9d45060c7befecb9eebd7e602f97f5d9b4f8a2f56"},"codex_1007": {"_note": "Arm cc-codex-cu-1007-browser (Amendment 10): the ChatGPT/Codex app 26.1007.21159 (build 20052) from the public appcast, installed in a clone of the bench VM only and never launched. The Computer Use service is not in the app bundle (the app installs it at run time), so it stays 26.929.1001365.",diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench.py b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench.pyindex c297870c7d..4ff6ccac1c 100644--- a/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench.py+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tests/test_bench.py@@ -10,6 +10,8 @@ from __future__ import annotationsimport dataclassesimport jsonimport os+import plistlib+import subprocessimport sysimport tempfileimport unittest@@ -213,6 +215,81 @@ class LedgerTest(unittest.TestCase):self.assertFalse(core.budget_allows(12.0, 4.0, 15.0))[email protected](sys.platform == "darwin", "assembly requires macOS plutil and zsh")+class MainAssemblyTest(unittest.TestCase):+ def setUp(self) -> None:+ tmp = tempfile.TemporaryDirectory()+ self.addCleanup(tmp.cleanup)+ self.home = Path(tmp.name)+ self.main = self.home / "bench-work/cua-main"+ self.rust = self.main / "src/libs/cua-driver/rust"+ release = self.rust / "target/release"+ release.mkdir(parents=True)+ for name in ("cua-driver", "cua-cursor-theme"):+ binary = release / name+ binary.write_text("#!/bin/sh\necho fixture-driver\n")+ binary.chmod(0o755)+ skills = self.rust / "Skills/cua-driver"+ skills.mkdir(parents=True)+ (skills / "SKILL.md").write_text("fixture skill\n")+ self.sha = "123456789abcdef0123456789abcdef0123456789"+ (self.main / "SOURCE_SHA").write_text(self.sha + "\n")+ donor = self.home / "bench-work/cua-0.34.0/CuaDriver-0.34.0.app/Contents"+ (donor / "Resources").mkdir(parents=True)+ (donor / "Resources/AppIcon.icns").write_bytes(b"fixture icon")+ with (donor / "Info.plist").open("wb") as stream:+ plistlib.dump(+ {"CFBundleShortVersionString": "0.34.0", "CFBundleVersion": "0.34.0"}, stream+ )+ shim = self.home / "bin"+ shim.mkdir()+ codesign = shim / "codesign"+ codesign.write_text("#!/bin/sh\nexit 0\n")+ codesign.chmod(0o755)+ self.env = {**os.environ, "HOME": str(self.home), "PATH": f"{shim}:{os.environ['PATH']}"}++ def test_bundle_versions_follow_workspace_package(self) -> None:+ (self.rust / "Cargo.toml").write_text(+ '[package]\nversion = "0.0.1"\n'+ '[workspace.package]\nversion = "9.8.7"\n'+ '[workspace.dependencies]\nversion = "0.0.2"\n'+ )+ result = subprocess.run(+ ["/bin/zsh", str(HERE.parent / "tools/vm_main_build/assemble_main.sh")],+ env=self.env,+ capture_output=True,+ text=True,+ timeout=30,+ )+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ with (self.main / "CuaDriverBenchMain.app/Contents/Info.plist").open("rb") as stream:+ info = plistlib.load(stream)+ self.assertEqual(info["CFBundleShortVersionString"], "9.8.7")+ self.assertEqual(info["CFBundleVersion"], "9.8.7." + self.sha[:9])++ def test_missing_workspace_version_fails(self) -> None:+ for workspace in (+ "",+ '[workspace.package]\nedition = "2021"\n',+ '[workspace.package]\nversion = ""\n',+ ):+ with self.subTest(workspace=workspace):+ (self.rust / "Cargo.toml").write_text(+ '[package]\nversion = "0.0.1"\n'+ + workspace+ + '[workspace.dependencies]\nversion = "0.0.2"\n'+ )+ result = subprocess.run(+ ["/bin/zsh", str(HERE.parent / "tools/vm_main_build/assemble_main.sh")],+ env=self.env,+ capture_output=True,+ text=True,+ timeout=30,+ )+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("no [workspace.package] version", result.stderr)++class PinsAndGateTest(unittest.TestCase):def test_compare_pins_flags_every_difference_and_missing_values(self) -> None:expected = {"a": "1", "b": "2", "c": "3", "release": "url", "cua_driver_git_sha": "x"}diff --git a/libs/cua-driver-bench/automated-eval/macos_bench/tools/vm_main_build/assemble_main.sh b/libs/cua-driver-bench/automated-eval/macos_bench/tools/vm_main_build/assemble_main.shindex 3be6619007..c34e1665b3 100755--- a/libs/cua-driver-bench/automated-eval/macos_bench/tools/vm_main_build/assemble_main.sh+++ b/libs/cua-driver-bench/automated-eval/macos_bench/tools/vm_main_build/assemble_main.sh@@ -9,6 +9,8 @@ R=$M/src/libs/cua-driver/rustA=$M/CuaDriverBenchMain.appOLD=~/bench-work/cua-0.34.0/CuaDriver-0.34.0.appSHA=$(cat $M/SOURCE_SHA)+VERSION=$(awk -F '"' '/^\[workspace\.package\][[:space:]]*$/ { package=1; next } /^\[/ { package=0 } package && /^[[:space:]]*version[[:space:]]*=/ { print $2; exit }' "$R/Cargo.toml")+[[ -n $VERSION ]] || { echo "no [workspace.package] version in $R/Cargo.toml" >&2; exit 1; }rm -rf $A; mkdir -p $A/Contents/MacOS $A/Contents/Resourcescp $R/target/release/cua-driver $A/Contents/MacOS/cua-drivercp $R/target/release/cua-cursor-theme $A/Contents/MacOS/cua-cursor-theme@@ -17,8 +19,8 @@ cp $OLD/Contents/Info.plist $A/Contents/Info.plistplutil -replace CFBundleIdentifier -string com.trycua.driver.benchmain $A/Contents/Info.plistplutil -replace CFBundleName -string "Cua Driver Bench Main" $A/Contents/Info.plistplutil -replace CFBundleDisplayName -string "Cua Driver Bench Main" $A/Contents/Info.plist-plutil -replace CFBundleShortVersionString -string "0.34.0" $A/Contents/Info.plist-plutil -replace CFBundleVersion -string "0.34.0.${SHA:0:9}" $A/Contents/Info.plist+plutil -replace CFBundleShortVersionString -string "$VERSION" $A/Contents/Info.plist+plutil -replace CFBundleVersion -string "${VERSION}.${SHA:0:9}" $A/Contents/Info.plistcat > $M/ent.plist <<PL<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">