Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver): choose the jev-use S1 modality and report its checkpoint (#4205)
Francesco Bonacci7a82121900ea
diff --git a/libs/cua-driver/examples/jev-use/decision-models.md b/libs/cua-driver/examples/jev-use/decision-models.mdindex f1dc89a6c3..4c6512eea6 100644--- a/libs/cua-driver/examples/jev-use/decision-models.md+++ b/libs/cua-driver/examples/jev-use/decision-models.md@@ -130,8 +130,29 @@ request schema permits up to 32 candidates. The S1 multimodal adapter requiresan existing local screenshot path and an explicit screenshot capture ID equalto the request's `capture_id`. The caller must supply the image from thatcapture; the adapter checks identity but cannot authenticate image contents.-The path and image are never added to a TypeSafe request. The CLI currently-exposes only the text adapter.+The path and image are never added to a TypeSafe request.++`choose_decision.py --model s1` loads the text adapter by default. Select the+multimodal adapter with `--s1-modality multimodal` (or `S1_MODALITY=multimodal`)+and pass the capture's image with `--screenshot <png>` and+`--screenshot-capture-id <capture_id>`; the chooser returns `error` when the+capture ID differs from the request. `verify_decision_cli.py --model s1+--screenshot <png>` runs the same check against a fixture, binding the image to+the fixture's capture ID.++The response's `model` field names the checkpoint as+`<adapter>[@<revision>]:<modality>`, for example+`cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79:text`. The adapter name+and revision come from a Hugging Face cache snapshot path or from the metadata+that `hf download --local-dir` writes; otherwise the directory name is used+without a revision. Set `S1_ADAPTER_ID` (for example `cua-ai/cua-s1-4b-0.2`) and+`S1_ADAPTER_REVISION` to state them explicitly. The base model is not part of+this identity, so record `S1_BASE_MODEL_PATH`'s revision alongside the+evidence.++The chooser supports only Cua-S1-4B PEFT adapters (`cua-s1-4b-0.2` and+`cua-s1-4b-0.1`). `cua-s1-nano-0.1` and `cua-s1-forms` are byte-level scorers+with a different input format; `S1_ADAPTER_PATH` pointing at them fails setup.## Verification scopediff --git a/libs/cua-driver/examples/jev-use/python/choose_decision.py b/libs/cua-driver/examples/jev-use/python/choose_decision.pyindex d208cd95d6..89268a2dc6 100644--- a/libs/cua-driver/examples/jev-use/python/choose_decision.py+++ b/libs/cua-driver/examples/jev-use/python/choose_decision.py@@ -11,11 +11,13 @@ from typing import Anyfrom choose_action import MAX_INPUT_BYTES, validate_requestfrom decision_models import (+ S1_MODALITIES,DecisionRequest,ModelScores,S1DecisionModel,TypeSafeDecisionModel,choose,+ s1_model_identity,)@@ -37,7 +39,24 @@ class MockDecisionModel:)-def local_s1_model() -> S1DecisionModel:+def _resolve_s1_adapter(adapter: Path, modality: str) -> Path:+ """Return the PEFT adapter directory the chooser will load for `modality`."""+ for candidate in (adapter / modality, adapter):+ if (candidate / "adapter_config.json").is_file():+ return candidate+ raise ValueError(+ f"S1_ADAPTER_PATH has no Cua-S1-4B PEFT adapter for modality {modality!r} "+ f"(expected {modality}/adapter_config.json or adapter_config.json); "+ "cua-s1-nano and cua-s1-forms checkpoints are not supported by this chooser"+ )+++def local_s1_model(+ *,+ modality: str | None = None,+ screenshot: Path | None = None,+ screenshot_capture_id: str | None = None,+) -> S1DecisionModel:base_value = os.environ["S1_BASE_MODEL_PATH"]adapter_value = os.environ["S1_ADAPTER_PATH"]if not base_value.strip() or not adapter_value.strip():@@ -46,6 +65,27 @@ def local_s1_model() -> S1DecisionModel:adapter = Path(adapter_value).expanduser()if not base.is_dir() or not adapter.is_dir():raise ValueError("S1_BASE_MODEL_PATH and S1_ADAPTER_PATH must be local directories")+ modality = modality or os.environ.get("S1_MODALITY", "").strip() or "text"+ if modality not in S1_MODALITIES:+ raise ValueError(f"S1 modality must be one of {', '.join(S1_MODALITIES)}")+ if modality == "multimodal":+ if screenshot is None or not screenshot_capture_id:+ raise ValueError(+ "multimodal S1 requires --screenshot and --screenshot-capture-id "+ "from the same capture as the request"+ )+ screenshot = screenshot.expanduser()+ if not screenshot.is_file():+ raise ValueError("--screenshot must be an existing local image file")+ elif screenshot is not None or screenshot_capture_id is not None:+ raise ValueError("--screenshot and --screenshot-capture-id require multimodal S1")+ _resolve_s1_adapter(adapter, modality)+ name = s1_model_identity(+ adapter,+ modality,+ repo_id=os.environ.get("S1_ADAPTER_ID"),+ revision=os.environ.get("S1_ADAPTER_REVISION"),+ )from cua_s1.four_b import FourBModelreturn S1DecisionModel(@@ -54,8 +94,12 @@ def local_s1_model() -> S1DecisionModel:lora_adapter_path=adapter,device=os.environ.get("S1_DEVICE", "cpu"),dtype=os.environ.get("S1_DTYPE", "float16"),- modality="text",- )+ modality=modality,+ ),+ modality=modality,+ screenshot_path=screenshot,+ screenshot_capture_id=screenshot_capture_id,+ name=name,)@@ -67,7 +111,25 @@ def choose_request(value: Any, model: Any) -> dict[str, Any]:def main() -> None:parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("--model", choices=("mock", "jev", "s1"), default="mock")+ parser.add_argument(+ "--s1-modality",+ choices=S1_MODALITIES,+ help="S1 adapter to load (default: $S1_MODALITY, else text)",+ )+ parser.add_argument(+ "--screenshot",+ type=Path,+ help="multimodal S1 only: local image from the request's capture",+ )+ parser.add_argument(+ "--screenshot-capture-id",+ help="multimodal S1 only: capture ID of --screenshot; must equal the request capture_id",+ )args = parser.parse_args()+ if args.model != "s1" and (+ args.s1_modality or args.screenshot is not None or args.screenshot_capture_id is not None+ ):+ parser.error("--s1-modality, --screenshot, and --screenshot-capture-id require --model s1")raw = sys.stdin.buffer.read(MAX_INPUT_BYTES + 1)if len(raw) > MAX_INPUT_BYTES:raise SystemExit("request exceeds input limit")@@ -80,7 +142,11 @@ def main() -> None:model = MockDecisionModel()elif args.model == "s1":try:- model = local_s1_model()+ model = local_s1_model(+ modality=args.s1_modality,+ screenshot=args.screenshot,+ screenshot_capture_id=args.screenshot_capture_id,+ )except (KeyError, ValueError, ImportError) as error:raise SystemExit(f"S1 setup failed: {error}") from Noneelse:diff --git a/libs/cua-driver/examples/jev-use/python/decision_models.py b/libs/cua-driver/examples/jev-use/python/decision_models.pyindex a41908f581..8d2a2bc63a 100644--- a/libs/cua-driver/examples/jev-use/python/decision_models.py+++ b/libs/cua-driver/examples/jev-use/python/decision_models.py@@ -8,6 +8,7 @@ from __future__ import annotationsimport mathimport json+import refrom dataclasses import dataclassfrom pathlib import Pathfrom types import MappingProxyType@@ -196,7 +197,81 @@ def visual_regions_as_text(request: DecisionRequest) -> str:return "\n".join(lines)+S1_MODALITIES = ("text", "multimodal")+_HF_SNAPSHOT_PARENT = "snapshots"+_REVISION_PATTERN = re.compile(r"[0-9a-f]{40}")+_UNSAFE_IDENTITY_CHARS = re.compile(r"[^A-Za-z0-9._/-]")+++def _hf_local_dir_revision(root: Path, modality: str) -> str | None:+ """Read the commit that `hf download --local-dir` recorded for the adapter.++ `huggingface_hub` writes `<local-dir>/.cache/huggingface/download/<file>.metadata`+ whose first line is the resolved commit hash. Only the selected modality's+ adapter files are consulted, and they must agree.+ """+ metadata_dir = root / ".cache" / "huggingface" / "download"+ revisions: set[str] = set()+ for prefix in (Path(modality), Path()):+ for name in ("adapter_config.json", "adapter_model.safetensors"):+ candidate = metadata_dir / prefix / f"{name}.metadata"+ try:+ first_line = candidate.read_text(encoding="utf-8").splitlines()[0].strip()+ except (OSError, IndexError, UnicodeDecodeError):+ continue+ if _REVISION_PATTERN.fullmatch(first_line):+ revisions.add(first_line)+ if revisions:+ break+ return revisions.pop() if len(revisions) == 1 else None+++def s1_model_identity(+ adapter_path: Path,+ modality: str,+ *,+ repo_id: str | None = None,+ revision: str | None = None,+) -> str:+ """Name the S1 checkpoint that produced a decision, for evidence logs.++ The result is `<adapter>[@<revision>]:<modality>`, for example+ `cua-ai/cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79:text`.+ `repo_id` and `revision` override detection. Otherwise the adapter name+ and revision come from a Hugging Face cache snapshot path+ (`models--<org>--<name>/snapshots/<commit>`), from the metadata that+ `hf download --local-dir` leaves in the adapter directory, or, failing+ both, from the adapter directory name with no revision. Detection never+ claims a Hub repository or revision it cannot read locally.+ """+ if modality not in S1_MODALITIES:+ raise ValueError(f"unknown S1 modality: {modality!r}")+ root = Path(adapter_path).expanduser()+ if root.name in S1_MODALITIES:+ root = root.parent+ label = repo_id.strip() if repo_id and repo_id.strip() else None+ detected_revision: str | None = None+ parts = root.parts+ if len(parts) >= 3 and parts[-2] == _HF_SNAPSHOT_PARENT and parts[-3].startswith("models--"):+ if label is None:+ label = parts[-3].removeprefix("models--").replace("--", "/")+ if _REVISION_PATTERN.fullmatch(parts[-1]):+ detected_revision = parts[-1]+ else:+ detected_revision = _hf_local_dir_revision(root, modality)+ if label is None:+ label = root.name or "cua-s1-adapter"+ chosen_revision = revision.strip() if revision and revision.strip() else detected_revision+ label = _UNSAFE_IDENTITY_CHARS.sub("_", label)[:64]+ identity = label+ if chosen_revision:+ identity += "@" + _UNSAFE_IDENTITY_CHARS.sub("_", chosen_revision)[:40]+ return f"{identity}:{modality}"++class S1DecisionModel:+ # Default identity for callers that construct the adapter directly without+ # naming the checkpoint. `choose_decision.py` passes a derived identity.name = "cua-s1-4b-local"def __init__(@@ -206,7 +281,12 @@ class S1DecisionModel:modality: Literal["text", "multimodal"] = "text",screenshot_path: Path | None = None,screenshot_capture_id: str | None = None,+ name: str | None = None,) -> None:+ if name is not None:+ if not isinstance(name, str) or not 0 < len(name) <= 128:+ raise ValueError("S1 model identity must be a bounded string")+ self.name = nameself.scorer = scorerself.modality = modalityself.screenshot_path = screenshot_pathdiff --git a/libs/cua-driver/examples/jev-use/python/tests/test_decision_models.py b/libs/cua-driver/examples/jev-use/python/tests/test_decision_models.pyindex 9e699a0cc5..167e72db9e 100644--- a/libs/cua-driver/examples/jev-use/python/tests/test_decision_models.py+++ b/libs/cua-driver/examples/jev-use/python/tests/test_decision_models.py@@ -12,8 +12,24 @@ from types import SimpleNamespaceBASE = Path(__file__).resolve().parents[2]sys.path.insert(0, str(BASE / "python"))-from choose_decision import MockDecisionModel, choose_request-from decision_models import S1DecisionModel, TypeSafeDecisionModel+from choose_decision import MockDecisionModel, choose_request, local_s1_model+from decision_models import S1DecisionModel, TypeSafeDecisionModel, s1_model_identity++REVISION = "16818868b0cc7813808aae4e87b417657046ab79"+OTHER_REVISION = "88d8b8a90c2da4470d005cc23ec8665a6442ebe1"+++def write_adapter(root: Path, modality: str, revision: str | None = None) -> None:+ """Lay out a PEFT adapter like `hf download --local-dir` does."""+ (root / modality).mkdir(parents=True, exist_ok=True)+ (root / modality / "adapter_config.json").write_text("{}", encoding="utf-8")+ if revision:+ metadata = root / ".cache/huggingface/download" / modality+ metadata.mkdir(parents=True, exist_ok=True)+ for name in ("adapter_config.json", "adapter_model.safetensors"):+ (metadata / f"{name}.metadata").write_text(+ f"{revision}\n0123abcd\n1790377747.1\n", encoding="utf-8"+ )def request() -> dict:@@ -246,6 +262,147 @@ class DecisionModelsTest(unittest.TestCase):self.assertEqual(result.stdout, "")self.assertIn("S1 setup failed", result.stderr)+ def test_s1_identity_uses_hf_local_dir_revision_for_the_selected_modality(self) -> None:+ with tempfile.TemporaryDirectory() as directory:+ adapter = Path(directory) / "cua-s1-4b-0.2"+ write_adapter(adapter, "text", REVISION)+ write_adapter(adapter, "multimodal", REVISION)+ self.assertEqual(s1_model_identity(adapter, "text"), f"cua-s1-4b-0.2@{REVISION}:text")+ self.assertEqual(+ s1_model_identity(adapter / "multimodal", "multimodal"),+ f"cua-s1-4b-0.2@{REVISION}:multimodal",+ )+ self.assertEqual(+ s1_model_identity(adapter, "text", repo_id="cua-ai/cua-s1-4b-0.2"),+ f"cua-ai/cua-s1-4b-0.2@{REVISION}:text",+ )++ def test_s1_identity_omits_conflicting_or_missing_revisions(self) -> None:+ with tempfile.TemporaryDirectory() as directory:+ plain = Path(directory) / "my adapter"+ write_adapter(plain, "text")+ self.assertEqual(s1_model_identity(plain, "text"), "my_adapter:text")+ mixed = Path(directory) / "mixed"+ write_adapter(mixed, "text", REVISION)+ metadata = mixed / ".cache/huggingface/download/text/adapter_model.safetensors.metadata"+ metadata.write_text(f"{OTHER_REVISION}\n", encoding="utf-8")+ self.assertEqual(s1_model_identity(mixed, "text"), "mixed:text")+ self.assertEqual(s1_model_identity(mixed, "text", revision="v0.2"), "[email protected]:text")++ def test_s1_identity_reads_hugging_face_cache_snapshots(self) -> None:+ snapshot = Path("/cache/hub/models--cua-ai--cua-s1-4b-0.1/snapshots") / OTHER_REVISION+ self.assertEqual(+ s1_model_identity(snapshot, "multimodal"),+ f"cua-ai/cua-s1-4b-0.1@{OTHER_REVISION}:multimodal",+ )+ self.assertEqual(+ s1_model_identity(snapshot / "text", "text"),+ f"cua-ai/cua-s1-4b-0.1@{OTHER_REVISION}:text",+ )++ def test_s1_identity_is_bounded_and_rejects_unknown_modality(self) -> None:+ identity = s1_model_identity(Path("/x") / ("a" * 300), "text", revision="r" * 300)+ self.assertLessEqual(len(identity), 128)+ with self.assertRaises(ValueError):+ s1_model_identity(Path("/x/adapter"), "audio")++ def test_s1_adapter_reports_explicit_identity_and_keeps_the_default(self) -> None:+ scores = {"submit-form": 1.0, "reobserve": 0.0, "abstain": 0.0}+ named = S1DecisionModel(FakeScorer(scores), name=f"cua-s1-4b-0.2@{REVISION}:text")+ self.assertEqual(+ choose_request(request(), named)["model"], f"cua-s1-4b-0.2@{REVISION}:text"+ )+ self.assertEqual(S1DecisionModel(FakeScorer(scores)).name, "cua-s1-4b-local")+ with self.assertRaises(ValueError):+ S1DecisionModel(FakeScorer(scores), name="")++ def _local_s1(self, env: dict[str, str], **kwargs):+ previous = {name: os.environ.get(name) for name in env}+ os.environ.update(env)+ try:+ return local_s1_model(**kwargs)+ finally:+ for name, value in previous.items():+ if value is None:+ os.environ.pop(name, None)+ else:+ os.environ[name] = value++ def test_local_s1_selects_modality_and_binds_the_screenshot(self) -> None:+ try:+ import cua_s1.four_b # noqa: F401+ except ImportError:+ self.skipTest("cua-s1 is not installed")+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ base = root / "Qwen3.5-4B"+ base.mkdir()+ adapter = root / "cua-s1-4b-0.2"+ write_adapter(adapter, "text", REVISION)+ write_adapter(adapter, "multimodal", REVISION)+ screenshot = root / "capture.png"+ screenshot.write_bytes(b"fixture")+ env = {+ "S1_BASE_MODEL_PATH": str(base),+ "S1_ADAPTER_PATH": str(adapter),+ "S1_MODALITY": "",+ "S1_ADAPTER_ID": "",+ "S1_ADAPTER_REVISION": "",+ }++ text = self._local_s1(env)+ self.assertEqual((text.modality, text.scorer.modality), ("text", "text"))+ self.assertEqual(text.name, f"cua-s1-4b-0.2@{REVISION}:text")+ self.assertEqual(text.scorer.lora_adapter_path, adapter)++ multimodal = self._local_s1(+ {**env, "S1_MODALITY": "multimodal"},+ screenshot=screenshot,+ screenshot_capture_id="capture-1",+ )+ self.assertEqual(multimodal.scorer.modality, "multimodal")+ self.assertEqual(multimodal.screenshot_path, screenshot)+ self.assertEqual(multimodal.screenshot_capture_id, "capture-1")+ self.assertEqual(multimodal.name, f"cua-s1-4b-0.2@{REVISION}:multimodal")++ with self.assertRaisesRegex(ValueError, "requires --screenshot"):+ self._local_s1(env, modality="multimodal")+ with self.assertRaisesRegex(ValueError, "require multimodal"):+ self._local_s1(env, screenshot=screenshot, screenshot_capture_id="capture-1")+ with self.assertRaisesRegex(ValueError, "modality must be one of"):+ self._local_s1({**env, "S1_MODALITY": "audio"})++ def test_local_s1_rejects_non_peft_checkpoints(self) -> None:+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ base = root / "base"+ base.mkdir()+ nano = root / "cua-s1-nano-0.1" / "text"+ nano.mkdir(parents=True)+ (nano / "model.safetensors").write_bytes(b"")+ env = {"S1_BASE_MODEL_PATH": str(base), "S1_ADAPTER_PATH": str(nano.parent)}+ with self.assertRaisesRegex(ValueError, "not supported by this chooser"):+ self._local_s1(env)++ def test_cli_rejects_s1_options_for_other_models(self) -> None:+ result = subprocess.run(+ [+ sys.executable,+ str(BASE / "python/choose_decision.py"),+ "--model",+ "mock",+ "--s1-modality",+ "multimodal",+ ],+ input=json.dumps(request()),+ text=True,+ capture_output=True,+ check=False,+ )+ self.assertEqual(result.returncode, 2)+ self.assertEqual(result.stdout, "")+ self.assertIn("require --model s1", result.stderr)+def test_typesafe_request_contains_no_screenshot_or_action_arguments(self) -> None:class FakeClient:sent = Nonediff --git a/libs/cua-driver/examples/jev-use/verify_decision_cli.py b/libs/cua-driver/examples/jev-use/verify_decision_cli.pyindex c507c7c490..27f4b85304 100644--- a/libs/cua-driver/examples/jev-use/verify_decision_cli.py+++ b/libs/cua-driver/examples/jev-use/verify_decision_cli.py@@ -14,7 +14,14 @@ def main() -> None:parser.add_argument("--model", choices=("mock", "jev", "s1"), required=True)parser.add_argument("--expected-id", default="submit-form")parser.add_argument("--fixture", choices=("positive", "negative"), default="positive")+ parser.add_argument(+ "--screenshot",+ type=Path,+ help="run the multimodal S1 adapter with this image, bound to the fixture capture",+ )args = parser.parse_args()+ if args.screenshot is not None and args.model != "s1":+ parser.error("--screenshot requires --model s1")root = Path(__file__).resolve().parentfixture_name = (@@ -23,8 +30,18 @@ def main() -> None:else "jev-choice-negative-v1.json")request = json.loads((root / "fixtures" / fixture_name).read_text(encoding="utf-8"))+ command = [sys.executable, str(root / "python/choose_decision.py"), "--model", args.model]+ if args.screenshot is not None:+ command += [+ "--s1-modality",+ "multimodal",+ "--screenshot",+ str(args.screenshot),+ "--screenshot-capture-id",+ request["capture_id"],+ ]result = subprocess.run(- [sys.executable, str(root / "python/choose_decision.py"), "--model", args.model],+ command,input=json.dumps(request),text=True,capture_output=True,