Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): choose the jev-use S1 modality and report its checkpoint (#4205)

Francesco Bonacci7a82121900ea
diff --git a/libs/cua-driver/examples/jev-use/decision-models.md b/libs/cua-driver/examples/jev-use/decision-models.md
index f1dc89a6c3..4c6512eea6 100644
--- a/libs/cua-driver/examples/jev-use/decision-models.md
+++ b/libs/cua-driver/examples/jev-use/decision-models.md
@@ -130,8 +130,29 @@ request schema permits up to 32 candidates. The S1 multimodal adapter requires
an existing local screenshot path and an explicit screenshot capture ID equal
to the request's `capture_id`. The caller must supply the image from that
capture; the adapter checks identity but cannot authenticate image contents.
-The path and image are never added to a TypeSafe request. The CLI currently
-exposes only the text adapter.
+The path and image are never added to a TypeSafe request.
+
+`choose_decision.py --model s1` loads the text adapter by default. Select the
+multimodal adapter with `--s1-modality multimodal` (or `S1_MODALITY=multimodal`)
+and pass the capture's image with `--screenshot <png>` and
+`--screenshot-capture-id <capture_id>`; the chooser returns `error` when the
+capture ID differs from the request. `verify_decision_cli.py --model s1
+--screenshot <png>` runs the same check against a fixture, binding the image to
+the fixture's capture ID.
+
+The response's `model` field names the checkpoint as
+`<adapter>[@<revision>]:<modality>`, for example
+`cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79:text`. The adapter name
+and revision come from a Hugging Face cache snapshot path or from the metadata
+that `hf download --local-dir` writes; otherwise the directory name is used
+without a revision. Set `S1_ADAPTER_ID` (for example `cua-ai/cua-s1-4b-0.2`) and
+`S1_ADAPTER_REVISION` to state them explicitly. The base model is not part of
+this identity, so record `S1_BASE_MODEL_PATH`'s revision alongside the
+evidence.
+
+The chooser supports only Cua-S1-4B PEFT adapters (`cua-s1-4b-0.2` and
+`cua-s1-4b-0.1`). `cua-s1-nano-0.1` and `cua-s1-forms` are byte-level scorers
+with a different input format; `S1_ADAPTER_PATH` pointing at them fails setup.
## Verification scope
diff --git a/libs/cua-driver/examples/jev-use/python/choose_decision.py b/libs/cua-driver/examples/jev-use/python/choose_decision.py
index d208cd95d6..89268a2dc6 100644
--- a/libs/cua-driver/examples/jev-use/python/choose_decision.py
+++ b/libs/cua-driver/examples/jev-use/python/choose_decision.py
@@ -11,11 +11,13 @@ from typing import Any
from choose_action import MAX_INPUT_BYTES, validate_request
from decision_models import (
+ S1_MODALITIES,
DecisionRequest,
ModelScores,
S1DecisionModel,
TypeSafeDecisionModel,
choose,
+ s1_model_identity,
)
@@ -37,7 +39,24 @@ class MockDecisionModel:
)
-def local_s1_model() -> S1DecisionModel:
+def _resolve_s1_adapter(adapter: Path, modality: str) -> Path:
+ """Return the PEFT adapter directory the chooser will load for `modality`."""
+ for candidate in (adapter / modality, adapter):
+ if (candidate / "adapter_config.json").is_file():
+ return candidate
+ raise ValueError(
+ f"S1_ADAPTER_PATH has no Cua-S1-4B PEFT adapter for modality {modality!r} "
+ f"(expected {modality}/adapter_config.json or adapter_config.json); "
+ "cua-s1-nano and cua-s1-forms checkpoints are not supported by this chooser"
+ )
+
+
+def local_s1_model(
+ *,
+ modality: str | None = None,
+ screenshot: Path | None = None,
+ screenshot_capture_id: str | None = None,
+) -> S1DecisionModel:
base_value = os.environ["S1_BASE_MODEL_PATH"]
adapter_value = os.environ["S1_ADAPTER_PATH"]
if not base_value.strip() or not adapter_value.strip():
@@ -46,6 +65,27 @@ def local_s1_model() -> S1DecisionModel:
adapter = Path(adapter_value).expanduser()
if not base.is_dir() or not adapter.is_dir():
raise ValueError("S1_BASE_MODEL_PATH and S1_ADAPTER_PATH must be local directories")
+ modality = modality or os.environ.get("S1_MODALITY", "").strip() or "text"
+ if modality not in S1_MODALITIES:
+ raise ValueError(f"S1 modality must be one of {', '.join(S1_MODALITIES)}")
+ if modality == "multimodal":
+ if screenshot is None or not screenshot_capture_id:
+ raise ValueError(
+ "multimodal S1 requires --screenshot and --screenshot-capture-id "
+ "from the same capture as the request"
+ )
+ screenshot = screenshot.expanduser()
+ if not screenshot.is_file():
+ raise ValueError("--screenshot must be an existing local image file")
+ elif screenshot is not None or screenshot_capture_id is not None:
+ raise ValueError("--screenshot and --screenshot-capture-id require multimodal S1")
+ _resolve_s1_adapter(adapter, modality)
+ name = s1_model_identity(
+ adapter,
+ modality,
+ repo_id=os.environ.get("S1_ADAPTER_ID"),
+ revision=os.environ.get("S1_ADAPTER_REVISION"),
+ )
from cua_s1.four_b import FourBModel
return S1DecisionModel(
@@ -54,8 +94,12 @@ def local_s1_model() -> S1DecisionModel:
lora_adapter_path=adapter,
device=os.environ.get("S1_DEVICE", "cpu"),
dtype=os.environ.get("S1_DTYPE", "float16"),
- modality="text",
- )
+ modality=modality,
+ ),
+ modality=modality,
+ screenshot_path=screenshot,
+ screenshot_capture_id=screenshot_capture_id,
+ name=name,
)
@@ -67,7 +111,25 @@ def choose_request(value: Any, model: Any) -> dict[str, Any]:
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", choices=("mock", "jev", "s1"), default="mock")
+ parser.add_argument(
+ "--s1-modality",
+ choices=S1_MODALITIES,
+ help="S1 adapter to load (default: $S1_MODALITY, else text)",
+ )
+ parser.add_argument(
+ "--screenshot",
+ type=Path,
+ help="multimodal S1 only: local image from the request's capture",
+ )
+ parser.add_argument(
+ "--screenshot-capture-id",
+ help="multimodal S1 only: capture ID of --screenshot; must equal the request capture_id",
+ )
args = parser.parse_args()
+ if args.model != "s1" and (
+ args.s1_modality or args.screenshot is not None or args.screenshot_capture_id is not None
+ ):
+ parser.error("--s1-modality, --screenshot, and --screenshot-capture-id require --model s1")
raw = sys.stdin.buffer.read(MAX_INPUT_BYTES + 1)
if len(raw) > MAX_INPUT_BYTES:
raise SystemExit("request exceeds input limit")
@@ -80,7 +142,11 @@ def main() -> None:
model = MockDecisionModel()
elif args.model == "s1":
try:
- model = local_s1_model()
+ model = local_s1_model(
+ modality=args.s1_modality,
+ screenshot=args.screenshot,
+ screenshot_capture_id=args.screenshot_capture_id,
+ )
except (KeyError, ValueError, ImportError) as error:
raise SystemExit(f"S1 setup failed: {error}") from None
else:
diff --git a/libs/cua-driver/examples/jev-use/python/decision_models.py b/libs/cua-driver/examples/jev-use/python/decision_models.py
index a41908f581..8d2a2bc63a 100644
--- a/libs/cua-driver/examples/jev-use/python/decision_models.py
+++ b/libs/cua-driver/examples/jev-use/python/decision_models.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import math
import json
+import re
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
@@ -196,7 +197,81 @@ def visual_regions_as_text(request: DecisionRequest) -> str:
return "\n".join(lines)
+S1_MODALITIES = ("text", "multimodal")
+_HF_SNAPSHOT_PARENT = "snapshots"
+_REVISION_PATTERN = re.compile(r"[0-9a-f]{40}")
+_UNSAFE_IDENTITY_CHARS = re.compile(r"[^A-Za-z0-9._/-]")
+
+
+def _hf_local_dir_revision(root: Path, modality: str) -> str | None:
+ """Read the commit that `hf download --local-dir` recorded for the adapter.
+
+ `huggingface_hub` writes `<local-dir>/.cache/huggingface/download/<file>.metadata`
+ whose first line is the resolved commit hash. Only the selected modality's
+ adapter files are consulted, and they must agree.
+ """
+ metadata_dir = root / ".cache" / "huggingface" / "download"
+ revisions: set[str] = set()
+ for prefix in (Path(modality), Path()):
+ for name in ("adapter_config.json", "adapter_model.safetensors"):
+ candidate = metadata_dir / prefix / f"{name}.metadata"
+ try:
+ first_line = candidate.read_text(encoding="utf-8").splitlines()[0].strip()
+ except (OSError, IndexError, UnicodeDecodeError):
+ continue
+ if _REVISION_PATTERN.fullmatch(first_line):
+ revisions.add(first_line)
+ if revisions:
+ break
+ return revisions.pop() if len(revisions) == 1 else None
+
+
+def s1_model_identity(
+ adapter_path: Path,
+ modality: str,
+ *,
+ repo_id: str | None = None,
+ revision: str | None = None,
+) -> str:
+ """Name the S1 checkpoint that produced a decision, for evidence logs.
+
+ The result is `<adapter>[@<revision>]:<modality>`, for example
+ `cua-ai/cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79:text`.
+ `repo_id` and `revision` override detection. Otherwise the adapter name
+ and revision come from a Hugging Face cache snapshot path
+ (`models--<org>--<name>/snapshots/<commit>`), from the metadata that
+ `hf download --local-dir` leaves in the adapter directory, or, failing
+ both, from the adapter directory name with no revision. Detection never
+ claims a Hub repository or revision it cannot read locally.
+ """
+ if modality not in S1_MODALITIES:
+ raise ValueError(f"unknown S1 modality: {modality!r}")
+ root = Path(adapter_path).expanduser()
+ if root.name in S1_MODALITIES:
+ root = root.parent
+ label = repo_id.strip() if repo_id and repo_id.strip() else None
+ detected_revision: str | None = None
+ parts = root.parts
+ if len(parts) >= 3 and parts[-2] == _HF_SNAPSHOT_PARENT and parts[-3].startswith("models--"):
+ if label is None:
+ label = parts[-3].removeprefix("models--").replace("--", "/")
+ if _REVISION_PATTERN.fullmatch(parts[-1]):
+ detected_revision = parts[-1]
+ else:
+ detected_revision = _hf_local_dir_revision(root, modality)
+ if label is None:
+ label = root.name or "cua-s1-adapter"
+ chosen_revision = revision.strip() if revision and revision.strip() else detected_revision
+ label = _UNSAFE_IDENTITY_CHARS.sub("_", label)[:64]
+ identity = label
+ if chosen_revision:
+ identity += "@" + _UNSAFE_IDENTITY_CHARS.sub("_", chosen_revision)[:40]
+ return f"{identity}:{modality}"
+
+
class S1DecisionModel:
+ # Default identity for callers that construct the adapter directly without
+ # naming the checkpoint. `choose_decision.py` passes a derived identity.
name = "cua-s1-4b-local"
def __init__(
@@ -206,7 +281,12 @@ class S1DecisionModel:
modality: Literal["text", "multimodal"] = "text",
screenshot_path: Path | None = None,
screenshot_capture_id: str | None = None,
+ name: str | None = None,
) -> None:
+ if name is not None:
+ if not isinstance(name, str) or not 0 < len(name) <= 128:
+ raise ValueError("S1 model identity must be a bounded string")
+ self.name = name
self.scorer = scorer
self.modality = modality
self.screenshot_path = screenshot_path
diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_decision_models.py b/libs/cua-driver/examples/jev-use/python/tests/test_decision_models.py
index 9e699a0cc5..167e72db9e 100644
--- a/libs/cua-driver/examples/jev-use/python/tests/test_decision_models.py
+++ b/libs/cua-driver/examples/jev-use/python/tests/test_decision_models.py
@@ -12,8 +12,24 @@ from types import SimpleNamespace
BASE = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(BASE / "python"))
-from choose_decision import MockDecisionModel, choose_request
-from decision_models import S1DecisionModel, TypeSafeDecisionModel
+from choose_decision import MockDecisionModel, choose_request, local_s1_model
+from decision_models import S1DecisionModel, TypeSafeDecisionModel, s1_model_identity
+
+REVISION = "16818868b0cc7813808aae4e87b417657046ab79"
+OTHER_REVISION = "88d8b8a90c2da4470d005cc23ec8665a6442ebe1"
+
+
+def write_adapter(root: Path, modality: str, revision: str | None = None) -> None:
+ """Lay out a PEFT adapter like `hf download --local-dir` does."""
+ (root / modality).mkdir(parents=True, exist_ok=True)
+ (root / modality / "adapter_config.json").write_text("{}", encoding="utf-8")
+ if revision:
+ metadata = root / ".cache/huggingface/download" / modality
+ metadata.mkdir(parents=True, exist_ok=True)
+ for name in ("adapter_config.json", "adapter_model.safetensors"):
+ (metadata / f"{name}.metadata").write_text(
+ f"{revision}\n0123abcd\n1790377747.1\n", encoding="utf-8"
+ )
def request() -> dict:
@@ -246,6 +262,147 @@ class DecisionModelsTest(unittest.TestCase):
self.assertEqual(result.stdout, "")
self.assertIn("S1 setup failed", result.stderr)
+ def test_s1_identity_uses_hf_local_dir_revision_for_the_selected_modality(self) -> None:
+ with tempfile.TemporaryDirectory() as directory:
+ adapter = Path(directory) / "cua-s1-4b-0.2"
+ write_adapter(adapter, "text", REVISION)
+ write_adapter(adapter, "multimodal", REVISION)
+ self.assertEqual(s1_model_identity(adapter, "text"), f"cua-s1-4b-0.2@{REVISION}:text")
+ self.assertEqual(
+ s1_model_identity(adapter / "multimodal", "multimodal"),
+ f"cua-s1-4b-0.2@{REVISION}:multimodal",
+ )
+ self.assertEqual(
+ s1_model_identity(adapter, "text", repo_id="cua-ai/cua-s1-4b-0.2"),
+ f"cua-ai/cua-s1-4b-0.2@{REVISION}:text",
+ )
+
+ def test_s1_identity_omits_conflicting_or_missing_revisions(self) -> None:
+ with tempfile.TemporaryDirectory() as directory:
+ plain = Path(directory) / "my adapter"
+ write_adapter(plain, "text")
+ self.assertEqual(s1_model_identity(plain, "text"), "my_adapter:text")
+ mixed = Path(directory) / "mixed"
+ write_adapter(mixed, "text", REVISION)
+ metadata = mixed / ".cache/huggingface/download/text/adapter_model.safetensors.metadata"
+ metadata.write_text(f"{OTHER_REVISION}\n", encoding="utf-8")
+ self.assertEqual(s1_model_identity(mixed, "text"), "mixed:text")
+ self.assertEqual(s1_model_identity(mixed, "text", revision="v0.2"), "[email protected]:text")
+
+ def test_s1_identity_reads_hugging_face_cache_snapshots(self) -> None:
+ snapshot = Path("/cache/hub/models--cua-ai--cua-s1-4b-0.1/snapshots") / OTHER_REVISION
+ self.assertEqual(
+ s1_model_identity(snapshot, "multimodal"),
+ f"cua-ai/cua-s1-4b-0.1@{OTHER_REVISION}:multimodal",
+ )
+ self.assertEqual(
+ s1_model_identity(snapshot / "text", "text"),
+ f"cua-ai/cua-s1-4b-0.1@{OTHER_REVISION}:text",
+ )
+
+ def test_s1_identity_is_bounded_and_rejects_unknown_modality(self) -> None:
+ identity = s1_model_identity(Path("/x") / ("a" * 300), "text", revision="r" * 300)
+ self.assertLessEqual(len(identity), 128)
+ with self.assertRaises(ValueError):
+ s1_model_identity(Path("/x/adapter"), "audio")
+
+ def test_s1_adapter_reports_explicit_identity_and_keeps_the_default(self) -> None:
+ scores = {"submit-form": 1.0, "reobserve": 0.0, "abstain": 0.0}
+ named = S1DecisionModel(FakeScorer(scores), name=f"cua-s1-4b-0.2@{REVISION}:text")
+ self.assertEqual(
+ choose_request(request(), named)["model"], f"cua-s1-4b-0.2@{REVISION}:text"
+ )
+ self.assertEqual(S1DecisionModel(FakeScorer(scores)).name, "cua-s1-4b-local")
+ with self.assertRaises(ValueError):
+ S1DecisionModel(FakeScorer(scores), name="")
+
+ def _local_s1(self, env: dict[str, str], **kwargs):
+ previous = {name: os.environ.get(name) for name in env}
+ os.environ.update(env)
+ try:
+ return local_s1_model(**kwargs)
+ finally:
+ for name, value in previous.items():
+ if value is None:
+ os.environ.pop(name, None)
+ else:
+ os.environ[name] = value
+
+ def test_local_s1_selects_modality_and_binds_the_screenshot(self) -> None:
+ try:
+ import cua_s1.four_b # noqa: F401
+ except ImportError:
+ self.skipTest("cua-s1 is not installed")
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ base = root / "Qwen3.5-4B"
+ base.mkdir()
+ adapter = root / "cua-s1-4b-0.2"
+ write_adapter(adapter, "text", REVISION)
+ write_adapter(adapter, "multimodal", REVISION)
+ screenshot = root / "capture.png"
+ screenshot.write_bytes(b"fixture")
+ env = {
+ "S1_BASE_MODEL_PATH": str(base),
+ "S1_ADAPTER_PATH": str(adapter),
+ "S1_MODALITY": "",
+ "S1_ADAPTER_ID": "",
+ "S1_ADAPTER_REVISION": "",
+ }
+
+ text = self._local_s1(env)
+ self.assertEqual((text.modality, text.scorer.modality), ("text", "text"))
+ self.assertEqual(text.name, f"cua-s1-4b-0.2@{REVISION}:text")
+ self.assertEqual(text.scorer.lora_adapter_path, adapter)
+
+ multimodal = self._local_s1(
+ {**env, "S1_MODALITY": "multimodal"},
+ screenshot=screenshot,
+ screenshot_capture_id="capture-1",
+ )
+ self.assertEqual(multimodal.scorer.modality, "multimodal")
+ self.assertEqual(multimodal.screenshot_path, screenshot)
+ self.assertEqual(multimodal.screenshot_capture_id, "capture-1")
+ self.assertEqual(multimodal.name, f"cua-s1-4b-0.2@{REVISION}:multimodal")
+
+ with self.assertRaisesRegex(ValueError, "requires --screenshot"):
+ self._local_s1(env, modality="multimodal")
+ with self.assertRaisesRegex(ValueError, "require multimodal"):
+ self._local_s1(env, screenshot=screenshot, screenshot_capture_id="capture-1")
+ with self.assertRaisesRegex(ValueError, "modality must be one of"):
+ self._local_s1({**env, "S1_MODALITY": "audio"})
+
+ def test_local_s1_rejects_non_peft_checkpoints(self) -> None:
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ base = root / "base"
+ base.mkdir()
+ nano = root / "cua-s1-nano-0.1" / "text"
+ nano.mkdir(parents=True)
+ (nano / "model.safetensors").write_bytes(b"")
+ env = {"S1_BASE_MODEL_PATH": str(base), "S1_ADAPTER_PATH": str(nano.parent)}
+ with self.assertRaisesRegex(ValueError, "not supported by this chooser"):
+ self._local_s1(env)
+
+ def test_cli_rejects_s1_options_for_other_models(self) -> None:
+ result = subprocess.run(
+ [
+ sys.executable,
+ str(BASE / "python/choose_decision.py"),
+ "--model",
+ "mock",
+ "--s1-modality",
+ "multimodal",
+ ],
+ input=json.dumps(request()),
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ self.assertEqual(result.returncode, 2)
+ self.assertEqual(result.stdout, "")
+ self.assertIn("require --model s1", result.stderr)
+
def test_typesafe_request_contains_no_screenshot_or_action_arguments(self) -> None:
class FakeClient:
sent = None
diff --git a/libs/cua-driver/examples/jev-use/verify_decision_cli.py b/libs/cua-driver/examples/jev-use/verify_decision_cli.py
index c507c7c490..27f4b85304 100644
--- a/libs/cua-driver/examples/jev-use/verify_decision_cli.py
+++ b/libs/cua-driver/examples/jev-use/verify_decision_cli.py
@@ -14,7 +14,14 @@ def main() -> None:
parser.add_argument("--model", choices=("mock", "jev", "s1"), required=True)
parser.add_argument("--expected-id", default="submit-form")
parser.add_argument("--fixture", choices=("positive", "negative"), default="positive")
+ parser.add_argument(
+ "--screenshot",
+ type=Path,
+ help="run the multimodal S1 adapter with this image, bound to the fixture capture",
+ )
args = parser.parse_args()
+ if args.screenshot is not None and args.model != "s1":
+ parser.error("--screenshot requires --model s1")
root = Path(__file__).resolve().parent
fixture_name = (
@@ -23,8 +30,18 @@ def main() -> None:
else "jev-choice-negative-v1.json"
)
request = json.loads((root / "fixtures" / fixture_name).read_text(encoding="utf-8"))
+ command = [sys.executable, str(root / "python/choose_decision.py"), "--model", args.model]
+ if args.screenshot is not None:
+ command += [
+ "--s1-modality",
+ "multimodal",
+ "--screenshot",
+ str(args.screenshot),
+ "--screenshot-capture-id",
+ request["capture_id"],
+ ]
result = subprocess.run(
- [sys.executable, str(root / "python/choose_decision.py"), "--model", args.model],
+ command,
input=json.dumps(request),
text=True,
capture_output=True,