Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci(cua-s1): add a weights-backed S1 smoke and an S1 + OmniParser Driver E2E row (#4253)

Francesco Bonaccie1c44bd87f55
diff --git a/.github/scripts/tests/test_ci_cua_s1_weights_workflow.py b/.github/scripts/tests/test_ci_cua_s1_weights_workflow.py
new file mode 100644
index 0000000000..00e1b36836
--- /dev/null
+++ b/.github/scripts/tests/test_ci_cua_s1_weights_workflow.py
@@ -0,0 +1,200 @@
+"""Keep the weights-backed Cua-S1 checks pinned, verified, secret-free, and cache-free."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import re
+from pathlib import Path
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[3]
+WORKFLOW = ROOT / ".github/workflows/ci-cua-s1-weights.yml"
+LOCK = ROOT / "libs/cua-s1/ci/weights.lock.json"
+FETCH = ROOT / "libs/cua-s1/ci/fetch_pinned_weights.py"
+README = ROOT / "libs/cua-s1/README.md"
+LINUX_RUNNER = ROOT / "scripts/ci/linux/run-rust-e2e.sh"
+ROW = "perception_s1_decision_loop_test"
+
+
+def load() -> dict:
+ return yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))
+
+
+def fetch_module():
+ spec = importlib.util.spec_from_file_location("fetch_pinned_weights", FETCH)
+ module = importlib.util.module_from_spec(spec)
+ assert spec.loader is not None
+ spec.loader.exec_module(module)
+ return module
+
+
+def test_workflow_is_read_only_secret_free_and_never_caches_weights() -> None:
+ text = WORKFLOW.read_text(encoding="utf-8")
+ workflow = load()
+ assert workflow["permissions"] == {"contents": "read"}
+ assert "secrets." not in text
+ assert "HF_TOKEN" not in text
+ assert "pull_request_target" not in workflow[True]
+ # The weights exceed half of the repository's 10 GB Actions cache.
+ assert "actions/cache" not in text
+ for job in workflow["jobs"].values():
+ for step in job["steps"]:
+ if "setup-uv" in step.get("uses", ""):
+ assert step["with"]["cache-dependency-glob"] == "libs/cua-s1/python/uv.lock"
+
+
+def test_triggers_are_nightly_dispatch_and_path_filtered_pull_requests() -> None:
+ triggers = load()[True]
+ assert triggers["schedule"]
+ assert "workflow_dispatch" in triggers
+ assert "push" not in triggers
+ paths = triggers["pull_request"]["paths"]
+ for required in (
+ "libs/cua-s1/**",
+ "libs/cua-driver/examples/jev-use/python/choose_decision.py",
+ "libs/cua-driver/examples/jev-use/python/decision_models.py",
+ "libs/cua-driver/examples/jev-use/verify_decision_cli.py",
+ "libs/cua-driver/examples/jev-use/fixtures/jev-choice-*",
+ f"libs/cua-driver/rust/crates/cua-driver-e2e/tests/{ROW}.rs",
+ ".github/workflows/ci-cua-s1-weights.yml",
+ ):
+ assert required in paths, required
+
+
+def test_jobs_use_standard_hosted_linux_and_cpu_without_float32() -> None:
+ workflow = load()
+ for name, job in workflow["jobs"].items():
+ assert job["runs-on"] == "ubuntu-latest", name
+ options = workflow[True]["workflow_dispatch"]["inputs"]["dtype"]["options"]
+ assert "float32" not in options # 18.7 GB of float32 weights exceed 16 GB of RAM
+ assert workflow["jobs"]["smoke"]["env"]["S1_DEVICE"] == "cpu"
+ assert workflow["jobs"]["driver-e2e"]["env"]["S1_DEVICE"] == "cpu"
+
+
+def test_smoke_runs_the_documented_verifier_in_both_modalities() -> None:
+ steps = {step.get("name"): step for step in load()["jobs"]["smoke"]["steps"]}
+ documented = steps["Run the documented chooser smoke (cold process per decision)"]["run"]
+ assert "verify_decision_cli.py" in documented
+ assert "for modality in text multimodal" in documented
+ assert "--fixture negative --expected-id abstain" in documented
+ assert "jev-choice-request-v1.png" in documented
+ assert "jev-choice-negative-v1.png" in documented
+ assert "fetch_pinned_weights.py" in steps["Download and verify the pinned weights"]["run"]
+ assert "GITHUB_STEP_SUMMARY" in steps["Publish latency and memory summary"]["run"]
+
+
+def test_fixture_screenshots_exist_for_both_fixtures() -> None:
+ fixtures = ROOT / "libs/cua-driver/examples/jev-use/fixtures"
+ for stem in ("jev-choice-request-v1", "jev-choice-negative-v1"):
+ assert (fixtures / f"{stem}.json").is_file()
+ assert (fixtures / f"{stem}.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
+
+
+def test_lock_pins_match_the_documented_revisions() -> None:
+ fetch = fetch_module()
+ artifacts = {item["repo_id"]: item for item in fetch.load_lock(LOCK)}
+ assert set(artifacts) == {"Qwen/Qwen3.5-4B", "cua-ai/cua-s1-4b-0.2"}
+ readme = README.read_text(encoding="utf-8")
+ for repo_id, artifact in artifacts.items():
+ assert re.search(
+ rf"{re.escape(repo_id)}\s*\\?\n?\s*--revision {artifact['revision']}", readme
+ ), repo_id
+ base = artifacts["Qwen/Qwen3.5-4B"]["files"]
+ assert {
+ "model.safetensors-00001-of-00002.safetensors",
+ "model.safetensors-00002-of-00002.safetensors",
+ "config.json",
+ "tokenizer.json",
+ } <= set(base)
+ adapter = artifacts["cua-ai/cua-s1-4b-0.2"]["files"]
+ for modality in ("text", "multimodal"):
+ assert f"{modality}/adapter_config.json" in adapter
+ assert f"{modality}/adapter_model.safetensors" in adapter
+
+
+def test_fetch_rejects_unsafe_locks(tmp_path: Path) -> None:
+ fetch = fetch_module()
+ document = json.loads(LOCK.read_text(encoding="utf-8"))
+ document["artifacts"][1]["files"] = {"../escape": {"size": 1, "sha256": "0" * 64}}
+ unsafe = tmp_path / "lock.json"
+ unsafe.write_text(json.dumps(document), encoding="utf-8")
+ with pytest.raises(ValueError, match="unsafe file path"):
+ fetch.load_lock(unsafe)
+ document["artifacts"][1]["files"] = {"a": {"size": 1, "sha256": "0" * 64}}
+ document["artifacts"][1]["revision"] = "main"
+ unsafe.write_text(json.dumps(document), encoding="utf-8")
+ with pytest.raises(ValueError, match="full commit SHA"):
+ fetch.load_lock(unsafe)
+
+
+def test_verify_detects_missing_extra_and_tampered_files(tmp_path: Path) -> None:
+ fetch = fetch_module()
+ import hashlib
+
+ good = b"weights"
+ artifact = {
+ "name": "demo",
+ "files": {
+ "a.safetensors": {"size": len(good), "sha256": hashlib.sha256(good).hexdigest()},
+ "sub/b.json": {"size": 2, "sha256": hashlib.sha256(b"{}").hexdigest()},
+ },
+ }
+ (tmp_path / "sub").mkdir()
+ (tmp_path / "a.safetensors").write_bytes(good)
+ (tmp_path / "sub/b.json").write_bytes(b"{}")
+ (tmp_path / ".cache/huggingface").mkdir(parents=True)
+ (tmp_path / ".cache/huggingface/meta").write_text("ignored")
+ assert fetch.verify_artifact(tmp_path, artifact) == []
+
+ (tmp_path / "a.safetensors").write_bytes(b"weightz")
+ (tmp_path / "extra.bin").write_bytes(b"x")
+ (tmp_path / "sub/b.json").unlink()
+ problems = "\n".join(fetch.verify_artifact(tmp_path, artifact))
+ assert "a.safetensors: sha256" in problems
+ assert "sub/b.json: missing" in problems
+ assert "extra.bin: not in the lock" in problems
+
+
+def test_driver_row_runs_through_the_canonical_linux_runner_only_in_its_lane() -> None:
+ job = load()["jobs"]["driver-e2e"]
+ steps = {step.get("name"): step for step in job["steps"]}
+ row = steps["Run the S1 + OmniParser capture-loop row"]
+ assert row["env"]["CUA_E2E_INTERNAL_LANE"] == "s1-perception"
+ assert "scripts/ci/linux/run-rust-e2e.sh" in row["run"]
+ assert "xvfb-run" in row["run"]
+ download = steps["Download the published cua-perception release"]["run"]
+ assert "releases/download/${tag}" in download
+ assert "sha256sum --check --strict" in download
+ assert job["env"]["S1_MODALITY"] == "text"
+
+ runner = LINUX_RUNNER.read_text(encoding="utf-8")
+ assert f"--test {ROW}" in runner
+ lane = runner.split('if [[ "${SUITE}" == s1-perception ]]; then', 1)[1].split("\nfi\n", 1)[0]
+ assert ROW in lane
+ # The canonical lanes never select the model-backed row.
+ assert runner.count(ROW) == 1
+ for required in (
+ "CUA_E2E_PERCEPTION_CATALOG",
+ "CUA_E2E_PERCEPTION_VERSION",
+ "CUA_E2E_S1_PYTHON",
+ "S1_BASE_MODEL_PATH",
+ "S1_ADAPTER_PATH",
+ ):
+ assert required in runner
+
+
+def test_driver_row_records_a_typed_limitation_without_avx512_bf16() -> None:
+ steps = load()["jobs"]["driver-e2e"]["steps"]
+ names = [step.get("name") or step.get("uses") for step in steps]
+ check = steps[names.index("Check the runner CPU against the capture-lifetime budget")]
+ assert check["id"] == "cpu"
+ assert "avx512_bf16" in check["run"]
+ assert "cua-e2e-limitation-v1" in check["run"]
+ assert "perception-s1-decision-loop-limitation.json" in check["run"]
+ gated = steps[names.index(check["name"]) + 1 : names.index("Publish row summary")]
+ assert gated and all(step["if"] == "steps.cpu.outputs.supported == 'true'" for step in gated)
+ for name in ("Publish row summary", "Upload row evidence"):
+ assert steps[names.index(name)]["if"] == "always()"
diff --git a/.github/workflows/ci-cua-s1-weights.yml b/.github/workflows/ci-cua-s1-weights.yml
new file mode 100644
index 0000000000..d4f34b3924
--- /dev/null
+++ b/.github/workflows/ci-cua-s1-weights.yml
@@ -0,0 +1,450 @@
+name: "CI: cua-s1 weights"
+
+# Weights-backed Cua-S1-4B checks on a standard GitHub-hosted Linux runner.
+# The pinned public weights (~9.5 GB, no token) are downloaded from Hugging
+# Face on every run and verified against libs/cua-s1/ci/weights.lock.json.
+# They are deliberately not stored in the Actions cache: the repository's
+# 10 GB cache limit would evict every other workflow's cache.
+on:
+ schedule:
+ - cron: "37 4 * * *"
+ workflow_dispatch:
+ inputs:
+ dtype:
+ # bfloat16 uses the runner's AVX512-BF16; float16 has no native CPU
+ # support there and is about 8x slower. float32 weights (18.7 GB)
+ # do not fit in the runner's 16 GB of memory.
+ description: "Torch dtype for CPU inference"
+ required: false
+ type: choice
+ default: bfloat16
+ options:
+ - bfloat16
+ - float16
+ pull_request:
+ paths:
+ - "libs/cua-s1/**"
+ - "libs/cua-driver/examples/jev-use/python/choose_action.py"
+ - "libs/cua-driver/examples/jev-use/python/choose_decision.py"
+ - "libs/cua-driver/examples/jev-use/python/decision_models.py"
+ - "libs/cua-driver/examples/jev-use/python/jev_adapter.py"
+ - "libs/cua-driver/examples/jev-use/verify_decision_cli.py"
+ - "libs/cua-driver/examples/jev-use/fixtures/jev-choice-*"
+ - "libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_s1_decision_loop_test.rs"
+ - ".github/workflows/ci-cua-s1-weights.yml"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ci-cua-s1-weights-${{ github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+ CUA_TELEMETRY_ENABLED: "false"
+ HF_HUB_DISABLE_TELEMETRY: "1"
+ PYTHONHASHSEED: "0"
+ WANDB_MODE: disabled
+
+jobs:
+ smoke:
+ name: S1 chooser smoke with pinned weights (CPU, text + multimodal)
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+ env:
+ S1_DEVICE: cpu
+ S1_DTYPE: ${{ inputs.dtype || 'bfloat16' }}
+ defaults:
+ run:
+ shell: bash
+ working-directory: libs/cua-s1/python
+ steps:
+ - name: Check out Cua
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+ - name: Free disk space for the weights and the CUDA-enabled torch wheel
+ working-directory: .
+ run: |
+ set -euo pipefail
+ df -h /
+ # Preinstalled toolchains this job never uses.
+ sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup \
+ /opt/hostedtoolcache/CodeQL /usr/local/share/boost /usr/share/swift
+ sudo docker image prune --all --force >/dev/null 2>&1 || true
+ df -h /
+ {
+ echo "S1_MODELS=$RUNNER_TEMP/s1-models"
+ echo "S1_EVIDENCE=$RUNNER_TEMP/s1-evidence"
+ } >> "$GITHUB_ENV"
+ mkdir -p "$RUNNER_TEMP/s1-models" "$RUNNER_TEMP/s1-evidence"
+
+ - name: Record runner hardware
+ working-directory: .
+ run: |
+ set -euo pipefail
+ {
+ echo "cpu_model=$(lscpu | sed -n 's/^Model name: *//p' | head -n1)"
+ echo "nproc=$(nproc)"
+ echo "mem_total_mb=$(free -m | awk '/^Mem:/ {print $2}')"
+ echo "mem_available_mb=$(free -m | awk '/^Mem:/ {print $7}')"
+ flags="$(grep -m1 '^flags' /proc/cpuinfo)"
+ for flag in avx2 avx512f avx512_bf16 amx_bf16 avx512_fp16; do
+ if grep -qw "$flag" <<<"$flags"; then echo "$flag=yes"; else echo "$flag=no"; fi
+ done
+ echo "disk_free=$(df -h / | awk 'NR==2 {print $4}')"
+ } | tee "$S1_EVIDENCE/runner.env"
+
+ - name: Set up uv and Python
+ uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+ with:
+ python-version: "3.12"
+ enable-cache: true
+ cache-dependency-glob: libs/cua-s1/python/uv.lock
+
+ - name: Install the pinned four-b inference environment
+ run: uv sync --frozen --extra four-b
+
+ - name: Download and verify the pinned weights
+ run: |
+ set -euo pipefail
+ .venv/bin/python ../ci/fetch_pinned_weights.py --dest "$S1_MODELS" \
+ | tee "$S1_EVIDENCE/weights.json"
+ du -sh "$S1_MODELS"/*
+ df -h /
+
+ - name: Run the documented chooser smoke (cold process per decision)
+ env:
+ HF_HUB_OFFLINE: "1"
+ TRANSFORMERS_OFFLINE: "1"
+ run: |
+ set -euo pipefail
+ export S1_BASE_MODEL_PATH="$S1_MODELS/Qwen3.5-4B"
+ export S1_ADAPTER_PATH="$S1_MODELS/cua-s1-4b-0.2"
+ jev=../../cua-driver/examples/jev-use
+ : > "$S1_EVIDENCE/cold.jsonl"
+ for modality in text multimodal; do
+ for fixture in positive negative; do
+ args=(--model s1)
+ if [[ "$fixture" == negative ]]; then
+ args+=(--fixture negative --expected-id abstain)
+ png="$jev/fixtures/jev-choice-negative-v1.png"
+ else
+ png="$jev/fixtures/jev-choice-request-v1.png"
+ fi
+ if [[ "$modality" == multimodal ]]; then
+ args+=(--screenshot "$png")
+ fi
+ echo "::group::$modality $fixture"
+ timing="$S1_EVIDENCE/cold-$modality-$fixture.time"
+ status=0
+ /usr/bin/time -v -o "$timing" \
+ .venv/bin/python "$jev/verify_decision_cli.py" "${args[@]}" \
+ > "$S1_EVIDENCE/cold-$modality-$fixture.out" || status=$?
+ cat "$S1_EVIDENCE/cold-$modality-$fixture.out"
+ wall="$(sed -n 's/.*Elapsed (wall clock) time (h:mm:ss or m:ss): //p' "$timing")"
+ rss_kb="$(sed -n 's/.*Maximum resident set size (kbytes): //p' "$timing")"
+ jq -nc --arg modality "$modality" --arg fixture "$fixture" \
+ --arg wall "$wall" --argjson rss_kb "${rss_kb:-0}" --argjson status "$status" \
+ --arg out "$(cat "$S1_EVIDENCE/cold-$modality-$fixture.out")" \
+ '{modality: $modality, fixture: $fixture, exit: $status, wall: $wall,
+ peak_rss_mb: ($rss_kb / 1024 | floor), output: $out}' \
+ >> "$S1_EVIDENCE/cold.jsonl"
+ echo "::endgroup::"
+ if [[ "$status" != 0 ]]; then
+ echo "::error::documented S1 smoke failed: $modality $fixture"
+ exit 1
+ fi
+ grep -Fq '@16818868b0cc7813808aae4e87b417657046ab79:'"$modality" \
+ "$S1_EVIDENCE/cold-$modality-$fixture.out"
+ done
+ done
+
+ - name: Measure warm per-decision latency
+ env:
+ HF_HUB_OFFLINE: "1"
+ TRANSFORMERS_OFFLINE: "1"
+ run: |
+ set -euo pipefail
+ export S1_BASE_MODEL_PATH="$S1_MODELS/Qwen3.5-4B"
+ export S1_ADAPTER_PATH="$S1_MODELS/cua-s1-4b-0.2"
+ for modality in text multimodal; do
+ echo "::group::warm $modality"
+ # Multimodal decisions take minutes on runners without AVX512-BF16.
+ repeats=3
+ [[ "$modality" == multimodal ]] && repeats=1
+ S1_MODALITY="$modality" /usr/bin/time -v -o "$S1_EVIDENCE/warm-$modality.time" \
+ .venv/bin/python ../ci/warm_chooser.py --smoke --repeats "$repeats" \
+ > "$S1_EVIDENCE/warm-$modality.json"
+ cat "$S1_EVIDENCE/warm-$modality.json"
+ echo "::endgroup::"
+ done
+
+ - name: Publish latency and memory summary
+ if: always()
+ run: |
+ python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY"
+ import json, os, statistics
+ from pathlib import Path
+
+ evidence = Path(os.environ["S1_EVIDENCE"])
+ def read(name):
+ path = evidence / name
+ return path.read_text() if path.exists() else ""
+ runner = dict(line.split("=", 1) for line in read("runner.env").splitlines() if "=" in line)
+ print(f"## Cua-S1-4B weights smoke ({os.environ['S1_DEVICE']}, {os.environ['S1_DTYPE']})\n")
+ print(f"Runner: {runner.get('cpu_model', '?')}, {runner.get('nproc', '?')} vCPU, "
+ f"{runner.get('mem_total_mb', '?')} MB RAM; avx512_bf16={runner.get('avx512_bf16', '?')}, "
+ f"amx_bf16={runner.get('amx_bf16', '?')}\n")
+ weights = read("weights.json")
+ if weights:
+ for artifact in json.loads(weights)["artifacts"]:
+ print(f"- `{artifact['repo_id']}@{artifact['revision'][:12]}`: {artifact['files']} files, "
+ f"{artifact['bytes'] / 1e9:.2f} GB, downloaded in {artifact['download_s']} s, "
+ f"verified={artifact['verified']}")
+ print()
+ cold = [json.loads(line) for line in read("cold.jsonl").splitlines() if line.strip()]
+ if cold:
+ print("### Documented smoke (`verify_decision_cli.py --model s1`, one cold process per decision)\n")
+ print("| Modality | Fixture | Result | Wall (load + decision) | Peak RSS |")
+ print("| --- | --- | --- | --- | --- |")
+ for row in cold:
+ result = row["output"] if row["exit"] == 0 else f"exit {row['exit']}"
+ print(f"| {row['modality']} | {row['fixture']} | `{result}` | {row['wall']} | {row['peak_rss_mb']} MB |")
+ print()
+ print("### Warm decisions (model resident; `libs/cua-s1/ci/warm_chooser.py --smoke`)\n")
+ print("| Modality | Load | Fixture | Choice | Latency per decision (ms) | Peak RSS |")
+ print("| --- | --- | --- | --- | --- | --- |")
+ for modality in ("text", "multimodal"):
+ raw = read(f"warm-{modality}.json").strip()
+ if not raw:
+ print(f"| {modality} | not run | | | | |")
+ continue
+ report = json.loads(raw.splitlines()[-1])
+ for fixture in ("positive", "negative"):
+ rows = [d for d in report["decisions"] if d["fixture"] == fixture]
+ latencies = [d["latency_ms"] for d in rows if "latency_ms" in d]
+ choice = ", ".join(sorted({f"{d.get('selected_id')} ({'ok' if d['ok'] else 'FAIL'})" for d in rows}))
+ median = statistics.median(latencies) if latencies else "n/a"
+ print(f"| {modality} | {report['load_s']} s | {fixture} | {choice} | "
+ f"median {median} (all: {latencies}) | {report['peak_rss_mb']} MB |")
+ PY
+
+ - name: Upload smoke evidence
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: cua-s1-weights-smoke
+ path: ${{ runner.temp }}/s1-evidence
+ if-no-files-found: ignore
+ retention-days: 14
+
+ driver-e2e:
+ # Canonical Linux X11 row for the model-backed perception loop: Driver
+ # captures the Electron harness, the published cua-perception extension
+ # parses the capture, Cua-S1-4B chooses one region, Driver clicks it with
+ # the same capture_id inside the 60 s lifetime, the fixture journal
+ # verifies the effect, and stale captures are refused. It runs here, not
+ # in e2e-rust-linux.yml, because it needs the 9.5 GB of pinned weights.
+ name: Linux X11 / S1 + OmniParser capture-loop row
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+ env:
+ CUA_E2E_SOURCE_SHA: ${{ github.sha }}
+ CUA_E2E_PERCEPTION_VERSION: "0.2.1"
+ # SHA-256 of cua-perception-0.2.1-x86_64-unknown-linux-gnu.tar.gz as
+ # published; the Driver also verifies the signed catalog.
+ PERCEPTION_ARCHIVE_SHA256: b4d76b626df9ce1a48b8036f7313412525295d0cb818d82ecbe3d9cca5fd1d08
+ S1_DEVICE: cpu
+ S1_DTYPE: ${{ inputs.dtype || 'bfloat16' }}
+ S1_MODALITY: text
+ defaults:
+ run:
+ shell: bash
+ steps:
+ - name: Check out Cua
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+ - name: Check the runner CPU against the capture-lifetime budget
+ id: cpu
+ run: |
+ set -euo pipefail
+ # Hosted ubuntu-latest VMs vary: some expose AVX512-BF16, some hide
+ # AVX-512 entirely. Without native BF16 a warm Cua-S1-4B text
+ # decision took 50-67 s for the 3-candidate fixture (5.5-7.3 s
+ # with it), so the row cannot decide inside the 60 s capture
+ # lifetime. Record that as a typed limitation instead of a flaky
+ # failure or a vacuous pass; the smoke job still runs.
+ artifacts=artifacts/cua-driver/linux
+ mkdir -p "$artifacts"
+ flags="$(grep -m1 '^flags' /proc/cpuinfo)"
+ cpu_model="$(lscpu | sed -n 's/^Model name: *//p' | head -n1)"
+ echo "cpu_model=$cpu_model nproc=$(nproc) mem_total_mb=$(free -m | awk '/^Mem:/ {print $2}')" \
+ | tee "$artifacts/runner.txt"
+ if grep -qw avx512_bf16 <<<"$flags"; then
+ echo "supported=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "supported=false" >> "$GITHUB_OUTPUT"
+ reason="This hosted runner ($cpu_model) does not expose AVX512-BF16; CPU bfloat16 Cua-S1-4B decisions take longer than the 60 s capture lifetime there."
+ jq -n --arg reason "$reason" '{
+ schema: "cua-e2e-limitation-v1",
+ platform: "linux",
+ display_server: "x11",
+ harness: "electron",
+ test: "perception-s1-decision-loop",
+ status: "not_covered",
+ reason: $reason
+ }' > "$artifacts/perception-s1-decision-loop-limitation.json"
+ echo "::warning title=S1 row not run::$reason Re-run the job to draw another runner."
+ fi
+
+ - name: Free disk space for the weights, Rust build, and extension
+ if: steps.cpu.outputs.supported == 'true'
+ run: |
+ set -euo pipefail
+ df -h /
+ sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup \
+ /opt/hostedtoolcache/CodeQL /usr/local/share/boost /usr/share/swift
+ sudo docker image prune --all --force >/dev/null 2>&1 || true
+ df -h /
+ {
+ echo "S1_MODELS=$RUNNER_TEMP/s1-models"
+ echo "S1_BASE_MODEL_PATH=$RUNNER_TEMP/s1-models/Qwen3.5-4B"
+ echo "S1_ADAPTER_PATH=$RUNNER_TEMP/s1-models/cua-s1-4b-0.2"
+ echo "CUA_E2E_S1_PYTHON=$GITHUB_WORKSPACE/libs/cua-s1/python/.venv/bin/python"
+ echo "PERCEPTION_DIR=$RUNNER_TEMP/cua-perception"
+ } >> "$GITHUB_ENV"
+
+ - uses: dtolnay/rust-toolchain@4716b85f2fac3e324e64fa2810f6b5c3905760a5 # 1.97.1
+ if: steps.cpu.outputs.supported == 'true'
+ - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
+ if: steps.cpu.outputs.supported == 'true'
+ with:
+ workspaces: "libs/cua-driver/rust -> target"
+ key: e2e-s1-perception
+ save-if: ${{ github.event_name == 'schedule' }}
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+ if: steps.cpu.outputs.supported == 'true'
+ with:
+ node-version: "22"
+
+ - name: Install GUI and Rust build dependencies
+ if: steps.cpu.outputs.supported == 'true'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends \
+ xvfb dbus-x11 at-spi2-core openbox picom python3-gi gir1.2-gtk-3.0 \
+ libgtk-3-dev clang pkg-config libdbus-1-dev libpipewire-0.3-dev \
+ libspa-0.2-dev libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev \
+ libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \
+ libayatana-appindicator3-dev librsvg2-dev ffmpeg jq
+
+ - name: Set up uv and Python
+ if: steps.cpu.outputs.supported == 'true'
+ uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+ with:
+ python-version: "3.12"
+ enable-cache: true
+ cache-dependency-glob: libs/cua-s1/python/uv.lock
+
+ - name: Install the pinned four-b inference environment
+ if: steps.cpu.outputs.supported == 'true'
+ working-directory: libs/cua-s1/python
+ run: uv sync --frozen --extra four-b
+
+ - name: Download and verify the pinned weights
+ if: steps.cpu.outputs.supported == 'true'
+ run: |
+ set -euo pipefail
+ "$CUA_E2E_S1_PYTHON" libs/cua-s1/ci/fetch_pinned_weights.py --dest "$S1_MODELS"
+ df -h /
+
+ - name: Download the published cua-perception release
+ if: steps.cpu.outputs.supported == 'true'
+ run: |
+ set -euo pipefail
+ tag="cua-perception-v${CUA_E2E_PERCEPTION_VERSION}"
+ stem="cua-perception-${CUA_E2E_PERCEPTION_VERSION}-x86_64-unknown-linux-gnu"
+ base="https://github.com/${GITHUB_REPOSITORY}/releases/download/${tag}"
+ mkdir -p "$PERCEPTION_DIR"
+ cd "$PERCEPTION_DIR"
+ for asset in "${stem}.catalog.json" "${stem}.tar.gz" SHA256SUMS; do
+ curl --fail --silent --show-error --location --retry 3 -o "$asset" "$base/$asset"
+ done
+ grep -E " \*?(${stem}\.catalog\.json|${stem}\.tar\.gz)$" SHA256SUMS | sha256sum --check --strict
+ echo "${PERCEPTION_ARCHIVE_SHA256} ${stem}.tar.gz" | sha256sum --check --strict
+ echo "CUA_E2E_PERCEPTION_CATALOG=$PERCEPTION_DIR/${stem}.catalog.json" >> "$GITHUB_ENV"
+
+ - name: Run the S1 + OmniParser capture-loop row
+ if: steps.cpu.outputs.supported == 'true'
+ env:
+ CUA_E2E_INTERNAL_LANE: s1-perception
+ run: |
+ set -euo pipefail
+ # The row's anonymous memory (resident model ~8.4 GB, Driver,
+ # Electron, OmniParser worker, recorder) fits in 16 GB. With swap
+ # enabled, the kernel swaps the resident model out while it copies
+ # the weights, and the next decisions page it back in from disk.
+ sudo swapoff -a
+ free -m
+ vmstat -S M 5 > "$RUNNER_TEMP/vmstat.log" &
+ vmstat_pid=$!
+ # Top CPU and memory consumers, to explain any latency outlier.
+ while sleep 15; do
+ date -u +%H:%M:%S
+ ps -eo pcpu,rss,comm --sort=-pcpu | head -n 8
+ done > "$RUNNER_TEMP/top.log" 2>&1 &
+ top_pid=$!
+ trap 'kill "$vmstat_pid" "$top_pid" 2>/dev/null || true' EXIT
+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \
+ dbus-run-session -- bash -lc \
+ "openbox >/tmp/cua-openbox.log 2>&1 & picom --backend xrender --config /dev/null >/tmp/cua-picom.log 2>&1 & sleep 2; scripts/ci/linux/run-rust-e2e.sh"
+
+ - name: Publish row summary
+ if: always()
+ run: |
+ set -euo pipefail
+ artifacts=artifacts/cua-driver/linux
+ {
+ echo "## S1 + OmniParser capture-loop row (Linux X11, S1 ${S1_MODALITY} on ${S1_DEVICE}/${S1_DTYPE})"
+ echo
+ if [[ -f "$artifacts/s1-perception/timing.json" ]]; then
+ jq -r '
+ "| Measure | Value |", "| --- | --- |",
+ "| Extension | cua-perception \(.extension.version) (\(.extension.trust)) |",
+ "| S1 checkpoint | `\(.s1.model)` (load \(.s1.load_s) s, resident copy \(.s1.resident_s) s, warm-up decision \(.s1.warmup_ms) ms, anonymous RSS \(.s1.rss_anon_mb) MB) |",
+ "| Regions parsed / candidates | \(.regions_parsed) / \(.candidates) |",
+ "| S1 choice | `\(.selected_id)` \(.selected_text | tojson) (confidence \(.confidence)) |",
+ "| Parse (warm worker) | \(.parse_ms) ms (first parse \(.warm_parse_ms) ms) |",
+ "| S1 decision | \(.s1_decision_ms) ms |",
+ "| Capture age at click | \(.capture_age_at_click_ms) ms of 60000 |",
+ "| Click | \(.click_ms) ms, route \(.click_route) |",
+ "| Fixture oracle | \(.fixture_counter) |"
+ ' "$artifacts/s1-perception/timing.json"
+ elif [[ -f "$artifacts/perception-s1-decision-loop-limitation.json" ]]; then
+ echo "**Not run:** $(jq -r .reason "$artifacts/perception-s1-decision-loop-limitation.json")"
+ else
+ echo "No timing record was produced."
+ fi
+ echo
+ echo "Runner: $(cat "$artifacts/runner.txt" 2>/dev/null || echo unknown)"
+ if [[ -f "$RUNNER_TEMP/vmstat.log" ]]; then
+ min_free="$(awk 'NR>2 {avail=$4+$5+$6; if (min=="" || avail<min) min=avail} END {print min}' "$RUNNER_TEMP/vmstat.log")"
+ echo
+ echo "Lowest free + buffer + cache memory during the row: ${min_free} MB (vmstat, 5 s samples)."
+ fi
+ echo
+ if [[ -f "$artifacts/summary.md" ]]; then
+ cat "$artifacts/summary.md"
+ fi
+ } >> "$GITHUB_STEP_SUMMARY"
+ cp "$RUNNER_TEMP/vmstat.log" "$RUNNER_TEMP/top.log" "$artifacts/" 2>/dev/null || true
+
+ - name: Upload row evidence
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: cua-s1-perception-linux-x11
+ path: artifacts/cua-driver/linux
+ if-no-files-found: ignore
+ compression-level: 0
+ retention-days: 14
diff --git a/.github/workflows/ci-cua-s1.yml b/.github/workflows/ci-cua-s1.yml
index 6648da0219..c5df69db16 100644
--- a/.github/workflows/ci-cua-s1.yml
+++ b/.github/workflows/ci-cua-s1.yml
@@ -54,8 +54,8 @@ jobs:
- name: Check formatting and lint
run: |
- uv run --frozen ruff format --check src tests ../evals ../training
- uv run --frozen ruff check src tests ../evals ../training
+ uv run --frozen ruff format --check src tests ../evals ../training ../ci
+ uv run --frozen ruff check src tests ../evals ../training ../ci
- name: Check package imports
run: |
diff --git a/.github/workflows/ci-test-scripts.yml b/.github/workflows/ci-test-scripts.yml
index 0236318069..1e1fd49a36 100644
--- a/.github/workflows/ci-test-scripts.yml
+++ b/.github/workflows/ci-test-scripts.yml
@@ -58,6 +58,8 @@ on:
- "scripts/ci/**"
- "scripts/ci/macos/**"
- "libs/cua-driver/tests/perception-demo/**"
+ - "libs/cua-s1/ci/**"
+ - "libs/cua-s1/README.md"
# Release-wiring, channel, and installer tests read these inputs.
- ".github/release-notes/**"
- ".github/release-state/**"
diff --git a/libs/cua-driver/docs/test-harnesses-guide.md b/libs/cua-driver/docs/test-harnesses-guide.md
index 79d004d414..d567bbf0d4 100644
--- a/libs/cua-driver/docs/test-harnesses-guide.md
+++ b/libs/cua-driver/docs/test-harnesses-guide.md
@@ -639,5 +639,6 @@ Perception tests have four owners, and each tests different code:
| `libs/cua-driver/tests/perception-demo/` | Visual demo evidence sanitizing, caching, and envelopes | `ci-test-scripts.yml` |
| `.github/scripts/tests/test_cua_perception_*.py`, `test_perception_release.py` | Perception release, review-trigger, and review-pipeline workflows and `.github/scripts` helpers | `ci-test-scripts.yml`, `ci-cua-perception-release.yml` |
| `libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_capture_loop_test.rs` | Driver's desktop capture, parse, capture-bound click, and reobserve loop with a deterministic developer-only worker | Capture lane of the canonical macOS, Windows, and Linux X11 desktop E2E |
+| `libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_s1_decision_loop_test.rs` | The same loop with the published extension (OmniParser) and a resident Cua-S1-4B choosing the region, plus consumed and expired capture refusals | `s1-perception` lane of the Linux X11 runner, selected only by `ci-cua-s1-weights.yml` |
Add a perception test to the directory that owns the code it exercises.
diff --git a/libs/cua-driver/docs/test-matrix.md b/libs/cua-driver/docs/test-matrix.md
index 5b3f4cba21..e8dd7cc538 100644
--- a/libs/cua-driver/docs/test-matrix.md
+++ b/libs/cua-driver/docs/test-matrix.md
@@ -120,6 +120,35 @@ instance of the installed app with an isolated extension home and socket.
Wayland lanes record a coverage limitation instead of running the row. Model
quality belongs to the `cua-perception` crate and its artifact-gated lanes.
+## Harness E2E: Model-Backed Perception Decision Loop
+
+`perception_s1_decision_loop_test.rs` runs the same capture-bound loop with
+real models, in the `s1-perception` lane of the Linux X11 runner
+(`scripts/ci/linux/run-rust-e2e.sh`). Only
+`.github/workflows/ci-cua-s1-weights.yml` selects that lane (nightly, manual
+dispatch, and pull requests that touch Cua-S1, the jev-use chooser, or the
+row), because it needs the pinned Cua-S1-4B weights:
+
+| Cell | Coverage |
+| --- | --- |
+| `linux-electron-perception-s1-capture-click-px-foreground` | The published `cua-perception` release, installed from its signed catalog as `publisher-verified`, parses a retained Electron window capture with OmniParser; a resident Cua-S1-4B (text adapter, CPU) chooses one of the parsed text regions for the goal "Increment the counter once."; Driver clicks the region center once with the same `capture_id` within the 60-second capture lifetime; the fixture journal must reach `counter=1`; the consumed capture is refused with `capture_not_found`, an unused capture past its lifetime is refused with `capture_expired`, and the counter stays at 1 |
+
+The model runs on CPU in `bfloat16`, which fits the capture lifetime only on
+hosted runners that expose AVX512-BF16. With 16 offered regions, decisions
+took 20 to 38 s and the click landed 23 to 42 s after the capture, so the row
+offers at most 12. The same
+`ubuntu-latest` label sometimes lands on a VM without AVX-512, where one text
+decision takes over 50 s. On such a runner the job skips the row, writes a
+`cua-e2e-limitation-v1` record (`perception-s1-decision-loop-limitation.json`),
+and warns; re-run the job to draw another runner.
+
+The row writes the parse, the `cua.jev_choice_request_v1` request, the S1
+decision, and a timing record under `artifacts/cua-driver/linux/s1-perception/`
+next to the trajectory video. macOS and Windows lanes do not run it: the
+hosted macOS arm64 runner has 7 GB of memory, less than the 9.3 GB of 16-bit
+base weights, and the Windows lane has no provisioned weights. Their
+capture-bound loop is covered by the model-free row above.
+
## Harness E2E: Native Windows
Windows native harnesses are repo-local applications built from source:
diff --git a/libs/cua-driver/examples/jev-use/fixtures/jev-choice-negative-v1.png b/libs/cua-driver/examples/jev-use/fixtures/jev-choice-negative-v1.png
new file mode 100644
index 0000000000..f219275c5f
Binary files /dev/null and b/libs/cua-driver/examples/jev-use/fixtures/jev-choice-negative-v1.png differ
diff --git a/libs/cua-driver/examples/jev-use/fixtures/jev-choice-request-v1.png b/libs/cua-driver/examples/jev-use/fixtures/jev-choice-request-v1.png
new file mode 100644
index 0000000000..ca373dff4a
Binary files /dev/null and b/libs/cua-driver/examples/jev-use/fixtures/jev-choice-request-v1.png differ
diff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_s1_decision_loop_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_s1_decision_loop_test.rs
new file mode 100644
index 0000000000..b8e34ea28e
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_s1_decision_loop_test.rs
@@ -0,0 +1,712 @@
+//! Canonical desktop row for the model-backed perception decision loop.
+//!
+//! The row drives the complete loop that `jev-use` documents for a local
+//! decision model, with real models at every step:
+//!
+//! 1. Cua Driver captures the Electron web harness window and retains the
+//! capture (`get_window_state` -> `capture_id`).
+//! 2. The published, publisher-verified `cua-perception` extension, installed
+//! from its signed release catalog, parses that exact capture with
+//! OmniParser (`parse_visual_regions`).
+//! 3. Cua-S1-4B, loaded once from the pinned public weights and kept resident
+//! (`libs/cua-s1/ci/warm_chooser.py`), chooses one closed candidate built
+//! from the parsed text regions.
+//! 4. Driver clicks the chosen region's center once with the same
+//! `capture_id`, inside the 60-second capture lifetime.
+//! 5. The fixture's loopback journal, which the Driver never reads, proves the
+//! effect (`counter=1`).
+//! 6. Stale captures are refused without effect: the consumed capture returns
+//! `capture_not_found`, and an unused capture older than the lifetime
+//! returns `capture_expired`. The counter stays at 1.
+//!
+//! The row needs about 10 GB of free memory for the model, so it runs only in
+//! the weights-backed Linux X11 lane (`.github/workflows/ci-cua-s1-weights.yml`
+//! through `scripts/ci/linux/run-rust-e2e.sh` with the `s1-perception` lane).
+//! That lane supplies:
+//!
+//! - `CUA_E2E_PERCEPTION_CATALOG`: the release catalog JSON, with its archive
+//! next to it;
+//! - `CUA_E2E_PERCEPTION_VERSION`: the extension version the catalog names;
+//! - `CUA_E2E_S1_PYTHON`: a Python with the `cua-s1[four-b]` environment;
+//! - `S1_BASE_MODEL_PATH`, `S1_ADAPTER_PATH`, `S1_DEVICE`, `S1_DTYPE`, and
+//! optionally `S1_MODALITY`, exactly as the jev-use chooser reads them.
+//!
+//! ```text
+//! cargo test -p cua-driver-e2e --test perception_s1_decision_loop_test -- \
+//! --ignored --nocapture --test-threads=1
+//! ```
+
+#![cfg(target_os = "linux")]
+
+use std::collections::HashSet;
+use std::fs;
+use std::io::{BufRead, BufReader, Write};
+use std::path::{Path, PathBuf};
+use std::process::{Child, ChildStdin, Command, Stdio};
+use std::sync::mpsc::{self, Receiver};
+use std::thread;
+use std::time::{Duration, Instant};
+
+use cua_driver_testkit::e2e::{
+ execute_case, recording_evidence, shared_web_route, CaseSpec, Delivery, DisplayServer,
+ Evidence, Observation, OracleKind, Platform, Scope, Targeting,
+};
+use cua_driver_testkit::{
+ driver_binary, harness_app, spawn_in_job, Driver, FixtureJournal, McpDriver, ToolResponse,
+};
+use serde_json::{json, Value};
+
+const FIXTURE_TITLE: &str = "CuaTestHarness Electron";
+const GOAL: &str = "Increment the counter once.";
+/// Driver's capture lifetime (`CaptureRegistryConfig::default().ttl`).
+const CAPTURE_TTL: Duration = Duration::from_secs(60);
+/// S1 scores one letter per option (at most 26 with reobserve/abstain). Each
+/// offered region lengthens the single prefill pass. On a 4-vCPU hosted runner
+/// with AVX512-BF16, 16 regions took 20 to 38 s per decision while the row
+/// recorded video, so 12 keep parse + decision + click well inside the
+/// capture lifetime.
+const MAX_REGION_CANDIDATES: usize = 12;
+const S1_READY_TIMEOUT: Duration = Duration::from_secs(900);
+const S1_DECISION_TIMEOUT: Duration = Duration::from_secs(120);
+
+fn required_env(name: &str) -> String {
+ std::env::var(name)
+ .ok()
+ .filter(|value| !value.trim().is_empty())
+ .unwrap_or_else(|| panic!("{name} is required by the S1 perception row"))
+}
+
+fn repo_root() -> PathBuf {
+ Path::new(env!("CARGO_MANIFEST_DIR"))
+ .ancestors()
+ .nth(5)
+ .expect("repository root above crates/cua-driver-e2e")
+ .to_path_buf()
+}
+
+fn evidence_dir() -> PathBuf {
+ let root = std::env::var_os("CUA_E2E_RESULTS_FILE")
+ .map(PathBuf::from)
+ .and_then(|results| results.parent().map(Path::to_path_buf))
+ .unwrap_or_else(std::env::temp_dir);
+ let directory = root.join("s1-perception");
+ fs::create_dir_all(&directory).expect("create S1 perception evidence directory");
+ directory
+}
+
+fn write_evidence(name: &str, value: &Value) {
+ let path = evidence_dir().join(name);
+ fs::write(&path, serde_json::to_vec_pretty(value).unwrap())
+ .unwrap_or_else(|error| panic!("write {path:?}: {error}"));
+}
+
+// ---------------------------------------------------------------- S1
+
+/// One resident Cua-S1-4B chooser speaking JSON lines over stdio.
+struct WarmChooser {
+ child: Child,
+ stdin: ChildStdin,
+ lines: Receiver<String>,
+ ready: Value,
+}
+
+impl WarmChooser {
+ fn start() -> Self {
+ let python = required_env("CUA_E2E_S1_PYTHON");
+ for name in ["S1_BASE_MODEL_PATH", "S1_ADAPTER_PATH"] {
+ required_env(name);
+ }
+ let script = repo_root().join("libs/cua-s1/ci/warm_chooser.py");
+ let mut child = Command::new(python)
+ .arg(&script)
+ .arg("--warmup")
+ .env("HF_HUB_OFFLINE", "1")
+ .env("TRANSFORMERS_OFFLINE", "1")
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::inherit())
+ .spawn()
+ .unwrap_or_else(|error| panic!("start {script:?}: {error}"));
+ let stdin = child.stdin.take().expect("chooser stdin");
+ let stdout = child.stdout.take().expect("chooser stdout");
+ let (sender, lines) = mpsc::channel();
+ thread::spawn(move || {
+ for line in BufReader::new(stdout).lines().map_while(Result::ok) {
+ if sender.send(line).is_err() {
+ break;
+ }
+ }
+ });
+ let started = Instant::now();
+ let line = lines
+ .recv_timeout(S1_READY_TIMEOUT)
+ .unwrap_or_else(|_| panic!("S1 chooser was not ready within {S1_READY_TIMEOUT:?}"));
+ let ready: Value = serde_json::from_str(&line).expect("S1 ready line is JSON");
+ assert_eq!(ready["ready"], true, "S1 chooser failed to start: {ready}");
+ eprintln!(
+ "[s1-perception] chooser ready in {:.1}s: {ready}",
+ started.elapsed().as_secs_f64()
+ );
+ Self {
+ child,
+ stdin,
+ lines,
+ ready,
+ }
+ }
+
+ fn decide(&mut self, request: &Value, screenshot: Option<&Path>) -> (Value, u64) {
+ let message = json!({"request": request, "screenshot": screenshot});
+ writeln!(self.stdin, "{message}").expect("send request to S1 chooser");
+ self.stdin.flush().expect("flush S1 chooser request");
+ let line = self
+ .lines
+ .recv_timeout(S1_DECISION_TIMEOUT)
+ .unwrap_or_else(|_| panic!("S1 decision took longer than {S1_DECISION_TIMEOUT:?}"));
+ let reply: Value = serde_json::from_str(&line).expect("S1 reply is JSON");
+ assert!(reply.get("error").is_none(), "S1 chooser refused: {reply}");
+ let latency = reply["latency_ms"].as_u64().expect("S1 latency_ms");
+ (reply["decision"].clone(), latency)
+ }
+}
+
+impl Drop for WarmChooser {
+ fn drop(&mut self) {
+ let _ = self.child.kill();
+ let _ = self.child.wait();
+ }
+}
+
+// ---------------------------------------------------------------- extension
+
+fn run_driver_cli(home: &Path, args: &[&str]) -> std::process::Output {
+ Command::new(driver_binary())
+ .args(args)
+ .env("CUA_DRIVER_RS_HOME", home)
+ .env("CUA_DRIVER_CLI_TELEMETRY_CHILD", "1")
+ .env("CUA_DRIVER_RS_TELEMETRY_ENABLED", "false")
+ .stdin(Stdio::null())
+ .output()
+ .expect("run cua-driver")
+}
+
+/// Install the published extension from its signed release catalog, the
+/// documented user path, and require publisher-verified trust.
+fn install_published_extension(home: &Path) -> Value {
+ let catalog = required_env("CUA_E2E_PERCEPTION_CATALOG");
+ let version = required_env("CUA_E2E_PERCEPTION_VERSION");
+ let install = run_driver_cli(
+ home,
+ &[
+ "extension",
+ "install",
+ "cua-perception",
+ "--catalog",
+ &catalog,
+ ],
+ );
+ assert!(
+ install.status.success(),
+ "published extension install failed: stdout={} stderr={}",
+ String::from_utf8_lossy(&install.stdout),
+ String::from_utf8_lossy(&install.stderr)
+ );
+ let status = run_driver_cli(home, &["extension", "status", "cua-perception", "--json"]);
+ assert!(
+ status.status.success(),
+ "extension status failed: {}",
+ String::from_utf8_lossy(&status.stderr)
+ );
+ let status: Value = serde_json::from_slice(&status.stdout).expect("extension status JSON");
+ assert_eq!(status["installed"], true, "extension status: {status}");
+ assert_eq!(
+ status["trust"], "publisher-verified",
+ "extension status: {status}"
+ );
+ assert_eq!(
+ status["active_version"], version,
+ "extension status: {status}"
+ );
+ status
+}
+
+// ---------------------------------------------------------------- fixture
+
+struct Fixture {
+ pid: u32,
+ window_id: u64,
+ journal: FixtureJournal,
+}
+
+fn launch_electron(driver: &mut McpDriver) -> Fixture {
+ let journal = FixtureJournal::start();
+ let cdp_port = std::net::TcpListener::bind(("127.0.0.1", 0))
+ .and_then(|listener| listener.local_addr())
+ .expect("allocate a loopback CDP port")
+ .port();
+ let window_ids = |driver: &mut McpDriver| -> HashSet<u64> {
+ driver.call("list_windows", json!({})).structured()["windows"]
+ .as_array()
+ .map(|windows| {
+ windows
+ .iter()
+ .filter_map(|window| window["window_id"].as_u64())
+ .collect()
+ })
+ .unwrap_or_default()
+ };
+ let before = window_ids(driver);
+ let path = harness_app("harness-electron", "CuaTestHarness.Electron");
+ assert!(
+ path.exists(),
+ "the Electron harness was not staged at {path:?}"
+ );
+ let mut command = Command::new(path);
+ command
+ .args([
+ "--no-sandbox",
+ "--disable-gpu",
+ "--force-renderer-accessibility",
+ ])
+ .env("CUA_E2E_FIXTURE_JOURNAL_URL", journal.url())
+ .env("CUA_ELECTRON_CDP_PORT", cdp_port.to_string())
+ .stdout(Stdio::inherit())
+ .stderr(Stdio::inherit());
+ let child = spawn_in_job(&mut command).expect("launch the Electron harness");
+ driver.reaper().push(child);
+
+ let deadline = Instant::now() + Duration::from_secs(30);
+ while Instant::now() < deadline {
+ let windows = driver.call("list_windows", json!({}));
+ let found = windows.structured()["windows"]
+ .as_array()
+ .into_iter()
+ .flatten()
+ .find(|window| {
+ window["window_id"]
+ .as_u64()
+ .is_some_and(|id| !before.contains(&id))
+ && window["title"]
+ .as_str()
+ .is_some_and(|title| title.contains(FIXTURE_TITLE))
+ })
+ .and_then(|window| {
+ Some((
+ window["pid"].as_u64()? as u32,
+ window["window_id"].as_u64()?,
+ ))
+ });
+ if let Some((pid, window_id)) = found {
+ driver.reaper().track_pid(pid);
+ while Instant::now() < deadline {
+ if journal.text("lbl-counter").as_deref() == Some("counter=0") {
+ return Fixture {
+ pid,
+ window_id,
+ journal,
+ };
+ }
+ thread::sleep(Duration::from_millis(100));
+ }
+ panic!("Electron harness page never published its initial state");
+ }
+ thread::sleep(Duration::from_millis(250));
+ }
+ panic!("Electron harness window did not appear");
+}
+
+fn capture_window(driver: &mut McpDriver, fixture: &Fixture) -> (String, Instant) {
+ let state = driver.call(
+ "get_window_state",
+ json!({
+ "pid": fixture.pid as i64,
+ "window_id": fixture.window_id,
+ "include_accessibility_tree": false
+ }),
+ );
+ let captured_at = Instant::now();
+ assert!(
+ !state.is_error(),
+ "get_window_state failed: {}",
+ state.text()
+ );
+ let capture_id = state.structured()["capture_id"]
+ .as_str()
+ .unwrap_or_else(|| panic!("window observation omitted capture_id: {}", state.text()))
+ .to_owned();
+ (capture_id, captured_at)
+}
+
+fn parse_regions(driver: &mut McpDriver, capture_id: &str) -> ToolResponse {
+ driver.call(
+ "parse_visual_regions",
+ json!({
+ "capture_id": capture_id,
+ "options": {"kinds": ["text", "icon"], "min_confidence": 0.3, "max_regions": 100}
+ }),
+ )
+}
+
+fn validated_parse(
+ parsed: &ToolResponse,
+ fixture: &Fixture,
+ capture_id: &str,
+ version: &str,
+) -> Value {
+ assert!(
+ !parsed.is_error(),
+ "parse_visual_regions failed: {}; structured={}",
+ parsed.text(),
+ parsed.structured()
+ );
+ let result = parsed.structured().clone();
+ assert_eq!(result["schema"], "cua.visual_regions_v1");
+ assert_eq!(result["capture"]["capture_id"], capture_id);
+ assert_eq!(result["capture"]["source"]["kind"], "window");
+ assert_eq!(result["capture"]["source"]["pid"], fixture.pid);
+ assert_eq!(result["capture"]["source"]["window_id"], fixture.window_id);
+ assert_eq!(result["parser"]["extension_id"], "cua-perception");
+ assert_eq!(result["parser"]["extension_version"], version);
+ assert_eq!(
+ result["capture"]["action_coordinate_space"]["kind"], "screenshot_pixels",
+ "region centers are clicked in screenshot pixels: {}",
+ result["capture"]
+ );
+ result
+}
+
+fn center(region: &Value) -> (f64, f64) {
+ let bounds = &region["bounds"];
+ let value = |key: &str| bounds[key].as_f64().expect("numeric region bounds");
+ (
+ value("x") + value("width") / 2.0,
+ value("y") + value("height") / 2.0,
+ )
+}
+
+/// Build a `cua.jev_choice_request_v1` from parsed regions with the caller
+/// policy jev-use documents: every candidate is a supplied region, plus
+/// `reobserve` and `abstain`. Text regions that carry letters are offered in
+/// reading order; the request carries exactly those regions.
+fn choice_request(parse: &Value, capture_id: &str) -> (Value, Vec<Value>) {
+ let mut texts: Vec<Value> = parse["regions"]
+ .as_array()
+ .into_iter()
+ .flatten()
+ .filter(|region| region["kind"] == "text")
+ .filter(|region| {
+ region["text"]
+ .as_str()
+ .is_some_and(|text| text.chars().filter(|c| c.is_alphabetic()).count() >= 2)
+ })
+ .cloned()
+ .collect();
+ texts.sort_by(|left, right| {
+ let key = |region: &Value| {
+ (
+ region["bounds"]["y"].as_i64().unwrap_or(0) / 12,
+ region["bounds"]["x"].as_i64().unwrap_or(0),
+ )
+ };
+ key(left).cmp(&key(right))
+ });
+ texts.truncate(MAX_REGION_CANDIDATES);
+ let regions: Vec<Value> = texts
+ .iter()
+ .map(|region| {
+ json!({
+ "id": region["id"],
+ "kind": "text",
+ "bounds": region["bounds"],
+ "text": region["text"],
+ "confidence": region["confidence"],
+ "interactive": region["interactive"],
+ })
+ })
+ .collect();
+ let mut candidates: Vec<Value> = texts
+ .iter()
+ .map(|region| {
+ let (x, y) = center(region);
+ let text: String = region["text"]
+ .as_str()
+ .unwrap_or("")
+ .chars()
+ .take(60)
+ .collect();
+ json!({
+ "id": format!("click-{}", region["id"].as_str().unwrap_or("region")),
+ "description": format!("Click the text region {text:?} at ({x:.0},{y:.0})."),
+ })
+ })
+ .collect();
+ candidates.push(json!({
+ "id": "reobserve",
+ "description": "Discard this decision set and obtain a fresh observation."
+ }));
+ candidates.push(json!({
+ "id": "abstain",
+ "description": "Stop without acting if no supplied action is safe."
+ }));
+ (
+ json!({
+ "schema": "cua.jev_choice_request_v1",
+ "goal": GOAL,
+ "capture_id": capture_id,
+ "regions": regions,
+ "history": [],
+ "candidates": candidates,
+ }),
+ texts,
+ )
+}
+
+fn wait_for_counter(journal: &FixtureJournal, expected: &str, within: Duration) -> bool {
+ let deadline = Instant::now() + within;
+ loop {
+ if journal.text("lbl-counter").as_deref() == Some(expected) {
+ return true;
+ }
+ if Instant::now() >= deadline {
+ return false;
+ }
+ thread::sleep(Duration::from_millis(50));
+ }
+}
+
+fn click_args(fixture: &Fixture, x: f64, y: f64, capture_id: &str) -> Value {
+ json!({
+ "pid": fixture.pid as i64,
+ "window_id": fixture.window_id,
+ "x": x,
+ "y": y,
+ "capture_id": capture_id,
+ "delivery_mode": "foreground"
+ })
+}
+
+fn assert_refused(response: &ToolResponse, code: &str, what: &str) {
+ assert!(
+ response.is_error(),
+ "{what} authorized a click: {}",
+ response.text()
+ );
+ assert_eq!(
+ response.structured()["code"],
+ code,
+ "{what} used an unexpected refusal: {}",
+ response.structured()
+ );
+ assert_eq!(response.structured()["effect"], "refused", "{what}");
+}
+
+// ---------------------------------------------------------------- row
+
+/// S1 chooses among OmniParser regions of a live capture; Driver clicks the
+/// choice with that capture inside its lifetime; the fixture journal verifies
+/// the effect; consumed and expired captures are refused without effect.
+#[test]
+#[ignore]
+fn s1_chooses_a_parsed_region_and_the_capture_bound_click_is_state_verified() {
+ let route = shared_web_route(
+ Platform::current(),
+ DisplayServer::current(),
+ "left_click",
+ Targeting::Px,
+ Delivery::Foreground,
+ )
+ .expect("foreground pixel click route");
+ let case = CaseSpec::delivered(
+ format!(
+ "{}-electron-perception-s1-capture-click-px-foreground",
+ std::env::consts::OS
+ ),
+ "electron",
+ "electron",
+ "perception_s1_capture_click",
+ Targeting::Px,
+ Delivery::Foreground,
+ Scope::Window,
+ route,
+ vec![OracleKind::FixtureState, OracleKind::Protocol],
+ );
+ let cell_id = case.cell_id.clone();
+ execute_case(case, |evidence| {
+ let version = required_env("CUA_E2E_PERCEPTION_VERSION");
+ // Load and warm the model before any capture exists, like a live
+ // agent would; its lifetime starts at the capture, not at startup.
+ let mut chooser = WarmChooser::start();
+
+ let home = tempfile::Builder::new()
+ .prefix("cua-perception-s1-home-")
+ .tempdir()
+ .expect("create isolated extension home");
+ let status = install_published_extension(home.path());
+ let home_str = home.path().to_str().expect("UTF-8 extension home");
+ let mut driver =
+ McpDriver::spawn_named_with_env(&cell_id, &[("CUA_DRIVER_RS_HOME", home_str)])
+ .expect("start the source-built Driver with the extension home");
+ *evidence = recording_evidence(driver.recording_dir());
+ let fixture = launch_electron(&mut driver);
+ let raised = driver.call(
+ "bring_to_front",
+ json!({"pid": fixture.pid as i64, "window_id": fixture.window_id}),
+ );
+ eprintln!("[s1-perception] bring_to_front: {}", raised.text());
+ thread::sleep(Duration::from_millis(500));
+
+ // Warm the extension worker, and keep this capture unused so it can
+ // prove expiry at the end.
+ let (expiring_capture, expiring_at) = capture_window(&mut driver, &fixture);
+ let warm_started = Instant::now();
+ let warm = parse_regions(&mut driver, &expiring_capture);
+ validated_parse(&warm, &fixture, &expiring_capture, &version);
+ let warm_parse_ms = warm_started.elapsed().as_millis() as u64;
+ driver.start_behavior_recording();
+
+ // 1. Observe and retain the native capture.
+ let (capture_id, captured_at) = capture_window(&mut driver, &fixture);
+ // 2. The published extension parses that exact capture.
+ let parse_started = Instant::now();
+ let parsed = parse_regions(&mut driver, &capture_id);
+ let parse_ms = parse_started.elapsed().as_millis() as u64;
+ let parse = validated_parse(&parsed, &fixture, &capture_id, &version);
+ assert_eq!(
+ fixture.journal.text("lbl-counter").as_deref(),
+ Some("counter=0"),
+ "parsing must not change fixture state"
+ );
+ // 3. S1 chooses one closed candidate.
+ let (request, offered) = choice_request(&parse, &capture_id);
+ write_evidence("parse.json", &parse);
+ write_evidence("request.json", &request);
+ let (decision, s1_ms) = chooser.decide(&request, None);
+ write_evidence("decision.json", &decision);
+ let decided_age = captured_at.elapsed();
+ assert_eq!(decision["schema"], "cua.decision_choice_v1", "{decision}");
+ assert_eq!(decision["capture_id"], capture_id, "{decision}");
+ assert_eq!(
+ decision["kind"], "selected",
+ "S1 did not select an action: {decision}"
+ );
+ let selected = decision["selected_id"].as_str().expect("selected_id");
+ let region = offered
+ .iter()
+ .find(|region| format!("click-{}", region["id"].as_str().unwrap_or("")) == selected)
+ .unwrap_or_else(|| panic!("S1 selected an unknown candidate: {decision}"));
+ let (x, y) = center(region);
+
+ // 4. One capture-bound click inside the capture lifetime.
+ let click_age = captured_at.elapsed();
+ assert!(
+ click_age < CAPTURE_TTL,
+ "the loop took {click_age:?}, beyond the {CAPTURE_TTL:?} capture lifetime"
+ );
+ let args = click_args(&fixture, x, y, &capture_id);
+ let click_started = Instant::now();
+ let click = driver.call("click", args.clone());
+ let click_ms = click_started.elapsed().as_millis() as u64;
+ assert!(
+ !click.is_error(),
+ "capture-bound click failed: {}; structured={}",
+ click.text(),
+ click.structured()
+ );
+ // 5. The independent fixture oracle.
+ let delivered = wait_for_counter(&fixture.journal, "counter=1", Duration::from_secs(10));
+ let timing = json!({
+ "cell_id": cell_id,
+ "extension": {"version": status["active_version"], "trust": status["trust"]},
+ "parser": parse["parser"],
+ "s1": chooser.ready,
+ "goal": GOAL,
+ "regions_parsed": parse["regions"].as_array().map(Vec::len),
+ "candidates": request["candidates"].as_array().map(Vec::len),
+ "selected_id": selected,
+ "selected_text": region["text"],
+ "confidence": decision["confidence"],
+ "warm_parse_ms": warm_parse_ms,
+ "parse_ms": parse_ms,
+ "s1_decision_ms": s1_ms,
+ "capture_age_at_decision_ms": decided_age.as_millis() as u64,
+ "capture_age_at_click_ms": click_age.as_millis() as u64,
+ "click_ms": click_ms,
+ "click_route": click.action_route(),
+ "fixture_counter": fixture.journal.text("lbl-counter"),
+ });
+ write_evidence("timing.json", &timing);
+ eprintln!("[s1-perception] {timing}");
+ assert!(
+ delivered,
+ "S1 chose {selected} ({:?}) but the fixture never reached counter=1: {}",
+ region["text"],
+ fixture.journal.snapshot()
+ );
+
+ // 6a. The consumed capture is refused.
+ let reused = driver.call("click", args);
+ assert_refused(&reused, "capture_not_found", "a consumed capture");
+ // 6b. An unused capture past its lifetime is refused.
+ let remaining = CAPTURE_TTL.saturating_sub(expiring_at.elapsed());
+ thread::sleep(remaining + Duration::from_secs(2));
+ let expired = driver.call("click", click_args(&fixture, x, y, &expiring_capture));
+ assert_refused(&expired, "capture_expired", "an expired capture");
+ thread::sleep(Duration::from_millis(750));
+ assert_eq!(
+ fixture.journal.text("lbl-counter").as_deref(),
+ Some("counter=1"),
+ "a refused stale capture changed the fixture: {}",
+ fixture.journal.snapshot()
+ );
+ Observation::delivered(
+ vec![OracleKind::FixtureState, OracleKind::Protocol],
+ Evidence::default(),
+ )
+ });
+}
+
+/// The request builder offers only letter-bearing text regions, in reading
+/// order, capped for S1's 26-letter option space, plus reobserve/abstain.
+#[test]
+fn choice_request_offers_bounded_text_regions_in_reading_order() {
+ let region = |id: &str, kind: &str, text: &str, x: i64, y: i64| {
+ json!({"id": id, "kind": kind, "text": text, "confidence": 0.9, "interactive": false,
+ "bounds": {"x": x, "y": y, "width": 40, "height": 20}})
+ };
+ let mut regions = vec![
+ region("text-2", "text", "Reset", 200, 101),
+ region("text-1", "text", "Increment", 100, 100),
+ region("text-3", "text", "0", 300, 100),
+ region("icon-1", "icon", "", 10, 10),
+ region("text-4", "text", "Heading", 20, 20),
+ ];
+ for index in 0..40 {
+ regions.push(region(
+ &format!("text-x{index}"),
+ "text",
+ "Filler",
+ 10,
+ 500 + index * 30,
+ ));
+ }
+ let (request, offered) = choice_request(&json!({"regions": regions}), "capture_x");
+ let ids: Vec<&str> = request["candidates"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|candidate| candidate["id"].as_str().unwrap())
+ .collect();
+ assert_eq!(&ids[..3], ["click-text-4", "click-text-1", "click-text-2"]);
+ assert_eq!(ids.len(), MAX_REGION_CANDIDATES + 2);
+ assert_eq!(&ids[ids.len() - 2..], ["reobserve", "abstain"]);
+ assert_eq!(offered.len(), MAX_REGION_CANDIDATES);
+ assert_eq!(
+ request["regions"].as_array().unwrap().len(),
+ MAX_REGION_CANDIDATES
+ );
+ assert_eq!(request["capture_id"], "capture_x");
+ assert_eq!(
+ request["candidates"][1]["description"],
+ "Click the text region \"Increment\" at (120,110)."
+ );
+}
diff --git a/libs/cua-s1/README.md b/libs/cua-s1/README.md
index 68d19611c3..28c276e003 100644
--- a/libs/cua-s1/README.md
+++ b/libs/cua-s1/README.md
@@ -168,6 +168,26 @@ times are medians of five warm calls:
| `cua-s1-4b-0.2` text | `cpu`, `float32` | 18.3 s | 19.5 s | 32.1 s |
| `cua-s1-4b-0.2` multimodal | `cpu`, `float32` | 7.8 s | 34.2 s | 50.2 s |
+The weights-backed CI job (`.github/workflows/ci-cua-s1-weights.yml`) also
+measures a standard GitHub-hosted `ubuntu-latest` runner on every run: 4 vCPU
+(AMD EPYC 9V74, two Torch threads), 16 GB of memory, Python 3.12, and the same
+lock. The same runner label comes with and without AVX-512, and that decides
+the speed of `bfloat16` on CPU. On 2026-09-26 it measured `cua-s1-4b-0.2` on
+`cpu` with `bfloat16`:
+
+| Modality | Runner CPU flags | Warm decision, 3-candidate fixture | Cold `verify_decision_cli.py` (load + decision) | Peak RSS |
+| ---------- | ---------------- | ---------------------------------- | ----------------------------------------------- | -------- |
+| text | AVX512-BF16 | 5.5 to 7.3 s | 13 to 25 s | 9.4 GB |
+| multimodal | AVX512-BF16 | 47 to 50 s (1280x800) | 53 to 55 s | 10.6 GB |
+| text | AVX2, no AVX-512 | 51 to 68 s | 66 to 74 s | 9.4 GB |
+| multimodal | AVX2, no AVX-512 | 360 to 379 s (1280x800) | 364 to 383 s | 10.6 GB |
+
+Use `bfloat16` on CPU. `float16` has no native CPU support on these runners:
+the cold verifier took 114 to 191 s for text and about 10 minutes for
+multimodal on an AVX512-BF16 runner. `float32` weights (18.7 GB) do not fit in
+16 GB. Peak RSS counts memory-mapped weight pages. The job summary of each run
+records the runner's flags and the current numbers.
+
Multimodal rows used 1280x800 screenshots; a 2560x1600 screenshot took 22 to
30 s per decision on `mps`, so downscale large captures before scoring. The
CPU rows ran while other workloads kept the host's load average near 16, so
@@ -291,7 +311,18 @@ object, names a different `capture_id`, omits or adds a candidate, or selects
anything other than `submit-form`. To check the checked-in negative fixture,
add `--fixture negative --expected-id abstain`. To run the multimodal
adapter, add `--screenshot <png>` with an image of the fixture's form; the
-verifier binds it to the fixture's capture ID.
+verifier binds it to the fixture's capture ID. The fixtures directory has one
+for each request: `jev-choice-request-v1.png` and `jev-choice-negative-v1.png`.
+
+CI runs this smoke with real weights in `.github/workflows/ci-cua-s1-weights.yml`
+(nightly, on manual dispatch, and on pull requests that change Cua-S1 or the
+chooser). It downloads the two pinned artifacts on every run and verifies each
+file's size and SHA-256 against `libs/cua-s1/ci/weights.lock.json` before
+loading it, runs the verifier for both fixtures in text and multimodal modes,
+and records warm latency with `libs/cua-s1/ci/warm_chooser.py`, which keeps one
+model resident and answers decisions over stdio. The same workflow runs the
+Cua Driver desktop row that lets Cua-S1 choose among OmniParser regions of a
+live capture (see `libs/cua-driver/docs/test-matrix.md`).
To see the full response, run the chooser directly:
diff --git a/libs/cua-s1/ci/fetch_pinned_weights.py b/libs/cua-s1/ci/fetch_pinned_weights.py
new file mode 100644
index 0000000000..c5361ba487
--- /dev/null
+++ b/libs/cua-s1/ci/fetch_pinned_weights.py
@@ -0,0 +1,145 @@
+"""Download the pinned public Cua-S1-4B weights and verify every file.
+
+Reads ``weights.lock.json`` (next to this script by default), downloads each
+listed file from Hugging Face at the exact pinned commit into
+``<dest>/<artifact name>``, and verifies its size and SHA-256 before anything
+loads it. No token is used or needed: every artifact is public.
+
+ python libs/cua-s1/ci/fetch_pinned_weights.py --dest "$RUNNER_TEMP/s1-models"
+
+Prints one JSON summary line. Exits nonzero on any missing, extra, or
+mismatched file. ``--verify-only`` checks an existing directory without
+network access.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+LOCK_SCHEMA = "cua-s1/weights-lock/v1"
+_REVISION = re.compile(r"[0-9a-f]{40}")
+_SHA256 = re.compile(r"[0-9a-f]{64}")
+_CHUNK = 8 * 1024 * 1024
+
+
+def load_lock(path: Path) -> list[dict[str, Any]]:
+ document = json.loads(path.read_text(encoding="utf-8"))
+ if document.get("schema") != LOCK_SCHEMA:
+ raise ValueError(f"{path}: unsupported schema {document.get('schema')!r}")
+ artifacts = document["artifacts"]
+ names = set()
+ for artifact in artifacts:
+ name = artifact["name"]
+ if not re.fullmatch(r"[A-Za-z0-9._-]+", name) or name in names:
+ raise ValueError(f"invalid or duplicate artifact name: {name!r}")
+ names.add(name)
+ if not _REVISION.fullmatch(artifact["revision"]):
+ raise ValueError(f"{name}: revision must be a full commit SHA")
+ if not artifact["files"]:
+ raise ValueError(f"{name}: no files listed")
+ for relative, expected in artifact["files"].items():
+ parts = Path(relative).parts
+ if Path(relative).is_absolute() or ".." in parts or not parts:
+ raise ValueError(f"{name}: unsafe file path {relative!r}")
+ if not _SHA256.fullmatch(expected["sha256"]) or int(expected["size"]) < 0:
+ raise ValueError(f"{name}/{relative}: invalid size or sha256")
+ return artifacts
+
+
+def sha256_file(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as handle:
+ while chunk := handle.read(_CHUNK):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def verify_artifact(root: Path, artifact: dict[str, Any]) -> list[str]:
+ """Return a list of problems; empty means every listed file matches."""
+ problems = []
+ expected_files = artifact["files"]
+ for relative, expected in expected_files.items():
+ path = root / relative
+ if not path.is_file() or path.is_symlink():
+ problems.append(f"{artifact['name']}/{relative}: missing")
+ continue
+ size = path.stat().st_size
+ if size != int(expected["size"]):
+ problems.append(f"{artifact['name']}/{relative}: size {size} != {expected['size']}")
+ continue
+ actual = sha256_file(path)
+ if actual != expected["sha256"]:
+ problems.append(
+ f"{artifact['name']}/{relative}: sha256 {actual} != {expected['sha256']}"
+ )
+ for path in sorted(root.rglob("*")):
+ relative = path.relative_to(root).as_posix()
+ if relative == ".cache" or relative.startswith(".cache/") or path.is_dir():
+ continue
+ if relative not in expected_files:
+ problems.append(f"{artifact['name']}/{relative}: not in the lock")
+ return problems
+
+
+def download_artifact(root: Path, artifact: dict[str, Any]) -> None:
+ from huggingface_hub import snapshot_download
+
+ snapshot_download(
+ repo_id=artifact["repo_id"],
+ revision=artifact["revision"],
+ local_dir=str(root),
+ allow_patterns=sorted(artifact["files"]),
+ token=False,
+ )
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--lock", type=Path, default=Path(__file__).with_name("weights.lock.json"))
+ parser.add_argument("--dest", type=Path, required=True)
+ parser.add_argument("--verify-only", action="store_true")
+ args = parser.parse_args(argv)
+
+ artifacts = load_lock(args.lock)
+ args.dest.mkdir(parents=True, exist_ok=True)
+ os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
+ summary: dict[str, Any] = {"artifacts": []}
+ problems: list[str] = []
+ for artifact in artifacts:
+ root = args.dest / artifact["name"]
+ started = time.perf_counter()
+ if not args.verify_only:
+ download_artifact(root, artifact)
+ downloaded = time.perf_counter()
+ found = verify_artifact(root, artifact)
+ problems.extend(found)
+ summary["artifacts"].append(
+ {
+ "name": artifact["name"],
+ "repo_id": artifact["repo_id"],
+ "revision": artifact["revision"],
+ "path": str(root),
+ "files": len(artifact["files"]),
+ "bytes": sum(int(item["size"]) for item in artifact["files"].values()),
+ "download_s": round(downloaded - started, 1),
+ "verify_s": round(time.perf_counter() - downloaded, 1),
+ "verified": not found,
+ }
+ )
+ summary["verified"] = not problems
+ print(json.dumps(summary, separators=(",", ":")))
+ for problem in problems:
+ print(f"weights verification failed: {problem}", file=sys.stderr)
+ return 0 if not problems else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/libs/cua-s1/ci/warm_chooser.py b/libs/cua-s1/ci/warm_chooser.py
new file mode 100644
index 0000000000..28e5d67388
--- /dev/null
+++ b/libs/cua-s1/ci/warm_chooser.py
@@ -0,0 +1,297 @@
+"""Keep one Cua-S1-4B chooser resident and answer decisions over stdio.
+
+The jev-use chooser (`choose_decision.py --model s1`) loads the model for every
+decision, which takes tens of seconds on CPU. Live loops must decide within the
+Cua Driver's 60-second capture lifetime, so the S1 docs recommend keeping one
+loaded model resident. This helper does exactly that for CI and the desktop
+E2E row, using the repository's validation and scoring code unchanged
+(`choose_action.validate_request`, `decision_models.S1DecisionModel`,
+`decision_models.choose`). It never dispatches actions.
+
+Configuration is the chooser's: `S1_BASE_MODEL_PATH`, `S1_ADAPTER_PATH`,
+`S1_DEVICE`, `S1_DTYPE`, and `S1_MODALITY` (one modality per process).
+`S1_RESIDENT=0` keeps the weights memory-mapped instead of copying them into
+process memory (see `make_resident`).
+
+Serve mode (default) prints one `{"ready": true, ...}` line after loading,
+then reads one JSON object per stdin line:
+
+ {"request": <cua.jev_choice_request_v1>, "screenshot": "<png path>"}
+
+`screenshot` is required for multimodal and is bound to the request's
+capture. Each reply is one line: `{"decision": <cua.decision_choice_v1>,
+"latency_ms": <int>}` or `{"error": "<detail>"}`.
+
+Smoke mode (`--smoke`) runs the checked-in positive and negative fixtures
+(with their screenshots for multimodal), checks them exactly as
+`verify_decision_cli.py` does, prints one JSON report, and exits nonzero on
+any mismatch.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import platform
+import resource
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+REPO = Path(__file__).resolve().parents[3]
+JEV_USE = REPO / "libs/cua-driver/examples/jev-use"
+sys.path.insert(0, str(JEV_USE / "python"))
+
+from choose_action import MAX_INPUT_BYTES, validate_request # noqa: E402
+from choose_decision import _resolve_s1_adapter # noqa: E402
+from decision_models import ( # noqa: E402
+ S1_MODALITIES,
+ DecisionRequest,
+ S1DecisionModel,
+ choose,
+ s1_model_identity,
+)
+
+FIXTURES = {
+ "positive": ("jev-choice-request-v1", "submit-form"),
+ "negative": ("jev-choice-negative-v1", "abstain"),
+}
+RESPONSE_KEYS = {
+ "schema",
+ "kind",
+ "capture_id",
+ "selected_id",
+ "model",
+ "confidence",
+ "probabilities",
+ "reason",
+}
+
+
+def peak_rss_mb() -> float:
+ peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+ # Linux reports KiB; macOS reports bytes.
+ return round(peak / (1024 * 1024 if sys.platform == "darwin" else 1024), 1)
+
+
+def make_resident(model: Any) -> None:
+ """Copy memory-mapped weights into process memory.
+
+ Transformers maps the safetensors shards and loads same-dtype CPU weights
+ as file-backed pages. Under desktop memory pressure (Driver, browser,
+ OmniParser worker, video recorder) the kernel evicts those clean pages,
+ and every forward pass rereads up to 9 GB from disk. Anonymous copies stay
+ resident; one tensor is copied at a time, so the peak cost is one tensor.
+ """
+ import torch
+
+ with torch.no_grad():
+ for tensor in [*model.parameters(), *model.buffers()]:
+ tensor.data = tensor.data.clone()
+
+
+def rss_breakdown_mb() -> dict[str, float]:
+ """Current anonymous and file-backed resident memory (Linux only)."""
+ try:
+ status = Path("/proc/self/status").read_text(encoding="utf-8")
+ except OSError:
+ return {}
+ values = {}
+ for line in status.splitlines():
+ key, _, rest = line.partition(":")
+ if key in {"RssAnon", "RssFile"}:
+ values[key.lower().replace("rss", "rss_") + "_mb"] = round(
+ int(rest.split()[0]) / 1024, 1
+ )
+ return values
+
+
+class WarmChooser:
+ def __init__(self) -> None:
+ base = Path(os.environ["S1_BASE_MODEL_PATH"]).expanduser()
+ adapter = Path(os.environ["S1_ADAPTER_PATH"]).expanduser()
+ if not base.is_dir() or not adapter.is_dir():
+ raise ValueError("S1_BASE_MODEL_PATH and S1_ADAPTER_PATH must be local directories")
+ self.modality = os.environ.get("S1_MODALITY", "").strip() or "text"
+ if self.modality not in S1_MODALITIES:
+ raise ValueError(f"S1_MODALITY must be one of {', '.join(S1_MODALITIES)}")
+ _resolve_s1_adapter(adapter, self.modality)
+ self.device = os.environ.get("S1_DEVICE", "cpu")
+ self.dtype = os.environ.get("S1_DTYPE", "bfloat16")
+ self.name = s1_model_identity(
+ adapter,
+ self.modality,
+ repo_id=os.environ.get("S1_ADAPTER_ID"),
+ revision=os.environ.get("S1_ADAPTER_REVISION"),
+ )
+ from cua_s1.four_b import FourBModel
+
+ started = time.perf_counter()
+ self.scorer = FourBModel(
+ base_model=str(base),
+ lora_adapter_path=adapter,
+ device=self.device,
+ dtype=self.dtype,
+ modality=self.modality,
+ )
+ self.scorer.load()
+ self.load_s = round(time.perf_counter() - started, 2)
+ self.resident_s: float | None = None
+ self.warmup_ms: int | None = None
+ if os.environ.get("S1_RESIDENT", "1") != "0":
+ started = time.perf_counter()
+ make_resident(self.scorer._model)
+ self.resident_s = round(time.perf_counter() - started, 2)
+
+ def decide(self, raw: Any, screenshot: str | None) -> tuple[dict[str, Any], int]:
+ if len(json.dumps(raw).encode()) > MAX_INPUT_BYTES:
+ raise ValueError("request exceeds input limit")
+ request = DecisionRequest.from_validated(validate_request(raw))
+ shot = None
+ if self.modality == "multimodal":
+ if not screenshot or not Path(screenshot).is_file():
+ raise ValueError("multimodal S1 requires an existing screenshot path")
+ shot = Path(screenshot)
+ elif screenshot:
+ raise ValueError("screenshot requires S1_MODALITY=multimodal")
+ model = S1DecisionModel(
+ self.scorer,
+ modality=self.modality,
+ screenshot_path=shot,
+ screenshot_capture_id=request.capture_id if shot else None,
+ name=self.name,
+ )
+ started = time.perf_counter()
+ result = choose(model, request).to_wire()
+ return result, round((time.perf_counter() - started) * 1000)
+
+ def describe(self) -> dict[str, Any]:
+ import torch
+
+ return {
+ "model": self.name,
+ "modality": self.modality,
+ "device": self.device,
+ "dtype": self.dtype,
+ "load_s": self.load_s,
+ "resident_s": self.resident_s,
+ "warmup_ms": self.warmup_ms,
+ "torch": torch.__version__,
+ "torch_threads": torch.get_num_threads(),
+ "cpu_count": os.cpu_count(),
+ "machine": platform.machine(),
+ "peak_rss_mb": peak_rss_mb(),
+ **rss_breakdown_mb(),
+ }
+
+
+def check_response(response: dict[str, Any], request: dict[str, Any], expected_id: str) -> None:
+ """The same checks `verify_decision_cli.py` applies to the chooser output."""
+ if set(response) != RESPONSE_KEYS or response["schema"] != "cua.decision_choice_v1":
+ raise RuntimeError("decision model returned an unsupported response")
+ if response["capture_id"] != request["capture_id"]:
+ raise RuntimeError("decision model returned a different capture")
+ expected_kind = expected_id if expected_id in {"reobserve", "abstain"} else "selected"
+ if (response["kind"], response["selected_id"]) != (expected_kind, expected_id):
+ raise RuntimeError(
+ f"decision model chose {response['kind']}/{response['selected_id']}, "
+ f"expected {expected_kind}/{expected_id}"
+ )
+ if set(response["probabilities"]) != {item["id"] for item in request["candidates"]}:
+ raise RuntimeError("decision model returned a different candidate set")
+
+
+def smoke(chooser: WarmChooser, repeats: int) -> int:
+ report: dict[str, Any] = {**chooser.describe(), "decisions": []}
+ failures = 0
+ for fixture, (stem, expected_id) in FIXTURES.items():
+ request = json.loads((JEV_USE / "fixtures" / f"{stem}.json").read_text(encoding="utf-8"))
+ screenshot = (
+ str(JEV_USE / "fixtures" / f"{stem}.png") if chooser.modality == "multimodal" else None
+ )
+ for attempt in range(1, repeats + 1):
+ entry: dict[str, Any] = {
+ "fixture": fixture,
+ "attempt": attempt,
+ "expected": expected_id,
+ }
+ try:
+ response, latency_ms = chooser.decide(request, screenshot)
+ entry.update(
+ latency_ms=latency_ms,
+ kind=response["kind"],
+ selected_id=response["selected_id"],
+ confidence=response["confidence"],
+ )
+ check_response(response, request, expected_id)
+ entry["ok"] = True
+ except Exception as error: # noqa: BLE001 - reported, then fails the run
+ entry.update(ok=False, error=str(error)[:300])
+ failures += 1
+ report["decisions"].append(entry)
+ print(json.dumps(entry), file=sys.stderr, flush=True)
+ report["peak_rss_mb"] = peak_rss_mb()
+ report["ok"] = failures == 0
+ print(json.dumps(report, separators=(",", ":")), flush=True)
+ return 0 if failures == 0 else 1
+
+
+def serve(chooser: WarmChooser, out: Any) -> int:
+ def reply(value: dict[str, Any]) -> None:
+ out.write(json.dumps(value, separators=(",", ":")) + "\n")
+ out.flush()
+
+ reply({"ready": True, **chooser.describe()})
+ for line in sys.stdin:
+ if not line.strip():
+ continue
+ try:
+ message = json.loads(line)
+ if not isinstance(message, dict) or "request" not in message:
+ raise ValueError("expected an object with a request")
+ decision, latency_ms = chooser.decide(message["request"], message.get("screenshot"))
+ reply({"decision": decision, "latency_ms": latency_ms, "peak_rss_mb": peak_rss_mb()})
+ except (ValueError, KeyError, TypeError, json.JSONDecodeError) as error:
+ reply({"error": str(error)[:300]})
+ return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--smoke", action="store_true", help="run the fixture checks and exit")
+ parser.add_argument("--repeats", type=int, default=2, help="decisions per fixture in --smoke")
+ parser.add_argument(
+ "--warmup",
+ action="store_true",
+ help="serve mode: run one positive-fixture decision before reporting ready",
+ )
+ args = parser.parse_args(argv)
+
+ # Keep the protocol stream clean: libraries may print while loading.
+ protocol = sys.stdout
+ sys.stdout = sys.stderr
+ try:
+ chooser = WarmChooser()
+ except (KeyError, ValueError, ImportError) as error:
+ protocol.write(json.dumps({"ready": False, "error": f"S1 setup failed: {error}"}) + "\n")
+ protocol.flush()
+ return 2
+ if args.smoke:
+ sys.stdout = protocol
+ return smoke(chooser, max(1, args.repeats))
+ if args.warmup:
+ stem, _ = FIXTURES["positive"]
+ request = json.loads((JEV_USE / "fixtures" / f"{stem}.json").read_text(encoding="utf-8"))
+ screenshot = (
+ str(JEV_USE / "fixtures" / f"{stem}.png") if chooser.modality == "multimodal" else None
+ )
+ _, latency_ms = chooser.decide(request, screenshot)
+ chooser.warmup_ms = latency_ms
+ print(f"warmup decision: {latency_ms} ms", file=sys.stderr, flush=True)
+ return serve(chooser, protocol)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/libs/cua-s1/ci/weights.lock.json b/libs/cua-s1/ci/weights.lock.json
new file mode 100644
index 0000000000..dfb9748b45
--- /dev/null
+++ b/libs/cua-s1/ci/weights.lock.json
@@ -0,0 +1,102 @@
+{
+ "schema": "cua-s1/weights-lock/v1",
+ "description": "Pinned public Hugging Face artifacts for the weights-backed Cua-S1-4B smoke. Every listed file is downloaded at the pinned revision and verified by size and SHA-256; keep in sync with libs/cua-s1/README.md.",
+ "artifacts": [
+ {
+ "name": "Qwen3.5-4B",
+ "repo_id": "Qwen/Qwen3.5-4B",
+ "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
+ "role": "base",
+ "files": {
+ ".gitattributes": {
+ "size": 1570,
+ "sha256": "34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930"
+ },
+ "LICENSE": {
+ "size": 11544,
+ "sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a"
+ },
+ "README.md": {
+ "size": 77661,
+ "sha256": "1406be1b6b8fd8a6545870da516912804756593628a1d0fb0a7965211e82a7bb"
+ },
+ "chat_template.jinja": {
+ "size": 7756,
+ "sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715"
+ },
+ "config.json": {
+ "size": 3161,
+ "sha256": "ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670"
+ },
+ "merges.txt": {
+ "size": 3353259,
+ "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d"
+ },
+ "model.safetensors-00001-of-00002.safetensors": {
+ "size": 5329398688,
+ "sha256": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61"
+ },
+ "model.safetensors-00002-of-00002.safetensors": {
+ "size": 3990429408,
+ "sha256": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188"
+ },
+ "model.safetensors.index.json": {
+ "size": 76196,
+ "sha256": "cf3f798ee02ba45f9622aa8892a47369ab667d0afbf154ee7c2212de42e6302d"
+ },
+ "preprocessor_config.json": {
+ "size": 390,
+ "sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
+ },
+ "tokenizer.json": {
+ "size": 12807982,
+ "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42"
+ },
+ "tokenizer_config.json": {
+ "size": 16710,
+ "sha256": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8"
+ },
+ "video_preprocessor_config.json": {
+ "size": 385,
+ "sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13"
+ },
+ "vocab.json": {
+ "size": 6722759,
+ "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003"
+ }
+ }
+ },
+ {
+ "name": "cua-s1-4b-0.2",
+ "repo_id": "cua-ai/cua-s1-4b-0.2",
+ "revision": "16818868b0cc7813808aae4e87b417657046ab79",
+ "role": "adapter",
+ "files": {
+ ".gitattributes": {
+ "size": 1519,
+ "sha256": "11ad7efa24975ee4b0c3c3a38ed18737f0658a5f75a0a96787b576a78a023361"
+ },
+ "README.md": {
+ "size": 1134,
+ "sha256": "0e89cb4aea205e08c13f8af7fbfdfd0854ba0f6c57602071238771d0e1f8c7cd"
+ },
+ "multimodal/adapter_config.json": {
+ "size": 1129,
+ "sha256": "f80edac43dd7605317ae8189613c75d13a029fa5d8a4870c04f1ba7097a8fab5"
+ },
+ "multimodal/adapter_model.safetensors": {
+ "size": 101665112,
+ "sha256": "38ecd5a9191a436c1739db29104517f0aff4c70d0736d9cd810d3d49cfd652ea"
+ },
+ "text/adapter_config.json": {
+ "size": 1091,
+ "sha256": "c246fce1fe1d44160ae5f246f9881dfeabb4fcd67fe4a777cef10a1dbcf16540"
+ },
+ "text/adapter_model.safetensors": {
+ "size": 84968408,
+ "sha256": "9b59c5aed96171a50b26526613766bbf44347a5c7af70f81efe6bcc6e9dbfb0e"
+ }
+ }
+ }
+ ]
+}
diff --git a/scripts/ci/linux/run-rust-e2e.sh b/scripts/ci/linux/run-rust-e2e.sh
index cfe0749080..504c749aa0 100755
--- a/scripts/ci/linux/run-rust-e2e.sh
+++ b/scripts/ci/linux/run-rust-e2e.sh
@@ -31,6 +31,22 @@ done
case "$SUITE" in
shared|native|capture|all) ;;
+ s1-perception)
+ # Model-backed perception decision row. It is not part of `all`: it needs
+ # the published cua-perception catalog and the pinned Cua-S1-4B weights,
+ # which only .github/workflows/ci-cua-s1-weights.yml provisions.
+ if [[ -n "${WAYLAND_DISPLAY:-}" && -z "${DISPLAY:-}" ]]; then
+ echo "the s1-perception lane runs on X11 only" >&2
+ exit 2
+ fi
+ for required in CUA_E2E_PERCEPTION_CATALOG CUA_E2E_PERCEPTION_VERSION \
+ CUA_E2E_S1_PYTHON S1_BASE_MODEL_PATH S1_ADAPTER_PATH; do
+ if [[ -z "${!required:-}" ]]; then
+ echo "${required} is required for the s1-perception lane" >&2
+ exit 2
+ fi
+ done
+ ;;
*) echo "unsupported internal lane: $SUITE" >&2; exit 2 ;;
esac
@@ -127,6 +143,7 @@ if [[ "${BUILD_FIXTURES}" == 1 ]]; then
fi
;;
capture) FIXTURE_TARGETS="electron,gtk3" ;;
+ s1-perception) FIXTURE_TARGETS="electron" ;;
*)
FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri},gtk3"
if [[ -z "${WAYLAND_DISPLAY:-}" || -n "${DISPLAY:-}" ]]; then
@@ -408,6 +425,13 @@ if [[ "${SUITE}" == capture || "${SUITE}" == all ]]; then
fi
fi
+if [[ "${SUITE}" == s1-perception ]]; then
+ run_test perception-s1-decision-loop \
+ cargo test -p cua-driver-e2e "${CARGO_DRIVER_FEATURE_ARGS[@]}" \
+ --test perception_s1_decision_loop_test -- \
+ --ignored --nocapture --test-threads=1
+fi
+
if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; then
run_computer_history_gate
fi