Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
ci(cua-s1): add a weights-backed S1 smoke and an S1 + OmniParser Driver E2E row (#4253)
Francesco Bonaccie1c44bd87f55
diff --git a/.github/scripts/tests/test_ci_cua_s1_weights_workflow.py b/.github/scripts/tests/test_ci_cua_s1_weights_workflow.pynew file mode 100644index 0000000000..00e1b36836--- /dev/null+++ b/.github/scripts/tests/test_ci_cua_s1_weights_workflow.py@@ -0,0 +1,200 @@+"""Keep the weights-backed Cua-S1 checks pinned, verified, secret-free, and cache-free."""++from __future__ import annotations++import importlib.util+import json+import re+from pathlib import Path++import pytest+import yaml++ROOT = Path(__file__).resolve().parents[3]+WORKFLOW = ROOT / ".github/workflows/ci-cua-s1-weights.yml"+LOCK = ROOT / "libs/cua-s1/ci/weights.lock.json"+FETCH = ROOT / "libs/cua-s1/ci/fetch_pinned_weights.py"+README = ROOT / "libs/cua-s1/README.md"+LINUX_RUNNER = ROOT / "scripts/ci/linux/run-rust-e2e.sh"+ROW = "perception_s1_decision_loop_test"+++def load() -> dict:+ return yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))+++def fetch_module():+ spec = importlib.util.spec_from_file_location("fetch_pinned_weights", FETCH)+ module = importlib.util.module_from_spec(spec)+ assert spec.loader is not None+ spec.loader.exec_module(module)+ return module+++def test_workflow_is_read_only_secret_free_and_never_caches_weights() -> None:+ text = WORKFLOW.read_text(encoding="utf-8")+ workflow = load()+ assert workflow["permissions"] == {"contents": "read"}+ assert "secrets." not in text+ assert "HF_TOKEN" not in text+ assert "pull_request_target" not in workflow[True]+ # The weights exceed half of the repository's 10 GB Actions cache.+ assert "actions/cache" not in text+ for job in workflow["jobs"].values():+ for step in job["steps"]:+ if "setup-uv" in step.get("uses", ""):+ assert step["with"]["cache-dependency-glob"] == "libs/cua-s1/python/uv.lock"+++def test_triggers_are_nightly_dispatch_and_path_filtered_pull_requests() -> None:+ triggers = load()[True]+ assert triggers["schedule"]+ assert "workflow_dispatch" in triggers+ assert "push" not in triggers+ paths = triggers["pull_request"]["paths"]+ for required in (+ "libs/cua-s1/**",+ "libs/cua-driver/examples/jev-use/python/choose_decision.py",+ "libs/cua-driver/examples/jev-use/python/decision_models.py",+ "libs/cua-driver/examples/jev-use/verify_decision_cli.py",+ "libs/cua-driver/examples/jev-use/fixtures/jev-choice-*",+ f"libs/cua-driver/rust/crates/cua-driver-e2e/tests/{ROW}.rs",+ ".github/workflows/ci-cua-s1-weights.yml",+ ):+ assert required in paths, required+++def test_jobs_use_standard_hosted_linux_and_cpu_without_float32() -> None:+ workflow = load()+ for name, job in workflow["jobs"].items():+ assert job["runs-on"] == "ubuntu-latest", name+ options = workflow[True]["workflow_dispatch"]["inputs"]["dtype"]["options"]+ assert "float32" not in options # 18.7 GB of float32 weights exceed 16 GB of RAM+ assert workflow["jobs"]["smoke"]["env"]["S1_DEVICE"] == "cpu"+ assert workflow["jobs"]["driver-e2e"]["env"]["S1_DEVICE"] == "cpu"+++def test_smoke_runs_the_documented_verifier_in_both_modalities() -> None:+ steps = {step.get("name"): step for step in load()["jobs"]["smoke"]["steps"]}+ documented = steps["Run the documented chooser smoke (cold process per decision)"]["run"]+ assert "verify_decision_cli.py" in documented+ assert "for modality in text multimodal" in documented+ assert "--fixture negative --expected-id abstain" in documented+ assert "jev-choice-request-v1.png" in documented+ assert "jev-choice-negative-v1.png" in documented+ assert "fetch_pinned_weights.py" in steps["Download and verify the pinned weights"]["run"]+ assert "GITHUB_STEP_SUMMARY" in steps["Publish latency and memory summary"]["run"]+++def test_fixture_screenshots_exist_for_both_fixtures() -> None:+ fixtures = ROOT / "libs/cua-driver/examples/jev-use/fixtures"+ for stem in ("jev-choice-request-v1", "jev-choice-negative-v1"):+ assert (fixtures / f"{stem}.json").is_file()+ assert (fixtures / f"{stem}.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n")+++def test_lock_pins_match_the_documented_revisions() -> None:+ fetch = fetch_module()+ artifacts = {item["repo_id"]: item for item in fetch.load_lock(LOCK)}+ assert set(artifacts) == {"Qwen/Qwen3.5-4B", "cua-ai/cua-s1-4b-0.2"}+ readme = README.read_text(encoding="utf-8")+ for repo_id, artifact in artifacts.items():+ assert re.search(+ rf"{re.escape(repo_id)}\s*\\?\n?\s*--revision {artifact['revision']}", readme+ ), repo_id+ base = artifacts["Qwen/Qwen3.5-4B"]["files"]+ assert {+ "model.safetensors-00001-of-00002.safetensors",+ "model.safetensors-00002-of-00002.safetensors",+ "config.json",+ "tokenizer.json",+ } <= set(base)+ adapter = artifacts["cua-ai/cua-s1-4b-0.2"]["files"]+ for modality in ("text", "multimodal"):+ assert f"{modality}/adapter_config.json" in adapter+ assert f"{modality}/adapter_model.safetensors" in adapter+++def test_fetch_rejects_unsafe_locks(tmp_path: Path) -> None:+ fetch = fetch_module()+ document = json.loads(LOCK.read_text(encoding="utf-8"))+ document["artifacts"][1]["files"] = {"../escape": {"size": 1, "sha256": "0" * 64}}+ unsafe = tmp_path / "lock.json"+ unsafe.write_text(json.dumps(document), encoding="utf-8")+ with pytest.raises(ValueError, match="unsafe file path"):+ fetch.load_lock(unsafe)+ document["artifacts"][1]["files"] = {"a": {"size": 1, "sha256": "0" * 64}}+ document["artifacts"][1]["revision"] = "main"+ unsafe.write_text(json.dumps(document), encoding="utf-8")+ with pytest.raises(ValueError, match="full commit SHA"):+ fetch.load_lock(unsafe)+++def test_verify_detects_missing_extra_and_tampered_files(tmp_path: Path) -> None:+ fetch = fetch_module()+ import hashlib++ good = b"weights"+ artifact = {+ "name": "demo",+ "files": {+ "a.safetensors": {"size": len(good), "sha256": hashlib.sha256(good).hexdigest()},+ "sub/b.json": {"size": 2, "sha256": hashlib.sha256(b"{}").hexdigest()},+ },+ }+ (tmp_path / "sub").mkdir()+ (tmp_path / "a.safetensors").write_bytes(good)+ (tmp_path / "sub/b.json").write_bytes(b"{}")+ (tmp_path / ".cache/huggingface").mkdir(parents=True)+ (tmp_path / ".cache/huggingface/meta").write_text("ignored")+ assert fetch.verify_artifact(tmp_path, artifact) == []++ (tmp_path / "a.safetensors").write_bytes(b"weightz")+ (tmp_path / "extra.bin").write_bytes(b"x")+ (tmp_path / "sub/b.json").unlink()+ problems = "\n".join(fetch.verify_artifact(tmp_path, artifact))+ assert "a.safetensors: sha256" in problems+ assert "sub/b.json: missing" in problems+ assert "extra.bin: not in the lock" in problems+++def test_driver_row_runs_through_the_canonical_linux_runner_only_in_its_lane() -> None:+ job = load()["jobs"]["driver-e2e"]+ steps = {step.get("name"): step for step in job["steps"]}+ row = steps["Run the S1 + OmniParser capture-loop row"]+ assert row["env"]["CUA_E2E_INTERNAL_LANE"] == "s1-perception"+ assert "scripts/ci/linux/run-rust-e2e.sh" in row["run"]+ assert "xvfb-run" in row["run"]+ download = steps["Download the published cua-perception release"]["run"]+ assert "releases/download/${tag}" in download+ assert "sha256sum --check --strict" in download+ assert job["env"]["S1_MODALITY"] == "text"++ runner = LINUX_RUNNER.read_text(encoding="utf-8")+ assert f"--test {ROW}" in runner+ lane = runner.split('if [[ "${SUITE}" == s1-perception ]]; then', 1)[1].split("\nfi\n", 1)[0]+ assert ROW in lane+ # The canonical lanes never select the model-backed row.+ assert runner.count(ROW) == 1+ for required in (+ "CUA_E2E_PERCEPTION_CATALOG",+ "CUA_E2E_PERCEPTION_VERSION",+ "CUA_E2E_S1_PYTHON",+ "S1_BASE_MODEL_PATH",+ "S1_ADAPTER_PATH",+ ):+ assert required in runner+++def test_driver_row_records_a_typed_limitation_without_avx512_bf16() -> None:+ steps = load()["jobs"]["driver-e2e"]["steps"]+ names = [step.get("name") or step.get("uses") for step in steps]+ check = steps[names.index("Check the runner CPU against the capture-lifetime budget")]+ assert check["id"] == "cpu"+ assert "avx512_bf16" in check["run"]+ assert "cua-e2e-limitation-v1" in check["run"]+ assert "perception-s1-decision-loop-limitation.json" in check["run"]+ gated = steps[names.index(check["name"]) + 1 : names.index("Publish row summary")]+ assert gated and all(step["if"] == "steps.cpu.outputs.supported == 'true'" for step in gated)+ for name in ("Publish row summary", "Upload row evidence"):+ assert steps[names.index(name)]["if"] == "always()"diff --git a/.github/workflows/ci-cua-s1-weights.yml b/.github/workflows/ci-cua-s1-weights.ymlnew file mode 100644index 0000000000..d4f34b3924--- /dev/null+++ b/.github/workflows/ci-cua-s1-weights.yml@@ -0,0 +1,450 @@+name: "CI: cua-s1 weights"++# Weights-backed Cua-S1-4B checks on a standard GitHub-hosted Linux runner.+# The pinned public weights (~9.5 GB, no token) are downloaded from Hugging+# Face on every run and verified against libs/cua-s1/ci/weights.lock.json.+# They are deliberately not stored in the Actions cache: the repository's+# 10 GB cache limit would evict every other workflow's cache.+on:+ schedule:+ - cron: "37 4 * * *"+ workflow_dispatch:+ inputs:+ dtype:+ # bfloat16 uses the runner's AVX512-BF16; float16 has no native CPU+ # support there and is about 8x slower. float32 weights (18.7 GB)+ # do not fit in the runner's 16 GB of memory.+ description: "Torch dtype for CPU inference"+ required: false+ type: choice+ default: bfloat16+ options:+ - bfloat16+ - float16+ pull_request:+ paths:+ - "libs/cua-s1/**"+ - "libs/cua-driver/examples/jev-use/python/choose_action.py"+ - "libs/cua-driver/examples/jev-use/python/choose_decision.py"+ - "libs/cua-driver/examples/jev-use/python/decision_models.py"+ - "libs/cua-driver/examples/jev-use/python/jev_adapter.py"+ - "libs/cua-driver/examples/jev-use/verify_decision_cli.py"+ - "libs/cua-driver/examples/jev-use/fixtures/jev-choice-*"+ - "libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_s1_decision_loop_test.rs"+ - ".github/workflows/ci-cua-s1-weights.yml"++permissions:+ contents: read++concurrency:+ group: ci-cua-s1-weights-${{ github.ref }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}++env:+ CUA_TELEMETRY_ENABLED: "false"+ HF_HUB_DISABLE_TELEMETRY: "1"+ PYTHONHASHSEED: "0"+ WANDB_MODE: disabled++jobs:+ smoke:+ name: S1 chooser smoke with pinned weights (CPU, text + multimodal)+ runs-on: ubuntu-latest+ timeout-minutes: 90+ env:+ S1_DEVICE: cpu+ S1_DTYPE: ${{ inputs.dtype || 'bfloat16' }}+ defaults:+ run:+ shell: bash+ working-directory: libs/cua-s1/python+ steps:+ - name: Check out Cua+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4++ - name: Free disk space for the weights and the CUDA-enabled torch wheel+ working-directory: .+ run: |+ set -euo pipefail+ df -h /+ # Preinstalled toolchains this job never uses.+ sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup \+ /opt/hostedtoolcache/CodeQL /usr/local/share/boost /usr/share/swift+ sudo docker image prune --all --force >/dev/null 2>&1 || true+ df -h /+ {+ echo "S1_MODELS=$RUNNER_TEMP/s1-models"+ echo "S1_EVIDENCE=$RUNNER_TEMP/s1-evidence"+ } >> "$GITHUB_ENV"+ mkdir -p "$RUNNER_TEMP/s1-models" "$RUNNER_TEMP/s1-evidence"++ - name: Record runner hardware+ working-directory: .+ run: |+ set -euo pipefail+ {+ echo "cpu_model=$(lscpu | sed -n 's/^Model name: *//p' | head -n1)"+ echo "nproc=$(nproc)"+ echo "mem_total_mb=$(free -m | awk '/^Mem:/ {print $2}')"+ echo "mem_available_mb=$(free -m | awk '/^Mem:/ {print $7}')"+ flags="$(grep -m1 '^flags' /proc/cpuinfo)"+ for flag in avx2 avx512f avx512_bf16 amx_bf16 avx512_fp16; do+ if grep -qw "$flag" <<<"$flags"; then echo "$flag=yes"; else echo "$flag=no"; fi+ done+ echo "disk_free=$(df -h / | awk 'NR==2 {print $4}')"+ } | tee "$S1_EVIDENCE/runner.env"++ - name: Set up uv and Python+ uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6+ with:+ python-version: "3.12"+ enable-cache: true+ cache-dependency-glob: libs/cua-s1/python/uv.lock++ - name: Install the pinned four-b inference environment+ run: uv sync --frozen --extra four-b++ - name: Download and verify the pinned weights+ run: |+ set -euo pipefail+ .venv/bin/python ../ci/fetch_pinned_weights.py --dest "$S1_MODELS" \+ | tee "$S1_EVIDENCE/weights.json"+ du -sh "$S1_MODELS"/*+ df -h /++ - name: Run the documented chooser smoke (cold process per decision)+ env:+ HF_HUB_OFFLINE: "1"+ TRANSFORMERS_OFFLINE: "1"+ run: |+ set -euo pipefail+ export S1_BASE_MODEL_PATH="$S1_MODELS/Qwen3.5-4B"+ export S1_ADAPTER_PATH="$S1_MODELS/cua-s1-4b-0.2"+ jev=../../cua-driver/examples/jev-use+ : > "$S1_EVIDENCE/cold.jsonl"+ for modality in text multimodal; do+ for fixture in positive negative; do+ args=(--model s1)+ if [[ "$fixture" == negative ]]; then+ args+=(--fixture negative --expected-id abstain)+ png="$jev/fixtures/jev-choice-negative-v1.png"+ else+ png="$jev/fixtures/jev-choice-request-v1.png"+ fi+ if [[ "$modality" == multimodal ]]; then+ args+=(--screenshot "$png")+ fi+ echo "::group::$modality $fixture"+ timing="$S1_EVIDENCE/cold-$modality-$fixture.time"+ status=0+ /usr/bin/time -v -o "$timing" \+ .venv/bin/python "$jev/verify_decision_cli.py" "${args[@]}" \+ > "$S1_EVIDENCE/cold-$modality-$fixture.out" || status=$?+ cat "$S1_EVIDENCE/cold-$modality-$fixture.out"+ wall="$(sed -n 's/.*Elapsed (wall clock) time (h:mm:ss or m:ss): //p' "$timing")"+ rss_kb="$(sed -n 's/.*Maximum resident set size (kbytes): //p' "$timing")"+ jq -nc --arg modality "$modality" --arg fixture "$fixture" \+ --arg wall "$wall" --argjson rss_kb "${rss_kb:-0}" --argjson status "$status" \+ --arg out "$(cat "$S1_EVIDENCE/cold-$modality-$fixture.out")" \+ '{modality: $modality, fixture: $fixture, exit: $status, wall: $wall,+ peak_rss_mb: ($rss_kb / 1024 | floor), output: $out}' \+ >> "$S1_EVIDENCE/cold.jsonl"+ echo "::endgroup::"+ if [[ "$status" != 0 ]]; then+ echo "::error::documented S1 smoke failed: $modality $fixture"+ exit 1+ fi+ grep -Fq '@16818868b0cc7813808aae4e87b417657046ab79:'"$modality" \+ "$S1_EVIDENCE/cold-$modality-$fixture.out"+ done+ done++ - name: Measure warm per-decision latency+ env:+ HF_HUB_OFFLINE: "1"+ TRANSFORMERS_OFFLINE: "1"+ run: |+ set -euo pipefail+ export S1_BASE_MODEL_PATH="$S1_MODELS/Qwen3.5-4B"+ export S1_ADAPTER_PATH="$S1_MODELS/cua-s1-4b-0.2"+ for modality in text multimodal; do+ echo "::group::warm $modality"+ # Multimodal decisions take minutes on runners without AVX512-BF16.+ repeats=3+ [[ "$modality" == multimodal ]] && repeats=1+ S1_MODALITY="$modality" /usr/bin/time -v -o "$S1_EVIDENCE/warm-$modality.time" \+ .venv/bin/python ../ci/warm_chooser.py --smoke --repeats "$repeats" \+ > "$S1_EVIDENCE/warm-$modality.json"+ cat "$S1_EVIDENCE/warm-$modality.json"+ echo "::endgroup::"+ done++ - name: Publish latency and memory summary+ if: always()+ run: |+ python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY"+ import json, os, statistics+ from pathlib import Path++ evidence = Path(os.environ["S1_EVIDENCE"])+ def read(name):+ path = evidence / name+ return path.read_text() if path.exists() else ""+ runner = dict(line.split("=", 1) for line in read("runner.env").splitlines() if "=" in line)+ print(f"## Cua-S1-4B weights smoke ({os.environ['S1_DEVICE']}, {os.environ['S1_DTYPE']})\n")+ print(f"Runner: {runner.get('cpu_model', '?')}, {runner.get('nproc', '?')} vCPU, "+ f"{runner.get('mem_total_mb', '?')} MB RAM; avx512_bf16={runner.get('avx512_bf16', '?')}, "+ f"amx_bf16={runner.get('amx_bf16', '?')}\n")+ weights = read("weights.json")+ if weights:+ for artifact in json.loads(weights)["artifacts"]:+ print(f"- `{artifact['repo_id']}@{artifact['revision'][:12]}`: {artifact['files']} files, "+ f"{artifact['bytes'] / 1e9:.2f} GB, downloaded in {artifact['download_s']} s, "+ f"verified={artifact['verified']}")+ print()+ cold = [json.loads(line) for line in read("cold.jsonl").splitlines() if line.strip()]+ if cold:+ print("### Documented smoke (`verify_decision_cli.py --model s1`, one cold process per decision)\n")+ print("| Modality | Fixture | Result | Wall (load + decision) | Peak RSS |")+ print("| --- | --- | --- | --- | --- |")+ for row in cold:+ result = row["output"] if row["exit"] == 0 else f"exit {row['exit']}"+ print(f"| {row['modality']} | {row['fixture']} | `{result}` | {row['wall']} | {row['peak_rss_mb']} MB |")+ print()+ print("### Warm decisions (model resident; `libs/cua-s1/ci/warm_chooser.py --smoke`)\n")+ print("| Modality | Load | Fixture | Choice | Latency per decision (ms) | Peak RSS |")+ print("| --- | --- | --- | --- | --- | --- |")+ for modality in ("text", "multimodal"):+ raw = read(f"warm-{modality}.json").strip()+ if not raw:+ print(f"| {modality} | not run | | | | |")+ continue+ report = json.loads(raw.splitlines()[-1])+ for fixture in ("positive", "negative"):+ rows = [d for d in report["decisions"] if d["fixture"] == fixture]+ latencies = [d["latency_ms"] for d in rows if "latency_ms" in d]+ choice = ", ".join(sorted({f"{d.get('selected_id')} ({'ok' if d['ok'] else 'FAIL'})" for d in rows}))+ median = statistics.median(latencies) if latencies else "n/a"+ print(f"| {modality} | {report['load_s']} s | {fixture} | {choice} | "+ f"median {median} (all: {latencies}) | {report['peak_rss_mb']} MB |")+ PY++ - name: Upload smoke evidence+ if: always()+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4+ with:+ name: cua-s1-weights-smoke+ path: ${{ runner.temp }}/s1-evidence+ if-no-files-found: ignore+ retention-days: 14++ driver-e2e:+ # Canonical Linux X11 row for the model-backed perception loop: Driver+ # captures the Electron harness, the published cua-perception extension+ # parses the capture, Cua-S1-4B chooses one region, Driver clicks it with+ # the same capture_id inside the 60 s lifetime, the fixture journal+ # verifies the effect, and stale captures are refused. It runs here, not+ # in e2e-rust-linux.yml, because it needs the 9.5 GB of pinned weights.+ name: Linux X11 / S1 + OmniParser capture-loop row+ runs-on: ubuntu-latest+ timeout-minutes: 90+ env:+ CUA_E2E_SOURCE_SHA: ${{ github.sha }}+ CUA_E2E_PERCEPTION_VERSION: "0.2.1"+ # SHA-256 of cua-perception-0.2.1-x86_64-unknown-linux-gnu.tar.gz as+ # published; the Driver also verifies the signed catalog.+ PERCEPTION_ARCHIVE_SHA256: b4d76b626df9ce1a48b8036f7313412525295d0cb818d82ecbe3d9cca5fd1d08+ S1_DEVICE: cpu+ S1_DTYPE: ${{ inputs.dtype || 'bfloat16' }}+ S1_MODALITY: text+ defaults:+ run:+ shell: bash+ steps:+ - name: Check out Cua+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4++ - name: Check the runner CPU against the capture-lifetime budget+ id: cpu+ run: |+ set -euo pipefail+ # Hosted ubuntu-latest VMs vary: some expose AVX512-BF16, some hide+ # AVX-512 entirely. Without native BF16 a warm Cua-S1-4B text+ # decision took 50-67 s for the 3-candidate fixture (5.5-7.3 s+ # with it), so the row cannot decide inside the 60 s capture+ # lifetime. Record that as a typed limitation instead of a flaky+ # failure or a vacuous pass; the smoke job still runs.+ artifacts=artifacts/cua-driver/linux+ mkdir -p "$artifacts"+ flags="$(grep -m1 '^flags' /proc/cpuinfo)"+ cpu_model="$(lscpu | sed -n 's/^Model name: *//p' | head -n1)"+ echo "cpu_model=$cpu_model nproc=$(nproc) mem_total_mb=$(free -m | awk '/^Mem:/ {print $2}')" \+ | tee "$artifacts/runner.txt"+ if grep -qw avx512_bf16 <<<"$flags"; then+ echo "supported=true" >> "$GITHUB_OUTPUT"+ else+ echo "supported=false" >> "$GITHUB_OUTPUT"+ reason="This hosted runner ($cpu_model) does not expose AVX512-BF16; CPU bfloat16 Cua-S1-4B decisions take longer than the 60 s capture lifetime there."+ jq -n --arg reason "$reason" '{+ schema: "cua-e2e-limitation-v1",+ platform: "linux",+ display_server: "x11",+ harness: "electron",+ test: "perception-s1-decision-loop",+ status: "not_covered",+ reason: $reason+ }' > "$artifacts/perception-s1-decision-loop-limitation.json"+ echo "::warning title=S1 row not run::$reason Re-run the job to draw another runner."+ fi++ - name: Free disk space for the weights, Rust build, and extension+ if: steps.cpu.outputs.supported == 'true'+ run: |+ set -euo pipefail+ df -h /+ sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup \+ /opt/hostedtoolcache/CodeQL /usr/local/share/boost /usr/share/swift+ sudo docker image prune --all --force >/dev/null 2>&1 || true+ df -h /+ {+ echo "S1_MODELS=$RUNNER_TEMP/s1-models"+ echo "S1_BASE_MODEL_PATH=$RUNNER_TEMP/s1-models/Qwen3.5-4B"+ echo "S1_ADAPTER_PATH=$RUNNER_TEMP/s1-models/cua-s1-4b-0.2"+ echo "CUA_E2E_S1_PYTHON=$GITHUB_WORKSPACE/libs/cua-s1/python/.venv/bin/python"+ echo "PERCEPTION_DIR=$RUNNER_TEMP/cua-perception"+ } >> "$GITHUB_ENV"++ - uses: dtolnay/rust-toolchain@4716b85f2fac3e324e64fa2810f6b5c3905760a5 # 1.97.1+ if: steps.cpu.outputs.supported == 'true'+ - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2+ if: steps.cpu.outputs.supported == 'true'+ with:+ workspaces: "libs/cua-driver/rust -> target"+ key: e2e-s1-perception+ save-if: ${{ github.event_name == 'schedule' }}+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4+ if: steps.cpu.outputs.supported == 'true'+ with:+ node-version: "22"++ - name: Install GUI and Rust build dependencies+ if: steps.cpu.outputs.supported == 'true'+ run: |+ sudo apt-get update+ sudo apt-get install -y --no-install-recommends \+ xvfb dbus-x11 at-spi2-core openbox picom python3-gi gir1.2-gtk-3.0 \+ libgtk-3-dev clang pkg-config libdbus-1-dev libpipewire-0.3-dev \+ libspa-0.2-dev libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev \+ libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \+ libayatana-appindicator3-dev librsvg2-dev ffmpeg jq++ - name: Set up uv and Python+ if: steps.cpu.outputs.supported == 'true'+ uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6+ with:+ python-version: "3.12"+ enable-cache: true+ cache-dependency-glob: libs/cua-s1/python/uv.lock++ - name: Install the pinned four-b inference environment+ if: steps.cpu.outputs.supported == 'true'+ working-directory: libs/cua-s1/python+ run: uv sync --frozen --extra four-b++ - name: Download and verify the pinned weights+ if: steps.cpu.outputs.supported == 'true'+ run: |+ set -euo pipefail+ "$CUA_E2E_S1_PYTHON" libs/cua-s1/ci/fetch_pinned_weights.py --dest "$S1_MODELS"+ df -h /++ - name: Download the published cua-perception release+ if: steps.cpu.outputs.supported == 'true'+ run: |+ set -euo pipefail+ tag="cua-perception-v${CUA_E2E_PERCEPTION_VERSION}"+ stem="cua-perception-${CUA_E2E_PERCEPTION_VERSION}-x86_64-unknown-linux-gnu"+ base="https://github.com/${GITHUB_REPOSITORY}/releases/download/${tag}"+ mkdir -p "$PERCEPTION_DIR"+ cd "$PERCEPTION_DIR"+ for asset in "${stem}.catalog.json" "${stem}.tar.gz" SHA256SUMS; do+ curl --fail --silent --show-error --location --retry 3 -o "$asset" "$base/$asset"+ done+ grep -E " \*?(${stem}\.catalog\.json|${stem}\.tar\.gz)$" SHA256SUMS | sha256sum --check --strict+ echo "${PERCEPTION_ARCHIVE_SHA256} ${stem}.tar.gz" | sha256sum --check --strict+ echo "CUA_E2E_PERCEPTION_CATALOG=$PERCEPTION_DIR/${stem}.catalog.json" >> "$GITHUB_ENV"++ - name: Run the S1 + OmniParser capture-loop row+ if: steps.cpu.outputs.supported == 'true'+ env:+ CUA_E2E_INTERNAL_LANE: s1-perception+ run: |+ set -euo pipefail+ # The row's anonymous memory (resident model ~8.4 GB, Driver,+ # Electron, OmniParser worker, recorder) fits in 16 GB. With swap+ # enabled, the kernel swaps the resident model out while it copies+ # the weights, and the next decisions page it back in from disk.+ sudo swapoff -a+ free -m+ vmstat -S M 5 > "$RUNNER_TEMP/vmstat.log" &+ vmstat_pid=$!+ # Top CPU and memory consumers, to explain any latency outlier.+ while sleep 15; do+ date -u +%H:%M:%S+ ps -eo pcpu,rss,comm --sort=-pcpu | head -n 8+ done > "$RUNNER_TEMP/top.log" 2>&1 &+ top_pid=$!+ trap 'kill "$vmstat_pid" "$top_pid" 2>/dev/null || true' EXIT+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \+ dbus-run-session -- bash -lc \+ "openbox >/tmp/cua-openbox.log 2>&1 & picom --backend xrender --config /dev/null >/tmp/cua-picom.log 2>&1 & sleep 2; scripts/ci/linux/run-rust-e2e.sh"++ - name: Publish row summary+ if: always()+ run: |+ set -euo pipefail+ artifacts=artifacts/cua-driver/linux+ {+ echo "## S1 + OmniParser capture-loop row (Linux X11, S1 ${S1_MODALITY} on ${S1_DEVICE}/${S1_DTYPE})"+ echo+ if [[ -f "$artifacts/s1-perception/timing.json" ]]; then+ jq -r '+ "| Measure | Value |", "| --- | --- |",+ "| Extension | cua-perception \(.extension.version) (\(.extension.trust)) |",+ "| S1 checkpoint | `\(.s1.model)` (load \(.s1.load_s) s, resident copy \(.s1.resident_s) s, warm-up decision \(.s1.warmup_ms) ms, anonymous RSS \(.s1.rss_anon_mb) MB) |",+ "| Regions parsed / candidates | \(.regions_parsed) / \(.candidates) |",+ "| S1 choice | `\(.selected_id)` \(.selected_text | tojson) (confidence \(.confidence)) |",+ "| Parse (warm worker) | \(.parse_ms) ms (first parse \(.warm_parse_ms) ms) |",+ "| S1 decision | \(.s1_decision_ms) ms |",+ "| Capture age at click | \(.capture_age_at_click_ms) ms of 60000 |",+ "| Click | \(.click_ms) ms, route \(.click_route) |",+ "| Fixture oracle | \(.fixture_counter) |"+ ' "$artifacts/s1-perception/timing.json"+ elif [[ -f "$artifacts/perception-s1-decision-loop-limitation.json" ]]; then+ echo "**Not run:** $(jq -r .reason "$artifacts/perception-s1-decision-loop-limitation.json")"+ else+ echo "No timing record was produced."+ fi+ echo+ echo "Runner: $(cat "$artifacts/runner.txt" 2>/dev/null || echo unknown)"+ if [[ -f "$RUNNER_TEMP/vmstat.log" ]]; then+ min_free="$(awk 'NR>2 {avail=$4+$5+$6; if (min=="" || avail<min) min=avail} END {print min}' "$RUNNER_TEMP/vmstat.log")"+ echo+ echo "Lowest free + buffer + cache memory during the row: ${min_free} MB (vmstat, 5 s samples)."+ fi+ echo+ if [[ -f "$artifacts/summary.md" ]]; then+ cat "$artifacts/summary.md"+ fi+ } >> "$GITHUB_STEP_SUMMARY"+ cp "$RUNNER_TEMP/vmstat.log" "$RUNNER_TEMP/top.log" "$artifacts/" 2>/dev/null || true++ - name: Upload row evidence+ if: always()+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4+ with:+ name: cua-s1-perception-linux-x11+ path: artifacts/cua-driver/linux+ if-no-files-found: ignore+ compression-level: 0+ retention-days: 14diff --git a/.github/workflows/ci-cua-s1.yml b/.github/workflows/ci-cua-s1.ymlindex 6648da0219..c5df69db16 100644--- a/.github/workflows/ci-cua-s1.yml+++ b/.github/workflows/ci-cua-s1.yml@@ -54,8 +54,8 @@ jobs:- name: Check formatting and lintrun: |- uv run --frozen ruff format --check src tests ../evals ../training- uv run --frozen ruff check src tests ../evals ../training+ uv run --frozen ruff format --check src tests ../evals ../training ../ci+ uv run --frozen ruff check src tests ../evals ../training ../ci- name: Check package importsrun: |diff --git a/.github/workflows/ci-test-scripts.yml b/.github/workflows/ci-test-scripts.ymlindex 0236318069..1e1fd49a36 100644--- a/.github/workflows/ci-test-scripts.yml+++ b/.github/workflows/ci-test-scripts.yml@@ -58,6 +58,8 @@ on:- "scripts/ci/**"- "scripts/ci/macos/**"- "libs/cua-driver/tests/perception-demo/**"+ - "libs/cua-s1/ci/**"+ - "libs/cua-s1/README.md"# Release-wiring, channel, and installer tests read these inputs.- ".github/release-notes/**"- ".github/release-state/**"diff --git a/libs/cua-driver/docs/test-harnesses-guide.md b/libs/cua-driver/docs/test-harnesses-guide.mdindex 79d004d414..d567bbf0d4 100644--- a/libs/cua-driver/docs/test-harnesses-guide.md+++ b/libs/cua-driver/docs/test-harnesses-guide.md@@ -639,5 +639,6 @@ Perception tests have four owners, and each tests different code:| `libs/cua-driver/tests/perception-demo/` | Visual demo evidence sanitizing, caching, and envelopes | `ci-test-scripts.yml` || `.github/scripts/tests/test_cua_perception_*.py`, `test_perception_release.py` | Perception release, review-trigger, and review-pipeline workflows and `.github/scripts` helpers | `ci-test-scripts.yml`, `ci-cua-perception-release.yml` || `libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_capture_loop_test.rs` | Driver's desktop capture, parse, capture-bound click, and reobserve loop with a deterministic developer-only worker | Capture lane of the canonical macOS, Windows, and Linux X11 desktop E2E |+| `libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_s1_decision_loop_test.rs` | The same loop with the published extension (OmniParser) and a resident Cua-S1-4B choosing the region, plus consumed and expired capture refusals | `s1-perception` lane of the Linux X11 runner, selected only by `ci-cua-s1-weights.yml` |Add a perception test to the directory that owns the code it exercises.diff --git a/libs/cua-driver/docs/test-matrix.md b/libs/cua-driver/docs/test-matrix.mdindex 5b3f4cba21..e8dd7cc538 100644--- a/libs/cua-driver/docs/test-matrix.md+++ b/libs/cua-driver/docs/test-matrix.md@@ -120,6 +120,35 @@ instance of the installed app with an isolated extension home and socket.Wayland lanes record a coverage limitation instead of running the row. Modelquality belongs to the `cua-perception` crate and its artifact-gated lanes.+## Harness E2E: Model-Backed Perception Decision Loop++`perception_s1_decision_loop_test.rs` runs the same capture-bound loop with+real models, in the `s1-perception` lane of the Linux X11 runner+(`scripts/ci/linux/run-rust-e2e.sh`). Only+`.github/workflows/ci-cua-s1-weights.yml` selects that lane (nightly, manual+dispatch, and pull requests that touch Cua-S1, the jev-use chooser, or the+row), because it needs the pinned Cua-S1-4B weights:++| Cell | Coverage |+| --- | --- |+| `linux-electron-perception-s1-capture-click-px-foreground` | The published `cua-perception` release, installed from its signed catalog as `publisher-verified`, parses a retained Electron window capture with OmniParser; a resident Cua-S1-4B (text adapter, CPU) chooses one of the parsed text regions for the goal "Increment the counter once."; Driver clicks the region center once with the same `capture_id` within the 60-second capture lifetime; the fixture journal must reach `counter=1`; the consumed capture is refused with `capture_not_found`, an unused capture past its lifetime is refused with `capture_expired`, and the counter stays at 1 |++The model runs on CPU in `bfloat16`, which fits the capture lifetime only on+hosted runners that expose AVX512-BF16. With 16 offered regions, decisions+took 20 to 38 s and the click landed 23 to 42 s after the capture, so the row+offers at most 12. The same+`ubuntu-latest` label sometimes lands on a VM without AVX-512, where one text+decision takes over 50 s. On such a runner the job skips the row, writes a+`cua-e2e-limitation-v1` record (`perception-s1-decision-loop-limitation.json`),+and warns; re-run the job to draw another runner.++The row writes the parse, the `cua.jev_choice_request_v1` request, the S1+decision, and a timing record under `artifacts/cua-driver/linux/s1-perception/`+next to the trajectory video. macOS and Windows lanes do not run it: the+hosted macOS arm64 runner has 7 GB of memory, less than the 9.3 GB of 16-bit+base weights, and the Windows lane has no provisioned weights. Their+capture-bound loop is covered by the model-free row above.+## Harness E2E: Native WindowsWindows native harnesses are repo-local applications built from source:diff --git a/libs/cua-driver/examples/jev-use/fixtures/jev-choice-negative-v1.png b/libs/cua-driver/examples/jev-use/fixtures/jev-choice-negative-v1.pngnew file mode 100644index 0000000000..f219275c5fBinary files /dev/null and b/libs/cua-driver/examples/jev-use/fixtures/jev-choice-negative-v1.png differdiff --git a/libs/cua-driver/examples/jev-use/fixtures/jev-choice-request-v1.png b/libs/cua-driver/examples/jev-use/fixtures/jev-choice-request-v1.pngnew file mode 100644index 0000000000..ca373dff4aBinary files /dev/null and b/libs/cua-driver/examples/jev-use/fixtures/jev-choice-request-v1.png differdiff --git a/libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_s1_decision_loop_test.rs b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_s1_decision_loop_test.rsnew file mode 100644index 0000000000..b8e34ea28e--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver-e2e/tests/perception_s1_decision_loop_test.rs@@ -0,0 +1,712 @@+//! Canonical desktop row for the model-backed perception decision loop.+//!+//! The row drives the complete loop that `jev-use` documents for a local+//! decision model, with real models at every step:+//!+//! 1. Cua Driver captures the Electron web harness window and retains the+//! capture (`get_window_state` -> `capture_id`).+//! 2. The published, publisher-verified `cua-perception` extension, installed+//! from its signed release catalog, parses that exact capture with+//! OmniParser (`parse_visual_regions`).+//! 3. Cua-S1-4B, loaded once from the pinned public weights and kept resident+//! (`libs/cua-s1/ci/warm_chooser.py`), chooses one closed candidate built+//! from the parsed text regions.+//! 4. Driver clicks the chosen region's center once with the same+//! `capture_id`, inside the 60-second capture lifetime.+//! 5. The fixture's loopback journal, which the Driver never reads, proves the+//! effect (`counter=1`).+//! 6. Stale captures are refused without effect: the consumed capture returns+//! `capture_not_found`, and an unused capture older than the lifetime+//! returns `capture_expired`. The counter stays at 1.+//!+//! The row needs about 10 GB of free memory for the model, so it runs only in+//! the weights-backed Linux X11 lane (`.github/workflows/ci-cua-s1-weights.yml`+//! through `scripts/ci/linux/run-rust-e2e.sh` with the `s1-perception` lane).+//! That lane supplies:+//!+//! - `CUA_E2E_PERCEPTION_CATALOG`: the release catalog JSON, with its archive+//! next to it;+//! - `CUA_E2E_PERCEPTION_VERSION`: the extension version the catalog names;+//! - `CUA_E2E_S1_PYTHON`: a Python with the `cua-s1[four-b]` environment;+//! - `S1_BASE_MODEL_PATH`, `S1_ADAPTER_PATH`, `S1_DEVICE`, `S1_DTYPE`, and+//! optionally `S1_MODALITY`, exactly as the jev-use chooser reads them.+//!+//! ```text+//! cargo test -p cua-driver-e2e --test perception_s1_decision_loop_test -- \+//! --ignored --nocapture --test-threads=1+//! ```++#![cfg(target_os = "linux")]++use std::collections::HashSet;+use std::fs;+use std::io::{BufRead, BufReader, Write};+use std::path::{Path, PathBuf};+use std::process::{Child, ChildStdin, Command, Stdio};+use std::sync::mpsc::{self, Receiver};+use std::thread;+use std::time::{Duration, Instant};++use cua_driver_testkit::e2e::{+ execute_case, recording_evidence, shared_web_route, CaseSpec, Delivery, DisplayServer,+ Evidence, Observation, OracleKind, Platform, Scope, Targeting,+};+use cua_driver_testkit::{+ driver_binary, harness_app, spawn_in_job, Driver, FixtureJournal, McpDriver, ToolResponse,+};+use serde_json::{json, Value};++const FIXTURE_TITLE: &str = "CuaTestHarness Electron";+const GOAL: &str = "Increment the counter once.";+/// Driver's capture lifetime (`CaptureRegistryConfig::default().ttl`).+const CAPTURE_TTL: Duration = Duration::from_secs(60);+/// S1 scores one letter per option (at most 26 with reobserve/abstain). Each+/// offered region lengthens the single prefill pass. On a 4-vCPU hosted runner+/// with AVX512-BF16, 16 regions took 20 to 38 s per decision while the row+/// recorded video, so 12 keep parse + decision + click well inside the+/// capture lifetime.+const MAX_REGION_CANDIDATES: usize = 12;+const S1_READY_TIMEOUT: Duration = Duration::from_secs(900);+const S1_DECISION_TIMEOUT: Duration = Duration::from_secs(120);++fn required_env(name: &str) -> String {+ std::env::var(name)+ .ok()+ .filter(|value| !value.trim().is_empty())+ .unwrap_or_else(|| panic!("{name} is required by the S1 perception row"))+}++fn repo_root() -> PathBuf {+ Path::new(env!("CARGO_MANIFEST_DIR"))+ .ancestors()+ .nth(5)+ .expect("repository root above crates/cua-driver-e2e")+ .to_path_buf()+}++fn evidence_dir() -> PathBuf {+ let root = std::env::var_os("CUA_E2E_RESULTS_FILE")+ .map(PathBuf::from)+ .and_then(|results| results.parent().map(Path::to_path_buf))+ .unwrap_or_else(std::env::temp_dir);+ let directory = root.join("s1-perception");+ fs::create_dir_all(&directory).expect("create S1 perception evidence directory");+ directory+}++fn write_evidence(name: &str, value: &Value) {+ let path = evidence_dir().join(name);+ fs::write(&path, serde_json::to_vec_pretty(value).unwrap())+ .unwrap_or_else(|error| panic!("write {path:?}: {error}"));+}++// ---------------------------------------------------------------- S1++/// One resident Cua-S1-4B chooser speaking JSON lines over stdio.+struct WarmChooser {+ child: Child,+ stdin: ChildStdin,+ lines: Receiver<String>,+ ready: Value,+}++impl WarmChooser {+ fn start() -> Self {+ let python = required_env("CUA_E2E_S1_PYTHON");+ for name in ["S1_BASE_MODEL_PATH", "S1_ADAPTER_PATH"] {+ required_env(name);+ }+ let script = repo_root().join("libs/cua-s1/ci/warm_chooser.py");+ let mut child = Command::new(python)+ .arg(&script)+ .arg("--warmup")+ .env("HF_HUB_OFFLINE", "1")+ .env("TRANSFORMERS_OFFLINE", "1")+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::inherit())+ .spawn()+ .unwrap_or_else(|error| panic!("start {script:?}: {error}"));+ let stdin = child.stdin.take().expect("chooser stdin");+ let stdout = child.stdout.take().expect("chooser stdout");+ let (sender, lines) = mpsc::channel();+ thread::spawn(move || {+ for line in BufReader::new(stdout).lines().map_while(Result::ok) {+ if sender.send(line).is_err() {+ break;+ }+ }+ });+ let started = Instant::now();+ let line = lines+ .recv_timeout(S1_READY_TIMEOUT)+ .unwrap_or_else(|_| panic!("S1 chooser was not ready within {S1_READY_TIMEOUT:?}"));+ let ready: Value = serde_json::from_str(&line).expect("S1 ready line is JSON");+ assert_eq!(ready["ready"], true, "S1 chooser failed to start: {ready}");+ eprintln!(+ "[s1-perception] chooser ready in {:.1}s: {ready}",+ started.elapsed().as_secs_f64()+ );+ Self {+ child,+ stdin,+ lines,+ ready,+ }+ }++ fn decide(&mut self, request: &Value, screenshot: Option<&Path>) -> (Value, u64) {+ let message = json!({"request": request, "screenshot": screenshot});+ writeln!(self.stdin, "{message}").expect("send request to S1 chooser");+ self.stdin.flush().expect("flush S1 chooser request");+ let line = self+ .lines+ .recv_timeout(S1_DECISION_TIMEOUT)+ .unwrap_or_else(|_| panic!("S1 decision took longer than {S1_DECISION_TIMEOUT:?}"));+ let reply: Value = serde_json::from_str(&line).expect("S1 reply is JSON");+ assert!(reply.get("error").is_none(), "S1 chooser refused: {reply}");+ let latency = reply["latency_ms"].as_u64().expect("S1 latency_ms");+ (reply["decision"].clone(), latency)+ }+}++impl Drop for WarmChooser {+ fn drop(&mut self) {+ let _ = self.child.kill();+ let _ = self.child.wait();+ }+}++// ---------------------------------------------------------------- extension++fn run_driver_cli(home: &Path, args: &[&str]) -> std::process::Output {+ Command::new(driver_binary())+ .args(args)+ .env("CUA_DRIVER_RS_HOME", home)+ .env("CUA_DRIVER_CLI_TELEMETRY_CHILD", "1")+ .env("CUA_DRIVER_RS_TELEMETRY_ENABLED", "false")+ .stdin(Stdio::null())+ .output()+ .expect("run cua-driver")+}++/// Install the published extension from its signed release catalog, the+/// documented user path, and require publisher-verified trust.+fn install_published_extension(home: &Path) -> Value {+ let catalog = required_env("CUA_E2E_PERCEPTION_CATALOG");+ let version = required_env("CUA_E2E_PERCEPTION_VERSION");+ let install = run_driver_cli(+ home,+ &[+ "extension",+ "install",+ "cua-perception",+ "--catalog",+ &catalog,+ ],+ );+ assert!(+ install.status.success(),+ "published extension install failed: stdout={} stderr={}",+ String::from_utf8_lossy(&install.stdout),+ String::from_utf8_lossy(&install.stderr)+ );+ let status = run_driver_cli(home, &["extension", "status", "cua-perception", "--json"]);+ assert!(+ status.status.success(),+ "extension status failed: {}",+ String::from_utf8_lossy(&status.stderr)+ );+ let status: Value = serde_json::from_slice(&status.stdout).expect("extension status JSON");+ assert_eq!(status["installed"], true, "extension status: {status}");+ assert_eq!(+ status["trust"], "publisher-verified",+ "extension status: {status}"+ );+ assert_eq!(+ status["active_version"], version,+ "extension status: {status}"+ );+ status+}++// ---------------------------------------------------------------- fixture++struct Fixture {+ pid: u32,+ window_id: u64,+ journal: FixtureJournal,+}++fn launch_electron(driver: &mut McpDriver) -> Fixture {+ let journal = FixtureJournal::start();+ let cdp_port = std::net::TcpListener::bind(("127.0.0.1", 0))+ .and_then(|listener| listener.local_addr())+ .expect("allocate a loopback CDP port")+ .port();+ let window_ids = |driver: &mut McpDriver| -> HashSet<u64> {+ driver.call("list_windows", json!({})).structured()["windows"]+ .as_array()+ .map(|windows| {+ windows+ .iter()+ .filter_map(|window| window["window_id"].as_u64())+ .collect()+ })+ .unwrap_or_default()+ };+ let before = window_ids(driver);+ let path = harness_app("harness-electron", "CuaTestHarness.Electron");+ assert!(+ path.exists(),+ "the Electron harness was not staged at {path:?}"+ );+ let mut command = Command::new(path);+ command+ .args([+ "--no-sandbox",+ "--disable-gpu",+ "--force-renderer-accessibility",+ ])+ .env("CUA_E2E_FIXTURE_JOURNAL_URL", journal.url())+ .env("CUA_ELECTRON_CDP_PORT", cdp_port.to_string())+ .stdout(Stdio::inherit())+ .stderr(Stdio::inherit());+ let child = spawn_in_job(&mut command).expect("launch the Electron harness");+ driver.reaper().push(child);++ let deadline = Instant::now() + Duration::from_secs(30);+ while Instant::now() < deadline {+ let windows = driver.call("list_windows", json!({}));+ let found = windows.structured()["windows"]+ .as_array()+ .into_iter()+ .flatten()+ .find(|window| {+ window["window_id"]+ .as_u64()+ .is_some_and(|id| !before.contains(&id))+ && window["title"]+ .as_str()+ .is_some_and(|title| title.contains(FIXTURE_TITLE))+ })+ .and_then(|window| {+ Some((+ window["pid"].as_u64()? as u32,+ window["window_id"].as_u64()?,+ ))+ });+ if let Some((pid, window_id)) = found {+ driver.reaper().track_pid(pid);+ while Instant::now() < deadline {+ if journal.text("lbl-counter").as_deref() == Some("counter=0") {+ return Fixture {+ pid,+ window_id,+ journal,+ };+ }+ thread::sleep(Duration::from_millis(100));+ }+ panic!("Electron harness page never published its initial state");+ }+ thread::sleep(Duration::from_millis(250));+ }+ panic!("Electron harness window did not appear");+}++fn capture_window(driver: &mut McpDriver, fixture: &Fixture) -> (String, Instant) {+ let state = driver.call(+ "get_window_state",+ json!({+ "pid": fixture.pid as i64,+ "window_id": fixture.window_id,+ "include_accessibility_tree": false+ }),+ );+ let captured_at = Instant::now();+ assert!(+ !state.is_error(),+ "get_window_state failed: {}",+ state.text()+ );+ let capture_id = state.structured()["capture_id"]+ .as_str()+ .unwrap_or_else(|| panic!("window observation omitted capture_id: {}", state.text()))+ .to_owned();+ (capture_id, captured_at)+}++fn parse_regions(driver: &mut McpDriver, capture_id: &str) -> ToolResponse {+ driver.call(+ "parse_visual_regions",+ json!({+ "capture_id": capture_id,+ "options": {"kinds": ["text", "icon"], "min_confidence": 0.3, "max_regions": 100}+ }),+ )+}++fn validated_parse(+ parsed: &ToolResponse,+ fixture: &Fixture,+ capture_id: &str,+ version: &str,+) -> Value {+ assert!(+ !parsed.is_error(),+ "parse_visual_regions failed: {}; structured={}",+ parsed.text(),+ parsed.structured()+ );+ let result = parsed.structured().clone();+ assert_eq!(result["schema"], "cua.visual_regions_v1");+ assert_eq!(result["capture"]["capture_id"], capture_id);+ assert_eq!(result["capture"]["source"]["kind"], "window");+ assert_eq!(result["capture"]["source"]["pid"], fixture.pid);+ assert_eq!(result["capture"]["source"]["window_id"], fixture.window_id);+ assert_eq!(result["parser"]["extension_id"], "cua-perception");+ assert_eq!(result["parser"]["extension_version"], version);+ assert_eq!(+ result["capture"]["action_coordinate_space"]["kind"], "screenshot_pixels",+ "region centers are clicked in screenshot pixels: {}",+ result["capture"]+ );+ result+}++fn center(region: &Value) -> (f64, f64) {+ let bounds = ®ion["bounds"];+ let value = |key: &str| bounds[key].as_f64().expect("numeric region bounds");+ (+ value("x") + value("width") / 2.0,+ value("y") + value("height") / 2.0,+ )+}++/// Build a `cua.jev_choice_request_v1` from parsed regions with the caller+/// policy jev-use documents: every candidate is a supplied region, plus+/// `reobserve` and `abstain`. Text regions that carry letters are offered in+/// reading order; the request carries exactly those regions.+fn choice_request(parse: &Value, capture_id: &str) -> (Value, Vec<Value>) {+ let mut texts: Vec<Value> = parse["regions"]+ .as_array()+ .into_iter()+ .flatten()+ .filter(|region| region["kind"] == "text")+ .filter(|region| {+ region["text"]+ .as_str()+ .is_some_and(|text| text.chars().filter(|c| c.is_alphabetic()).count() >= 2)+ })+ .cloned()+ .collect();+ texts.sort_by(|left, right| {+ let key = |region: &Value| {+ (+ region["bounds"]["y"].as_i64().unwrap_or(0) / 12,+ region["bounds"]["x"].as_i64().unwrap_or(0),+ )+ };+ key(left).cmp(&key(right))+ });+ texts.truncate(MAX_REGION_CANDIDATES);+ let regions: Vec<Value> = texts+ .iter()+ .map(|region| {+ json!({+ "id": region["id"],+ "kind": "text",+ "bounds": region["bounds"],+ "text": region["text"],+ "confidence": region["confidence"],+ "interactive": region["interactive"],+ })+ })+ .collect();+ let mut candidates: Vec<Value> = texts+ .iter()+ .map(|region| {+ let (x, y) = center(region);+ let text: String = region["text"]+ .as_str()+ .unwrap_or("")+ .chars()+ .take(60)+ .collect();+ json!({+ "id": format!("click-{}", region["id"].as_str().unwrap_or("region")),+ "description": format!("Click the text region {text:?} at ({x:.0},{y:.0})."),+ })+ })+ .collect();+ candidates.push(json!({+ "id": "reobserve",+ "description": "Discard this decision set and obtain a fresh observation."+ }));+ candidates.push(json!({+ "id": "abstain",+ "description": "Stop without acting if no supplied action is safe."+ }));+ (+ json!({+ "schema": "cua.jev_choice_request_v1",+ "goal": GOAL,+ "capture_id": capture_id,+ "regions": regions,+ "history": [],+ "candidates": candidates,+ }),+ texts,+ )+}++fn wait_for_counter(journal: &FixtureJournal, expected: &str, within: Duration) -> bool {+ let deadline = Instant::now() + within;+ loop {+ if journal.text("lbl-counter").as_deref() == Some(expected) {+ return true;+ }+ if Instant::now() >= deadline {+ return false;+ }+ thread::sleep(Duration::from_millis(50));+ }+}++fn click_args(fixture: &Fixture, x: f64, y: f64, capture_id: &str) -> Value {+ json!({+ "pid": fixture.pid as i64,+ "window_id": fixture.window_id,+ "x": x,+ "y": y,+ "capture_id": capture_id,+ "delivery_mode": "foreground"+ })+}++fn assert_refused(response: &ToolResponse, code: &str, what: &str) {+ assert!(+ response.is_error(),+ "{what} authorized a click: {}",+ response.text()+ );+ assert_eq!(+ response.structured()["code"],+ code,+ "{what} used an unexpected refusal: {}",+ response.structured()+ );+ assert_eq!(response.structured()["effect"], "refused", "{what}");+}++// ---------------------------------------------------------------- row++/// S1 chooses among OmniParser regions of a live capture; Driver clicks the+/// choice with that capture inside its lifetime; the fixture journal verifies+/// the effect; consumed and expired captures are refused without effect.+#[test]+#[ignore]+fn s1_chooses_a_parsed_region_and_the_capture_bound_click_is_state_verified() {+ let route = shared_web_route(+ Platform::current(),+ DisplayServer::current(),+ "left_click",+ Targeting::Px,+ Delivery::Foreground,+ )+ .expect("foreground pixel click route");+ let case = CaseSpec::delivered(+ format!(+ "{}-electron-perception-s1-capture-click-px-foreground",+ std::env::consts::OS+ ),+ "electron",+ "electron",+ "perception_s1_capture_click",+ Targeting::Px,+ Delivery::Foreground,+ Scope::Window,+ route,+ vec![OracleKind::FixtureState, OracleKind::Protocol],+ );+ let cell_id = case.cell_id.clone();+ execute_case(case, |evidence| {+ let version = required_env("CUA_E2E_PERCEPTION_VERSION");+ // Load and warm the model before any capture exists, like a live+ // agent would; its lifetime starts at the capture, not at startup.+ let mut chooser = WarmChooser::start();++ let home = tempfile::Builder::new()+ .prefix("cua-perception-s1-home-")+ .tempdir()+ .expect("create isolated extension home");+ let status = install_published_extension(home.path());+ let home_str = home.path().to_str().expect("UTF-8 extension home");+ let mut driver =+ McpDriver::spawn_named_with_env(&cell_id, &[("CUA_DRIVER_RS_HOME", home_str)])+ .expect("start the source-built Driver with the extension home");+ *evidence = recording_evidence(driver.recording_dir());+ let fixture = launch_electron(&mut driver);+ let raised = driver.call(+ "bring_to_front",+ json!({"pid": fixture.pid as i64, "window_id": fixture.window_id}),+ );+ eprintln!("[s1-perception] bring_to_front: {}", raised.text());+ thread::sleep(Duration::from_millis(500));++ // Warm the extension worker, and keep this capture unused so it can+ // prove expiry at the end.+ let (expiring_capture, expiring_at) = capture_window(&mut driver, &fixture);+ let warm_started = Instant::now();+ let warm = parse_regions(&mut driver, &expiring_capture);+ validated_parse(&warm, &fixture, &expiring_capture, &version);+ let warm_parse_ms = warm_started.elapsed().as_millis() as u64;+ driver.start_behavior_recording();++ // 1. Observe and retain the native capture.+ let (capture_id, captured_at) = capture_window(&mut driver, &fixture);+ // 2. The published extension parses that exact capture.+ let parse_started = Instant::now();+ let parsed = parse_regions(&mut driver, &capture_id);+ let parse_ms = parse_started.elapsed().as_millis() as u64;+ let parse = validated_parse(&parsed, &fixture, &capture_id, &version);+ assert_eq!(+ fixture.journal.text("lbl-counter").as_deref(),+ Some("counter=0"),+ "parsing must not change fixture state"+ );+ // 3. S1 chooses one closed candidate.+ let (request, offered) = choice_request(&parse, &capture_id);+ write_evidence("parse.json", &parse);+ write_evidence("request.json", &request);+ let (decision, s1_ms) = chooser.decide(&request, None);+ write_evidence("decision.json", &decision);+ let decided_age = captured_at.elapsed();+ assert_eq!(decision["schema"], "cua.decision_choice_v1", "{decision}");+ assert_eq!(decision["capture_id"], capture_id, "{decision}");+ assert_eq!(+ decision["kind"], "selected",+ "S1 did not select an action: {decision}"+ );+ let selected = decision["selected_id"].as_str().expect("selected_id");+ let region = offered+ .iter()+ .find(|region| format!("click-{}", region["id"].as_str().unwrap_or("")) == selected)+ .unwrap_or_else(|| panic!("S1 selected an unknown candidate: {decision}"));+ let (x, y) = center(region);++ // 4. One capture-bound click inside the capture lifetime.+ let click_age = captured_at.elapsed();+ assert!(+ click_age < CAPTURE_TTL,+ "the loop took {click_age:?}, beyond the {CAPTURE_TTL:?} capture lifetime"+ );+ let args = click_args(&fixture, x, y, &capture_id);+ let click_started = Instant::now();+ let click = driver.call("click", args.clone());+ let click_ms = click_started.elapsed().as_millis() as u64;+ assert!(+ !click.is_error(),+ "capture-bound click failed: {}; structured={}",+ click.text(),+ click.structured()+ );+ // 5. The independent fixture oracle.+ let delivered = wait_for_counter(&fixture.journal, "counter=1", Duration::from_secs(10));+ let timing = json!({+ "cell_id": cell_id,+ "extension": {"version": status["active_version"], "trust": status["trust"]},+ "parser": parse["parser"],+ "s1": chooser.ready,+ "goal": GOAL,+ "regions_parsed": parse["regions"].as_array().map(Vec::len),+ "candidates": request["candidates"].as_array().map(Vec::len),+ "selected_id": selected,+ "selected_text": region["text"],+ "confidence": decision["confidence"],+ "warm_parse_ms": warm_parse_ms,+ "parse_ms": parse_ms,+ "s1_decision_ms": s1_ms,+ "capture_age_at_decision_ms": decided_age.as_millis() as u64,+ "capture_age_at_click_ms": click_age.as_millis() as u64,+ "click_ms": click_ms,+ "click_route": click.action_route(),+ "fixture_counter": fixture.journal.text("lbl-counter"),+ });+ write_evidence("timing.json", &timing);+ eprintln!("[s1-perception] {timing}");+ assert!(+ delivered,+ "S1 chose {selected} ({:?}) but the fixture never reached counter=1: {}",+ region["text"],+ fixture.journal.snapshot()+ );++ // 6a. The consumed capture is refused.+ let reused = driver.call("click", args);+ assert_refused(&reused, "capture_not_found", "a consumed capture");+ // 6b. An unused capture past its lifetime is refused.+ let remaining = CAPTURE_TTL.saturating_sub(expiring_at.elapsed());+ thread::sleep(remaining + Duration::from_secs(2));+ let expired = driver.call("click", click_args(&fixture, x, y, &expiring_capture));+ assert_refused(&expired, "capture_expired", "an expired capture");+ thread::sleep(Duration::from_millis(750));+ assert_eq!(+ fixture.journal.text("lbl-counter").as_deref(),+ Some("counter=1"),+ "a refused stale capture changed the fixture: {}",+ fixture.journal.snapshot()+ );+ Observation::delivered(+ vec![OracleKind::FixtureState, OracleKind::Protocol],+ Evidence::default(),+ )+ });+}++/// The request builder offers only letter-bearing text regions, in reading+/// order, capped for S1's 26-letter option space, plus reobserve/abstain.+#[test]+fn choice_request_offers_bounded_text_regions_in_reading_order() {+ let region = |id: &str, kind: &str, text: &str, x: i64, y: i64| {+ json!({"id": id, "kind": kind, "text": text, "confidence": 0.9, "interactive": false,+ "bounds": {"x": x, "y": y, "width": 40, "height": 20}})+ };+ let mut regions = vec![+ region("text-2", "text", "Reset", 200, 101),+ region("text-1", "text", "Increment", 100, 100),+ region("text-3", "text", "0", 300, 100),+ region("icon-1", "icon", "", 10, 10),+ region("text-4", "text", "Heading", 20, 20),+ ];+ for index in 0..40 {+ regions.push(region(+ &format!("text-x{index}"),+ "text",+ "Filler",+ 10,+ 500 + index * 30,+ ));+ }+ let (request, offered) = choice_request(&json!({"regions": regions}), "capture_x");+ let ids: Vec<&str> = request["candidates"]+ .as_array()+ .unwrap()+ .iter()+ .map(|candidate| candidate["id"].as_str().unwrap())+ .collect();+ assert_eq!(&ids[..3], ["click-text-4", "click-text-1", "click-text-2"]);+ assert_eq!(ids.len(), MAX_REGION_CANDIDATES + 2);+ assert_eq!(&ids[ids.len() - 2..], ["reobserve", "abstain"]);+ assert_eq!(offered.len(), MAX_REGION_CANDIDATES);+ assert_eq!(+ request["regions"].as_array().unwrap().len(),+ MAX_REGION_CANDIDATES+ );+ assert_eq!(request["capture_id"], "capture_x");+ assert_eq!(+ request["candidates"][1]["description"],+ "Click the text region \"Increment\" at (120,110)."+ );+}diff --git a/libs/cua-s1/README.md b/libs/cua-s1/README.mdindex 68d19611c3..28c276e003 100644--- a/libs/cua-s1/README.md+++ b/libs/cua-s1/README.md@@ -168,6 +168,26 @@ times are medians of five warm calls:| `cua-s1-4b-0.2` text | `cpu`, `float32` | 18.3 s | 19.5 s | 32.1 s || `cua-s1-4b-0.2` multimodal | `cpu`, `float32` | 7.8 s | 34.2 s | 50.2 s |+The weights-backed CI job (`.github/workflows/ci-cua-s1-weights.yml`) also+measures a standard GitHub-hosted `ubuntu-latest` runner on every run: 4 vCPU+(AMD EPYC 9V74, two Torch threads), 16 GB of memory, Python 3.12, and the same+lock. The same runner label comes with and without AVX-512, and that decides+the speed of `bfloat16` on CPU. On 2026-09-26 it measured `cua-s1-4b-0.2` on+`cpu` with `bfloat16`:++| Modality | Runner CPU flags | Warm decision, 3-candidate fixture | Cold `verify_decision_cli.py` (load + decision) | Peak RSS |+| ---------- | ---------------- | ---------------------------------- | ----------------------------------------------- | -------- |+| text | AVX512-BF16 | 5.5 to 7.3 s | 13 to 25 s | 9.4 GB |+| multimodal | AVX512-BF16 | 47 to 50 s (1280x800) | 53 to 55 s | 10.6 GB |+| text | AVX2, no AVX-512 | 51 to 68 s | 66 to 74 s | 9.4 GB |+| multimodal | AVX2, no AVX-512 | 360 to 379 s (1280x800) | 364 to 383 s | 10.6 GB |++Use `bfloat16` on CPU. `float16` has no native CPU support on these runners:+the cold verifier took 114 to 191 s for text and about 10 minutes for+multimodal on an AVX512-BF16 runner. `float32` weights (18.7 GB) do not fit in+16 GB. Peak RSS counts memory-mapped weight pages. The job summary of each run+records the runner's flags and the current numbers.+Multimodal rows used 1280x800 screenshots; a 2560x1600 screenshot took 22 to30 s per decision on `mps`, so downscale large captures before scoring. TheCPU rows ran while other workloads kept the host's load average near 16, so@@ -291,7 +311,18 @@ object, names a different `capture_id`, omits or adds a candidate, or selectsanything other than `submit-form`. To check the checked-in negative fixture,add `--fixture negative --expected-id abstain`. To run the multimodaladapter, add `--screenshot <png>` with an image of the fixture's form; the-verifier binds it to the fixture's capture ID.+verifier binds it to the fixture's capture ID. The fixtures directory has one+for each request: `jev-choice-request-v1.png` and `jev-choice-negative-v1.png`.++CI runs this smoke with real weights in `.github/workflows/ci-cua-s1-weights.yml`+(nightly, on manual dispatch, and on pull requests that change Cua-S1 or the+chooser). It downloads the two pinned artifacts on every run and verifies each+file's size and SHA-256 against `libs/cua-s1/ci/weights.lock.json` before+loading it, runs the verifier for both fixtures in text and multimodal modes,+and records warm latency with `libs/cua-s1/ci/warm_chooser.py`, which keeps one+model resident and answers decisions over stdio. The same workflow runs the+Cua Driver desktop row that lets Cua-S1 choose among OmniParser regions of a+live capture (see `libs/cua-driver/docs/test-matrix.md`).To see the full response, run the chooser directly:diff --git a/libs/cua-s1/ci/fetch_pinned_weights.py b/libs/cua-s1/ci/fetch_pinned_weights.pynew file mode 100644index 0000000000..c5361ba487--- /dev/null+++ b/libs/cua-s1/ci/fetch_pinned_weights.py@@ -0,0 +1,145 @@+"""Download the pinned public Cua-S1-4B weights and verify every file.++Reads ``weights.lock.json`` (next to this script by default), downloads each+listed file from Hugging Face at the exact pinned commit into+``<dest>/<artifact name>``, and verifies its size and SHA-256 before anything+loads it. No token is used or needed: every artifact is public.++ python libs/cua-s1/ci/fetch_pinned_weights.py --dest "$RUNNER_TEMP/s1-models"++Prints one JSON summary line. Exits nonzero on any missing, extra, or+mismatched file. ``--verify-only`` checks an existing directory without+network access.+"""++from __future__ import annotations++import argparse+import hashlib+import json+import os+import re+import sys+import time+from pathlib import Path+from typing import Any++LOCK_SCHEMA = "cua-s1/weights-lock/v1"+_REVISION = re.compile(r"[0-9a-f]{40}")+_SHA256 = re.compile(r"[0-9a-f]{64}")+_CHUNK = 8 * 1024 * 1024+++def load_lock(path: Path) -> list[dict[str, Any]]:+ document = json.loads(path.read_text(encoding="utf-8"))+ if document.get("schema") != LOCK_SCHEMA:+ raise ValueError(f"{path}: unsupported schema {document.get('schema')!r}")+ artifacts = document["artifacts"]+ names = set()+ for artifact in artifacts:+ name = artifact["name"]+ if not re.fullmatch(r"[A-Za-z0-9._-]+", name) or name in names:+ raise ValueError(f"invalid or duplicate artifact name: {name!r}")+ names.add(name)+ if not _REVISION.fullmatch(artifact["revision"]):+ raise ValueError(f"{name}: revision must be a full commit SHA")+ if not artifact["files"]:+ raise ValueError(f"{name}: no files listed")+ for relative, expected in artifact["files"].items():+ parts = Path(relative).parts+ if Path(relative).is_absolute() or ".." in parts or not parts:+ raise ValueError(f"{name}: unsafe file path {relative!r}")+ if not _SHA256.fullmatch(expected["sha256"]) or int(expected["size"]) < 0:+ raise ValueError(f"{name}/{relative}: invalid size or sha256")+ return artifacts+++def sha256_file(path: Path) -> str:+ digest = hashlib.sha256()+ with path.open("rb") as handle:+ while chunk := handle.read(_CHUNK):+ digest.update(chunk)+ return digest.hexdigest()+++def verify_artifact(root: Path, artifact: dict[str, Any]) -> list[str]:+ """Return a list of problems; empty means every listed file matches."""+ problems = []+ expected_files = artifact["files"]+ for relative, expected in expected_files.items():+ path = root / relative+ if not path.is_file() or path.is_symlink():+ problems.append(f"{artifact['name']}/{relative}: missing")+ continue+ size = path.stat().st_size+ if size != int(expected["size"]):+ problems.append(f"{artifact['name']}/{relative}: size {size} != {expected['size']}")+ continue+ actual = sha256_file(path)+ if actual != expected["sha256"]:+ problems.append(+ f"{artifact['name']}/{relative}: sha256 {actual} != {expected['sha256']}"+ )+ for path in sorted(root.rglob("*")):+ relative = path.relative_to(root).as_posix()+ if relative == ".cache" or relative.startswith(".cache/") or path.is_dir():+ continue+ if relative not in expected_files:+ problems.append(f"{artifact['name']}/{relative}: not in the lock")+ return problems+++def download_artifact(root: Path, artifact: dict[str, Any]) -> None:+ from huggingface_hub import snapshot_download++ snapshot_download(+ repo_id=artifact["repo_id"],+ revision=artifact["revision"],+ local_dir=str(root),+ allow_patterns=sorted(artifact["files"]),+ token=False,+ )+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--lock", type=Path, default=Path(__file__).with_name("weights.lock.json"))+ parser.add_argument("--dest", type=Path, required=True)+ parser.add_argument("--verify-only", action="store_true")+ args = parser.parse_args(argv)++ artifacts = load_lock(args.lock)+ args.dest.mkdir(parents=True, exist_ok=True)+ os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")+ summary: dict[str, Any] = {"artifacts": []}+ problems: list[str] = []+ for artifact in artifacts:+ root = args.dest / artifact["name"]+ started = time.perf_counter()+ if not args.verify_only:+ download_artifact(root, artifact)+ downloaded = time.perf_counter()+ found = verify_artifact(root, artifact)+ problems.extend(found)+ summary["artifacts"].append(+ {+ "name": artifact["name"],+ "repo_id": artifact["repo_id"],+ "revision": artifact["revision"],+ "path": str(root),+ "files": len(artifact["files"]),+ "bytes": sum(int(item["size"]) for item in artifact["files"].values()),+ "download_s": round(downloaded - started, 1),+ "verify_s": round(time.perf_counter() - downloaded, 1),+ "verified": not found,+ }+ )+ summary["verified"] = not problems+ print(json.dumps(summary, separators=(",", ":")))+ for problem in problems:+ print(f"weights verification failed: {problem}", file=sys.stderr)+ return 0 if not problems else 1+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/libs/cua-s1/ci/warm_chooser.py b/libs/cua-s1/ci/warm_chooser.pynew file mode 100644index 0000000000..28e5d67388--- /dev/null+++ b/libs/cua-s1/ci/warm_chooser.py@@ -0,0 +1,297 @@+"""Keep one Cua-S1-4B chooser resident and answer decisions over stdio.++The jev-use chooser (`choose_decision.py --model s1`) loads the model for every+decision, which takes tens of seconds on CPU. Live loops must decide within the+Cua Driver's 60-second capture lifetime, so the S1 docs recommend keeping one+loaded model resident. This helper does exactly that for CI and the desktop+E2E row, using the repository's validation and scoring code unchanged+(`choose_action.validate_request`, `decision_models.S1DecisionModel`,+`decision_models.choose`). It never dispatches actions.++Configuration is the chooser's: `S1_BASE_MODEL_PATH`, `S1_ADAPTER_PATH`,+`S1_DEVICE`, `S1_DTYPE`, and `S1_MODALITY` (one modality per process).+`S1_RESIDENT=0` keeps the weights memory-mapped instead of copying them into+process memory (see `make_resident`).++Serve mode (default) prints one `{"ready": true, ...}` line after loading,+then reads one JSON object per stdin line:++ {"request": <cua.jev_choice_request_v1>, "screenshot": "<png path>"}++`screenshot` is required for multimodal and is bound to the request's+capture. Each reply is one line: `{"decision": <cua.decision_choice_v1>,+"latency_ms": <int>}` or `{"error": "<detail>"}`.++Smoke mode (`--smoke`) runs the checked-in positive and negative fixtures+(with their screenshots for multimodal), checks them exactly as+`verify_decision_cli.py` does, prints one JSON report, and exits nonzero on+any mismatch.+"""++from __future__ import annotations++import argparse+import json+import os+import platform+import resource+import sys+import time+from pathlib import Path+from typing import Any++REPO = Path(__file__).resolve().parents[3]+JEV_USE = REPO / "libs/cua-driver/examples/jev-use"+sys.path.insert(0, str(JEV_USE / "python"))++from choose_action import MAX_INPUT_BYTES, validate_request # noqa: E402+from choose_decision import _resolve_s1_adapter # noqa: E402+from decision_models import ( # noqa: E402+ S1_MODALITIES,+ DecisionRequest,+ S1DecisionModel,+ choose,+ s1_model_identity,+)++FIXTURES = {+ "positive": ("jev-choice-request-v1", "submit-form"),+ "negative": ("jev-choice-negative-v1", "abstain"),+}+RESPONSE_KEYS = {+ "schema",+ "kind",+ "capture_id",+ "selected_id",+ "model",+ "confidence",+ "probabilities",+ "reason",+}+++def peak_rss_mb() -> float:+ peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss+ # Linux reports KiB; macOS reports bytes.+ return round(peak / (1024 * 1024 if sys.platform == "darwin" else 1024), 1)+++def make_resident(model: Any) -> None:+ """Copy memory-mapped weights into process memory.++ Transformers maps the safetensors shards and loads same-dtype CPU weights+ as file-backed pages. Under desktop memory pressure (Driver, browser,+ OmniParser worker, video recorder) the kernel evicts those clean pages,+ and every forward pass rereads up to 9 GB from disk. Anonymous copies stay+ resident; one tensor is copied at a time, so the peak cost is one tensor.+ """+ import torch++ with torch.no_grad():+ for tensor in [*model.parameters(), *model.buffers()]:+ tensor.data = tensor.data.clone()+++def rss_breakdown_mb() -> dict[str, float]:+ """Current anonymous and file-backed resident memory (Linux only)."""+ try:+ status = Path("/proc/self/status").read_text(encoding="utf-8")+ except OSError:+ return {}+ values = {}+ for line in status.splitlines():+ key, _, rest = line.partition(":")+ if key in {"RssAnon", "RssFile"}:+ values[key.lower().replace("rss", "rss_") + "_mb"] = round(+ int(rest.split()[0]) / 1024, 1+ )+ return values+++class WarmChooser:+ def __init__(self) -> None:+ base = Path(os.environ["S1_BASE_MODEL_PATH"]).expanduser()+ adapter = Path(os.environ["S1_ADAPTER_PATH"]).expanduser()+ if not base.is_dir() or not adapter.is_dir():+ raise ValueError("S1_BASE_MODEL_PATH and S1_ADAPTER_PATH must be local directories")+ self.modality = os.environ.get("S1_MODALITY", "").strip() or "text"+ if self.modality not in S1_MODALITIES:+ raise ValueError(f"S1_MODALITY must be one of {', '.join(S1_MODALITIES)}")+ _resolve_s1_adapter(adapter, self.modality)+ self.device = os.environ.get("S1_DEVICE", "cpu")+ self.dtype = os.environ.get("S1_DTYPE", "bfloat16")+ self.name = s1_model_identity(+ adapter,+ self.modality,+ repo_id=os.environ.get("S1_ADAPTER_ID"),+ revision=os.environ.get("S1_ADAPTER_REVISION"),+ )+ from cua_s1.four_b import FourBModel++ started = time.perf_counter()+ self.scorer = FourBModel(+ base_model=str(base),+ lora_adapter_path=adapter,+ device=self.device,+ dtype=self.dtype,+ modality=self.modality,+ )+ self.scorer.load()+ self.load_s = round(time.perf_counter() - started, 2)+ self.resident_s: float | None = None+ self.warmup_ms: int | None = None+ if os.environ.get("S1_RESIDENT", "1") != "0":+ started = time.perf_counter()+ make_resident(self.scorer._model)+ self.resident_s = round(time.perf_counter() - started, 2)++ def decide(self, raw: Any, screenshot: str | None) -> tuple[dict[str, Any], int]:+ if len(json.dumps(raw).encode()) > MAX_INPUT_BYTES:+ raise ValueError("request exceeds input limit")+ request = DecisionRequest.from_validated(validate_request(raw))+ shot = None+ if self.modality == "multimodal":+ if not screenshot or not Path(screenshot).is_file():+ raise ValueError("multimodal S1 requires an existing screenshot path")+ shot = Path(screenshot)+ elif screenshot:+ raise ValueError("screenshot requires S1_MODALITY=multimodal")+ model = S1DecisionModel(+ self.scorer,+ modality=self.modality,+ screenshot_path=shot,+ screenshot_capture_id=request.capture_id if shot else None,+ name=self.name,+ )+ started = time.perf_counter()+ result = choose(model, request).to_wire()+ return result, round((time.perf_counter() - started) * 1000)++ def describe(self) -> dict[str, Any]:+ import torch++ return {+ "model": self.name,+ "modality": self.modality,+ "device": self.device,+ "dtype": self.dtype,+ "load_s": self.load_s,+ "resident_s": self.resident_s,+ "warmup_ms": self.warmup_ms,+ "torch": torch.__version__,+ "torch_threads": torch.get_num_threads(),+ "cpu_count": os.cpu_count(),+ "machine": platform.machine(),+ "peak_rss_mb": peak_rss_mb(),+ **rss_breakdown_mb(),+ }+++def check_response(response: dict[str, Any], request: dict[str, Any], expected_id: str) -> None:+ """The same checks `verify_decision_cli.py` applies to the chooser output."""+ if set(response) != RESPONSE_KEYS or response["schema"] != "cua.decision_choice_v1":+ raise RuntimeError("decision model returned an unsupported response")+ if response["capture_id"] != request["capture_id"]:+ raise RuntimeError("decision model returned a different capture")+ expected_kind = expected_id if expected_id in {"reobserve", "abstain"} else "selected"+ if (response["kind"], response["selected_id"]) != (expected_kind, expected_id):+ raise RuntimeError(+ f"decision model chose {response['kind']}/{response['selected_id']}, "+ f"expected {expected_kind}/{expected_id}"+ )+ if set(response["probabilities"]) != {item["id"] for item in request["candidates"]}:+ raise RuntimeError("decision model returned a different candidate set")+++def smoke(chooser: WarmChooser, repeats: int) -> int:+ report: dict[str, Any] = {**chooser.describe(), "decisions": []}+ failures = 0+ for fixture, (stem, expected_id) in FIXTURES.items():+ request = json.loads((JEV_USE / "fixtures" / f"{stem}.json").read_text(encoding="utf-8"))+ screenshot = (+ str(JEV_USE / "fixtures" / f"{stem}.png") if chooser.modality == "multimodal" else None+ )+ for attempt in range(1, repeats + 1):+ entry: dict[str, Any] = {+ "fixture": fixture,+ "attempt": attempt,+ "expected": expected_id,+ }+ try:+ response, latency_ms = chooser.decide(request, screenshot)+ entry.update(+ latency_ms=latency_ms,+ kind=response["kind"],+ selected_id=response["selected_id"],+ confidence=response["confidence"],+ )+ check_response(response, request, expected_id)+ entry["ok"] = True+ except Exception as error: # noqa: BLE001 - reported, then fails the run+ entry.update(ok=False, error=str(error)[:300])+ failures += 1+ report["decisions"].append(entry)+ print(json.dumps(entry), file=sys.stderr, flush=True)+ report["peak_rss_mb"] = peak_rss_mb()+ report["ok"] = failures == 0+ print(json.dumps(report, separators=(",", ":")), flush=True)+ return 0 if failures == 0 else 1+++def serve(chooser: WarmChooser, out: Any) -> int:+ def reply(value: dict[str, Any]) -> None:+ out.write(json.dumps(value, separators=(",", ":")) + "\n")+ out.flush()++ reply({"ready": True, **chooser.describe()})+ for line in sys.stdin:+ if not line.strip():+ continue+ try:+ message = json.loads(line)+ if not isinstance(message, dict) or "request" not in message:+ raise ValueError("expected an object with a request")+ decision, latency_ms = chooser.decide(message["request"], message.get("screenshot"))+ reply({"decision": decision, "latency_ms": latency_ms, "peak_rss_mb": peak_rss_mb()})+ except (ValueError, KeyError, TypeError, json.JSONDecodeError) as error:+ reply({"error": str(error)[:300]})+ return 0+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--smoke", action="store_true", help="run the fixture checks and exit")+ parser.add_argument("--repeats", type=int, default=2, help="decisions per fixture in --smoke")+ parser.add_argument(+ "--warmup",+ action="store_true",+ help="serve mode: run one positive-fixture decision before reporting ready",+ )+ args = parser.parse_args(argv)++ # Keep the protocol stream clean: libraries may print while loading.+ protocol = sys.stdout+ sys.stdout = sys.stderr+ try:+ chooser = WarmChooser()+ except (KeyError, ValueError, ImportError) as error:+ protocol.write(json.dumps({"ready": False, "error": f"S1 setup failed: {error}"}) + "\n")+ protocol.flush()+ return 2+ if args.smoke:+ sys.stdout = protocol+ return smoke(chooser, max(1, args.repeats))+ if args.warmup:+ stem, _ = FIXTURES["positive"]+ request = json.loads((JEV_USE / "fixtures" / f"{stem}.json").read_text(encoding="utf-8"))+ screenshot = (+ str(JEV_USE / "fixtures" / f"{stem}.png") if chooser.modality == "multimodal" else None+ )+ _, latency_ms = chooser.decide(request, screenshot)+ chooser.warmup_ms = latency_ms+ print(f"warmup decision: {latency_ms} ms", file=sys.stderr, flush=True)+ return serve(chooser, protocol)+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/libs/cua-s1/ci/weights.lock.json b/libs/cua-s1/ci/weights.lock.jsonnew file mode 100644index 0000000000..dfb9748b45--- /dev/null+++ b/libs/cua-s1/ci/weights.lock.json@@ -0,0 +1,102 @@+{+ "schema": "cua-s1/weights-lock/v1",+ "description": "Pinned public Hugging Face artifacts for the weights-backed Cua-S1-4B smoke. Every listed file is downloaded at the pinned revision and verified by size and SHA-256; keep in sync with libs/cua-s1/README.md.",+ "artifacts": [+ {+ "name": "Qwen3.5-4B",+ "repo_id": "Qwen/Qwen3.5-4B",+ "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",+ "role": "base",+ "files": {+ ".gitattributes": {+ "size": 1570,+ "sha256": "34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930"+ },+ "LICENSE": {+ "size": 11544,+ "sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a"+ },+ "README.md": {+ "size": 77661,+ "sha256": "1406be1b6b8fd8a6545870da516912804756593628a1d0fb0a7965211e82a7bb"+ },+ "chat_template.jinja": {+ "size": 7756,+ "sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715"+ },+ "config.json": {+ "size": 3161,+ "sha256": "ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670"+ },+ "merges.txt": {+ "size": 3353259,+ "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d"+ },+ "model.safetensors-00001-of-00002.safetensors": {+ "size": 5329398688,+ "sha256": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61"+ },+ "model.safetensors-00002-of-00002.safetensors": {+ "size": 3990429408,+ "sha256": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188"+ },+ "model.safetensors.index.json": {+ "size": 76196,+ "sha256": "cf3f798ee02ba45f9622aa8892a47369ab667d0afbf154ee7c2212de42e6302d"+ },+ "preprocessor_config.json": {+ "size": 390,+ "sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"+ },+ "tokenizer.json": {+ "size": 12807982,+ "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42"+ },+ "tokenizer_config.json": {+ "size": 16710,+ "sha256": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8"+ },+ "video_preprocessor_config.json": {+ "size": 385,+ "sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13"+ },+ "vocab.json": {+ "size": 6722759,+ "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003"+ }+ }+ },+ {+ "name": "cua-s1-4b-0.2",+ "repo_id": "cua-ai/cua-s1-4b-0.2",+ "revision": "16818868b0cc7813808aae4e87b417657046ab79",+ "role": "adapter",+ "files": {+ ".gitattributes": {+ "size": 1519,+ "sha256": "11ad7efa24975ee4b0c3c3a38ed18737f0658a5f75a0a96787b576a78a023361"+ },+ "README.md": {+ "size": 1134,+ "sha256": "0e89cb4aea205e08c13f8af7fbfdfd0854ba0f6c57602071238771d0e1f8c7cd"+ },+ "multimodal/adapter_config.json": {+ "size": 1129,+ "sha256": "f80edac43dd7605317ae8189613c75d13a029fa5d8a4870c04f1ba7097a8fab5"+ },+ "multimodal/adapter_model.safetensors": {+ "size": 101665112,+ "sha256": "38ecd5a9191a436c1739db29104517f0aff4c70d0736d9cd810d3d49cfd652ea"+ },+ "text/adapter_config.json": {+ "size": 1091,+ "sha256": "c246fce1fe1d44160ae5f246f9881dfeabb4fcd67fe4a777cef10a1dbcf16540"+ },+ "text/adapter_model.safetensors": {+ "size": 84968408,+ "sha256": "9b59c5aed96171a50b26526613766bbf44347a5c7af70f81efe6bcc6e9dbfb0e"+ }+ }+ }+ ]+}diff --git a/scripts/ci/linux/run-rust-e2e.sh b/scripts/ci/linux/run-rust-e2e.shindex cfe0749080..504c749aa0 100755--- a/scripts/ci/linux/run-rust-e2e.sh+++ b/scripts/ci/linux/run-rust-e2e.sh@@ -31,6 +31,22 @@ donecase "$SUITE" inshared|native|capture|all) ;;+ s1-perception)+ # Model-backed perception decision row. It is not part of `all`: it needs+ # the published cua-perception catalog and the pinned Cua-S1-4B weights,+ # which only .github/workflows/ci-cua-s1-weights.yml provisions.+ if [[ -n "${WAYLAND_DISPLAY:-}" && -z "${DISPLAY:-}" ]]; then+ echo "the s1-perception lane runs on X11 only" >&2+ exit 2+ fi+ for required in CUA_E2E_PERCEPTION_CATALOG CUA_E2E_PERCEPTION_VERSION \+ CUA_E2E_S1_PYTHON S1_BASE_MODEL_PATH S1_ADAPTER_PATH; do+ if [[ -z "${!required:-}" ]]; then+ echo "${required} is required for the s1-perception lane" >&2+ exit 2+ fi+ done+ ;;*) echo "unsupported internal lane: $SUITE" >&2; exit 2 ;;esac@@ -127,6 +143,7 @@ if [[ "${BUILD_FIXTURES}" == 1 ]]; thenfi;;capture) FIXTURE_TARGETS="electron,gtk3" ;;+ s1-perception) FIXTURE_TARGETS="electron" ;;*)FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri},gtk3"if [[ -z "${WAYLAND_DISPLAY:-}" || -n "${DISPLAY:-}" ]]; then@@ -408,6 +425,13 @@ if [[ "${SUITE}" == capture || "${SUITE}" == all ]]; thenfifi+if [[ "${SUITE}" == s1-perception ]]; then+ run_test perception-s1-decision-loop \+ cargo test -p cua-driver-e2e "${CARGO_DRIVER_FEATURE_ARGS[@]}" \+ --test perception_s1_decision_loop_test -- \+ --ignored --nocapture --test-threads=1+fi+if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; thenrun_computer_history_gatefi