Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
ci(cua-sandbox): restore periodic live Fleet E2E (#3022)
r33drichards76ad749d8cc4
diff --git a/.github/scripts/tests/test_periodic_cua_sandbox_live.py b/.github/scripts/tests/test_periodic_cua_sandbox_live.pynew file mode 100644index 0000000000..d27aac2b98--- /dev/null+++ b/.github/scripts/tests/test_periodic_cua_sandbox_live.py@@ -0,0 +1,342 @@+"""Contract tests for the periodic Cua Sandbox live Fleet workflow."""++import json+import os+from pathlib import Path+import shutil+import subprocess+import sys+import tempfile+import unittest++import yaml+++REPO_ROOT = Path(__file__).resolve().parents[3]+WORKFLOW = REPO_ROOT / ".github/workflows/periodic-cua-sandbox-live.yml"+SANDBOX_ROOT = REPO_ROOT / "libs/python/cua-sandbox"+LIVE_TEST = SANDBOX_ROOT / "tests/live/test_fleet_ephemeral.py"+++class TestPeriodicCuaSandboxLive(unittest.TestCase):+ @staticmethod+ def workflow() -> dict[str, object]:+ # BaseLoader keeps GitHub Actions' `on` key as a string.+ return yaml.load(WORKFLOW.read_text(), Loader=yaml.BaseLoader)++ @staticmethod+ def steps_by_name(job: dict[str, object]) -> dict[str, dict[str, object]]:+ return {+ step["name"]: step for step in job["steps"] if isinstance(step, dict) and "name" in step+ }++ def run_prepare_matrix(self, event_name: str, requested_lane: str) -> dict[str, object]:+ prepare_script = self.workflow()["jobs"]["prepare"]["steps"][0]["run"]+ with tempfile.TemporaryDirectory() as temporary_directory:+ github_output = Path(temporary_directory) / "github-output"+ environment = os.environ | {+ "EVENT_NAME": event_name,+ "REQUESTED_LANE": requested_lane,+ "GITHUB_OUTPUT": str(github_output),+ }+ subprocess.run(+ ["bash", "-c", prepare_script],+ env=environment,+ check=True,+ capture_output=True,+ text=True,+ )+ outputs = dict(+ line.split("=", 1) for line in github_output.read_text().splitlines() if "=" in line+ )+ return json.loads(outputs["matrix"])++ def test_trigger_and_lane_structure(self) -> None:+ workflow = self.workflow()+ triggers = workflow["on"]+ self.assertEqual(triggers["schedule"], [{"cron": "7/15 * * * *"}])+ self.assertEqual(triggers["push"]["branches"], ["main"])+ self.assertEqual(+ triggers["push"]["paths"],+ [+ "libs/python/cua-sandbox/**",+ "libs/python/cua-fleet/**",+ ".github/workflows/periodic-cua-sandbox-live.yml",+ ".github/scripts/tests/test_periodic_cua_sandbox_live.py",+ ],+ )+ inputs = triggers["workflow_dispatch"]["inputs"]+ self.assertEqual(+ inputs["lane"]["options"],+ ["both", "main-source", "published-package"],+ )+ self.assertEqual(inputs["force_failure"]["type"], "boolean")++ prepare = workflow["jobs"]["prepare"]+ self.assertEqual(prepare["outputs"]["matrix"], "${{ steps.matrix.outputs.matrix }}")+ prepare_script = prepare["steps"][0]["run"]+ self.assertIn('[[ "$EVENT_NAME" == "push" ]]', prepare_script)+ self.assertIn('"lane":"main-source"', prepare_script)+ self.assertIn('"lane":"published-package"', prepare_script)++ def test_prepare_matrix_selects_lanes_for_each_trigger(self) -> None:+ expected_matrices = {+ ("push", ""): {"include": [{"lane": "main-source"}]},+ ("push", "both"): {"include": [{"lane": "main-source"}]},+ ("push", "published-package"): {"include": [{"lane": "main-source"}]},+ ("schedule", ""): {+ "include": [+ {"lane": "main-source"},+ {"lane": "published-package"},+ ]+ },+ ("schedule", "main-source"): {+ "include": [+ {"lane": "main-source"},+ {"lane": "published-package"},+ ]+ },+ ("workflow_dispatch", "both"): {+ "include": [+ {"lane": "main-source"},+ {"lane": "published-package"},+ ]+ },+ ("workflow_dispatch", "main-source"): {"include": [{"lane": "main-source"}]},+ ("workflow_dispatch", "published-package"): {+ "include": [{"lane": "published-package"}]+ },+ }++ for inputs, expected_matrix in expected_matrices.items():+ with self.subTest(event_name=inputs[0], requested_lane=inputs[1]):+ self.assertEqual(self.run_prepare_matrix(*inputs), expected_matrix)++ def test_docs_describe_the_remediated_workflow(self) -> None:+ docs = (+ REPO_ROOT / "docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.md",+ REPO_ROOT / "docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.md",+ )+ required_contract = (+ "libs/python/cua-fleet/**",+ "Check Fleet OAuth credentials",+ "step-scoped OAuth credentials",+ "CUA_LIVE_E2E_SOURCE_SHA",+ "periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}",+ "Prepare isolated live test suite",+ "CUA_LIVE_E2E_TEST_ROOT",+ 'tee -a "$GITHUB_OUTPUT"',+ "Write controlled failure diagnostics",+ "persistent reconciled resources",+ "claim-only cleanup",+ "cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",+ )+ stale_contract = (+ "Concurrency is scoped\nper lane",+ "python - <<'PY' >> \"$GITHUB_OUTPUT\"",+ "Emergency namespace cleanup",+ "namespace leak",+ "automatic namespace cleanup",+ "leave no test namespace behind",+ "no remaining namespace",+ "namespace is absent",+ )++ for document in docs:+ content = document.read_text()+ with self.subTest(document=document):+ for expected in required_contract:+ self.assertIn(expected, content)+ for stale in stale_contract:+ self.assertNotIn(stale, content)++ def test_live_job_security_and_execution_structure(self) -> None:+ workflow = self.workflow()+ live = workflow["jobs"]["live"]+ steps = self.steps_by_name(live)++ self.assertEqual(live["timeout-minutes"], "25")+ self.assertEqual(+ live["strategy"]["matrix"],+ "${{ fromJSON(needs.prepare.outputs.matrix) }}",+ )+ self.assertEqual(+ live["concurrency"]["group"],+ "periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}",+ )+ self.assertEqual(+ live["concurrency"]["cancel-in-progress"],+ "${{ github.event_name == 'schedule' }}",+ )+ expected_oauth_env = {+ "CUA_CLIENT_ID": "${{ secrets.CUA_CLIENT_ID }}",+ "CUA_CLIENT_SECRET": "${{ secrets.CUA_CLIENT_SECRET }}",+ }+ self.assertNotIn("CUA_CLIENT_ID", live["env"])+ self.assertNotIn("CUA_CLIENT_SECRET", live["env"])+ self.assertNotIn("CUA_API_KEY", live["env"])+ oauth_steps = {+ step["name"]: step["env"]+ for step in live["steps"]+ if "CUA_CLIENT_ID" in step.get("env", {}) or "CUA_CLIENT_SECRET" in step.get("env", {})+ }+ self.assertEqual(+ oauth_steps,+ {+ "Check Fleet OAuth credentials": expected_oauth_env,+ "Run live Fleet smoke": expected_oauth_env,+ },+ )+ self.assertEqual(live["env"]["CUA_LIVE_E2E_EVENT"], "${{ github.event_name }}")+ self.assertEqual(+ live["env"]["CUA_LIVE_E2E_NAMESPACE"],+ "cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",+ )+ self.assertNotIn("github.run_id", live["env"]["CUA_LIVE_E2E_NAMESPACE"])+ self.assertNotIn("github.run_attempt", live["env"]["CUA_LIVE_E2E_NAMESPACE"])++ step_names = [step["name"] for step in live["steps"]]+ self.assertLess(+ step_names.index("Checkout main"),+ step_names.index("Record checked out source SHA"),+ )++ checkout = steps["Checkout main"]+ self.assertEqual(+ checkout["uses"],+ "actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5",+ )+ self.assertEqual(+ checkout["with"]["ref"],+ "${{ github.event_name == 'push' && github.sha || 'main' }}",+ )+ self.assertEqual(+ steps["Set up Python"]["uses"],+ "actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065",+ )+ self.assertIn("-e libs/python/cua-sandbox", steps["Install main source"]["run"])+ self.assertIn("--upgrade cua-sandbox", steps["Install published package"]["run"])++ preflight_step = steps["Check Fleet OAuth credentials"]+ self.assertEqual(preflight_step["env"], expected_oauth_env)+ preflight = preflight_step["run"]+ self.assertIn("CUA_CLIENT_ID", preflight)+ self.assertIn("CUA_CLIENT_SECRET", preflight)+ self.assertIn("exit 1", preflight)++ source_sha = steps["Record checked out source SHA"]+ self.assertEqual(source_sha["id"], "source_sha")+ self.assertIn("git rev-parse HEAD", source_sha["run"])+ self.assertIn("source_sha=", source_sha["run"])+ self.assertIn("GITHUB_OUTPUT", source_sha["run"])+ self.assertIn("CUA_LIVE_E2E_SOURCE_SHA", source_sha["run"])+ self.assertIn("GITHUB_ENV", source_sha["run"])++ isolated_suite = steps["Prepare isolated live test suite"]["run"]+ self.assertIn("CUA_LIVE_E2E_TEST_ROOT", isolated_suite)+ self.assertIn("tests/live", isolated_suite)+ live_step = steps["Run live Fleet smoke"]+ self.assertEqual(live_step["env"], expected_oauth_env)+ live_run = live_step["run"]+ self.assertIn("$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_ephemeral.py", live_run)+ self.assertNotIn("libs/python/cua-sandbox/tests/live", live_run)+ self.assertIn('PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT', live_run)++ def test_failure_diagnostics_alerting_and_pinned_actions(self) -> None:+ workflow = self.workflow()+ live = workflow["jobs"]["live"]+ steps = self.steps_by_name(live)++ for step in live["steps"]:+ if "uses" in step:+ self.assertRegex(step["uses"], r"^[^@]+@[0-9a-f]{40}$")+ self.assertEqual(+ steps["Upload failure diagnostics"]["uses"],+ "actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08",+ )+ self.assertEqual(steps["Upload failure diagnostics"]["if"], "failure()")+ self.assertEqual(steps["Upload failure diagnostics"]["with"]["retention-days"], "7")++ controlled_summary = steps["Write controlled failure diagnostics"]+ self.assertEqual(+ controlled_summary["if"],+ "github.event_name == 'workflow_dispatch' && inputs.force_failure",+ )+ self.assertIn("summary.json", controlled_summary["run"])+ self.assertIn("ControlledFailure", controlled_summary["run"])+ self.assertIn('os.environ["CUA_LIVE_E2E_SOURCE_SHA"]', controlled_summary["run"])+ self.assertNotIn('os.environ.get("GITHUB_SHA")', controlled_summary["run"])+ self.assertEqual(+ steps["Controlled alert test failure"]["if"],+ "github.event_name == 'workflow_dispatch' && inputs.force_failure",+ )++ versions = steps["Record installed versions"]+ self.assertIn("GITHUB_OUTPUT", versions["run"])+ self.assertNotIn("steps.versions.outputs", versions["run"])++ alert = steps["Alert Alertmanager"]+ self.assertEqual(alert["if"], "failure()")+ self.assertIn("https://am.cua.ai/api/v2/alerts", alert["run"])+ self.assertIn("PeriodicCuaSandboxLiveE2EFailed", alert["run"])+ self.assertIn('"lane": "${{ matrix.lane }}"', alert["run"])+ self.assertIn("${{ steps.versions.outputs.sandbox }}", alert["run"])+ self.assertIn("${{ steps.source_sha.outputs.source_sha }}", alert["run"])+ self.assertNotIn('"source_sha": "${{ github.sha }}"', alert["run"])++ def test_cleanup_is_claim_only_and_inventory_is_diagnostic(self) -> None:+ workflow = WORKFLOW.read_text()+ live_test = LIVE_TEST.read_text()++ self.assertNotIn("cleanup_namespace", workflow)+ self.assertNotIn("delete_namespace", workflow)+ self.assertNotIn("Emergency namespace cleanup", workflow)+ self.assertIn("Sandbox.ephemeral", live_test)+ self.assertNotIn("wait_namespace_absent", live_test)+ self.assertNotIn("namespace_leak", live_test)+ self.assertIn("wait_claims_absent", live_test)+ self.assertIn("claim_leak", live_test)+ self.assertIn("persistent_resources", live_test)+ self.assertIn("unexpected_inventory", live_test)+ self.assertIn("module_origins", live_test)+ self.assertIn("cua_sandbox", live_test)++ def test_isolated_suite_cannot_import_checkout_cua_sandbox(self) -> None:+ with tempfile.TemporaryDirectory() as temporary_directory:+ temporary_root = Path(temporary_directory)+ suite_root = temporary_root / "live-suite"+ copied_live = suite_root / "tests/live"+ copied_live.mkdir(parents=True)+ shutil.copy2(SANDBOX_ROOT / "tests/__init__.py", suite_root / "tests")+ shutil.copy2(SANDBOX_ROOT / "tests/live/__init__.py", copied_live)++ installed_root = temporary_root / "installed"+ installed_package = installed_root / "cua_sandbox"+ installed_package.mkdir(parents=True)+ (installed_package / "__init__.py").write_text("MARKER = 'published-wheel'\n")+ probe = copied_live / "import_origin_probe.py"+ probe.write_text(+ "import cua_sandbox\n"+ "from pathlib import Path\n"+ "print(Path(cua_sandbox.__file__).resolve())\n"+ )++ environment = os.environ | {+ "PYTHONPATH": os.pathsep.join((str(suite_root), str(installed_root)))+ }+ result = subprocess.run(+ [sys.executable, str(probe)],+ cwd=REPO_ROOT,+ env=environment,+ check=True,+ capture_output=True,+ text=True,+ )++ origin = Path(result.stdout.strip()).resolve()+ self.assertEqual(origin, (installed_package / "__init__.py").resolve())+ self.assertNotIn(SANDBOX_ROOT.resolve(), origin.parents)+++if __name__ == "__main__":+ unittest.main()diff --git a/.github/workflows/ci-test-scripts.yml b/.github/workflows/ci-test-scripts.ymlindex 4075128b87..df2811aa90 100644--- a/.github/workflows/ci-test-scripts.yml+++ b/.github/workflows/ci-test-scripts.yml@@ -17,6 +17,7 @@ on:- ".release-please-manifest.json"- "release-please-config.json"- ".github/workflows/ci-test-scripts.yml"+ - ".github/workflows/periodic-cua-sandbox-live.yml"- ".github/workflows/ci-contributor-attribution.yml"- "docs/release-backfill-runbook.md"- "libs/cua-driver/scripts/**"@@ -42,7 +43,7 @@ jobs:- name: Install dependenciesrun: |python -m pip install --upgrade pip- pip install jsonschema pytest toml+ pip install jsonschema pyyaml pytest toml- name: Run testsrun: |diff --git a/.github/workflows/periodic-cua-sandbox-live.yml b/.github/workflows/periodic-cua-sandbox-live.ymlnew file mode 100644index 0000000000..3844e4743c--- /dev/null+++ b/.github/workflows/periodic-cua-sandbox-live.yml@@ -0,0 +1,212 @@+name: "Periodic: Cua Sandbox Live Fleet E2E"++on:+ schedule:+ - cron: "7/15 * * * *"+ push:+ branches: [main]+ paths:+ - "libs/python/cua-sandbox/**"+ - "libs/python/cua-fleet/**"+ - ".github/workflows/periodic-cua-sandbox-live.yml"+ - ".github/scripts/tests/test_periodic_cua_sandbox_live.py"+ workflow_dispatch:+ inputs:+ lane:+ description: "Lane to run"+ required: true+ default: both+ type: choice+ options: [both, main-source, published-package]+ force_failure:+ description: "Fail after setup to certify alerting"+ required: true+ default: false+ type: boolean++permissions:+ contents: read++jobs:+ prepare:+ runs-on: ubuntu-latest+ outputs:+ matrix: ${{ steps.matrix.outputs.matrix }}+ steps:+ - id: matrix+ shell: bash+ env:+ EVENT_NAME: ${{ github.event_name }}+ REQUESTED_LANE: ${{ inputs.lane }}+ run: |+ set -euo pipefail+ if [[ "$EVENT_NAME" == "push" ]]; then+ matrix='{"include":[{"lane":"main-source"}]}'+ elif [[ "$EVENT_NAME" == "workflow_dispatch" && "$REQUESTED_LANE" != "both" ]]; then+ matrix="{\"include\":[{\"lane\":\"$REQUESTED_LANE\"}]}"+ else+ matrix='{"include":[{"lane":"main-source"},{"lane":"published-package"}]}'+ fi+ echo "matrix=$matrix" >> "$GITHUB_OUTPUT"++ live:+ needs: prepare+ runs-on: ubuntu-latest+ timeout-minutes: 25+ strategy:+ fail-fast: false+ matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }}+ concurrency:+ group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}+ cancel-in-progress: ${{ github.event_name == 'schedule' }}+ env:+ CUA_FLEET_BASE_URL: https://run.cua.ai+ CUA_LIVE_E2E_LANE: ${{ matrix.lane }}+ CUA_LIVE_E2E_EVENT: ${{ github.event_name }}+ CUA_LIVE_E2E_NAMESPACE: cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}+ CUA_LIVE_E2E_ARTIFACT_DIR: /tmp/cua-live-e2e+ CUA_TELEMETRY_ENABLED: "false"+ steps:+ - name: Check Fleet OAuth credentials+ env:+ CUA_CLIENT_ID: ${{ secrets.CUA_CLIENT_ID }}+ CUA_CLIENT_SECRET: ${{ secrets.CUA_CLIENT_SECRET }}+ shell: bash+ run: |+ set -euo pipefail+ if [[ -z "$CUA_CLIENT_ID" || -z "$CUA_CLIENT_SECRET" ]]; then+ echo "CUA_CLIENT_ID and CUA_CLIENT_SECRET must both be configured."+ exit 1+ fi++ - name: Checkout main+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4+ with:+ ref: ${{ github.event_name == 'push' && github.sha || 'main' }}++ - name: Record checked out source SHA+ id: source_sha+ shell: bash+ run: |+ set -euo pipefail+ source_sha="$(git rev-parse HEAD)"+ echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT"+ echo "CUA_LIVE_E2E_SOURCE_SHA=$source_sha" >> "$GITHUB_ENV"++ - name: Set up Python+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5+ with:+ python-version: "3.12"++ - name: Install uv+ run: python -m pip install uv++ - name: Install main source+ if: matrix.lane == 'main-source'+ run: |+ uv pip install --system \+ --index https://wheels.cua.ai/simple \+ --default-index https://pypi.org/simple \+ -e libs/python/cua-sandbox pytest pytest-asyncio++ - name: Install published package+ if: matrix.lane == 'published-package'+ run: |+ uv pip install --system \+ --index https://wheels.cua.ai/simple \+ --default-index https://pypi.org/simple \+ --upgrade cua-sandbox pytest pytest-asyncio++ - name: Prepare isolated live test suite+ shell: bash+ run: |+ set -euo pipefail+ suite_root="$(mktemp -d /tmp/cua-live-e2e-suite.XXXXXX)"+ mkdir -p "$suite_root/tests/live"+ cp libs/python/cua-sandbox/tests/__init__.py "$suite_root/tests/"+ cp libs/python/cua-sandbox/tests/live/*.py "$suite_root/tests/live/"+ echo "CUA_LIVE_E2E_TEST_ROOT=$suite_root" >> "$GITHUB_ENV"++ - name: Record installed versions+ id: versions+ shell: bash+ run: |+ set -o pipefail+ python - <<'PY' | tee -a "$GITHUB_OUTPUT"+ from importlib.metadata import version+ print("sandbox=" + version("cua-sandbox"))+ print("fleet=" + version("cua-fleet"))+ PY++ - name: Write controlled failure diagnostics+ if: github.event_name == 'workflow_dispatch' && inputs.force_failure+ run: |+ python - <<'PY'+ import json+ import os+ from pathlib import Path++ artifact_dir = Path(os.environ["CUA_LIVE_E2E_ARTIFACT_DIR"])+ artifact_dir.mkdir(parents=True, exist_ok=True)+ (artifact_dir / "summary.json").write_text(+ json.dumps(+ {+ "lane": os.environ["CUA_LIVE_E2E_LANE"],+ "namespace": os.environ["CUA_LIVE_E2E_NAMESPACE"],+ "source_sha": os.environ["CUA_LIVE_E2E_SOURCE_SHA"],+ "error": {"type": "ControlledFailure"},+ },+ indent=2,+ sort_keys=True,+ )+ + "\n"+ )+ PY++ - name: Controlled alert test failure+ if: github.event_name == 'workflow_dispatch' && inputs.force_failure+ run: exit 1++ - name: Run live Fleet smoke+ if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.force_failure) }}+ env:+ CUA_CLIENT_ID: ${{ secrets.CUA_CLIENT_ID }}+ CUA_CLIENT_SECRET: ${{ secrets.CUA_CLIENT_SECRET }}+ run: |+ PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT" python -m pytest -q -s \+ "$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_ephemeral.py"++ - name: Upload failure diagnostics+ if: failure()+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4+ with:+ name: cua-sandbox-live-${{ matrix.lane }}-${{ github.run_id }}-${{ github.run_attempt }}+ path: /tmp/cua-live-e2e+ if-no-files-found: warn+ retention-days: 7++ - name: Alert Alertmanager+ if: failure()+ shell: bash+ run: |+ set -euo pipefail+ curl --silent --show-error --fail \+ -X POST https://am.cua.ai/api/v2/alerts \+ -H 'Content-Type: application/json' \+ -d '[{+ "labels": {+ "alertname": "PeriodicCuaSandboxLiveE2EFailed",+ "severity": "critical",+ "service": "cua-sandbox",+ "job": "periodic-cua-sandbox-live",+ "lane": "${{ matrix.lane }}"+ },+ "annotations": {+ "summary": "Cua Sandbox live Fleet E2E failed (${{ matrix.lane }})",+ "description": "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",+ "source_sha": "${{ steps.source_sha.outputs.source_sha }}",+ "package_version": "${{ steps.versions.outputs.sandbox }}",+ "image_digest": "sha256:5b9cb82f482834f7541901b87be956e7544d0db13fabc0b372cbc5eca5a74180"+ },+ "generatorURL": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"+ }]'diff --git a/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.md b/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.mdnew file mode 100644index 0000000000..0c9655eed4--- /dev/null+++ b/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.md@@ -0,0 +1,903 @@+# Periodic Cua Sandbox Live Fleet E2E Implementation Plan++> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.++**Goal:** Add a 15-minute live Fleet smoke that independently certifies repository `main` and the latest published `cua-sandbox`, verifies explicit port `8000`, screen and shell access, detects claim leaks, and alerts Alertmanager on failure.++**Architecture:** A public-API-only Python support module owns Fleet inspection and diagnostic-only leak detection; a thin opt-in pytest test owns the live scenario. One trigger-aware GitHub Actions workflow selects source and published lanes, while a repository-side contract test locks down schedule, security, cleanup, and alerting behavior.++**Tech Stack:** Python 3.12, pytest/pytest-asyncio, `cua-sandbox`, `fleet_sdk`, httpx, GitHub Actions, Alertmanager v2 API.++## Global Constraints++- Schedule must remain `7/15 * * * *`, running at `:07`, `:22`, `:37`, and `:52` UTC.+- Scheduled runs execute both `main-source` and `published-package`; relevant pushes to `main` execute only `main-source`.+- Manual dispatch accepts `both`, `main-source`, or `published-package`; only manual dispatch may set `force_failure=true`.+- Use image `296062593712.dkr.ecr.us-west-2.amazonaws.com/desktop-workspace-duo@sha256:5b9cb82f482834f7541901b87be956e7544d0db13fabc0b372cbc5eca5a74180`.+- Provision with `cpu=4`, `memory_mb=4096`, `server_port=8000`, `time_to_start=900`, `request_timeout=60`, and `telemetry_enabled=False`.+- Authenticate only with `CUA_CLIENT_ID`, `CUA_CLIENT_SECRET`, `CUA_FLEET_BASE_URL=https://run.cua.ai`, and the default Cyclops token endpoint.+- Do not use `CUA_API_KEY`, legacy `/api/keys`, namespace-scoped key creation, repository-private SDK helpers, or mutable image tags.+- A leaked claim is a test failure; collect sanitized diagnostics without explicit deletion.+- Upload diagnostics only on failure and never include credentials, tokens, or authorization headers.+- Pin `actions/checkout`, `actions/setup-python`, and `actions/upload-artifact` by full commit SHA.+- No nightly suite and no required pull request live check.++---++### Task 1: Add Offline Tests For Live Support Logic++**Files:**+- Create: `libs/python/cua-sandbox/tests/live/__init__.py`+- Create: `libs/python/cua-sandbox/tests/live/fleet_e2e_support.py`+- Create: `libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py`++**Interfaces:**+- Consumes: public `fleet_sdk` records and exceptions; no `cua_sandbox` private modules.+- Produces: `HttpxFleetClient`, `build_fleet_client()`, `build_namespace_name()`, `wait_claims_absent()`, `collect_resource_inventory()`, `assert_template_contract()`, and `write_summary()` for diagnostic-only live pytest cleanup verification.++- [ ] **Step 1: Create the live test package marker**++Create an empty `libs/python/cua-sandbox/tests/live/__init__.py` so support code can be imported consistently by pytest.++- [ ] **Step 2: Write failing unit tests for naming, template assertions, and cleanup polling**++Create `libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py` with fakes that do not contact live infrastructure:++```python+from __future__ import annotations++import json+from types import SimpleNamespace++import pytest++from tests.live.fleet_e2e_support import (+ assert_template_contract,+ build_namespace_name,+ collect_resource_inventory,+ wait_claims_absent,+)+++def test_build_namespace_name_is_stable_for_each_lane_and_event_class() -> None:+ name = build_namespace_name("published-package", "workflow_dispatch")+ assert name == "cua-live-published-package-manual"+ assert len(name) <= 63+++def test_assert_template_contract_accepts_server_port_8000() -> None:+ probes = SimpleNamespace(+ to_json=lambda: json.dumps(+ {"readinessProbe": {"tcpSocket": {"port": 8000}}}+ )+ )+ template = SimpleNamespace(+ spec=SimpleNamespace(+ vm_template=SimpleNamespace(+ services=[SimpleNamespace(name="server", target_port=8000)],+ probes=probes,+ )+ )+ )+ assert_template_contract(template, expected_port=8000)+++def test_assert_template_contract_rejects_wrong_service_port() -> None:+ probes = SimpleNamespace(+ to_json=lambda: json.dumps(+ {"readinessProbe": {"tcpSocket": {"port": 8000}}}+ )+ )+ template = SimpleNamespace(+ spec=SimpleNamespace(+ vm_template=SimpleNamespace(+ services=[SimpleNamespace(name="server", target_port=5000)],+ probes=probes,+ )+ )+ )+ with pytest.raises(AssertionError, match="target_port"):+ assert_template_contract(template, expected_port=8000)+++async def test_collect_resource_inventory_lists_owned_resources() -> None:+ def resource(name: str):+ return SimpleNamespace(metadata=SimpleNamespace(name=name))++ class FakeClient:+ async def get_namespace(self, name: str):+ return SimpleNamespace(name=name)++ async def list_templates(self, name: str):+ return [resource("template-a")]++ async def list_pools(self, name: str):+ return [resource("pool-a")]++ async def list_claims(self, name: str):+ return [resource("claim-a")]++ assert await collect_resource_inventory(FakeClient(), "demo") == {+ "templates": ["template-a"],+ "pools": ["pool-a"],+ "claims": ["claim-a"],+ }+++async def test_wait_claims_absent_polls_until_claims_are_gone(monkeypatch) -> None:+ class StatusError(Exception):+ def __init__(self, status: int) -> None:+ self.status = status++ calls = 0++ class FakeClient:+ async def get_namespace(self, name: str):+ nonlocal calls+ calls += 1+ if calls == 1:+ return SimpleNamespace(name=name)+ raise StatusError(404)++ monkeypatch.setattr(+ "tests.live.fleet_e2e_support.is_not_found_error",+ lambda error: getattr(error, "status", None) == 404,+ )+ monkeypatch.setattr("asyncio.sleep", lambda _: _completed())++ assert await wait_claims_absent(FakeClient(), "demo", timeout=1, interval=0)+ assert calls == 2+++async def _completed() -> None:+ return None+```++- [ ] **Step 3: Run the focused test and verify it fails**++Run:++```bash+cd libs/python/cua-sandbox+uv sync --group dev+.venv/bin/python -m pytest -q tests/test_live_fleet_e2e_support.py+```++Expected: FAIL during collection because `tests.live.fleet_e2e_support` does not yet export the required helpers.++- [ ] **Step 4: Implement the minimal public Fleet support module**++Create `libs/python/cua-sandbox/tests/live/fleet_e2e_support.py` with these concrete interfaces:++```python+from __future__ import annotations++import asyncio+import json+import os+from pathlib import Path+import re+import time+from typing import Any++import httpx+from fleet_sdk import (+ CyclopsClient,+ CyclopsConfiguration,+ CyclopsCredentials,+ HttpClient,+ HttpError,+ HttpHeader,+ HttpRequest,+ HttpResponse,+ SdkError,+)++DEFAULT_BASE_URL = "https://run.cua.ai"+DEFAULT_TOKEN_URL = (+ "https://auth.cua.ai/realms/cyclops-cs/protocol/openid-connect/token"+)+++class HttpxFleetClient(HttpClient):+ def __init__(self) -> None:+ self._client = httpx.AsyncClient(timeout=60.0)++ async def execute(self, request: HttpRequest) -> HttpResponse:+ try:+ response = await self._client.request(+ request.method,+ request.url,+ headers={header.name: header.value for header in request.headers},+ content=request.body,+ )+ except httpx.TransportError as error:+ raise HttpError.Transport(str(error)) from error+ return HttpResponse(+ status=response.status_code,+ headers=[+ HttpHeader(name=name, value=value)+ for name, value in response.headers.multi_items()+ ],+ body=response.content,+ )++ async def aclose(self) -> None:+ await self._client.aclose()+++def build_namespace_name(lane: str, event_name: str) -> str:+ event_class = {"schedule": "schedule", "push": "push", "workflow_dispatch": "manual"}.get(+ event_name, "manual"+ )+ raw = f"cua-live-{lane}-{event_class}".lower()+ normalized = re.sub(r"[^a-z0-9-]+", "-", raw).strip("-")+ return normalized[:63].rstrip("-")+++def build_fleet_client() -> tuple[CyclopsClient, HttpxFleetClient]:+ client_id = os.environ["CUA_CLIENT_ID"]+ client_secret = os.environ["CUA_CLIENT_SECRET"]+ http_client = HttpxFleetClient()+ configuration = CyclopsConfiguration(+ base_url=os.environ.get("CUA_FLEET_BASE_URL", DEFAULT_BASE_URL),+ token_url=os.environ.get("CUA_TOKEN_URL", DEFAULT_TOKEN_URL),+ credentials=CyclopsCredentials(client_id, client_secret),+ pool_poll_interval_ms=2000,+ pool_poll_limit=300,+ claim_poll_interval_ms=2000,+ claim_poll_limit=300,+ )+ return CyclopsClient.connect(configuration, http_client), http_client+++async def wait_claims_absent(+ client: CyclopsClient,+ name: str,+ *,+ timeout: float = 180.0,+ interval: float = 5.0,+) -> bool:+ deadline = time.monotonic() + timeout+ while time.monotonic() < deadline:+ if not await client.list_claims(name):+ return True+ await asyncio.sleep(interval)+ return not await client.list_claims(name)+++async def collect_resource_inventory(+ client: CyclopsClient, name: str+) -> dict[str, list[str]]:+ templates = await client.list_templates(name)+ pools = await client.list_pools(name)+ claims = await client.list_claims(name)+ return {+ "templates": [item.metadata.name for item in templates],+ "pools": [item.metadata.name for item in pools],+ "claims": [item.metadata.name for item in claims],+ }+++def assert_template_contract(template: Any, expected_port: int) -> None:+ vm_template = template.spec.vm_template+ server = next(service for service in vm_template.services if service.name == "server")+ assert server.target_port == expected_port, (+ f"server target_port={server.target_port}, expected {expected_port}"+ )+ probes = json.loads(vm_template.probes.to_json())+ observed = probes["readinessProbe"]["tcpSocket"]["port"]+ assert observed == expected_port, (+ f"readiness probe port={observed}, expected {expected_port}"+ )+++def write_summary(path: Path, summary: dict[str, Any]) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")+```++- [ ] **Step 5: Run the focused test and verify it passes**++Run:++```bash+cd libs/python/cua-sandbox+.venv/bin/python -m pytest -q tests/test_live_fleet_e2e_support.py+```++Expected: `5 passed`.++- [ ] **Step 6: Commit the offline support layer**++```bash+git add \+ libs/python/cua-sandbox/tests/live/__init__.py \+ libs/python/cua-sandbox/tests/live/fleet_e2e_support.py \+ libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py+git commit -m "test(cua-sandbox): add live Fleet E2E support"+```++---++### Task 2: Add The Opt-In Live Fleet Pytest++**Files:**+- Create: `libs/python/cua-sandbox/tests/live/test_fleet_ephemeral.py`+- Modify: `libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py`++**Interfaces:**+- Consumes: Task 1 support functions, public `cua_sandbox.Image`, `cua_sandbox.Sandbox`, and public `fleet_sdk` client methods.+- Produces: one credential-gated pytest scenario and a sanitized JSON result at `CUA_LIVE_E2E_ARTIFACT_DIR/summary.json`.++- [ ] **Step 1: Add a failing offline test for credential gating**++Append this test to `tests/test_live_fleet_e2e_support.py`:++```python+def test_live_test_requires_both_oauth_values(monkeypatch) -> None:+ monkeypatch.delenv("CUA_CLIENT_ID", raising=False)+ monkeypatch.delenv("CUA_CLIENT_SECRET", raising=False)+ from tests.live.test_fleet_ephemeral import has_oauth_credentials++ assert not has_oauth_credentials()+ monkeypatch.setenv("CUA_CLIENT_ID", "client")+ assert not has_oauth_credentials()+ monkeypatch.setenv("CUA_CLIENT_SECRET", "secret")+ assert has_oauth_credentials()+```++- [ ] **Step 2: Run the test and verify it fails**++Run:++```bash+cd libs/python/cua-sandbox+.venv/bin/python -m pytest -q \+ tests/test_live_fleet_e2e_support.py::test_live_test_requires_both_oauth_values+```++Expected: FAIL because `tests.live.test_fleet_ephemeral` does not exist.++- [ ] **Step 3: Implement the live scenario**++Create `libs/python/cua-sandbox/tests/live/test_fleet_ephemeral.py`:++```python+from __future__ import annotations++from importlib.metadata import version+import os+from pathlib import Path+import time++import pytest+from cua_sandbox import Image, Sandbox++from tests.live.fleet_e2e_support import (+ assert_template_contract,+ build_fleet_client,+ build_namespace_name,+ collect_resource_inventory,+ wait_claims_absent,+ write_summary,+)++IMAGE = (+ "296062593712.dkr.ecr.us-west-2.amazonaws.com/desktop-workspace-duo"+ "@sha256:5b9cb82f482834f7541901b87be956e7544d0db13fabc0b372cbc5eca5a74180"+)+++def has_oauth_credentials() -> bool:+ return bool(os.environ.get("CUA_CLIENT_ID") and os.environ.get("CUA_CLIENT_SECRET"))+++pytestmark = [+ pytest.mark.asyncio,+ pytest.mark.skipif(not has_oauth_credentials(), reason="Fleet OAuth credentials not set"),+]+++async def test_fleet_ephemeral_live() -> None:+ lane = os.environ.get("CUA_LIVE_E2E_LANE", "local")+ namespace = os.environ.get("CUA_LIVE_E2E_NAMESPACE") or build_namespace_name(+ lane,+ os.environ.get("CUA_LIVE_E2E_EVENT", "manual"),+ )+ artifact_dir = Path(+ os.environ.get("CUA_LIVE_E2E_ARTIFACT_DIR", "/tmp/cua-live-e2e")+ )+ summary = {+ "lane": lane,+ "namespace": namespace,+ "image": IMAGE,+ "source_sha": os.environ.get("GITHUB_SHA"),+ "packages": {+ "cua-sandbox": version("cua-sandbox"),+ "cua-fleet": version("cua-fleet"),+ },+ }+ fleet, http_client = build_fleet_client()+ primary_error: BaseException | None = None++ try:+ started = time.monotonic()+ async with Sandbox.ephemeral(+ Image.from_registry(IMAGE),+ name=namespace,+ cpu=4,+ memory_mb=4096,+ server_port=8000,+ time_to_start=900,+ request_timeout=60,+ telemetry_enabled=False,+ ) as sandbox:+ summary["provision_seconds"] = time.monotonic() - started+ summary["sandbox_name"] = sandbox.name+ template = await fleet.get_template(namespace, namespace)+ assert_template_contract(template, expected_port=8000)++ width, height = await sandbox.screen.size()+ summary["screen"] = {"width": width, "height": height}+ assert (width, height) == (1024, 768)++ screenshot = await sandbox.screenshot()+ artifact_dir.mkdir(parents=True, exist_ok=True)+ (artifact_dir / "screen.png").write_bytes(screenshot)+ assert screenshot.startswith(b"\x89PNG\r\n\x1a\n")+ assert len(screenshot) > 1000++ result = await sandbox.shell.run("uname -s")+ summary["shell"] = {+ "success": result.success,+ "stdout": result.stdout.strip(),+ "stderr": result.stderr.strip(),+ }+ assert result.success+ assert result.stdout.strip() == "Linux"+ except BaseException as error:+ primary_error = error+ summary["error"] = {"type": type(error).__name__}+ raise+ finally:+ try:+ cleanup_started = time.monotonic()+ claims_absent = await wait_claims_absent(fleet, namespace)+ inventory = await collect_resource_inventory(fleet, namespace)+ expected_inventory = {"templates": [namespace], "pools": [namespace], "claims": []}+ summary["claims_absent"] = claims_absent+ summary["persistent_resources"] = inventory+ if not claims_absent:+ summary["claim_leak"] = True+ pytest.fail(f"claims remain in namespace {namespace} after Sandbox.ephemeral()")+ if inventory != expected_inventory:+ summary["unexpected_inventory"] = True+ pytest.fail(f"unexpected reconciled resource inventory: {inventory}")+ summary["cleanup_seconds"] = time.monotonic() - cleanup_started+ finally:+ write_summary(artifact_dir / "summary.json", summary)+ await http_client.aclose()+```++- [ ] **Step 4: Verify credential-free collection skips the live test**++Run:++```bash+cd libs/python/cua-sandbox+env -u CUA_CLIENT_ID -u CUA_CLIENT_SECRET \+ .venv/bin/python -m pytest -q tests/live/test_fleet_ephemeral.py+```++Expected: `1 skipped` and no network request.++- [ ] **Step 5: Run all offline live-support tests**++Run:++```bash+cd libs/python/cua-sandbox+.venv/bin/python -m pytest -q \+ tests/test_live_fleet_e2e_support.py \+ tests/live/test_fleet_ephemeral.py+```++Expected without credentials: `6 passed, 1 skipped`.++- [ ] **Step 6: Commit the live scenario**++```bash+git add \+ libs/python/cua-sandbox/tests/live/test_fleet_ephemeral.py \+ libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py+git commit -m "test(cua-sandbox): add live Fleet ephemeral smoke"+```++---++### Task 3: Add And Lock The Trigger-Aware Periodic Workflow++**Files:**+- Create: `.github/scripts/tests/test_periodic_cua_sandbox_live.py`+- Create: `.github/workflows/periodic-cua-sandbox-live.yml`+- Modify: `.github/workflows/ci-test-scripts.yml`+- Modify: `libs/python/cua-sandbox/tests/live/test_fleet_ephemeral.py`+- Test: `.github/scripts/tests/test_periodic_cua_sandbox_live.py`++**Implementation contract:** The workflow runs every 15 minutes, has one source+lane for relevant `main` pushes, and uses the selected manual lane or both+lanes. It never explicitly deletes a namespace: `Sandbox.ephemeral()` cleanup+is verified with diagnostic-only leak detection.++```yaml+on:+ schedule:+ - cron: "7/15 * * * *"+ push:+ branches: [main]+ paths:+ - "libs/python/cua-sandbox/**"+ - "libs/python/cua-fleet/**"+ - ".github/workflows/periodic-cua-sandbox-live.yml"+ - ".github/scripts/tests/test_periodic_cua_sandbox_live.py"+```++The `prepare` job emits `main-source` for every `push`, both lanes for+`schedule`, and the requested `workflow_dispatch` lane. The contract test+extracts this shell script from parsed YAML, runs it with a temporary+`GITHUB_OUTPUT`, parses its JSON matrix, and covers push, schedule, and every+manual selection. This makes a push-to-both or ignored manual selection fail+CI.++```yaml+concurrency:+ group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}+ cancel-in-progress: ${{ github.event_name == 'schedule' }}+```++The event-and-lane group lets a new schedule cancel only an older schedule for+the same lane. Push and manual evidence use separate groups and are never+cancelled by a schedule.++```yaml+steps:+ - name: Check Fleet OAuth credentials+ run: |+ if [[ -z "$CUA_CLIENT_ID" || -z "$CUA_CLIENT_SECRET" ]]; then+ exit 1+ fi+ - name: Prepare isolated live test suite+ run: |+ suite_root="$(mktemp -d /tmp/cua-live-e2e-suite.XXXXXX)"+ mkdir -p "$suite_root/tests/live"+ cp libs/python/cua-sandbox/tests/__init__.py "$suite_root/tests/"+ cp libs/python/cua-sandbox/tests/live/*.py "$suite_root/tests/live/"+ echo "CUA_LIVE_E2E_TEST_ROOT=$suite_root" >> "$GITHUB_ENV"+ - name: Record installed versions+ run: |+ python - <<'PY' | tee -a "$GITHUB_OUTPUT"+ from importlib.metadata import version+ print("sandbox=" + version("cua-sandbox"))+ print("fleet=" + version("cua-fleet"))+ PY+ - name: Write controlled failure diagnostics+ if: github.event_name == 'workflow_dispatch' && inputs.force_failure+ run: |+ python - <<'PY'+ import json+ import os+ from pathlib import Path++ artifact_dir = Path(os.environ["CUA_LIVE_E2E_ARTIFACT_DIR"])+ artifact_dir.mkdir(parents=True, exist_ok=True)+ (artifact_dir / "summary.json").write_text(+ json.dumps(+ {+ "lane": os.environ["CUA_LIVE_E2E_LANE"],+ "namespace": os.environ["CUA_LIVE_E2E_NAMESPACE"],+ "source_sha": os.environ.get("GITHUB_SHA"),+ "error": {"type": "ControlledFailure"},+ },+ indent=2,+ sort_keys=True,+ )+ + "\n"+ )+ PY+ - name: Run live Fleet smoke+ run: PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT" python -m pytest -q -s "$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_ephemeral.py"+```++Both lanes install their respective SDK package before this copied suite runs.+The source lane resolves `cua_sandbox` through its editable install; the+published lane resolves it from site-packages because the checkout package root+is absent from `PYTHONPATH`. The live summary records the resolved+`cua_sandbox` module origin. Forced failures create a sanitized `summary.json`+before the failure-only artifact upload; the versions step writes outputs for+the later Alertmanager payload without reading `steps.versions.outputs` inside+its own step.++The repository-side contract parses YAML with `yaml.BaseLoader`, rejects+unpinned actions and any `cleanup_namespace` or `delete_namespace` workflow+reference, executes the matrix preparation script, and verifies credentials,+isolation, diagnostics, and lane-specific Alertmanager payloads. Scripts CI+installs `pyyaml` and path-filters both the contract and workflow.++- [ ] **Validation**++```bash+python -m pytest -q .github/scripts/tests/test_periodic_cua_sandbox_live.py+python -m pytest -q .github/scripts/tests+python - <<'PY'+from pathlib import Path+import yaml+yaml.load(+ Path('.github/workflows/periodic-cua-sandbox-live.yml').read_text(),+ Loader=yaml.BaseLoader,+)+PY+actionlint .github/workflows/periodic-cua-sandbox-live.yml+git diff --check+```++Expected: the workflow contracts, scripts tests, YAML parse, actionlint, and+whitespace check pass without contacting live infrastructure.++---++### Task 4: Run Focused And Repository Validation++**Files:**+- Modify only if validation reveals task-scoped defects.++**Interfaces:**+- Consumes: Tasks 1-3.+- Produces: fresh offline evidence that the live test is safely skipped without credentials and workflow contracts are enforced by CI.++- [ ] **Step 1: Run sandbox live-support tests without credentials**++```bash+cd libs/python/cua-sandbox+env -u CUA_CLIENT_ID -u CUA_CLIENT_SECRET \+ .venv/bin/python -m pytest -q \+ tests/test_live_fleet_e2e_support.py \+ tests/live/test_fleet_ephemeral.py+```++Expected: `6 passed, 1 skipped`.++- [ ] **Step 2: Run existing Fleet regression tests**++```bash+cd libs/python/cua-sandbox+.venv/bin/python -m pytest -q \+ tests/test_fleet_cloud_transport.py \+ tests/test_vm_cleanup.py \+ -k 'server_port or custom_services'+```++Expected: `38 passed, 32 deselected`.++- [ ] **Step 3: Run scripts CI tests**++```bash+python -m pytest -q .github/scripts/tests+```++Expected: all tests pass with no failures.++- [ ] **Step 4: Run lint and diff validation**++```bash+cd libs/python/cua-sandbox+.venv/bin/python -m ruff check \+ tests/live/fleet_e2e_support.py \+ tests/live/test_fleet_ephemeral.py \+ tests/test_live_fleet_e2e_support.py+.venv/bin/python -m ruff format --check \+ tests/live/fleet_e2e_support.py \+ tests/live/test_fleet_ephemeral.py \+ tests/test_live_fleet_e2e_support.py+cd ../../..+git diff --check+```++Expected: all commands exit `0`.++- [ ] **Step 5: Commit any validation-only corrections**++If Task 4 required scoped corrections:++```bash+git add \+ .github/scripts/tests/test_periodic_cua_sandbox_live.py \+ .github/workflows/ci-test-scripts.yml \+ .github/workflows/periodic-cua-sandbox-live.yml \+ libs/python/cua-sandbox/tests/live \+ libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py+git commit -m "test(cua-sandbox): harden periodic live E2E"+```++If no corrections were needed, do not create an empty commit.++---++### Task 5: Certify Both Live Lanes And Roll Out The Schedule++**Files:**+- No code changes unless live evidence exposes a task-scoped defect.++**Interfaces:**+- Consumes: repository secrets, the exact candidate SHA, and Tasks 1-4.+- Produces: live source and published-package evidence, cleanup confirmation, Alertmanager verification, and an enabled periodic monitor on `main`.++- [ ] **Step 1: Verify the published package contains `server_port` before enabling the published lane**++Run in an isolated environment:++```bash+CHECK_ROOT=$(mktemp -d /tmp/cua-published-check.XXXXXX)+trap 'rm -rf "$CHECK_ROOT"' EXIT+uv venv --python 3.12 "$CHECK_ROOT/venv"+uv pip install \+ --python "$CHECK_ROOT/venv/bin/python" \+ --index https://wheels.cua.ai/simple \+ --default-index https://pypi.org/simple \+ --upgrade cua-sandbox+"$CHECK_ROOT/venv/bin/python" - <<'PY'+import inspect+from cua_sandbox import Sandbox+assert "server_port" in inspect.signature(Sandbox.ephemeral).parameters+PY+```++Expected: exit `0`. If it fails, publish the `cua-sandbox` release containing PR #2997 before enabling scheduled alerts; do not weaken the live assertion.++- [ ] **Step 2: Run the source lane against live Fleet on the exact candidate SHA**++Set the OAuth secrets in the environment and run:++```bash+export CUA_FLEET_BASE_URL=https://run.cua.ai+export CUA_LIVE_E2E_LANE=main-source+export CUA_LIVE_E2E_NAMESPACE="cua-live-main-source-manual"+export CUA_LIVE_E2E_ARTIFACT_DIR=/tmp/cua-live-main-source+LIVE_TEST_ROOT=$(mktemp -d /tmp/cua-live-e2e-suite.XXXXXX)+mkdir -p "$LIVE_TEST_ROOT/tests/live"+cp libs/python/cua-sandbox/tests/__init__.py "$LIVE_TEST_ROOT/tests/"+cp libs/python/cua-sandbox/tests/live/*.py "$LIVE_TEST_ROOT/tests/live/"+PYTHONPATH="$LIVE_TEST_ROOT" \+ libs/python/cua-sandbox/.venv/bin/python -m pytest -q -s \+ "$LIVE_TEST_ROOT/tests/live/test_fleet_ephemeral.py"+```++Expected: PASS; summary reports `1024x768`, `Linux`, port `8000`, the editable `cua_sandbox` module origin, and claim-only cleanup.++- [ ] **Step 3: Run the published lane in an isolated environment**++Run from the repository root with the OAuth secrets already exported:++```bash+PUBLISHED_ROOT=$(mktemp -d /tmp/cua-published-live.XXXXXX)+trap 'rm -rf "$PUBLISHED_ROOT"' EXIT+uv venv --python 3.12 "$PUBLISHED_ROOT/venv"+uv pip install \+ --python "$PUBLISHED_ROOT/venv/bin/python" \+ --index https://wheels.cua.ai/simple \+ --default-index https://pypi.org/simple \+ --upgrade cua-sandbox pytest pytest-asyncio+export CUA_FLEET_BASE_URL=https://run.cua.ai+export CUA_LIVE_E2E_LANE=published-package+export CUA_LIVE_E2E_NAMESPACE="cua-live-published-package-manual"+export CUA_LIVE_E2E_ARTIFACT_DIR=/tmp/cua-live-published-package+LIVE_TEST_ROOT=$(mktemp -d /tmp/cua-live-e2e-suite.XXXXXX)+mkdir -p "$LIVE_TEST_ROOT/tests/live"+cp libs/python/cua-sandbox/tests/__init__.py "$LIVE_TEST_ROOT/tests/"+cp libs/python/cua-sandbox/tests/live/*.py "$LIVE_TEST_ROOT/tests/live/"+PYTHONPATH="$LIVE_TEST_ROOT" \+ "$PUBLISHED_ROOT/venv/bin/python" -m pytest -q -s \+ "$LIVE_TEST_ROOT/tests/live/test_fleet_ephemeral.py"+```++Expected: PASS with the installed package version in `summary.json` and the expected persistent pool/template and no claims.++- [ ] **Step 4: Push the branch and open the implementation PR**++```bash+git push -u origin codex/periodic-live-sandbox-e2e+cat > /tmp/periodic-live-e2e-pr.md <<'EOF'+## What changed++- restore a Fleet-backed live `cua-sandbox` smoke every 15 minutes+- test repository `main` and the latest published package independently+- verify port `8000`, screen dimensions, screenshot, shell, and claim-only cleanup+- notify Alertmanager with lane-specific failure context++## Verification++- offline support and workflow contract tests pass+- focused Fleet regression tests pass+- source and published live lanes pass against the pinned Duo image+EOF+gh pr create \+ --repo trycua/cua \+ --title "ci(cua-sandbox): restore periodic live Fleet E2E" \+ --body-file /tmp/periodic-live-e2e-pr.md+```++The PR body must include the candidate SHA, both live lane results, package versions, namespace names, and cleanup confirmation.++- [ ] **Step 5: Merge the implementation PR after review and green CI**++```bash+PR_NUMBER=$(gh pr view --repo trycua/cua --json number --jq '.number')+gh pr checks --repo trycua/cua "$PR_NUMBER" --watch+gh pr merge --repo trycua/cua "$PR_NUMBER" --squash+```++Expected: the PR is merged and `.github/workflows/periodic-cua-sandbox-live.yml` exists on `main`.++- [ ] **Step 6: Verify the Alertmanager path with a controlled failure**++After the workflow exists on `main`, dispatch the manual-only controlled failure:++```bash+gh workflow run periodic-cua-sandbox-live.yml \+ --repo trycua/cua \+ --ref main \+ -f lane=main-source \+ -f force_failure=true+RUN_ID=$(gh run list \+ --repo trycua/cua \+ --workflow periodic-cua-sandbox-live.yml \+ --event workflow_dispatch \+ --limit 1 \+ --json databaseId \+ --jq '.[0].databaseId')+gh run watch "$RUN_ID" --repo trycua/cua --exit-status || true+curl -fsS --get https://am.cua.ai/api/v2/alerts \+ --data-urlencode 'filter=alertname="PeriodicCuaSandboxLiveE2EFailed"' \+ | jq -e '.[] | select(.labels.service == "cua-sandbox" and .labels.lane == "main-source")'+```++Expected: the workflow fails intentionally and Alertmanager returns one matching lane-specific alert. Because `force_failure` is restricted to `workflow_dispatch`, scheduled and push runs cannot activate it.++- [ ] **Step 7: Observe two consecutive scheduled intervals**++After merge, inspect two consecutive `:07/:22/:37/:52` runs. Confirm both lanes pass, a newer schedule cancels only an older scheduled run of the same lane, artifacts are absent on success, and each dedicated namespace retains only its named pool/template with no claims.++- [ ] **Step 8: Record rollout evidence**++Update the merged PR or implementation issue with workflow run links, observed versions, cleanup evidence, and the Alertmanager test. The rollout is complete only after two consecutive scheduled runs pass for both lanes.++## Live Evidence Remediation++The monitor uses reusable, dedicated namespaces instead of per-run namespaces.+Each lane has one DNS-safe namespace for each event class:++- `cua-live-<lane>-schedule` for scheduled runs+- `cua-live-<lane>-push` for pushes+- `cua-live-<lane>-manual` for `workflow_dispatch`++The event-and-lane concurrency group serializes use of each deterministic claim;+only scheduled runs cancel an older scheduled run in the same lane. Fleet+reconciliation preserves the namespace, pool, and template, all named after the+namespace. `Sandbox.ephemeral()` is verified with claim-only cleanup: after+exit the monitor polls until claims are absent, records persistent reconciled+resources, and requires exactly the named pool/template with zero claims. The+read-only poll and inventory run after every provisioning attempt, including one+that fails before yielding a sandbox. It never explicitly deletes a namespace,+pool, or template.+++Workflow namespace expression: `cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}`.++persistent reconciled resources are intentionally retained between runs; only deterministic claims are ephemeral.++The workflow uses step-scoped OAuth credentials only for credential preflight and the+live pytest step. After checkout, `git rev-parse HEAD` is exported as+`CUA_LIVE_E2E_SOURCE_SHA` and is the source SHA recorded by live, controlled-failure,+and Alertmanager evidence.diff --git a/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.md b/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.mdnew file mode 100644index 0000000000..9cb325f45b--- /dev/null+++ b/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.md@@ -0,0 +1,255 @@+# Periodic Cua Sandbox Live Fleet E2E Design++## Problem++The `cua-sandbox` repository has focused unit and contract coverage for Fleet+resource generation, but ordinary pull request CI does not prove that the SDK+can provision a real sandbox through `https://run.cua.ai`, reach the guest+computer-server, and remove the resources it created.++The repository previously had+`.github/workflows/periodic-test-linux-cloud-vm.yml`. It ran a Linux cloud VM+smoke every 15 minutes, used a repository secret, and notified Alertmanager on+failure. That workflow was removed on July 10, 2026 as a legacy cloud VM test.+The replacement should retain its fast detection and operational alerting while+testing the current Fleet-backed public SDK contract.++## Goals++- Exercise Fleet-backed `Sandbox.ephemeral()` against live production+ infrastructure every 15 minutes.+- Test both the current repository `main` source and the latest published+ `cua-sandbox` package.+- Run the source lane immediately after relevant changes merge to `main`.+- Verify guest access, generated Fleet port configuration, and claim-only+ cleanup with persistent reconciled resources.+- Alert through Alertmanager with enough lane and version context to triage a+ failure.+- Prevent failed or superseded runs from accumulating live infrastructure.++## Non-Goals++- The workflow is not a required pull request check.+- There is no separate nightly or broad lifecycle suite.+- The workflow does not test snapshots, forks, Android, Windows, or multiple+ regions.+- The live smoke does not test `server_port=5000` until a pinned image serves+ the computer-server `/cmd` API on that port. Existing contract tests remain+ authoritative for non-default port propagation.+- The workflow does not use the legacy `/api/keys` API or a namespace-scoped+ key.++## Trigger And Lane Model++`.github/workflows/periodic-cua-sandbox-live.yml` has three triggers:++1. `schedule` uses `7/15 * * * *`, preserving the `:07`, `:22`, `:37`, and+ `:52` cadence.+2. `push` is limited to `main` and these paths: `libs/python/cua-sandbox/**`,+ `libs/python/cua-fleet/**`,+ `.github/workflows/periodic-cua-sandbox-live.yml`, and+ `.github/scripts/tests/test_periodic_cua_sandbox_live.py`.+3. `workflow_dispatch` accepts `both`, `main-source`, or `published-package`,+ plus manual-only `force_failure`.++The preparation script emits a JSON matrix: every `push` selects only+`main-source`; every `schedule` selects both lanes; manual runs select their+requested lane or both. The workflow contract executes this extracted shell+script in `bash` with a temporary `GITHUB_OUTPUT` and parses the emitted JSON,+so a push-to-both mutation or ignored manual selection fails CI.++The two lanes use `fail-fast: false` and this concurrency contract:++```yaml+concurrency:+ group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}+ cancel-in-progress: ${{ github.event_name == 'schedule' }}+```++The event-and-lane grouping means a new schedule can cancel only an older+schedule for the same lane. Push and manual runs use distinct groups and are+allowed to finish.++## Installation Isolation++`main-source` installs `libs/python/cua-sandbox` editable;+`published-package` installs the current published `cua-sandbox` wheel. Both+lanes then run `Prepare isolated live test suite`, which copies only+`tests/__init__.py` and `tests/live/*.py` into a temporary+`CUA_LIVE_E2E_TEST_ROOT`, then runs:++```bash+PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT" python -m pytest -q -s \+ "$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_ephemeral.py"+```++The checkout package root is not on that test path. Therefore the source lane+uses its editable install while the published lane imports `cua_sandbox` from+site-packages. The live summary records the resolved `cua_sandbox` module+origin alongside package versions and source SHA. After checkout, `Record checked out source SHA`+uses `git rev-parse HEAD` to publish `CUA_LIVE_E2E_SOURCE_SHA`; the live summary,+controlled-failure summary, and Alertmanager annotation use that exact value rather+than the triggering event SHA.++## Authentication And Secrets++Fleet authentication uses only masked GitHub Actions secrets `CUA_CLIENT_ID`+and `CUA_CLIENT_SECRET`, with `CUA_FLEET_BASE_URL=https://run.cua.ai`. Before+checkout, installation, or pytest, `Check Fleet OAuth credentials` exits with+an error if either value is empty. This prevents production monitoring from+passing through the live test's credential-free pytest skip.++The workflow uses step-scoped OAuth credentials only on `Check Fleet OAuth credentials`+and `Run live Fleet smoke`; checkout, setup, package installation, and copied-suite+preparation do not inherit either secret.++No credential value, access token, or authorization header is written to logs+or uploaded artifacts. The workflow does not create temporary user keys.++## Live Scenario++Place the stable scenario in+`libs/python/cua-sandbox/tests/live/test_fleet_ephemeral.py`. The test is+opt-in and skips unless the OAuth environment is configured, keeping ordinary+local and pull request test runs credential-free.++Each lane and event class uses a reusable DNS-safe namespace. The workflow sets+`cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}`, yielding `schedule`, `push`, or `manual` namespaces.+Event-and-lane concurrency serializes each deterministic claim.++Provision with the exact certified image:++```text+296062593712.dkr.ecr.us-west-2.amazonaws.com/desktop-workspace-duo@sha256:5b9cb82f482834f7541901b87be956e7544d0db13fabc0b372cbc5eca5a74180+```++Use the public SDK with:++- `cpu=4`+- `memory_mb=4096`+- `server_port=8000`+- `time_to_start=900`+- `request_timeout=60`+- `telemetry_enabled=False`++The pinned digest isolates SDK and Fleet regressions from unrelated image-tag+movement. Image publication health belongs in a separate image workflow.++## Assertions++While the sandbox context is active, the test verifies:++1. the sandbox has a non-empty name and is bound to the expected namespace;+2. the generated Fleet template exposes the named `server` Service with+ `targetPort=8000`;+3. the generated readiness probe uses `tcpSocket.port=8000`;+4. `sandbox.screen.size()` returns exactly `1024x768`;+5. a screenshot starts with the PNG signature and has a nontrivial payload;+6. `sandbox.shell.run("uname -s")` succeeds and returns `Linux`.++The direct Fleet template assertions are deliberate. Guest screen and shell+success prove connectivity, while the template checks prove that the public+`server_port` input reached live infrastructure rather than succeeding through+an unrelated default or stale resource.++## Cleanup And Leak Detection++Fleet reconciliation intentionally preserves each dedicated namespace, pool, and+template. After `Sandbox.ephemeral()` exits, the monitor performs claim-only+cleanup verification after every provisioning attempt, including a failure before the context yields:++1. preserve the original scenario exception, if any;+2. poll until claims are absent;+3. collect a sanitized inventory; and+4. after successful provisioning, require exactly the pool/template named after the namespace and zero claims; pre-yield failures retain diagnostic inventory without imposing that invariant.++A remaining claim or any unexpected inventory is a diagnostic failure. The test+and workflow never explicitly delete a namespace, pool, or template: name-only+deletes can race with reconciliation. Cleanup errors are reported alongside the+primary exception.++## Diagnostics And Artifacts++The live test writes a sanitized JSON summary containing lane, source SHA,+installed package versions, the resolved `cua_sandbox` module origin, namespace+and resource names, assertion timing, screen and shell observations, claim-only cleanup results and persistent reconciled resources. On failure only, the+workflow uploads this summary and related diagnostics with short retention.++A manual `force_failure` first runs `Write controlled failure diagnostics` to+create a sanitized `summary.json` containing `ControlledFailure`, then exits+nonzero so the failure-only artifact path is itself certifiable. The versions+step uses `tee -a "$GITHUB_OUTPUT"` to log and publish `sandbox` and `fleet`+outputs for the Alertmanager payload; it never reads its own step outputs.++Cleanup remains diagnostic-only and without explicit deletion: the workflow+never calls `cleanup_namespace` or `delete_namespace`.++## Alertmanager++Each lane has an `if: failure()` notification step posting to+`https://am.cua.ai/api/v2/alerts`. Use an alert such as+`PeriodicCuaSandboxLiveE2EFailed` with labels:++- `severity=critical`+- `service=cua-sandbox`+- `job=periodic-cua-sandbox-live`+- `lane=main-source|published-package`++Annotations include the failed GitHub Actions run, the source SHA or installed+package version, the pinned image digest, and a link to the workflow dashboard.+Do not include credentials or raw authorization failures that may contain+headers.++The lane label lets Alertmanager group repeated failures without combining a+source regression with a published-package or shared-infrastructure failure.++## Workflow Contract Coverage++The repository-side contract parses the workflow with `yaml.BaseLoader` and+asserts triggers, path filters, the executed preparation matrix, checkout ref,+event-and-lane concurrency, credential preflight, copied-suite isolation,+version output handling, controlled-failure diagnostics, failure-only artifacts,+Alertmanager labels, full-SHA action pins, and absence of explicit deletion.+Scripts CI installs `pyyaml` and runs this contract when the workflow changes.++## Rollout++1. Add the live test, workflow, and workflow contract test with the schedule+ temporarily disabled or guarded.+2. Manually dispatch `main-source` and verify assertions, diagnostics, and+ complete claim-only cleanup.+3. Manually dispatch `published-package` and verify the installed release plus+ cleanup behavior.+4. Exercise the Alertmanager payload without exposing secrets, then resolve the+ test alert.+5. Enable the `7/15 * * * *` schedule.+6. Observe at least two consecutive scheduled runs for both lanes before+ considering rollout complete.++## Success Criteria++- Both lanes pass against live Fleet infrastructure with the pinned Duo image.+- A relevant merge to `main` receives an immediate source-lane result.+- Every scheduled interval starts both lanes; a newer schedule can cancel only+ an older scheduled run of the same lane, while push and manual runs finish.+- Port, screen, screenshot, and shell assertions all exercise the public SDK.+- Normal runs retain only the named persistent pool/template with no claims; failed provisioning records read-only claim and inventory diagnostics without deleting resources.+- A forced failure produces one actionable, lane-specific Alertmanager alert+ and sanitized failure artifacts.++## Live Evidence Remediation++The monitor uses reusable, dedicated namespaces instead of per-run namespaces.+Each lane has one DNS-safe namespace for each event class:++- `cua-live-<lane>-schedule` for scheduled runs+- `cua-live-<lane>-push` for pushes+- `cua-live-<lane>-manual` for `workflow_dispatch`++The event-and-lane concurrency group serializes use of each deterministic claim;+only scheduled runs cancel an older scheduled run in the same lane. Fleet+reconciliation preserves the namespace, pool, and template, all named after the+namespace. `Sandbox.ephemeral()` is verified with claim-only cleanup: after+exit the monitor polls until claims are absent, records persistent reconciled+resources, and requires exactly the named pool/template with zero claims. It+never explicitly deletes a namespace, pool, or template.diff --git a/libs/python/cua-sandbox/tests/live/__init__.py b/libs/python/cua-sandbox/tests/live/__init__.pynew file mode 100644index 0000000000..e69de29bb2diff --git a/libs/python/cua-sandbox/tests/live/fleet_e2e_support.py b/libs/python/cua-sandbox/tests/live/fleet_e2e_support.pynew file mode 100644index 0000000000..d756bac37e--- /dev/null+++ b/libs/python/cua-sandbox/tests/live/fleet_e2e_support.py@@ -0,0 +1,165 @@+from __future__ import annotations++import asyncio+import json+import os+import re+import time+from pathlib import Path+from typing import Any++import httpx+from fleet_sdk import (+ CyclopsClient,+ CyclopsConfiguration,+ CyclopsCredentials,+ HttpClient,+ HttpError,+ HttpHeader,+ HttpRequest,+ HttpResponse,+ SdkError,+)++DEFAULT_BASE_URL = "https://run.cua.ai"+DEFAULT_TOKEN_URL = "https://auth.cua.ai/realms/cyclops-cs/protocol/openid-connect/token"+SENSITIVE_SUMMARY_KEY_NAMES = {"api_key", "apikey"}+SENSITIVE_SUMMARY_KEY_PARTS = {"authorization", "password", "secret", "token"}+++class HttpxFleetClient(HttpClient):+ def __init__(self) -> None:+ self._client = httpx.AsyncClient(timeout=60.0)++ async def execute(self, request: HttpRequest) -> HttpResponse:+ try:+ response = await self._client.request(+ request.method,+ request.url,+ headers={header.name: header.value for header in request.headers},+ content=request.body,+ )+ except httpx.TransportError as error:+ raise HttpError.Transport(str(error)) from error+ return HttpResponse(+ status=response.status_code,+ headers=[+ HttpHeader(name=name, value=value) for name, value in response.headers.multi_items()+ ],+ body=response.content,+ )++ async def aclose(self) -> None:+ await self._client.aclose()+++def build_namespace_name(lane: str, event_name: str) -> str:+ event_class = {+ "schedule": "schedule",+ "push": "push",+ "workflow_dispatch": "manual",+ }.get(event_name, "manual")+ raw = f"cua-live-{lane}-{event_class}".lower()+ normalized = re.sub(r"[^a-z0-9-]+", "-", raw).strip("-")+ return normalized[:63].rstrip("-")+++def build_fleet_client() -> tuple[CyclopsClient, HttpxFleetClient]:+ client_id = os.environ["CUA_CLIENT_ID"]+ client_secret = os.environ["CUA_CLIENT_SECRET"]+ http_client = HttpxFleetClient()+ configuration = CyclopsConfiguration(+ base_url=os.environ.get("CUA_FLEET_BASE_URL", DEFAULT_BASE_URL),+ token_url=os.environ.get("CUA_TOKEN_URL", DEFAULT_TOKEN_URL),+ credentials=CyclopsCredentials(client_id, client_secret),+ pool_poll_interval_ms=2000,+ pool_poll_limit=300,+ claim_poll_interval_ms=2000,+ claim_poll_limit=300,+ )+ return CyclopsClient.connect(configuration, http_client), http_client+++def is_not_found_error(error: BaseException) -> bool:+ return isinstance(error, SdkError.Status) and error.status == 404+++async def wait_claims_absent(+ client: CyclopsClient,+ name: str,+ *,+ timeout: float = 180.0,+ interval: float = 5.0,+) -> bool:+ deadline = time.monotonic() + timeout+ while time.monotonic() < deadline:+ try:+ if not await client.list_claims(name):+ return True+ except Exception as error:+ if is_not_found_error(error):+ return True+ raise+ await asyncio.sleep(interval)+ try:+ return not await client.list_claims(name)+ except Exception as error:+ if is_not_found_error(error):+ return True+ raise+++async def collect_resource_inventory(client: CyclopsClient, name: str) -> dict[str, list[str]]:+ try:+ templates = await client.list_templates(name)+ pools = await client.list_pools(name)+ claims = await client.list_claims(name)+ except Exception as error:+ if is_not_found_error(error):+ return {"templates": [], "pools": [], "claims": []}+ raise+ return {+ "templates": [item.metadata.name for item in templates],+ "pools": [item.metadata.name for item in pools],+ "claims": [item.metadata.name for item in claims],+ }+++def assert_template_contract(template: Any, expected_port: int) -> None:+ vm_template = template.spec.vm_template+ server = next((service for service in vm_template.services if service.name == "server"), None)+ assert server is not None, "server service is required"+ assert (+ server.target_port == expected_port+ ), f"server target_port={server.target_port}, expected {expected_port}"+ probes = json.loads(vm_template.probes.to_json())+ observed = probes["readinessProbe"]["tcpSocket"]["port"]+ assert observed == expected_port, f"readiness probe port={observed}, expected {expected_port}"+++def _is_sensitive_summary_key(key: object) -> bool:+ separated = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", str(key))+ normalized = re.sub(r"[^a-z0-9]+", "_", separated.lower()).strip("_")+ return (+ normalized in SENSITIVE_SUMMARY_KEY_NAMES+ or normalized.endswith("_api_key")+ or any(part in SENSITIVE_SUMMARY_KEY_PARTS for part in normalized.split("_"))+ )+++def _redact_summary(value: Any) -> Any:+ if isinstance(value, dict):+ return {+ key: "<redacted>" if _is_sensitive_summary_key(key) else _redact_summary(item)+ for key, item in value.items()+ }+ if isinstance(value, list):+ return [_redact_summary(item) for item in value]+ if isinstance(value, tuple):+ return tuple(_redact_summary(item) for item in value)+ return value+++def write_summary(path: Path, summary: dict[str, Any]) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(json.dumps(_redact_summary(summary), indent=2, sort_keys=True) + "\n")diff --git a/libs/python/cua-sandbox/tests/live/test_fleet_ephemeral.py b/libs/python/cua-sandbox/tests/live/test_fleet_ephemeral.pynew file mode 100644index 0000000000..0d3e2b4a06--- /dev/null+++ b/libs/python/cua-sandbox/tests/live/test_fleet_ephemeral.py@@ -0,0 +1,202 @@+from __future__ import annotations++import os+import re+import time+from importlib.metadata import version+from pathlib import Path++import cua_sandbox+import pytest+from cua_sandbox import Image, Sandbox++from tests.live.fleet_e2e_support import (+ assert_template_contract,+ build_fleet_client,+ build_namespace_name,+ collect_resource_inventory,+ wait_claims_absent,+ write_summary,+)++IMAGE = (+ "296062593712.dkr.ecr.us-west-2.amazonaws.com/desktop-workspace-duo"+ "@sha256:5b9cb82f482834f7541901b87be956e7544d0db13fabc0b372cbc5eca5a74180"+)+++def has_oauth_credentials() -> bool:+ return bool(os.environ.get("CUA_CLIENT_ID") and os.environ.get("CUA_CLIENT_SECRET"))+++def selected_namespace() -> str:+ lane = os.environ.get("CUA_LIVE_E2E_LANE", "local")+ namespace = os.environ.get("CUA_LIVE_E2E_NAMESPACE") or build_namespace_name(+ lane,+ os.environ.get("CUA_LIVE_E2E_EVENT", os.environ.get("GITHUB_EVENT_NAME", "manual")),+ )+ if not namespace.startswith("cua-live-"):+ raise ValueError("CUA_LIVE_E2E_NAMESPACE must start with cua-live-")+ if len(namespace) > 63 or re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", namespace) is None:+ raise ValueError("CUA_LIVE_E2E_NAMESPACE must be a DNS-1123 label of at most 63 characters")+ return namespace+++pytestmark = [+ pytest.mark.asyncio,+ pytest.mark.skipif(not has_oauth_credentials(), reason="Fleet OAuth credentials not set"),+]+++async def run_fleet_ephemeral_live() -> None:+ lane = os.environ.get("CUA_LIVE_E2E_LANE", "local")+ namespace = selected_namespace()+ artifact_dir = Path(os.environ.get("CUA_LIVE_E2E_ARTIFACT_DIR", "/tmp/cua-live-e2e"))+ summary = {+ "lane": lane,+ "namespace": namespace,+ "image": IMAGE,+ "source_sha": os.environ.get("CUA_LIVE_E2E_SOURCE_SHA") or os.environ.get("GITHUB_SHA"),+ "packages": {+ "cua-sandbox": version("cua-sandbox"),+ "cua-fleet": version("cua-fleet"),+ },+ "module_origins": {+ "cua_sandbox": str(Path(cua_sandbox.__file__).resolve()),+ },+ }+ fleet, http_client = build_fleet_client()+ primary_error: BaseException | None = None+ cleanup_error: BaseException | None = None+ close_error: BaseException | None = None+ summary_error: BaseException | None = None+ provisioning_attempted = False+ sandbox_yielded = False++ def record_cleanup_error(error: BaseException) -> None:+ nonlocal cleanup_error+ error_summary = {"type": type(error).__name__}+ if cleanup_error is None:+ cleanup_error = error+ summary["cleanup_error"] = error_summary+ else:+ summary.setdefault("cleanup_secondary_errors", []).append(error_summary)++ try:+ provisioning_attempted = True+ started = time.monotonic()+ async with Sandbox.ephemeral(+ Image.from_registry(IMAGE),+ name=namespace,+ cpu=4,+ memory_mb=4096,+ server_port=8000,+ time_to_start=900,+ request_timeout=60,+ telemetry_enabled=False,+ ) as sandbox:+ sandbox_yielded = True+ summary["provision_seconds"] = time.monotonic() - started+ summary["sandbox_name"] = sandbox.name+ try:+ assert (+ isinstance(sandbox.name, str) and sandbox.name+ ), "sandbox name must be a non-empty string"+ assert (+ sandbox.name == namespace+ ), f"sandbox name {sandbox.name!r} must equal namespace {namespace!r}"++ template = await fleet.get_template(namespace, namespace)+ assert_template_contract(template, expected_port=8000)++ width, height = await sandbox.screen.size()+ summary["screen"] = {"width": width, "height": height}+ assert (width, height) == (1024, 768)++ screenshot = await sandbox.screenshot()+ artifact_dir.mkdir(parents=True, exist_ok=True)+ (artifact_dir / "screen.png").write_bytes(screenshot)+ assert screenshot.startswith(b"\x89PNG\r\n\x1a\n")+ assert len(screenshot) > 1000++ result = await sandbox.shell.run("uname -s")+ summary["shell"] = {+ "success": result.success,+ "stdout": result.stdout.strip(),+ "stderr": result.stderr.strip(),+ }+ assert result.success+ assert result.stdout.strip() == "Linux"+ except BaseException as error:+ primary_error = error+ summary["error"] = {"type": type(error).__name__}+ except BaseException as error:+ if primary_error is None:+ if sandbox_yielded:+ record_cleanup_error(error)+ else:+ primary_error = error+ summary["error"] = {"type": type(error).__name__}+ else:+ summary["context_exit_error"] = {"type": type(error).__name__}+ finally:+ if provisioning_attempted:+ cleanup_started = time.monotonic()+ claims_absent: bool | None = None+ inventory: dict[str, list[str]] | None = None+ try:+ claims_absent = await wait_claims_absent(fleet, namespace)+ summary["claims_absent"] = claims_absent+ except BaseException as error:+ record_cleanup_error(error)+ try:+ inventory = await collect_resource_inventory(fleet, namespace)+ summary["persistent_resources"] = inventory+ except BaseException as error:+ record_cleanup_error(error)++ if claims_absent is False:+ try:+ summary["claim_leak"] = True+ pytest.fail(f"claims remain in namespace {namespace} after Sandbox.ephemeral()")+ except BaseException as error:+ record_cleanup_error(error)+ if sandbox_yielded and inventory is not None:+ expected_inventory = {"templates": [namespace], "pools": [namespace], "claims": []}+ if inventory != expected_inventory:+ try:+ summary["unexpected_inventory"] = True+ pytest.fail(+ "unexpected reconciled resource inventory "+ f"for namespace {namespace}: {inventory}"+ )+ except BaseException as error:+ record_cleanup_error(error)+ summary["cleanup_seconds"] = time.monotonic() - cleanup_started+ if not sandbox_yielded:+ summary["provisioning"] = {"attempted": True, "sandbox_yielded": False}++ try:+ await http_client.aclose()+ except BaseException as error:+ close_error = error+ summary["close_error"] = {"type": type(error).__name__}++ try:+ write_summary(artifact_dir / "summary.json", summary)+ except BaseException as error:+ summary_error = error+ summary["summary_error"] = {"type": type(error).__name__}++ if primary_error is not None:+ raise primary_error+ if cleanup_error is not None:+ raise cleanup_error+ if close_error is not None:+ raise close_error+ if summary_error is not None:+ raise summary_error+++async def test_fleet_ephemeral_live() -> None:+ await run_fleet_ephemeral_live()diff --git a/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py b/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.pynew file mode 100644index 0000000000..ff9de869e3--- /dev/null+++ b/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py@@ -0,0 +1,835 @@+from __future__ import annotations++import json+import re+from types import SimpleNamespace++import pytest+from cua_sandbox import Image+from fleet_sdk import SdkError++from tests.live.fleet_e2e_support import (+ assert_template_contract,+ build_namespace_name,+ collect_resource_inventory,+ wait_claims_absent,+ write_summary,+)+++def test_live_test_requires_both_oauth_values(monkeypatch) -> None:+ monkeypatch.delenv("CUA_CLIENT_ID", raising=False)+ monkeypatch.delenv("CUA_CLIENT_SECRET", raising=False)+ from tests.live.test_fleet_ephemeral import has_oauth_credentials++ assert not has_oauth_credentials()+ monkeypatch.setenv("CUA_CLIENT_ID", "client")+ assert not has_oauth_credentials()+ monkeypatch.setenv("CUA_CLIENT_SECRET", "secret")+ assert has_oauth_credentials()+++ ("event_name", "event_class"),+ [("schedule", "schedule"), ("push", "push"), ("workflow_dispatch", "manual")],+)+def test_build_namespace_name_is_stable_for_each_lane_and_event_class(+ event_name: str, event_class: str+) -> None:+ name = build_namespace_name("published-package", event_name)+ assert name == f"cua-live-published-package-{event_class}"+ assert len(name) <= 63+++def test_build_namespace_name_normalizes_invalid_overlong_lane_input() -> None:+ name = build_namespace_name("Published_Package!!!" + "X" * 100, "workflow_dispatch")++ assert len(name) <= 63+ assert re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", name)+ assert name.startswith("cua-live-published-package-")+++def test_assert_template_contract_accepts_server_port_8000() -> None:+ probes = SimpleNamespace(+ to_json=lambda: json.dumps({"readinessProbe": {"tcpSocket": {"port": 8000}}})+ )+ template = SimpleNamespace(+ spec=SimpleNamespace(+ vm_template=SimpleNamespace(+ services=[SimpleNamespace(name="server", target_port=8000)],+ probes=probes,+ )+ )+ )+ assert_template_contract(template, expected_port=8000)+++def test_assert_template_contract_rejects_wrong_service_port() -> None:+ probes = SimpleNamespace(+ to_json=lambda: json.dumps({"readinessProbe": {"tcpSocket": {"port": 8000}}})+ )+ template = SimpleNamespace(+ spec=SimpleNamespace(+ vm_template=SimpleNamespace(+ services=[SimpleNamespace(name="server", target_port=5000)],+ probes=probes,+ )+ )+ )+ with pytest.raises(AssertionError, match="target_port"):+ assert_template_contract(template, expected_port=8000)+++def test_assert_template_contract_rejects_missing_server_service() -> None:+ probes = SimpleNamespace(+ to_json=lambda: json.dumps({"readinessProbe": {"tcpSocket": {"port": 8000}}})+ )+ template = SimpleNamespace(+ spec=SimpleNamespace(+ vm_template=SimpleNamespace(+ services=[SimpleNamespace(name="other", target_port=8000)],+ probes=probes,+ )+ )+ )++ with pytest.raises(AssertionError, match="server service"):+ assert_template_contract(template, expected_port=8000)+++async def test_collect_resource_inventory_lists_owned_resources() -> None:+ def resource(name: str):+ return SimpleNamespace(metadata=SimpleNamespace(name=name))++ class FakeClient:+ async def list_templates(self, name: str):+ return [resource("template-a")]++ async def list_pools(self, name: str):+ return [resource("pool-a")]++ async def list_claims(self, name: str):+ return [resource("claim-a")]++ assert await collect_resource_inventory(FakeClient(), "demo") == {+ "templates": ["template-a"],+ "pools": ["pool-a"],+ "claims": ["claim-a"],+ }+++async def test_wait_claims_absent_polls_until_claims_are_gone() -> None:+ calls = 0++ class FakeClient:+ async def list_claims(self, name: str):+ nonlocal calls+ calls += 1+ if calls == 1:+ return [SimpleNamespace(metadata=SimpleNamespace(name="claim-a"))]+ return []++ assert await wait_claims_absent(FakeClient(), "demo", timeout=1, interval=0)+ assert calls == 2+++async def test_wait_claims_absent_returns_true_for_public_sdk_404() -> None:+ class FakeClient:+ async def list_claims(self, name: str):+ raise SdkError.Status("list claims", 404, b"not found")++ assert await wait_claims_absent(FakeClient(), "demo", timeout=1, interval=0)+++async def test_wait_claims_absent_times_out_when_claims_remain() -> None:+ class FakeClient:+ async def list_claims(self, name: str):+ return [SimpleNamespace(metadata=SimpleNamespace(name="claim-a"))]++ assert not await wait_claims_absent(FakeClient(), "demo", timeout=0, interval=0)+++async def test_wait_claims_absent_propagates_non_404_sdk_errors() -> None:+ class FakeClient:+ async def list_claims(self, name: str):+ raise SdkError.Status("list claims", 500, b"failure")++ with pytest.raises(SdkError.Status):+ await wait_claims_absent(FakeClient(), "demo", timeout=1, interval=0)+++async def test_collect_resource_inventory_returns_empty_for_public_sdk_404() -> None:+ class FakeClient:+ async def list_templates(self, name: str):+ raise SdkError.Status("list templates", 404, b"not found")++ assert await collect_resource_inventory(FakeClient(), "demo") == {+ "templates": [],+ "pools": [],+ "claims": [],+ }+++async def test_collect_resource_inventory_propagates_non_404_sdk_errors() -> None:+ class FakeClient:+ async def list_templates(self, name: str):+ raise SdkError.Status("list templates", 500, b"failure")++ with pytest.raises(SdkError.Status):+ await collect_resource_inventory(FakeClient(), "demo")+++def test_write_summary_recursively_redacts_sensitive_values(tmp_path) -> None:+ path = tmp_path / "summary.json"+ write_summary(+ path,+ {+ "authorization": "Bearer top-secret",+ "token": "token-value",+ "client_secret": "client-secret-value",+ "api_key": "api-key-value",+ "clientSecret": "camel-client-secret-value",+ "accessToken": "access-token-value",+ "refreshToken": "refresh-token-value",+ "nested": {+ "Authorization": "Nested Bearer secret",+ "x-api-key": "header-api-key-value",+ "items": [{"TOKEN": "nested-token"}, {"safe": "visible"}],+ },+ },+ )++ rendered = path.read_text()+ summary = json.loads(rendered)++ for secret in (+ "Bearer top-secret",+ "token-value",+ "client-secret-value",+ "api-key-value",+ "camel-client-secret-value",+ "access-token-value",+ "refresh-token-value",+ "header-api-key-value",+ "Nested Bearer secret",+ "nested-token",+ ):+ assert secret not in rendered+ assert summary["authorization"] == "<redacted>"+ assert summary["token"] == "<redacted>"+ assert summary["client_secret"] == "<redacted>"+ assert summary["api_key"] == "<redacted>"+ assert summary["clientSecret"] == "<redacted>"+ assert summary["accessToken"] == "<redacted>"+ assert summary["refreshToken"] == "<redacted>"+ assert summary["nested"]["Authorization"] == "<redacted>"+ assert summary["nested"]["x-api-key"] == "<redacted>"+ assert summary["nested"]["items"][0]["TOKEN"] == "<redacted>"+ assert summary["nested"]["items"][1]["safe"] == "visible"+++def test_live_cleanup_exposes_no_explicit_namespace_deletion_api() -> None:+ from cua_sandbox import Sandbox as CuaSandbox+ from cua_sandbox.transport.fleet_cloud import FleetCloudTransport++ from tests.live import fleet_e2e_support+ from tests.live import test_fleet_ephemeral as live_test++ assert not hasattr(fleet_e2e_support, "cleanup_namespace")+ assert not hasattr(live_test, "cleanup_namespace")+ assert not hasattr(CuaSandbox, "owns_namespace")+ assert not hasattr(FleetCloudTransport, "owns_namespace")+++def test_selected_namespace_rejects_unsafe_override(monkeypatch) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "shared-production")++ with pytest.raises(ValueError, match="must start with cua-live-"):+ live_test.selected_namespace()+++async def test_existing_safe_namespace_is_reused_without_explicit_deletion(+ monkeypatch, tmp_path+) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ calls: list[str] = []+ summaries = []++ class FakeHttpClient:+ async def aclose(self) -> None:+ calls.append("close")++ class FakeSandbox:+ name = "cua-live-existing"++ class screen:+ @staticmethod+ async def size():+ return (1024, 768)++ @staticmethod+ async def screenshot():+ return b"\x89PNG\r\n\x1a\n" + b"x" * 1001++ class shell:+ @staticmethod+ async def run(command: str):+ return SimpleNamespace(success=True, stdout="Linux\n", stderr="")++ class FakeEphemeral:+ async def __aenter__(self):+ return FakeSandbox()++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ class FakeFleet:+ async def get_template(self, namespace: str, name: str):+ return object()++ def ephemeral(*args, **kwargs):+ calls.append("provision")+ return FakeEphemeral()++ async def wait_claims_absent(fleet, namespace: str) -> bool:+ return True++ async def collect_resource_inventory(fleet, namespace: str):+ return {"templates": [namespace], "pools": [namespace], "claims": []}++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-existing")+ monkeypatch.setenv("CUA_LIVE_E2E_ARTIFACT_DIR", str(tmp_path))+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (FakeFleet(), FakeHttpClient()))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", ephemeral)+ monkeypatch.setattr(live_test, "assert_template_contract", lambda template, expected_port: None)+ monkeypatch.setattr(live_test, "wait_claims_absent", wait_claims_absent)+ monkeypatch.setattr(live_test, "collect_resource_inventory", collect_resource_inventory)+ monkeypatch.setattr(live_test, "write_summary", lambda path, summary: summaries.append(summary))++ await live_test.run_fleet_ephemeral_live()++ assert calls == ["provision", "close"]+ assert summaries[-1]["persistent_resources"] == {+ "templates": ["cua-live-existing"],+ "pools": ["cua-live-existing"],+ "claims": [],+ }++[email protected]("cleanup_stage", ["poll", "inventory"])+async def test_cleanup_failure_does_not_mask_primary_failure(+ monkeypatch, cleanup_stage: str+) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ class PrimaryFailure(Exception):+ pass++ class CleanupFailure(Exception):+ pass++ class FakeSandbox:+ name = "cua-live-primary-failure"++ class FailingEphemeral:+ async def __aenter__(self):+ return FakeSandbox()++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ class FakeFleet:+ async def get_template(self, namespace: str, name: str):+ raise PrimaryFailure("provision failed")++ class FakeHttpClient:+ async def aclose(self) -> None:+ return None++ summaries = []++ async def wait_claims_absent(fleet, namespace: str) -> bool:+ if cleanup_stage == "poll":+ raise CleanupFailure("poll failed")+ return False++ async def collect_resource_inventory(fleet, namespace: str):+ raise CleanupFailure("inventory failed")++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-primary-failure")+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (FakeFleet(), FakeHttpClient()))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", lambda *args, **kwargs: FailingEphemeral())+ monkeypatch.setattr(live_test, "wait_claims_absent", wait_claims_absent)+ monkeypatch.setattr(live_test, "collect_resource_inventory", collect_resource_inventory)+ monkeypatch.setattr(live_test, "write_summary", lambda path, summary: summaries.append(summary))++ with pytest.raises(PrimaryFailure, match="provision failed"):+ await live_test.run_fleet_ephemeral_live()++ assert summaries[-1]["cleanup_error"] == {"type": "CleanupFailure"}+++async def test_summary_write_failure_still_closes_and_preserves_primary_error(monkeypatch) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ class SummaryWriteFailure(Exception):+ pass++ class FakeHttpClient:+ closed = False++ async def aclose(self) -> None:+ self.closed = True++ class FailingEphemeral:+ async def __aenter__(self):+ raise RuntimeError("primary failure")++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ http_client = FakeHttpClient()++ def write_summary(path, summary) -> None:+ raise SummaryWriteFailure("disk unavailable")++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-summary-failure")+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (object(), http_client))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", lambda *args, **kwargs: FailingEphemeral())+ monkeypatch.setattr(live_test, "write_summary", write_summary)++ with pytest.raises(RuntimeError, match="primary failure"):+ await live_test.run_fleet_ephemeral_live()++ assert http_client.closed+++async def test_close_failure_does_not_mask_primary_error(monkeypatch) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ class CloseFailure(Exception):+ pass++ class FakeHttpClient:+ async def aclose(self) -> None:+ raise CloseFailure("close failed")++ class FailingEphemeral:+ async def __aenter__(self):+ raise RuntimeError("primary failure")++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ summaries = []++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-close-failure")+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (object(), FakeHttpClient()))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", lambda *args, **kwargs: FailingEphemeral())+ monkeypatch.setattr(live_test, "write_summary", lambda path, summary: summaries.append(summary))++ with pytest.raises(RuntimeError, match="primary failure"):+ await live_test.run_fleet_ephemeral_live()++ assert summaries[-1]["close_error"] == {"type": "CloseFailure"}+++ "namespace",+ ["cua-live-UPPER", "cua-live-", "cua-live-" + "a" * 55, "cua-live-name_"],+)+def test_selected_namespace_requires_dns_1123_label(monkeypatch, namespace: str) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", namespace)++ with pytest.raises(ValueError, match="DNS-1123 label"):+ live_test.selected_namespace()+++async def test_claim_leak_records_persistent_inventory_without_deletion(+ monkeypatch, tmp_path+) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ summaries = []++ class FakeSandbox:+ name = "cua-live-raced"++ class screen:+ @staticmethod+ async def size():+ return (1024, 768)++ @staticmethod+ async def screenshot():+ return b"\x89PNG\r\n\x1a\n" + b"x" * 1001++ class shell:+ @staticmethod+ async def run(command: str):+ return SimpleNamespace(success=True, stdout="Linux\n", stderr="")++ class FakeEphemeral:+ async def __aenter__(self):+ return FakeSandbox()++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ class FakeHttpClient:+ async def aclose(self) -> None:+ return None++ class FakeFleet:+ async def get_template(self, namespace: str, name: str):+ return object()++ async def wait_claims_absent(fleet, namespace: str) -> bool:+ return False++ async def collect_resource_inventory(fleet, namespace: str):+ return {"templates": [namespace], "pools": [namespace], "claims": ["claim-a"]}++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-raced")+ monkeypatch.setenv("CUA_LIVE_E2E_ARTIFACT_DIR", str(tmp_path))+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (FakeFleet(), FakeHttpClient()))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", lambda *args, **kwargs: FakeEphemeral())+ monkeypatch.setattr(live_test, "assert_template_contract", lambda template, expected_port: None)+ monkeypatch.setattr(live_test, "wait_claims_absent", wait_claims_absent)+ monkeypatch.setattr(live_test, "collect_resource_inventory", collect_resource_inventory)+ monkeypatch.setattr(live_test, "write_summary", lambda path, summary: summaries.append(summary))++ with pytest.raises(pytest.fail.Exception, match="claims remain"):+ await live_test.run_fleet_ephemeral_live()++ assert summaries[-1]["claim_leak"] is True+ assert summaries[-1]["persistent_resources"] == {+ "templates": ["cua-live-raced"],+ "pools": ["cua-live-raced"],+ "claims": ["claim-a"],+ }+++async def test_provisioning_failure_before_yield_never_deletes_namespace(monkeypatch) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ class ProvisioningFailure(Exception):+ pass++ summaries = []++ class FailingEphemeral:+ async def __aenter__(self):+ raise ProvisioningFailure("provisioning failed")++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ class FakeHttpClient:+ async def aclose(self) -> None:+ return None++ async def wait_claims_absent(fleet, namespace: str) -> bool:+ return False++ async def collect_resource_inventory(fleet, namespace: str):+ return {"templates": [], "pools": [], "claims": []}++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-provisioning-failure")+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (object(), FakeHttpClient()))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", lambda *args, **kwargs: FailingEphemeral())+ monkeypatch.setattr(live_test, "wait_claims_absent", wait_claims_absent)+ monkeypatch.setattr(live_test, "collect_resource_inventory", collect_resource_inventory)+ monkeypatch.setattr(live_test, "write_summary", lambda path, summary: summaries.append(summary))++ with pytest.raises(ProvisioningFailure, match="provisioning failed"):+ await live_test.run_fleet_ephemeral_live()++ assert summaries[-1]["provisioning"] == {"attempted": True, "sandbox_yielded": False}+++async def test_pre_yield_claim_leak_is_recorded_without_masking_provisioning_error(+ monkeypatch, tmp_path+) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ class ProvisioningFailure(Exception):+ pass++ summaries = []++ class FailingEphemeral:+ async def __aenter__(self):+ raise ProvisioningFailure("provisioning failed")++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ class FakeHttpClient:+ async def aclose(self) -> None:+ return None++ async def wait_claims_absent(fleet, namespace: str) -> bool:+ return False++ async def collect_resource_inventory(fleet, namespace: str):+ return {"templates": [namespace], "pools": [namespace], "claims": ["claim-a"]}++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-pre-yield-claim")+ monkeypatch.setenv("CUA_LIVE_E2E_ARTIFACT_DIR", str(tmp_path))+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (object(), FakeHttpClient()))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", lambda *args, **kwargs: FailingEphemeral())+ monkeypatch.setattr(live_test, "wait_claims_absent", wait_claims_absent)+ monkeypatch.setattr(live_test, "collect_resource_inventory", collect_resource_inventory)+ monkeypatch.setattr(live_test, "write_summary", lambda path, summary: summaries.append(summary))++ with pytest.raises(ProvisioningFailure, match="provisioning failed"):+ await live_test.run_fleet_ephemeral_live()++ assert summaries[-1]["claim_leak"] is True+ assert summaries[-1]["persistent_resources"]["claims"] == ["claim-a"]+ assert summaries[-1]["cleanup_error"] == {"type": "Failed"}+++async def test_pre_yield_missing_namespace_records_empty_inventory_without_cleanup_failure(+ monkeypatch, tmp_path+) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ class ProvisioningFailure(Exception):+ pass++ summaries = []++ class FailingEphemeral:+ async def __aenter__(self):+ raise ProvisioningFailure("provisioning failed")++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ class FakeHttpClient:+ async def aclose(self) -> None:+ return None++ async def wait_claims_absent(fleet, namespace: str) -> bool:+ return True++ async def collect_resource_inventory(fleet, namespace: str):+ return {"templates": [], "pools": [], "claims": []}++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-pre-yield-missing")+ monkeypatch.setenv("CUA_LIVE_E2E_ARTIFACT_DIR", str(tmp_path))+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (object(), FakeHttpClient()))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", lambda *args, **kwargs: FailingEphemeral())+ monkeypatch.setattr(live_test, "wait_claims_absent", wait_claims_absent)+ monkeypatch.setattr(live_test, "collect_resource_inventory", collect_resource_inventory)+ monkeypatch.setattr(live_test, "write_summary", lambda path, summary: summaries.append(summary))++ with pytest.raises(ProvisioningFailure, match="provisioning failed"):+ await live_test.run_fleet_ephemeral_live()++ assert summaries[-1]["provisioning"] == {"attempted": True, "sandbox_yielded": False}+ assert summaries[-1]["persistent_resources"] == {"templates": [], "pools": [], "claims": []}+ assert "cleanup_error" not in summaries[-1]+++async def test_live_runner_uses_pinned_ephemeral_configuration(monkeypatch) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ class StopProvisioning(Exception):+ pass++ captured = {}++ class FailingEphemeral:+ async def __aenter__(self):+ raise StopProvisioning("stop after capturing arguments")++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ class FakeHttpClient:+ async def aclose(self) -> None:+ return None++ def ephemeral(image, **kwargs):+ captured["image"] = image+ captured["kwargs"] = kwargs+ return FailingEphemeral()++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-configuration")+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (object(), FakeHttpClient()))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", ephemeral)+ monkeypatch.setattr(live_test, "write_summary", lambda path, summary: None)++ with pytest.raises(StopProvisioning, match="capturing arguments"):+ await live_test.run_fleet_ephemeral_live()++ assert captured["image"] == Image.from_registry(live_test.IMAGE)+ assert captured["kwargs"] == {+ "name": "cua-live-configuration",+ "cpu": 4,+ "memory_mb": 4096,+ "server_port": 8000,+ "time_to_start": 900,+ "request_timeout": 60,+ "telemetry_enabled": False,+ }+++ ("sandbox_name", "message"),+ [("", "non-empty string"), ("cua-live-unexpected", "must equal namespace")],+)+async def test_invalid_sandbox_identity_preserves_primary_error_and_runs_cleanup(+ monkeypatch, tmp_path, sandbox_name: str, message: str+) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ summaries = []++ class FakeSandbox:+ name = sandbox_name++ class FakeEphemeral:+ async def __aenter__(self):+ return FakeSandbox()++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ class FakeFleet:+ async def get_template(self, namespace: str, name: str):+ raise AssertionError("template inspection must not follow an invalid identity")++ class FakeHttpClient:+ async def aclose(self) -> None:+ return None++ async def wait_claims_absent(fleet, namespace: str) -> bool:+ return False++ async def collect_resource_inventory(fleet, namespace: str):+ return {"templates": [namespace], "pools": [namespace], "claims": ["claim-a"]}++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-identity")+ monkeypatch.setenv("CUA_LIVE_E2E_SOURCE_SHA", "checked-out-sha")+ monkeypatch.setenv("GITHUB_SHA", "event-sha")+ monkeypatch.setenv("CUA_LIVE_E2E_ARTIFACT_DIR", str(tmp_path))+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (FakeFleet(), FakeHttpClient()))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", lambda *args, **kwargs: FakeEphemeral())+ monkeypatch.setattr(live_test, "wait_claims_absent", wait_claims_absent)+ monkeypatch.setattr(live_test, "collect_resource_inventory", collect_resource_inventory)+ monkeypatch.setattr(live_test, "write_summary", lambda path, summary: summaries.append(summary))++ with pytest.raises(AssertionError, match=message):+ await live_test.run_fleet_ephemeral_live()++ assert summaries[-1]["source_sha"] == "checked-out-sha"+ assert summaries[-1]["claim_leak"] is True+ assert summaries[-1]["persistent_resources"]["claims"] == ["claim-a"]+ assert summaries[-1]["cleanup_error"] == {"type": "Failed"}+++ ("close_fails", "summary_fails"),+ [(True, False), (False, True), (True, True)],+)+async def test_cleanup_error_precedes_close_and_summary_failures(+ monkeypatch, tmp_path, close_fails: bool, summary_fails: bool+) -> None:+ from tests.live import test_fleet_ephemeral as live_test++ class CloseFailure(Exception):+ pass++ class SummaryFailure(Exception):+ pass++ summaries = []++ class FakeSandbox:+ name = "cua-live-owned"++ class screen:+ @staticmethod+ async def size():+ return (1024, 768)++ @staticmethod+ async def screenshot():+ return b"\x89PNG\r\n\x1a\n" + b"x" * 1001++ class shell:+ @staticmethod+ async def run(command: str):+ return SimpleNamespace(success=True, stdout="Linux\n", stderr="")++ class FakeEphemeral:+ async def __aenter__(self):+ return FakeSandbox()++ async def __aexit__(self, exc_type, exc_value, traceback) -> None:+ return None++ class FakeFleet:+ async def get_template(self, namespace: str, name: str):+ return object()++ class FakeHttpClient:+ async def aclose(self) -> None:+ if close_fails:+ raise CloseFailure("close failed")++ async def wait_claims_absent(fleet, namespace: str) -> bool:+ return False++ async def collect_resource_inventory(fleet, namespace: str):+ return {"templates": [namespace], "pools": [namespace], "claims": ["claim-a"]}++ def write_summary(path, summary) -> None:+ summaries.append(summary)+ if summary_fails:+ raise SummaryFailure("summary failed")++ monkeypatch.setenv("CUA_LIVE_E2E_NAMESPACE", "cua-live-owned")+ monkeypatch.setenv("CUA_LIVE_E2E_ARTIFACT_DIR", str(tmp_path))+ monkeypatch.setattr(live_test, "build_fleet_client", lambda: (FakeFleet(), FakeHttpClient()))+ monkeypatch.setattr(live_test.Sandbox, "ephemeral", lambda *args, **kwargs: FakeEphemeral())+ monkeypatch.setattr(live_test, "assert_template_contract", lambda template, expected_port: None)+ monkeypatch.setattr(live_test, "wait_claims_absent", wait_claims_absent)+ monkeypatch.setattr(live_test, "collect_resource_inventory", collect_resource_inventory)+ monkeypatch.setattr(live_test, "write_summary", write_summary)++ with pytest.raises(pytest.fail.Exception, match="claims remain"):+ await live_test.run_fleet_ephemeral_live()++ assert summaries[-1]["cleanup_error"] == {"type": "Failed"}+ if close_fails:+ assert summaries[-1]["close_error"] == {"type": "CloseFailure"}+ if summary_fails:+ assert summaries[-1]["summary_error"] == {"type": "SummaryFailure"}diff --git a/libs/python/cua-sandbox/tests/test_vm_cleanup.py b/libs/python/cua-sandbox/tests/test_vm_cleanup.pyindex ab1815f868..710aa58250 100644--- a/libs/python/cua-sandbox/tests/test_vm_cleanup.py+++ b/libs/python/cua-sandbox/tests/test_vm_cleanup.py@@ -292,6 +292,7 @@ class TestFleetServerPortForwarding:transport.connect = AsyncMock()with (+ patch.object(Sandbox, "_uses_fleet", return_value=True),patch("cua_sandbox.sandbox.FleetCloudTransport",return_value=transport,