Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
test(cua-sandbox): exercise persistent pre-provisioned pools in periodic live E2E (#3218)
r33drichards9b625d7dceb9
diff --git a/.github/scripts/tests/test_periodic_cua_sandbox_live.py b/.github/scripts/tests/test_periodic_cua_sandbox_live.pyindex 277b01c703..316ab91512 100644--- a/.github/scripts/tests/test_periodic_cua_sandbox_live.py+++ b/.github/scripts/tests/test_periodic_cua_sandbox_live.py@@ -15,6 +15,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3]WORKFLOW = REPO_ROOT / ".github/workflows/periodic-cua-sandbox-live.yml"SANDBOX_ROOT = REPO_ROOT / "libs/python/cua-sandbox"LIVE_TEST = SANDBOX_ROOT / "tests/live/test_fleet_ephemeral.py"+POOL_LIVE_TEST = SANDBOX_ROOT / "tests/live/test_fleet_pool_persistent.py"class TestPeriodicCuaSandboxLive(unittest.TestCase):@@ -29,13 +30,16 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):step["name"]: step for step in job["steps"] if isinstance(step, dict) and "name" in step}- def run_prepare_matrix(self, event_name: str, requested_lane: str) -> dict[str, object]:+ def run_prepare_matrix(+ self, event_name: str, requested_lane: str, requested_suite: str = ""+ ) -> dict[str, object]:prepare_script = self.workflow()["jobs"]["prepare"]["steps"][0]["run"]with tempfile.TemporaryDirectory() as temporary_directory:github_output = Path(temporary_directory) / "github-output"environment = os.environ | {"EVENT_NAME": event_name,"REQUESTED_LANE": requested_lane,+ "REQUESTED_SUITE": requested_suite,"GITHUB_OUTPUT": str(github_output),}subprocess.run(@@ -69,14 +73,17 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):inputs["lane"]["options"],["both", "main-source", "published-package"],)+ self.assertEqual(inputs["suite"]["options"], ["both", "ephemeral", "pool"])+ self.assertEqual(inputs["suite"]["default"], "both")self.assertEqual(inputs["force_failure"]["type"], "boolean")prepare = workflow["jobs"]["prepare"]self.assertEqual(prepare["outputs"]["matrix"], "${{ steps.matrix.outputs.matrix }}")prepare_script = prepare["steps"][0]["run"]self.assertIn('[[ "$EVENT_NAME" == "push" ]]', prepare_script)- self.assertIn('"lane":"main-source"', prepare_script)- self.assertIn('"lane":"published-package"', prepare_script)+ self.assertIn('{"lane":"main-source","suite":"ephemeral"}', prepare_script)+ self.assertIn('lanes=("main-source" "published-package")', prepare_script)+ self.assertIn('suites=("ephemeral" "pool")', prepare_script)def test_jobs_only_run_in_the_upstream_repository(self) -> None:workflow = self.workflow()@@ -87,37 +94,34 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):"github.repository == 'trycua/cua'",)- def test_prepare_matrix_selects_lanes_for_each_trigger(self) -> None:+ def test_prepare_matrix_selects_lanes_and_suites_for_each_trigger(self) -> None:+ main_ephemeral = {"lane": "main-source", "suite": "ephemeral"}+ main_pool = {"lane": "main-source", "suite": "pool"}+ published_ephemeral = {"lane": "published-package", "suite": "ephemeral"}+ published_pool = {"lane": "published-package", "suite": "pool"}+ all_lanes_and_suites = {+ "include": [main_ephemeral, main_pool, published_ephemeral, published_pool]+ }expected_matrices = {- ("push", ""): {"include": [{"lane": "main-source"}]},- ("push", "both"): {"include": [{"lane": "main-source"}]},- ("push", "published-package"): {"include": [{"lane": "main-source"}]},- ("schedule", ""): {- "include": [- {"lane": "main-source"},- {"lane": "published-package"},- ]- },- ("schedule", "main-source"): {- "include": [- {"lane": "main-source"},- {"lane": "published-package"},- ]- },- ("workflow_dispatch", "both"): {- "include": [- {"lane": "main-source"},- {"lane": "published-package"},- ]- },- ("workflow_dispatch", "main-source"): {"include": [{"lane": "main-source"}]},- ("workflow_dispatch", "published-package"): {- "include": [{"lane": "published-package"}]+ ("push", "", ""): {"include": [main_ephemeral]},+ ("push", "both", "both"): {"include": [main_ephemeral]},+ ("push", "published-package", "pool"): {"include": [main_ephemeral]},+ ("schedule", "", ""): all_lanes_and_suites,+ ("schedule", "main-source", "pool"): all_lanes_and_suites,+ ("workflow_dispatch", "both", "both"): all_lanes_and_suites,+ ("workflow_dispatch", "main-source", ""): {"include": [main_ephemeral, main_pool]},+ ("workflow_dispatch", "main-source", "both"): {"include": [main_ephemeral, main_pool]},+ ("workflow_dispatch", "main-source", "pool"): {"include": [main_pool]},+ ("workflow_dispatch", "both", "pool"): {"include": [main_pool, published_pool]},+ ("workflow_dispatch", "published-package", "ephemeral"): {+ "include": [published_ephemeral]},}for inputs, expected_matrix in expected_matrices.items():- with self.subTest(event_name=inputs[0], requested_lane=inputs[1]):+ with self.subTest(+ event_name=inputs[0], requested_lane=inputs[1], requested_suite=inputs[2]+ ):self.assertEqual(self.run_prepare_matrix(*inputs), expected_matrix)def test_docs_describe_the_remediated_workflow(self) -> None:@@ -139,6 +143,14 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):"persistent reconciled resources","claim-only cleanup","cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",+ "periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }}",+ "cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",+ "cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",+ "Run live Fleet pool smoke",+ "test_fleet_pool_persistent.py",+ "WarmPoolAutoscaling(min_pool_size=0, initial_pool_size=0, max_pool_size=1)",+ "pool_pre_existed",+ "claim-only release",)stale_contract = ("Concurrency is scoped\nper lane",@@ -171,7 +183,7 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):)self.assertEqual(live["concurrency"]["group"],- "periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}",+ "periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }}",)self.assertEqual(live["concurrency"]["cancel-in-progress"],@@ -194,15 +206,30 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):{"Check Fleet OAuth credentials": expected_oauth_env,"Run live Fleet smoke": expected_oauth_env,+ "Run live Fleet pool smoke": expected_oauth_env,},)self.assertEqual(live["env"]["CUA_LIVE_E2E_EVENT"], "${{ github.event_name }}")+ self.assertEqual(live["env"]["CUA_LIVE_E2E_SUITE"], "${{ matrix.suite }}")self.assertEqual(live["env"]["CUA_LIVE_E2E_NAMESPACE"],"cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",)- self.assertNotIn("github.run_id", live["env"]["CUA_LIVE_E2E_NAMESPACE"])- self.assertNotIn("github.run_attempt", live["env"]["CUA_LIVE_E2E_NAMESPACE"])+ self.assertEqual(+ live["env"]["CUA_LIVE_E2E_POOL_WARM_NAMESPACE"],+ "cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",+ )+ self.assertEqual(+ live["env"]["CUA_LIVE_E2E_POOL_COLD_NAMESPACE"],+ "cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",+ )+ for namespace_env in (+ "CUA_LIVE_E2E_NAMESPACE",+ "CUA_LIVE_E2E_POOL_WARM_NAMESPACE",+ "CUA_LIVE_E2E_POOL_COLD_NAMESPACE",+ ):+ self.assertNotIn("github.run_id", live["env"][namespace_env])+ self.assertNotIn("github.run_attempt", live["env"][namespace_env])step_names = [step["name"] for step in live["steps"]]self.assertLess(@@ -246,10 +273,19 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):self.assertIn("tests/live", isolated_suite)live_step = steps["Run live Fleet smoke"]self.assertEqual(live_step["env"], expected_oauth_env)+ self.assertIn("matrix.suite == 'ephemeral'", live_step["if"])live_run = live_step["run"]self.assertIn("$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_ephemeral.py", live_run)self.assertNotIn("libs/python/cua-sandbox/tests/live", live_run)self.assertIn('PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT', live_run)+ pool_step = steps["Run live Fleet pool smoke"]+ self.assertEqual(pool_step["env"], expected_oauth_env)+ self.assertIn("matrix.suite == 'pool'", pool_step["if"])+ self.assertIn("force_failure", pool_step["if"])+ pool_run = pool_step["run"]+ self.assertIn("$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_pool_persistent.py", pool_run)+ self.assertNotIn("libs/python/cua-sandbox/tests/live", pool_run)+ self.assertIn('PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT', pool_run)def test_failure_diagnostics_alerting_and_pinned_actions(self) -> None:workflow = self.workflow()@@ -265,6 +301,11 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):)self.assertEqual(steps["Upload failure diagnostics"]["if"], "failure()")self.assertEqual(steps["Upload failure diagnostics"]["with"]["retention-days"], "7")+ self.assertEqual(+ steps["Upload failure diagnostics"]["with"]["name"],+ "cua-sandbox-live-${{ matrix.lane }}-${{ matrix.suite }}"+ "-${{ github.run_id }}-${{ github.run_attempt }}",+ )controlled_summary = steps["Write controlled failure diagnostics"]self.assertEqual(@@ -274,6 +315,7 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):self.assertIn("summary.json", controlled_summary["run"])self.assertIn("ControlledFailure", controlled_summary["run"])self.assertIn('os.environ["CUA_LIVE_E2E_SOURCE_SHA"]', controlled_summary["run"])+ self.assertIn('os.environ["CUA_LIVE_E2E_SUITE"]', controlled_summary["run"])self.assertNotIn('os.environ.get("GITHUB_SHA")', controlled_summary["run"])self.assertEqual(steps["Controlled alert test failure"]["if"],@@ -289,6 +331,7 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):self.assertIn("https://am.cua.ai/api/v2/alerts", alert["run"])self.assertIn("PeriodicCuaSandboxLiveE2EFailed", alert["run"])self.assertIn('"lane": "${{ matrix.lane }}"', alert["run"])+ self.assertIn('"suite": "${{ matrix.suite }}"', alert["run"])self.assertIn("${{ steps.versions.outputs.sandbox }}", alert["run"])self.assertIn("${{ steps.source_sha.outputs.source_sha }}", alert["run"])self.assertNotIn('"source_sha": "${{ github.sha }}"', alert["run"])@@ -310,6 +353,28 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):self.assertIn("module_origins", live_test)self.assertIn("cua_sandbox", live_test)+ def test_pool_suite_is_claim_only_and_pool_is_persistent(self) -> None:+ workflow = WORKFLOW.read_text()+ pool_test = POOL_LIVE_TEST.read_text()++ self.assertIn("Run live Fleet pool smoke", workflow)+ self.assertIn("Sandbox.ephemeral", pool_test)+ self.assertIn("Pool.apply", pool_test)+ self.assertIn(+ "WarmPoolAutoscaling(min_pool_size=0, initial_pool_size=0, max_pool_size=1)",+ pool_test,+ )+ self.assertIn("wait_claims_absent", pool_test)+ self.assertIn("claim_leak", pool_test)+ self.assertIn("persistent_resources", pool_test)+ self.assertIn("unexpected_inventory", pool_test)+ self.assertIn("module_origins", pool_test)+ self.assertIn("pool_pre_existed", pool_test)+ self.assertNotIn("pool.delete(", pool_test)+ self.assertNotIn("delete_pool", pool_test)+ self.assertNotIn("delete_namespace", pool_test)+ self.assertNotIn("keep_pool", pool_test)+def test_isolated_suite_cannot_import_checkout_cua_sandbox(self) -> None:with tempfile.TemporaryDirectory() as temporary_directory:temporary_root = Path(temporary_directory)diff --git a/.github/workflows/ci-public-containerdisk-refs.yml b/.github/workflows/ci-public-containerdisk-refs.ymlindex 5a7bd2c201..c8cd4e88b5 100644--- a/.github/workflows/ci-public-containerdisk-refs.yml+++ b/.github/workflows/ci-public-containerdisk-refs.yml@@ -10,6 +10,7 @@ on:- "infra/fleets-wif-smoke/**"- "libs/fleet/backend/auth/**"- "libs/python/cua-sandbox/tests/live/test_fleet_ephemeral.py"+ - "libs/python/cua-sandbox/tests/live/test_fleet_pool_persistent.py"- "tests/test_public_containerdisk_refs.py"permissions:diff --git a/.github/workflows/periodic-cua-sandbox-live.yml b/.github/workflows/periodic-cua-sandbox-live.ymlindex 66cb68f9cd..e23ea05839 100644--- a/.github/workflows/periodic-cua-sandbox-live.yml+++ b/.github/workflows/periodic-cua-sandbox-live.yml@@ -18,6 +18,12 @@ on:default: bothtype: choiceoptions: [both, main-source, published-package]+ suite:+ description: "Suite to run"+ required: true+ default: both+ type: choice+ options: [both, ephemeral, pool]force_failure:description: "Fail after setup to certify alerting"required: true@@ -39,14 +45,27 @@ jobs:env:EVENT_NAME: ${{ github.event_name }}REQUESTED_LANE: ${{ inputs.lane }}+ REQUESTED_SUITE: ${{ inputs.suite }}run: |set -euo pipefailif [[ "$EVENT_NAME" == "push" ]]; then- matrix='{"include":[{"lane":"main-source"}]}'- elif [[ "$EVENT_NAME" == "workflow_dispatch" && "$REQUESTED_LANE" != "both" ]]; then- matrix="{\"include\":[{\"lane\":\"$REQUESTED_LANE\"}]}"+ matrix='{"include":[{"lane":"main-source","suite":"ephemeral"}]}'else- matrix='{"include":[{"lane":"main-source"},{"lane":"published-package"}]}'+ lanes=("main-source" "published-package")+ if [[ "$EVENT_NAME" == "workflow_dispatch" && "$REQUESTED_LANE" != "both" ]]; then+ lanes=("$REQUESTED_LANE")+ fi+ suites=("ephemeral" "pool")+ if [[ "$EVENT_NAME" == "workflow_dispatch" && "${REQUESTED_SUITE:-both}" != "both" ]]; then+ suites=("$REQUESTED_SUITE")+ fi+ include=""+ for lane in "${lanes[@]}"; do+ for suite in "${suites[@]}"; do+ include+="{\"lane\":\"$lane\",\"suite\":\"$suite\"},"+ done+ done+ matrix="{\"include\":[${include%,}]}"fiecho "matrix=$matrix" >> "$GITHUB_OUTPUT"@@ -59,13 +78,16 @@ jobs:fail-fast: falsematrix: ${{ fromJSON(needs.prepare.outputs.matrix) }}concurrency:- group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}+ group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }}cancel-in-progress: ${{ github.event_name == 'schedule' }}env:CUA_FLEET_BASE_URL: https://run.cua.aiCUA_LIVE_E2E_LANE: ${{ matrix.lane }}+ CUA_LIVE_E2E_SUITE: ${{ matrix.suite }}CUA_LIVE_E2E_EVENT: ${{ github.event_name }}CUA_LIVE_E2E_NAMESPACE: cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}+ CUA_LIVE_E2E_POOL_WARM_NAMESPACE: cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}+ CUA_LIVE_E2E_POOL_COLD_NAMESPACE: cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}CUA_LIVE_E2E_ARTIFACT_DIR: /tmp/cua-live-e2eCUA_TELEMETRY_ENABLED: "false"steps:@@ -154,6 +176,7 @@ jobs:json.dumps({"lane": os.environ["CUA_LIVE_E2E_LANE"],+ "suite": os.environ["CUA_LIVE_E2E_SUITE"],"namespace": os.environ["CUA_LIVE_E2E_NAMESPACE"],"source_sha": os.environ["CUA_LIVE_E2E_SOURCE_SHA"],"error": {"type": "ControlledFailure"},@@ -170,7 +193,7 @@ jobs:run: exit 1- name: Run live Fleet smoke- if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.force_failure) }}+ if: ${{ matrix.suite == 'ephemeral' && !(github.event_name == 'workflow_dispatch' && inputs.force_failure) }}env:CUA_CLIENT_ID: ${{ secrets.CUA_CLIENT_ID }}CUA_CLIENT_SECRET: ${{ secrets.CUA_CLIENT_SECRET }}@@ -178,11 +201,20 @@ jobs:PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT" python -m pytest -q -s \"$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_ephemeral.py"+ - name: Run live Fleet pool smoke+ if: ${{ matrix.suite == 'pool' && !(github.event_name == 'workflow_dispatch' && inputs.force_failure) }}+ env:+ CUA_CLIENT_ID: ${{ secrets.CUA_CLIENT_ID }}+ CUA_CLIENT_SECRET: ${{ secrets.CUA_CLIENT_SECRET }}+ run: |+ PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT" python -m pytest -q -s \+ "$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_pool_persistent.py"+- name: Upload failure diagnosticsif: failure()uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4with:- name: cua-sandbox-live-${{ matrix.lane }}-${{ github.run_id }}-${{ github.run_attempt }}+ name: cua-sandbox-live-${{ matrix.lane }}-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}path: /tmp/cua-live-e2eif-no-files-found: warnretention-days: 7@@ -201,10 +233,11 @@ jobs:"severity": "critical","service": "cua-sandbox","job": "periodic-cua-sandbox-live",- "lane": "${{ matrix.lane }}"+ "lane": "${{ matrix.lane }}",+ "suite": "${{ matrix.suite }}"},"annotations": {- "summary": "Cua Sandbox live Fleet E2E failed (${{ matrix.lane }})",+ "summary": "Cua Sandbox live Fleet E2E failed (${{ matrix.lane }}/${{ matrix.suite }})","description": "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}","source_sha": "${{ steps.source_sha.outputs.source_sha }}","package_version": "${{ steps.versions.outputs.sandbox }}",diff --git a/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.md b/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.mdindex 2aeb54fb3d..84d286adda 100644--- a/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.md+++ b/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.md@@ -906,3 +906,36 @@ The workflow uses step-scoped OAuth credentials only for credential preflight anlive pytest step. After checkout, `git rev-parse HEAD` is exported as`CUA_LIVE_E2E_SOURCE_SHA` and is the source SHA recorded by live, controlled-failure,and Alertmanager evidence.++## Persistent Pool Suite++The workflow later gained a second suite that claims from persistent,+pre-provisioned Fleet pools instead of creating and destroying a pool per run.+The prepare job emits a lane-and-suite matrix: pushes stay on the `ephemeral`+suite only, while scheduled runs cross both lanes with both suites and manual+dispatch selects lane and suite combinations. The concurrency group is+`periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }}`,+so a newer schedule cancels only an older scheduled run of the same lane and+suite.++The suite's step, `Run live Fleet pool smoke`, executes+`tests/live/test_fleet_pool_persistent.py` from the same isolated copied suite+with step-scoped OAuth credentials. Two pool namespaces per lane and event+class are set by the workflow:++- `cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}`+- `cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}`++The warm pool keeps `replicas=1` so claims bind to pre-provisioned capacity;+the cold pool expresses scale-to-zero with+`WarmPoolAutoscaling(min_pool_size=0, initial_pool_size=0, max_pool_size=1)`+because pool reconciliation rejects `replicas` below one. Each run records+`pool_pre_existed` and replica counts from `Pool.get`, reconciles the pinned+configuration idempotently with `Pool.apply`, claims through+`Sandbox.ephemeral(pool=..., name=...)` with the claim name fixed to the+namespace, and exits with a claim-only release. After release the monitor+polls until claims are absent and requires the reconciled inventory to contain+exactly the named pool and template with zero claims; the pool and template+deliberately persist between runs. The warm mode asserts a claim-acquisition+bound only when the pool pre-existed with a ready replica. Failure artifacts+and Alertmanager labels carry the suite alongside the lane.diff --git a/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.md b/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.mdindex 755d8eca63..35d6240620 100644--- a/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.md+++ b/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.md@@ -18,6 +18,8 @@ testing the current Fleet-backed public SDK contract.- Exercise Fleet-backed `Sandbox.ephemeral()` against live productioninfrastructure every 15 minutes.+- Exercise claiming from persistent, pre-provisioned Fleet pools — both a warm+ pool and a scale-to-zero pool — on the same cadence.- Test both the current repository `main` source and the latest published`cua-sandbox` package.- Run the source lane immediately after relevant changes merge to `main`.@@ -49,31 +51,34 @@ testing the current Fleet-backed public SDK contract.`libs/python/cua-fleet/**`,`.github/workflows/periodic-cua-sandbox-live.yml`, and`.github/scripts/tests/test_periodic_cua_sandbox_live.py`.-3. `workflow_dispatch` accepts `both`, `main-source`, or `published-package`,- plus manual-only `force_failure`.+3. `workflow_dispatch` accepts a lane (`both`, `main-source`, or+ `published-package`), a suite (`both`, `ephemeral`, or `pool`), plus+ manual-only `force_failure`.Both jobs are guarded with `if: github.repository == 'trycua/cua'`. A forkthat syncs `main` or enables the schedule therefore never runs the live smoke,never fails the credential preflight, and never posts a fork-originated`PeriodicCuaSandboxLiveE2EFailed` alert to the public Alertmanager endpoint.-The preparation script emits a JSON matrix: every `push` selects only-`main-source`; every `schedule` selects both lanes; manual runs select their-requested lane or both. The workflow contract executes this extracted shell-script in `bash` with a temporary `GITHUB_OUTPUT` and parses the emitted JSON,-so a push-to-both mutation or ignored manual selection fails CI.+The preparation script emits a JSON lane-and-suite matrix: every `push`+selects only `main-source` with the `ephemeral` suite; every `schedule`+selects both lanes crossed with both suites; manual runs select their+requested lane and suite combinations. The workflow contract executes this+extracted shell script in `bash` with a temporary `GITHUB_OUTPUT` and parses+the emitted JSON, so a push-to-both mutation or ignored manual selection fails+CI.-The two lanes use `fail-fast: false` and this concurrency contract:+The matrix jobs use `fail-fast: false` and this concurrency contract:```yamlconcurrency:- group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}+ group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }}cancel-in-progress: ${{ github.event_name == 'schedule' }}```-The event-and-lane grouping means a new schedule can cancel only an older-schedule for the same lane. Push and manual runs use distinct groups and are-allowed to finish.+The event-lane-and-suite grouping means a new schedule can cancel only an+older schedule for the same lane and suite. Push and manual runs use distinct+groups and are allowed to finish.## Installation Isolation@@ -173,6 +178,47 @@ and workflow never explicitly delete a namespace, pool, or template: name-onlydeletes can race with reconciliation. Cleanup errors are reported alongside theprimary exception.+## Persistent Pool Suite++The `pool` suite exercises the pre-provisioned pool consumer path that the+`ephemeral` suite cannot: pools that survive across runs and hand out claims.+`Run live Fleet pool smoke` executes+`libs/python/cua-sandbox/tests/live/test_fleet_pool_persistent.py` from the+same isolated copied suite. The suite runs on `schedule` and+`workflow_dispatch` only; pushes keep running only the `ephemeral` suite.++Each lane and event class owns two persistent pool namespaces, set by the+workflow as:++- `cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}`+- `cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}`++The warm mode keeps `replicas=1`, so a scheduled claim binds to an+already-running sandbox and releasing the claim recycles that sandbox back+into the pool. The cold mode expresses scale-to-zero with+`WarmPoolAutoscaling(min_pool_size=0, initial_pool_size=0, max_pool_size=1)`+because pool reconciliation rejects `replicas` below one; a claim then+cold-starts capacity through autoscaler demand.++Each run observes the pool first with `Pool.get`, recording+`pool_pre_existed` and the replica counts, then reconciles the pinned+configuration with `Pool.apply` using the same certified image digest,+`cpu=4`, and `memory_mb=4096`. Reconciliation is idempotent: it bootstraps a+missing pool, heals drift, and never deletes. The claim uses+`Sandbox.ephemeral(pool=..., name=...)` with the claim name fixed to the+namespace, so an interrupted run's claim is adopted and released by the next+run. Exiting the context performs a claim-only release: the pool and template+must persist.++After release the monitor polls until claims are absent and requires the+reconciled inventory to contain exactly the named pool and template with zero+claims — the persistence mirror of the ephemeral suite's empty-inventory+invariant. Replica counts after release are recorded as telemetry only,+because warm-pool recycling and autoscaler decay are server-controlled. The+warm mode additionally asserts a claim-acquisition bound only when the pool+pre-existed with at least one ready replica; bootstrap runs record timing+without enforcing it.+## Diagnostics And ArtifactsThe live test writes a sanitized JSON summary containing lane, source SHA,@@ -199,21 +245,25 @@ Each lane has an `if: failure()` notification step posting to- `service=cua-sandbox`- `job=periodic-cua-sandbox-live`- `lane=main-source|published-package`+- `suite=ephemeral|pool`Annotations include the failed GitHub Actions run, the source SHA or installedpackage version, the pinned image digest, and a link to the workflow dashboard.Do not include credentials or raw authorization failures that may containheaders.-The lane label lets Alertmanager group repeated failures without combining a-source regression with a published-package or shared-infrastructure failure.+The lane and suite labels let Alertmanager group repeated failures without+combining a source regression with a published-package failure, or an+ephemeral provisioning failure with a persistent pool claim failure. Failure+artifacts are named per lane and suite so concurrent matrix jobs never collide+on upload.## Workflow Contract CoverageThe repository-side contract parses the workflow with `yaml.BaseLoader` andasserts triggers, path filters, the upstream-repository fork guard on bothjobs, the executed preparation matrix, checkout ref,-event-and-lane concurrency, credential preflight, copied-suite isolation,+event-lane-and-suite concurrency, credential preflight, copied-suite isolation,version output handling, controlled-failure diagnostics, failure-only artifacts,Alertmanager labels, full-SHA action pins, and absence of explicit deletion.Scripts CI installs `pyyaml` and runs this contract when the workflow changes.@@ -226,10 +276,14 @@ Scripts CI installs `pyyaml` and runs this contract when the workflow changes.complete claim-only cleanup.3. Manually dispatch `published-package` and verify the installed release pluscleanup behavior.-4. Exercise the Alertmanager payload without exposing secrets, then resolve the+4. Manually dispatch the `pool` suite twice per lane: the first run+ bootstraps both persistent pools (`pool_pre_existed` false), the second+ proves a warm claim against pre-provisioned capacity and records cold+ scale-to-zero telemetry.+5. Exercise the Alertmanager payload without exposing secrets, then resolve thetest alert.-5. Enable the `7/15 * * * *` schedule.-6. Observe at least two consecutive scheduled runs for both lanes before+6. Enable the `7/15 * * * *` schedule.+7. Observe at least two consecutive scheduled runs for both lanes beforeconsidering rollout complete.## Success Criteria@@ -240,8 +294,10 @@ Scripts CI installs `pyyaml` and runs this contract when the workflow changes.an older scheduled run of the same lane, while push and manual runs finish.- Port, screen, screenshot, and shell assertions all exercise the public SDK.- Normal runs retain only the named persistent pool/template with no claims; failed provisioning records read-only claim and inventory diagnostics without deleting resources.-- A forced failure produces one actionable, lane-specific Alertmanager alert- and sanitized failure artifacts.+- Pool-suite runs claim from and release back to persistent pools that survive+ the run, with the warm pool binding against pre-provisioned capacity.+- A forced failure produces one actionable, lane- and suite-specific+ Alertmanager alert and sanitized failure artifacts.## Live Evidence Remediation@@ -252,10 +308,15 @@ Each lane has one DNS-safe namespace for each event class:- `cua-live-<lane>-push` for pushes- `cua-live-<lane>-manual` for `workflow_dispatch`-The event-and-lane concurrency group serializes use of each deterministic claim;-only scheduled runs cancel an older scheduled run in the same lane. Fleet-reconciliation preserves the namespace, pool, and template, all named after the-namespace. `Sandbox.ephemeral()` is verified with claim-only cleanup: after-exit the monitor polls until claims are absent, records persistent reconciled-resources, and requires exactly the named pool/template with zero claims. It-never explicitly deletes a namespace, pool, or template.+The persistent pool suite adds `cua-live-pool-warm-<lane>-<event-class>` and+`cua-live-pool-cold-<lane>-<event-class>` namespaces whose pools and templates+deliberately outlive every run.++The event-lane-and-suite concurrency group serializes use of each+deterministic claim; only scheduled runs cancel an older scheduled run in the+same lane and suite. Fleet reconciliation preserves the namespace, pool, and+template, all named after the namespace. `Sandbox.ephemeral()` is verified+with claim-only cleanup: after exit the monitor polls until claims are absent,+records persistent reconciled resources, and requires exactly the named+pool/template with zero claims. It never explicitly deletes a namespace, pool,+or template.diff --git a/libs/python/cua-sandbox/tests/live/fleet_e2e_support.py b/libs/python/cua-sandbox/tests/live/fleet_e2e_support.pyindex d756bac37e..b60320fa91 100644--- a/libs/python/cua-sandbox/tests/live/fleet_e2e_support.py+++ b/libs/python/cua-sandbox/tests/live/fleet_e2e_support.py@@ -53,17 +53,31 @@ class HttpxFleetClient(HttpClient):await self._client.aclose()-def build_namespace_name(lane: str, event_name: str) -> str:- event_class = {+def has_oauth_credentials() -> bool:+ return bool(os.environ.get("CUA_CLIENT_ID") and os.environ.get("CUA_CLIENT_SECRET"))+++def _event_class(event_name: str) -> str:+ return {"schedule": "schedule","push": "push","workflow_dispatch": "manual",}.get(event_name, "manual")- raw = f"cua-live-{lane}-{event_class}".lower()- normalized = re.sub(r"[^a-z0-9-]+", "-", raw).strip("-")+++def _normalize_namespace_name(raw: str) -> str:+ normalized = re.sub(r"[^a-z0-9-]+", "-", raw.lower()).strip("-")return normalized[:63].rstrip("-")+def build_namespace_name(lane: str, event_name: str) -> str:+ return _normalize_namespace_name(f"cua-live-{lane}-{_event_class(event_name)}")+++def build_pool_namespace_name(mode: str, lane: str, event_name: str) -> str:+ return _normalize_namespace_name(f"cua-live-pool-{mode}-{lane}-{_event_class(event_name)}")++def build_fleet_client() -> tuple[CyclopsClient, HttpxFleetClient]:client_id = os.environ["CUA_CLIENT_ID"]client_secret = os.environ["CUA_CLIENT_SECRET"]diff --git a/libs/python/cua-sandbox/tests/live/test_fleet_pool_persistent.py b/libs/python/cua-sandbox/tests/live/test_fleet_pool_persistent.pynew file mode 100644index 0000000000..8c4ed1966f--- /dev/null+++ b/libs/python/cua-sandbox/tests/live/test_fleet_pool_persistent.py@@ -0,0 +1,284 @@+from __future__ import annotations++import os+import re+import time+from importlib.metadata import version+from pathlib import Path++import cua_sandbox+import pytest+from cua_sandbox import Image, Pool, Sandbox, WarmPoolAutoscaling++from tests.live.fleet_e2e_support import (+ assert_template_contract,+ build_fleet_client,+ build_pool_namespace_name,+ collect_resource_inventory,+ has_oauth_credentials,+ is_not_found_error,+ wait_claims_absent,+ write_summary,+)++IMAGE = (+ "public.ecr.aws/k5j5w0x5/cua-ubuntu-24.04"+ "@sha256:82702ebdd32d1f8fc05f2ea409a7c67d0ba9f8f8e4e9f1a89ce40989d5f4475d"+)++POOL_CPU = 4+POOL_MEMORY_MB = 4096+WARM_TIME_TO_START = 180+COLD_TIME_TO_START = 900+WARM_BIND_SLA_SECONDS = 300++MODE_ENV = {+ "warm": "CUA_LIVE_E2E_POOL_WARM_NAMESPACE",+ "cold": "CUA_LIVE_E2E_POOL_COLD_NAMESPACE",+}+++def cold_autoscaling() -> WarmPoolAutoscaling:+ # replicas < 1 is rejected by Pool.apply, so scale-to-zero is expressed+ # through autoscaling: KEDA owns spec.replicas and decays it to zero.+ return WarmPoolAutoscaling(min_pool_size=0, initial_pool_size=0, max_pool_size=1)+++def selected_pool_namespace(mode: str) -> str:+ lane = os.environ.get("CUA_LIVE_E2E_LANE", "local")+ namespace = os.environ.get(MODE_ENV[mode]) or build_pool_namespace_name(+ mode,+ lane,+ os.environ.get("CUA_LIVE_E2E_EVENT", os.environ.get("GITHUB_EVENT_NAME", "manual")),+ )+ if not namespace.startswith(f"cua-live-pool-{mode}-"):+ raise ValueError(f"{MODE_ENV[mode]} must start with cua-live-pool-{mode}-")+ if len(namespace) > 63 or re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", namespace) is None:+ raise ValueError(f"{MODE_ENV[mode]} must be a DNS-1123 label of at most 63 characters")+ return namespace+++pytestmark = [+ pytest.mark.asyncio,+ pytest.mark.skipif(not has_oauth_credentials(), reason="Fleet OAuth credentials not set"),+]+++async def run_fleet_pool_live(mode: str) -> None:+ lane = os.environ.get("CUA_LIVE_E2E_LANE", "local")+ namespace = selected_pool_namespace(mode)+ artifact_dir = Path(os.environ.get("CUA_LIVE_E2E_ARTIFACT_DIR", "/tmp/cua-live-e2e"))+ summary = {+ "lane": lane,+ "mode": mode,+ "namespace": namespace,+ "image": IMAGE,+ "source_sha": os.environ.get("CUA_LIVE_E2E_SOURCE_SHA") or os.environ.get("GITHUB_SHA"),+ "packages": {+ "cua-sandbox": version("cua-sandbox"),+ "cua-fleet": version("cua-fleet"),+ },+ "module_origins": {+ "cua_sandbox": str(Path(cua_sandbox.__file__).resolve()),+ },+ }+ fleet, http_client = build_fleet_client()+ primary_error: BaseException | None = None+ cleanup_error: BaseException | None = None+ close_error: BaseException | None = None+ summary_error: BaseException | None = None+ pool_applied = False+ sandbox_yielded = False++ def record_cleanup_error(error: BaseException) -> None:+ nonlocal cleanup_error+ error_summary = {"type": type(error).__name__}+ if cleanup_error is None:+ cleanup_error = error+ summary["cleanup_error"] = error_summary+ else:+ summary.setdefault("cleanup_secondary_errors", []).append(error_summary)++ try:+ try:+ existing = await Pool.get(namespace)+ except BaseException as error:+ if not is_not_found_error(error):+ raise+ summary["pool_pre_existed"] = False+ else:+ summary["pool_pre_existed"] = True+ summary["spec_replicas_before"] = existing.resource.spec.replicas+ status = getattr(existing.resource, "status", None)+ summary["ready_replicas_before"] = (+ (getattr(status, "ready_replicas", None) or 0) if status is not None else 0+ )++ apply_started = time.monotonic()+ pool = await Pool.apply(+ Image.from_registry(IMAGE),+ name=namespace,+ replicas=1,+ cpu=POOL_CPU,+ memory_mb=POOL_MEMORY_MB,+ autoscaling=None if mode == "warm" else cold_autoscaling(),+ )+ pool_applied = True+ summary["apply_seconds"] = time.monotonic() - apply_started+ assert pool.name == namespace, f"pool name {pool.name!r} must equal namespace {namespace!r}"++ template = await fleet.get_template(namespace, namespace)+ assert_template_contract(template, expected_port=8000)++ claim_started = time.monotonic()+ async with Sandbox.ephemeral(+ pool=namespace,+ name=namespace,+ time_to_start=WARM_TIME_TO_START if mode == "warm" else COLD_TIME_TO_START,+ telemetry_enabled=False,+ ) as sandbox:+ sandbox_yielded = True+ claim_seconds = time.monotonic() - claim_started+ summary["claim_seconds"] = claim_seconds+ summary["sandbox_name"] = sandbox.name+ sandbox_claim_name = getattr(sandbox, "claim_name", None)+ sandbox_pool_name = getattr(sandbox, "pool_name", None)+ summary["claim_name"] = sandbox_claim_name or sandbox.name+ summary["pool_name"] = sandbox_pool_name or namespace+ try:+ assert (+ isinstance(sandbox.name, str) and sandbox.name+ ), "sandbox name must be a non-empty string"+ if sandbox_claim_name is not None:+ assert sandbox_claim_name == namespace, (+ f"claim name {sandbox_claim_name!r} must equal "+ f"requested name {namespace!r}"+ )+ if sandbox_pool_name is not None:+ assert (+ sandbox_pool_name == namespace+ ), f"pool name {sandbox_pool_name!r} must equal namespace {namespace!r}"++ warm_bind_sla_applied = (+ mode == "warm"+ and summary.get("pool_pre_existed") is True+ and summary.get("ready_replicas_before", 0) >= 1+ )+ summary["warm_bind_sla_applied"] = warm_bind_sla_applied+ if warm_bind_sla_applied:+ assert claim_seconds < WARM_BIND_SLA_SECONDS, (+ f"pre-provisioned warm claim took {claim_seconds:.1f}s, "+ f"expected under {WARM_BIND_SLA_SECONDS}s"+ )++ width, height = await sandbox.screen.size()+ summary["screen"] = {"width": width, "height": height}+ assert (width, height) == (1024, 768)++ screenshot = await sandbox.screenshot()+ artifact_dir.mkdir(parents=True, exist_ok=True)+ (artifact_dir / f"screen-pool-{mode}.png").write_bytes(screenshot)+ assert screenshot.startswith(b"\x89PNG\r\n\x1a\n")+ assert len(screenshot) > 1000++ result = await sandbox.shell.run("uname -s")+ summary["shell"] = {+ "success": result.success,+ "stdout": result.stdout.strip(),+ "stderr": result.stderr.strip(),+ }+ assert result.success+ assert result.stdout.strip() == "Linux"+ except BaseException as error:+ primary_error = error+ summary["error"] = {"type": type(error).__name__}+ except BaseException as error:+ if primary_error is None:+ if sandbox_yielded:+ record_cleanup_error(error)+ else:+ primary_error = error+ summary["error"] = {"type": type(error).__name__}+ else:+ summary["context_exit_error"] = {"type": type(error).__name__}+ finally:+ if pool_applied:+ cleanup_started = time.monotonic()+ claims_absent: bool | None = None+ inventory: dict[str, list[str]] | None = None+ try:+ claims_absent = await wait_claims_absent(fleet, namespace)+ summary["claims_absent"] = claims_absent+ except BaseException as error:+ record_cleanup_error(error)+ try:+ inventory = await collect_resource_inventory(fleet, namespace)+ summary["persistent_resources"] = inventory+ except BaseException as error:+ record_cleanup_error(error)++ if claims_absent is False:+ try:+ summary["claim_leak"] = True+ pytest.fail(f"claims remain in namespace {namespace} after claim-only release")+ except BaseException as error:+ record_cleanup_error(error)+ if inventory is not None:+ expected_inventory = {+ "templates": [namespace],+ "pools": [namespace],+ "claims": [],+ }+ if inventory != expected_inventory:+ try:+ summary["unexpected_inventory"] = True+ pytest.fail(+ f"persistent pool inventory for namespace {namespace} must be "+ f"{expected_inventory}, got {inventory}"+ )+ except BaseException as error:+ record_cleanup_error(error)+ try:+ refreshed = await Pool.get(namespace)+ summary["spec_replicas_after"] = refreshed.resource.spec.replicas+ status = getattr(refreshed.resource, "status", None)+ summary["ready_replicas_after"] = (+ (getattr(status, "ready_replicas", None) or 0) if status is not None else 0+ )+ except BaseException as error:+ record_cleanup_error(error)+ summary["cleanup_seconds"] = time.monotonic() - cleanup_started+ if not sandbox_yielded:+ summary["provisioning"] = {+ "pool_applied": pool_applied,+ "sandbox_yielded": False,+ }++ try:+ await http_client.aclose()+ except BaseException as error:+ close_error = error+ summary["close_error"] = {"type": type(error).__name__}++ try:+ write_summary(artifact_dir / f"summary-pool-{mode}.json", summary)+ except BaseException as error:+ summary_error = error+ summary["summary_error"] = {"type": type(error).__name__}++ if primary_error is not None:+ raise primary_error+ if cleanup_error is not None:+ raise cleanup_error+ if close_error is not None:+ raise close_error+ if summary_error is not None:+ raise summary_error+++async def test_fleet_pool_warm_live() -> None:+ await run_fleet_pool_live("warm")+++async def test_fleet_pool_cold_live() -> None:+ await run_fleet_pool_live("cold")diff --git a/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py b/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.pyindex 3055122634..6cc5f26f40 100644--- a/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py+++ b/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py@@ -11,7 +11,9 @@ from fleet_sdk import SdkErrorfrom tests.live.fleet_e2e_support import (assert_template_contract,build_namespace_name,+ build_pool_namespace_name,collect_resource_inventory,+ has_oauth_credentials,wait_claims_absent,write_summary,)@@ -49,6 +51,39 @@ def test_build_namespace_name_normalizes_invalid_overlong_lane_input() -> None:assert name.startswith("cua-live-published-package-")[email protected]("mode", ["warm", "cold"])+ ("event_name", "event_class"),+ [("schedule", "schedule"), ("push", "push"), ("workflow_dispatch", "manual")],+)+def test_build_pool_namespace_name_is_stable_for_each_mode_lane_and_event_class(+ mode: str, event_name: str, event_class: str+) -> None:+ name = build_pool_namespace_name(mode, "published-package", event_name)+ assert name == f"cua-live-pool-{mode}-published-package-{event_class}"+ assert len(name) <= 63+++def test_build_pool_namespace_name_normalizes_invalid_overlong_input() -> None:+ name = build_pool_namespace_name(+ "warm", "Published_Package!!!" + "X" * 100, "workflow_dispatch"+ )++ assert len(name) <= 63+ assert re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", name)+ assert name.startswith("cua-live-pool-warm-published-package-")+++def test_support_has_oauth_credentials_requires_both_values(monkeypatch) -> None:+ monkeypatch.delenv("CUA_CLIENT_ID", raising=False)+ monkeypatch.delenv("CUA_CLIENT_SECRET", raising=False)+ assert not has_oauth_credentials()+ monkeypatch.setenv("CUA_CLIENT_ID", "client")+ assert not has_oauth_credentials()+ monkeypatch.setenv("CUA_CLIENT_SECRET", "secret")+ assert has_oauth_credentials()++def test_assert_template_contract_accepts_server_port_8000() -> None:probes = SimpleNamespace(to_json=lambda: json.dumps({"readinessProbe": {"tcpSocket": {"port": 8000}}})diff --git a/libs/python/cua-sandbox/tests/test_live_fleet_pool_support.py b/libs/python/cua-sandbox/tests/test_live_fleet_pool_support.pynew file mode 100644index 0000000000..654471b5e1--- /dev/null+++ b/libs/python/cua-sandbox/tests/test_live_fleet_pool_support.py@@ -0,0 +1,348 @@+from __future__ import annotations++from types import SimpleNamespace++import pytest+from fleet_sdk import SdkError++from tests.live import test_fleet_pool_persistent as pool_live++PNG = b"\x89PNG\r\n\x1a\n" + b"0" * 2000+++def make_pool_resource(namespace: str, *, spec_replicas: int = 1, ready_replicas: int = 1):+ return SimpleNamespace(+ metadata=SimpleNamespace(namespace=namespace, name=namespace),+ spec=SimpleNamespace(replicas=spec_replicas),+ status=SimpleNamespace(ready_replicas=ready_replicas),+ )+++def not_found(operation: str = "get pool") -> SdkError.Status:+ return SdkError.Status(operation, 404, b"not found")+++class FakePoolHandle:+ def __init__(self, resource) -> None:+ self._resource = resource++ @property+ def name(self) -> str:+ return self._resource.metadata.name++ @property+ def resource(self):+ return self._resource+++class FakeSandbox:+ def __init__(self, name: str, shell_stdout: str) -> None:+ self.name = name+ self.claim_name = name+ self.pool_name = name+ self.screen = SimpleNamespace(size=self._size)+ self.shell = SimpleNamespace(run=self._run)+ self._shell_stdout = shell_stdout++ async def _size(self) -> tuple[int, int]:+ return (1024, 768)++ async def _run(self, command: str):+ return SimpleNamespace(success=True, stdout=self._shell_stdout, stderr="")++ async def screenshot(self) -> bytes:+ return PNG+++class _FakeEphemeral:+ def __init__(self, sandbox: FakeSandbox) -> None:+ self._sandbox = sandbox++ async def __aenter__(self) -> FakeSandbox:+ return self._sandbox++ async def __aexit__(self, exc_type, exc, traceback) -> bool:+ return False+++class PoolRunnerHarness:+ def __init__(self) -> None:+ self.get_results: list = []+ self.apply_calls: list[dict] = []+ self.apply_error: BaseException | None = None+ self.ephemeral_calls: list[dict] = []+ self.shell_stdout = "Linux"+ self.claims_absent: object = True+ self.claims_absent_calls: list[str] = []+ self.inventory: object = None+ self.inventory_calls: list[str] = []+ self.template_contract_calls: list[tuple] = []+ self.summaries: dict[str, dict] = {}+++def install_pool_runner(monkeypatch, tmp_path, *, mode: str, namespace: str) -> PoolRunnerHarness:+ harness = PoolRunnerHarness()+ monkeypatch.setenv("CUA_LIVE_E2E_LANE", "test")+ monkeypatch.setenv("CUA_LIVE_E2E_EVENT", "schedule")+ monkeypatch.setenv("CUA_LIVE_E2E_ARTIFACT_DIR", str(tmp_path))+ monkeypatch.setenv(pool_live.MODE_ENV[mode], namespace)++ class FakePool:+ @staticmethod+ async def get(name: str) -> FakePoolHandle:+ assert harness.get_results, "unexpected Pool.get call"+ result = harness.get_results.pop(0)+ if isinstance(result, BaseException):+ raise result+ return FakePoolHandle(result)++ @staticmethod+ async def apply(image, **kwargs) -> FakePoolHandle:+ harness.apply_calls.append({"image": image, **kwargs})+ if harness.apply_error is not None:+ raise harness.apply_error+ return FakePoolHandle(make_pool_resource(kwargs["name"]))++ class FakeSandboxApi:+ @staticmethod+ def ephemeral(**kwargs) -> _FakeEphemeral:+ harness.ephemeral_calls.append(kwargs)+ return _FakeEphemeral(FakeSandbox(kwargs["name"], harness.shell_stdout))++ class FakeFleet:+ async def get_template(self, template_namespace: str, name: str):+ return SimpleNamespace(namespace=template_namespace, name=name)++ class FakeHttpClient:+ async def aclose(self) -> None:+ return None++ async def fake_wait_claims_absent(fleet, name: str) -> bool:+ harness.claims_absent_calls.append(name)+ if isinstance(harness.claims_absent, BaseException):+ raise harness.claims_absent+ return bool(harness.claims_absent)++ async def fake_collect_resource_inventory(fleet, name: str):+ harness.inventory_calls.append(name)+ if isinstance(harness.inventory, BaseException):+ raise harness.inventory+ if harness.inventory is None:+ return {"templates": [name], "pools": [name], "claims": []}+ return harness.inventory++ def fake_assert_template_contract(template, expected_port: int) -> None:+ harness.template_contract_calls.append((template, expected_port))++ def fake_write_summary(path, summary) -> None:+ harness.summaries[path.name] = summary++ monkeypatch.setattr(pool_live, "Pool", FakePool)+ monkeypatch.setattr(pool_live, "Sandbox", FakeSandboxApi)+ monkeypatch.setattr(pool_live, "build_fleet_client", lambda: (FakeFleet(), FakeHttpClient()))+ monkeypatch.setattr(pool_live, "wait_claims_absent", fake_wait_claims_absent)+ monkeypatch.setattr(pool_live, "collect_resource_inventory", fake_collect_resource_inventory)+ monkeypatch.setattr(pool_live, "assert_template_contract", fake_assert_template_contract)+ monkeypatch.setattr(pool_live, "write_summary", fake_write_summary)+ return harness+++async def test_pool_live_runner_uses_pinned_warm_configuration(monkeypatch, tmp_path) -> None:+ namespace = "cua-live-pool-warm-test-schedule"+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)+ harness.get_results = [+ make_pool_resource(namespace, ready_replicas=1),+ make_pool_resource(namespace, ready_replicas=1),+ ]++ await pool_live.run_fleet_pool_live("warm")++ (apply_call,) = harness.apply_calls+ assert apply_call["name"] == namespace+ assert apply_call["replicas"] == 1+ assert apply_call["cpu"] == 4+ assert apply_call["memory_mb"] == 4096+ assert apply_call["autoscaling"] is None++ (ephemeral_call,) = harness.ephemeral_calls+ assert ephemeral_call["pool"] == namespace+ assert isinstance(ephemeral_call["pool"], str)+ assert ephemeral_call["name"] == namespace+ assert ephemeral_call["time_to_start"] == 180+ assert ephemeral_call["telemetry_enabled"] is False+ for rejected in ("image", "cpu", "memory_mb", "server_port", "replicas"):+ assert rejected not in ephemeral_call++ ((_, contract_port),) = harness.template_contract_calls+ assert contract_port == 8000+ summary = harness.summaries["summary-pool-warm.json"]+ assert summary["pool_pre_existed"] is True+ assert summary["ready_replicas_before"] == 1+ assert summary["warm_bind_sla_applied"] is True+ assert summary["claims_absent"] is True+ assert summary["persistent_resources"] == {+ "templates": [namespace],+ "pools": [namespace],+ "claims": [],+ }+ assert summary["spec_replicas_after"] == 1+++async def test_pool_live_runner_uses_pinned_cold_configuration(monkeypatch, tmp_path) -> None:+ namespace = "cua-live-pool-cold-test-schedule"+ harness = install_pool_runner(monkeypatch, tmp_path, mode="cold", namespace=namespace)+ harness.get_results = [+ make_pool_resource(namespace, ready_replicas=0),+ make_pool_resource(namespace, ready_replicas=0),+ ]++ await pool_live.run_fleet_pool_live("cold")++ (apply_call,) = harness.apply_calls+ assert apply_call["replicas"] == 1+ autoscaling = apply_call["autoscaling"]+ assert autoscaling is not None+ assert autoscaling.min_pool_size == 0+ assert autoscaling.initial_pool_size == 0+ assert autoscaling.max_pool_size == 1++ (ephemeral_call,) = harness.ephemeral_calls+ assert ephemeral_call["time_to_start"] == 900++ summary = harness.summaries["summary-pool-cold.json"]+ assert summary["warm_bind_sla_applied"] is False+++async def test_pool_pre_existed_false_is_recorded_for_public_sdk_404(monkeypatch, tmp_path) -> None:+ namespace = "cua-live-pool-warm-test-schedule"+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)+ harness.get_results = [not_found(), make_pool_resource(namespace)]++ await pool_live.run_fleet_pool_live("warm")++ summary = harness.summaries["summary-pool-warm.json"]+ assert summary["pool_pre_existed"] is False+ assert "ready_replicas_before" not in summary+ assert summary["warm_bind_sla_applied"] is False+++async def test_empty_inventory_fails_as_unexpected(monkeypatch, tmp_path) -> None:+ namespace = "cua-live-pool-warm-test-schedule"+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)+ harness.get_results = [make_pool_resource(namespace), make_pool_resource(namespace)]+ harness.inventory = {"templates": [], "pools": [], "claims": []}++ with pytest.raises(pytest.fail.Exception, match="persistent pool inventory"):+ await pool_live.run_fleet_pool_live("warm")++ summary = harness.summaries["summary-pool-warm.json"]+ assert summary["unexpected_inventory"] is True+++async def test_claim_leak_fails_and_is_recorded(monkeypatch, tmp_path) -> None:+ namespace = "cua-live-pool-warm-test-schedule"+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)+ harness.get_results = [make_pool_resource(namespace), make_pool_resource(namespace)]+ harness.claims_absent = False++ with pytest.raises(pytest.fail.Exception, match="claims remain"):+ await pool_live.run_fleet_pool_live("warm")++ summary = harness.summaries["summary-pool-warm.json"]+ assert summary["claim_leak"] is True+++async def test_warm_sla_not_applied_without_ready_replicas(monkeypatch, tmp_path) -> None:+ namespace = "cua-live-pool-warm-test-schedule"+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)+ harness.get_results = [+ make_pool_resource(namespace, ready_replicas=0),+ make_pool_resource(namespace, ready_replicas=1),+ ]+ monkeypatch.setattr(pool_live, "WARM_BIND_SLA_SECONDS", -1.0)++ await pool_live.run_fleet_pool_live("warm")++ summary = harness.summaries["summary-pool-warm.json"]+ assert summary["warm_bind_sla_applied"] is False+ assert "error" not in summary+++async def test_warm_sla_enforced_with_ready_replicas(monkeypatch, tmp_path) -> None:+ namespace = "cua-live-pool-warm-test-schedule"+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)+ harness.get_results = [+ make_pool_resource(namespace, ready_replicas=1),+ make_pool_resource(namespace, ready_replicas=1),+ ]+ monkeypatch.setattr(pool_live, "WARM_BIND_SLA_SECONDS", -1.0)++ with pytest.raises(AssertionError, match="pre-provisioned warm claim"):+ await pool_live.run_fleet_pool_live("warm")++ summary = harness.summaries["summary-pool-warm.json"]+ assert summary["warm_bind_sla_applied"] is True+ assert summary["error"] == {"type": "AssertionError"}+++async def test_cleanup_failure_does_not_mask_primary_failure(monkeypatch, tmp_path) -> None:+ namespace = "cua-live-pool-warm-test-schedule"+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)+ harness.get_results = [make_pool_resource(namespace), make_pool_resource(namespace)]+ harness.shell_stdout = "Darwin"+ harness.inventory = RuntimeError("inventory unavailable")++ with pytest.raises(AssertionError):+ await pool_live.run_fleet_pool_live("warm")++ summary = harness.summaries["summary-pool-warm.json"]+ assert summary["error"] == {"type": "AssertionError"}+ assert summary["cleanup_error"] == {"type": "RuntimeError"}+++async def test_apply_failure_skips_persistence_verification(monkeypatch, tmp_path) -> None:+ namespace = "cua-live-pool-warm-test-schedule"+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)+ harness.get_results = [not_found()]+ harness.apply_error = RuntimeError("apply rejected")++ with pytest.raises(RuntimeError, match="apply rejected"):+ await pool_live.run_fleet_pool_live("warm")++ assert harness.claims_absent_calls == []+ assert harness.inventory_calls == []+ assert harness.ephemeral_calls == []+ summary = harness.summaries["summary-pool-warm.json"]+ assert summary["provisioning"] == {"pool_applied": False, "sandbox_yielded": False}+ assert summary["error"] == {"type": "RuntimeError"}+++def test_selected_pool_namespace_rejects_foreign_prefix(monkeypatch) -> None:+ monkeypatch.setenv("CUA_LIVE_E2E_POOL_WARM_NAMESPACE", "cua-live-warm-other")++ with pytest.raises(ValueError, match="must start with cua-live-pool-warm-"):+ pool_live.selected_pool_namespace("warm")+++def test_selected_pool_namespace_requires_dns_1123_label(monkeypatch) -> None:+ monkeypatch.setenv("CUA_LIVE_E2E_POOL_COLD_NAMESPACE", "cua-live-pool-cold-" + "x" * 60)++ with pytest.raises(ValueError, match="DNS-1123"):+ pool_live.selected_pool_namespace("cold")+++def test_selected_pool_namespace_builds_default_from_lane_and_event(monkeypatch) -> None:+ monkeypatch.delenv("CUA_LIVE_E2E_POOL_WARM_NAMESPACE", raising=False)+ monkeypatch.setenv("CUA_LIVE_E2E_LANE", "main-source")+ monkeypatch.setenv("CUA_LIVE_E2E_EVENT", "schedule")++ assert pool_live.selected_pool_namespace("warm") == "cua-live-pool-warm-main-source-schedule"