Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

test(cua-sandbox): exercise persistent pre-provisioned pools in periodic live E2E (#3218)

r33drichards9b625d7dceb9
diff --git a/.github/scripts/tests/test_periodic_cua_sandbox_live.py b/.github/scripts/tests/test_periodic_cua_sandbox_live.py
index 277b01c703..316ab91512 100644
--- a/.github/scripts/tests/test_periodic_cua_sandbox_live.py
+++ b/.github/scripts/tests/test_periodic_cua_sandbox_live.py
@@ -15,6 +15,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3]
WORKFLOW = REPO_ROOT / ".github/workflows/periodic-cua-sandbox-live.yml"
SANDBOX_ROOT = REPO_ROOT / "libs/python/cua-sandbox"
LIVE_TEST = SANDBOX_ROOT / "tests/live/test_fleet_ephemeral.py"
+POOL_LIVE_TEST = SANDBOX_ROOT / "tests/live/test_fleet_pool_persistent.py"
class TestPeriodicCuaSandboxLive(unittest.TestCase):
@@ -29,13 +30,16 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
step["name"]: step for step in job["steps"] if isinstance(step, dict) and "name" in step
}
- def run_prepare_matrix(self, event_name: str, requested_lane: str) -> dict[str, object]:
+ def run_prepare_matrix(
+ self, event_name: str, requested_lane: str, requested_suite: str = ""
+ ) -> dict[str, object]:
prepare_script = self.workflow()["jobs"]["prepare"]["steps"][0]["run"]
with tempfile.TemporaryDirectory() as temporary_directory:
github_output = Path(temporary_directory) / "github-output"
environment = os.environ | {
"EVENT_NAME": event_name,
"REQUESTED_LANE": requested_lane,
+ "REQUESTED_SUITE": requested_suite,
"GITHUB_OUTPUT": str(github_output),
}
subprocess.run(
@@ -69,14 +73,17 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
inputs["lane"]["options"],
["both", "main-source", "published-package"],
)
+ self.assertEqual(inputs["suite"]["options"], ["both", "ephemeral", "pool"])
+ self.assertEqual(inputs["suite"]["default"], "both")
self.assertEqual(inputs["force_failure"]["type"], "boolean")
prepare = workflow["jobs"]["prepare"]
self.assertEqual(prepare["outputs"]["matrix"], "${{ steps.matrix.outputs.matrix }}")
prepare_script = prepare["steps"][0]["run"]
self.assertIn('[[ "$EVENT_NAME" == "push" ]]', prepare_script)
- self.assertIn('"lane":"main-source"', prepare_script)
- self.assertIn('"lane":"published-package"', prepare_script)
+ self.assertIn('{"lane":"main-source","suite":"ephemeral"}', prepare_script)
+ self.assertIn('lanes=("main-source" "published-package")', prepare_script)
+ self.assertIn('suites=("ephemeral" "pool")', prepare_script)
def test_jobs_only_run_in_the_upstream_repository(self) -> None:
workflow = self.workflow()
@@ -87,37 +94,34 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
"github.repository == 'trycua/cua'",
)
- def test_prepare_matrix_selects_lanes_for_each_trigger(self) -> None:
+ def test_prepare_matrix_selects_lanes_and_suites_for_each_trigger(self) -> None:
+ main_ephemeral = {"lane": "main-source", "suite": "ephemeral"}
+ main_pool = {"lane": "main-source", "suite": "pool"}
+ published_ephemeral = {"lane": "published-package", "suite": "ephemeral"}
+ published_pool = {"lane": "published-package", "suite": "pool"}
+ all_lanes_and_suites = {
+ "include": [main_ephemeral, main_pool, published_ephemeral, published_pool]
+ }
expected_matrices = {
- ("push", ""): {"include": [{"lane": "main-source"}]},
- ("push", "both"): {"include": [{"lane": "main-source"}]},
- ("push", "published-package"): {"include": [{"lane": "main-source"}]},
- ("schedule", ""): {
- "include": [
- {"lane": "main-source"},
- {"lane": "published-package"},
- ]
- },
- ("schedule", "main-source"): {
- "include": [
- {"lane": "main-source"},
- {"lane": "published-package"},
- ]
- },
- ("workflow_dispatch", "both"): {
- "include": [
- {"lane": "main-source"},
- {"lane": "published-package"},
- ]
- },
- ("workflow_dispatch", "main-source"): {"include": [{"lane": "main-source"}]},
- ("workflow_dispatch", "published-package"): {
- "include": [{"lane": "published-package"}]
+ ("push", "", ""): {"include": [main_ephemeral]},
+ ("push", "both", "both"): {"include": [main_ephemeral]},
+ ("push", "published-package", "pool"): {"include": [main_ephemeral]},
+ ("schedule", "", ""): all_lanes_and_suites,
+ ("schedule", "main-source", "pool"): all_lanes_and_suites,
+ ("workflow_dispatch", "both", "both"): all_lanes_and_suites,
+ ("workflow_dispatch", "main-source", ""): {"include": [main_ephemeral, main_pool]},
+ ("workflow_dispatch", "main-source", "both"): {"include": [main_ephemeral, main_pool]},
+ ("workflow_dispatch", "main-source", "pool"): {"include": [main_pool]},
+ ("workflow_dispatch", "both", "pool"): {"include": [main_pool, published_pool]},
+ ("workflow_dispatch", "published-package", "ephemeral"): {
+ "include": [published_ephemeral]
},
}
for inputs, expected_matrix in expected_matrices.items():
- with self.subTest(event_name=inputs[0], requested_lane=inputs[1]):
+ with self.subTest(
+ event_name=inputs[0], requested_lane=inputs[1], requested_suite=inputs[2]
+ ):
self.assertEqual(self.run_prepare_matrix(*inputs), expected_matrix)
def test_docs_describe_the_remediated_workflow(self) -> None:
@@ -139,6 +143,14 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
"persistent reconciled resources",
"claim-only cleanup",
"cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",
+ "periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }}",
+ "cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",
+ "cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",
+ "Run live Fleet pool smoke",
+ "test_fleet_pool_persistent.py",
+ "WarmPoolAutoscaling(min_pool_size=0, initial_pool_size=0, max_pool_size=1)",
+ "pool_pre_existed",
+ "claim-only release",
)
stale_contract = (
"Concurrency is scoped\nper lane",
@@ -171,7 +183,7 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
)
self.assertEqual(
live["concurrency"]["group"],
- "periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}",
+ "periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }}",
)
self.assertEqual(
live["concurrency"]["cancel-in-progress"],
@@ -194,15 +206,30 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
{
"Check Fleet OAuth credentials": expected_oauth_env,
"Run live Fleet smoke": expected_oauth_env,
+ "Run live Fleet pool smoke": expected_oauth_env,
},
)
self.assertEqual(live["env"]["CUA_LIVE_E2E_EVENT"], "${{ github.event_name }}")
+ self.assertEqual(live["env"]["CUA_LIVE_E2E_SUITE"], "${{ matrix.suite }}")
self.assertEqual(
live["env"]["CUA_LIVE_E2E_NAMESPACE"],
"cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",
)
- self.assertNotIn("github.run_id", live["env"]["CUA_LIVE_E2E_NAMESPACE"])
- self.assertNotIn("github.run_attempt", live["env"]["CUA_LIVE_E2E_NAMESPACE"])
+ self.assertEqual(
+ live["env"]["CUA_LIVE_E2E_POOL_WARM_NAMESPACE"],
+ "cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",
+ )
+ self.assertEqual(
+ live["env"]["CUA_LIVE_E2E_POOL_COLD_NAMESPACE"],
+ "cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}",
+ )
+ for namespace_env in (
+ "CUA_LIVE_E2E_NAMESPACE",
+ "CUA_LIVE_E2E_POOL_WARM_NAMESPACE",
+ "CUA_LIVE_E2E_POOL_COLD_NAMESPACE",
+ ):
+ self.assertNotIn("github.run_id", live["env"][namespace_env])
+ self.assertNotIn("github.run_attempt", live["env"][namespace_env])
step_names = [step["name"] for step in live["steps"]]
self.assertLess(
@@ -246,10 +273,19 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
self.assertIn("tests/live", isolated_suite)
live_step = steps["Run live Fleet smoke"]
self.assertEqual(live_step["env"], expected_oauth_env)
+ self.assertIn("matrix.suite == 'ephemeral'", live_step["if"])
live_run = live_step["run"]
self.assertIn("$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_ephemeral.py", live_run)
self.assertNotIn("libs/python/cua-sandbox/tests/live", live_run)
self.assertIn('PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT', live_run)
+ pool_step = steps["Run live Fleet pool smoke"]
+ self.assertEqual(pool_step["env"], expected_oauth_env)
+ self.assertIn("matrix.suite == 'pool'", pool_step["if"])
+ self.assertIn("force_failure", pool_step["if"])
+ pool_run = pool_step["run"]
+ self.assertIn("$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_pool_persistent.py", pool_run)
+ self.assertNotIn("libs/python/cua-sandbox/tests/live", pool_run)
+ self.assertIn('PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT', pool_run)
def test_failure_diagnostics_alerting_and_pinned_actions(self) -> None:
workflow = self.workflow()
@@ -265,6 +301,11 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
)
self.assertEqual(steps["Upload failure diagnostics"]["if"], "failure()")
self.assertEqual(steps["Upload failure diagnostics"]["with"]["retention-days"], "7")
+ self.assertEqual(
+ steps["Upload failure diagnostics"]["with"]["name"],
+ "cua-sandbox-live-${{ matrix.lane }}-${{ matrix.suite }}"
+ "-${{ github.run_id }}-${{ github.run_attempt }}",
+ )
controlled_summary = steps["Write controlled failure diagnostics"]
self.assertEqual(
@@ -274,6 +315,7 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
self.assertIn("summary.json", controlled_summary["run"])
self.assertIn("ControlledFailure", controlled_summary["run"])
self.assertIn('os.environ["CUA_LIVE_E2E_SOURCE_SHA"]', controlled_summary["run"])
+ self.assertIn('os.environ["CUA_LIVE_E2E_SUITE"]', controlled_summary["run"])
self.assertNotIn('os.environ.get("GITHUB_SHA")', controlled_summary["run"])
self.assertEqual(
steps["Controlled alert test failure"]["if"],
@@ -289,6 +331,7 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
self.assertIn("https://am.cua.ai/api/v2/alerts", alert["run"])
self.assertIn("PeriodicCuaSandboxLiveE2EFailed", alert["run"])
self.assertIn('"lane": "${{ matrix.lane }}"', alert["run"])
+ self.assertIn('"suite": "${{ matrix.suite }}"', alert["run"])
self.assertIn("${{ steps.versions.outputs.sandbox }}", alert["run"])
self.assertIn("${{ steps.source_sha.outputs.source_sha }}", alert["run"])
self.assertNotIn('"source_sha": "${{ github.sha }}"', alert["run"])
@@ -310,6 +353,28 @@ class TestPeriodicCuaSandboxLive(unittest.TestCase):
self.assertIn("module_origins", live_test)
self.assertIn("cua_sandbox", live_test)
+ def test_pool_suite_is_claim_only_and_pool_is_persistent(self) -> None:
+ workflow = WORKFLOW.read_text()
+ pool_test = POOL_LIVE_TEST.read_text()
+
+ self.assertIn("Run live Fleet pool smoke", workflow)
+ self.assertIn("Sandbox.ephemeral", pool_test)
+ self.assertIn("Pool.apply", pool_test)
+ self.assertIn(
+ "WarmPoolAutoscaling(min_pool_size=0, initial_pool_size=0, max_pool_size=1)",
+ pool_test,
+ )
+ self.assertIn("wait_claims_absent", pool_test)
+ self.assertIn("claim_leak", pool_test)
+ self.assertIn("persistent_resources", pool_test)
+ self.assertIn("unexpected_inventory", pool_test)
+ self.assertIn("module_origins", pool_test)
+ self.assertIn("pool_pre_existed", pool_test)
+ self.assertNotIn("pool.delete(", pool_test)
+ self.assertNotIn("delete_pool", pool_test)
+ self.assertNotIn("delete_namespace", pool_test)
+ self.assertNotIn("keep_pool", pool_test)
+
def test_isolated_suite_cannot_import_checkout_cua_sandbox(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
temporary_root = Path(temporary_directory)
diff --git a/.github/workflows/ci-public-containerdisk-refs.yml b/.github/workflows/ci-public-containerdisk-refs.yml
index 5a7bd2c201..c8cd4e88b5 100644
--- a/.github/workflows/ci-public-containerdisk-refs.yml
+++ b/.github/workflows/ci-public-containerdisk-refs.yml
@@ -10,6 +10,7 @@ on:
- "infra/fleets-wif-smoke/**"
- "libs/fleet/backend/auth/**"
- "libs/python/cua-sandbox/tests/live/test_fleet_ephemeral.py"
+ - "libs/python/cua-sandbox/tests/live/test_fleet_pool_persistent.py"
- "tests/test_public_containerdisk_refs.py"
permissions:
diff --git a/.github/workflows/periodic-cua-sandbox-live.yml b/.github/workflows/periodic-cua-sandbox-live.yml
index 66cb68f9cd..e23ea05839 100644
--- a/.github/workflows/periodic-cua-sandbox-live.yml
+++ b/.github/workflows/periodic-cua-sandbox-live.yml
@@ -18,6 +18,12 @@ on:
default: both
type: choice
options: [both, main-source, published-package]
+ suite:
+ description: "Suite to run"
+ required: true
+ default: both
+ type: choice
+ options: [both, ephemeral, pool]
force_failure:
description: "Fail after setup to certify alerting"
required: true
@@ -39,14 +45,27 @@ jobs:
env:
EVENT_NAME: ${{ github.event_name }}
REQUESTED_LANE: ${{ inputs.lane }}
+ REQUESTED_SUITE: ${{ inputs.suite }}
run: |
set -euo pipefail
if [[ "$EVENT_NAME" == "push" ]]; then
- matrix='{"include":[{"lane":"main-source"}]}'
- elif [[ "$EVENT_NAME" == "workflow_dispatch" && "$REQUESTED_LANE" != "both" ]]; then
- matrix="{\"include\":[{\"lane\":\"$REQUESTED_LANE\"}]}"
+ matrix='{"include":[{"lane":"main-source","suite":"ephemeral"}]}'
else
- matrix='{"include":[{"lane":"main-source"},{"lane":"published-package"}]}'
+ lanes=("main-source" "published-package")
+ if [[ "$EVENT_NAME" == "workflow_dispatch" && "$REQUESTED_LANE" != "both" ]]; then
+ lanes=("$REQUESTED_LANE")
+ fi
+ suites=("ephemeral" "pool")
+ if [[ "$EVENT_NAME" == "workflow_dispatch" && "${REQUESTED_SUITE:-both}" != "both" ]]; then
+ suites=("$REQUESTED_SUITE")
+ fi
+ include=""
+ for lane in "${lanes[@]}"; do
+ for suite in "${suites[@]}"; do
+ include+="{\"lane\":\"$lane\",\"suite\":\"$suite\"},"
+ done
+ done
+ matrix="{\"include\":[${include%,}]}"
fi
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
@@ -59,13 +78,16 @@ jobs:
fail-fast: false
matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }}
concurrency:
- group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}
+ group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }}
cancel-in-progress: ${{ github.event_name == 'schedule' }}
env:
CUA_FLEET_BASE_URL: https://run.cua.ai
CUA_LIVE_E2E_LANE: ${{ matrix.lane }}
+ CUA_LIVE_E2E_SUITE: ${{ matrix.suite }}
CUA_LIVE_E2E_EVENT: ${{ github.event_name }}
CUA_LIVE_E2E_NAMESPACE: cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}
+ CUA_LIVE_E2E_POOL_WARM_NAMESPACE: cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}
+ CUA_LIVE_E2E_POOL_COLD_NAMESPACE: cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}
CUA_LIVE_E2E_ARTIFACT_DIR: /tmp/cua-live-e2e
CUA_TELEMETRY_ENABLED: "false"
steps:
@@ -154,6 +176,7 @@ jobs:
json.dumps(
{
"lane": os.environ["CUA_LIVE_E2E_LANE"],
+ "suite": os.environ["CUA_LIVE_E2E_SUITE"],
"namespace": os.environ["CUA_LIVE_E2E_NAMESPACE"],
"source_sha": os.environ["CUA_LIVE_E2E_SOURCE_SHA"],
"error": {"type": "ControlledFailure"},
@@ -170,7 +193,7 @@ jobs:
run: exit 1
- name: Run live Fleet smoke
- if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.force_failure) }}
+ if: ${{ matrix.suite == 'ephemeral' && !(github.event_name == 'workflow_dispatch' && inputs.force_failure) }}
env:
CUA_CLIENT_ID: ${{ secrets.CUA_CLIENT_ID }}
CUA_CLIENT_SECRET: ${{ secrets.CUA_CLIENT_SECRET }}
@@ -178,11 +201,20 @@ jobs:
PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT" python -m pytest -q -s \
"$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_ephemeral.py"
+ - name: Run live Fleet pool smoke
+ if: ${{ matrix.suite == 'pool' && !(github.event_name == 'workflow_dispatch' && inputs.force_failure) }}
+ env:
+ CUA_CLIENT_ID: ${{ secrets.CUA_CLIENT_ID }}
+ CUA_CLIENT_SECRET: ${{ secrets.CUA_CLIENT_SECRET }}
+ run: |
+ PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT" python -m pytest -q -s \
+ "$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_pool_persistent.py"
+
- name: Upload failure diagnostics
if: failure()
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
with:
- name: cua-sandbox-live-${{ matrix.lane }}-${{ github.run_id }}-${{ github.run_attempt }}
+ name: cua-sandbox-live-${{ matrix.lane }}-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
path: /tmp/cua-live-e2e
if-no-files-found: warn
retention-days: 7
@@ -201,10 +233,11 @@ jobs:
"severity": "critical",
"service": "cua-sandbox",
"job": "periodic-cua-sandbox-live",
- "lane": "${{ matrix.lane }}"
+ "lane": "${{ matrix.lane }}",
+ "suite": "${{ matrix.suite }}"
},
"annotations": {
- "summary": "Cua Sandbox live Fleet E2E failed (${{ matrix.lane }})",
+ "summary": "Cua Sandbox live Fleet E2E failed (${{ matrix.lane }}/${{ matrix.suite }})",
"description": "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
"source_sha": "${{ steps.source_sha.outputs.source_sha }}",
"package_version": "${{ steps.versions.outputs.sandbox }}",
diff --git a/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.md b/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.md
index 2aeb54fb3d..84d286adda 100644
--- a/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.md
+++ b/docs/superpowers/plans/2026-08-09-periodic-cua-sandbox-live-e2e.md
@@ -906,3 +906,36 @@ The workflow uses step-scoped OAuth credentials only for credential preflight an
live pytest step. After checkout, `git rev-parse HEAD` is exported as
`CUA_LIVE_E2E_SOURCE_SHA` and is the source SHA recorded by live, controlled-failure,
and Alertmanager evidence.
+
+## Persistent Pool Suite
+
+The workflow later gained a second suite that claims from persistent,
+pre-provisioned Fleet pools instead of creating and destroying a pool per run.
+The prepare job emits a lane-and-suite matrix: pushes stay on the `ephemeral`
+suite only, while scheduled runs cross both lanes with both suites and manual
+dispatch selects lane and suite combinations. The concurrency group is
+`periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }}`,
+so a newer schedule cancels only an older scheduled run of the same lane and
+suite.
+
+The suite's step, `Run live Fleet pool smoke`, executes
+`tests/live/test_fleet_pool_persistent.py` from the same isolated copied suite
+with step-scoped OAuth credentials. Two pool namespaces per lane and event
+class are set by the workflow:
+
+- `cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}`
+- `cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}`
+
+The warm pool keeps `replicas=1` so claims bind to pre-provisioned capacity;
+the cold pool expresses scale-to-zero with
+`WarmPoolAutoscaling(min_pool_size=0, initial_pool_size=0, max_pool_size=1)`
+because pool reconciliation rejects `replicas` below one. Each run records
+`pool_pre_existed` and replica counts from `Pool.get`, reconciles the pinned
+configuration idempotently with `Pool.apply`, claims through
+`Sandbox.ephemeral(pool=..., name=...)` with the claim name fixed to the
+namespace, and exits with a claim-only release. After release the monitor
+polls until claims are absent and requires the reconciled inventory to contain
+exactly the named pool and template with zero claims; the pool and template
+deliberately persist between runs. The warm mode asserts a claim-acquisition
+bound only when the pool pre-existed with a ready replica. Failure artifacts
+and Alertmanager labels carry the suite alongside the lane.
diff --git a/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.md b/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.md
index 755d8eca63..35d6240620 100644
--- a/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.md
+++ b/docs/superpowers/specs/2026-08-09-periodic-cua-sandbox-live-e2e-design.md
@@ -18,6 +18,8 @@ testing the current Fleet-backed public SDK contract.
- Exercise Fleet-backed `Sandbox.ephemeral()` against live production
infrastructure every 15 minutes.
+- Exercise claiming from persistent, pre-provisioned Fleet pools — both a warm
+ pool and a scale-to-zero pool — on the same cadence.
- Test both the current repository `main` source and the latest published
`cua-sandbox` package.
- Run the source lane immediately after relevant changes merge to `main`.
@@ -49,31 +51,34 @@ testing the current Fleet-backed public SDK contract.
`libs/python/cua-fleet/**`,
`.github/workflows/periodic-cua-sandbox-live.yml`, and
`.github/scripts/tests/test_periodic_cua_sandbox_live.py`.
-3. `workflow_dispatch` accepts `both`, `main-source`, or `published-package`,
- plus manual-only `force_failure`.
+3. `workflow_dispatch` accepts a lane (`both`, `main-source`, or
+ `published-package`), a suite (`both`, `ephemeral`, or `pool`), plus
+ manual-only `force_failure`.
Both jobs are guarded with `if: github.repository == 'trycua/cua'`. A fork
that syncs `main` or enables the schedule therefore never runs the live smoke,
never fails the credential preflight, and never posts a fork-originated
`PeriodicCuaSandboxLiveE2EFailed` alert to the public Alertmanager endpoint.
-The preparation script emits a JSON matrix: every `push` selects only
-`main-source`; every `schedule` selects both lanes; manual runs select their
-requested lane or both. The workflow contract executes this extracted shell
-script in `bash` with a temporary `GITHUB_OUTPUT` and parses the emitted JSON,
-so a push-to-both mutation or ignored manual selection fails CI.
+The preparation script emits a JSON lane-and-suite matrix: every `push`
+selects only `main-source` with the `ephemeral` suite; every `schedule`
+selects both lanes crossed with both suites; manual runs select their
+requested lane and suite combinations. The workflow contract executes this
+extracted shell script in `bash` with a temporary `GITHUB_OUTPUT` and parses
+the emitted JSON, so a push-to-both mutation or ignored manual selection fails
+CI.
-The two lanes use `fail-fast: false` and this concurrency contract:
+The matrix jobs use `fail-fast: false` and this concurrency contract:
```yaml
concurrency:
- group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}
+ group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }}
cancel-in-progress: ${{ github.event_name == 'schedule' }}
```
-The event-and-lane grouping means a new schedule can cancel only an older
-schedule for the same lane. Push and manual runs use distinct groups and are
-allowed to finish.
+The event-lane-and-suite grouping means a new schedule can cancel only an
+older schedule for the same lane and suite. Push and manual runs use distinct
+groups and are allowed to finish.
## Installation Isolation
@@ -173,6 +178,47 @@ and workflow never explicitly delete a namespace, pool, or template: name-only
deletes can race with reconciliation. Cleanup errors are reported alongside the
primary exception.
+## Persistent Pool Suite
+
+The `pool` suite exercises the pre-provisioned pool consumer path that the
+`ephemeral` suite cannot: pools that survive across runs and hand out claims.
+`Run live Fleet pool smoke` executes
+`libs/python/cua-sandbox/tests/live/test_fleet_pool_persistent.py` from the
+same isolated copied suite. The suite runs on `schedule` and
+`workflow_dispatch` only; pushes keep running only the `ephemeral` suite.
+
+Each lane and event class owns two persistent pool namespaces, set by the
+workflow as:
+
+- `cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}`
+- `cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }}`
+
+The warm mode keeps `replicas=1`, so a scheduled claim binds to an
+already-running sandbox and releasing the claim recycles that sandbox back
+into the pool. The cold mode expresses scale-to-zero with
+`WarmPoolAutoscaling(min_pool_size=0, initial_pool_size=0, max_pool_size=1)`
+because pool reconciliation rejects `replicas` below one; a claim then
+cold-starts capacity through autoscaler demand.
+
+Each run observes the pool first with `Pool.get`, recording
+`pool_pre_existed` and the replica counts, then reconciles the pinned
+configuration with `Pool.apply` using the same certified image digest,
+`cpu=4`, and `memory_mb=4096`. Reconciliation is idempotent: it bootstraps a
+missing pool, heals drift, and never deletes. The claim uses
+`Sandbox.ephemeral(pool=..., name=...)` with the claim name fixed to the
+namespace, so an interrupted run's claim is adopted and released by the next
+run. Exiting the context performs a claim-only release: the pool and template
+must persist.
+
+After release the monitor polls until claims are absent and requires the
+reconciled inventory to contain exactly the named pool and template with zero
+claims — the persistence mirror of the ephemeral suite's empty-inventory
+invariant. Replica counts after release are recorded as telemetry only,
+because warm-pool recycling and autoscaler decay are server-controlled. The
+warm mode additionally asserts a claim-acquisition bound only when the pool
+pre-existed with at least one ready replica; bootstrap runs record timing
+without enforcing it.
+
## Diagnostics And Artifacts
The live test writes a sanitized JSON summary containing lane, source SHA,
@@ -199,21 +245,25 @@ Each lane has an `if: failure()` notification step posting to
- `service=cua-sandbox`
- `job=periodic-cua-sandbox-live`
- `lane=main-source|published-package`
+- `suite=ephemeral|pool`
Annotations include the failed GitHub Actions run, the source SHA or installed
package version, the pinned image digest, and a link to the workflow dashboard.
Do not include credentials or raw authorization failures that may contain
headers.
-The lane label lets Alertmanager group repeated failures without combining a
-source regression with a published-package or shared-infrastructure failure.
+The lane and suite labels let Alertmanager group repeated failures without
+combining a source regression with a published-package failure, or an
+ephemeral provisioning failure with a persistent pool claim failure. Failure
+artifacts are named per lane and suite so concurrent matrix jobs never collide
+on upload.
## Workflow Contract Coverage
The repository-side contract parses the workflow with `yaml.BaseLoader` and
asserts triggers, path filters, the upstream-repository fork guard on both
jobs, the executed preparation matrix, checkout ref,
-event-and-lane concurrency, credential preflight, copied-suite isolation,
+event-lane-and-suite concurrency, credential preflight, copied-suite isolation,
version output handling, controlled-failure diagnostics, failure-only artifacts,
Alertmanager labels, full-SHA action pins, and absence of explicit deletion.
Scripts CI installs `pyyaml` and runs this contract when the workflow changes.
@@ -226,10 +276,14 @@ Scripts CI installs `pyyaml` and runs this contract when the workflow changes.
complete claim-only cleanup.
3. Manually dispatch `published-package` and verify the installed release plus
cleanup behavior.
-4. Exercise the Alertmanager payload without exposing secrets, then resolve the
+4. Manually dispatch the `pool` suite twice per lane: the first run
+ bootstraps both persistent pools (`pool_pre_existed` false), the second
+ proves a warm claim against pre-provisioned capacity and records cold
+ scale-to-zero telemetry.
+5. Exercise the Alertmanager payload without exposing secrets, then resolve the
test alert.
-5. Enable the `7/15 * * * *` schedule.
-6. Observe at least two consecutive scheduled runs for both lanes before
+6. Enable the `7/15 * * * *` schedule.
+7. Observe at least two consecutive scheduled runs for both lanes before
considering rollout complete.
## Success Criteria
@@ -240,8 +294,10 @@ Scripts CI installs `pyyaml` and runs this contract when the workflow changes.
an older scheduled run of the same lane, while push and manual runs finish.
- Port, screen, screenshot, and shell assertions all exercise the public SDK.
- Normal runs retain only the named persistent pool/template with no claims; failed provisioning records read-only claim and inventory diagnostics without deleting resources.
-- A forced failure produces one actionable, lane-specific Alertmanager alert
- and sanitized failure artifacts.
+- Pool-suite runs claim from and release back to persistent pools that survive
+ the run, with the warm pool binding against pre-provisioned capacity.
+- A forced failure produces one actionable, lane- and suite-specific
+ Alertmanager alert and sanitized failure artifacts.
## Live Evidence Remediation
@@ -252,10 +308,15 @@ Each lane has one DNS-safe namespace for each event class:
- `cua-live-<lane>-push` for pushes
- `cua-live-<lane>-manual` for `workflow_dispatch`
-The event-and-lane concurrency group serializes use of each deterministic claim;
-only scheduled runs cancel an older scheduled run in the same lane. Fleet
-reconciliation preserves the namespace, pool, and template, all named after the
-namespace. `Sandbox.ephemeral()` is verified with claim-only cleanup: after
-exit the monitor polls until claims are absent, records persistent reconciled
-resources, and requires exactly the named pool/template with zero claims. It
-never explicitly deletes a namespace, pool, or template.
+The persistent pool suite adds `cua-live-pool-warm-<lane>-<event-class>` and
+`cua-live-pool-cold-<lane>-<event-class>` namespaces whose pools and templates
+deliberately outlive every run.
+
+The event-lane-and-suite concurrency group serializes use of each
+deterministic claim; only scheduled runs cancel an older scheduled run in the
+same lane and suite. Fleet reconciliation preserves the namespace, pool, and
+template, all named after the namespace. `Sandbox.ephemeral()` is verified
+with claim-only cleanup: after exit the monitor polls until claims are absent,
+records persistent reconciled resources, and requires exactly the named
+pool/template with zero claims. It never explicitly deletes a namespace, pool,
+or template.
diff --git a/libs/python/cua-sandbox/tests/live/fleet_e2e_support.py b/libs/python/cua-sandbox/tests/live/fleet_e2e_support.py
index d756bac37e..b60320fa91 100644
--- a/libs/python/cua-sandbox/tests/live/fleet_e2e_support.py
+++ b/libs/python/cua-sandbox/tests/live/fleet_e2e_support.py
@@ -53,17 +53,31 @@ class HttpxFleetClient(HttpClient):
await self._client.aclose()
-def build_namespace_name(lane: str, event_name: str) -> str:
- event_class = {
+def has_oauth_credentials() -> bool:
+ return bool(os.environ.get("CUA_CLIENT_ID") and os.environ.get("CUA_CLIENT_SECRET"))
+
+
+def _event_class(event_name: str) -> str:
+ return {
"schedule": "schedule",
"push": "push",
"workflow_dispatch": "manual",
}.get(event_name, "manual")
- raw = f"cua-live-{lane}-{event_class}".lower()
- normalized = re.sub(r"[^a-z0-9-]+", "-", raw).strip("-")
+
+
+def _normalize_namespace_name(raw: str) -> str:
+ normalized = re.sub(r"[^a-z0-9-]+", "-", raw.lower()).strip("-")
return normalized[:63].rstrip("-")
+def build_namespace_name(lane: str, event_name: str) -> str:
+ return _normalize_namespace_name(f"cua-live-{lane}-{_event_class(event_name)}")
+
+
+def build_pool_namespace_name(mode: str, lane: str, event_name: str) -> str:
+ return _normalize_namespace_name(f"cua-live-pool-{mode}-{lane}-{_event_class(event_name)}")
+
+
def build_fleet_client() -> tuple[CyclopsClient, HttpxFleetClient]:
client_id = os.environ["CUA_CLIENT_ID"]
client_secret = os.environ["CUA_CLIENT_SECRET"]
diff --git a/libs/python/cua-sandbox/tests/live/test_fleet_pool_persistent.py b/libs/python/cua-sandbox/tests/live/test_fleet_pool_persistent.py
new file mode 100644
index 0000000000..8c4ed1966f
--- /dev/null
+++ b/libs/python/cua-sandbox/tests/live/test_fleet_pool_persistent.py
@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import os
+import re
+import time
+from importlib.metadata import version
+from pathlib import Path
+
+import cua_sandbox
+import pytest
+from cua_sandbox import Image, Pool, Sandbox, WarmPoolAutoscaling
+
+from tests.live.fleet_e2e_support import (
+ assert_template_contract,
+ build_fleet_client,
+ build_pool_namespace_name,
+ collect_resource_inventory,
+ has_oauth_credentials,
+ is_not_found_error,
+ wait_claims_absent,
+ write_summary,
+)
+
+IMAGE = (
+ "public.ecr.aws/k5j5w0x5/cua-ubuntu-24.04"
+ "@sha256:82702ebdd32d1f8fc05f2ea409a7c67d0ba9f8f8e4e9f1a89ce40989d5f4475d"
+)
+
+POOL_CPU = 4
+POOL_MEMORY_MB = 4096
+WARM_TIME_TO_START = 180
+COLD_TIME_TO_START = 900
+WARM_BIND_SLA_SECONDS = 300
+
+MODE_ENV = {
+ "warm": "CUA_LIVE_E2E_POOL_WARM_NAMESPACE",
+ "cold": "CUA_LIVE_E2E_POOL_COLD_NAMESPACE",
+}
+
+
+def cold_autoscaling() -> WarmPoolAutoscaling:
+ # replicas < 1 is rejected by Pool.apply, so scale-to-zero is expressed
+ # through autoscaling: KEDA owns spec.replicas and decays it to zero.
+ return WarmPoolAutoscaling(min_pool_size=0, initial_pool_size=0, max_pool_size=1)
+
+
+def selected_pool_namespace(mode: str) -> str:
+ lane = os.environ.get("CUA_LIVE_E2E_LANE", "local")
+ namespace = os.environ.get(MODE_ENV[mode]) or build_pool_namespace_name(
+ mode,
+ lane,
+ os.environ.get("CUA_LIVE_E2E_EVENT", os.environ.get("GITHUB_EVENT_NAME", "manual")),
+ )
+ if not namespace.startswith(f"cua-live-pool-{mode}-"):
+ raise ValueError(f"{MODE_ENV[mode]} must start with cua-live-pool-{mode}-")
+ if len(namespace) > 63 or re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", namespace) is None:
+ raise ValueError(f"{MODE_ENV[mode]} must be a DNS-1123 label of at most 63 characters")
+ return namespace
+
+
+pytestmark = [
+ pytest.mark.asyncio,
+ pytest.mark.skipif(not has_oauth_credentials(), reason="Fleet OAuth credentials not set"),
+]
+
+
+async def run_fleet_pool_live(mode: str) -> None:
+ lane = os.environ.get("CUA_LIVE_E2E_LANE", "local")
+ namespace = selected_pool_namespace(mode)
+ artifact_dir = Path(os.environ.get("CUA_LIVE_E2E_ARTIFACT_DIR", "/tmp/cua-live-e2e"))
+ summary = {
+ "lane": lane,
+ "mode": mode,
+ "namespace": namespace,
+ "image": IMAGE,
+ "source_sha": os.environ.get("CUA_LIVE_E2E_SOURCE_SHA") or os.environ.get("GITHUB_SHA"),
+ "packages": {
+ "cua-sandbox": version("cua-sandbox"),
+ "cua-fleet": version("cua-fleet"),
+ },
+ "module_origins": {
+ "cua_sandbox": str(Path(cua_sandbox.__file__).resolve()),
+ },
+ }
+ fleet, http_client = build_fleet_client()
+ primary_error: BaseException | None = None
+ cleanup_error: BaseException | None = None
+ close_error: BaseException | None = None
+ summary_error: BaseException | None = None
+ pool_applied = False
+ sandbox_yielded = False
+
+ def record_cleanup_error(error: BaseException) -> None:
+ nonlocal cleanup_error
+ error_summary = {"type": type(error).__name__}
+ if cleanup_error is None:
+ cleanup_error = error
+ summary["cleanup_error"] = error_summary
+ else:
+ summary.setdefault("cleanup_secondary_errors", []).append(error_summary)
+
+ try:
+ try:
+ existing = await Pool.get(namespace)
+ except BaseException as error:
+ if not is_not_found_error(error):
+ raise
+ summary["pool_pre_existed"] = False
+ else:
+ summary["pool_pre_existed"] = True
+ summary["spec_replicas_before"] = existing.resource.spec.replicas
+ status = getattr(existing.resource, "status", None)
+ summary["ready_replicas_before"] = (
+ (getattr(status, "ready_replicas", None) or 0) if status is not None else 0
+ )
+
+ apply_started = time.monotonic()
+ pool = await Pool.apply(
+ Image.from_registry(IMAGE),
+ name=namespace,
+ replicas=1,
+ cpu=POOL_CPU,
+ memory_mb=POOL_MEMORY_MB,
+ autoscaling=None if mode == "warm" else cold_autoscaling(),
+ )
+ pool_applied = True
+ summary["apply_seconds"] = time.monotonic() - apply_started
+ assert pool.name == namespace, f"pool name {pool.name!r} must equal namespace {namespace!r}"
+
+ template = await fleet.get_template(namespace, namespace)
+ assert_template_contract(template, expected_port=8000)
+
+ claim_started = time.monotonic()
+ async with Sandbox.ephemeral(
+ pool=namespace,
+ name=namespace,
+ time_to_start=WARM_TIME_TO_START if mode == "warm" else COLD_TIME_TO_START,
+ telemetry_enabled=False,
+ ) as sandbox:
+ sandbox_yielded = True
+ claim_seconds = time.monotonic() - claim_started
+ summary["claim_seconds"] = claim_seconds
+ summary["sandbox_name"] = sandbox.name
+ sandbox_claim_name = getattr(sandbox, "claim_name", None)
+ sandbox_pool_name = getattr(sandbox, "pool_name", None)
+ summary["claim_name"] = sandbox_claim_name or sandbox.name
+ summary["pool_name"] = sandbox_pool_name or namespace
+ try:
+ assert (
+ isinstance(sandbox.name, str) and sandbox.name
+ ), "sandbox name must be a non-empty string"
+ if sandbox_claim_name is not None:
+ assert sandbox_claim_name == namespace, (
+ f"claim name {sandbox_claim_name!r} must equal "
+ f"requested name {namespace!r}"
+ )
+ if sandbox_pool_name is not None:
+ assert (
+ sandbox_pool_name == namespace
+ ), f"pool name {sandbox_pool_name!r} must equal namespace {namespace!r}"
+
+ warm_bind_sla_applied = (
+ mode == "warm"
+ and summary.get("pool_pre_existed") is True
+ and summary.get("ready_replicas_before", 0) >= 1
+ )
+ summary["warm_bind_sla_applied"] = warm_bind_sla_applied
+ if warm_bind_sla_applied:
+ assert claim_seconds < WARM_BIND_SLA_SECONDS, (
+ f"pre-provisioned warm claim took {claim_seconds:.1f}s, "
+ f"expected under {WARM_BIND_SLA_SECONDS}s"
+ )
+
+ width, height = await sandbox.screen.size()
+ summary["screen"] = {"width": width, "height": height}
+ assert (width, height) == (1024, 768)
+
+ screenshot = await sandbox.screenshot()
+ artifact_dir.mkdir(parents=True, exist_ok=True)
+ (artifact_dir / f"screen-pool-{mode}.png").write_bytes(screenshot)
+ assert screenshot.startswith(b"\x89PNG\r\n\x1a\n")
+ assert len(screenshot) > 1000
+
+ result = await sandbox.shell.run("uname -s")
+ summary["shell"] = {
+ "success": result.success,
+ "stdout": result.stdout.strip(),
+ "stderr": result.stderr.strip(),
+ }
+ assert result.success
+ assert result.stdout.strip() == "Linux"
+ except BaseException as error:
+ primary_error = error
+ summary["error"] = {"type": type(error).__name__}
+ except BaseException as error:
+ if primary_error is None:
+ if sandbox_yielded:
+ record_cleanup_error(error)
+ else:
+ primary_error = error
+ summary["error"] = {"type": type(error).__name__}
+ else:
+ summary["context_exit_error"] = {"type": type(error).__name__}
+ finally:
+ if pool_applied:
+ cleanup_started = time.monotonic()
+ claims_absent: bool | None = None
+ inventory: dict[str, list[str]] | None = None
+ try:
+ claims_absent = await wait_claims_absent(fleet, namespace)
+ summary["claims_absent"] = claims_absent
+ except BaseException as error:
+ record_cleanup_error(error)
+ try:
+ inventory = await collect_resource_inventory(fleet, namespace)
+ summary["persistent_resources"] = inventory
+ except BaseException as error:
+ record_cleanup_error(error)
+
+ if claims_absent is False:
+ try:
+ summary["claim_leak"] = True
+ pytest.fail(f"claims remain in namespace {namespace} after claim-only release")
+ except BaseException as error:
+ record_cleanup_error(error)
+ if inventory is not None:
+ expected_inventory = {
+ "templates": [namespace],
+ "pools": [namespace],
+ "claims": [],
+ }
+ if inventory != expected_inventory:
+ try:
+ summary["unexpected_inventory"] = True
+ pytest.fail(
+ f"persistent pool inventory for namespace {namespace} must be "
+ f"{expected_inventory}, got {inventory}"
+ )
+ except BaseException as error:
+ record_cleanup_error(error)
+ try:
+ refreshed = await Pool.get(namespace)
+ summary["spec_replicas_after"] = refreshed.resource.spec.replicas
+ status = getattr(refreshed.resource, "status", None)
+ summary["ready_replicas_after"] = (
+ (getattr(status, "ready_replicas", None) or 0) if status is not None else 0
+ )
+ except BaseException as error:
+ record_cleanup_error(error)
+ summary["cleanup_seconds"] = time.monotonic() - cleanup_started
+ if not sandbox_yielded:
+ summary["provisioning"] = {
+ "pool_applied": pool_applied,
+ "sandbox_yielded": False,
+ }
+
+ try:
+ await http_client.aclose()
+ except BaseException as error:
+ close_error = error
+ summary["close_error"] = {"type": type(error).__name__}
+
+ try:
+ write_summary(artifact_dir / f"summary-pool-{mode}.json", summary)
+ except BaseException as error:
+ summary_error = error
+ summary["summary_error"] = {"type": type(error).__name__}
+
+ if primary_error is not None:
+ raise primary_error
+ if cleanup_error is not None:
+ raise cleanup_error
+ if close_error is not None:
+ raise close_error
+ if summary_error is not None:
+ raise summary_error
+
+
+async def test_fleet_pool_warm_live() -> None:
+ await run_fleet_pool_live("warm")
+
+
+async def test_fleet_pool_cold_live() -> None:
+ await run_fleet_pool_live("cold")
diff --git a/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py b/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py
index 3055122634..6cc5f26f40 100644
--- a/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py
+++ b/libs/python/cua-sandbox/tests/test_live_fleet_e2e_support.py
@@ -11,7 +11,9 @@ from fleet_sdk import SdkError
from tests.live.fleet_e2e_support import (
assert_template_contract,
build_namespace_name,
+ build_pool_namespace_name,
collect_resource_inventory,
+ has_oauth_credentials,
wait_claims_absent,
write_summary,
)
@@ -49,6 +51,39 @@ def test_build_namespace_name_normalizes_invalid_overlong_lane_input() -> None:
assert name.startswith("cua-live-published-package-")
[email protected]("mode", ["warm", "cold"])
+ ("event_name", "event_class"),
+ [("schedule", "schedule"), ("push", "push"), ("workflow_dispatch", "manual")],
+)
+def test_build_pool_namespace_name_is_stable_for_each_mode_lane_and_event_class(
+ mode: str, event_name: str, event_class: str
+) -> None:
+ name = build_pool_namespace_name(mode, "published-package", event_name)
+ assert name == f"cua-live-pool-{mode}-published-package-{event_class}"
+ assert len(name) <= 63
+
+
+def test_build_pool_namespace_name_normalizes_invalid_overlong_input() -> None:
+ name = build_pool_namespace_name(
+ "warm", "Published_Package!!!" + "X" * 100, "workflow_dispatch"
+ )
+
+ assert len(name) <= 63
+ assert re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", name)
+ assert name.startswith("cua-live-pool-warm-published-package-")
+
+
+def test_support_has_oauth_credentials_requires_both_values(monkeypatch) -> None:
+ monkeypatch.delenv("CUA_CLIENT_ID", raising=False)
+ monkeypatch.delenv("CUA_CLIENT_SECRET", raising=False)
+ assert not has_oauth_credentials()
+ monkeypatch.setenv("CUA_CLIENT_ID", "client")
+ assert not has_oauth_credentials()
+ monkeypatch.setenv("CUA_CLIENT_SECRET", "secret")
+ assert has_oauth_credentials()
+
+
def test_assert_template_contract_accepts_server_port_8000() -> None:
probes = SimpleNamespace(
to_json=lambda: json.dumps({"readinessProbe": {"tcpSocket": {"port": 8000}}})
diff --git a/libs/python/cua-sandbox/tests/test_live_fleet_pool_support.py b/libs/python/cua-sandbox/tests/test_live_fleet_pool_support.py
new file mode 100644
index 0000000000..654471b5e1
--- /dev/null
+++ b/libs/python/cua-sandbox/tests/test_live_fleet_pool_support.py
@@ -0,0 +1,348 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from fleet_sdk import SdkError
+
+from tests.live import test_fleet_pool_persistent as pool_live
+
+PNG = b"\x89PNG\r\n\x1a\n" + b"0" * 2000
+
+
+def make_pool_resource(namespace: str, *, spec_replicas: int = 1, ready_replicas: int = 1):
+ return SimpleNamespace(
+ metadata=SimpleNamespace(namespace=namespace, name=namespace),
+ spec=SimpleNamespace(replicas=spec_replicas),
+ status=SimpleNamespace(ready_replicas=ready_replicas),
+ )
+
+
+def not_found(operation: str = "get pool") -> SdkError.Status:
+ return SdkError.Status(operation, 404, b"not found")
+
+
+class FakePoolHandle:
+ def __init__(self, resource) -> None:
+ self._resource = resource
+
+ @property
+ def name(self) -> str:
+ return self._resource.metadata.name
+
+ @property
+ def resource(self):
+ return self._resource
+
+
+class FakeSandbox:
+ def __init__(self, name: str, shell_stdout: str) -> None:
+ self.name = name
+ self.claim_name = name
+ self.pool_name = name
+ self.screen = SimpleNamespace(size=self._size)
+ self.shell = SimpleNamespace(run=self._run)
+ self._shell_stdout = shell_stdout
+
+ async def _size(self) -> tuple[int, int]:
+ return (1024, 768)
+
+ async def _run(self, command: str):
+ return SimpleNamespace(success=True, stdout=self._shell_stdout, stderr="")
+
+ async def screenshot(self) -> bytes:
+ return PNG
+
+
+class _FakeEphemeral:
+ def __init__(self, sandbox: FakeSandbox) -> None:
+ self._sandbox = sandbox
+
+ async def __aenter__(self) -> FakeSandbox:
+ return self._sandbox
+
+ async def __aexit__(self, exc_type, exc, traceback) -> bool:
+ return False
+
+
+class PoolRunnerHarness:
+ def __init__(self) -> None:
+ self.get_results: list = []
+ self.apply_calls: list[dict] = []
+ self.apply_error: BaseException | None = None
+ self.ephemeral_calls: list[dict] = []
+ self.shell_stdout = "Linux"
+ self.claims_absent: object = True
+ self.claims_absent_calls: list[str] = []
+ self.inventory: object = None
+ self.inventory_calls: list[str] = []
+ self.template_contract_calls: list[tuple] = []
+ self.summaries: dict[str, dict] = {}
+
+
+def install_pool_runner(monkeypatch, tmp_path, *, mode: str, namespace: str) -> PoolRunnerHarness:
+ harness = PoolRunnerHarness()
+ monkeypatch.setenv("CUA_LIVE_E2E_LANE", "test")
+ monkeypatch.setenv("CUA_LIVE_E2E_EVENT", "schedule")
+ monkeypatch.setenv("CUA_LIVE_E2E_ARTIFACT_DIR", str(tmp_path))
+ monkeypatch.setenv(pool_live.MODE_ENV[mode], namespace)
+
+ class FakePool:
+ @staticmethod
+ async def get(name: str) -> FakePoolHandle:
+ assert harness.get_results, "unexpected Pool.get call"
+ result = harness.get_results.pop(0)
+ if isinstance(result, BaseException):
+ raise result
+ return FakePoolHandle(result)
+
+ @staticmethod
+ async def apply(image, **kwargs) -> FakePoolHandle:
+ harness.apply_calls.append({"image": image, **kwargs})
+ if harness.apply_error is not None:
+ raise harness.apply_error
+ return FakePoolHandle(make_pool_resource(kwargs["name"]))
+
+ class FakeSandboxApi:
+ @staticmethod
+ def ephemeral(**kwargs) -> _FakeEphemeral:
+ harness.ephemeral_calls.append(kwargs)
+ return _FakeEphemeral(FakeSandbox(kwargs["name"], harness.shell_stdout))
+
+ class FakeFleet:
+ async def get_template(self, template_namespace: str, name: str):
+ return SimpleNamespace(namespace=template_namespace, name=name)
+
+ class FakeHttpClient:
+ async def aclose(self) -> None:
+ return None
+
+ async def fake_wait_claims_absent(fleet, name: str) -> bool:
+ harness.claims_absent_calls.append(name)
+ if isinstance(harness.claims_absent, BaseException):
+ raise harness.claims_absent
+ return bool(harness.claims_absent)
+
+ async def fake_collect_resource_inventory(fleet, name: str):
+ harness.inventory_calls.append(name)
+ if isinstance(harness.inventory, BaseException):
+ raise harness.inventory
+ if harness.inventory is None:
+ return {"templates": [name], "pools": [name], "claims": []}
+ return harness.inventory
+
+ def fake_assert_template_contract(template, expected_port: int) -> None:
+ harness.template_contract_calls.append((template, expected_port))
+
+ def fake_write_summary(path, summary) -> None:
+ harness.summaries[path.name] = summary
+
+ monkeypatch.setattr(pool_live, "Pool", FakePool)
+ monkeypatch.setattr(pool_live, "Sandbox", FakeSandboxApi)
+ monkeypatch.setattr(pool_live, "build_fleet_client", lambda: (FakeFleet(), FakeHttpClient()))
+ monkeypatch.setattr(pool_live, "wait_claims_absent", fake_wait_claims_absent)
+ monkeypatch.setattr(pool_live, "collect_resource_inventory", fake_collect_resource_inventory)
+ monkeypatch.setattr(pool_live, "assert_template_contract", fake_assert_template_contract)
+ monkeypatch.setattr(pool_live, "write_summary", fake_write_summary)
+ return harness
+
+
+async def test_pool_live_runner_uses_pinned_warm_configuration(monkeypatch, tmp_path) -> None:
+ namespace = "cua-live-pool-warm-test-schedule"
+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)
+ harness.get_results = [
+ make_pool_resource(namespace, ready_replicas=1),
+ make_pool_resource(namespace, ready_replicas=1),
+ ]
+
+ await pool_live.run_fleet_pool_live("warm")
+
+ (apply_call,) = harness.apply_calls
+ assert apply_call["name"] == namespace
+ assert apply_call["replicas"] == 1
+ assert apply_call["cpu"] == 4
+ assert apply_call["memory_mb"] == 4096
+ assert apply_call["autoscaling"] is None
+
+ (ephemeral_call,) = harness.ephemeral_calls
+ assert ephemeral_call["pool"] == namespace
+ assert isinstance(ephemeral_call["pool"], str)
+ assert ephemeral_call["name"] == namespace
+ assert ephemeral_call["time_to_start"] == 180
+ assert ephemeral_call["telemetry_enabled"] is False
+ for rejected in ("image", "cpu", "memory_mb", "server_port", "replicas"):
+ assert rejected not in ephemeral_call
+
+ ((_, contract_port),) = harness.template_contract_calls
+ assert contract_port == 8000
+ summary = harness.summaries["summary-pool-warm.json"]
+ assert summary["pool_pre_existed"] is True
+ assert summary["ready_replicas_before"] == 1
+ assert summary["warm_bind_sla_applied"] is True
+ assert summary["claims_absent"] is True
+ assert summary["persistent_resources"] == {
+ "templates": [namespace],
+ "pools": [namespace],
+ "claims": [],
+ }
+ assert summary["spec_replicas_after"] == 1
+
+
+async def test_pool_live_runner_uses_pinned_cold_configuration(monkeypatch, tmp_path) -> None:
+ namespace = "cua-live-pool-cold-test-schedule"
+ harness = install_pool_runner(monkeypatch, tmp_path, mode="cold", namespace=namespace)
+ harness.get_results = [
+ make_pool_resource(namespace, ready_replicas=0),
+ make_pool_resource(namespace, ready_replicas=0),
+ ]
+
+ await pool_live.run_fleet_pool_live("cold")
+
+ (apply_call,) = harness.apply_calls
+ assert apply_call["replicas"] == 1
+ autoscaling = apply_call["autoscaling"]
+ assert autoscaling is not None
+ assert autoscaling.min_pool_size == 0
+ assert autoscaling.initial_pool_size == 0
+ assert autoscaling.max_pool_size == 1
+
+ (ephemeral_call,) = harness.ephemeral_calls
+ assert ephemeral_call["time_to_start"] == 900
+
+ summary = harness.summaries["summary-pool-cold.json"]
+ assert summary["warm_bind_sla_applied"] is False
+
+
+async def test_pool_pre_existed_false_is_recorded_for_public_sdk_404(monkeypatch, tmp_path) -> None:
+ namespace = "cua-live-pool-warm-test-schedule"
+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)
+ harness.get_results = [not_found(), make_pool_resource(namespace)]
+
+ await pool_live.run_fleet_pool_live("warm")
+
+ summary = harness.summaries["summary-pool-warm.json"]
+ assert summary["pool_pre_existed"] is False
+ assert "ready_replicas_before" not in summary
+ assert summary["warm_bind_sla_applied"] is False
+
+
+async def test_empty_inventory_fails_as_unexpected(monkeypatch, tmp_path) -> None:
+ namespace = "cua-live-pool-warm-test-schedule"
+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)
+ harness.get_results = [make_pool_resource(namespace), make_pool_resource(namespace)]
+ harness.inventory = {"templates": [], "pools": [], "claims": []}
+
+ with pytest.raises(pytest.fail.Exception, match="persistent pool inventory"):
+ await pool_live.run_fleet_pool_live("warm")
+
+ summary = harness.summaries["summary-pool-warm.json"]
+ assert summary["unexpected_inventory"] is True
+
+
+async def test_claim_leak_fails_and_is_recorded(monkeypatch, tmp_path) -> None:
+ namespace = "cua-live-pool-warm-test-schedule"
+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)
+ harness.get_results = [make_pool_resource(namespace), make_pool_resource(namespace)]
+ harness.claims_absent = False
+
+ with pytest.raises(pytest.fail.Exception, match="claims remain"):
+ await pool_live.run_fleet_pool_live("warm")
+
+ summary = harness.summaries["summary-pool-warm.json"]
+ assert summary["claim_leak"] is True
+
+
+async def test_warm_sla_not_applied_without_ready_replicas(monkeypatch, tmp_path) -> None:
+ namespace = "cua-live-pool-warm-test-schedule"
+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)
+ harness.get_results = [
+ make_pool_resource(namespace, ready_replicas=0),
+ make_pool_resource(namespace, ready_replicas=1),
+ ]
+ monkeypatch.setattr(pool_live, "WARM_BIND_SLA_SECONDS", -1.0)
+
+ await pool_live.run_fleet_pool_live("warm")
+
+ summary = harness.summaries["summary-pool-warm.json"]
+ assert summary["warm_bind_sla_applied"] is False
+ assert "error" not in summary
+
+
+async def test_warm_sla_enforced_with_ready_replicas(monkeypatch, tmp_path) -> None:
+ namespace = "cua-live-pool-warm-test-schedule"
+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)
+ harness.get_results = [
+ make_pool_resource(namespace, ready_replicas=1),
+ make_pool_resource(namespace, ready_replicas=1),
+ ]
+ monkeypatch.setattr(pool_live, "WARM_BIND_SLA_SECONDS", -1.0)
+
+ with pytest.raises(AssertionError, match="pre-provisioned warm claim"):
+ await pool_live.run_fleet_pool_live("warm")
+
+ summary = harness.summaries["summary-pool-warm.json"]
+ assert summary["warm_bind_sla_applied"] is True
+ assert summary["error"] == {"type": "AssertionError"}
+
+
+async def test_cleanup_failure_does_not_mask_primary_failure(monkeypatch, tmp_path) -> None:
+ namespace = "cua-live-pool-warm-test-schedule"
+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)
+ harness.get_results = [make_pool_resource(namespace), make_pool_resource(namespace)]
+ harness.shell_stdout = "Darwin"
+ harness.inventory = RuntimeError("inventory unavailable")
+
+ with pytest.raises(AssertionError):
+ await pool_live.run_fleet_pool_live("warm")
+
+ summary = harness.summaries["summary-pool-warm.json"]
+ assert summary["error"] == {"type": "AssertionError"}
+ assert summary["cleanup_error"] == {"type": "RuntimeError"}
+
+
+async def test_apply_failure_skips_persistence_verification(monkeypatch, tmp_path) -> None:
+ namespace = "cua-live-pool-warm-test-schedule"
+ harness = install_pool_runner(monkeypatch, tmp_path, mode="warm", namespace=namespace)
+ harness.get_results = [not_found()]
+ harness.apply_error = RuntimeError("apply rejected")
+
+ with pytest.raises(RuntimeError, match="apply rejected"):
+ await pool_live.run_fleet_pool_live("warm")
+
+ assert harness.claims_absent_calls == []
+ assert harness.inventory_calls == []
+ assert harness.ephemeral_calls == []
+ summary = harness.summaries["summary-pool-warm.json"]
+ assert summary["provisioning"] == {"pool_applied": False, "sandbox_yielded": False}
+ assert summary["error"] == {"type": "RuntimeError"}
+
+
+def test_selected_pool_namespace_rejects_foreign_prefix(monkeypatch) -> None:
+ monkeypatch.setenv("CUA_LIVE_E2E_POOL_WARM_NAMESPACE", "cua-live-warm-other")
+
+ with pytest.raises(ValueError, match="must start with cua-live-pool-warm-"):
+ pool_live.selected_pool_namespace("warm")
+
+
+def test_selected_pool_namespace_requires_dns_1123_label(monkeypatch) -> None:
+ monkeypatch.setenv("CUA_LIVE_E2E_POOL_COLD_NAMESPACE", "cua-live-pool-cold-" + "x" * 60)
+
+ with pytest.raises(ValueError, match="DNS-1123"):
+ pool_live.selected_pool_namespace("cold")
+
+
+def test_selected_pool_namespace_builds_default_from_lane_and_event(monkeypatch) -> None:
+ monkeypatch.delenv("CUA_LIVE_E2E_POOL_WARM_NAMESPACE", raising=False)
+ monkeypatch.setenv("CUA_LIVE_E2E_LANE", "main-source")
+ monkeypatch.setenv("CUA_LIVE_E2E_EVENT", "schedule")
+
+ assert pool_live.selected_pool_namespace("warm") == "cua-live-pool-warm-main-source-schedule"