Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(spaces): warm pools, a prepared Linux Space and agent harnesses ready in about one second (behind the warm-pools experiment) (#6472)
ddupontd592250ea79f
diff --git a/apps/cua-spaces-desktop/src/bridge/settings.ts b/apps/cua-spaces-desktop/src/bridge/settings.tsindex bbcdabbff5..848725f6c1 100644--- a/apps/cua-spaces-desktop/src/bridge/settings.ts+++ b/apps/cua-spaces-desktop/src/bridge/settings.ts@@ -96,12 +96,12 @@ export function experimentsPage(model: AppModel) {return { ...page, sections: page.sections.map((s) => ({ ...s, rows: s.rows.filter((r) => !WEB_UI_ROWS.has(r.id)) })) };}-/** The experiments the daemon enforces (Secure Spaces, Child Spaces, Ask first, CI runners, Public TCP shares, Phone app) as the cua home keeps them: what `cua experiments enable` set shows the next time Settings is read. */+/** The experiments the daemon enforces (Secure Spaces, Child Spaces, Ask first, CI runners, Public TCP shares, Phone app, Warm start) as the cua home keeps them: what `cua experiments enable` set shows the next time Settings is read. */export function refreshDaemonExperiments(model: AppModel): void {const kept = model.native.appSettingsLoad(model.settingsPath).experiments;const now = model.settings.experiments;- if (kept.secureSpaces === now.secureSpaces && kept.childSpaces === now.childSpaces && kept.askFirst === now.askFirst && kept.ciRunners === now.ciRunners && kept.tcpShares === now.tcpShares && kept.mobile === now.mobile) return;- model.settings = { ...model.settings, experiments: { ...now, secureSpaces: kept.secureSpaces, childSpaces: kept.childSpaces, askFirst: kept.askFirst, ciRunners: kept.ciRunners, tcpShares: kept.tcpShares, mobile: kept.mobile } };+ if (kept.secureSpaces === now.secureSpaces && kept.childSpaces === now.childSpaces && kept.askFirst === now.askFirst && kept.ciRunners === now.ciRunners && kept.tcpShares === now.tcpShares && kept.mobile === now.mobile && kept.warmPools === now.warmPools) return;+ model.settings = { ...model.settings, experiments: { ...now, secureSpaces: kept.secureSpaces, childSpaces: kept.childSpaces, askFirst: kept.askFirst, ciRunners: kept.ciRunners, tcpShares: kept.tcpShares, mobile: kept.mobile, warmPools: kept.warmPools } };}/** `settings.get`'s answer: General, Experiments, the update channel (null:diff --git a/apps/cua-spaces-desktop/src/native/generated/cua_spaces_ffi.ts b/apps/cua-spaces-desktop/src/native/generated/cua_spaces_ffi.tsindex 84b5cb6a9f..e16ce8d86d 100644--- a/apps/cua-spaces-desktop/src/native/generated/cua_spaces_ffi.ts+++ b/apps/cua-spaces-desktop/src/native/generated/cua_spaces_ffi.ts@@ -14603,7 +14603,13 @@ export type AppExperiments = {* push events). Off unless turned on; the daemon enforces it (the switch* is kept in the cua home).*/- mobile: boolean+ mobile: boolean,+ /**+ * Warm pools: one Linux Space kept ready, with its coding apps+ * installed, so the next Create from Home starts at once. Off unless+ * turned on; the daemon enforces it (the switch is kept in the cua home).+ */+ warmPools: boolean}/**@@ -14636,7 +14642,8 @@ const FfiConverterTypeAppExperiments = (() => {askFirst: FfiConverterBool.read(from),ciRunners: FfiConverterBool.read(from),tcpShares: FfiConverterBool.read(from),- mobile: FfiConverterBool.read(from)+ mobile: FfiConverterBool.read(from),+ warmPools: FfiConverterBool.read(from)};}write(value: TypeName, into: RustBuffer): void {@@ -14650,6 +14657,7 @@ const FfiConverterTypeAppExperiments = (() => {FfiConverterBool.write(value.ciRunners, into);FfiConverterBool.write(value.tcpShares, into);FfiConverterBool.write(value.mobile, into);+ FfiConverterBool.write(value.warmPools, into);}allocationSize(value: TypeName): number {return FfiConverterBool.allocationSize(value.cuaVolume) +@@ -14661,7 +14669,8 @@ const FfiConverterTypeAppExperiments = (() => {FfiConverterBool.allocationSize(value.askFirst) +FfiConverterBool.allocationSize(value.ciRunners) +FfiConverterBool.allocationSize(value.tcpShares) +- FfiConverterBool.allocationSize(value.mobile);+ FfiConverterBool.allocationSize(value.mobile) ++ FfiConverterBool.allocationSize(value.warmPools);}};diff --git a/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Models/AppModel.swift b/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Models/AppModel.swiftindex 630c678f50..57039a438c 100644--- a/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Models/AppModel.swift+++ b/apps/cua-spaces-macos/Sources/CuaSpacesMacKit/Models/AppModel.swift@@ -1973,7 +1973,7 @@ public final class AppModel {// MARK: - Experiments- /// The experiments the daemon enforces (Secure Spaces, Child Spaces, CI runners, Public TCP shares, Phone app) as+ /// The experiments the daemon enforces (Secure Spaces, Child Spaces, CI runners, Public TCP shares, Phone app, Warm start) as/// the cua home keeps them: what `cua experiments enable` set shows here/// the next time Settings is read.public func refreshDaemonExperiments() {@@ -1983,13 +1983,15 @@ public final class AppModel {|| kept.askFirst != settings.experiments.askFirst|| kept.ciRunners != settings.experiments.ciRunners|| kept.tcpShares != settings.experiments.tcpShares- || kept.mobile != settings.experiments.mobile else { return }+ || kept.mobile != settings.experiments.mobile+ || kept.warmPools != settings.experiments.warmPools else { return }settings.experiments.secureSpaces = kept.secureSpacessettings.experiments.childSpaces = kept.childSpacessettings.experiments.askFirst = kept.askFirstsettings.experiments.ciRunners = kept.ciRunnerssettings.experiments.tcpShares = kept.tcpSharessettings.experiments.mobile = kept.mobile+ settings.experiments.warmPools = kept.warmPools}/// Settings, Experiments: a switch and one line per experiment.diff --git a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/FirstRunTests.swift b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/FirstRunTests.swiftindex 430bebcd45..2458a94fd7 100644--- a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/FirstRunTests.swift+++ b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/FirstRunTests.swift@@ -25,7 +25,7 @@ struct FirstRunTests {onboarding: OnboardingModel(statePath: state.path),settingsPath: dir.appendingPathComponent("settings.json").path, telemetry: telemetry,loginItem: loginItem)- model.settings.experiments = AppExperiments(cuaVolume: false, yourCloud: false, sharing: true, webUi: true, secureSpaces: false, childSpaces: false, askFirst: false, ciRunners: false, tcpShares: false, mobile: false)+ model.settings.experiments = AppExperiments(cuaVolume: false, yourCloud: false, sharing: true, webUi: true, secureSpaces: false, childSpaces: false, askFirst: false, ciRunners: false, tcpShares: false, mobile: false, warmPools: false)let bridge = WebUIBridge(model: model, allowedOrigins: [])bridge.notifySpaceReady = { _ in }return (bridge, model, telemetry, state)diff --git a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/MainUIRouteTests.swift b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/MainUIRouteTests.swiftindex 2002f6d400..c4fef43bdc 100644--- a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/MainUIRouteTests.swift+++ b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/MainUIRouteTests.swift@@ -21,7 +21,7 @@ struct MainUIRouteTests {onboarding: OnboardingModel(statePath: dir.appendingPathComponent("onboarding.json").path),settingsPath: dir.appendingPathComponent("settings.json").path,telemetry: FixtureTelemetry())- m.settings.experiments = AppExperiments(cuaVolume: true, yourCloud: false, sharing: false, webUi: true, secureSpaces: false, childSpaces: false, askFirst: false, ciRunners: false, tcpShares: false, mobile: false)+ m.settings.experiments = AppExperiments(cuaVolume: true, yourCloud: false, sharing: false, webUi: true, secureSpaces: false, childSpaces: false, askFirst: false, ciRunners: false, tcpShares: false, mobile: false, warmPools: false)return m}diff --git a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/ViewModelTests.swift b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/ViewModelTests.swiftindex 839831a948..395b0a2896 100644--- a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/ViewModelTests.swift+++ b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/ViewModelTests.swift@@ -432,7 +432,7 @@ struct ViewModelTests {let telemetry = FixtureTelemetry()let model = makeModel(telemetry: telemetry)let toggles = model.experimentsPage.sections.flatMap(\.rows).filter { $0.kind == .toggle }- #expect(toggles.map(\.id) == ["experiment:cua_volume", "experiment:your_cloud", "experiment:sharing", "experiment:secure_spaces", "experiment:child_spaces", "experiment:ask_first", "experiment:ci_runners", "experiment:tcp_shares", "experiment:mobile"])+ #expect(toggles.map(\.id) == ["experiment:cua_volume", "experiment:your_cloud", "experiment:sharing", "experiment:secure_spaces", "experiment:child_spaces", "experiment:ask_first", "experiment:ci_runners", "experiment:tcp_shares", "experiment:mobile", "experiment:warm_pools"])// Sharing and Your cloud are on unless turned off; no New UI row.let onByDefault: Set<String> = ["experiment:sharing", "experiment:your_cloud"]#expect(toggles.allSatisfy { r indiff --git a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/WebUIHostTests.swift b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/WebUIHostTests.swiftindex f078e62b26..5911259af6 100644--- a/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/WebUIHostTests.swift+++ b/apps/cua-spaces-macos/Tests/CuaSpacesMacTests/WebUIHostTests.swift@@ -47,7 +47,7 @@ struct WebUIHostTests {}@Test func coreRecordsEncodeAsPlainValues() {- let x = AppExperiments(cuaVolume: true, yourCloud: false, sharing: false, webUi: true, secureSpaces: false, childSpaces: false, askFirst: false, ciRunners: false, tcpShares: false, mobile: false)+ let x = AppExperiments(cuaVolume: true, yourCloud: false, sharing: false, webUi: true, secureSpaces: false, childSpaces: false, askFirst: false, ciRunners: false, tcpShares: false, mobile: false, warmPools: false)let v = BridgeValue.encode(x) as? [String: Any]#expect(v?["cuaVolume"] as? Bool == true)#expect(v?["webUi"] as? Bool == true)diff --git a/apps/cua-spaces-web/e2e/settings-checks.ts b/apps/cua-spaces-web/e2e/settings-checks.tsindex d2d7b5dc58..0219c38553 100644--- a/apps/cua-spaces-web/e2e/settings-checks.ts+++ b/apps/cua-spaces-web/e2e/settings-checks.ts@@ -198,6 +198,7 @@ const LABS: Record<string, { label?: string; note: string }> = {"experiment:ci_runners": { note: "Run GitHub Actions jobs in a fresh Space each, on this computer. Adds CI Runners to the sidebar." },"experiment:tcp_shares": { note: "Share a game server or another TCP service in a Space at a public address." },"experiment:mobile": { note: "Your computers answer the Cua Spaces phone app: start a task, follow the thread and review changes." },+ "experiment:warm_pools": { note: "Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once." },};async function checkExperiments(page: Page, checkpoints: ParityCheckpoint[]): Promise<number> {diff --git a/apps/cua-spaces-web/src/__tests__/settings-advanced.test.tsx b/apps/cua-spaces-web/src/__tests__/settings-advanced.test.tsxindex 4c32d35e27..e7f4eeb837 100644--- a/apps/cua-spaces-web/src/__tests__/settings-advanced.test.tsx+++ b/apps/cua-spaces-web/src/__tests__/settings-advanced.test.tsx@@ -69,6 +69,7 @@ describe.skipIf(!wasmBuilt)("account usage and self-hosting in Settings (real ro["experiment:ci_runners", /CI runners/],["experiment:tcp_shares", /Public TCP shares/],["experiment:mobile", /Phone app/],+ ["experiment:warm_pools", /Warm start/],] as const;it("shows a switch in Labs for every experiment, off, with its line, before any is on", async () => {diff --git a/apps/cua-spaces-web/src/bridge/__tests__/experiments-defaults.test.ts b/apps/cua-spaces-web/src/bridge/__tests__/experiments-defaults.test.tsindex fcaf6642e6..d90d499085 100644--- a/apps/cua-spaces-web/src/bridge/__tests__/experiments-defaults.test.ts+++ b/apps/cua-spaces-web/src/bridge/__tests__/experiments-defaults.test.ts@@ -6,7 +6,7 @@ import { readExperiments } from "../ops/volume";describe("stored experiments", () => {it("reads sharing and your cloud as on unless stored off, and the rest as off", () => {- expect(readExperiments(undefined)).toEqual({ cuaVolume: false, yourCloud: true, sharing: true, webUi: false, secureSpaces: false, childSpaces: false, askFirst: false, ciRunners: false, tcpShares: false, mobile: false });+ expect(readExperiments(undefined)).toEqual({ cuaVolume: false, yourCloud: true, sharing: true, webUi: false, secureSpaces: false, childSpaces: false, askFirst: false, ciRunners: false, tcpShares: false, mobile: false, warmPools: false });expect(readExperiments(JSON.stringify({ cuaVolume: true }))).toMatchObject({ cuaVolume: true, sharing: true });expect(readExperiments(JSON.stringify({ sharing: false }))).toMatchObject({ sharing: false });expect(readExperiments(JSON.stringify({ yourCloud: false }))).toMatchObject({ yourCloud: false });diff --git a/apps/cua-spaces-web/src/bridge/adapters/demo/volume.ts b/apps/cua-spaces-web/src/bridge/adapters/demo/volume.tsindex 7344041dbb..7ba077061a 100644--- a/apps/cua-spaces-web/src/bridge/adapters/demo/volume.ts+++ b/apps/cua-spaces-web/src/bridge/adapters/demo/volume.ts@@ -39,7 +39,7 @@ export interface DemoVolumeState {export function demoVolumeState(now: number, onboarded: boolean, platform: DemoPlatform = MAC_DEMO_PLATFORM, askFirst = false, ciRunners = false): DemoVolumeState {return {- experiments: { cuaVolume: true, yourCloud: true, sharing: true, webUi: false, secureSpaces: false, childSpaces: false, askFirst, ciRunners, tcpShares: false, mobile: false },+ experiments: { cuaVolume: true, yourCloud: true, sharing: true, webUi: false, secureSpaces: false, childSpaces: false, askFirst, ciRunners, tcpShares: false, mobile: false, warmPools: false },requests: [{ id: "req-research", principal: "agent:researcher", prefix: "agents/writer/outputs/", mode: "r", reason: "Cite the launch draft in the weekly summary" },],diff --git a/apps/cua-spaces-web/src/bridge/contracts/bridge-shapes.json b/apps/cua-spaces-web/src/bridge/contracts/bridge-shapes.jsonindex 63aca656e6..698e2b0cd3 100644--- a/apps/cua-spaces-web/src/bridge/contracts/bridge-shapes.json+++ b/apps/cua-spaces-web/src/bridge/contracts/bridge-shapes.json@@ -2060,6 +2060,9 @@},"mobile": {"type": "boolean"+ },+ "warmPools": {+ "type": "boolean"}}},diff --git a/apps/cua-spaces-web/src/bridge/contracts/shapes.ts b/apps/cua-spaces-web/src/bridge/contracts/shapes.tsindex d914653cbd..24ca41646a 100644--- a/apps/cua-spaces-web/src/bridge/contracts/shapes.ts+++ b/apps/cua-spaces-web/src/bridge/contracts/shapes.ts@@ -463,7 +463,7 @@ const SettingsShape = ref(const ExperimentsShape = ref("Experiments",- obj<Experiments>()({ cuaVolume: opt(bool), yourCloud: opt(bool), sharing: opt(bool), webUi: opt(bool), secureSpaces: opt(bool), childSpaces: opt(bool), askFirst: opt(bool), ciRunners: opt(bool), tcpShares: opt(bool), mobile: opt(bool) }),+ obj<Experiments>()({ cuaVolume: opt(bool), yourCloud: opt(bool), sharing: opt(bool), webUi: opt(bool), secureSpaces: opt(bool), childSpaces: opt(bool), askFirst: opt(bool), ciRunners: opt(bool), tcpShares: opt(bool), mobile: opt(bool), warmPools: opt(bool) }),);const RepoCacheShape = ref(diff --git a/apps/cua-spaces-web/src/bridge/contracts/volume.ts b/apps/cua-spaces-web/src/bridge/contracts/volume.tsindex 496c1b5e41..232a8d5573 100644--- a/apps/cua-spaces-web/src/bridge/contracts/volume.ts+++ b/apps/cua-spaces-web/src/bridge/contracts/volume.ts@@ -40,9 +40,10 @@ export interface Experiments {tcpShares?: boolean;/** Mobile: the computer answers the Cua Spaces phone app. Off unless turned on; the daemon enforces it. */mobile?: boolean;+ warmPools?: boolean;}-export const NO_EXPERIMENTS: Required<Experiments> = { cuaVolume: false, yourCloud: true, sharing: true, webUi: false, secureSpaces: false, childSpaces: false, askFirst: false, ciRunners: false, tcpShares: false, mobile: false };+export const NO_EXPERIMENTS: Required<Experiments> = { cuaVolume: false, yourCloud: true, sharing: true, webUi: false, secureSpaces: false, childSpaces: false, askFirst: false, ciRunners: false, tcpShares: false, mobile: false, warmPools: false };/* ---- The daemon's answers (volume_*) ---------------------------------------- */diff --git a/apps/cua-spaces-web/src/bridge/ops/settings.ts b/apps/cua-spaces-web/src/bridge/ops/settings.tsindex b28b270692..83a82e23d7 100644--- a/apps/cua-spaces-web/src/bridge/ops/settings.ts+++ b/apps/cua-spaces-web/src/bridge/ops/settings.ts@@ -324,6 +324,7 @@ const EXPERIMENT_ROWS: Record<keyof Experiments, string> = {ciRunners: "experiment:ci_runners",tcpShares: "experiment:tcp_shares",mobile: "experiment:mobile",+ warmPools: "experiment:warm_pools",};/**diff --git a/apps/cua-spaces-web/src/bridge/ops/volume.ts b/apps/cua-spaces-web/src/bridge/ops/volume.tsindex ebb026e942..0dc2af6bf5 100644--- a/apps/cua-spaces-web/src/bridge/ops/volume.ts+++ b/apps/cua-spaces-web/src/bridge/ops/volume.ts@@ -138,6 +138,7 @@ export function readExperiments(raw: string | undefined): Experiments {ciRunners: stored.ciRunners === true,tcpShares: stored.tcpShares === true,mobile: stored.mobile === true,+ warmPools: stored.warmPools === true,};} catch {return NO_EXPERIMENTS;diff --git a/apps/cua-spaces-web/src/components/settings/advanced-tab.tsx b/apps/cua-spaces-web/src/components/settings/advanced-tab.tsxindex 1292e10249..ad60396016 100644--- a/apps/cua-spaces-web/src/components/settings/advanced-tab.tsx+++ b/apps/cua-spaces-web/src/components/settings/advanced-tab.tsx@@ -20,7 +20,7 @@ const SELF_HOSTED = "self-hosted";const BETA_HELP = "New features sooner, with rough edges. Back on Stable, this version stays until a newer stable release.";/** Experiments that stay as Labs. "Your cloud" and "Sharing" have no switch any more: the plan decides on the server. */-const LABS = new Set(["experiment:cua_volume", "experiment:secure_spaces", "experiment:child_spaces", "experiment:ask_first", "experiment:ci_runners", "experiment:tcp_shares", "experiment:mobile"]);+const LABS = new Set(["experiment:cua_volume", "experiment:secure_spaces", "experiment:child_spaces", "experiment:ask_first", "experiment:ci_runners", "experiment:tcp_shares", "experiment:mobile", "experiment:warm_pools"]);/*** Settings, Advanced: early releases, Labs, which software runs Spaces on@@ -159,6 +159,7 @@ function Labs() {"experiment:ci_runners": "Run GitHub Actions jobs in a fresh Space each, on this computer. Adds CI Runners to the sidebar.","experiment:tcp_shares": "Share a game server or another TCP service in a Space at a public address.","experiment:mobile": "Your computers answer the Cua Spaces phone app: start a task, follow the thread and review changes.",+ "experiment:warm_pools": "Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once.",};return (<SettingsGroup title="Labs">diff --git a/apps/cua-spaces/src-tauri/Cargo.lock b/apps/cua-spaces/src-tauri/Cargo.lockindex 1d3398a4df..fd40812159 100644--- a/apps/cua-spaces/src-tauri/Cargo.lock+++ b/apps/cua-spaces/src-tauri/Cargo.lock@@ -1835,6 +1835,7 @@ dependencies = ["axum","base64 0.22.1","bytes",+ "cua-agents","cua-cluster","cua-create-failure","cua-disk",diff --git a/docs/content/docs/spaces/concepts/runtime-conformance.mdx b/docs/content/docs/spaces/concepts/runtime-conformance.mdxindex 262c86788a..f1236df131 100644--- a/docs/content/docs/spaces/concepts/runtime-conformance.mdx+++ b/docs/content/docs/spaces/concepts/runtime-conformance.mdx@@ -33,6 +33,10 @@ does with them, and shows how Cua checks them.- **A coding agent session starts.** The app installs and the agent answers.- **Ready means answering.** Time to ready is measured from the create callto a Space that answers.+- **A warm claim is an ordinary Space.** A Space made from a member Cua kept+ ready answers with its own token and nothing else opens it, not even the+ member's own secret, and it behaves as one that started cold. A runtime+ that cannot claim a member refuses a pool in plain words and starts cold.- **Delete leaves nothing.** No Space, process, container or network remains.A runtime that cannot keep a promise refuses it at once, in plain words that@@ -41,20 +45,24 @@ without the promise, is a bug.## What each runtime says-| Runtime | Guest | Default network | Pause | Checkpoint and fork | KVM |-|---|---|---|---|---|---|-| `builtin-gvisor` | linux | enforced on macos, enforced on windows | yes | yes | no |-| `builtin-runc` | linux | enforced on macos, enforced on windows | yes | yes | no |-| `engine-gvisor` | linux | enforced on linux | yes | yes | no |-| `engine-runc` | linux | enforced on linux | yes | yes | no |-| `firecracker-microvm` | linux | enforced on linux | yes | yes | yes |-| `vz-microvm` | linux | enforced on macos | yes | yes | no |-| `containerization-microvm` | linux | refused on macos | no | no | no |-| `qemu-vm` | linux | enforced on linux, enforced on macos, enforced on windows | yes | yes | no |-| `lume-macos-vm` | macos | enforced on macos | no | no | no |+| Runtime | Guest | Default network | Pause | Checkpoint and fork | KVM | Warm start |+|---|---|---|---|---|---|---|+| `builtin-gvisor` | linux | enforced on macos, enforced on windows | yes | yes | no | claim, apps ready ahead |+| `builtin-runc` | linux | enforced on macos, enforced on windows | yes | yes | no | claim, apps ready ahead |+| `engine-gvisor` | linux | enforced on linux | yes | yes | no | claim, apps ready ahead |+| `engine-runc` | linux | enforced on linux | yes | yes | no | claim, apps ready ahead |+| `firecracker-microvm` | linux | enforced on linux | yes | yes | yes | claim |+| `vz-microvm` | linux | enforced on macos | yes | yes | no | claim |+| `containerization-microvm` | linux | refused on macos | no | no | no | claim |+| `qemu-vm` | linux | enforced on linux, enforced on macos, enforced on windows | yes | yes | no | claim |+| `lume-macos-vm` | macos | enforced on macos | no | no | no | claim |Pause, checkpoint and fork are what the runtime can do with a running Space.-Where a runtime cannot, asking for one is refused with a reason. The built-in+Where a runtime cannot, asking for one is refused with a reason. Warm start is+whether a member kept ready can be claimed for a new Space; on a container the+member also answers before it is claimed, so the [warm start](/spaces/guides/warm-start)+experiment can install the coding apps in it ahead. The other runtimes install+them after the claim, as they do without a pool. The built-inruntimes run on this computer's built-in Linux runtime; `engine-` runtimes runon a container engine you run on a Linux host.diff --git a/docs/content/docs/spaces/guides/experiments.mdx b/docs/content/docs/spaces/guides/experiments.mdxindex 25f501787a..a6ac6ca7c2 100644--- a/docs/content/docs/spaces/guides/experiments.mdx+++ b/docs/content/docs/spaces/guides/experiments.mdx@@ -26,6 +26,7 @@ A command that belongs to an experiment that is off is left out of `--help`. If| `child-spaces` | A Space makes Spaces of its own, inside a budget for its tree, and keeps a command running. See [Child Spaces](/spaces/guides/child-spaces) and [Keep a command running](/spaces/guides/keep-a-command-running). || `tcp-shares` | Share a raw TCP service in a Space, such as a game server, at a public address. See [Share a game server](/spaces/guides/share-a-game-server). || `mobile` | Your computers answer the Cua Spaces phone app: the daemon serves a phone's tools (start a task, tool rows, changes and diffs, approval scopes) and reports events for push alerts. See [Cua Spaces on your phone](/spaces/guides/phone). |+| `warm-pools` | Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once. See [Warm start](/spaces/guides/warm-start). || `secure-spaces` | A Space reaches the internet and its own tree and nothing else on your network unless you open it, with a network setting for each Space. See [A Space's network](/cua-sdk/guides/egress). |## Turning one offdiff --git a/docs/content/docs/spaces/guides/warm-start.mdx b/docs/content/docs/spaces/guides/warm-start.mdxnew file mode 100644index 0000000000..a1924e312f--- /dev/null+++ b/docs/content/docs/spaces/guides/warm-start.mdx@@ -0,0 +1,42 @@+---+title: Warm start+description: Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once. An experiment, off until you turn it on.+---++When you ask for something on Home, Cua makes a Linux Space for the agent, puts the coding app in it and starts the app. The Space takes a few seconds to start and the app takes much longer to install the first time. Warm start does that work before you ask.++With the experiment on, this computer keeps one Linux Space ready in the background: the same Linux Space that Home creates for a new agent, with Codex, Hermes, Claude Code and OpenCode already installed in it. While the experiment is on, Home makes that Space from the Linux slim tier (`ghcr.io/trycua/linux:24.04-slim`, about 0.5 GB to download) instead of the full Linux image, so the Space it keeps ready is the one Create takes. Create takes it, which takes a fraction of a second, and the agent starts in a Space that already has its app. Cua starts the next warm Space as soon as the last one is taken, so the one after is ready when you have typed the next request.++## Turn it on++```bash skip="no-runner"+cua experiments enable warm-pools+```++Or open Settings, Advanced, Labs and turn on Warm start. Turn it off the same way. Off, nothing is kept ready and Create works as it always did.++## What it does++- Keeps at most one Linux Space ready. It is hidden: it is on no list and costs nothing you see until Create takes it.+- Installs the coding apps in it while it waits, so Create does not wait for an install. By default these are Codex, Hermes, Claude Code and OpenCode, the four people run most. To choose others, set `CUA_WARM_POOLS_HARNESSES` for the cua daemon to a comma separated list of app ids, or to `none` to keep a bare Space.+- Starts the next one at once after Create takes it. If you create again before it is ready, that Create waits for the one that is starting, which began before it, instead of starting another from scratch.+- Gives the Space back after two hours without a Create, and whenever this computer needs the memory or CPUs for work you started. A warm Space never makes a Create fail: if there is no room, or anything goes wrong, Create starts a Space the way it does without the experiment.+- Holds only what [`pool.memory_percent`](/cua-cli/reference/cli/account-and-setup) allows (15 percent of this computer's memory by default).++## Where it works++A warm Space is claimed on every kind of Space Cua runs, and its coding apps are installed ahead on containers, the kind Home creates on a Mac. A microVM or a VM that cannot answer before it is claimed is claimed warm, and its coding apps are installed right after the claim, as they are without the experiment. [One Space on every runtime](/spaces/concepts/runtime-conformance) lists what each runtime does, and the live conformance run checks a warm claim on each.++## See it++`cua daemon status` shows how many warm Spaces this computer holds. The daemon log says when a Space is claimed and how long the claim took. With the experiment on, every step of an agent's start is also written to `phases.jsonl` in the `persistent` folder of the Cua home, with the time since your message, so you can read how long a start took.++## How fast++Measured on a Mac with 36 GB of memory and Cua's built-in Linux runtime, creating ten Spaces in a row three seconds apart, then starting Claude Code in each and waiting for a session that accepts a message. With Warm start on, a Create takes the waiting Space in about 0.1 s and the session opens about 1 s later, so the agent is ready about 1.1 s after Create (1.0 to 1.4 s each). The next Space is ready again in about 2 s, which is faster than the next request. Without the experiment the same ten Creates took about 22 s each, most of it installing Claude Code (16 s) and opening its session (4 s). The very first Create after installing Cua still has to download the runtime and the image, about 70 s on a fast connection; once Warm start has finished its first Space, about two minutes after the app starts, Create takes about 1.2 s.++Per app, creating ten Spaces in a row three seconds apart and waiting for the app to accept a message (medians): Codex 0.9 s, Claude Code 1.1 s, OpenCode 1.8 s, Hermes 2.8 s; without the experiment 14 s, 21 s, 20 s and 46 s. Hermes and OpenCode spend that time loading their own code when a session opens (Hermes imports about a second of Python), which a Space cannot do before it has your credentials.++## What it costs++The Space Cua keeps ready is prepared once and saved: the saved copy takes about 5.7 GB of the runtime's disk (the slim image is 1.8 GB unpacked; the four apps add 2 GB), and the waiting Space holds about 200 MB of memory. Preparing the first one takes two to four minutes in the background; when the app starts again it finds the saved copy and has a Space ready in about 5 s.diff --git a/docs/content/docs/spaces/meta.json b/docs/content/docs/spaces/meta.jsonindex 4d34d62815..3cea9376a8 100644--- a/docs/content/docs/spaces/meta.json+++ b/docs/content/docs/spaces/meta.json@@ -40,6 +40,7 @@"guides/keep-a-command-running","guides/long-running-jobs","guides/cua-ci",+ "guides/warm-start","---Keep it working---","guides/troubleshoot","guides/experiments",diff --git a/examples/openkoalabot-example-tauri/src-tauri/Cargo.lock b/examples/openkoalabot-example-tauri/src-tauri/Cargo.lockindex e4f1f69d80..c654c3f782 100644--- a/examples/openkoalabot-example-tauri/src-tauri/Cargo.lock+++ b/examples/openkoalabot-example-tauri/src-tauri/Cargo.lock@@ -1826,6 +1826,7 @@ dependencies = ["axum","base64 0.22.1","bytes",+ "cua-agents","cua-cluster","cua-create-failure","cua-disk",diff --git a/examples/streaming/rust/Cargo.lock b/examples/streaming/rust/Cargo.lockindex a802ccb636..d939f58389 100644--- a/examples/streaming/rust/Cargo.lock+++ b/examples/streaming/rust/Cargo.lock@@ -1462,6 +1462,7 @@ dependencies = ["async-trait","base64 0.22.1","bytes",+ "cua-agents","cua-byoc","cua-cluster","cua-create-failure",diff --git a/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock b/libs/cua-spacesd/tests/spaces-e2e/Cargo.lockindex f7edadc859..5ba922e8a6 100644--- a/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock+++ b/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock@@ -2236,6 +2236,7 @@ dependencies = ["axum","base64 0.22.1","bytes",+ "cua-agents","cua-byoc","cua-cluster","cua-create-failure",diff --git a/libs/cua/Cargo.lock b/libs/cua/Cargo.lockindex 70e9656dc7..b08e5eac9f 100644--- a/libs/cua/Cargo.lock+++ b/libs/cua/Cargo.lock@@ -2688,6 +2688,7 @@ dependencies = ["axum","base64 0.22.1","bytes",+ "cua-agents","cua-byoc","cua-cluster","cua-contrib",diff --git a/libs/cua/crates/cua-agents/src/harness.rs b/libs/cua/crates/cua-agents/src/harness.rsindex 298614787f..5906cab323 100644--- a/libs/cua/crates/cua-agents/src/harness.rs+++ b/libs/cua/crates/cua-agents/src/harness.rs@@ -72,6 +72,10 @@ pub struct Harness {pub sandbox_mcp: bool,/// Supported by the SDK: installed, driven and proven live.pub ready: bool,+ /// Environment that lets the agent open a session without a real+ /// credential (a value no model ever sees): what a warm-up session and a+ /// measurement of start times run with. Empty: the first of `keys`.+ pub placeholder: &'static [(&'static str, &'static str)],/// What is known not to work, published with its capabilities.pub notes: &'static [&'static str],}@@ -97,6 +101,7 @@ pub const HARNESSES: &[Harness] = &[skills_agent: Some("claude-code"),sandbox_mcp: true,ready: true,+ placeholder: &[("ANTHROPIC_API_KEY", "placeholder-not-a-key")],notes: &[],},Harness {@@ -117,6 +122,10 @@ pub const HARNESSES: &[Harness] = &[skills_agent: Some("codex"),sandbox_mcp: true,ready: true,+ placeholder: &[(+ "CODEX_AUTH_JSON",+ r#"{"OPENAI_API_KEY":"placeholder-not-a-key"}"#,+ )],notes: &[],},Harness {@@ -133,6 +142,7 @@ pub const HARNESSES: &[Harness] = &[skills_agent: Some("gemini-cli"),sandbox_mcp: true,ready: true,+ placeholder: &[],notes: &["Google stopped serving free Gemini CLI accounts on 2026-06-18; use a paid API key",],@@ -151,6 +161,7 @@ pub const HARNESSES: &[Harness] = &[skills_agent: Some("antigravity"),sandbox_mcp: true,ready: true,+ placeholder: &[],notes: &["runs Google's ACP server (the IDE's agent) headless; there is no interactive terminal CLI","the server is proprietary and about 1 GB; its sha256 is pinned by cua (Google publishes none)",@@ -173,6 +184,7 @@ pub const HARNESSES: &[Harness] = &[skills_agent: Some("opencode"),sandbox_mcp: true,ready: true,+ placeholder: &[("OPENAI_API_KEY", "placeholder-not-a-key")],notes: &[],},Harness {@@ -192,6 +204,7 @@ pub const HARNESSES: &[Harness] = &[skills_agent: Some("goose"),sandbox_mcp: true,ready: true,+ placeholder: &[],notes: &[],},Harness {@@ -211,6 +224,7 @@ pub const HARNESSES: &[Harness] = &[skills_agent: Some("pi"),sandbox_mcp: false,ready: true,+ placeholder: &[],notes: &["Pi has no MCP client (by design); MCP servers passed to a run are ignored","driven through the community pi-acp adapter (MIT), not a first-party ACP server",@@ -238,6 +252,7 @@ pub const HARNESSES: &[Harness] = &[skills_agent: None,sandbox_mcp: false,ready: true,+ placeholder: &[("OPENROUTER_API_KEY", "placeholder-not-a-key")],notes: &["hermes-acp does not take ACP session MCP servers; configure MCP in HERMES_HOME's config.yaml","installed from a pinned commit with uv sync --frozen (Hermes ships no wheel); needs git and Python 3.11 to 3.13 or uv's managed Python",@@ -261,6 +276,7 @@ pub const HARNESSES: &[Harness] = &[skills_agent: Some("openclaw"),sandbox_mcp: false,ready: true,+ placeholder: &[],notes: &["openclaw acp (bridge mode) refuses per-session MCP servers; configure MCP on the OpenClaw gateway","openclaw acp bridges to a Gateway; the run starts a loopback Gateway for itself",@@ -286,6 +302,25 @@ impl Harness {/// Short names people and apps use for a harness, and the id they mean.const ALIASES: &[(&str, &str)] = &[("codex", "openai-codex"), ("claude", "claude-code")];+impl Harness {+ /// The environment a session of this harness opens with when no real+ /// credential is available (see [`Harness::placeholder`]).+ pub fn placeholder_env(&self) -> std::collections::BTreeMap<String, String> {+ if self.placeholder.is_empty() {+ return self+ .keys+ .first()+ .map(|k| ((*k).to_string(), "placeholder-not-a-key".to_string()))+ .into_iter()+ .collect();+ }+ self.placeholder+ .iter()+ .map(|(k, v)| ((*k).to_string(), (*v).to_string()))+ .collect()+ }+}+/// A harness by id (or by one of its short names: `codex`, `claude`).pub fn harness(id: &str) -> Option<&'static Harness> {let id = ALIASES@@ -1510,4 +1545,14 @@ mod tests {.unwrap();assert!(l.files.is_empty());}++ #[test]+ fn every_ready_harness_has_a_placeholder_to_open_a_session_with() {+ for h in HARNESSES.iter().filter(|h| h.ready) {+ let env = h.placeholder_env();+ assert!(!env.is_empty(), "{}", h.id);+ // Never a real-looking secret.+ assert!(env.values().all(|v| v.contains("placeholder")), "{}", h.id);+ }+ }}diff --git a/libs/cua/crates/cua-agents/src/runs.rs b/libs/cua/crates/cua-agents/src/runs.rsindex cfa3380bce..da368c8a4e 100644--- a/libs/cua/crates/cua-agents/src/runs.rs+++ b/libs/cua/crates/cua-agents/src/runs.rs@@ -731,6 +731,50 @@ impl Agents {Ok(done)}+ /// Whether a start of `h` must run the install script: asked for, and+ /// something is missing. With every tool in place the script only prints+ /// that it found them, and each of its steps writes an event (a process+ /// or two in the guest, a quarter of a second each on a sandboxed+ /// kernel), so a start with the tools there skips it.+ async fn install_wanted(&self, h: &Harness, asked: bool) -> bool {+ asked && !self.tools_in_place(h).await+ }++ /// Whether the tools of `h` (and the runner's own) are installed under+ /// the guest user's home by this crate, in one guest call. A tool baked+ /// into the image, or one with operating system packages, always takes+ /// the script.+ async fn tools_in_place(&self, h: &Harness) -> bool {+ let mut ids: Vec<&str> = vec!["acp-sdk", "mcp-remote"];+ ids.extend(h.installs.iter().copied());+ let Ok(order) = installables::resolve(&ids) else {+ return false;+ };+ let mut tests = vec![];+ for id in &order {+ let Some(it) = installables::get(id) else {+ return false;+ };+ if !it.apt.is_empty() {+ return false;+ }+ tests.push(format!(+ "[ -f {} ]",+ quote(&format!(+ "{}/{}/{id}/{}/.cua-installed",+ self.home.trim_end_matches('/'),+ installables::TOOLS_DIR,+ it.version+ ))+ ));+ }+ let script = format!("{} && echo in-place", tests.join(" && "));+ matches!(+ self.sh(&script, Duration::from_secs(30)).await,+ Ok(out) if out.trim() == "in-place"+ )+ }+/// Puts what every start of `h` needs in the guest, once and in/// parallel: the bridge pipe, the runner and the cua skills, each skipped/// when it is already there. Safe to run ahead of a start (when an agent@@ -1160,7 +1204,7 @@ impl Agents {h,&rdir,&runner_dir,- opts.install.unwrap_or(true),+ self.install_wanted(h, opts.install.unwrap_or(true)).await,opts.repo.as_deref().map(|r| (r, opts.branch.as_deref(), cwd.as_str())),@@ -1227,11 +1271,19 @@ impl Agents {.and_then(|b| serde_json::from_slice(&b).ok()))}- /// One run's status. Never disturbs the run.+ /// One run's status. Never disturbs the run. The three guest reads it+ /// takes (meta, state, whether the process lives) go out together, so it+ /// costs one round trip.pub async fn status(&self, run_id: &str) -> Result<RunInfo> {valid_run_id(run_id)?;let rdir = self.run_dir(run_id);- let meta: Option<RunMeta> = match self.json_file(&format!("{rdir}/meta.json")).await {+ let (meta_path, state_path) = (format!("{rdir}/meta.json"), format!("{rdir}/state.json"));+ let (meta, state, alive) = tokio::join!(+ self.json_file(&meta_path),+ self.json_file(&state_path),+ self.alive(run_id)+ );+ let meta: Option<RunMeta> = match meta {Ok(m) => m.and_then(|v| serde_json::from_value(v).ok()),Err(e) => {return Ok(RunInfo {@@ -1253,13 +1305,7 @@ impl Agents {let Some(meta) = meta else {return Err(Error::NotFound(format!("no run {run_id} in this sandbox")));};- let state = self- .json_file(&format!("{rdir}/state.json"))- .await- .ok()- .flatten()- .unwrap_or(Value::Null);- let alive = self.alive(run_id).await;+ let state = state.ok().flatten().unwrap_or(Value::Null);Ok(info(run_id, meta, &state, alive))}@@ -1350,8 +1396,23 @@ impl Agents {model: Option<&str>,) -> Result<RunInfo> {let before = self.status(run_id).await?;+ self.send_after(run_id, text, files, &before, model).await?;+ self.status(run_id).await+ }++ /// [`Self::send_with_model`] for a caller that read the run's status+ /// just now (`before`): with a live runner it is one guest write, no+ /// status reads around it. The events say what the turn it starts did.+ pub async fn send_after(+ &self,+ run_id: &str,+ text: &str,+ files: Vec<Attachment>,+ before: &RunInfo,+ model: Option<&str>,+ ) -> Result<()> {if matches!(before.status, RunStatus::Unknown) {- return Err(Error::Guest(before.reason));+ return Err(Error::Guest(before.reason.clone()));}let rdir = self.run_dir(run_id);let mut paths = vec![];@@ -1390,7 +1451,7 @@ impl Agents {}self.spawn_run(run_id).await?;}- self.status(run_id).await+ Ok(())}/// Interrupts the turn in flight (ACP `session/cancel`); the sessiondiff --git a/libs/cua/crates/cua-daemon/Cargo.toml b/libs/cua/crates/cua-daemon/Cargo.tomlindex 48a2162911..c37c31d320 100644--- a/libs/cua/crates/cua-daemon/Cargo.toml+++ b/libs/cua/crates/cua-daemon/Cargo.toml@@ -43,6 +43,8 @@ cua-contrib = { workspace = true, optional = true }cua-create-failure.workspace = truecua-vmm = { workspace = true, default-features = true }cua-image.workspace = true+# The coding apps a warm-pool member gets before it is claimed (`warm-pools`).+cua-agents.workspace = true# Disk accounting, cache GC and orphan reaping.cua-disk.workspace = truecua-spaces = { workspace = true, optional = true }diff --git a/libs/cua/crates/cua-daemon/src/local.rs b/libs/cua/crates/cua-daemon/src/local.rsindex 8e60a1f503..cca24d6b99 100644--- a/libs/cua/crates/cua-daemon/src/local.rs+++ b/libs/cua/crates/cua-daemon/src/local.rs@@ -771,6 +771,14 @@ impl VmmLocal {Err(e) => tracing::warn!(%image, "pool.warm: {e}"),}}+ if cua_vmm::experiments::enabled(cua_vmm::experiments::Gate::WarmPools) {+ match self.keep_warm_space(pool).await {+ Ok(()) => {+ sources.insert(crate::pool::WARM_SOURCE.to_string());+ }+ Err(e) => tracing::warn!("warm-pools: {e}"),+ }+ }for np in pool.named_pools() {let source = format!("pool:{}", np.name);let outcome = async {@@ -794,10 +802,59 @@ impl VmmLocal {.insert(np.name.clone(), state);}pool.retain_explicit(|src| {- !(src.starts_with("warm:") || src.starts_with("pool:")) || sources.contains(src)+ !(src.starts_with("warm:")+ || src.starts_with("pool:")+ || src == crate::pool::WARM_SOURCE)+ || sources.contains(src)});}+ /// The `warm-pools` experiment: keeps one Linux Space ready, the Space+ /// the Spaces app's Home makes for a new agent, with the coding apps of+ /// [`warm_pools_harnesses`] installed in it. The shape is a plain local+ /// create of [`warm_pools_image`] (the size a create without one gets),+ /// so the create that Home makes claims it. It is kept ready after each+ /// claim at once, and drains after [`WARM_POOLS_IDLE`] without a create.+ async fn keep_warm_space(+ &self,+ pool: &crate::pool::WarmPool,+ ) -> std::result::Result<(), String> {+ use cua_sandbox_core::{CreateOptions, ProviderKind, Sandboxes, placement::On};+ let image = warm_pools_image();+ let mut o = CreateOptions::new(ProviderKind::Local, image.clone());+ if let Ok(r) = cua_sandbox_core::settings::Settings::load()+ .and_then(|s| s.resolve(Some(On::Local), None, None))+ {+ o.kind = r.kind;+ o.runtime = r.runtime;+ }+ o.apply_placement()+ .map_err(|e| format!("cannot run {image} here: {e}"))?;+ let spec = Sandboxes::local_start_spec(&o, "warm", Some("warm"));+ let p = self.prepare(&spec).await.map_err(|e| e.to_string())?;+ if p.injected {+ return Err("the image has no cua-spacesd".to_string());+ }+ let spacesd = p.image.as_ref().and_then(|i| i.spacesd) == Some(true);+ let key = pool+ .set_explicit_prepared(+ crate::pool::WARM_SOURCE,+ &p.spec,+ p.backend,+ spacesd,+ p.rt,+ 1,+ Some(1),+ None,+ Some(WARM_POOLS_IDLE),+ Some(warm_pools_harnesses()),+ )+ .ok_or_else(|| "this shape does not pool".to_string())?;+ pool.note_request(&p.spec, p.backend, spacesd, requested_words(&spec));+ tracing::debug!(shape = %key, "warm-pools: keeping one Linux Space ready");+ Ok(())+ }+/// Keeps `min` (at most `max`) members of `spec`'s shape for explicit/// pool `source`; the shape key, or why it does not pool here.async fn explicit(@@ -1156,6 +1213,86 @@ impl VmmLocal {/// settings, memory and strays are checked every 15th tick).pub const POOL_TICK: std::time::Duration = std::time::Duration::from_secs(2);+/// Opens a session of `h` in a member and closes it again (see+/// [`PoolHost::prepare`]), then removes the run it made.+async fn warm_session(+ agents: &cua_agents::Agents,+ id: &str,+ h: &cua_agents::harness::Harness,+) -> std::result::Result<(), String> {+ use cua_agents::{RunStatus, runs::RunOptions};+ let env = h.placeholder_env();+ let started = agents+ .start(+ id,+ "",+ RunOptions {+ env,+ skills: Some(false),+ idle_exit_secs: Some(120),+ ..Default::default()+ },+ )+ .await+ .map_err(|e| e.to_string())?;+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120);+ let mut result = Err("the session did not open in 120 s".to_string());+ while std::time::Instant::now() < deadline {+ match agents.status(&started.run_id).await {+ Ok(st) if st.status == RunStatus::Idle && st.accepts_message => {+ result = Ok(());+ break;+ }+ Ok(st) if st.status == RunStatus::Failed => {+ result = Err(format!("{} {:?}", st.reason, st.error));+ break;+ }+ _ => tokio::time::sleep(std::time::Duration::from_millis(100)).await,+ }+ }+ let _ = agents.stop(&started.run_id).await;+ let _ = agents+ .guest()+ .run(cua_spacesd_client::Command::shell(format!(+ "rm -rf '{}'",+ started.run_dir.replace('\'', "")+ )))+ .await;+ result+}++/// The image of the Linux Space the `warm-pools` experiment keeps ready: the+/// Linux slim tier, which the Spaces app's Home creates for a new agent while+/// the experiment is on (the wizard reads the same catalog file).+pub fn warm_pools_image() -> String {+ cua_image::catalog::entries()+ .iter()+ .find(|e| {+ e.published+ && e.os == "linux"+ && e.group == "canonical"+ && e.tier.as_deref() == Some("slim")+ && e.local.as_deref() == Some("container")+ })+ .map(|e| e.reference.clone())+ .unwrap_or_else(|| "ghcr.io/trycua/linux:24.04-slim".to_string())+}++/// The coding apps a warm Space is prepared for: `CUA_WARM_POOLS_HARNESSES`+/// (comma separated harness ids), else the four most used (Codex, Hermes, Claude Code, OpenCode).+pub fn warm_pools_harnesses() -> Vec<String> {+ let v = std::env::var("CUA_WARM_POOLS_HARNESSES")+ .unwrap_or_else(|_| "openai-codex,hermes,claude-code,opencode".into());+ v.split(',')+ .map(str::trim)+ .filter(|h| !h.is_empty() && *h != "none")+ .map(str::to_string)+ .collect()+}++/// How long the warm Space sits unused before it is given back.+pub const WARM_POOLS_IDLE: std::time::Duration = std::time::Duration::from_secs(2 * 60 * 60);+/// What the warm pool asks of this machine: cua-spacesd of its members, and/// the memory they draw on.struct PoolHost {@@ -1291,6 +1428,110 @@ impl crate::pool::PoolHost for PoolHost {Ok(())}+ fn can_prepare(&self, backend: BackendKind) -> bool {+ // The runtime's declaration (the conformance table): only a member+ // that answers before it is claimed, a container, takes its apps ahead.+ cua_vmm::conformance::capabilities(backend).prepare+ }++ async fn prepare(+ &self,+ endpoints: &Endpoints,+ secret: &str,+ harnesses: &[String],+ timeout: std::time::Duration,+ ) -> std::result::Result<(), String> {+ let run = async {+ let client = member_client(endpoints, secret).await?;+ let agents = cua_agents::Agents::new(client)+ .await+ .map_err(|e| e.to_string())?;+ let mut installs: Vec<&str> = Vec::new();+ for id in harnesses {+ let Some(h) = cua_agents::harness::harness(id).filter(|h| h.ready) else {+ tracing::warn!(harness = %id, "warm pool: not a harness this build can install");+ continue;+ };+ // The bridge, the runner and the skills, then the apps.+ agents.prewarm(h, true).await.map_err(|e| e.to_string())?;+ installs.extend(h.installs.iter().copied());+ }+ installs.sort_unstable();+ installs.dedup();+ if !installs.is_empty() {+ agents+ .ensure(&installs, |_| {})+ .await+ .map_err(|e| e.to_string())?;+ }+ // One session of each app is opened and closed here, with a key+ // that no model ever sees: the app's files are read once, so the+ // session of the Space that claims this one does not pay a cold+ // start for them. Nothing it wrote stays.+ for id in harnesses {+ let Some(h) = cua_agents::harness::harness(id).filter(|h| h.ready) else {+ continue;+ };+ if let Err(e) = warm_session(&agents, id, h).await {+ tracing::debug!(harness = %id, error = %e, "warm pool: the warm-up session did not open");+ }+ }+ Ok::<(), String>(())+ };+ match tokio::time::timeout(timeout, run).await {+ Ok(r) => r,+ Err(_) => Err(format!("preparing took longer than {timeout:?}")),+ }+ }++ async fn warm_files(&self, endpoints: &Endpoints, secret: &str, harnesses: &[String]) {+ // The directories the apps installed into, read once. The pool secret+ // stops opening the member when it is claimed: a failure is fine.+ let mut ids: Vec<&str> = vec!["acp-sdk", "mcp-remote"];+ for id in harnesses {+ if let Some(h) = cua_agents::harness::harness(id) {+ ids.extend(h.installs.iter().copied());+ }+ }+ let Ok(order) = cua_agents::installables::resolve(&ids) else {+ return;+ };+ let dirs: Vec<String> = order+ .iter()+ .filter_map(|id| {+ let it = cua_agents::installables::get(id)?;+ Some(format!("\"$HOME/.cua/tools/{id}/{}\"", it.version))+ })+ .collect();+ if dirs.is_empty() {+ return;+ }+ let script = format!(+ "find {} -type f -size -64M -exec cat {{}} + >/dev/null 2>&1; true",+ dirs.join(" ")+ );+ let run = async {+ let c = member_client(endpoints, secret).await?;+ c.run(+ cua_spacesd_client::Command::shell(script)+ .timeout(std::time::Duration::from_secs(120)),+ )+ .await+ .map(|_| ())+ .map_err(|e| e.to_string())+ };+ let began = std::time::Instant::now();+ match run.await {+ Ok(()) => tracing::debug!(+ ms = began.elapsed().as_millis() as u64,+ "warm pool: a member's apps were read once"+ ),+ Err(e) => {+ tracing::debug!(error = %e, "warm pool: could not read a member's apps ahead")+ }+ }+ }+async fn memory(&self, backend: BackendKind) -> Option<crate::pool::Memory> {let host = host_memory();if backend != BackendKind::Container {@@ -3295,6 +3536,51 @@ fn claim_delivers_token(backend: BackendKind) -> bool {#[cfg(test)]mod tests {+ #[test]+ fn members_are_prepared_where_the_conformance_table_says() {+ use crate::pool::PoolHost as _;+ let host = super::PoolHost {+ local: std::sync::Weak::new(),+ };+ for b in cua_vmm::egress::BACKENDS {+ let c = cua_vmm::conformance::capabilities(b);+ assert_eq!(host.can_prepare(b), c.prepare, "{b}");+ // A member that is prepared before its claim is a member that+ // can be claimed.+ assert!(!c.prepare || c.claim, "{b}");+ }+ assert!(host.can_prepare(cua_vmm::BackendKind::Container));+ assert!(!host.can_prepare(cua_vmm::BackendKind::Firecracker));+ }++ #[test]+ fn the_warm_space_is_the_image_home_creates_and_its_apps_can_be_chosen() {+ // The Spaces app's Home creates the Linux slim tier for a new agent+ // while the experiment is on (the wizard's `first_space_offers`); a+ // different image here would never be claimed.+ assert_eq!(super::warm_pools_image(), "ghcr.io/trycua/linux:24.04-slim");+ static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());+ let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());+ // SAFETY: serialised by ENV_LOCK; no other test reads this variable.+ unsafe { std::env::remove_var("CUA_WARM_POOLS_HARNESSES") };+ assert_eq!(+ super::warm_pools_harnesses(),+ ["openai-codex", "hermes", "claude-code", "opencode"]+ );+ unsafe { std::env::set_var("CUA_WARM_POOLS_HARNESSES", " claude-code , ,none ") };+ assert_eq!(super::warm_pools_harnesses(), ["claude-code"]);+ unsafe { std::env::set_var("CUA_WARM_POOLS_HARNESSES", "none") };+ assert!(super::warm_pools_harnesses().is_empty());+ unsafe { std::env::remove_var("CUA_WARM_POOLS_HARNESSES") };+ // Every default is a harness this build can install.+ for id in ["openai-codex", "hermes", "claude-code", "opencode"] {+ assert!(+ cua_agents::harness::harness(id).is_some_and(|h| h.ready),+ "{id}"+ );+ }+ }+#[test]fn a_macos_pull_is_sized_from_the_catalog() {let gib = 1u64 << 30;diff --git a/libs/cua/crates/cua-daemon/src/pool/mod.rs b/libs/cua/crates/cua-daemon/src/pool/mod.rsindex 3beab2e5be..3ef851f7f4 100644--- a/libs/cua/crates/cua-daemon/src/pool/mod.rs+++ b/libs/cua/crates/cua-daemon/src/pool/mod.rs@@ -69,6 +69,22 @@ const REFILL_DELAY: Duration = Duration::from_millis(1500);/// [`WarmPool::refill_later`]).const REFILL_MAX_WAIT: Duration = Duration::from_millis(7500);+/// The explicit-pool source of the `warm-pools` experiment's Linux Space+/// ([`crate::local::VmmLocal::apply_pools`]). A shape that has it is kept+/// warm on purpose for creates that follow each other: a refill starts the+/// moment a member is claimed, and a create that finds no member ready waits+/// for the one that is starting (it began before this create, so waiting is+/// never slower than a cold start of its own) instead of booting another.+pub const WARM_SOURCE: &str = "warm-pools";++/// The longest a create waits for a member that is starting+/// ([`WARM_SOURCE`]): about what a cold start and the install it saves take.+/// Past it the create starts cold, as it does without the experiment.+const WARM_WAIT_MAX: Duration = Duration::from_secs(30);++/// The longest the agent stack of a member takes to install.+const PREPARE_TIMEOUT: Duration = Duration::from_secs(900);+/// Consecutive failed member starts after which a shape stops refilling/// until it is used again.const MAX_FAILURES: u32 = 3;@@ -230,6 +246,39 @@ pub trait PoolHost: Send + Sync {/// it can list them without starting anything (container engines).async fn members_present(&self, backend: BackendKind) -> Option<Vec<String>>;+ /// Whether a member of `backend` can be prepared before it is claimed:+ /// it answers calls made with its pool secret (a container does; a+ /// microVM waits paused and a QEMU member restores memory only when it+ /// is claimed, so those are prepared after the claim, as before).+ fn can_prepare(&self, backend: BackendKind) -> bool {+ let _ = backend;+ false+ }++ /// Puts the agent stack of `harnesses` (bridge, runner, skills and the+ /// coding apps) in the member at `endpoints`, which answers to its pool+ /// `secret`, for at most `timeout`. Only called when+ /// [`PoolHost::can_prepare`] says the backend can.+ async fn prepare(+ &self,+ endpoints: &Endpoints,+ secret: &str,+ harnesses: &[String],+ timeout: Duration,+ ) -> Result<(), String> {+ let _ = (endpoints, secret, harnesses, timeout);+ Err("this runtime cannot prepare a member".into())+ }++ /// Reads the files of the apps `harnesses` once in the member at+ /// `endpoints`, so the first session of the Space that claims it does+ /// not read them from the engine's disk (a member booted from a saved+ /// filesystem has an empty file cache). Best effort, in the+ /// background, never needed for correctness.+ async fn warm_files(&self, endpoints: &Endpoints, secret: &str, harnesses: &[String]) {+ let _ = (endpoints, secret, harnesses);+ }+/// Whether this machine runs on battery now (no member then).fn on_battery(&self) -> bool;@@ -278,6 +327,8 @@ struct Member {name: String,secret: String,footprint_mb: u64,+ /// The agent stack of the shape's harnesses is in it already.+ prepared: bool,}/// A remembered shape.@@ -307,6 +358,15 @@ struct Shape {/// The words a create of it asked for (image, kind, runtime), as/// placement keys warm members.requested: Option<(String, String, String)>,+ /// The harnesses whose agent stack a member gets before it is ready+ /// ([`WarmPool::set_prepare`]); empty: members are bare.+ prepare: Vec<String>,+ /// Measured (EWMA, seconds): a member's preparation.+ prepare_s: Option<f64>,+ /// The saved filesystem of the first prepared member+ /// ([`cua_vmm::Runtime::save_prepared`]): its tag and the image the next+ /// members start from, already prepared.+ prepared_image: Option<(String, cua_vmm::ImageSource)>,}impl Shape {@@ -327,9 +387,44 @@ impl Shape {decision: None,requested: None,last_used: None,+ prepare: Vec::new(),+ prepare_s: None,+ prepared_image: None,}}+ /// Whether a create may wait for a member that is starting: the shape is+ /// kept warm on purpose for the `warm-pools` experiment and that pool is+ /// not idle.+ fn waits_for_refill(&self) -> bool {+ self.explicit+ .get(WARM_SOURCE)+ .is_some_and(|e| !self.idle(e))+ && self.refill_estimate().is_some_and(|d| d < WARM_WAIT_MAX)+ }++ /// Whether the shape is kept warm for the experiment, whatever the+ /// estimate (the refill after a claim starts at once).+ fn is_warm_source(&self) -> bool {+ self.explicit+ .get(WARM_SOURCE)+ .is_some_and(|e| !self.idle(e))+ }++ /// How long the member that is starting takes to be ready, as measured,+ /// and `None` before this shape has had a member (the first member of a+ /// new install pulls the image and installs the apps: a create does not+ /// wait for that, it starts cold).+ fn refill_estimate(&self) -> Option<Duration> {+ let boot = self.boot_s?;+ let secs = if self.prepared_image.is_some() || self.prepare.is_empty() {+ boot+ } else {+ boot + self.prepare_s?+ };+ Some(Duration::from_secs_f64(secs))+ }+/// Whether explicit pool `e` is idle now (no create of the shape for/// its idle time).fn idle(&self, e: &Explicit) -> bool {@@ -438,6 +533,9 @@ struct Record {template: StartSpec,#[serde(default)]keep: u32,+ /// The agent stack was installed before the member was ready.+ #[serde(default)]+ prepared: bool,}/// What the pool holds now ([`WarmPool::status`]).@@ -481,6 +579,11 @@ struct Inner {/// [`WarmPool::reclaim`] took members back for other work: the policy/// starts no member of its own until then.held_off_until: Mutex<Option<Instant>>,+ /// Woken when a member becomes ready or a start ends without one, so a+ /// create that waits for a starting member ([`WARM_SOURCE`]) looks again.+ arrived: tokio::sync::Notify,+ /// Shapes with a refill scheduled that has not started yet.+ due: Mutex<HashSet<String>>,}/// How long the policy starts no member after other work took capacity back.@@ -586,6 +689,23 @@ fn token_of(spec: &StartSpec) -> Option<String> {.cloned()}+/// The tag of the saved filesystem of a shape's prepared members: the shape,+/// the apps and this build (a new build installs the apps it pins).+fn prepared_tag(key: &str, harnesses: &[String]) -> String {+ use sha2::{Digest, Sha256};+ let mut h = Sha256::new();+ h.update(harnesses.join(",").as_bytes());+ h.update(env!("CARGO_PKG_VERSION").as_bytes());+ let d = h.finalize();+ format!(+ "warm-{key}-{}",+ d.iter()+ .take(4)+ .map(|b| format!("{b:02x}"))+ .collect::<String>()+ )+}+/// A fresh name for a member.fn member_name() -> String {format!(@@ -621,6 +741,8 @@ impl WarmPool {ticks: Default::default(),class: policy::HostClass::current(),held_off_until: Mutex::new(None),+ arrived: tokio::sync::Notify::new(),+ due: Mutex::new(HashSet::new()),}),}}@@ -747,12 +869,7 @@ impl WarmPool {let key = shape_key(spec, backend, spacesd)?;let token = token_of(spec)?;let arrived = self.now_s();- let (member, rt) = {- let mut shapes = self.inner.shapes.lock().unwrap();- let shape = shapes.get_mut(&key)?;- let m = shape.members.pop()?;- (m, shape.rt.clone())- };+ let (member, rt) = self.take_member(&key).await?;let started = Instant::now();let identity = ClaimIdentity {token: Some(token.clone()),@@ -796,6 +913,7 @@ impl WarmPool {member = %member.name,claim_ms,ready_ms = (ready_s * 1000.0) as u64,+ prepared = member.prepared,"claimed a warm Space");if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(&key) {@@ -804,10 +922,95 @@ impl WarmPool {s.last_used = Some(Instant::now());}self.evaluate(&key);- self.refill_later(&key, REFILL_DELAY);+ // A shape kept warm on purpose is refilled at once: the next create+ // (typed while this Space starts) should find a member ready. Others+ // wait for the claimed Space's first commands to pass.+ let warm = self+ .inner+ .shapes+ .lock()+ .unwrap()+ .get(&key)+ .is_some_and(|s| s.is_warm_source());+ self.refill_later(&key, if warm { Duration::ZERO } else { REFILL_DELAY });Some(inst)}+ /// Takes a ready member of shape `key`. A shape kept warm for the+ /// `warm-pools` experiment ([`WARM_SOURCE`]) whose member is starting+ /// is waited for (up to [`WARM_WAIT_MAX`]): that member began before this+ /// create, so waiting is never slower than a cold start of its own, and+ /// a cold start would not have the agent stack in it. Any other shape+ /// answers at once.+ async fn take_member(&self, key: &str) -> Option<(Member, Arc<dyn VmmRuntime>)> {+ let deadline = Instant::now() + WARM_WAIT_MAX;+ let mut waited = false;+ loop {+ // Registered before the look, so a member that arrives between+ // the look and the wait is not missed.+ let notified = self.inner.arrived.notified();+ tokio::pin!(notified);+ notified.as_mut().enable();+ let starting = {+ let mut shapes = self.inner.shapes.lock().unwrap();+ let shape = shapes.get_mut(key)?;+ // One with the coding apps in it first.+ let at = shape+ .members+ .iter()+ .rposition(|m| m.prepared)+ .or(shape.members.len().checked_sub(1));+ if let Some(m) = at.map(|i| shape.members.remove(i)) {+ if waited {+ tracing::info!(+ shape = key,+ "a create waited for the member that was starting"+ );+ }+ return Some((m, shape.rt.clone()));+ }+ shape.waits_for_refill()+ && (shape.starting > 0+ || self.inner.refilling.lock().unwrap().contains(key)+ || self.inner.due.lock().unwrap().contains(key))+ };+ let Some(left) = deadline.checked_duration_since(Instant::now()) else {+ tracing::warn!(+ shape = key,+ "warm-pools: the member that was starting did not arrive in time; starting cold"+ );+ return None;+ };+ if !starting {+ if waited || self.is_warm_shape(key) {+ tracing::info!(+ shape = key,+ "warm-pools: no prepared Space was ready and none is starting; starting cold"+ );+ }+ return None;+ }+ waited = true;+ if tokio::time::timeout(left, notified).await.is_err() {+ tracing::warn!(+ shape = key,+ "warm-pools: the member that was starting did not arrive in time; starting cold"+ );+ return None;+ }+ }+ }++ /// Whether shape `key` is kept warm for the `warm-pools` experiment.+ fn is_warm_shape(&self, key: &str) -> bool {+ self.inner+ .shapes+ .lock()+ .unwrap()+ .get(key)+ .is_some_and(|s| s.is_warm_source())+ }+/// Records a create of `spec`'s shape that started on the default tier/// (no member was ready) in `start_s` seconds on `rt`: the policy may/// now want members of it.@@ -858,6 +1061,28 @@ impl WarmPool {max: Option<u32>,until: Option<Instant>,idle: Option<Duration>,+ ) -> Option<String> {+ self.set_explicit_prepared(+ source, template, backend, spacesd, rt, min, max, until, idle, None,+ )+ }++ /// [`WarmPool::set_explicit`] that also sets the harnesses the shape's+ /// members are prepared for before their first start (`Some`; `None`+ /// leaves them as they are).+ #[allow(clippy::too_many_arguments)]+ pub fn set_explicit_prepared(+ &self,+ source: &str,+ template: &StartSpec,+ backend: BackendKind,+ spacesd: bool,+ rt: Arc<dyn VmmRuntime>,+ min: u32,+ max: Option<u32>,+ until: Option<Instant>,+ idle: Option<Duration>,+ prepare: Option<Vec<String>>,) -> Option<String> {let key = shape_key(template, backend, spacesd)?;{@@ -887,6 +1112,9 @@ impl WarmPool {},);shape.failures = 0;+ if let Some(p) = prepare {+ shape.prepare = p;+ }}self.refill_later(&key, Duration::ZERO);Some(key)@@ -988,6 +1216,7 @@ impl WarmPool {fn refill_later(&self, key: &str, delay: Duration) {let pool = self.clone();let key = key.to_string();+ self.inner.due.lock().unwrap().insert(key.clone());if let Ok(h) = tokio::runtime::Handle::try_current() {h.spawn(async move {if !delay.is_zero() {@@ -1027,13 +1256,17 @@ impl WarmPool {/// Starts members of shape `key` until it has its target, memory/// allowing. One refill per shape at a time.pub async fn refill(&self, key: &str) {- if !self.inner.refilling.lock().unwrap().insert(key.to_string()) {+ let started = self.inner.refilling.lock().unwrap().insert(key.to_string());+ self.inner.due.lock().unwrap().remove(key);+ if !started {return;}struct Done<'a>(&'a Inner, String);impl Drop for Done<'_> {fn drop(&mut self) {self.0.refilling.lock().unwrap().remove(&self.1);+ // A create waiting for a member looks again: none is coming.+ self.0.arrived.notify_waiters();}}let _done = Done(&self.inner, key.to_string());@@ -1069,7 +1302,7 @@ impl WarmPool {if keep > 0 && self.inner.host.memory(backend).await.is_none() {self.inner.host.wake(backend).await;}- if !self.memory_allows(backend, estimate).await {+ if !self.memory_allows(backend, estimate, key).await {tracing::debug!(shape = key,"warm pool: not enough memory for another member"@@ -1109,16 +1342,31 @@ impl WarmPool {tracing::warn!(shape = key, error = %e, "a warm-pool member did not start");}}+ drop(shapes);+ self.inner.arrived.notify_waiters();}}/// Whether one more member estimated at `estimate_mb` fits: the pool/// stays within `pool.memory_percent` of the memory and above the free/// floor.- async fn memory_allows(&self, backend: BackendKind, estimate_mb: u64) -> bool {+ ///+ /// The one warm Space of the `warm-pools` experiment ([`WARM_SOURCE`])+ /// is not held to the percentage while it is the only member held (on a+ /// built-in runtime of 4 GiB, 15 percent is less than one warm Space):+ /// it still needs the free floor, so it never takes memory a Space in use+ /// needs.+ async fn memory_allows(&self, backend: BackendKind, estimate_mb: u64, key: &str) -> bool {let Some(mem) = self.inner.host.memory(backend).await else {return false;};+ let one_warm_home = self+ .inner+ .shapes+ .lock()+ .unwrap()+ .get(key)+ .is_some_and(|s| s.is_warm_source());let pct = u64::from(self.settings().memory_percent);let held: u64 = {let shapes = self.inner.shapes.lock().unwrap();@@ -1131,6 +1379,9 @@ impl WarmPool {}).sum()};+ if one_warm_home && held == 0 {+ return fits(mem, 100, 0, estimate_mb);+ }fits(mem, pct, held, estimate_mb)}@@ -1147,13 +1398,26 @@ impl WarmPool {let shapes = self.inner.shapes.lock().unwrap();shapes.get(key).map(|s| s.keep()).unwrap_or(0)};- let record = Record {+ let prepare = {+ let shapes = self.inner.shapes.lock().unwrap();+ shapes+ .get(key)+ .map(|s| s.prepare.clone())+ .unwrap_or_default()+ };+ let prepare = if self.inner.host.can_prepare(backend) {+ prepare+ } else {+ Vec::new()+ };+ let mut record = Record {name: name.clone(),key: key.to_string(),backend,secret: secret.clone(),template: template.clone(),keep,+ prepared: false,};// Recorded and leased before anything exists: a crash from here on// leaves a lease whose process is gone, and the reaper removes it.@@ -1165,6 +1429,36 @@ impl WarmPool {let mut spec = template.clone();spec.name = name.clone();spec.pool_member = true;+ // A member of a shape whose first member was prepared boots from that+ // member's saved filesystem: its apps are in it already, so it is+ // ready in the time of a boot, not of an install.+ let mut saved = if prepare.is_empty() {+ None+ } else {+ self.inner+ .shapes+ .lock()+ .unwrap()+ .get(key)+ .and_then(|s| s.prepared_image.clone())+ };+ // A daemon that restarted: what the last one saved is still there.+ if saved.is_none() && !prepare.is_empty() {+ let tag = prepared_tag(key, &prepare);+ if let Ok(Some(image)) = rt.find_prepared(&tag).await {+ tracing::info!(+ shape = key,+ "warm-pool: found the prepared filesystem a previous daemon saved"+ );+ if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {+ s.prepared_image = Some((tag.clone(), image.clone()));+ }+ saved = Some((tag, image));+ }+ }+ if let Some((_, image)) = &saved {+ spec.image = image.clone();+ }spec.env.insert("CUA_ENV_TOKEN".into(), secret.clone());spec.labels.insert(cua_vmm::container::LABEL_POOL.into(),@@ -1178,8 +1472,17 @@ impl WarmPool {// Until its shape holds it (the refill moves it there), the sweep// of unknown members leaves it alone.self.inner.starting.lock().unwrap().insert(name.clone());+ let mut prepared = false;let ready = async {- let inst = rt.start(&spec).await.map_err(|e| e.to_string())?;+ let inst = rt.start(&spec).await.map_err(|e| e.to_string());+ if inst.is_err() && saved.is_some() {+ // The saved filesystem does not start any more (the engine+ // lost it): the next member is prepared from scratch.+ if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {+ s.prepared_image = None;+ }+ }+ let inst = inst?;// A container member runs with its pool secret: it is ready// once it serves a command and a screenshot. Other backends// return from `start` with a member ready to claim (a microVM@@ -1198,6 +1501,56 @@ impl WarmPool {Ok::<_, String>(inst)}.await;+ let booted = started.elapsed();+ // The agent stack goes in while the member is hidden, so the Space+ // that claims it has its coding apps. A member that cannot be+ // prepared is still a member: it is claimed bare, and the stack goes+ // in after the claim as it does without a pool.+ if ready.is_ok() && saved.is_some() {+ prepared = true;+ record.prepared = true;+ let _ = self.write_record(&record);+ } else if let Ok(inst) = &ready+ && !prepare.is_empty()+ {+ let began = Instant::now();+ match self+ .inner+ .host+ .prepare(&inst.endpoints, &secret, &prepare, PREPARE_TIMEOUT)+ .await+ {+ Ok(()) => {+ prepared = true;+ record.prepared = true;+ let _ = self.write_record(&record);+ let took = began.elapsed().as_secs_f64();+ if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {+ s.prepare_s = ewma(s.prepare_s, took);+ }+ tracing::info!(member = %name, harnesses = %prepare.join(","), prepare_ms = (took * 1000.0) as u64, "warm-pool member prepared");+ // Its filesystem is kept (before anyone can claim it),+ // so the next member boots from it.+ let tag = prepared_tag(key, &prepare);+ let saving = Instant::now();+ match rt.save_prepared(&name, &tag).await {+ Ok(Some(image)) => {+ tracing::info!(member = %name, save_ms = saving.elapsed().as_millis() as u64, "warm-pool: the prepared filesystem is kept for the next member");+ if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {+ s.prepared_image = Some((tag, image));+ }+ }+ Ok(None) => {}+ Err(e) => {+ tracing::warn!(member = %name, error = %e, "warm-pool: could not keep the prepared filesystem; the next member is prepared again");+ }+ }+ }+ Err(e) => {+ tracing::warn!(member = %name, error = %e, "warm-pool member could not be prepared; it is claimed bare");+ }+ }+ }match ready {Ok(_) => {}Err(e) => {@@ -1209,7 +1562,8 @@ impl WarmPool {};let footprint_mb = self.inner.host.used_mb(backend, &name).await.unwrap_or(0);if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {- s.boot_s = ewma(s.boot_s, started.elapsed().as_secs_f64());+ // The boot alone: preparing and saving are measured on their own.+ s.boot_s = ewma(s.boot_s, booted.as_secs_f64());}tracing::info!(member = %name,@@ -1218,10 +1572,24 @@ impl WarmPool {footprint_mb,"warm-pool member ready");+ // A member that booted from the saved filesystem has nothing in its+ // file cache: its apps are read now, while it waits to be claimed.+ if saved.is_some()+ && prepared+ && let Ok(inst) = &ready+ {+ let pool = self.clone();+ let (endpoints, secret, apps) =+ (inst.endpoints.clone(), secret.clone(), prepare.clone());+ tokio::spawn(async move {+ pool.inner.host.warm_files(&endpoints, &secret, &apps).await;+ });+ }Ok(Member {name,secret,footprint_mb,+ prepared,})}@@ -1247,6 +1615,7 @@ impl WarmPool {self.evaluate(k);}let mut drained = Vec::new();+ let mut saved_gone: Vec<(Arc<dyn VmmRuntime>, String)> = Vec::new();{let mut shapes = self.inner.shapes.lock().unwrap();for k in &keys {@@ -1261,7 +1630,11 @@ impl WarmPool {.last().is_none_or(|t| now - t > policy::HORIZON_S);if stale {- shapes.remove(k);+ if let Some(gone) = shapes.remove(k)+ && let Some((tag, _)) = gone.prepared_image+ {+ saved_gone.push((gone.rt.clone(), tag));+ }continue;}let excess = shape@@ -1275,6 +1648,9 @@ impl WarmPool {}}}+ for (rt, tag) in saved_gone {+ self.drop_saved(rt, tag);+ }for (rt, m, why) in drained {tracing::info!(member = %m.name, reason = %why, "warm pool: a member is no longer wanted");self.discard(rt, m.name.clone());@@ -1473,6 +1849,7 @@ impl WarmPool {name: r.name.clone(),secret: r.secret.clone(),footprint_mb,+ prepared: r.prepared,});}}@@ -1562,16 +1939,23 @@ impl WarmPool {}tokio::time::sleep(Duration::from_millis(100)).await;}+ let mut saved: Vec<(Arc<dyn VmmRuntime>, String)> = Vec::new();let all: Vec<(Arc<dyn VmmRuntime>, Member)> = {let mut shapes = self.inner.shapes.lock().unwrap();shapes.drain().flat_map(|(_, s)| {let rt = s.rt.clone();+ if let Some((tag, _)) = &s.prepared_image {+ saved.push((rt.clone(), tag.clone()));+ }s.members.into_iter().map(move |m| (rt.clone(), m))}).collect()};+ // The saved filesystems stay: a daemon that starts again finds them+ // ([`prepared_tag`]) instead of preparing a member for minutes.+ drop(saved);for (rt, m) in all {match rt.delete(&m.name).await {Ok(()) | Err(VmmError::NotFound(_)) => self.forget_member(&m.name),@@ -1602,12 +1986,26 @@ impl WarmPool {fn forget_shape(&self, key: &str) {let gone = self.inner.shapes.lock().unwrap().remove(key);if let Some(s) = gone {+ if let Some((tag, _)) = &s.prepared_image {+ self.drop_saved(s.rt.clone(), tag.clone());+ }for m in s.members {self.discard(s.rt.clone(), m.name);}}}+ /// Deletes the saved filesystem `tag` in the background.+ fn drop_saved(&self, rt: Arc<dyn VmmRuntime>, tag: String) {+ if let Ok(h) = tokio::runtime::Handle::try_current() {+ h.spawn(async move {+ if let Err(e) = rt.drop_prepared(&tag).await {+ tracing::debug!(%tag, error = %e, "warm pool: could not delete a saved filesystem");+ }+ });+ }+ }+/// Drops member `name`'s record and lease.fn forget_member(&self, name: &str) {self.remove_record(name);@@ -1679,6 +2077,15 @@ mod tests {no_claims: bool,starts: AtomicU32,fail_starts: AtomicBool,+ /// How long a start takes, in ms (a booting member).+ start_ms: std::sync::atomic::AtomicU64,+ /// Whether `save_prepared` keeps a filesystem, and the tags kept+ /// and dropped.+ can_save: AtomicBool,+ saved: Mutex<Vec<String>>,+ dropped: Mutex<Vec<String>>,+ /// The image of every start, in order.+ images: Mutex<Vec<String>>,}fn instance(name: &str) -> Instance {@@ -1706,6 +2113,13 @@ mod tests {}async fn start(&self, spec: &StartSpec) -> VResult<Instance> {self.starts.fetch_add(1, SeqCst);+ if let ImageSource::Oci { reference } = &spec.image {+ self.images.lock().unwrap().push(reference.clone());+ }+ let ms = self.start_ms.load(SeqCst);+ if ms > 0 {+ tokio::time::sleep(Duration::from_millis(ms)).await;+ }if self.fail_starts.load(SeqCst) {return Err(VmmError::other("boom"));}@@ -1749,6 +2163,26 @@ mod tests {async fn fork(&self, _: &str, _: &str) -> VResult<()> {Ok(())}+ async fn save_prepared(&self, _: &str, tag: &str) -> VResult<Option<ImageSource>> {+ if !self.can_save.load(SeqCst) {+ return Ok(None);+ }+ self.saved.lock().unwrap().push(tag.to_string());+ Ok(Some(ImageSource::oci(format!("local/prepared:{tag}"))))+ }+ async fn find_prepared(&self, tag: &str) -> VResult<Option<ImageSource>> {+ Ok(self+ .saved+ .lock()+ .unwrap()+ .iter()+ .any(|t| t == tag)+ .then(|| ImageSource::oci(format!("local/prepared:{tag}"))))+ }+ async fn drop_prepared(&self, tag: &str) -> VResult<()> {+ self.dropped.lock().unwrap().push(tag.to_string());+ Ok(())+ }async fn checkpoint(&self, _: &str, _: &str) -> VResult<CheckpointInfo> {Err(VmmError::invalid("unused"))}@@ -1799,6 +2233,12 @@ mod tests {fail_rotate: AtomicBool,ready: AtomicBool,busy: AtomicBool,+ /// Whether members can be prepared, what was prepared (harnesses+ /// per member) and whether preparing fails.+ can_prepare: AtomicBool,+ prepared: Mutex<Vec<Vec<String>>>,+ fail_prepare: AtomicBool,+ prepare_ms: std::sync::atomic::AtomicU64,}impl Default for FakeHost {@@ -1812,6 +2252,10 @@ mod tests {fail_rotate: AtomicBool::new(false),ready: AtomicBool::new(true),busy: AtomicBool::new(false),+ can_prepare: AtomicBool::new(true),+ prepared: Mutex::new(vec![]),+ fail_prepare: AtomicBool::new(false),+ prepare_ms: Default::default(),}}}@@ -1847,6 +2291,26 @@ mod tests {.push((secret.to_string(), token.to_string()));Ok(())}+ fn can_prepare(&self, _: BackendKind) -> bool {+ self.can_prepare.load(SeqCst)+ }+ async fn prepare(+ &self,+ _: &Endpoints,+ _: &str,+ harnesses: &[String],+ _: Duration,+ ) -> Result<(), String> {+ let ms = self.prepare_ms.load(SeqCst);+ if ms > 0 {+ tokio::time::sleep(Duration::from_millis(ms)).await;+ }+ if self.fail_prepare.load(SeqCst) {+ return Err("no network".into());+ }+ self.prepared.lock().unwrap().push(harnesses.to_vec());+ Ok(())+ }async fn memory(&self, _: BackendKind) -> Option<Memory> {Some(*self.memory.lock().unwrap())}@@ -2475,4 +2939,383 @@ mod tests {assert!(f.rt.members().is_empty());assert!(f.state.leases().is_empty());}++ /// The `warm-pools` Space: one member, prepared with the coding+ /// apps, kept ready after every claim.+ async fn keep_warm_space(f: &Fixture, s: &StartSpec) -> String {+ let rt: Arc<dyn VmmRuntime> = f.rt.clone();+ f.pool+ .set_explicit_prepared(+ WARM_SOURCE,+ s,+ BackendKind::Container,+ true,+ rt,+ 1,+ Some(1),+ None,+ Some(Duration::from_secs(7200)),+ Some(vec!["claude-code".into(), "openai-codex".into()]),+ )+ .unwrap()+ }++ /// Waits until shape `key` holds `n` ready members.+ async fn until_ready(f: &Fixture, key: &str, n: u32) {+ for _ in 0..200 {+ if f.pool.shape_status(key).is_some_and(|s| s.0 >= n) {+ return;+ }+ tokio::time::sleep(Duration::from_millis(20)).await;+ }+ panic!(+ "shape {key} never had {n} ready member(s): {:?}",+ f.pool.shape_status(key)+ );+ }++ #[tokio::test]+ async fn a_warm_pools_member_has_its_coding_apps_before_it_is_ready() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ let key = keep_warm_space(&f, &spec("one", "t")).await;+ until_ready(&f, &key, 1).await;+ assert_eq!(+ f.host.prepared.lock().unwrap().clone(),+ vec![vec!["claude-code".to_string(), "openai-codex".to_string()]],+ "prepared once, for both apps, before it counted as ready"+ );+ // The record says so: a daemon that adopts it knows.+ let rec = f.pool.records();+ assert_eq!(rec.len(), 1);+ assert!(rec[0].prepared);+ }++ #[tokio::test]+ async fn a_backend_that_cannot_prepare_its_members_keeps_them_bare() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ f.host.can_prepare.store(false, SeqCst);+ let key = keep_warm_space(&f, &spec("one", "t")).await;+ until_ready(&f, &key, 1).await;+ assert!(f.host.prepared.lock().unwrap().is_empty());+ assert!(!f.pool.records()[0].prepared);+ // A claim still works: the stack goes in after it, as without a pool.+ let c = f+ .pool+ .claim(&spec("mine", "tok"), BackendKind::Container, true)+ .await;+ assert!(c.is_some());+ }++ #[tokio::test]+ async fn a_member_that_fails_to_prepare_is_still_a_member() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ f.host.fail_prepare.store(true, SeqCst);+ let key = keep_warm_space(&f, &spec("one", "t")).await;+ until_ready(&f, &key, 1).await;+ assert!(!f.pool.records()[0].prepared, "claimed bare, never refused");+ assert!(+ f.pool+ .claim(&spec("mine", "tok"), BackendKind::Container, true)+ .await+ .is_some()+ );+ }++ #[tokio::test]+ async fn the_next_create_finds_a_member_because_a_claim_refills_at_once() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ let key = keep_warm_space(&f, &spec("one", "t")).await;+ until_ready(&f, &key, 1).await;+ for i in 0..4 {+ let got = f+ .pool+ .claim(+ &spec(&format!("s{i}"), &format!("t{i}")),+ BackendKind::Container,+ true,+ )+ .await;+ assert!(got.is_some(), "create {i} claims a ready member");+ // Not the 1.5 s the other shapes wait: a few polls, not seconds.+ until_ready(&f, &key, 1).await;+ }+ assert_eq!(+ f.host.prepared.lock().unwrap().len(),+ 5,+ "one member each, all prepared"+ );+ }++ #[tokio::test]+ async fn a_create_waits_for_the_member_that_is_starting_instead_of_booting_another() {+ let rt = FakeRuntime::default();+ rt.start_ms.store(150, SeqCst);+ let f = fixture(PoolSettings::default(), rt);+ let key = keep_warm_space(&f, &spec("one", "t")).await;+ // The very first member (a new install) is not waited for: the+ // create starts cold.+ let t = Instant::now();+ assert!(+ f.pool+ .claim(&spec("early", "t0"), BackendKind::Container, true)+ .await+ .is_none(),+ "no member has been ready yet: cold"+ );+ assert!(t.elapsed() < Duration::from_millis(100));+ until_ready(&f, &key, 1).await;+ assert!(+ f.pool+ .claim(&spec("first", "t1"), BackendKind::Container, true)+ .await+ .is_some()+ );+ // Now a member is starting and its boot is known: the next create+ // waits for it.+ let t = Instant::now();+ let got = f+ .pool+ .claim(&spec("mine", "tok"), BackendKind::Container, true)+ .await;+ assert!(got.is_some(), "claimed the member that was starting");+ assert!(+ t.elapsed() >= Duration::from_millis(80),+ "it waited for the boot"+ );+ until_ready(&f, &key, 1).await;+ }++ #[tokio::test]+ async fn a_shape_that_is_not_kept_warm_never_makes_a_create_wait() {+ let rt = FakeRuntime::default();+ rt.start_ms.store(300, SeqCst);+ let f = fixture(PoolSettings::default(), rt);+ let rt2: Arc<dyn VmmRuntime> = f.rt.clone();+ // An ordinary explicit pool (`pool.warm`): a member is starting.+ let key = f+ .pool+ .keep(&spec("one", "t"), BackendKind::Container, true, rt2, 1)+ .unwrap();+ let pool = f.pool.clone();+ let refill = tokio::spawn(async move { pool.refill(&key).await });+ tokio::time::sleep(Duration::from_millis(50)).await;+ let t = Instant::now();+ let got = f+ .pool+ .claim(&spec("mine", "tok"), BackendKind::Container, true)+ .await;+ assert!(got.is_none(), "no member ready: the create starts cold");+ assert!(+ t.elapsed() < Duration::from_millis(100),+ "and does not wait"+ );+ refill.await.unwrap();+ }++ #[tokio::test]+ async fn a_create_stops_waiting_when_the_member_fails_to_start() {+ let rt = FakeRuntime::default();+ rt.start_ms.store(100, SeqCst);+ rt.fail_starts.store(true, SeqCst);+ let f = fixture(PoolSettings::default(), rt);+ keep_warm_space(&f, &spec("one", "t")).await;+ let t = Instant::now();+ let got = f+ .pool+ .claim(&spec("mine", "tok"), BackendKind::Container, true)+ .await;+ assert!(got.is_none(), "cold start, never a refused create");+ assert!(+ t.elapsed() < Duration::from_secs(5),+ "released when the start failed"+ );+ }++ #[tokio::test]+ async fn the_warm_space_goes_back_when_nobody_creates() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ let rt: Arc<dyn VmmRuntime> = f.rt.clone();+ let key = f+ .pool+ .set_explicit_prepared(+ WARM_SOURCE,+ &spec("one", "t"),+ BackendKind::Container,+ true,+ rt,+ 1,+ Some(1),+ None,+ Some(Duration::from_millis(1)),+ Some(vec!["claude-code".into()]),+ )+ .unwrap();+ until_ready(&f, &key, 1).await;+ tokio::time::sleep(Duration::from_millis(20)).await;+ // Idle now: the next tick gives the member back, nothing refills it,+ // and a create never waits for one.+ f.pool.tick().await;+ tokio::time::sleep(Duration::from_millis(100)).await;+ f.pool.refill(&key).await;+ assert_eq!(+ f.rt.members().len(),+ 0,+ "an idle warm Space keeps no member"+ );+ let t = Instant::now();+ assert!(+ f.pool+ .claim(&spec("m", "t"), BackendKind::Container, true)+ .await+ .is_none()+ );+ assert!(t.elapsed() < Duration::from_millis(100));+ // Withdrawn (the experiment turned off): the source is gone.+ f.pool.retain_explicit(|src| src != WARM_SOURCE);+ f.pool.tick().await;+ assert_eq!(f.pool.shape_status(&key).map(|s| s.0).unwrap_or(0), 0);+ }++ #[tokio::test]+ async fn the_one_warm_space_is_not_held_to_the_percentage_but_to_the_free_floor() {+ // A small engine VM: 15 percent of it is less than one member.+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ *f.host.memory.lock().unwrap() = Memory {+ total_mb: 3000,+ available_mb: 2500,+ };+ let rt: Arc<dyn VmmRuntime> = f.rt.clone();+ let plain = f+ .pool+ .keep(+ &spec("a", "t").memory_mb(1024),+ BackendKind::Container,+ true,+ rt,+ 1,+ )+ .unwrap();+ f.pool.refill(&plain).await;+ assert_eq!(+ f.rt.members().len(),+ 0,+ "an ordinary pool respects the share"+ );+ let key = keep_warm_space(&f, &spec("one", "t")).await;+ until_ready(&f, &key, 1).await;+ assert_eq!(+ f.rt.members().len(),+ 1,+ "the warm Space gets its one member"+ );+ // Not below the floor: with 1.2 GB free it starts nothing.+ f.pool+ .claim(&spec("m", "tok"), BackendKind::Container, true)+ .await+ .unwrap();+ *f.host.memory.lock().unwrap() = Memory {+ total_mb: 3000,+ available_mb: 1200,+ };+ f.pool.refill(&key).await;+ assert_eq!(f.rt.members().len(), 0, "the free floor still holds");+ }++ #[tokio::test]+ async fn later_members_boot_from_the_filesystem_of_the_first_prepared_one() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ f.rt.can_save.store(true, SeqCst);+ let key = keep_warm_space(&f, &spec("one", "t")).await;+ until_ready(&f, &key, 1).await;+ assert_eq!(+ f.rt.saved.lock().unwrap().len(),+ 1,+ "kept once, before it was ready"+ );+ for i in 0..3 {+ assert!(+ f.pool+ .claim(+ &spec(&format!("s{i}"), &format!("t{i}")),+ BackendKind::Container,+ true+ )+ .await+ .is_some()+ );+ until_ready(&f, &key, 1).await;+ }+ let images = f.rt.images.lock().unwrap().clone();+ assert_eq!(images.len(), 4);+ assert!(+ !images[0].starts_with("local/prepared:"),+ "the first boots the image"+ );+ assert!(+ images[1..]+ .iter()+ .all(|i| i.starts_with("local/prepared:warm-")),+ "{images:?}"+ );+ // Prepared once; the others only booted.+ assert_eq!(f.host.prepared.lock().unwrap().len(), 1);+ assert_eq!(f.rt.saved.lock().unwrap().len(), 1);+ assert!(f.pool.records().iter().all(|r| r.prepared));+ // A daemon that stops keeps it for the next one; it goes with its+ // shape (forgotten after the policy's horizon without a create).+ f.pool.retain_explicit(|src| src != WARM_SOURCE);+ f.pool.drain().await;+ assert!(f.rt.dropped.lock().unwrap().is_empty());+ }++ #[tokio::test]+ async fn a_runtime_that_cannot_save_prepares_every_member() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ let key = keep_warm_space(&f, &spec("one", "t")).await;+ until_ready(&f, &key, 1).await;+ f.pool+ .claim(&spec("s", "t1"), BackendKind::Container, true)+ .await+ .unwrap();+ until_ready(&f, &key, 1).await;+ assert_eq!(+ f.host.prepared.lock().unwrap().len(),+ 2,+ "each member prepared"+ );+ assert!(+ f.rt.images+ .lock()+ .unwrap()+ .iter()+ .all(|i| !i.starts_with("local/prepared:"))+ );+ }++ #[tokio::test]+ async fn a_restarted_daemon_finds_the_filesystem_the_last_one_saved() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ f.rt.can_save.store(true, SeqCst);+ // What a previous daemon saved for this shape and these apps.+ let s0 = spec("one", "t");+ let key = shape_key(&s0, BackendKind::Container, true).unwrap();+ let tag = prepared_tag(+ &key,+ &["claude-code".to_string(), "openai-codex".to_string()],+ );+ f.rt.saved.lock().unwrap().push(tag.clone());+ let key2 = keep_warm_space(&f, &s0).await;+ assert_eq!(key, key2);+ until_ready(&f, &key, 1).await;+ let images = f.rt.images.lock().unwrap().clone();+ assert_eq!(images.len(), 1);+ assert!(+ images[0].starts_with("local/prepared:warm-"),+ "the first member booted from it: {images:?}"+ );+ assert!(+ f.host.prepared.lock().unwrap().is_empty(),+ "and was not prepared again"+ );+ }}diff --git a/libs/cua/crates/cua-daemon/tests/warm_pools_live.rs b/libs/cua/crates/cua-daemon/tests/warm_pools_live.rsnew file mode 100644index 0000000000..48a49bbbd8--- /dev/null+++ b/libs/cua/crates/cua-daemon/tests/warm_pools_live.rs@@ -0,0 +1,301 @@+//! Opt-in: the `warm-pools` experiment end to end on this computer's+//! container engine, measured the way Home uses it. A person asks for one+//! thing, then another a few seconds later, and so on: each ask is a create+//! of the Linux Space Home makes for an agent, then the coding app starts in+//! it. The test makes N such creates in a row, `CUA_WARM_LIVE_GAP_MS` apart,+//! and writes one JSON line for each:+//!+//! - `create_ms`: from the create call to a Space that answers with its own+//! token (the claim, when a warm Space was ready);+//! - `apps_ms`: the agent stack check at the start of the agent (a no-op+//! when the coding app is in the Space already);+//! - `agent_ms`: from the start of the coding app to an open session that+//! accepts a message (no model is asked anything);+//! - `total_ms`: the three together, from the create call to "the agent is+//! ready for the first message";+//! - `warm_ready`: warm Spaces the daemon held when the create began.+//!+//! `CUA_WARM_LIVE_MODE=warm` (default) turns the experiment on and waits for+//! the prepared warm Space first; `cold` leaves it off, so every create+//! starts a Space and installs the coding app, as without the experiment.+//!+//! ```text+//! CUA_HOME=$HOME/projects/.cua-work/warmpool/home CUA_WARM_LIVE=1 \+//! CUA_WARM_LIVE_OUT=/some/dir/warm.jsonl \+//! cargo test -p cua-daemon --test warm_pools_live -- --ignored --nocapture+//! ```+//!+//! Needs a throwaway `CUA_HOME` (never `~/.cua`). Image+//! `CUA_WARM_LIVE_IMAGE` (default: what Home creates), app+//! `CUA_WARM_LIVE_HARNESS` (default `claude-code`), count `CUA_WARM_LIVE_N`+//! (10), gap `CUA_WARM_LIVE_GAP_MS` (3000). Everything made is deleted on+//! every path. Run it under the no-UI sandbox profile on a desktop.++use std::sync::Arc;+use std::time::{Duration, Instant};++use cua_agents::{Agents, RunStatus, runs::RunOptions};+use cua_daemon::{CreateRequest, Runtime, RuntimeConfig, local::VmmLocal};+use cua_spacesd_client::{ConnectOptions, SpacesdClient, TransportPreference};++async fn client(url: &str, token: &str) -> SpacesdClient {+ let o = ConnectOptions::parse(url)+ .unwrap()+ .token(token)+ .transport(TransportPreference::Native)+ .probe(false);+ SpacesdClient::connect(o).await.unwrap()+}++fn env_u64(name: &str, default: u64) -> u64 {+ std::env::var(name)+ .ok()+ .and_then(|v| v.parse().ok())+ .unwrap_or(default)+}++/// Warm Spaces ready now and whether one of them has its apps installed.+fn pool_records(state: &std::path::Path) -> (usize, usize) {+ let mut all = 0;+ let mut prepared = 0;+ if let Ok(rd) = std::fs::read_dir(state.join(".pool")) {+ for e in rd.flatten() {+ if let Some(v) = std::fs::read(e.path())+ .ok()+ .and_then(|b| serde_json::from_slice::<serde_json::Value>(&b).ok())+ && v.get("secret").is_some()+ {+ all += 1;+ if v["prepared"].as_bool() == Some(true) {+ prepared += 1;+ }+ }+ }+ }+ (all, prepared)+}++#[tokio::test(flavor = "multi_thread")]+#[ignore = "runs Spaces on this computer's container engine; set CUA_WARM_LIVE=1 and a throwaway CUA_HOME"]+async fn sequential_creates_start_the_agent_from_a_warm_home() {+ if std::env::var("CUA_WARM_LIVE").as_deref() != Ok("1") {+ eprintln!("skipped: set CUA_WARM_LIVE=1");+ return;+ }+ let home = std::path::PathBuf::from(+ std::env::var_os("CUA_HOME").expect("set CUA_HOME to a throwaway home"),+ );+ let real = std::env::var_os("HOME").map(|h| std::path::PathBuf::from(h).join(".cua"));+ assert!(Some(&home) != real.as_ref(), "never the user's ~/.cua");+ if std::env::var_os("RUST_LOG").is_some() {+ let _ = tracing_subscriber::fmt()+ .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())+ .with_target(false)+ .try_init();+ }+ let warm = std::env::var("CUA_WARM_LIVE_MODE").as_deref() != Ok("cold");+ let image = std::env::var("CUA_WARM_LIVE_IMAGE")+ .unwrap_or_else(|_| cua_daemon::local::warm_pools_image());+ // The apps measured, one after the other against the same prepared+ // Space (comma separated; default: the four most used).+ let harnesses: Vec<String> = std::env::var("CUA_WARM_LIVE_HARNESS")+ .unwrap_or_else(|_| "openai-codex,hermes,claude-code,opencode".into())+ .split(',')+ .map(|h| h.trim().to_string())+ .filter(|h| !h.is_empty())+ .collect();+ let n = env_u64("CUA_WARM_LIVE_N", 10) as usize;+ let gap = Duration::from_millis(env_u64("CUA_WARM_LIVE_GAP_MS", 3000));+ let out_path = std::env::var_os("CUA_WARM_LIVE_OUT").map(std::path::PathBuf::from);+ // The experiment, for this process only.+ cua_vmm::experiments::set_enabled(cua_vmm::experiments::Gate::WarmPools, Some(warm));+ // The apps the warm Space is prepared for: the daemon's default, or+ // `CUA_WARM_POOLS_HARNESSES` as given.+ let state = home.join("sandboxes");+ let vmm = Arc::new(VmmLocal::default());+ let rt = Runtime::new(RuntimeConfig {+ state_dir: Some(state.clone()),+ vmm: Some(vmm.clone()),+ ..Default::default()+ })+ .unwrap();+ let (stop, rx) = tokio::sync::watch::channel(false);+ let pool_task = vmm+ .enable_pool(cua_sandbox_core::StateStore::new(state.clone()), rx)+ .expect("pool enabled once");+ let tag = format!("{:06x}", std::process::id() & 0xff_ffff);+ let names: Vec<String> = harnesses+ .iter()+ .flat_map(|h| {+ let (tag, h) = (tag.clone(), h.replace(['-', '_'], ""));+ (0..n).map(move |i| format!("wp-{tag}-{h}-{i}"))+ })+ .collect();+ let request = |name: &str| CreateRequest {+ location: Some("local".into()),+ name: Some(name.into()),+ image: image.clone(),+ ..Default::default()+ };+ let mut lines: Vec<serde_json::Value> = vec![];+ let outcome = futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(async {+ if warm {+ // The first warm Space is made and prepared (the image is+ // pulled and the app installed once); creates start after it.+ let began = Instant::now();+ let deadline = Instant::now() + Duration::from_secs(env_u64("CUA_WARM_LIVE_WAIT_S", 2400));+ loop {+ let ready = vmm.pool().map(|p| p.status().members).unwrap_or(0);+ if ready >= 1 && pool_records(&state).1 >= 1 {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "no prepared warm Space within the wait: {:?}",+ vmm.pool().map(|p| p.status())+ );+ tokio::time::sleep(Duration::from_millis(500)).await;+ }+ eprintln!(+ "prepared warm Space ready after {:.0}s",+ began.elapsed().as_secs_f64()+ );+ } else {+ // Without the experiment the image is pulled once, outside the+ // timing, so the numbers are starts and not downloads.+ // (`CUA_WARM_LIVE_NO_PULL=1` keeps it in: the first create of a+ // fresh install, with the runtime and the image still to fetch.)+ if std::env::var_os("CUA_WARM_LIVE_NO_PULL").is_none() {+ let warmup = format!("wp-{tag}-pull");+ rt.create(request(&warmup)).await.unwrap();+ rt.delete(&warmup).await.unwrap();+ }+ }+ for (k, name) in names.iter().enumerate() {+ let (harness, i) = (&harnesses[k / n], k % n);+ if i > 0 {+ tokio::time::sleep(gap).await;+ }+ let warm_ready = vmm.pool().map(|p| p.status().members).unwrap_or(0);+ let t0 = Instant::now();+ let made = rt.create(request(name)).await.unwrap();+ let create_ms = t0.elapsed().as_millis() as u64;+ let sb = rt.handle(&made.id).await.unwrap();+ let token = sb.env_token().expect("a token");+ let c = client(&made.endpoints["env"], &token).await;+ let agents = Agents::new(c).await.unwrap();+ // What every start of the app does first: the stack check.+ let h = cua_agents::harness::harness(harness).expect("a harness");+ let t1 = Instant::now();+ agents.prewarm(h, true).await.unwrap();+ agents.ensure(h.installs, |_| {}).await.unwrap();+ let apps_ms = t1.elapsed().as_millis() as u64;+ let t2 = Instant::now();+ // A credential no model ever sees: no turn is taken.+ let env = h.placeholder_env();+ let started = agents+ .start(+ harness,+ "",+ RunOptions {+ env,+ install: Some(true),+ skills: Some(false),+ idle_exit_secs: Some(900),+ ..Default::default()+ },+ )+ .await+ .unwrap();+ let mut ready_at = None;+ let mut failed: Option<String> = None;+ let deadline = Instant::now() + Duration::from_secs(900);+ while Instant::now() < deadline {+ if let Ok(st) = agents.status(&started.run_id).await {+ if st.status == RunStatus::Idle && st.accepts_message {+ ready_at = Some(t2.elapsed().as_millis() as u64);+ break;+ }+ if st.status == RunStatus::Failed {+ failed = Some(format!("{} {:?}", st.reason, st.error));+ break;+ }+ }+ tokio::time::sleep(Duration::from_millis(50)).await;+ }+ if let Some(why) = failed {+ let line = serde_json::json!({+ "harness": harness, "i": i, "error": why.chars().take(300).collect::<String>(),+ "create_ms": create_ms, "mode": if warm { "warm" } else { "cold" },+ });+ eprintln!("{line}");+ lines.push(line);+ let _ = agents.stop(&started.run_id).await;+ let _ = rt.delete(name).await;+ continue;+ }+ let agent_ms = ready_at.expect("the agent session opened within 15 minutes");+ let line = serde_json::json!({+ "harness": harness, "i": i, "mode": if warm { "warm" } else { "cold" }, "gap_ms": gap.as_millis() as u64,+ "warm_ready": warm_ready, "create_ms": create_ms, "apps_ms": apps_ms,+ "agent_ms": agent_ms, "total_ms": t0.elapsed().as_millis() as u64,+ });+ eprintln!("{line}");+ lines.push(line);+ if std::env::var_os("CUA_WARM_LIVE_PROFILE").is_some() {+ // The runner's own steps, in the guest's clock: where the start went.+ if let Ok(page) = agents.events(&started.run_id, 0, 500).await {+ let first = page.events.first().map(|e| e.ts_ms).unwrap_or(0);+ let steps: Vec<String> = page+ .events+ .iter()+ .map(|e| {+ format!(+ "{}+{}",+ e.raw.get("type").and_then(|v| v.as_str()).unwrap_or(e.kind),+ e.ts_ms.saturating_sub(first)+ )+ })+ .collect();+ eprintln!("profile {i}: {}", steps.join(" "));+ }+ }+ let _ = agents.stop(&started.run_id).await;+ if let Some(hold) = std::env::var("CUA_WARM_LIVE_HOLD_S").ok().and_then(|v| v.parse::<u64>().ok())+ && k + 1 == names.len()+ {+ // Left running for a person to look at (`docker exec`).+ eprintln!("holding {name} for {hold} s");+ tokio::time::sleep(Duration::from_secs(hold)).await;+ }+ let _ = rt.delete(name).await;+ }+ for name in &names {+ let _ = rt.delete(name).await;+ }+ }))+ .await;+ let _ = stop.send(true);+ let _ = pool_task.await;+ if let Some(p) = vmm.pool() {+ p.drain().await;+ }+ for name in &names {+ let _ = rt.delete(name).await;+ }+ if let Some(path) = out_path {+ let text: String = lines.iter().map(|l| format!("{l}\n")).collect();+ std::fs::write(path, text).unwrap();+ }+ if let Err(panic) = outcome {+ std::panic::resume_unwind(panic);+ }+ let left = vmm.container().await.unwrap().pool_members().await.unwrap();+ assert!(left.is_empty(), "members left behind: {left:?}");+ use cua_sandbox_core::LocalRuntime as _;+ let listed = vmm.list().await.unwrap();+ assert!(+ listed.iter().all(|s| !names.contains(&s.name)),+ "Spaces left behind"+ );+}diff --git a/libs/cua/crates/cua-spaces-app-core/parity/golden/experiments.json b/libs/cua/crates/cua-spaces-app-core/parity/golden/experiments.jsonindex 7c463b41bd..d29e331132 100644--- a/libs/cua/crates/cua-spaces-app-core/parity/golden/experiments.json+++ b/libs/cua/crates/cua-spaces-app-core/parity/golden/experiments.json@@ -107,7 +107,9 @@"- experiment:tcp_shares toggle: Public TCP shares {On|*Off}","- experiment:tcp_shares-note note: Share a game server or another TCP service in a Space at a public address.","- experiment:mobile toggle: Phone app {On|*Off}",- "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts."+ "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts.",+ "- experiment:warm_pools toggle: Warm start {On|*Off}",+ "- experiment:warm_pools-note note: Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."],"volumePage": null,"wizard": {@@ -257,7 +259,9 @@"- experiment:tcp_shares toggle: Public TCP shares {On|*Off}","- experiment:tcp_shares-note note: Share a game server or another TCP service in a Space at a public address.","- experiment:mobile toggle: Phone app {On|*Off}",- "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts."+ "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts.",+ "- experiment:warm_pools toggle: Warm start {On|*Off}",+ "- experiment:warm_pools-note note: Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."],"volumePage": "Volume","wizard": {@@ -406,7 +410,9 @@"- experiment:tcp_shares toggle: Public TCP shares {On|*Off}","- experiment:tcp_shares-note note: Share a game server or another TCP service in a Space at a public address.","- experiment:mobile toggle: Phone app {On|*Off}",- "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts."+ "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts.",+ "- experiment:warm_pools toggle: Warm start {On|*Off}",+ "- experiment:warm_pools-note note: Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."],"volumePage": "Volume","wizard": {@@ -539,7 +545,9 @@"- experiment:tcp_shares toggle: Public TCP shares {On|*Off}","- experiment:tcp_shares-note note: Share a game server or another TCP service in a Space at a public address.","- experiment:mobile toggle: Phone app {On|*Off}",- "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts."+ "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts.",+ "- experiment:warm_pools toggle: Warm start {On|*Off}",+ "- experiment:warm_pools-note note: Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."],"volumePage": "Volume","wizard": {@@ -672,7 +680,9 @@"- experiment:tcp_shares toggle: Public TCP shares {On|*Off}","- experiment:tcp_shares-note note: Share a game server or another TCP service in a Space at a public address.","- experiment:mobile toggle: Phone app {On|*Off}",- "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts."+ "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts.",+ "- experiment:warm_pools toggle: Warm start {On|*Off}",+ "- experiment:warm_pools-note note: Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."],"volumePage": "Volume","wizard": {@@ -805,7 +815,9 @@"- experiment:tcp_shares toggle: Public TCP shares {On|*Off}","- experiment:tcp_shares-note note: Share a game server or another TCP service in a Space at a public address.","- experiment:mobile toggle: Phone app {On|*Off}",- "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts."+ "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts.",+ "- experiment:warm_pools toggle: Warm start {On|*Off}",+ "- experiment:warm_pools-note note: Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."],"volumePage": "Volume","wizard": {@@ -943,7 +955,9 @@"- experiment:tcp_shares toggle: Public TCP shares {On|*Off}","- experiment:tcp_shares-note note: Share a game server or another TCP service in a Space at a public address.","- experiment:mobile toggle: Phone app {On|*Off}",- "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts."+ "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts.",+ "- experiment:warm_pools toggle: Warm start {On|*Off}",+ "- experiment:warm_pools-note note: Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."],"volumePage": null,"wizard": {@@ -1082,7 +1096,9 @@"- experiment:tcp_shares toggle: Public TCP shares {On|*Off}","- experiment:tcp_shares-note note: Share a game server or another TCP service in a Space at a public address.","- experiment:mobile toggle: Phone app {On|*Off}",- "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts."+ "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts.",+ "- experiment:warm_pools toggle: Warm start {On|*Off}",+ "- experiment:warm_pools-note note: Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."],"volumePage": null,"wizard": {@@ -1222,7 +1238,9 @@"- experiment:tcp_shares toggle: Public TCP shares {On|*Off}","- experiment:tcp_shares-note note: Share a game server or another TCP service in a Space at a public address.","- experiment:mobile toggle: Phone app {On|*Off}",- "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts."+ "- experiment:mobile-note note: Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts.",+ "- experiment:warm_pools toggle: Warm start {On|*Off}",+ "- experiment:warm_pools-note note: Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."],"volumePage": null,"wizard": {diff --git a/libs/cua/crates/cua-spaces-app-core/src/experiments.rs b/libs/cua/crates/cua-spaces-app-core/src/experiments.rsindex ddea46e01b..ede6d7478f 100644--- a/libs/cua/crates/cua-spaces-app-core/src/experiments.rs+++ b/libs/cua/crates/cua-spaces-app-core/src/experiments.rs@@ -23,6 +23,8 @@//! asks on a card before something leaves its Space (a push to GitHub),//! and the host acts only after its owner answers.//! - Public TCP shares ([`Experiment::TcpShares`]): enforced the same way.+//! - Warm start ([`Experiment::WarmPools`]): enforced the same way. The cua+//! daemon keeps one prepared warm Space ready on this computer.//!//! Off only hides the entry points. Nothing is undone: a mounted Volume//! stays mounted (and keeps syncing), a connected cloud stays connected@@ -76,6 +78,10 @@ pub struct Experiments {/// push events). Off unless turned on; the daemon enforces it (the switch/// is kept in the cua home).pub mobile: bool,+ /// Warm pools: one Linux Space kept ready, with its coding apps+ /// installed, so the next Create from Home starts at once. Off unless+ /// turned on; the daemon enforces it (the switch is kept in the cua home).+ pub warm_pools: bool,}impl Default for Experiments {@@ -91,6 +97,7 @@ impl Default for Experiments {ci_runners: false,tcp_shares: false,mobile: false,+ warm_pools: false,}}}@@ -119,10 +126,12 @@ pub enum Experiment {TcpShares,/// Mobile.Mobile,+ /// Warm start.+ WarmPools,}/// Every experiment, in the order Settings lists them.-pub const ALL: [Experiment; 10] = [+pub const ALL: [Experiment; 11] = [Experiment::CuaVolume,Experiment::YourCloud,Experiment::Sharing,@@ -133,6 +142,7 @@ pub const ALL: [Experiment; 10] = [Experiment::CiRunners,Experiment::TcpShares,Experiment::Mobile,+ Experiment::WarmPools,];impl Experiment {@@ -150,6 +160,7 @@ impl Experiment {Experiment::CiRunners => "ci_runners",Experiment::TcpShares => "tcp_shares",Experiment::Mobile => "mobile",+ Experiment::WarmPools => "warm_pools",}}@@ -171,6 +182,7 @@ impl Experiment {Experiment::CiRunners => "CI runners",Experiment::TcpShares => "Public TCP shares",Experiment::Mobile => "Phone app",+ Experiment::WarmPools => "Warm start",}}@@ -201,6 +213,9 @@ impl Experiment {Experiment::Mobile => {"Your computers answer the Cua Spaces phone app: start a task, follow the thread, review changes and get alerts."}+ Experiment::WarmPools => {+ "Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."+ }}}}@@ -219,6 +234,7 @@ impl Experiments {Experiment::CiRunners => self.ci_runners,Experiment::TcpShares => self.tcp_shares,Experiment::Mobile => self.mobile,+ Experiment::WarmPools => self.warm_pools,}}@@ -235,6 +251,7 @@ impl Experiments {Experiment::CiRunners => self.ci_runners = on,Experiment::TcpShares => self.tcp_shares = on,Experiment::Mobile => self.mobile = on,+ Experiment::WarmPools => self.warm_pools = on,}self}@@ -356,6 +373,8 @@ mod tests {"experiment:tcp_shares-note","experiment:mobile","experiment:mobile-note",+ "experiment:warm_pools",+ "experiment:warm_pools-note",]);for r in switches(&p) {@@ -403,7 +422,8 @@ mod tests {"ask_first","ci_runners","tcp_shares",- "mobile"+ "mobile",+ "warm_pools"]);}@@ -439,7 +459,7 @@ mod tests {let back = serde_json::to_value(x).unwrap();assert_eq!(back,- serde_json::json!({"cuaVolume": true, "yourCloud": true, "sharing": true, "webUi": false, "secureSpaces": false, "childSpaces": false, "askFirst": false, "ciRunners": false, "tcpShares": false, "mobile": false})+ serde_json::json!({"cuaVolume": true, "yourCloud": true, "sharing": true, "webUi": false, "secureSpaces": false, "childSpaces": false, "askFirst": false, "ciRunners": false, "tcpShares": false, "mobile": false, "warmPools": false}));// An explicit off is kept.let off: Experiments = serde_json::from_str(r#"{"yourCloud":false}"#).unwrap();diff --git a/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs b/libs/cua/crates/cua-spaces-app-core/src/telemetry.rsindex cc43beb58c..37a5461f08 100644--- a/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs+++ b/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs@@ -2252,7 +2252,8 @@ mod tests {"ask_first".into(),"ci_runners".into(),"tcp_shares".into(),- "mobile".into()+ "mobile".into(),+ "warm_pools".into()]}]);diff --git a/libs/cua/crates/cua-spaces-app-core/src/wizard.rs b/libs/cua/crates/cua-spaces-app-core/src/wizard.rsindex 24ce4525da..b1a4834d57 100644--- a/libs/cua/crates/cua-spaces-app-core/src/wizard.rs+++ b/libs/cua/crates/cua-spaces-app-core/src/wizard.rs@@ -4220,6 +4220,23 @@ pub(crate) fn arch_word(arch: Option<&str>) -> Option<&'static str> {}}+/// The catalog's Linux slim tier as a container (about 0.5 GB to pull), when+/// published: what the first Linux Space of Ask first is while the `warm-pools`+/// experiment is on, because that is the Space it keeps ready.+fn slim_linux_ref() -> Option<String> {+ catalog()+ .images+ .iter()+ .find(|i| {+ i.published+ && i.os == SpaceOs::Linux+ && i.tier == Some(ImageTier::Slim)+ && i.group == "canonical"+ && !i.image_ref.ends_with("-disk")+ })+ .map(|i| i.image_ref.clone())+}+/// The catalog's Omarchy (an amd64 VM), when published.fn omarchy_ref() -> Option<String> {catalog()@@ -4264,7 +4281,14 @@ pub fn first_space_offers(env: &WizardEnv, host_os: SpaceOs) -> Vec<FirstSpaceOf..env.clone()};let arch = arch_word(local.host_arch.as_deref());- let linux = view_on(&initial(&local), &local, host_os).image.image_ref;+ let mut linux = view_on(&initial(&local), &local, host_os).image.image_ref;+ // The Space the `warm-pools` experiment keeps ready is the slim tier: the+ // first Linux Space is made from it too, so Create claims it.+ if env.experiments.warm_pools+ && let Some(slim) = slim_linux_ref()+ {+ linux = slim;+ }let mut wanted: Vec<(&str, String)> = vec![("linux", linux)];if arch == Some("amd64")&& let Some(r) = omarchy_ref()@@ -4703,6 +4727,27 @@ mod tests {let offers = first_space_offers(&e, SpaceOs::Linux);let ids: Vec<&str> = offers.iter().map(|o| o.id.as_str()).collect();assert_eq!(ids, ["linux", "omarchy"]);+ // The Linux Space Home creates for a new agent is the full image;+ // with the `warm-pools` experiment on it is the slim tier, which the+ // daemon keeps ready at the same size (`cua_daemon::local::+ // warm_pools_image` asserts the same reference), so a change to+ // either must change both.+ assert_eq!(offers[0].image, "ghcr.io/trycua/linux:24.04");+ let warm = WizardEnv {+ experiments: crate::experiments::Experiments {+ warm_pools: true,+ ..e.experiments+ },+ ..e.clone()+ };+ let slim = first_space_offers(&warm, SpaceOs::Linux);+ assert_eq!(slim[0].image, "ghcr.io/trycua/linux:24.04-slim");+ assert_eq!(slim[0].os, SpaceOs::Linux);+ assert_eq!(+ (slim[0].args.cpus, slim[0].args.memory_mb),+ (Some(2), Some(4096))+ );+ assert_eq!(slim[0].args.image, "ghcr.io/trycua/linux:24.04-slim");let o = &offers[1];assert_eq!(o.image, "ghcr.io/trycua/omarchy:edge");assert_eq!(o.os, SpaceOs::Linux);diff --git a/libs/cua/crates/cua-spaces-ext/src/persistent/members.rs b/libs/cua/crates/cua-spaces-ext/src/persistent/members.rsindex cf05cf222d..199472f706 100644--- a/libs/cua/crates/cua-spaces-ext/src/persistent/members.rs+++ b/libs/cua/crates/cua-spaces-ext/src/persistent/members.rs@@ -831,10 +831,10 @@ impl Persistent {entry: &str,) -> Result<Delivered> {let picked_ms = super::phases::now_ms();- let (base, cold) = match self.current_turn(rec).await {- Some(Some(turn)) => (turn, false),- Some(None) => return Ok(Delivered::Busy),- None => (0, true),+ let (base, cold, known) = match self.current_turn_info(rec).await {+ Some((Some(turn), info)) => (turn, false, Some(info)),+ Some((None, _)) => return Ok(Delivered::Busy),+ None => (0, true, None),};let holder = Holder::new(&rec.name, rec.k(), rec.drive_epoch());let dispatch = || ClaimEntries {@@ -899,12 +899,19 @@ impl Persistent {"starting"}),);- let sent = Box::pin(self.send_session(rec, &framed, entry)).await;+ let sent = Box::pin(self.send_session(rec, &framed, entry, known)).await;if sent.is_err() {self.set_stage(rec, None);}match sent {Ok(d) => {+ super::phases::local(+ &d.run_id,+ "delivered",+ super::phases::now_ms().saturating_sub(posted_ms),+ d.started,+ &rec.harness,+ );if rec.brief.is_some() {let run = d.run_id.clone();let _ = self.update_in(rec, |r| r.briefed_run = Some(run));@@ -1004,13 +1011,19 @@ impl Persistent {/// The turn of `rec`'s run when it can take a message: `None` (no/// run), `Some(None)` (mid turn), `Some(Some(turn))`.async fn current_turn(&self, rec: &AgentRecord) -> Option<Option<u32>> {+ self.current_turn_info(rec).await.map(|(turn, _)| turn)+ }++ /// [`Self::current_turn`] with the status it was read from, so a+ /// delivery that follows does not read it again.+ async fn current_turn_info(&self, rec: &AgentRecord) -> Option<(Option<u32>, ag::RunInfo)> {let run = rec.run_id.as_ref()?;let agents = self.cached_agents(&rec.space).await.ok()?;match agents.status(run).await {- Ok(st) if st.status == RunStatus::Running => Some(None),+ Ok(st) if st.status == RunStatus::Running => Some((None, st)),// A durable run being re-driven: its turn goes on.- Ok(st) if Persistent::redriving(rec, st.status) => Some(None),- Ok(st) => Some(Some(st.turn)),+ Ok(st) if Persistent::redriving(rec, st.status) => Some((None, st)),+ Ok(st) => Some((Some(st.turn), st)),Err(_) => None,}}diff --git a/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs b/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rsindex 441017de65..118d214515 100644--- a/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs+++ b/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs@@ -492,8 +492,14 @@ pub(crate) struct Live {wake: wake::LiveWake,/// When a standby run may be started again after one that failed, by member.standby_retry: HashMap<String, std::time::Instant>,+ /// The credential hash an agent's run would start with now, by agent+ /// name, and when it was worked out.+ cred_hashes: HashMap<String, (std::time::Instant, Option<String>)>,}+/// How long a worked-out credential hash is reused.+const CRED_HASH_TTL: Duration = Duration::from_secs(5);+/// The persistent agents of one cua home. Cheap to clone.#[derive(Clone)]pub struct Persistent {@@ -552,6 +558,7 @@ impl Persistent {admins: Arc<std::sync::RwLock<std::collections::HashSet<String>>>,) -> Persistent {let dir = spaces.home_dir().join(DIR);+ phases::set_sink(&dir);Persistent {spaces,drive,@@ -1135,11 +1142,20 @@ impl Persistent {async fn send_from(&self, name: &str, text: &str, entry: &str) -> Result<Delivery> {let rec = self.get(name)?;self.not_external(&rec)?;- self.send_session(&rec, text, entry).await+ self.send_session(&rec, text, entry, None).await}- /// [`Self::send_from`] to the session `view` shows.- async fn send_session(&self, view: &AgentRecord, text: &str, entry: &str) -> Result<Delivery> {+ /// [`Self::send_from`] to the session `view` shows. `known` is the run's+ /// status as the caller read it a moment ago (the dispatcher just did):+ /// with it a warm delivery reads nothing more from the guest and is one+ /// write.+ async fn send_session(+ &self,+ view: &AgentRecord,+ text: &str,+ entry: &str,+ known: Option<ag::RunInfo>,+ ) -> Result<Delivery> {let rec = self.get(&view.name)?.session(view.k());let name = rec.name.as_str();if rec.paused {@@ -1147,7 +1163,11 @@ impl Persistent {}if let Some(run) = &rec.run_id {let agents = self.agents_for(&rec.space).await?;- match agents.status(run).await {+ let status = match known {+ Some(st) => Ok(st),+ None => agents.status(run).await,+ };+ match status {Ok(st) if st.status == RunStatus::Running => {return Err(Error::Agent(format!("{name} is in the middle of a turn; try again when it ends"@@ -1203,7 +1223,7 @@ impl Persistent {self.take_lease(&rec).await?;}agents- .send_with_model(run, text, vec![], rec.model.as_deref())+ .send_after(run, text, vec![], &st, rec.model.as_deref()).await?;return Ok(Delivery {run_id: run.clone(),@@ -1251,16 +1271,39 @@ impl Persistent {run: &str,st: &ag::RunInfo,) -> bool {- if let Some(started) = st.meta.as_ref().and_then(|m| m.cred_hash.as_deref())- && let Ok((env, _)) = cua_spaces::agents::run_env(- &rec.harness,- &rec.env,- &rec.env_from_host,- rec.base_url.is_some(),- )- && ag::runs::credential_hash(&rec.harness, &env) != started- {- return true;+ if let Some(started) = st.meta.as_ref().and_then(|m| m.cred_hash.as_deref()) {+ // Reading the keys costs a keychain round trip, and a message+ // does not need them fresher than a few seconds.+ let cached = self+ .live+ .lock()+ .await+ .cred_hashes+ .get(&rec.name)+ .filter(|(at, _)| at.elapsed() < CRED_HASH_TTL)+ .map(|(_, h)| h.clone());+ let now = match cached {+ Some(h) => h,+ None => {+ let h = cua_spaces::agents::run_env(+ &rec.harness,+ &rec.env,+ &rec.env_from_host,+ rec.base_url.is_some(),+ )+ .ok()+ .map(|(env, _)| ag::runs::credential_hash(&rec.harness, &env));+ self.live+ .lock()+ .await+ .cred_hashes+ .insert(rec.name.clone(), (Instant::now(), h.clone()));+ h+ }+ };+ if now.is_some_and(|h| h != started) {+ return true;+ }}if st.stop_reason.as_deref() == Some("error")&& let Ok(r) = agents.result(run).awaitdiff --git a/libs/cua/crates/cua-spaces-ext/src/persistent/phases.rs b/libs/cua/crates/cua-spaces-ext/src/persistent/phases.rsindex 08987ee9cb..b0bd57bfef 100644--- a/libs/cua/crates/cua-spaces-ext/src/persistent/phases.rs+++ b/libs/cua/crates/cua-spaces-ext/src/persistent/phases.rs@@ -141,8 +141,62 @@ pub(crate) struct Join {pub posted_ms: u64,}+// --- the local record ----------------------------------------------------------++fn sink() -> &'static Mutex<Option<std::path::PathBuf>> {+ static S: OnceLock<Mutex<Option<std::path::PathBuf>>> = OnceLock::new();+ S.get_or_init(Default::default)+}++/// Keeps every step this process reports in `<dir>/phases.jsonl`, one JSON+/// line each (`ts_ms`, `run`, `phase`, `ms` since the message, `cold`,+/// `harness`), so a bench or a person can read how long a join took without+/// telemetry (which is anonymous and goes elsewhere). The newest lines+/// matter: the file is cut to its last 2000 lines when it passes 1 MiB. Only+/// while the `warm-pools` experiment is on.+pub(crate) fn set_sink(dir: &Path) {+ if let Ok(mut s) = sink().lock() {+ *s = Some(dir.join("phases.jsonl"));+ }+}++fn record(run: &str, phase: &str, ms: u64, cold: bool, harness: &str) {+ if !cua_spaces::experiments::enabled(cua_spaces::experiments::Gate::WarmPools) {+ return;+ }+ let Some(path) = sink().lock().ok().and_then(|s| s.clone()) else {+ return;+ };+ let line = serde_json::json!({+ "ts_ms": now_ms(), "run": run, "phase": phase, "ms": ms, "cold": cold, "harness": harness,+ });+ use std::io::Write as _;+ if let Ok(meta) = std::fs::metadata(&path)+ && meta.len() > 1 << 20+ && let Ok(text) = std::fs::read_to_string(&path)+ {+ let keep: Vec<&str> = text.lines().rev().take(2000).collect();+ let kept = keep.into_iter().rev().collect::<Vec<_>>().join("\n");+ let _ = std::fs::write(&path, format!("{kept}\n"));+ }+ if let Ok(mut f) = std::fs::OpenOptions::new()+ .create(true)+ .append(true)+ .open(&path)+ {+ let _ = writeln!(f, "{line}");+ }+}++/// A step that only the local record keeps (not part of the telemetry's+/// fixed words): `ms` since the message.+pub(crate) fn local(run: &str, phase: &str, ms: u64, cold: bool, harness: &str) {+ record(run, phase, ms, cold, harness);+}+impl Join {fn send(&self, phase: &str, ms: u64, tool: &str, failure: &str) {+ record(&self.run_id, phase, ms, self.cold, &self.harness);cua_telemetry::global().agent_join_phase(&self.run_id,phase,@@ -222,7 +276,11 @@ pub(crate) fn follow(let mut seen_first_token = false;let mut installed: HashSet<String> = HashSet::new();let mut reported: HashSet<&'static str> = HashSet::new();- let mut every = Duration::from_millis(200);+ // While the first words are awaited the events are read every 25 ms+ // (a local read): the step is stamped when this host sees it, so a+ // slow poll would be a slow step. Only with `warm-pools` on.+ let quick = cua_spaces::experiments::enabled(cua_spaces::experiments::Gate::WarmPools);+ let mut every = Duration::from_millis(if quick { 25 } else { 200 });while started.elapsed() < FOLLOW_FOR {let page = match agents.events(&join.run_id, cursor, 200).await {Ok(p) => p,@@ -289,6 +347,15 @@ pub(crate) fn follow("update" if turn_open && !seen_first_token && ev.kind == "message" => {seen_first_token = true;join.send("first_token", ms, "none", "none");+ // When this host read it: the host's own clock only.+ record(+ &join.run_id,+ "first_token_seen",+ now_ms().saturating_sub(join.posted_ms),+ join.cold,+ &join.harness,+ );+ every = Duration::from_millis(200);}"turn_ended" if turn_open => {join.send("turn_ended", ms, "none", "none");@@ -307,6 +374,9 @@ pub(crate) fn follow(tokio::time::sleep(every).await;if started.elapsed() > Duration::from_secs(30) {every = Duration::from_secs(1);+ } else if quick && !seen_first_token && started.elapsed() > Duration::from_secs(8) {+ // A slow start (a cold install): no need to read this often.+ every = Duration::from_millis(200);}}}@@ -338,4 +408,59 @@ mod tests {assert_eq!(failure_word("npm install failed"), "install");assert_eq!(failure_word("boom"), "other");}++ /// The local record of a join exists only with the `warm-pools`+ /// experiment on: off, nothing is written.+ #[test]+ fn the_local_record_is_kept_only_with_warm_pools_on() {+ let dir = tempfile::tempdir().unwrap();+ set_sink(dir.path());+ let file = dir.path().join("phases.jsonl");+ {+ let _off =+ cua_spaces::experiments::hold(cua_spaces::experiments::Gate::WarmPools, false);+ local("run-a", "delivered", 12, false, "claude-code");+ assert!(!file.exists(), "off: no file");+ }+ {+ let _on = cua_spaces::experiments::hold(cua_spaces::experiments::Gate::WarmPools, true);+ local("run-a", "delivered", 12, false, "claude-code");+ local("run-a", "first_token_seen", 840, false, "claude-code");+ }+ let lines: Vec<serde_json::Value> = std::fs::read_to_string(&file)+ .unwrap()+ .lines()+ .map(|l| serde_json::from_str(l).unwrap())+ .collect();+ assert_eq!(lines.len(), 2);+ assert_eq!(lines[0]["phase"], "delivered");+ assert_eq!(lines[0]["ms"], 12);+ assert_eq!(lines[1]["phase"], "first_token_seen");+ assert_eq!(lines[1]["harness"], "claude-code");+ assert!(lines[1]["ts_ms"].as_u64().unwrap() > 0);+ }++ /// Every harness id cua-agents knows, and its short names, count as+ /// themselves in telemetry: an id outside the schema's list reports+ /// `other`, which hid which agents people run (5217 events in 14 days).+ #[test]+ fn every_harness_and_its_short_names_are_reported_as_themselves() {+ for id in cua_agents::harness::ids() {+ assert_eq!(cua_telemetry::events::harness(id), id, "{id}");+ }+ for (short, id) in [+ ("codex", "openai-codex"),+ ("claude", "claude-code"),+ ("Codex", "openai-codex"),+ (" claude-code ", "claude-code"),+ ] {+ assert_eq!(cua_telemetry::events::harness(short), id, "{short}");+ assert_eq!(+ cua_agents::harness::harness(short.trim().to_ascii_lowercase().as_str())+ .map(|h| h.id),+ Some(id)+ );+ }+ assert_eq!(cua_telemetry::events::harness("not-a-harness"), "other");+ }}diff --git a/libs/cua/crates/cua-spaces-ffi/src/app_core.rs b/libs/cua/crates/cua-spaces-ffi/src/app_core.rsindex 0131a7b2ff..d6c711c468 100644--- a/libs/cua/crates/cua-spaces-ffi/src/app_core.rs+++ b/libs/cua/crates/cua-spaces-ffi/src/app_core.rs@@ -2205,6 +2205,9 @@ fn sync_experiments_from_home(if let Some(on) = switch(home, Gate::Mobile) {experiments.mobile = on;}+ if let Some(on) = switch(home, Gate::WarmPools) {+ experiments.warm_pools = on;+ }experiments}@@ -2220,7 +2223,8 @@ fn sync_experiments_to_home(set_switch(home, Gate::AskFirst, experiments.ask_first)?;set_switch(home, Gate::CiRunners, experiments.ci_runners)?;set_switch(home, Gate::TcpShares, experiments.tcp_shares)?;- set_switch(home, Gate::Mobile, experiments.mobile)+ set_switch(home, Gate::Mobile, experiments.mobile)?;+ set_switch(home, Gate::WarmPools, experiments.warm_pools)}/// Writes the settings file, and keeps the experiments the daemon enforces@@ -2277,7 +2281,8 @@ mod experiment_sync_tests {|| n == "ask-first"|| n == "ci-runners"|| n == "tcp-shares"- || n == "mobile"),+ || n == "mobile"+ || n == "warm-pools"),"{kept:?}");}diff --git a/libs/cua/crates/cua-spaces-ffi/src/app_core_types.rs b/libs/cua/crates/cua-spaces-ffi/src/app_core_types.rsindex a88095c8e5..c5ada63c5e 100644--- a/libs/cua/crates/cua-spaces-ffi/src/app_core_types.rs+++ b/libs/cua/crates/cua-spaces-ffi/src/app_core_types.rs@@ -6991,6 +6991,10 @@ pub struct AppExperiments {/// push events). Off unless turned on; the daemon enforces it (the switch/// is kept in the cua home).pub mobile: bool,+ /// Warm pools: one Linux Space kept ready, with its coding apps+ /// installed, so the next Create from Home starts at once. Off unless+ /// turned on; the daemon enforces it (the switch is kept in the cua home).+ pub warm_pools: bool,}/// A runtime the host found for one kind (the SDK's `RuntimeDetected`).diff --git a/libs/cua/crates/cua-telemetry/src/events.rs b/libs/cua/crates/cua-telemetry/src/events.rsindex 7a1cd47ba5..bb2c6a3f19 100644--- a/libs/cua/crates/cua-telemetry/src/events.rs+++ b/libs/cua/crates/cua-telemetry/src/events.rs@@ -225,8 +225,17 @@ pub fn teleport_app(id: &str) -> String {}}-/// A cua-agents harness id, else `other`.+/// A cua-agents harness id, else `other`. The short names a person or an+/// agent types (`codex`, `claude`: `cua_agents::harness::harness` accepts+/// them) and ids in another case count as the harness they name; before,+/// they all reported `other`.pub fn harness(id: &str) -> &'static str {+ let id = id.trim().to_ascii_lowercase();+ let id = match id.as_str() {+ "codex" => "openai-codex",+ "claude" => "claude-code",+ other => other,+ };pick(schema::HARNESSES, id, "other")}diff --git a/libs/cua/crates/cua-telemetry/src/schema.rs b/libs/cua/crates/cua-telemetry/src/schema.rsindex b66fbc5dfc..490a95cf93 100644--- a/libs/cua/crates/cua-telemetry/src/schema.rs+++ b/libs/cua/crates/cua-telemetry/src/schema.rs@@ -717,6 +717,7 @@ pub const EXPERIMENTS: &[&str] = &["ci_runners","tcp_shares","mobile",+ "warm_pools",];/// The Spaces app's interface language, as a fixed set: the languages it/// ships or plans to; any other language reads `other`. No region, no text.@@ -1815,7 +1816,7 @@ pub const EVENTS: &[EventSpec] = &[p("experiments_on",Kind::Set(EXPERIMENTS),- "The Spaces app's experiments turned on in Settings, Experiments on this install (cua_volume, your_cloud, sharing, secure_spaces, child_spaces, ask_first, ci_runners, tcp_shares, mobile, joined with +), or none.",+ "The Spaces app's experiments turned on in Settings, Experiments on this install (cua_volume, your_cloud, sharing, secure_spaces, child_spaces, ask_first, ci_runners, tcp_shares, mobile, warm_pools, joined with +), or none.",),p("ui_locale",@@ -2250,7 +2251,7 @@ pub const EVENTS: &[EventSpec] = &[p("experiment",Kind::Enum(EXPERIMENTS),- "Which experiment: cua_volume, your_cloud, sharing, secure_spaces, child_spaces, ask_first, ci_runners, tcp_shares or mobile.",+ "Which experiment: cua_volume, your_cloud, sharing, secure_spaces, child_spaces, ask_first, ci_runners, tcp_shares, mobile or warm_pools.",),],purpose: "Which experiments people try, and which they turn off again.",diff --git a/libs/cua/crates/cua-vmm/src/conformance.rs b/libs/cua/crates/cua-vmm/src/conformance.rsindex 3412844eb7..0cb08d9145 100644--- a/libs/cua/crates/cua-vmm/src/conformance.rs+++ b/libs/cua/crates/cua-vmm/src/conformance.rs@@ -70,11 +70,16 @@ pub enum Case {/// Deleting leaves nothing: no Space, no process, no container, no/// network.Teardown,+ /// A warm claim: a Space made from a member the daemon kept ready+ /// answers with its own token, nothing else opens it (not the member's+ /// pool secret), and it behaves as a cold one; a runtime without warm+ /// claims says so and starts cold.+ Warm,}impl Case {/// Every case, in the order the live run takes them.- pub const ALL: [Case; 9] = [+ pub const ALL: [Case; 10] = [Case::Ready,Case::Policy,Case::Dns,@@ -83,6 +88,7 @@ impl Case {Case::Children,Case::Agent,Case::Lifecycle,+ Case::Warm,Case::Teardown,];@@ -98,6 +104,7 @@ impl Case {Case::Ready => "ready",Case::Lifecycle => "lifecycle",Case::Teardown => "teardown",+ Case::Warm => "warm",}}@@ -117,6 +124,9 @@ impl Case {Case::Ready => "time from create to a Space that answers",Case::Lifecycle => "stop and resume, and the declared pause, checkpoint and fork",Case::Teardown => "delete leaves no Space, process, container or network",+ Case::Warm => {+ "a Space claimed from a warm member answers with its own token and only that one, or the runtime says it starts cold"+ }}}}@@ -165,6 +175,15 @@ pub struct Capabilities {/// Guests the host cannot reach by address need the runtime's own/// forward (`true`), else the guest is dialed at its own address.pub forwarded: bool,+ /// A member kept ready can be claimed by renaming it to a Space and+ /// giving it that Space's token ([`crate::Runtime::claim`]). Without it+ /// a pool is refused and Spaces start cold.+ pub claim: bool,+ /// A member answers calls made with its pool secret before it is+ /// claimed (a container does), so its coding apps can be installed+ /// ahead (the `warm-pools` experiment). Without it they go in after the+ /// claim, as without a pool.+ pub prepare: bool,}/// What `backend` declares. No wildcard: a new backend does not build@@ -182,6 +201,8 @@ pub fn capabilities(backend: BackendKind) -> Capabilities {kvm: false,nested_virt: false,forwarded: true,+ claim: true,+ prepare: true,},BackendKind::Qemu => Capabilities {guests: &["linux", "windows", "android", "macos"],@@ -192,6 +213,8 @@ pub fn capabilities(backend: BackendKind) -> Capabilities {kvm: false,nested_virt: false,forwarded: true,+ claim: true,+ prepare: false,},BackendKind::QemuDocker => Capabilities {guests: &[],@@ -202,6 +225,8 @@ pub fn capabilities(backend: BackendKind) -> Capabilities {kvm: false,nested_virt: false,forwarded: true,+ claim: false,+ prepare: false,},BackendKind::Lume => Capabilities {guests: &["macos"],@@ -212,6 +237,8 @@ pub fn capabilities(backend: BackendKind) -> Capabilities {kvm: false,nested_virt: false,forwarded: false,+ claim: true,+ prepare: false,},BackendKind::Firecracker => Capabilities {guests: &["linux"],@@ -222,6 +249,8 @@ pub fn capabilities(backend: BackendKind) -> Capabilities {kvm: true,nested_virt: true,forwarded: false,+ claim: true,+ prepare: false,},// Apple's runtime helper owns the VM: no pause, no snapshots.BackendKind::Containerization => Capabilities {@@ -233,6 +262,8 @@ pub fn capabilities(backend: BackendKind) -> Capabilities {kvm: false,nested_virt: false,forwarded: false,+ claim: true,+ prepare: false,},BackendKind::Vz => Capabilities {guests: &["linux"],@@ -243,6 +274,8 @@ pub fn capabilities(backend: BackendKind) -> Capabilities {kvm: false,nested_virt: false,forwarded: false,+ claim: true,+ prepare: false,},}}@@ -275,6 +308,18 @@ pub struct Entry {const LINUX_CASES: &[Case] = &Case::ALL;const VM_CASES: &[Case] = &[+ Case::Ready,+ Case::Policy,+ Case::Dns,+ Case::Download,+ Case::Lifecycle,+ Case::Warm,+ Case::Teardown,+];+/// A macOS VM is claimed only from a pool kept warm on purpose, and a Mac+/// runs two of them: its warm start is covered by the Lume tests, not by a+/// third guest in the live run.+const MACOS_CASES: &[Case] = &[Case::Ready,Case::Policy,Case::Dns,@@ -382,7 +427,7 @@ pub fn entries() -> &'static [Entry] {hosts: &["macos"],guest: "macos",needs: "an Apple silicon Mac with Lume and a macOS image",- cases: VM_CASES,+ cases: MACOS_CASES,not_runnable: None,},Entry {@@ -424,6 +469,9 @@ pub fn expect(entry: &Entry, case: Case, os: &str) -> Expect {}match case {Case::Policy => policy_expectation(entry.backend, os),+ Case::Warm if !capabilities(entry.backend).claim => Expect::Refused(+ "this runtime has no warm claims: a pool is refused and its Spaces start cold",+ ),_ => Expect::Pass,}}@@ -473,6 +521,8 @@ pub fn entries_json() -> Value {"fork": c.fork,"kvm": c.kvm,"nested_virt": c.nested_virt,+ "claim": c.claim,+ "prepare": c.prepare,},"hosts": hosts,})@@ -485,7 +535,7 @@ pub fn entries_json() -> Value {/// docs; a test keeps them in step).pub fn promises_table_markdown() -> String {let mut out = String::from(- "| Runtime | Guest | Default network | Pause | Checkpoint and fork | KVM |\n|---|---|---|---|---|---|\n",+ "| Runtime | Guest | Default network | Pause | Checkpoint and fork | KVM | Warm start |\n|---|---|---|---|---|---|---|\n",);for e in entries().iter().filter(|e| e.not_runnable.is_none()) {let c = capabilities(e.backend);@@ -501,14 +551,20 @@ pub fn promises_table_markdown() -> String {}).collect();let yes = |b: bool| if b { "yes" } else { "no" };+ let warm = match (c.claim, c.prepare) {+ (true, true) => "claim, apps ready ahead",+ (true, false) => "claim",+ _ => "no",+ };out.push_str(&format!(- "| `{}` | {} | {} | {} | {} | {} |\n",+ "| `{}` | {} | {} | {} | {} | {} | {} |\n",e.id,e.guest,net.join(", "),yes(c.pause),yes(c.checkpoint && c.fork),yes(c.kvm),+ warm,));}out@@ -709,4 +765,35 @@ mod tests {}}}++ /// A runtime that cannot claim says so in the matrix and is not asked+ /// for a warm case; one that can, is. Every runnable Linux guest takes+ /// the warm case; the macOS VM's is covered by its own tests.+ #[test]+ fn the_warm_case_follows_what_the_runtime_declares() {+ for e in entries() {+ let c = capabilities(e.backend);+ // A member that can be prepared can be claimed.+ assert!(!c.prepare || c.claim, "{}", e.id);+ if e.cases.contains(&Case::Warm) {+ assert!(+ c.claim,+ "{} is asked for warm claims but declares none",+ e.id+ );+ assert_eq!(expect(e, Case::Warm, "linux"), Expect::Pass, "{}", e.id);+ }+ }+ assert!(entry("builtin-gvisor").unwrap().cases.contains(&Case::Warm));+ assert!(capabilities(BackendKind::Container).prepare);+ assert!(+ !capabilities(BackendKind::Firecracker).prepare,+ "paused members are unreachable until claimed"+ );+ assert!(matches!(+ expect(entry("lume-macos-vm").unwrap(), Case::Warm, "macos"),+ Expect::Refused(_)+ ));+ assert!(!capabilities(BackendKind::QemuDocker).claim);+ }}diff --git a/libs/cua/crates/cua-vmm/src/container/mod.rs b/libs/cua/crates/cua-vmm/src/container/mod.rsindex c995defd4e..2123d6e227 100644--- a/libs/cua/crates/cua-vmm/src/container/mod.rs+++ b/libs/cua/crates/cua-vmm/src/container/mod.rs@@ -54,6 +54,9 @@ pub const BASE_REPO: &str = "cua-vmm/base";/// Engine repository of checkpoints and of the intermediate images this/// backend commits for port rebinds and forks.pub const CKPT_REPO: &str = "cua-vmm/checkpoint";+/// Engine repository of the filesystems of prepared warm-pool members+/// ([`Runtime::save_prepared`]): the image a prepared member starts from.+pub const PREPARED_REPO: &str = "cua-vmm/prepared";/// On every container, image and volume the SDK creates: `true`. Garbage/// collection only ever touches objects that carry it (plus the images the@@ -101,6 +104,7 @@ pub fn image_kind(repo: &str, tag: &str) -> &'static str {CKPT_REPO if tag.ends_with("-rebind") || tag.starts_with("fork-") => "intermediate",CKPT_REPO => "checkpoint",BASE_REPO => "base",+ PREPARED_REPO => "prepared",_ => "build-image",}}@@ -1698,6 +1702,54 @@ impl Runtime for ContainerRuntime {))}+ async fn save_prepared(&self, name: &str, tag: &str) -> Result<Option<ImageSource>> {+ validate_name(tag)?;+ let i = self+ .inspect(name)+ .await?+ .ok_or_else(|| VmmError::NotFound(name.into()))?;+ // The root filesystem as it is now. gVisor keeps its writes in its+ // own overlay, which a commit cannot see: the rootfs is streamed out.+ if Self::isolation_from(&i) == Isolation::Gvisor && Self::status_from(&i) == Status::Running+ {+ self.export_import_as(name, PREPARED_REPO, tag, config_changes(i.config.as_ref()))+ .await?;+ } else {+ self.commit(name, PREPARED_REPO, tag, true).await?;+ }+ Ok(Some(ImageSource::oci(format!("{PREPARED_REPO}:{tag}"))))+ }++ async fn find_prepared(&self, tag: &str) -> Result<Option<ImageSource>> {+ let image = format!("{PREPARED_REPO}:{tag}");+ Ok(self+ .docker+ .inspect_image(&image)+ .await+ .ok()+ .map(|_| ImageSource::oci(image)))+ }++ async fn drop_prepared(&self, tag: &str) -> Result<()> {+ let image = format!("{PREPARED_REPO}:{tag}");+ match self+ .docker+ .remove_image(+ &image,+ Some(bollard::query_parameters::RemoveImageOptions {+ force: true,+ ..Default::default()+ }),+ None,+ )+ .await+ {+ Ok(_) => Ok(()),+ Err(e) if is_not_found(&e) => Ok(()),+ Err(e) => Err(engine_err(e)),+ }+ }+async fn delete_checkpoint(&self, checkpoint: &str) -> Result<()> {egress::forget(checkpoint);if let Some(rec) = snapshot::read_record("checkpoints", checkpoint) {diff --git a/libs/cua/crates/cua-vmm/src/experiments.rs b/libs/cua/crates/cua-vmm/src/experiments.rsindex 32575f1b20..571919abf1 100644--- a/libs/cua/crates/cua-vmm/src/experiments.rs+++ b/libs/cua/crates/cua-vmm/src/experiments.rs@@ -39,6 +39,10 @@ pub enum Gate {/// Windows on KVM: Windows Spaces from a Mac coordinator on a Cua-managed/// AWS x86 pool host with nested KVM.WindowsKvm,+ /// Warm pools: one prepared Linux Space is kept ready on this computer+ /// (its coding apps already installed), so the next Create from Home+ /// claims it instead of making one.+ WarmPools,}impl Gate {@@ -51,6 +55,7 @@ impl Gate {Gate::TcpShares,Gate::Mobile,Gate::WindowsKvm,+ Gate::WarmPools,];/// The stable name (`secure-spaces`): the file name, the CLI word.@@ -63,6 +68,7 @@ impl Gate {Gate::TcpShares => "tcp-shares",Gate::Mobile => "mobile",Gate::WindowsKvm => "windows-kvm",+ Gate::WarmPools => "warm-pools",}}@@ -76,6 +82,7 @@ impl Gate {Gate::TcpShares => "tcp_shares",Gate::Mobile => "mobile",Gate::WindowsKvm => "windows_kvm",+ Gate::WarmPools => "warm_pools",}}@@ -95,6 +102,7 @@ impl Gate {Gate::TcpShares => "Public TCP shares",Gate::Mobile => "Phone app",Gate::WindowsKvm => "Windows on KVM",+ Gate::WarmPools => "Warm start",}}@@ -122,6 +130,9 @@ impl Gate {Gate::WindowsKvm => {"Windows Spaces from a Mac run on a Cua-managed cloud host with nested virtualization."}+ Gate::WarmPools => {+ "Keep one Linux Space ready on this computer, with its coding apps installed, so the next Create from Home starts at once."+ }}}@@ -321,5 +332,8 @@ mod tests {assert_eq!(Gate::parse("nope"), None);assert_eq!(Gate::parse("Mobile"), Some(Gate::Mobile));assert_eq!(Gate::Mobile.id(), "mobile");+ assert_eq!(Gate::parse("warm_pools"), Some(Gate::WarmPools));+ assert_eq!(Gate::WarmPools.name(), "warm-pools");+ assert_eq!(Gate::WarmPools.id(), "warm_pools");}}diff --git a/libs/cua/crates/cua-vmm/src/managed.rs b/libs/cua/crates/cua-vmm/src/managed.rsindex b266dae354..2b620b31de 100644--- a/libs/cua/crates/cua-vmm/src/managed.rs+++ b/libs/cua/crates/cua-vmm/src/managed.rs@@ -1685,6 +1685,21 @@ mod engine_impl {self.engine().await?.claim(member, new_name, identity).await}+ async fn save_prepared(&self, name: &str, tag: &str) -> Result<Option<ImageSource>> {+ let _busy = self.busy();+ self.engine().await?.save_prepared(name, tag).await+ }++ async fn find_prepared(&self, tag: &str) -> Result<Option<ImageSource>> {+ let _busy = self.busy();+ self.engine().await?.find_prepared(tag).await+ }++ async fn drop_prepared(&self, tag: &str) -> Result<()> {+ let _busy = self.busy();+ self.engine().await?.drop_prepared(tag).await+ }+async fn stop(&self, name: &str) -> Result<()> {{let _busy = self.busy();diff --git a/libs/cua/crates/cua-vmm/src/runtime.rs b/libs/cua/crates/cua-vmm/src/runtime.rsindex 9904ecd3a1..69ed5aabe7 100644--- a/libs/cua/crates/cua-vmm/src/runtime.rs+++ b/libs/cua/crates/cua-vmm/src/runtime.rs@@ -85,6 +85,31 @@ pub trait Runtime: Send + Sync {})}+ /// Saves the filesystem of the running instance `name` as an image that+ /// instances start from instead of the one they were made from (what is+ /// installed in it stays), named `tag`, and returns it. Used by the warm+ /// pool so a member prepared once is not prepared again: the next one+ /// boots from this. `None`: this backend has no such image (its members+ /// are prepared one by one, or not at all).+ async fn save_prepared(&self, name: &str, tag: &str) -> Result<Option<ImageSource>> {+ let _ = (name, tag);+ Ok(None)+ }++ /// The image [`Runtime::save_prepared`] saved as `tag`, when the backend+ /// still has it (a daemon that restarted finds what the last one saved).+ async fn find_prepared(&self, tag: &str) -> Result<Option<ImageSource>> {+ let _ = tag;+ Ok(None)+ }++ /// Deletes an image saved by [`Runtime::save_prepared`] (not an error+ /// when it is gone).+ async fn drop_prepared(&self, tag: &str) -> Result<()> {+ let _ = tag;+ Ok(())+ }+/// Create a new stopped instance `new_name` from a stopped base, checkpoint/// or instance `source`, copy-on-write where the backend can.async fn fork(&self, source: &str, new_name: &str) -> Result<()>;diff --git a/libs/cua/scripts/conformance/entries.json b/libs/cua/scripts/conformance/entries.jsonindex 666e36b91f..94d88e5f5d 100644--- a/libs/cua/scripts/conformance/entries.json+++ b/libs/cua/scripts/conformance/entries.json@@ -32,6 +32,10 @@"case": "lifecycle","what": "stop and resume, and the declared pause, checkpoint and fork"},+ {+ "case": "warm",+ "what": "a Space claimed from a warm member answers with its own token and only that one, or the runtime says it starts cold"+ },{"case": "teardown","what": "delete leaves no Space, process, container or network"@@ -46,13 +50,15 @@"x86_64"],"checkpoint": true,+ "claim": true,"fork": true,"guests": ["linux"],"kvm": false,"nested_virt": false,- "pause": true+ "pause": true,+ "prepare": true},"guest": "linux","hosts": [@@ -98,6 +104,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -148,6 +159,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -171,13 +187,15 @@"x86_64"],"checkpoint": true,+ "claim": true,"fork": true,"guests": ["linux"],"kvm": false,"nested_virt": false,- "pause": true+ "pause": true,+ "prepare": true},"guest": "linux","hosts": [@@ -223,6 +241,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -273,6 +296,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -296,13 +324,15 @@"x86_64"],"checkpoint": true,+ "claim": true,"fork": true,"guests": ["linux"],"kvm": false,"nested_virt": false,- "pause": true+ "pause": true,+ "prepare": true},"guest": "linux","hosts": [@@ -348,6 +378,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -371,13 +406,15 @@"x86_64"],"checkpoint": true,+ "claim": true,"fork": true,"guests": ["linux"],"kvm": false,"nested_virt": false,- "pause": true+ "pause": true,+ "prepare": true},"guest": "linux","hosts": [@@ -423,6 +460,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -446,13 +488,15 @@"x86_64"],"checkpoint": true,+ "claim": true,"fork": true,"guests": ["linux"],"kvm": true,"nested_virt": true,- "pause": true+ "pause": true,+ "prepare": false},"guest": "linux","hosts": [@@ -498,6 +542,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -520,13 +569,15 @@"aarch64"],"checkpoint": true,+ "claim": true,"fork": true,"guests": ["linux"],"kvm": false,"nested_virt": false,- "pause": true+ "pause": true,+ "prepare": false},"guest": "linux","hosts": [@@ -572,6 +623,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -595,13 +651,15 @@"x86_64"],"checkpoint": false,+ "claim": true,"fork": false,"guests": ["linux"],"kvm": false,"nested_virt": false,- "pause": false+ "pause": false,+ "prepare": false},"guest": "linux","hosts": [@@ -647,6 +705,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -670,6 +733,7 @@"x86_64"],"checkpoint": true,+ "claim": true,"fork": true,"guests": ["linux",@@ -679,7 +743,8 @@],"kvm": false,"nested_virt": false,- "pause": true+ "pause": true,+ "prepare": false},"guest": "linux","hosts": [@@ -725,6 +790,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -775,6 +845,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -825,6 +900,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "pass",+ "why": null+ },{"case": "teardown","expect": "pass",@@ -847,13 +927,15 @@"aarch64"],"checkpoint": false,+ "claim": true,"fork": true,"guests": ["macos"],"kvm": false,"nested_virt": false,- "pause": false+ "pause": false,+ "prepare": false},"guest": "macos","hosts": [@@ -899,6 +981,11 @@"expect": "pass","why": null},+ {+ "case": "warm",+ "expect": "refused",+ "why": "not applicable to this guest"+ },{"case": "teardown","expect": "pass",@@ -919,11 +1006,13 @@"capabilities": {"archs": [],"checkpoint": false,+ "claim": false,"fork": false,"guests": [],"kvm": false,"nested_virt": false,- "pause": false+ "pause": false,+ "prepare": false},"guest": "linux","hosts": [],diff --git a/libs/cua/scripts/conformance/run.py b/libs/cua/scripts/conformance/run.pyindex c75f8eba84..5d5e942456 100755--- a/libs/cua/scripts/conformance/run.py+++ b/libs/cua/scripts/conformance/run.py@@ -338,6 +338,9 @@ def run_entry(entry, args, out_dir, log):# --- lifecycleif "lifecycle" in cases:row_lifecycle(run, entry, name, row)+ # --- warm claim+ if "warm" in cases:+ row_warm(run, entry, image, args, row)# --- cached ready timesif args.n > 0 and "ready" in cases:times = []@@ -578,6 +581,77 @@ def row_lifecycle(run, entry, name, row):row("lifecycle", "fail" if fails else "pass", "; ".join(fails + notes))+def row_warm(run, entry, image, args, row):+ """A pool of one: the member is claimed by a create, answers, is replaced, and the pool goes.++ The runtime says what it does: a runtime with no warm claims refuses the pool in plain words (its+ Spaces start cold); one that can claim hands out a Space that answers, and a second member is kept+ ready afterwards (the refill)."""+ caps = entry["capabilities"]+ pool, claimed = "conf-warm", "conf-warm-1"+ fails, notes = [], []+ t0 = time.monotonic()+ a = ["sandbox", "pool", "create", image, "--name", pool, "--min", "1", "--max", "1", "--kind", entry["kind"], "--cpu", "1", "--memory", "1GB", "--json"]+ if entry["runtime"]:+ a += ["--runtime", entry["runtime"]]+ rc, o, e, ms = run.sh(*a, timeout=300)+ if not caps["claim"]:+ said = (o + e).strip()+ if rc != 0 and len(said) >= 15:+ row("warm", "pass", "no warm claims here, said in plain words: " + said[:120], ms)+ else:+ row("warm", "fail", "this runtime declares no warm claims, but the pool was not refused in plain words")+ run.sh("sandbox", "pool", "rm", pool, timeout=120)+ return++ def ready():+ rc, o, e, _ = run.sh("sandbox", "pool", "ls", "--json", timeout=60)+ try:+ for p in json.loads(o):+ if p.get("name") == pool:+ return int(p.get("ready", 0)), (p.get("status") or p.get("reason") or "")+ except (ValueError, TypeError, AttributeError):+ pass+ return 0, (o + e).strip()[:120]++ if rc != 0:+ row("warm", "fail", "pool create: " + (e or o).strip()[:200])+ return+ try:+ n, why = 0, ""+ deadline = time.monotonic() + 900+ while time.monotonic() < deadline:+ n, why = ready()+ if n >= 1:+ break+ time.sleep(3)+ if n < 1:+ fails.append(f"no member was ready within 15 minutes ({why})")+ else:+ notes.append(f"member ready after {time.monotonic() - t0:.0f}s")+ rc, o, e, ms = run.sh("sandbox", "create", "--pool", pool, "--name", claimed, "--json", timeout=args.timeout)+ if rc != 0:+ fails.append("create from the pool: " + (e or o).strip()[:200])+ else:+ notes.append(f"claim {ms:.0f} ms")+ rc, o, e, _ = run.exec(claimed, "echo claimed-ok && cat /etc/machine-id 2>/dev/null | head -c 8")+ if "claimed-ok" not in o:+ fails.append("the claimed Space does not answer: " + (e or o).strip()[:120])+ # The pool keeps one member: a replacement comes.+ again = False+ for _ in range(200):+ if ready()[0] >= 1:+ again = True+ break+ time.sleep(3)+ if not again:+ fails.append("no replacement member became ready in 10 minutes")+ finally:+ run.sh("sandbox", "rm", claimed, "-f", timeout=300)+ run.sh("sandbox", "pool", "rm", pool, timeout=300)+ row("warm", "fail" if fails else "pass", "; ".join(fails + notes), (time.monotonic() - t0) * 1000)++def stop_apple_services(home):"""Apple's `container` services outlive a Space (cua never stops them, they are per user): theones this run's home started are stopped before the home goes."""diff --git a/libs/spaces-app-swift/Sources/CuaSpacesFFI/CuaSpacesFFI.swift b/libs/spaces-app-swift/Sources/CuaSpacesFFI/CuaSpacesFFI.swiftindex 9b21a81bad..9b64625c2e 100644--- a/libs/spaces-app-swift/Sources/CuaSpacesFFI/CuaSpacesFFI.swift+++ b/libs/spaces-app-swift/Sources/CuaSpacesFFI/CuaSpacesFFI.swift@@ -17105,6 +17105,12 @@ public struct AppExperiments: Equatable, Hashable {* is kept in the cua home).*/public var mobile: Bool+ /**+ * Warm pools: one Linux Space kept ready, with its coding apps+ * installed, so the next Create from Home starts at once. Off unless+ * turned on; the daemon enforces it (the switch is kept in the cua home).+ */+ public var warmPools: Bool// Default memberwise initializers are never public by default, so we// declare one manually.@@ -17151,7 +17157,12 @@ public struct AppExperiments: Equatable, Hashable {* Mobile: the computer answers the Cua Spaces phone app (its tools and* push events). Off unless turned on; the daemon enforces it (the switch* is kept in the cua home).- */mobile: Bool) {+ */mobile: Bool,+ /**+ * Warm pools: one Linux Space kept ready, with its coding apps+ * installed, so the next Create from Home starts at once. Off unless+ * turned on; the daemon enforces it (the switch is kept in the cua home).+ */warmPools: Bool) {self.cuaVolume = cuaVolumeself.yourCloud = yourCloudself.sharing = sharing@@ -17162,6 +17173,7 @@ public struct AppExperiments: Equatable, Hashable {self.ciRunners = ciRunnersself.tcpShares = tcpSharesself.mobile = mobile+ self.warmPools = warmPools}@@ -17189,7 +17201,8 @@ public struct FfiConverterTypeAppExperiments: FfiConverterRustBuffer {askFirst: FfiConverterBool.read(from: &buf),ciRunners: FfiConverterBool.read(from: &buf),tcpShares: FfiConverterBool.read(from: &buf),- mobile: FfiConverterBool.read(from: &buf)+ mobile: FfiConverterBool.read(from: &buf),+ warmPools: FfiConverterBool.read(from: &buf))}@@ -17204,6 +17217,7 @@ public struct FfiConverterTypeAppExperiments: FfiConverterRustBuffer {FfiConverterBool.write(value.ciRunners, into: &buf)FfiConverterBool.write(value.tcpShares, into: &buf)FfiConverterBool.write(value.mobile, into: &buf)+ FfiConverterBool.write(value.warmPools, into: &buf)}}diff --git a/tests/e2e/cua-sdk/rust/Cargo.lock b/tests/e2e/cua-sdk/rust/Cargo.lockindex d287dcb32e..7e5604ebcf 100644--- a/tests/e2e/cua-sdk/rust/Cargo.lock+++ b/tests/e2e/cua-sdk/rust/Cargo.lock@@ -1462,6 +1462,7 @@ dependencies = ["async-trait","base64 0.22.1","bytes",+ "cua-agents","cua-byoc","cua-cluster","cua-create-failure",diff --git a/tests/e2e/local-runtimes/Cargo.lock b/tests/e2e/local-runtimes/Cargo.lockindex de746258d7..18df43080e 100644--- a/tests/e2e/local-runtimes/Cargo.lock+++ b/tests/e2e/local-runtimes/Cargo.lock@@ -1241,6 +1241,7 @@ dependencies = ["async-trait","base64 0.22.1","bytes",+ "cua-agents","cua-create-failure","cua-disk","cua-home",