Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
perf(spaces): a new home is warm as soon as the person's agents are prepared (#6645)
ddupont397c66c05e97
diff --git a/libs/cua/crates/cua-daemon/src/local.rs b/libs/cua/crates/cua-daemon/src/local.rsindex 9a0c98290a..6b29fa0e75 100644--- a/libs/cua/crates/cua-daemon/src/local.rs+++ b/libs/cua/crates/cua-daemon/src/local.rs@@ -836,6 +836,7 @@ impl VmmLocal {return Err("the image has no cua-spacesd".to_string());}let spacesd = p.image.as_ref().and_then(|i| i.spacesd) == Some(true);+ let (harnesses, first) = warm_pools_ordered();let key = pool.set_explicit_prepared(crate::pool::WARM_SOURCE,@@ -847,9 +848,10 @@ impl VmmLocal {Some(1),None,Some(WARM_POOLS_IDLE),- Some(warm_pools_harnesses()),+ Some(harnesses),).ok_or_else(|| "this shape does not pool".to_string())?;+ pool.set_prepare_first(&key, first);pool.note_request(&p.spec, p.backend, spacesd, requested_words(&spec));tracing::debug!(shape = %key, "warm-pools: keeping one Linux Space ready");Ok(())@@ -1307,6 +1309,68 @@ pub fn warm_pools_harnesses() -> Vec<String> {.collect()}+/// Most harnesses a warm member is prepared for before it can be claimed;+/// the rest go into the saved filesystem in the background.+const WARM_FIRST_MAX: usize = 3;++/// The harness ids in `phases` (`persistent/phases.jsonl` lines, oldest+/// first) and `agents` (`persistent/agents.json`), most recently used first.+fn recent_harnesses(phases: &str, agents: &str) -> Vec<String> {+ let mut out: Vec<String> = Vec::new();+ let mut push = |h: &str| {+ if !h.is_empty() && !out.iter().any(|o| o == h) {+ out.push(h.to_string());+ }+ };+ for line in phases.lines().rev().take(400) {+ if let Ok(v) = serde_json::from_str::<serde_json::Value>(line)+ && let Some(h) = v.get("harness").and_then(|h| h.as_str())+ {+ push(h);+ }+ }+ if let Ok(serde_json::Value::Array(a)) = serde_json::from_str::<serde_json::Value>(agents) {+ for e in a.iter().rev() {+ if let Some(h) = e.get("harness").and_then(|h| h.as_str()) {+ push(h);+ }+ }+ }+ out+}++/// [`warm_pools_harnesses`] with the apps this person most likely uses+/// first, and how many of them (at most [`WARM_FIRST_MAX`]) a member is+/// prepared for before it can be claimed: Claude Code (the Home picker's+/// default), then the apps of their agents and recent runs. `0`: no+/// preference (`CUA_WARM_POOLS_HARNESSES` names the list), prepare all.+pub fn warm_pools_ordered() -> (Vec<String>, usize) {+ let all = warm_pools_harnesses();+ if std::env::var_os("CUA_WARM_POOLS_HARNESSES").is_some() {+ return (all, 0);+ }+ let dir = cua_home::cua_home().join("persistent");+ let read = |f: &str| std::fs::read_to_string(dir.join(f)).unwrap_or_default();+ order_for_warm(+ all,+ &recent_harnesses(&read("phases.jsonl"), &read("agents.json")),+ )+}++fn order_for_warm(all: Vec<String>, recent: &[String]) -> (Vec<String>, usize) {+ let mut first: Vec<String> = Vec::new();+ for h in std::iter::once(&"claude-code".to_string()).chain(recent.iter()) {+ if first.len() < WARM_FIRST_MAX && all.contains(h) && !first.contains(h) {+ first.push(h.clone());+ }+ }+ let n = first.len();+ let mut ordered = first;+ let rest: Vec<String> = all.into_iter().filter(|h| !ordered.contains(h)).collect();+ ordered.extend(rest);+ (ordered, n)+}+/// How long the warm Space sits unused before it is given back.pub const WARM_POOLS_IDLE: std::time::Duration = std::time::Duration::from_secs(2 * 60 * 60);@@ -1501,6 +1565,27 @@ impl crate::pool::PoolHost for PoolHost {}}+ fn pin(&self, harness: &str) -> String {+ // The apps a build installs for the harness, at their pinned versions.+ let Some(h) = cua_agents::harness::harness(harness) else {+ return harness.to_string();+ };+ let ids: Vec<&str> = h.installs.to_vec();+ let Ok(order) = cua_agents::installables::resolve(&ids) else {+ return harness.to_string();+ };+ order+ .iter()+ .filter_map(|id| {+ Some(format!(+ "{id}@{}",+ cua_agents::installables::get(id)?.version+ ))+ })+ .collect::<Vec<_>>()+ .join(",")+ }+async fn warm_files(&self, endpoints: &Endpoints, secret: &str, harnesses: &[String]) {// The directories the apps installed into, read once. The pool secret// stops opening the member when it is claimed: a failure is fine.@@ -3620,6 +3705,33 @@ mod tests {}}+ #[test]+ fn the_warm_member_is_prepared_for_the_agents_in_use_first() {+ let all: Vec<String> = [+ "openai-codex",+ "hermes",+ "claude-code",+ "opencode",+ "goose",+ "pi",+ ]+ .map(String::from)+ .to_vec();+ // Nothing used yet: Claude Code alone.+ let (o, n) = super::order_for_warm(all.clone(), &[]);+ assert_eq!((o[0].as_str(), n), ("claude-code", 1));+ assert_eq!(o.len(), all.len());+ // Recent runs and agents follow, newest first, three in all.+ let phases = "{\"harness\":\"goose\",\"phase\":\"posted\"}\n{\"harness\":\"hermes\",\"phase\":\"posted\"}\nnot json\n{\"harness\":\"hermes\"}\n";+ let agents = r#"[{"harness":"pi"},{"harness":"claude-code"},{"harness":"gone-harness"}]"#;+ let recent = super::recent_harnesses(phases, agents);+ assert_eq!(recent[0], "hermes");+ let (o, n) = super::order_for_warm(all.clone(), &recent);+ assert_eq!(n, 3);+ assert_eq!(&o[..3], ["claude-code", "hermes", "goose"]);+ assert_eq!(o.len(), all.len(), "the others follow, once");+ }+#[test]fn a_macos_pull_is_sized_from_the_catalog() {let gib = 1u64 << 30;diff --git a/libs/cua/crates/cua-daemon/src/pool/mod.rs b/libs/cua/crates/cua-daemon/src/pool/mod.rsindex 4786b5405a..886f05d78c 100644--- a/libs/cua/crates/cua-daemon/src/pool/mod.rs+++ b/libs/cua/crates/cua-daemon/src/pool/mod.rs@@ -286,6 +286,13 @@ pub trait PoolHost: Send + Sync {let _ = (endpoints, secret, harnesses);}+ /// What pins the install of `harness` in a prepared filesystem: the+ /// versions of the apps a build installs for it. A saved filesystem+ /// serves a build whose pins for the harnesses it wants are the same.+ fn pin(&self, harness: &str) -> String {+ harness.to_string()+ }+/// Whether this machine runs on battery now (no member then).fn on_battery(&self) -> bool;@@ -328,6 +335,17 @@ impl Ready {}}+/// The saved filesystem a member boots from.+#[derive(Clone)]+struct Chosen {+ tag: String,+ image: cua_vmm::ImageSource,+ /// What is installed in it (id to pin).+ have: std::collections::BTreeMap<String, String>,+ /// It has every app the shape wants.+ covers_all: bool,+}+/// A ready member.#[derive(Clone)]struct Member {@@ -371,6 +389,10 @@ struct Shape {/// The harnesses whose agent stack a member gets before it is ready/// ([`WarmPool::set_prepare`]); empty: members are bare.prepare: Vec<String>,+ /// How many of `prepare` (from the front) a member is prepared for before+ /// it can be claimed: the apps the person most likely uses first. The+ /// rest go into the saved filesystem in the background. 0: all of them.+ prepare_first: usize,/// Measured (EWMA, seconds): a member's preparation.prepare_s: Option<f64>,/// The saved filesystem of the first prepared member@@ -399,6 +421,7 @@ impl Shape {requested: None,last_used: None,prepare: Vec::new(),+ prepare_first: 0,prepare_s: None,prepared_image: None,}@@ -598,6 +621,12 @@ struct Inner {/// Why each shape's refill is deferred now, and when that was last/// logged ([`WarmPool::defer`]).deferred: Mutex<HashMap<String, (String, Instant)>>,+ /// Shapes whose saved filesystem is being built now, and (after a+ /// failure) when the next try may start.+ building: Mutex<HashSet<String>>,+ build_blocked: Mutex<HashMap<String, Instant>>,+ /// Serialises changes of the index of saved filesystems.+ index_lock: Mutex<()>,}/// How long the policy starts no member after other work took capacity back.@@ -760,16 +789,55 @@ fn token_of(spec: &StartSpec) -> Option<String> {.cloned()}-/// The tag of the saved filesystem of a shape's prepared members: the shape,-/// the apps and this build (a new build installs the apps it pins).-fn prepared_tag(key: &str, harnesses: &[String]) -> String {+/// What a saved filesystem is keyed by, apart from the apps in it: the+/// backend, the image it was made from and its platform. Nothing else of a+/// shape (network preset, size, ports, environment) is in the saved files,+/// so a change of those never throws a prepared filesystem away.+fn prepared_base(template: &StartSpec, backend: BackendKind) -> String {use sha2::{Digest, Sha256};+ let image = match &template.image {+ cua_vmm::ImageSource::Oci { reference } => reference.as_str(),+ _ => "",+ };let mut h = Sha256::new();- h.update(harnesses.join(",").as_bytes());- h.update(env!("CARGO_PKG_VERSION").as_bytes());+ for part in [+ backend.as_str(),+ image,+ &format!("{:?}", template.os),+ &format!("{:?}", template.arch),+ ] {+ h.update(part.as_bytes());+ h.update([0]);+ }+ h.finalize()+ .iter()+ .take(6)+ .map(|b| format!("{b:02x}"))+ .collect()+}++/// A saved filesystem: made from `base`, with these harnesses (and their+/// pins, [`PoolHost::pin`]) installed.+#[derive(Clone, Debug, Serialize, Deserialize)]+struct PreparedEntry {+ tag: String,+ base: String,+ harnesses: std::collections::BTreeMap<String, String>,+}++/// The tag of the saved filesystem of `base` with `harnesses` (id to pin).+fn prepared_tag(base: &str, harnesses: &std::collections::BTreeMap<String, String>) -> String {+ use sha2::{Digest, Sha256};+ let mut h = Sha256::new();+ for (id, pin) in harnesses {+ h.update(id.as_bytes());+ h.update([0]);+ h.update(pin.as_bytes());+ h.update([0]);+ }let d = h.finalize();format!(- "warm-{key}-{}",+ "warm-{base}-{}",d.iter().take(4).map(|b| format!("{b:02x}"))@@ -815,6 +883,9 @@ impl WarmPool {arrived: tokio::sync::Notify::new(),due: Mutex::new(HashSet::new()),deferred: Mutex::new(HashMap::new()),+ building: Mutex::new(HashSet::new()),+ build_blocked: Mutex::new(HashMap::new()),+ index_lock: Mutex::new(()),}),}}@@ -1272,6 +1343,14 @@ impl WarmPool {Some(key)}+ /// Sets how many of the shape's apps (from the front of its list) a member+ /// is prepared for before it can be claimed ([`Shape::prepare_first`]).+ pub fn set_prepare_first(&self, key: &str, n: usize) {+ if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {+ s.prepare_first = n;+ }+ }+/// Keeps `count` members of `template`'s shape regardless of use/// (`pool.warm`, keyed by the shape).pub fn keep(@@ -1602,6 +1681,291 @@ impl WarmPool {self.inner.deferred.lock().unwrap().remove(key);}+ fn index_path(&self) -> PathBuf {+ self.inner.dir.join("prepared-index.json")+ }++ fn index_load(&self) -> Vec<PreparedEntry> {+ std::fs::read(self.index_path())+ .ok()+ .and_then(|b| serde_json::from_slice(&b).ok())+ .unwrap_or_default()+ }++ fn index_update(&self, f: impl FnOnce(&mut Vec<PreparedEntry>)) {+ let _g = self.inner.index_lock.lock().unwrap();+ let mut v = self.index_load();+ f(&mut v);+ let path = self.index_path();+ if cua_home::guard_write(&path).is_ok() {+ let _ = std::fs::create_dir_all(&self.inner.dir);+ if let Ok(b) = serde_json::to_vec_pretty(&v) {+ let _ = cua_home::write_private(&path, &b);+ }+ }+ }++ fn index_remove(&self, tag: &str) {+ self.index_update(|v| v.retain(|e| e.tag != tag));+ }++ /// The pin of each of `harnesses` in this build.+ fn pins(&self, harnesses: &[String]) -> std::collections::BTreeMap<String, String> {+ harnesses+ .iter()+ .map(|h| (h.clone(), self.inner.host.pin(h)))+ .collect()+ }++ /// The saved filesystem a member of `template`'s shape boots from: one+ /// of the same image with at least the `first` apps in it (at these+ /// pins), the one with most of `all`. A superset is as good as an exact+ /// match; nothing but the image and the apps keys it, so a build, a+ /// network preset or a size change does not throw it away.+ async fn choose_prepared(+ &self,+ key: &str,+ template: &StartSpec,+ backend: BackendKind,+ rt: &dyn VmmRuntime,+ all: &[String],+ first: &[String],+ ) -> Option<Chosen> {+ let base = prepared_base(template, backend);+ let pins = self.pins(all);+ let mut cands: Vec<(usize, PreparedEntry)> = Vec::new();+ let mut why: Vec<String> = Vec::new();+ for e in self.index_load().into_iter().filter(|e| e.base == base) {+ let has = |h: &String| e.harnesses.get(h) == pins.get(h);+ if let Some(m) = first.iter().find(|h| !has(h)) {+ why.push(format!("{} lacks {m} at this build's pin", e.tag));+ continue;+ }+ cands.push((all.iter().filter(|h| has(h)).count(), e));+ }+ cands.sort_by(|a, b| {+ b.0.cmp(&a.0)+ .then(a.1.harnesses.len().cmp(&b.1.harnesses.len()))+ });+ for (n, e) in cands {+ match rt.find_prepared(&e.tag).await {+ Ok(Some(image)) => {+ tracing::info!(+ shape = key,+ tag = %e.tag,+ covers = n,+ wanted = all.len(),+ "warm-pool: prepared filesystem reused"+ );+ return Some(Chosen {+ covers_all: n == all.len(),+ tag: e.tag,+ image,+ have: e.harnesses,+ });+ }+ _ => {+ why.push(format!("{} is gone from the engine", e.tag));+ self.index_remove(&e.tag);+ }+ }+ }+ if why.is_empty() {+ why.push(format!("none saved for base {base}"));+ }+ tracing::info!(+ shape = key,+ wanted = %first.join(","),+ reason = %why.join("; "),+ "warm-pool: no prepared filesystem to reuse; the member is prepared"+ );+ None+ }++ /// Builds the saved filesystem with all of `all` in a member of its own+ /// (booted from `from`, which has some of them), in the background:+ /// the `first` apps are saved as soon as they are in, then the rest.+ /// Members that can be claimed never wait for a save.+ #[allow(clippy::too_many_arguments)]+ fn build_prepared(+ &self,+ key: &str,+ template: &StartSpec,+ backend: BackendKind,+ rt: Arc<dyn VmmRuntime>,+ all: Vec<String>,+ first: Vec<String>,+ from: Option<Chosen>,+ ) {+ if self.inner.closed.load(std::sync::atomic::Ordering::SeqCst) || !rt.keeps_prepared() {+ return;+ }+ if let Some(t) = self.inner.build_blocked.lock().unwrap().get(key)+ && *t > Instant::now()+ {+ return;+ }+ if !self.inner.building.lock().unwrap().insert(key.to_string()) {+ return;+ }+ let Ok(handle) = tokio::runtime::Handle::try_current() else {+ self.inner.building.lock().unwrap().remove(key);+ return;+ };+ let (pool, key, template) = (self.clone(), key.to_string(), template.clone());+ handle.spawn(async move {+ let began = Instant::now();+ let r = {+ let work = pool.build_inner(&key, &template, backend, rt, &all, &first, from);+ let closed = async {+ while !pool.inner.closed.load(std::sync::atomic::Ordering::SeqCst) {+ tokio::time::sleep(Duration::from_secs(1)).await;+ }+ };+ tokio::select! { r = work => r, _ = closed => Err("the pool closed".to_string()) }+ };+ pool.inner.building.lock().unwrap().remove(&key);+ match r {+ Ok(true) => tracing::info!(+ shape = %key,+ built_ms = began.elapsed().as_millis() as u64,+ since_start_s = pool.inner.epoch.elapsed().as_secs(),+ "warm-pool: the prepared filesystem has every app"+ ),+ // This runtime keeps nothing: members are prepared one by one.+ Ok(false) => {+ pool.inner+ .build_blocked+ .lock()+ .unwrap()+ .insert(key, Instant::now() + Duration::from_secs(3600));+ }+ Err(e) => {+ tracing::warn!(shape = %key, error = %e, "warm-pool: could not build the prepared filesystem; trying again later");+ pool.inner+ .build_blocked+ .lock()+ .unwrap()+ .insert(key, Instant::now() + Duration::from_secs(600));+ }+ }+ });+ }++ #[allow(clippy::too_many_arguments)]+ async fn build_inner(+ &self,+ key: &str,+ template: &StartSpec,+ backend: BackendKind,+ rt: Arc<dyn VmmRuntime>,+ all: &[String],+ first: &[String],+ from: Option<Chosen>,+ ) -> Result<bool, String> {+ let base = prepared_base(template, backend);+ let pins = self.pins(all);+ let name = member_name();+ let secret = pool_secret();+ let record = Record {+ name: name.clone(),+ key: key.to_string(),+ backend,+ secret: secret.clone(),+ template: template.clone(),+ keep: 0,+ prepared: false,+ };+ self.write_record(&record).map_err(|e| e.to_string())?;+ if let Err(e) = self.inner.leases.write_lease(&name, self.inner.pid) {+ self.remove_record(&name);+ return Err(e.to_string());+ }+ self.inner.starting.lock().unwrap().insert(name.clone());+ let work = async {+ let mut spec = template.clone();+ spec.name = name.clone();+ spec.pool_member = true;+ if let Some(c) = &from {+ spec.image = c.image.clone();+ }+ spec.env.insert("CUA_ENV_TOKEN".into(), secret.clone());+ spec.labels+ .insert(cua_vmm::container::LABEL_POOL.into(), "true".into());+ let inst = rt.start(&spec).await.map_err(|e| e.to_string())?;+ if backend == BackendKind::Container {+ self.inner+ .host+ .member_ready(+ &inst.endpoints,+ &secret,+ has_desktop(&spec),+ MEMBER_READY_TIMEOUT,+ )+ .await?;+ }+ let mut have = from.map(|c| c.have).unwrap_or_default();+ // The apps a member needs to be claimed, saved at once; then the+ // rest.+ for layer in [first, all] {+ let missing: Vec<String> = layer+ .iter()+ .filter(|h| have.get(*h) != pins.get(*h))+ .cloned()+ .collect();+ if missing.is_empty() {+ continue;+ }+ let began = Instant::now();+ self.inner+ .host+ .prepare(&inst.endpoints, &secret, &missing, PREPARE_TIMEOUT)+ .await?;+ for h in &missing {+ if let Some(p) = pins.get(h) {+ have.insert(h.clone(), p.clone());+ }+ }+ let tag = prepared_tag(&base, &have);+ let saving = Instant::now();+ match rt.save_prepared(&name, &tag).await {+ Ok(Some(_)) => {+ let entry = PreparedEntry {+ tag: tag.clone(),+ base: base.clone(),+ harnesses: have.clone(),+ };+ self.index_update(|v| {+ v.retain(|e| e.tag != tag);+ v.push(entry);+ });+ tracing::info!(+ shape = key,+ %tag,+ harnesses = %have.keys().cloned().collect::<Vec<_>>().join(","),+ prepare_ms = began.elapsed().as_millis() as u64 - saving.elapsed().as_millis() as u64,+ save_ms = saving.elapsed().as_millis() as u64,+ since_start_s = self.inner.epoch.elapsed().as_secs(),+ "warm-pool: the prepared filesystem is kept for the next member"+ );+ }+ Ok(None) => return Ok(false),+ Err(e) => return Err(e.to_string()),+ }+ }+ Ok(true)+ }+ .await;+ match rt.delete(&name).await {+ Ok(()) | Err(VmmError::NotFound(_)) => self.forget_member(&name),+ Err(e) => {+ tracing::warn!(member = %name, error = %e, "could not remove a warm-pool builder")+ }+ }+ self.inner.starting.lock().unwrap().remove(&name);+ work+ }+async fn start_member(&self,key: &str,@@ -1615,18 +1979,26 @@ impl WarmPool {let shapes = self.inner.shapes.lock().unwrap();shapes.get(key).map(|s| s.keep()).unwrap_or(0)};- let prepare = {+ let (all, first_n) = {let shapes = self.inner.shapes.lock().unwrap();shapes.get(key)- .map(|s| s.prepare.clone())+ .map(|s| (s.prepare.clone(), s.prepare_first)).unwrap_or_default()};- let prepare = if self.inner.host.can_prepare(backend) {- prepare+ let all = if self.inner.host.can_prepare(backend) {+ all} else {Vec::new()};+ // The apps a member must have before it can be claimed; the others+ // go into the saved filesystem in the background.+ let first: Vec<String> = all+ .iter()+ .take(if first_n == 0 { all.len() } else { first_n })+ .cloned()+ .collect();+ let prepare = first.clone();let mut record = Record {name: name.clone(),key: key.to_string(),@@ -1646,33 +2018,19 @@ impl WarmPool {let mut spec = template.clone();spec.name = name.clone();spec.pool_member = true;- // A member of a shape whose first member was prepared boots from that- // member's saved filesystem: its apps are in it already, so it is- // ready in the time of a boot, not of an install.- let mut saved = if prepare.is_empty() {+ // A member boots from a saved filesystem that has the apps it needs+ // (a build or a daemon before saved it): they are in it already, so+ // it is ready in the time of a boot, not of an install.+ let chosen = if all.is_empty() {None} else {- self.inner- .shapes- .lock()- .unwrap()- .get(key)- .and_then(|s| s.prepared_image.clone())+ self.choose_prepared(key, template, backend, rt.as_ref(), &all, &first)+ .await};- // A daemon that restarted: what the last one saved is still there.- if saved.is_none() && !prepare.is_empty() {- let tag = prepared_tag(key, &prepare);- if let Ok(Some(image)) = rt.find_prepared(&tag).await {- tracing::info!(- shape = key,- "warm-pool: found the prepared filesystem a previous daemon saved"- );- if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {- s.prepared_image = Some((tag.clone(), image.clone()));- }- saved = Some((tag, image));- }+ if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {+ s.prepared_image = chosen.as_ref().map(|c| (c.tag.clone(), c.image.clone()));}+ let saved = chosen.as_ref().map(|c| (c.tag.clone(), c.image.clone()));if let Some((_, image)) = &saved {spec.image = image.clone();}@@ -1692,12 +2050,15 @@ impl WarmPool {let mut prepared = false;let ready = async {let inst = rt.start(&spec).await.map_err(|e| e.to_string());- if inst.is_err() && saved.is_some() {+ if inst.is_err()+ && let Some((tag, _)) = &saved+ {// The saved filesystem does not start any more (the engine// lost it): the next member is prepared from scratch.if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {s.prepared_image = None;}+ self.index_remove(tag);}let inst = inst?;// A container member runs with its pool secret: it is ready@@ -1746,22 +2107,6 @@ impl WarmPool {s.prepare_s = ewma(s.prepare_s, took);}tracing::info!(member = %name, harnesses = %prepare.join(","), prepare_ms = (took * 1000.0) as u64, "warm-pool member prepared");- // Its filesystem is kept (before anyone can claim it),- // so the next member boots from it.- let tag = prepared_tag(key, &prepare);- let saving = Instant::now();- match rt.save_prepared(&name, &tag).await {- Ok(Some(image)) => {- tracing::info!(member = %name, save_ms = saving.elapsed().as_millis() as u64, "warm-pool: the prepared filesystem is kept for the next member");- if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {- s.prepared_image = Some((tag, image));- }- }- Ok(None) => {}- Err(e) => {- tracing::warn!(member = %name, error = %e, "warm-pool: could not keep the prepared filesystem; the next member is prepared again");- }- }}Err(e) => {tracing::warn!(member = %name, error = %e, "warm-pool member could not be prepared; it is claimed bare");@@ -1786,9 +2131,25 @@ impl WarmPool {member = %name,shape = key,ready_ms = started.elapsed().as_millis() as u64,+ claimable_after_start_s = self.inner.epoch.elapsed().as_secs_f64().round() as u64,+ prepared_for = %prepare.join(","),+ from = saved.as_ref().map_or("the image", |(t, _)| t.as_str()),footprint_mb,"warm-pool member ready");+ // The saved filesystem is built beside the members, never from one+ // that can be claimed (a claim changes what is in it).+ if !all.is_empty() && !chosen.as_ref().is_some_and(|c| c.covers_all) {+ self.build_prepared(+ key,+ template,+ backend,+ rt.clone(),+ all.clone(),+ first.clone(),+ chosen.clone(),+ );+ }// A member that booted from the saved filesystem has nothing in its// file cache: its apps are read now, while it waits to be claimed.if saved.is_some()@@ -2203,9 +2564,8 @@ impl WarmPool {fn forget_shape(&self, key: &str) {let gone = self.inner.shapes.lock().unwrap().remove(key);if let Some(s) = gone {- if let Some((tag, _)) = &s.prepared_image {- self.drop_saved(s.rt.clone(), tag.clone());- }+ // The saved filesystem stays (a shape that comes back finds it+ // by its image and apps; the disk budget removes the old).for m in s.members {self.discard(s.rt.clone(), m.name);}@@ -2380,6 +2740,9 @@ mod tests {async fn fork(&self, _: &str, _: &str) -> VResult<()> {Ok(())}+ fn keeps_prepared(&self) -> bool {+ self.can_save.load(SeqCst)+ }async fn save_prepared(&self, _: &str, tag: &str) -> VResult<Option<ImageSource>> {if !self.can_save.load(SeqCst) {return Ok(None);@@ -3562,17 +3925,26 @@ mod tests {assert_eq!(f.rt.members().len(), 0, "the free floor still holds");}+ /// Waits until the runtime kept `n` filesystems.+ async fn until_saved(f: &Fixture, n: usize) {+ for _ in 0..300 {+ if f.rt.saved.lock().unwrap().len() >= n {+ return;+ }+ tokio::time::sleep(Duration::from_millis(20)).await;+ }+ panic!("never saved {n}: {:?}", f.rt.saved.lock().unwrap());+ }+#[tokio::test]- async fn later_members_boot_from_the_filesystem_of_the_first_prepared_one() {+ async fn later_members_boot_from_the_filesystem_built_beside_the_first() {let f = fixture(PoolSettings::default(), FakeRuntime::default());f.rt.can_save.store(true, SeqCst);let key = keep_warm_space(&f, &spec("one", "t")).await;until_ready(&f, &key, 1).await;- assert_eq!(- f.rt.saved.lock().unwrap().len(),- 1,- "kept once, before it was ready"- );+ // The first member is claimable as prepared; its filesystem is kept+ // by a builder of its own, never by a member someone can claim.+ until_saved(&f, 1).await;for i in 0..3 {assert!(f.pool@@ -3587,28 +3959,142 @@ mod tests {until_ready(&f, &key, 1).await;}let images = f.rt.images.lock().unwrap().clone();- assert_eq!(images.len(), 4);assert!(!images[0].starts_with("local/prepared:"),"the first boots the image");assert!(- images[1..]- .iter()- .all(|i| i.starts_with("local/prepared:warm-")),+ images[images.len() - 1].starts_with("local/prepared:warm-"),"{images:?}");- // Prepared once; the others only booted.- assert_eq!(f.host.prepared.lock().unwrap().len(), 1);- assert_eq!(f.rt.saved.lock().unwrap().len(), 1);+ assert_eq!(f.rt.saved.lock().unwrap().len(), 1, "kept once");assert!(f.pool.records().iter().all(|r| r.prepared));- // A daemon that stops keeps it for the next one; it goes with its- // shape (forgotten after the policy's horizon without a create).+ // A daemon that stops keeps it for the next one.f.pool.retain_explicit(|src| src != WARM_SOURCE);f.pool.drain().await;assert!(f.rt.dropped.lock().unwrap().is_empty());}+ #[tokio::test]+ async fn the_first_member_is_claimable_after_the_first_apps_and_the_rest_follow() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ f.rt.can_save.store(true, SeqCst);+ let rt: Arc<dyn VmmRuntime> = f.rt.clone();+ let s0 = spec("one", "t");+ let key = f+ .pool+ .set_explicit_prepared(+ WARM_SOURCE,+ &s0,+ BackendKind::Container,+ true,+ rt,+ 1,+ Some(1),+ None,+ Some(Duration::from_secs(7200)),+ Some(vec!["claude-code".into(), "hermes".into(), "goose".into()]),+ )+ .unwrap();+ f.pool.set_prepare_first(&key, 1);+ until_ready(&f, &key, 1).await;+ assert_eq!(+ f.host.prepared.lock().unwrap()[0],+ ["claude-code"],+ "the member waited for the first app only"+ );+ // Beside it: the first app is kept at once, then the whole set.+ until_saved(&f, 2).await;+ let base = prepared_base(&s0, BackendKind::Container);+ let idx = f.pool.index_load();+ let sizes: Vec<usize> = idx.iter().map(|e| e.harnesses.len()).collect();+ assert!(sizes.contains(&1) && sizes.contains(&3), "{sizes:?}");+ assert!(idx.iter().all(|e| e.base == base));+ // The next member starts from the full one, with nothing to install.+ f.pool+ .claim(&spec("s", "t1"), BackendKind::Container, true)+ .await+ .unwrap();+ until_ready(&f, &key, 1).await;+ let before = f.host.prepared.lock().unwrap().len();+ let images = f.rt.images.lock().unwrap().clone();+ assert!(images.last().unwrap().starts_with("local/prepared:warm-"));+ assert_eq!(f.host.prepared.lock().unwrap().len(), before);+ let full = idx+ .iter()+ .max_by_key(|e| e.harnesses.len())+ .unwrap()+ .tag+ .clone();+ assert!(images.last().unwrap().ends_with(&full), "{images:?}");+ }++ #[tokio::test]+ async fn a_changed_network_preset_or_size_keeps_the_prepared_filesystem() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ f.rt.can_save.store(true, SeqCst);+ let s0 = spec("one", "t");+ let have: std::collections::BTreeMap<String, String> = ["claude-code", "openai-codex"]+ .map(|h| (h.to_string(), h.to_string()))+ .into();+ let base = prepared_base(&s0, BackendKind::Container);+ let tag = prepared_tag(&base, &have);+ f.rt.saved.lock().unwrap().push(tag.clone());+ f.pool.index_update(|v| {+ v.push(PreparedEntry {+ tag: tag.clone(),+ base,+ harnesses: have,+ })+ });+ // The same image under another network preset and a bigger size is+ // another shape, but the same saved files.+ let mut s1 = spec("two", "t");+ s1.restrict_network = true;+ s1.cpus = 4;+ assert_ne!(+ shape_key(&s0, BackendKind::Container, true),+ shape_key(&s1, BackendKind::Container, true)+ );+ let key = keep_warm_space(&f, &s1).await;+ until_ready(&f, &key, 1).await;+ assert!(+ f.host.prepared.lock().unwrap().is_empty(),+ "nothing prepared"+ );+ assert!(+ f.rt.images.lock().unwrap()[0].starts_with("local/prepared:warm-"),+ "booted from the kept filesystem"+ );+ }++ #[tokio::test]+ async fn a_filesystem_prepared_at_another_pin_is_not_reused() {+ let f = fixture(PoolSettings::default(), FakeRuntime::default());+ f.rt.can_save.store(true, SeqCst);+ let s0 = spec("one", "t");+ let have: std::collections::BTreeMap<String, String> = ["claude-code", "openai-codex"]+ .map(|h| (h.to_string(), "old-version".to_string()))+ .into();+ let base = prepared_base(&s0, BackendKind::Container);+ let tag = prepared_tag(&base, &have);+ f.rt.saved.lock().unwrap().push(tag.clone());+ f.pool.index_update(|v| {+ v.push(PreparedEntry {+ tag,+ base,+ harnesses: have,+ })+ });+ let key = keep_warm_space(&f, &s0).await;+ until_ready(&f, &key, 1).await;+ assert_eq!(+ f.host.prepared.lock().unwrap()[0],+ ["claude-code", "openai-codex"],+ "prepared again for the apps this build installs"+ );+ }+#[tokio::test]async fn a_runtime_that_cannot_save_prepares_every_member() {let f = fixture(PoolSettings::default(), FakeRuntime::default());@@ -3637,14 +4123,23 @@ mod tests {async fn a_restarted_daemon_finds_the_filesystem_the_last_one_saved() {let f = fixture(PoolSettings::default(), FakeRuntime::default());f.rt.can_save.store(true, SeqCst);- // What a previous daemon saved for this shape and these apps.+ // What a previous daemon saved for this image and these apps (more+ // than are wanted, and under another network preset: still reused).let s0 = spec("one", "t");let key = shape_key(&s0, BackendKind::Container, true).unwrap();- let tag = prepared_tag(- &key,- &["claude-code".to_string(), "openai-codex".to_string()],- );+ let have: std::collections::BTreeMap<String, String> =+ ["claude-code", "openai-codex", "pi"]+ .map(|h| (h.to_string(), h.to_string()))+ .into();+ let tag = prepared_tag(&prepared_base(&s0, BackendKind::Container), &have);f.rt.saved.lock().unwrap().push(tag.clone());+ f.pool.index_update(|v| {+ v.push(PreparedEntry {+ tag: tag.clone(),+ base: prepared_base(&s0, BackendKind::Container),+ harnesses: have.clone(),+ })+ });let key2 = keep_warm_space(&f, &s0).await;assert_eq!(key, key2);until_ready(&f, &key, 1).await;diff --git a/libs/cua/crates/cua-disk/src/gc.rs b/libs/cua/crates/cua-disk/src/gc.rsindex fcf36486f9..5d717a6ebd 100644--- a/libs/cua/crates/cua-disk/src/gc.rs+++ b/libs/cua/crates/cua-disk/src/gc.rs@@ -78,6 +78,10 @@ pub enum KeepReason {/// failed or was interrupted, and no later create from a newer base has/// succeeded.NoNewerBaseInUse,+ /// One of the two newest filesystems the warm pool saved with the apps+ /// installed: evicting it makes the next start prepare a member for+ /// minutes.+ LatestPrepared,}impl std::fmt::Display for KeepReason {@@ -88,6 +92,7 @@ impl std::fmt::Display for KeepReason {KeepReason::NoNewerBaseInUse => {f.write_str("kept for a retry: no newer base is in use")}+ KeepReason::LatestPrepared => f.write_str("a recent prepared warm-pool filesystem"),}}}@@ -207,6 +212,19 @@ pub fn kept_bases(report: &Report, pins: &[Pin]) -> HashMap<usize, KeepReason> {out.entry(n).or_insert(KeepReason::NoNewerBaseInUse);}}+ // The warm pool's saved filesystems are rebuilt in minutes and are+ // bigger than a small budget leaves room for: the newest two stay.+ let mut prepared: Vec<(usize, u64)> = report+ .items+ .iter()+ .enumerate()+ .filter(|(_, i)| !i.orphan && i.category.is_cache() && i.kind == "prepared")+ .map(|(n, i)| (n, i.created.unwrap_or(0)))+ .collect();+ prepared.sort_by_key(|(_, c)| std::cmp::Reverse(*c));+ for (n, _) in prepared.into_iter().take(2) {+ out.entry(n).or_insert(KeepReason::LatestPrepared);+ }out}@@ -687,6 +705,34 @@ mod tests {);}+ #[test]+ fn the_newest_prepared_warm_pool_filesystems_survive_a_small_budget() {+ use super::{KeepReason, kept_bases};+ use crate::scan::{Category, Item, Report};+ let item = |name: &str, created: u64| Item {+ category: Category::DockerImages,+ kind: "prepared".into(),+ name: name.into(),+ bytes: 6 << 30,+ created: Some(created),+ evictable: true,+ ..Default::default()+ };+ let report = Report {+ home: Default::default(),+ space: None,+ budget: "auto".into(),+ budget_bytes: Some(8 << 30),+ cache_bytes: 18 << 30,+ items: vec![item("old", 1), item("mid", 2), item("new", 3)],+ notes: vec![],+ };+ let kept = kept_bases(&report, &[]);+ assert_eq!(kept.get(&2), Some(&KeepReason::LatestPrepared));+ assert_eq!(kept.get(&1), Some(&KeepReason::LatestPrepared));+ assert!(!kept.contains_key(&0), "the oldest is still evictable");+ }+#[test]fn the_current_pins_are_the_macos_and_linux_defaults() {let pins = super::current_pins();diff --git a/libs/cua/crates/cua-vmm/src/container/mod.rs b/libs/cua/crates/cua-vmm/src/container/mod.rsindex 2123d6e227..d0f3ae886f 100644--- a/libs/cua/crates/cua-vmm/src/container/mod.rs+++ b/libs/cua/crates/cua-vmm/src/container/mod.rs@@ -1702,6 +1702,10 @@ impl Runtime for ContainerRuntime {))}+ fn keeps_prepared(&self) -> bool {+ true+ }+async fn save_prepared(&self, name: &str, tag: &str) -> Result<Option<ImageSource>> {validate_name(tag)?;let i = selfdiff --git a/libs/cua/crates/cua-vmm/src/managed.rs b/libs/cua/crates/cua-vmm/src/managed.rsindex 2b620b31de..c00f60346a 100644--- a/libs/cua/crates/cua-vmm/src/managed.rs+++ b/libs/cua/crates/cua-vmm/src/managed.rs@@ -1685,6 +1685,10 @@ mod engine_impl {self.engine().await?.claim(member, new_name, identity).await}+ fn keeps_prepared(&self) -> bool {+ true+ }+async fn save_prepared(&self, name: &str, tag: &str) -> Result<Option<ImageSource>> {let _busy = self.busy();self.engine().await?.save_prepared(name, tag).awaitdiff --git a/libs/cua/crates/cua-vmm/src/runtime.rs b/libs/cua/crates/cua-vmm/src/runtime.rsindex 69ed5aabe7..57a559ef73 100644--- a/libs/cua/crates/cua-vmm/src/runtime.rs+++ b/libs/cua/crates/cua-vmm/src/runtime.rs@@ -85,6 +85,11 @@ pub trait Runtime: Send + Sync {})}+ /// Whether [`Runtime::save_prepared`] keeps an image on this backend.+ fn keeps_prepared(&self) -> bool {+ false+ }+/// Saves the filesystem of the running instance `name` as an image that/// instances start from instead of the one they were made from (what is/// installed in it stays), named `tag`, and returns it. Used by the warm