Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

perf(spaces): a new home is warm as soon as the person's agents are prepared (#6645)

ddupont397c66c05e97
diff --git a/libs/cua/crates/cua-daemon/src/local.rs b/libs/cua/crates/cua-daemon/src/local.rs
index 9a0c98290a..6b29fa0e75 100644
--- a/libs/cua/crates/cua-daemon/src/local.rs
+++ b/libs/cua/crates/cua-daemon/src/local.rs
@@ -836,6 +836,7 @@ impl VmmLocal {
return Err("the image has no cua-spacesd".to_string());
}
let spacesd = p.image.as_ref().and_then(|i| i.spacesd) == Some(true);
+ let (harnesses, first) = warm_pools_ordered();
let key = pool
.set_explicit_prepared(
crate::pool::WARM_SOURCE,
@@ -847,9 +848,10 @@ impl VmmLocal {
Some(1),
None,
Some(WARM_POOLS_IDLE),
- Some(warm_pools_harnesses()),
+ Some(harnesses),
)
.ok_or_else(|| "this shape does not pool".to_string())?;
+ pool.set_prepare_first(&key, first);
pool.note_request(&p.spec, p.backend, spacesd, requested_words(&spec));
tracing::debug!(shape = %key, "warm-pools: keeping one Linux Space ready");
Ok(())
@@ -1307,6 +1309,68 @@ pub fn warm_pools_harnesses() -> Vec<String> {
.collect()
}
+/// Most harnesses a warm member is prepared for before it can be claimed;
+/// the rest go into the saved filesystem in the background.
+const WARM_FIRST_MAX: usize = 3;
+
+/// The harness ids in `phases` (`persistent/phases.jsonl` lines, oldest
+/// first) and `agents` (`persistent/agents.json`), most recently used first.
+fn recent_harnesses(phases: &str, agents: &str) -> Vec<String> {
+ let mut out: Vec<String> = Vec::new();
+ let mut push = |h: &str| {
+ if !h.is_empty() && !out.iter().any(|o| o == h) {
+ out.push(h.to_string());
+ }
+ };
+ for line in phases.lines().rev().take(400) {
+ if let Ok(v) = serde_json::from_str::<serde_json::Value>(line)
+ && let Some(h) = v.get("harness").and_then(|h| h.as_str())
+ {
+ push(h);
+ }
+ }
+ if let Ok(serde_json::Value::Array(a)) = serde_json::from_str::<serde_json::Value>(agents) {
+ for e in a.iter().rev() {
+ if let Some(h) = e.get("harness").and_then(|h| h.as_str()) {
+ push(h);
+ }
+ }
+ }
+ out
+}
+
+/// [`warm_pools_harnesses`] with the apps this person most likely uses
+/// first, and how many of them (at most [`WARM_FIRST_MAX`]) a member is
+/// prepared for before it can be claimed: Claude Code (the Home picker's
+/// default), then the apps of their agents and recent runs. `0`: no
+/// preference (`CUA_WARM_POOLS_HARNESSES` names the list), prepare all.
+pub fn warm_pools_ordered() -> (Vec<String>, usize) {
+ let all = warm_pools_harnesses();
+ if std::env::var_os("CUA_WARM_POOLS_HARNESSES").is_some() {
+ return (all, 0);
+ }
+ let dir = cua_home::cua_home().join("persistent");
+ let read = |f: &str| std::fs::read_to_string(dir.join(f)).unwrap_or_default();
+ order_for_warm(
+ all,
+ &recent_harnesses(&read("phases.jsonl"), &read("agents.json")),
+ )
+}
+
+fn order_for_warm(all: Vec<String>, recent: &[String]) -> (Vec<String>, usize) {
+ let mut first: Vec<String> = Vec::new();
+ for h in std::iter::once(&"claude-code".to_string()).chain(recent.iter()) {
+ if first.len() < WARM_FIRST_MAX && all.contains(h) && !first.contains(h) {
+ first.push(h.clone());
+ }
+ }
+ let n = first.len();
+ let mut ordered = first;
+ let rest: Vec<String> = all.into_iter().filter(|h| !ordered.contains(h)).collect();
+ ordered.extend(rest);
+ (ordered, n)
+}
+
/// How long the warm Space sits unused before it is given back.
pub const WARM_POOLS_IDLE: std::time::Duration = std::time::Duration::from_secs(2 * 60 * 60);
@@ -1501,6 +1565,27 @@ impl crate::pool::PoolHost for PoolHost {
}
}
+ fn pin(&self, harness: &str) -> String {
+ // The apps a build installs for the harness, at their pinned versions.
+ let Some(h) = cua_agents::harness::harness(harness) else {
+ return harness.to_string();
+ };
+ let ids: Vec<&str> = h.installs.to_vec();
+ let Ok(order) = cua_agents::installables::resolve(&ids) else {
+ return harness.to_string();
+ };
+ order
+ .iter()
+ .filter_map(|id| {
+ Some(format!(
+ "{id}@{}",
+ cua_agents::installables::get(id)?.version
+ ))
+ })
+ .collect::<Vec<_>>()
+ .join(",")
+ }
+
async fn warm_files(&self, endpoints: &Endpoints, secret: &str, harnesses: &[String]) {
// The directories the apps installed into, read once. The pool secret
// stops opening the member when it is claimed: a failure is fine.
@@ -3620,6 +3705,33 @@ mod tests {
}
}
+ #[test]
+ fn the_warm_member_is_prepared_for_the_agents_in_use_first() {
+ let all: Vec<String> = [
+ "openai-codex",
+ "hermes",
+ "claude-code",
+ "opencode",
+ "goose",
+ "pi",
+ ]
+ .map(String::from)
+ .to_vec();
+ // Nothing used yet: Claude Code alone.
+ let (o, n) = super::order_for_warm(all.clone(), &[]);
+ assert_eq!((o[0].as_str(), n), ("claude-code", 1));
+ assert_eq!(o.len(), all.len());
+ // Recent runs and agents follow, newest first, three in all.
+ let phases = "{\"harness\":\"goose\",\"phase\":\"posted\"}\n{\"harness\":\"hermes\",\"phase\":\"posted\"}\nnot json\n{\"harness\":\"hermes\"}\n";
+ let agents = r#"[{"harness":"pi"},{"harness":"claude-code"},{"harness":"gone-harness"}]"#;
+ let recent = super::recent_harnesses(phases, agents);
+ assert_eq!(recent[0], "hermes");
+ let (o, n) = super::order_for_warm(all.clone(), &recent);
+ assert_eq!(n, 3);
+ assert_eq!(&o[..3], ["claude-code", "hermes", "goose"]);
+ assert_eq!(o.len(), all.len(), "the others follow, once");
+ }
+
#[test]
fn a_macos_pull_is_sized_from_the_catalog() {
let gib = 1u64 << 30;
diff --git a/libs/cua/crates/cua-daemon/src/pool/mod.rs b/libs/cua/crates/cua-daemon/src/pool/mod.rs
index 4786b5405a..886f05d78c 100644
--- a/libs/cua/crates/cua-daemon/src/pool/mod.rs
+++ b/libs/cua/crates/cua-daemon/src/pool/mod.rs
@@ -286,6 +286,13 @@ pub trait PoolHost: Send + Sync {
let _ = (endpoints, secret, harnesses);
}
+ /// What pins the install of `harness` in a prepared filesystem: the
+ /// versions of the apps a build installs for it. A saved filesystem
+ /// serves a build whose pins for the harnesses it wants are the same.
+ fn pin(&self, harness: &str) -> String {
+ harness.to_string()
+ }
+
/// Whether this machine runs on battery now (no member then).
fn on_battery(&self) -> bool;
@@ -328,6 +335,17 @@ impl Ready {
}
}
+/// The saved filesystem a member boots from.
+#[derive(Clone)]
+struct Chosen {
+ tag: String,
+ image: cua_vmm::ImageSource,
+ /// What is installed in it (id to pin).
+ have: std::collections::BTreeMap<String, String>,
+ /// It has every app the shape wants.
+ covers_all: bool,
+}
+
/// A ready member.
#[derive(Clone)]
struct Member {
@@ -371,6 +389,10 @@ struct Shape {
/// The harnesses whose agent stack a member gets before it is ready
/// ([`WarmPool::set_prepare`]); empty: members are bare.
prepare: Vec<String>,
+ /// How many of `prepare` (from the front) a member is prepared for before
+ /// it can be claimed: the apps the person most likely uses first. The
+ /// rest go into the saved filesystem in the background. 0: all of them.
+ prepare_first: usize,
/// Measured (EWMA, seconds): a member's preparation.
prepare_s: Option<f64>,
/// The saved filesystem of the first prepared member
@@ -399,6 +421,7 @@ impl Shape {
requested: None,
last_used: None,
prepare: Vec::new(),
+ prepare_first: 0,
prepare_s: None,
prepared_image: None,
}
@@ -598,6 +621,12 @@ struct Inner {
/// Why each shape's refill is deferred now, and when that was last
/// logged ([`WarmPool::defer`]).
deferred: Mutex<HashMap<String, (String, Instant)>>,
+ /// Shapes whose saved filesystem is being built now, and (after a
+ /// failure) when the next try may start.
+ building: Mutex<HashSet<String>>,
+ build_blocked: Mutex<HashMap<String, Instant>>,
+ /// Serialises changes of the index of saved filesystems.
+ index_lock: Mutex<()>,
}
/// How long the policy starts no member after other work took capacity back.
@@ -760,16 +789,55 @@ fn token_of(spec: &StartSpec) -> Option<String> {
.cloned()
}
-/// The tag of the saved filesystem of a shape's prepared members: the shape,
-/// the apps and this build (a new build installs the apps it pins).
-fn prepared_tag(key: &str, harnesses: &[String]) -> String {
+/// What a saved filesystem is keyed by, apart from the apps in it: the
+/// backend, the image it was made from and its platform. Nothing else of a
+/// shape (network preset, size, ports, environment) is in the saved files,
+/// so a change of those never throws a prepared filesystem away.
+fn prepared_base(template: &StartSpec, backend: BackendKind) -> String {
use sha2::{Digest, Sha256};
+ let image = match &template.image {
+ cua_vmm::ImageSource::Oci { reference } => reference.as_str(),
+ _ => "",
+ };
let mut h = Sha256::new();
- h.update(harnesses.join(",").as_bytes());
- h.update(env!("CARGO_PKG_VERSION").as_bytes());
+ for part in [
+ backend.as_str(),
+ image,
+ &format!("{:?}", template.os),
+ &format!("{:?}", template.arch),
+ ] {
+ h.update(part.as_bytes());
+ h.update([0]);
+ }
+ h.finalize()
+ .iter()
+ .take(6)
+ .map(|b| format!("{b:02x}"))
+ .collect()
+}
+
+/// A saved filesystem: made from `base`, with these harnesses (and their
+/// pins, [`PoolHost::pin`]) installed.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+struct PreparedEntry {
+ tag: String,
+ base: String,
+ harnesses: std::collections::BTreeMap<String, String>,
+}
+
+/// The tag of the saved filesystem of `base` with `harnesses` (id to pin).
+fn prepared_tag(base: &str, harnesses: &std::collections::BTreeMap<String, String>) -> String {
+ use sha2::{Digest, Sha256};
+ let mut h = Sha256::new();
+ for (id, pin) in harnesses {
+ h.update(id.as_bytes());
+ h.update([0]);
+ h.update(pin.as_bytes());
+ h.update([0]);
+ }
let d = h.finalize();
format!(
- "warm-{key}-{}",
+ "warm-{base}-{}",
d.iter()
.take(4)
.map(|b| format!("{b:02x}"))
@@ -815,6 +883,9 @@ impl WarmPool {
arrived: tokio::sync::Notify::new(),
due: Mutex::new(HashSet::new()),
deferred: Mutex::new(HashMap::new()),
+ building: Mutex::new(HashSet::new()),
+ build_blocked: Mutex::new(HashMap::new()),
+ index_lock: Mutex::new(()),
}),
}
}
@@ -1272,6 +1343,14 @@ impl WarmPool {
Some(key)
}
+ /// Sets how many of the shape's apps (from the front of its list) a member
+ /// is prepared for before it can be claimed ([`Shape::prepare_first`]).
+ pub fn set_prepare_first(&self, key: &str, n: usize) {
+ if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {
+ s.prepare_first = n;
+ }
+ }
+
/// Keeps `count` members of `template`'s shape regardless of use
/// (`pool.warm`, keyed by the shape).
pub fn keep(
@@ -1602,6 +1681,291 @@ impl WarmPool {
self.inner.deferred.lock().unwrap().remove(key);
}
+ fn index_path(&self) -> PathBuf {
+ self.inner.dir.join("prepared-index.json")
+ }
+
+ fn index_load(&self) -> Vec<PreparedEntry> {
+ std::fs::read(self.index_path())
+ .ok()
+ .and_then(|b| serde_json::from_slice(&b).ok())
+ .unwrap_or_default()
+ }
+
+ fn index_update(&self, f: impl FnOnce(&mut Vec<PreparedEntry>)) {
+ let _g = self.inner.index_lock.lock().unwrap();
+ let mut v = self.index_load();
+ f(&mut v);
+ let path = self.index_path();
+ if cua_home::guard_write(&path).is_ok() {
+ let _ = std::fs::create_dir_all(&self.inner.dir);
+ if let Ok(b) = serde_json::to_vec_pretty(&v) {
+ let _ = cua_home::write_private(&path, &b);
+ }
+ }
+ }
+
+ fn index_remove(&self, tag: &str) {
+ self.index_update(|v| v.retain(|e| e.tag != tag));
+ }
+
+ /// The pin of each of `harnesses` in this build.
+ fn pins(&self, harnesses: &[String]) -> std::collections::BTreeMap<String, String> {
+ harnesses
+ .iter()
+ .map(|h| (h.clone(), self.inner.host.pin(h)))
+ .collect()
+ }
+
+ /// The saved filesystem a member of `template`'s shape boots from: one
+ /// of the same image with at least the `first` apps in it (at these
+ /// pins), the one with most of `all`. A superset is as good as an exact
+ /// match; nothing but the image and the apps keys it, so a build, a
+ /// network preset or a size change does not throw it away.
+ async fn choose_prepared(
+ &self,
+ key: &str,
+ template: &StartSpec,
+ backend: BackendKind,
+ rt: &dyn VmmRuntime,
+ all: &[String],
+ first: &[String],
+ ) -> Option<Chosen> {
+ let base = prepared_base(template, backend);
+ let pins = self.pins(all);
+ let mut cands: Vec<(usize, PreparedEntry)> = Vec::new();
+ let mut why: Vec<String> = Vec::new();
+ for e in self.index_load().into_iter().filter(|e| e.base == base) {
+ let has = |h: &String| e.harnesses.get(h) == pins.get(h);
+ if let Some(m) = first.iter().find(|h| !has(h)) {
+ why.push(format!("{} lacks {m} at this build's pin", e.tag));
+ continue;
+ }
+ cands.push((all.iter().filter(|h| has(h)).count(), e));
+ }
+ cands.sort_by(|a, b| {
+ b.0.cmp(&a.0)
+ .then(a.1.harnesses.len().cmp(&b.1.harnesses.len()))
+ });
+ for (n, e) in cands {
+ match rt.find_prepared(&e.tag).await {
+ Ok(Some(image)) => {
+ tracing::info!(
+ shape = key,
+ tag = %e.tag,
+ covers = n,
+ wanted = all.len(),
+ "warm-pool: prepared filesystem reused"
+ );
+ return Some(Chosen {
+ covers_all: n == all.len(),
+ tag: e.tag,
+ image,
+ have: e.harnesses,
+ });
+ }
+ _ => {
+ why.push(format!("{} is gone from the engine", e.tag));
+ self.index_remove(&e.tag);
+ }
+ }
+ }
+ if why.is_empty() {
+ why.push(format!("none saved for base {base}"));
+ }
+ tracing::info!(
+ shape = key,
+ wanted = %first.join(","),
+ reason = %why.join("; "),
+ "warm-pool: no prepared filesystem to reuse; the member is prepared"
+ );
+ None
+ }
+
+ /// Builds the saved filesystem with all of `all` in a member of its own
+ /// (booted from `from`, which has some of them), in the background:
+ /// the `first` apps are saved as soon as they are in, then the rest.
+ /// Members that can be claimed never wait for a save.
+ #[allow(clippy::too_many_arguments)]
+ fn build_prepared(
+ &self,
+ key: &str,
+ template: &StartSpec,
+ backend: BackendKind,
+ rt: Arc<dyn VmmRuntime>,
+ all: Vec<String>,
+ first: Vec<String>,
+ from: Option<Chosen>,
+ ) {
+ if self.inner.closed.load(std::sync::atomic::Ordering::SeqCst) || !rt.keeps_prepared() {
+ return;
+ }
+ if let Some(t) = self.inner.build_blocked.lock().unwrap().get(key)
+ && *t > Instant::now()
+ {
+ return;
+ }
+ if !self.inner.building.lock().unwrap().insert(key.to_string()) {
+ return;
+ }
+ let Ok(handle) = tokio::runtime::Handle::try_current() else {
+ self.inner.building.lock().unwrap().remove(key);
+ return;
+ };
+ let (pool, key, template) = (self.clone(), key.to_string(), template.clone());
+ handle.spawn(async move {
+ let began = Instant::now();
+ let r = {
+ let work = pool.build_inner(&key, &template, backend, rt, &all, &first, from);
+ let closed = async {
+ while !pool.inner.closed.load(std::sync::atomic::Ordering::SeqCst) {
+ tokio::time::sleep(Duration::from_secs(1)).await;
+ }
+ };
+ tokio::select! { r = work => r, _ = closed => Err("the pool closed".to_string()) }
+ };
+ pool.inner.building.lock().unwrap().remove(&key);
+ match r {
+ Ok(true) => tracing::info!(
+ shape = %key,
+ built_ms = began.elapsed().as_millis() as u64,
+ since_start_s = pool.inner.epoch.elapsed().as_secs(),
+ "warm-pool: the prepared filesystem has every app"
+ ),
+ // This runtime keeps nothing: members are prepared one by one.
+ Ok(false) => {
+ pool.inner
+ .build_blocked
+ .lock()
+ .unwrap()
+ .insert(key, Instant::now() + Duration::from_secs(3600));
+ }
+ Err(e) => {
+ tracing::warn!(shape = %key, error = %e, "warm-pool: could not build the prepared filesystem; trying again later");
+ pool.inner
+ .build_blocked
+ .lock()
+ .unwrap()
+ .insert(key, Instant::now() + Duration::from_secs(600));
+ }
+ }
+ });
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ async fn build_inner(
+ &self,
+ key: &str,
+ template: &StartSpec,
+ backend: BackendKind,
+ rt: Arc<dyn VmmRuntime>,
+ all: &[String],
+ first: &[String],
+ from: Option<Chosen>,
+ ) -> Result<bool, String> {
+ let base = prepared_base(template, backend);
+ let pins = self.pins(all);
+ let name = member_name();
+ let secret = pool_secret();
+ let record = Record {
+ name: name.clone(),
+ key: key.to_string(),
+ backend,
+ secret: secret.clone(),
+ template: template.clone(),
+ keep: 0,
+ prepared: false,
+ };
+ self.write_record(&record).map_err(|e| e.to_string())?;
+ if let Err(e) = self.inner.leases.write_lease(&name, self.inner.pid) {
+ self.remove_record(&name);
+ return Err(e.to_string());
+ }
+ self.inner.starting.lock().unwrap().insert(name.clone());
+ let work = async {
+ let mut spec = template.clone();
+ spec.name = name.clone();
+ spec.pool_member = true;
+ if let Some(c) = &from {
+ spec.image = c.image.clone();
+ }
+ spec.env.insert("CUA_ENV_TOKEN".into(), secret.clone());
+ spec.labels
+ .insert(cua_vmm::container::LABEL_POOL.into(), "true".into());
+ let inst = rt.start(&spec).await.map_err(|e| e.to_string())?;
+ if backend == BackendKind::Container {
+ self.inner
+ .host
+ .member_ready(
+ &inst.endpoints,
+ &secret,
+ has_desktop(&spec),
+ MEMBER_READY_TIMEOUT,
+ )
+ .await?;
+ }
+ let mut have = from.map(|c| c.have).unwrap_or_default();
+ // The apps a member needs to be claimed, saved at once; then the
+ // rest.
+ for layer in [first, all] {
+ let missing: Vec<String> = layer
+ .iter()
+ .filter(|h| have.get(*h) != pins.get(*h))
+ .cloned()
+ .collect();
+ if missing.is_empty() {
+ continue;
+ }
+ let began = Instant::now();
+ self.inner
+ .host
+ .prepare(&inst.endpoints, &secret, &missing, PREPARE_TIMEOUT)
+ .await?;
+ for h in &missing {
+ if let Some(p) = pins.get(h) {
+ have.insert(h.clone(), p.clone());
+ }
+ }
+ let tag = prepared_tag(&base, &have);
+ let saving = Instant::now();
+ match rt.save_prepared(&name, &tag).await {
+ Ok(Some(_)) => {
+ let entry = PreparedEntry {
+ tag: tag.clone(),
+ base: base.clone(),
+ harnesses: have.clone(),
+ };
+ self.index_update(|v| {
+ v.retain(|e| e.tag != tag);
+ v.push(entry);
+ });
+ tracing::info!(
+ shape = key,
+ %tag,
+ harnesses = %have.keys().cloned().collect::<Vec<_>>().join(","),
+ prepare_ms = began.elapsed().as_millis() as u64 - saving.elapsed().as_millis() as u64,
+ save_ms = saving.elapsed().as_millis() as u64,
+ since_start_s = self.inner.epoch.elapsed().as_secs(),
+ "warm-pool: the prepared filesystem is kept for the next member"
+ );
+ }
+ Ok(None) => return Ok(false),
+ Err(e) => return Err(e.to_string()),
+ }
+ }
+ Ok(true)
+ }
+ .await;
+ match rt.delete(&name).await {
+ Ok(()) | Err(VmmError::NotFound(_)) => self.forget_member(&name),
+ Err(e) => {
+ tracing::warn!(member = %name, error = %e, "could not remove a warm-pool builder")
+ }
+ }
+ self.inner.starting.lock().unwrap().remove(&name);
+ work
+ }
+
async fn start_member(
&self,
key: &str,
@@ -1615,18 +1979,26 @@ impl WarmPool {
let shapes = self.inner.shapes.lock().unwrap();
shapes.get(key).map(|s| s.keep()).unwrap_or(0)
};
- let prepare = {
+ let (all, first_n) = {
let shapes = self.inner.shapes.lock().unwrap();
shapes
.get(key)
- .map(|s| s.prepare.clone())
+ .map(|s| (s.prepare.clone(), s.prepare_first))
.unwrap_or_default()
};
- let prepare = if self.inner.host.can_prepare(backend) {
- prepare
+ let all = if self.inner.host.can_prepare(backend) {
+ all
} else {
Vec::new()
};
+ // The apps a member must have before it can be claimed; the others
+ // go into the saved filesystem in the background.
+ let first: Vec<String> = all
+ .iter()
+ .take(if first_n == 0 { all.len() } else { first_n })
+ .cloned()
+ .collect();
+ let prepare = first.clone();
let mut record = Record {
name: name.clone(),
key: key.to_string(),
@@ -1646,33 +2018,19 @@ impl WarmPool {
let mut spec = template.clone();
spec.name = name.clone();
spec.pool_member = true;
- // A member of a shape whose first member was prepared boots from that
- // member's saved filesystem: its apps are in it already, so it is
- // ready in the time of a boot, not of an install.
- let mut saved = if prepare.is_empty() {
+ // A member boots from a saved filesystem that has the apps it needs
+ // (a build or a daemon before saved it): they are in it already, so
+ // it is ready in the time of a boot, not of an install.
+ let chosen = if all.is_empty() {
None
} else {
- self.inner
- .shapes
- .lock()
- .unwrap()
- .get(key)
- .and_then(|s| s.prepared_image.clone())
+ self.choose_prepared(key, template, backend, rt.as_ref(), &all, &first)
+ .await
};
- // A daemon that restarted: what the last one saved is still there.
- if saved.is_none() && !prepare.is_empty() {
- let tag = prepared_tag(key, &prepare);
- if let Ok(Some(image)) = rt.find_prepared(&tag).await {
- tracing::info!(
- shape = key,
- "warm-pool: found the prepared filesystem a previous daemon saved"
- );
- if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {
- s.prepared_image = Some((tag.clone(), image.clone()));
- }
- saved = Some((tag, image));
- }
+ if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {
+ s.prepared_image = chosen.as_ref().map(|c| (c.tag.clone(), c.image.clone()));
}
+ let saved = chosen.as_ref().map(|c| (c.tag.clone(), c.image.clone()));
if let Some((_, image)) = &saved {
spec.image = image.clone();
}
@@ -1692,12 +2050,15 @@ impl WarmPool {
let mut prepared = false;
let ready = async {
let inst = rt.start(&spec).await.map_err(|e| e.to_string());
- if inst.is_err() && saved.is_some() {
+ if inst.is_err()
+ && let Some((tag, _)) = &saved
+ {
// The saved filesystem does not start any more (the engine
// lost it): the next member is prepared from scratch.
if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {
s.prepared_image = None;
}
+ self.index_remove(tag);
}
let inst = inst?;
// A container member runs with its pool secret: it is ready
@@ -1746,22 +2107,6 @@ impl WarmPool {
s.prepare_s = ewma(s.prepare_s, took);
}
tracing::info!(member = %name, harnesses = %prepare.join(","), prepare_ms = (took * 1000.0) as u64, "warm-pool member prepared");
- // Its filesystem is kept (before anyone can claim it),
- // so the next member boots from it.
- let tag = prepared_tag(key, &prepare);
- let saving = Instant::now();
- match rt.save_prepared(&name, &tag).await {
- Ok(Some(image)) => {
- tracing::info!(member = %name, save_ms = saving.elapsed().as_millis() as u64, "warm-pool: the prepared filesystem is kept for the next member");
- if let Some(s) = self.inner.shapes.lock().unwrap().get_mut(key) {
- s.prepared_image = Some((tag, image));
- }
- }
- Ok(None) => {}
- Err(e) => {
- tracing::warn!(member = %name, error = %e, "warm-pool: could not keep the prepared filesystem; the next member is prepared again");
- }
- }
}
Err(e) => {
tracing::warn!(member = %name, error = %e, "warm-pool member could not be prepared; it is claimed bare");
@@ -1786,9 +2131,25 @@ impl WarmPool {
member = %name,
shape = key,
ready_ms = started.elapsed().as_millis() as u64,
+ claimable_after_start_s = self.inner.epoch.elapsed().as_secs_f64().round() as u64,
+ prepared_for = %prepare.join(","),
+ from = saved.as_ref().map_or("the image", |(t, _)| t.as_str()),
footprint_mb,
"warm-pool member ready"
);
+ // The saved filesystem is built beside the members, never from one
+ // that can be claimed (a claim changes what is in it).
+ if !all.is_empty() && !chosen.as_ref().is_some_and(|c| c.covers_all) {
+ self.build_prepared(
+ key,
+ template,
+ backend,
+ rt.clone(),
+ all.clone(),
+ first.clone(),
+ chosen.clone(),
+ );
+ }
// A member that booted from the saved filesystem has nothing in its
// file cache: its apps are read now, while it waits to be claimed.
if saved.is_some()
@@ -2203,9 +2564,8 @@ impl WarmPool {
fn forget_shape(&self, key: &str) {
let gone = self.inner.shapes.lock().unwrap().remove(key);
if let Some(s) = gone {
- if let Some((tag, _)) = &s.prepared_image {
- self.drop_saved(s.rt.clone(), tag.clone());
- }
+ // The saved filesystem stays (a shape that comes back finds it
+ // by its image and apps; the disk budget removes the old).
for m in s.members {
self.discard(s.rt.clone(), m.name);
}
@@ -2380,6 +2740,9 @@ mod tests {
async fn fork(&self, _: &str, _: &str) -> VResult<()> {
Ok(())
}
+ fn keeps_prepared(&self) -> bool {
+ self.can_save.load(SeqCst)
+ }
async fn save_prepared(&self, _: &str, tag: &str) -> VResult<Option<ImageSource>> {
if !self.can_save.load(SeqCst) {
return Ok(None);
@@ -3562,17 +3925,26 @@ mod tests {
assert_eq!(f.rt.members().len(), 0, "the free floor still holds");
}
+ /// Waits until the runtime kept `n` filesystems.
+ async fn until_saved(f: &Fixture, n: usize) {
+ for _ in 0..300 {
+ if f.rt.saved.lock().unwrap().len() >= n {
+ return;
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ panic!("never saved {n}: {:?}", f.rt.saved.lock().unwrap());
+ }
+
#[tokio::test]
- async fn later_members_boot_from_the_filesystem_of_the_first_prepared_one() {
+ async fn later_members_boot_from_the_filesystem_built_beside_the_first() {
let f = fixture(PoolSettings::default(), FakeRuntime::default());
f.rt.can_save.store(true, SeqCst);
let key = keep_warm_space(&f, &spec("one", "t")).await;
until_ready(&f, &key, 1).await;
- assert_eq!(
- f.rt.saved.lock().unwrap().len(),
- 1,
- "kept once, before it was ready"
- );
+ // The first member is claimable as prepared; its filesystem is kept
+ // by a builder of its own, never by a member someone can claim.
+ until_saved(&f, 1).await;
for i in 0..3 {
assert!(
f.pool
@@ -3587,28 +3959,142 @@ mod tests {
until_ready(&f, &key, 1).await;
}
let images = f.rt.images.lock().unwrap().clone();
- assert_eq!(images.len(), 4);
assert!(
!images[0].starts_with("local/prepared:"),
"the first boots the image"
);
assert!(
- images[1..]
- .iter()
- .all(|i| i.starts_with("local/prepared:warm-")),
+ images[images.len() - 1].starts_with("local/prepared:warm-"),
"{images:?}"
);
- // Prepared once; the others only booted.
- assert_eq!(f.host.prepared.lock().unwrap().len(), 1);
- assert_eq!(f.rt.saved.lock().unwrap().len(), 1);
+ assert_eq!(f.rt.saved.lock().unwrap().len(), 1, "kept once");
assert!(f.pool.records().iter().all(|r| r.prepared));
- // A daemon that stops keeps it for the next one; it goes with its
- // shape (forgotten after the policy's horizon without a create).
+ // A daemon that stops keeps it for the next one.
f.pool.retain_explicit(|src| src != WARM_SOURCE);
f.pool.drain().await;
assert!(f.rt.dropped.lock().unwrap().is_empty());
}
+ #[tokio::test]
+ async fn the_first_member_is_claimable_after_the_first_apps_and_the_rest_follow() {
+ let f = fixture(PoolSettings::default(), FakeRuntime::default());
+ f.rt.can_save.store(true, SeqCst);
+ let rt: Arc<dyn VmmRuntime> = f.rt.clone();
+ let s0 = spec("one", "t");
+ let key = f
+ .pool
+ .set_explicit_prepared(
+ WARM_SOURCE,
+ &s0,
+ BackendKind::Container,
+ true,
+ rt,
+ 1,
+ Some(1),
+ None,
+ Some(Duration::from_secs(7200)),
+ Some(vec!["claude-code".into(), "hermes".into(), "goose".into()]),
+ )
+ .unwrap();
+ f.pool.set_prepare_first(&key, 1);
+ until_ready(&f, &key, 1).await;
+ assert_eq!(
+ f.host.prepared.lock().unwrap()[0],
+ ["claude-code"],
+ "the member waited for the first app only"
+ );
+ // Beside it: the first app is kept at once, then the whole set.
+ until_saved(&f, 2).await;
+ let base = prepared_base(&s0, BackendKind::Container);
+ let idx = f.pool.index_load();
+ let sizes: Vec<usize> = idx.iter().map(|e| e.harnesses.len()).collect();
+ assert!(sizes.contains(&1) && sizes.contains(&3), "{sizes:?}");
+ assert!(idx.iter().all(|e| e.base == base));
+ // The next member starts from the full one, with nothing to install.
+ f.pool
+ .claim(&spec("s", "t1"), BackendKind::Container, true)
+ .await
+ .unwrap();
+ until_ready(&f, &key, 1).await;
+ let before = f.host.prepared.lock().unwrap().len();
+ let images = f.rt.images.lock().unwrap().clone();
+ assert!(images.last().unwrap().starts_with("local/prepared:warm-"));
+ assert_eq!(f.host.prepared.lock().unwrap().len(), before);
+ let full = idx
+ .iter()
+ .max_by_key(|e| e.harnesses.len())
+ .unwrap()
+ .tag
+ .clone();
+ assert!(images.last().unwrap().ends_with(&full), "{images:?}");
+ }
+
+ #[tokio::test]
+ async fn a_changed_network_preset_or_size_keeps_the_prepared_filesystem() {
+ let f = fixture(PoolSettings::default(), FakeRuntime::default());
+ f.rt.can_save.store(true, SeqCst);
+ let s0 = spec("one", "t");
+ let have: std::collections::BTreeMap<String, String> = ["claude-code", "openai-codex"]
+ .map(|h| (h.to_string(), h.to_string()))
+ .into();
+ let base = prepared_base(&s0, BackendKind::Container);
+ let tag = prepared_tag(&base, &have);
+ f.rt.saved.lock().unwrap().push(tag.clone());
+ f.pool.index_update(|v| {
+ v.push(PreparedEntry {
+ tag: tag.clone(),
+ base,
+ harnesses: have,
+ })
+ });
+ // The same image under another network preset and a bigger size is
+ // another shape, but the same saved files.
+ let mut s1 = spec("two", "t");
+ s1.restrict_network = true;
+ s1.cpus = 4;
+ assert_ne!(
+ shape_key(&s0, BackendKind::Container, true),
+ shape_key(&s1, BackendKind::Container, true)
+ );
+ let key = keep_warm_space(&f, &s1).await;
+ until_ready(&f, &key, 1).await;
+ assert!(
+ f.host.prepared.lock().unwrap().is_empty(),
+ "nothing prepared"
+ );
+ assert!(
+ f.rt.images.lock().unwrap()[0].starts_with("local/prepared:warm-"),
+ "booted from the kept filesystem"
+ );
+ }
+
+ #[tokio::test]
+ async fn a_filesystem_prepared_at_another_pin_is_not_reused() {
+ let f = fixture(PoolSettings::default(), FakeRuntime::default());
+ f.rt.can_save.store(true, SeqCst);
+ let s0 = spec("one", "t");
+ let have: std::collections::BTreeMap<String, String> = ["claude-code", "openai-codex"]
+ .map(|h| (h.to_string(), "old-version".to_string()))
+ .into();
+ let base = prepared_base(&s0, BackendKind::Container);
+ let tag = prepared_tag(&base, &have);
+ f.rt.saved.lock().unwrap().push(tag.clone());
+ f.pool.index_update(|v| {
+ v.push(PreparedEntry {
+ tag,
+ base,
+ harnesses: have,
+ })
+ });
+ let key = keep_warm_space(&f, &s0).await;
+ until_ready(&f, &key, 1).await;
+ assert_eq!(
+ f.host.prepared.lock().unwrap()[0],
+ ["claude-code", "openai-codex"],
+ "prepared again for the apps this build installs"
+ );
+ }
+
#[tokio::test]
async fn a_runtime_that_cannot_save_prepares_every_member() {
let f = fixture(PoolSettings::default(), FakeRuntime::default());
@@ -3637,14 +4123,23 @@ mod tests {
async fn a_restarted_daemon_finds_the_filesystem_the_last_one_saved() {
let f = fixture(PoolSettings::default(), FakeRuntime::default());
f.rt.can_save.store(true, SeqCst);
- // What a previous daemon saved for this shape and these apps.
+ // What a previous daemon saved for this image and these apps (more
+ // than are wanted, and under another network preset: still reused).
let s0 = spec("one", "t");
let key = shape_key(&s0, BackendKind::Container, true).unwrap();
- let tag = prepared_tag(
- &key,
- &["claude-code".to_string(), "openai-codex".to_string()],
- );
+ let have: std::collections::BTreeMap<String, String> =
+ ["claude-code", "openai-codex", "pi"]
+ .map(|h| (h.to_string(), h.to_string()))
+ .into();
+ let tag = prepared_tag(&prepared_base(&s0, BackendKind::Container), &have);
f.rt.saved.lock().unwrap().push(tag.clone());
+ f.pool.index_update(|v| {
+ v.push(PreparedEntry {
+ tag: tag.clone(),
+ base: prepared_base(&s0, BackendKind::Container),
+ harnesses: have.clone(),
+ })
+ });
let key2 = keep_warm_space(&f, &s0).await;
assert_eq!(key, key2);
until_ready(&f, &key, 1).await;
diff --git a/libs/cua/crates/cua-disk/src/gc.rs b/libs/cua/crates/cua-disk/src/gc.rs
index fcf36486f9..5d717a6ebd 100644
--- a/libs/cua/crates/cua-disk/src/gc.rs
+++ b/libs/cua/crates/cua-disk/src/gc.rs
@@ -78,6 +78,10 @@ pub enum KeepReason {
/// failed or was interrupted, and no later create from a newer base has
/// succeeded.
NoNewerBaseInUse,
+ /// One of the two newest filesystems the warm pool saved with the apps
+ /// installed: evicting it makes the next start prepare a member for
+ /// minutes.
+ LatestPrepared,
}
impl std::fmt::Display for KeepReason {
@@ -88,6 +92,7 @@ impl std::fmt::Display for KeepReason {
KeepReason::NoNewerBaseInUse => {
f.write_str("kept for a retry: no newer base is in use")
}
+ KeepReason::LatestPrepared => f.write_str("a recent prepared warm-pool filesystem"),
}
}
}
@@ -207,6 +212,19 @@ pub fn kept_bases(report: &Report, pins: &[Pin]) -> HashMap<usize, KeepReason> {
out.entry(n).or_insert(KeepReason::NoNewerBaseInUse);
}
}
+ // The warm pool's saved filesystems are rebuilt in minutes and are
+ // bigger than a small budget leaves room for: the newest two stay.
+ let mut prepared: Vec<(usize, u64)> = report
+ .items
+ .iter()
+ .enumerate()
+ .filter(|(_, i)| !i.orphan && i.category.is_cache() && i.kind == "prepared")
+ .map(|(n, i)| (n, i.created.unwrap_or(0)))
+ .collect();
+ prepared.sort_by_key(|(_, c)| std::cmp::Reverse(*c));
+ for (n, _) in prepared.into_iter().take(2) {
+ out.entry(n).or_insert(KeepReason::LatestPrepared);
+ }
out
}
@@ -687,6 +705,34 @@ mod tests {
);
}
+ #[test]
+ fn the_newest_prepared_warm_pool_filesystems_survive_a_small_budget() {
+ use super::{KeepReason, kept_bases};
+ use crate::scan::{Category, Item, Report};
+ let item = |name: &str, created: u64| Item {
+ category: Category::DockerImages,
+ kind: "prepared".into(),
+ name: name.into(),
+ bytes: 6 << 30,
+ created: Some(created),
+ evictable: true,
+ ..Default::default()
+ };
+ let report = Report {
+ home: Default::default(),
+ space: None,
+ budget: "auto".into(),
+ budget_bytes: Some(8 << 30),
+ cache_bytes: 18 << 30,
+ items: vec![item("old", 1), item("mid", 2), item("new", 3)],
+ notes: vec![],
+ };
+ let kept = kept_bases(&report, &[]);
+ assert_eq!(kept.get(&2), Some(&KeepReason::LatestPrepared));
+ assert_eq!(kept.get(&1), Some(&KeepReason::LatestPrepared));
+ assert!(!kept.contains_key(&0), "the oldest is still evictable");
+ }
+
#[test]
fn the_current_pins_are_the_macos_and_linux_defaults() {
let pins = super::current_pins();
diff --git a/libs/cua/crates/cua-vmm/src/container/mod.rs b/libs/cua/crates/cua-vmm/src/container/mod.rs
index 2123d6e227..d0f3ae886f 100644
--- a/libs/cua/crates/cua-vmm/src/container/mod.rs
+++ b/libs/cua/crates/cua-vmm/src/container/mod.rs
@@ -1702,6 +1702,10 @@ impl Runtime for ContainerRuntime {
))
}
+ fn keeps_prepared(&self) -> bool {
+ true
+ }
+
async fn save_prepared(&self, name: &str, tag: &str) -> Result<Option<ImageSource>> {
validate_name(tag)?;
let i = self
diff --git a/libs/cua/crates/cua-vmm/src/managed.rs b/libs/cua/crates/cua-vmm/src/managed.rs
index 2b620b31de..c00f60346a 100644
--- a/libs/cua/crates/cua-vmm/src/managed.rs
+++ b/libs/cua/crates/cua-vmm/src/managed.rs
@@ -1685,6 +1685,10 @@ mod engine_impl {
self.engine().await?.claim(member, new_name, identity).await
}
+ fn keeps_prepared(&self) -> bool {
+ true
+ }
+
async fn save_prepared(&self, name: &str, tag: &str) -> Result<Option<ImageSource>> {
let _busy = self.busy();
self.engine().await?.save_prepared(name, tag).await
diff --git a/libs/cua/crates/cua-vmm/src/runtime.rs b/libs/cua/crates/cua-vmm/src/runtime.rs
index 69ed5aabe7..57a559ef73 100644
--- a/libs/cua/crates/cua-vmm/src/runtime.rs
+++ b/libs/cua/crates/cua-vmm/src/runtime.rs
@@ -85,6 +85,11 @@ pub trait Runtime: Send + Sync {
})
}
+ /// Whether [`Runtime::save_prepared`] keeps an image on this backend.
+ fn keeps_prepared(&self) -> bool {
+ false
+ }
+
/// Saves the filesystem of the running instance `name` as an image that
/// instances start from instead of the one they were made from (what is
/// installed in it stays), named `tag`, and returns it. Used by the warm