Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(spaces): say why a warm-pool refill waits, and end the agents of a deleted Space (#6632)

ddupont1c4cfba90f01
diff --git a/libs/cua/crates/cua-cli/src/persistent_cmd.rs b/libs/cua/crates/cua-cli/src/persistent_cmd.rs
index bd30fc16e4..c3811a7ebb 100644
--- a/libs/cua/crates/cua-cli/src/persistent_cmd.rs
+++ b/libs/cua/crates/cua-cli/src/persistent_cmd.rs
@@ -247,7 +247,9 @@ pub async fn run(
hint_line(out, "No agents. Add one with `cua agent add`.");
} else {
for a in all {
- let state = if a.paused {
+ let state = if a.space_state == "gone" {
+ "gone"
+ } else if a.paused {
"paused"
} else if a.run_id.is_some() {
"running"
diff --git a/libs/cua/crates/cua-daemon/src/pool/mod.rs b/libs/cua/crates/cua-daemon/src/pool/mod.rs
index 0645e5d92a..4786b5405a 100644
--- a/libs/cua/crates/cua-daemon/src/pool/mod.rs
+++ b/libs/cua/crates/cua-daemon/src/pool/mod.rs
@@ -89,6 +89,13 @@ const PREPARE_TIMEOUT: Duration = Duration::from_secs(900);
/// until it is used again.
const MAX_FAILURES: u32 = 3;
+/// How long a shape that failed [`MAX_FAILURES`] starts in a row waits
+/// before it tries one more (a registry or engine that was down recovers).
+const FAILURE_COOLDOWN: Duration = Duration::from_secs(300);
+
+/// How often a refill that stays deferred for the same reason says so again.
+const DEFER_LOG_EVERY: Duration = Duration::from_secs(120);
+
/// The estimate of a member's memory until one was measured: a container
/// Space at idle (a gVisor desktop is well under this). VMs count their
/// whole memory.
@@ -342,6 +349,9 @@ struct Shape {
members: Vec<Member>,
starting: u32,
failures: u32,
+ /// When the last member start failed (the cool-down after
+ /// [`MAX_FAILURES`] runs from it).
+ failed_at: Option<Instant>,
/// Measured idle footprint of a member (MiB).
footprint_mb: Option<u64>,
/// Its creates.
@@ -379,6 +389,7 @@ impl Shape {
members: vec![],
starting: 0,
failures: 0,
+ failed_at: None,
footprint_mb: None,
arrivals: Default::default(),
boot_s: None,
@@ -584,6 +595,9 @@ struct Inner {
arrived: tokio::sync::Notify,
/// Shapes with a refill scheduled that has not started yet.
due: Mutex<HashSet<String>>,
+ /// Why each shape's refill is deferred now, and when that was last
+ /// logged ([`WarmPool::defer`]).
+ deferred: Mutex<HashMap<String, (String, Instant)>>,
}
/// How long the policy starts no member after other work took capacity back.
@@ -800,6 +814,7 @@ impl WarmPool {
held_off_until: Mutex::new(None),
arrived: tokio::sync::Notify::new(),
due: Mutex::new(HashSet::new()),
+ deferred: Mutex::new(HashMap::new()),
}),
}
}
@@ -1416,9 +1431,26 @@ impl WarmPool {
let shapes = self.inner.shapes.lock().unwrap();
let Some(s) = shapes.get(key) else { return };
let have = s.members.len() as u32 + s.starting;
- if have >= s.target(auto) || s.failures >= MAX_FAILURES {
+ if have >= s.target(auto) {
+ self.undefer(key);
return;
}
+ if s.failures >= MAX_FAILURES {
+ let wait = s
+ .failed_at
+ .map(|t| FAILURE_COOLDOWN.saturating_sub(t.elapsed()))
+ .unwrap_or_default();
+ if !wait.is_zero() {
+ let why = format!(
+ "the last {} member starts failed; trying again in {}s",
+ s.failures,
+ wait.as_secs()
+ );
+ drop(shapes);
+ self.defer(key, "failing", why);
+ return;
+ }
+ }
(s.template.clone(), s.backend, s.rt.clone(), s.estimate_mb())
};
if !self.claims_work(backend, &rt).await {
@@ -1429,21 +1461,28 @@ impl WarmPool {
let shapes = self.inner.shapes.lock().unwrap();
shapes.get(key).map(|s| s.keep()).unwrap_or(0)
};
- if self.held_off() || (keep == 0 && self.inner.host.busy()) {
- tracing::debug!(
- shape = key,
- "warm pool: the machine's capacity is wanted elsewhere"
+ if self.held_off() {
+ self.defer(
+ key,
+ "held off",
+ "other work took capacity back a moment ago; no member starts for a while"
+ .into(),
+ );
+ return;
+ }
+ if keep == 0 && self.inner.host.busy() {
+ self.defer(
+ key,
+ "busy",
+ "the machine's CPUs are busy with other work".into(),
);
return;
}
if keep > 0 && self.inner.host.memory(backend).await.is_none() {
self.inner.host.wake(backend).await;
}
- if !self.memory_allows(backend, estimate, key).await {
- tracing::debug!(
- shape = key,
- "warm pool: not enough memory for another member"
- );
+ if let Err(why) = self.memory_allows(backend, estimate, key).await {
+ self.defer(key, "memory", why);
return;
}
{
@@ -1469,13 +1508,16 @@ impl WarmPool {
match started {
Ok(m) => {
s.failures = 0;
+ s.failed_at = None;
if s.footprint_mb.is_none() || m.footprint_mb > 0 {
s.footprint_mb = Some(m.footprint_mb);
}
s.members.push(m);
+ self.undefer(key);
}
Err(e) => {
s.failures += 1;
+ s.failed_at = Some(Instant::now());
tracing::warn!(shape = key, error = %e, "a warm-pool member did not start");
}
}
@@ -1493,9 +1535,14 @@ impl WarmPool {
/// built-in runtime of 4 GiB, 15 percent is less than one warm Space):
/// it still needs the free floor, so it never takes memory a Space in use
/// needs.
- async fn memory_allows(&self, backend: BackendKind, estimate_mb: u64, key: &str) -> bool {
+ async fn memory_allows(
+ &self,
+ backend: BackendKind,
+ estimate_mb: u64,
+ key: &str,
+ ) -> Result<(), String> {
let Some(mem) = self.inner.host.memory(backend).await else {
- return false;
+ return Err("the engine's memory cannot be read now".into());
};
let one_warm_home = self
.inner
@@ -1516,10 +1563,43 @@ impl WarmPool {
})
.sum()
};
- if one_warm_home && held == 0 {
- return fits(mem, 100, 0, estimate_mb);
+ let (pct, held) = if one_warm_home && held == 0 {
+ (100, 0)
+ } else {
+ (pct, held)
+ };
+ if fits(mem, pct, held, estimate_mb) {
+ return Ok(());
+ }
+ let floor = memory_floor_mb(mem.total_mb);
+ let used = mem.total_mb.saturating_sub(mem.available_mb);
+ Err(format!(
+ "not enough free memory: {} MiB of {} MiB in use by running Spaces, a member needs about {} MiB and {} MiB stay free (pool holds {} of its {} MiB share)",
+ used,
+ mem.total_mb,
+ estimate_mb,
+ floor,
+ held,
+ mem.total_mb * pct / 100
+ ))
+ }
+
+ /// Says why the refill of `key` waits, once, and again when the reason
+ /// changes or every [`DEFER_LOG_EVERY`] while it stays.
+ fn defer(&self, key: &str, kind: &str, why: String) {
+ let mut d = self.inner.deferred.lock().unwrap();
+ let fresh = d
+ .get(key)
+ .is_none_or(|(k, at)| k != kind || at.elapsed() >= DEFER_LOG_EVERY);
+ if fresh {
+ tracing::info!(shape = key, reason = %why, "warm pool: refill deferred");
+ d.insert(key.to_string(), (kind.to_string(), Instant::now()));
}
- fits(mem, pct, held, estimate_mb)
+ }
+
+ /// The refill of `key` is no longer deferred.
+ fn undefer(&self, key: &str) {
+ self.inner.deferred.lock().unwrap().remove(key);
}
async fn start_member(
@@ -2975,6 +3055,56 @@ mod tests {
assert_eq!(f.pool.status().members, 0);
}
+ #[tokio::test]
+ async fn a_refill_short_of_memory_says_why_and_goes_ahead_when_memory_frees() {
+ let f = fixture(PoolSettings::default(), FakeRuntime::default());
+ *f.host.memory.lock().unwrap() = Memory {
+ total_mb: 6144,
+ available_mb: 1500,
+ };
+ let key = warm(&f, &spec("one", "t")).await;
+ assert_eq!(f.rt.starts.load(SeqCst), 0);
+ let said = f.pool.inner.deferred.lock().unwrap().get(&key).cloned();
+ assert_eq!(said.map(|(kind, _)| kind), Some("memory".to_string()));
+ // A Space deleted: the next tick's refill starts the member.
+ *f.host.memory.lock().unwrap() = Memory {
+ total_mb: 6144,
+ available_mb: 5000,
+ };
+ f.pool.refill(&key).await;
+ assert_eq!(f.rt.starts.load(SeqCst), 1);
+ assert_eq!(f.pool.status().members, 1);
+ assert!(f.pool.inner.deferred.lock().unwrap().get(&key).is_none());
+ // Never more than the pool's size.
+ f.pool.refill(&key).await;
+ assert_eq!(f.rt.starts.load(SeqCst), 1);
+ }
+
+ #[tokio::test]
+ async fn failed_starts_wait_out_a_cooldown_and_then_try_again() {
+ let f = fixture(PoolSettings::default(), FakeRuntime::default());
+ f.rt.fail_starts.store(true, SeqCst);
+ let key = warm(&f, &spec("one", "t")).await;
+ assert_eq!(f.rt.starts.load(SeqCst), MAX_FAILURES);
+ // Still cooling down: no start, and the reason is recorded.
+ f.pool.refill(&key).await;
+ assert_eq!(f.rt.starts.load(SeqCst), MAX_FAILURES);
+ let said = f.pool.inner.deferred.lock().unwrap().get(&key).cloned();
+ assert_eq!(said.map(|(kind, _)| kind), Some("failing".to_string()));
+ // After the cooldown one more start is tried, and a good one resets.
+ f.rt.fail_starts.store(false, SeqCst);
+ f.pool
+ .inner
+ .shapes
+ .lock()
+ .unwrap()
+ .get_mut(&key)
+ .unwrap()
+ .failed_at = Some(Instant::now() - FAILURE_COOLDOWN - Duration::from_secs(1));
+ f.pool.refill(&key).await;
+ assert_eq!(f.pool.status().members, 1);
+ }
+
#[tokio::test]
async fn failed_starts_stop_refilling_and_leave_nothing() {
let f = fixture(PoolSettings::default(), FakeRuntime::default());
diff --git a/libs/cua/crates/cua-spaces-ext/src/lib.rs b/libs/cua/crates/cua-spaces-ext/src/lib.rs
index 11771fe069..2153a1a4d6 100644
--- a/libs/cua/crates/cua-spaces-ext/src/lib.rs
+++ b/libs/cua/crates/cua-spaces-ext/src/lib.rs
@@ -319,6 +319,16 @@ impl SpacesExtension for DriveExtension {
})
}
+ fn space_deleted<'a>(
+ &'a self,
+ spaces: &'a cua_spaces::Spaces,
+ space: &'a str,
+ ) -> BoxFuture<'a, ()> {
+ Box::pin(async move {
+ spaces.persistent().space_deleted(space).await;
+ })
+ }
+
fn as_any(&self) -> &dyn Any {
self
}
diff --git a/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs b/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs
index 4839d603b1..0cb935c69d 100644
--- a/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs
+++ b/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs
@@ -112,6 +112,10 @@ pub enum SpaceState {
Suspended,
/// A cloud Space we released on pause; resume creates it again.
Released,
+ /// The Space was deleted (or is not registered any more): the agent is
+ /// paused for good and nothing supervises it. Its home stays in the Cua
+ /// Volume; `agent rm` forgets the record.
+ Gone,
}
/// What resume needs to create a released cloud Space again.
@@ -490,6 +494,9 @@ pub(crate) struct Live {
joining: HashMap<String, u64>,
/// Agent-to-agent wakes per Space ([`wake`]).
wake: wake::LiveWake,
+ /// Local Spaces some agent names that the registry does not have, and
+ /// since when ([`Persistent::reconcile_gone`]).
+ missing: HashMap<String, std::time::Instant>,
/// When a standby run may be started again after one that failed, by member.
standby_retry: HashMap<String, std::time::Instant>,
/// The credential hash an agent's run would start with now, by agent
@@ -772,6 +779,111 @@ impl Persistent {
Ok(rec)
}
+ /// A Space was deleted: the persistent agents that lived in it go with
+ /// it, as `agent rm` would (run stopped, open tasks failed, routines and
+ /// standby sessions dropped, record forgotten; the agent's home stays
+ /// in the Cua Volume). An agent whose Space is only being replaced is
+ /// kept: a cloud Space released on pause (`recreate`) and a durable
+ /// member, whose copy the coordinator renews. Returns the names removed.
+ pub async fn space_deleted(&self, space: &str) -> Vec<String> {
+ let Ok(all) = self.list() else {
+ return vec![];
+ };
+ let mut removed = vec![];
+ for rec in all.into_iter().filter(|r| {
+ r.space == space
+ && r.recreate.is_none()
+ && r.space_state != SpaceState::Released
+ && !r.policy.durable
+ && r.durable_space.is_none()
+ }) {
+ match tokio::time::timeout(Duration::from_secs(30), self.remove(&rec.name)).await {
+ Ok(Ok(_)) => {
+ tracing::info!(agent = %rec.name, space, "persistent agents: removed an agent whose Space was deleted");
+ removed.push(rec.name);
+ }
+ Ok(Err(e)) => {
+ tracing::warn!(agent = %rec.name, space, "persistent agents: could not remove an agent of a deleted Space: {e}")
+ }
+ Err(_) => {
+ tracing::warn!(agent = %rec.name, space, "persistent agents: removing an agent of a deleted Space timed out")
+ }
+ }
+ }
+ removed
+ }
+
+ /// Marks `gone` the agents whose local Space is no longer registered
+ /// (deleted behind the supervisor's back: the sandbox CLI, an older
+ /// daemon, a crash between the delete and its cleanup), once the Space
+ /// has been missing for `grace` (a Space deleted and created again under
+ /// its name is not missed). A gone agent is paused and unsupervised, its
+ /// runs, sessions and routines are dropped, its home stays in the Cua
+ /// Volume, and `agent rm` forgets it. Returns the names marked.
+ pub async fn reconcile_gone(&self, grace: Duration) -> Vec<String> {
+ let (Ok(all), Ok(known)) = (self.list(), self.spaces.list()) else {
+ return vec![];
+ };
+ let known: std::collections::HashSet<String> = known.into_iter().map(|s| s.id).collect();
+ let now = Instant::now();
+ let mut due = vec![];
+ {
+ let mut live = self.live.lock().await;
+ live.missing.retain(|space, _| {
+ all.iter()
+ .any(|r| &r.space == space && r.space_state != SpaceState::Gone)
+ && !known.contains(space)
+ });
+ for rec in all.iter().filter(|r| {
+ r.space_state != SpaceState::Gone
+ && r.space.starts_with("local:")
+ && !known.contains(&r.space)
+ && r.recreate.is_none()
+ && !identity::is_external(r)
+ }) {
+ let since = *live.missing.entry(rec.space.clone()).or_insert(now);
+ if now.duration_since(since) >= grace {
+ due.push(rec.clone());
+ }
+ }
+ }
+ let mut marked = vec![];
+ for rec in due {
+ let name = rec.name.clone();
+ if let Err(e) = self.fail_open_tasks(&rec, &format!("{name}'s Space was deleted")) {
+ tracing::warn!(agent = %name, "open tasks not closed: {e}");
+ }
+ self.release_lease(&name).await;
+ if let Ok(store) = self.routine_store().await {
+ let mut s = store.lock().await;
+ for r in s.routines_for(&name) {
+ s.delete(&r.id);
+ }
+ }
+ let why = format!(
+ "its Space {} was deleted; its home stays in the Cua Volume (`agent rm {name}` forgets the agent)",
+ rec.space
+ );
+ let done = self.update(&name, |r| {
+ r.space_state = SpaceState::Gone;
+ r.paused = true;
+ r.run_id = None;
+ r.activity = None;
+ r.session_runs.clear();
+ r.relaunch = false;
+ r.last_error = Some(why.clone());
+ });
+ match done {
+ Ok(()) => {
+ tracing::info!(agent = %name, space = %rec.space, "persistent agents: {why}");
+ marked.push(name);
+ }
+ Err(e) => tracing::warn!(agent = %name, "could not mark an agent gone: {e}"),
+ }
+ }
+ marked
+ }
+
fn session(&self, rec: &AgentRecord) -> cua_volume::Session {
self.drive.session(crate::drive_runtime::agent_context(
&self.spaces,
@@ -1335,6 +1447,11 @@ impl Persistent {
let t0 = Instant::now();
let rec = self.get(name)?;
self.not_external(&rec)?;
+ if rec.space_state == SpaceState::Gone {
+ return Err(Error::invalid(format!(
+ "{name}'s Space was deleted; remove the agent with `agent rm {name}`"
+ )));
+ }
if rec.paused {
return Ok(PauseReport {
space_state: rec.space_state,
@@ -1391,12 +1508,18 @@ impl Persistent {
))
})?;
self.forget_space(&rec.space).await;
- self.spaces.delete(&rec.space).await?;
+ // Recorded before the delete: a Space's deletion removes
+ // the agents that lived in it, except one waiting to be
+ // created again ([`Persistent::space_deleted`]).
let recreate = Recreate {
image,
name: claim.clone(),
};
self.update(name, |r| r.recreate = Some(recreate))?;
+ if let Err(e) = self.spaces.delete(&rec.space).await {
+ let _ = self.update(name, |r| r.recreate = None);
+ return Err(e);
+ }
report.space_state = SpaceState::Released;
}
// Someone's own machine or an added address: only the agent
@@ -1434,6 +1557,11 @@ impl Persistent {
..Default::default()
};
match rec.space_state {
+ SpaceState::Gone => {
+ return Err(Error::invalid(format!(
+ "{name}'s Space was deleted; its home stays in the Cua Volume. Remove the agent with `agent rm {name}`"
+ )));
+ }
SpaceState::Running => {}
SpaceState::Suspended => {
if let SpaceId::Local { name: sandbox } = self.spaces.resolve(&rec.space)? {
@@ -1783,6 +1911,7 @@ impl Persistent {
}
Err(e) => report.errors.push(format!("routines: {e}")),
}
+ self.reconcile_gone(GONE_GRACE).await;
let records = match self.list() {
Ok(r) => r,
Err(e) => {
@@ -2456,6 +2585,10 @@ impl RoutineRunner for Runner {
}
}
+/// How long a local Space must be missing from the registry before the
+/// agents that name it are marked gone.
+const GONE_GRACE: Duration = Duration::from_secs(20);
+
/// How long the supervisor stays quiet about an error it already logged.
const ERROR_REPEAT: Duration = Duration::from_secs(300);
diff --git a/libs/cua/crates/cua-spaces-ext/tests/deleted_space_agents.rs b/libs/cua/crates/cua-spaces-ext/tests/deleted_space_agents.rs
new file mode 100644
index 0000000000..c4e1b9dfbf
--- /dev/null
+++ b/libs/cua/crates/cua-spaces-ext/tests/deleted_space_agents.rs
@@ -0,0 +1,155 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Deleting a Space ends the persistent agents that lived in it (no record
+//! left "running" for a Space that is gone, nothing supervises it, its
+//! schedules are dropped), and an agent whose local Space is not registered
+//! (deleted behind the daemon's back) is marked gone when the supervisor
+//! reconciles, with its home in the Cua Volume untouched. Against the
+//! in-process spacesd mock and a temporary cua home.
+
+use std::time::Duration;
+
+use cua_spaces::Spaces;
+use cua_spaces::mcp::McpServer;
+use cua_spaces_ext::SpacesPersistent as _;
+use cua_spaces_ext::persistent::SpaceState;
+use cua_spacesd_client::testing::{MockAuth, MockServer};
+use serde_json::{Value, json};
+
+struct Fx {
+ _srv: MockServer,
+ home: tempfile::TempDir,
+ spaces: Spaces,
+ space: String,
+ mcp: McpServer,
+}
+
+async fn setup() -> Fx {
+ let srv = MockServer::start(MockAuth::default()).await;
+ let home = tempfile::tempdir().unwrap();
+ let spaces = cua_spaces_ext::register(
+ Spaces::builder().home(home.path()),
+ cua_volume::Drive::open_local(home.path()),
+ None,
+ )
+ .build();
+ let info = spaces
+ .add(&format!("http://{}", srv.addr), None, Some("room".into()))
+ .await
+ .unwrap();
+ let mcp = McpServer::new(spaces.clone());
+ Fx {
+ _srv: srv,
+ home,
+ spaces,
+ space: info.id,
+ mcp,
+ }
+}
+
+async fn call(mcp: &McpServer, tool: &str, args: Value) -> Value {
+ let out = Box::pin(mcp.call(tool, args)).await;
+ cua_spaces::extension::outcome_json(out).unwrap_or_else(|e| panic!("{tool}: {e}"))
+}
+
+async fn add_agent(fx: &Fx, name: &str) {
+ call(
+ &fx.mcp,
+ "agent_add",
+ json!({"space": fx.space, "name": name, "harness": "claude-code"}),
+ )
+ .await;
+ call(
+ &fx.mcp,
+ "agent_schedule_add",
+ json!({"agent": name, "title": "Digest", "prompt": "summarize", "every_minutes": 60}),
+ )
+ .await;
+}
+
+#[tokio::test]
+async fn deleting_a_space_removes_its_agents_and_schedules() {
+ let fx = setup().await;
+ add_agent(&fx, "ada").await;
+ add_agent(&fx, "bob").await;
+ let p = fx.spaces.persistent();
+ assert_eq!(p.list().unwrap().len(), 2);
+ assert_eq!(p.routines(None).await.unwrap().len(), 2);
+
+ fx.spaces.delete(&fx.space).await.unwrap();
+
+ let p = fx.spaces.persistent();
+ assert!(p.list().unwrap().is_empty(), "no agent left for the Space");
+ assert!(p.routines(None).await.unwrap().is_empty());
+ // Nothing for the supervisor to tick, and no error to repeat.
+ let report = p.tick().await;
+ assert!(report.errors.is_empty(), "{:?}", report.errors);
+}
+
+#[tokio::test]
+async fn deleting_one_space_keeps_the_agents_of_another() {
+ let fx = setup().await;
+ add_agent(&fx, "ada").await;
+ // A second record that names a different Space.
+ let path = fx.home.path().join("persistent").join("agents.json");
+ let mut all: Vec<Value> = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
+ let mut other = all[0].clone();
+ other["name"] = json!("elsewhere");
+ other["space"] = json!("local:elsewhere");
+ all.push(other);
+ std::fs::write(&path, serde_json::to_vec(&all).unwrap()).unwrap();
+
+ fx.spaces.delete(&fx.space).await.unwrap();
+
+ let names: Vec<String> = fx
+ .spaces
+ .persistent()
+ .list()
+ .unwrap()
+ .into_iter()
+ .map(|a| a.name)
+ .collect();
+ assert_eq!(names, vec!["elsewhere".to_string()]);
+}
+
+#[tokio::test]
+async fn an_agent_of_a_space_missing_from_the_registry_is_marked_gone_and_left_alone() {
+ let fx = setup().await;
+ add_agent(&fx, "ada").await;
+ // The Space vanished without the delete path (a record that names a
+ // local Space the registry never had).
+ let path = fx.home.path().join("persistent").join("agents.json");
+ let mut all: Vec<Value> = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
+ all[0]["space"] = json!("local:home-deleted");
+ all[0]["run_id"] = json!("run-00000001");
+ std::fs::write(&path, serde_json::to_vec(&all).unwrap()).unwrap();
+ // The agent's home in the Cua Volume.
+ let home_marker = fx.home.path().join("keep-me");
+ std::fs::write(&home_marker, b"home").unwrap();
+
+ let p = fx.spaces.persistent();
+ // Not at once: a Space deleted and created again under its name is not
+ // missed.
+ assert!(p.reconcile_gone(Duration::from_secs(3600)).await.is_empty());
+ assert_eq!(p.list().unwrap()[0].space_state, SpaceState::Running);
+
+ let marked = p.reconcile_gone(Duration::ZERO).await;
+ assert_eq!(marked, vec!["ada".to_string()]);
+ let rec = p.get("ada").unwrap();
+ assert_eq!(rec.space_state, SpaceState::Gone);
+ assert!(rec.paused);
+ assert!(rec.run_id.is_none());
+ assert!(rec.last_error.unwrap().contains("was deleted"));
+ assert!(p.routines(Some("ada")).await.unwrap().is_empty());
+ assert!(home_marker.exists());
+
+ // Supervised no more: a pass is quiet, a second reconcile marks nothing,
+ // and resuming says to remove it.
+ assert!(p.tick().await.errors.is_empty());
+ assert!(p.reconcile_gone(Duration::ZERO).await.is_empty());
+ assert!(p.resume("ada", None).await.is_err());
+ // `agent rm` keeps working on it.
+ p.remove("ada").await.unwrap();
+ assert!(p.list().unwrap().is_empty());
+}
diff --git a/libs/cua/crates/cua-spaces/src/extension.rs b/libs/cua/crates/cua-spaces/src/extension.rs
index 37490e2461..8306bec9c7 100644
--- a/libs/cua/crates/cua-spaces/src/extension.rs
+++ b/libs/cua/crates/cua-spaces/src/extension.rs
@@ -214,6 +214,13 @@ pub trait SpacesExtension: Send + Sync + 'static {
Box::pin(async {})
}
+ /// A Space was deleted for good (its sandbox is gone): what lived in it
+ /// ends here. Not called when a Space is only forgotten or released.
+ fn space_deleted<'a>(&'a self, spaces: &'a Spaces, space: &'a str) -> BoxFuture<'a, ()> {
+ let _ = (spaces, space);
+ Box::pin(async {})
+ }
+
/// For [`Spaces::extension`] (downcasting to the concrete type).
fn as_any(&self) -> &dyn Any;
}
diff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rs
index 8064f88e2f..fea198e7c6 100644
--- a/libs/cua/crates/cua-spaces/src/spaces.rs
+++ b/libs/cua/crates/cua-spaces/src/spaces.rs
@@ -3478,6 +3478,11 @@ impl Spaces {
self.detach_attached_space(&id.to_string()).await;
}
let message = self.delete_space(&id).await?;
+ // What lived in the Space (its persistent agents) ends with it.
+ #[cfg(feature = "mcp")]
+ for e in &self.inner.extensions {
+ e.space_deleted(self, &id.to_string()).await;
+ }
// A Space this device attached to the relay as a machine of its own
// takes that machine with it.
#[cfg(feature = "spaces-agents")]