Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(spaces): a durable driver whose Space container is killed is noticed in seconds, not by the stall limit (#6233)
ddupont0aebcf2b4a0a
diff --git a/docs/content/docs/spaces/guides/keep-an-agent-running.mdx b/docs/content/docs/spaces/guides/keep-an-agent-running.mdxindex 621b64b545..571a6f6ec8 100644--- a/docs/content/docs/spaces/guides/keep-an-agent-running.mdx+++ b/docs/content/docs/spaces/guides/keep-an-agent-running.mdx@@ -68,6 +68,7 @@ cell, so read the times as rough).|---|---|---|| The agent process dies (`kill -9`) | The runner restarts it in place and resumes the session, in seconds | Yes, the recovery note lists what ran and what may have run halfway || The runner (the driver process) dies | Noticed at once; a new driver starts a new Space on the same machine, 40 to 80 s with Claude Code and Codex | Yes by the journal: the harness's own session files are gone with the old Space, so the recovery note carries the journal's summary instead of the conversation |+| The driver's Space is killed or runs out of memory | The machine sees the Space stop within a few seconds and ends the driver; a new driver starts a new Space on the same machine, about 55 s from the kill to the resumed agent with Claude Code | Same as above || The driver's Space is removed, or the driver freezes | Noticed only by the stall limit (`--stall-secs`, default 120 s): about 140 to 170 s from the fault to the resumed agent | Same as above || The machine running the agent stops answering | After about 65 to 75 s the session records the lost machine and starts the next drive on **another machine** | **No.** The task starts from the beginning |diff --git a/libs/cua/crates/cua-sandbox-core/src/sandbox.rs b/libs/cua/crates/cua-sandbox-core/src/sandbox.rsindex 34a066f2cf..8de0ee54b6 100644--- a/libs/cua/crates/cua-sandbox-core/src/sandbox.rs+++ b/libs/cua/crates/cua-sandbox-core/src/sandbox.rs@@ -1825,6 +1825,19 @@ impl Sandboxes {out}+ /// [`Self::note_unasked_stops`] for the one local sandbox `name`, looked+ /// at now: `Some` (and recorded as stopped) when it was running and its+ /// engine says it stopped without being asked to. A job's Space is+ /// watched this way while its command runs.+ pub async fn unasked_stop(&self, name: &str) -> Option<crate::StopReason> {+ self.inner.local.as_ref()?;+ let s = self.inner.state.load(name)?;+ if provider_of(&s) != ProviderKind::Local || s.status() != "running" {+ return None;+ }+ self.note_stop(name, s.runtime_type()).await+ }+/// [`Self::note_unasked_stops`] for one sandbox.async fn note_stop(&self, name: &str, runtime_type: &str) -> Option<crate::StopReason> {let rt = self.inner.local.as_ref()?;diff --git a/libs/cua/crates/cua-sandbox-core/tests/power.rs b/libs/cua/crates/cua-sandbox-core/tests/power.rsindex d947a41f90..68d9157bf5 100644--- a/libs/cua/crates/cua-sandbox-core/tests/power.rs+++ b/libs/cua/crates/cua-sandbox-core/tests/power.rs@@ -361,6 +361,29 @@ async fn the_watch_finds_a_stop_nobody_listed() {);}+#[tokio::test]+async fn one_sandbox_can_be_looked_at_for_an_unasked_stop() {+ let (_dir, rt, sbx) = env();+ sbx.state().save(&local("box", "runc")).unwrap();+ sbx.state().save(&local("other", "runc")).unwrap();+ rt.set("box", InstanceStatus::Running);+ rt.set("other", InstanceStatus::Running);+ // Running: nothing to say, and nothing recorded.+ assert_eq!(sbx.unasked_stop("box").await, None);+ assert_eq!(sbx.unasked_stop("missing").await, None);+ let killed = StopReason {+ exit_code: 137,+ out_of_memory: false,+ };+ rt.exit("box", killed);+ // Only the one asked about is looked at, and it is recorded as stopped.+ assert_eq!(sbx.unasked_stop("box").await, Some(killed));+ assert_eq!(sbx.stop_reason("box"), Some(killed));+ assert_eq!(sbx.stop_reason("other"), None);+ // Already recorded as stopped: it is not an unasked stop again.+ assert_eq!(sbx.unasked_stop("box").await, None);+}+#[tokio::test]async fn a_stop_that_was_asked_for_has_no_reason() {let (_dir, rt, sbx) = env();diff --git a/libs/cua/crates/cua-spaces/src/host_jobs.rs b/libs/cua/crates/cua-spaces/src/host_jobs.rsindex 24c6989d52..20b34e404b 100644--- a/libs/cua/crates/cua-spaces/src/host_jobs.rs+++ b/libs/cua/crates/cua-spaces/src/host_jobs.rs@@ -62,6 +62,9 @@ const MACOS_MIN: (u32, u64) = (4, 8192);/// Smallest macOS VM a member that asks for an accelerator gets.const MACOS_GPU_MIN: (u32, u64) = (6, 16384);+/// How often a running job's Space is looked at for having stopped on its own.+const SPACE_WATCH_EVERY: Duration = Duration::from_secs(3);+/// Where a macOS member sees its directories: links without spaces to the/// shared directories (`/Volumes/My Shared Files/<name>`), since a path with/// a space breaks the `#!` line of every console script a Python@@ -385,7 +388,7 @@ impl SpacesExecutor {let job = &job;out.create_ms = t0.elapsed().as_millis() as u32;let t1 = Instant::now();- let end = member::run(+ let run = member::run(MemberCommand {client: &client,job,@@ -406,8 +409,21 @@ impl SpacesExecutor {},space,cancel,- )- .await;+ );+ // A Space that stops on its own (killed, out of memory) ends the job+ // now: left to the command's output stream, the host would retry+ // reaching a Space that is gone for minutes before it said so.+ let end = tokio::select! {+ end = run => end,+ why = self.space_stopped(id) => {+ let note = format!("the job's Space stopped while its command ran: {why}");+ jobdir::append(+ &self.dirs.output_log(&job.job_id, job.attempt),+ format!("cua: {note}\n").as_bytes(),+ );+ ExecEnd::Error(note)+ }+ };drop(routed);drop(services);out.run_ms = t1.elapsed().as_millis() as u32;@@ -420,6 +436,18 @@ impl SpacesExecutor {Ok(end)}+ /// Resolves with why the job's Space `id` stopped without being asked+ /// to, as soon as it did (looked at every [`SPACE_WATCH_EVERY`]). Never+ /// resolves while it runs, and never for a Space that is not local.+ async fn space_stopped(&self, id: &str) -> String {+ loop {+ tokio::time::sleep(SPACE_WATCH_EVERY).await;+ if let Some(why) = self.spaces.unasked_stop_of(id).await {+ return why;+ }+ }+ }+/// Starts `job`'s host services ([`crate::job_services`]): their files/// in the work directory, their environment, the hotspot that reaches/// their ports, and the command wrapped in the port forwarder. Returnsdiff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rsindex 8028b310f2..fd75977ca1 100644--- a/libs/cua/crates/cua-spaces/src/spaces.rs+++ b/libs/cua/crates/cua-spaces/src/spaces.rs@@ -3429,6 +3429,21 @@ impl Spaces {.unwrap_or_default()}+ /// Why the local Space `id` stopped without being asked to, looked at+ /// now (its engine is asked, never started): `None` while it runs, for+ /// a Space that is not local, or one that was stopped on purpose. Unlike+ /// [`Self::stop_reason_of`] this does not wait for the next stop watch.+ pub async fn unasked_stop_of(&self, id: &str) -> Option<String> {+ let Ok(SpaceId::Local { name }) = SpaceId::parse(id) else {+ return None;+ };+ self.inner+ .sandboxes+ .unasked_stop(&name)+ .await+ .map(|r| r.words())+ }+/// [`Self::power_of`] for the sandbox `name` (a local Space's, or the/// cloud sandbox behind a `relay:` Space).fn sandbox_power(&self, name: &str) -> (String, String) {diff --git a/libs/cua/crates/cua-spaces/tests/host_jobs.rs b/libs/cua/crates/cua-spaces/tests/host_jobs.rsindex ec9e55c35f..2f76c884de 100644--- a/libs/cua/crates/cua-spaces/tests/host_jobs.rs+++ b/libs/cua/crates/cua-spaces/tests/host_jobs.rs@@ -29,6 +29,8 @@ struct FakeRuntime {started: Mutex<Vec<LocalStartSpec>>,deleted: Mutex<Vec<String>>,running: Mutex<BTreeMap<String, bool>>,+ /// Instances their engine says stopped on their own (a killed container).+ exits: Mutex<BTreeMap<String, cua_sandbox_core::StopReason>>,}#[async_trait]@@ -77,6 +79,13 @@ impl LocalRuntime for FakeRuntime {self.running.lock().unwrap().remove(name);Ok(())}+ async fn stop_reason(+ &self,+ name: &str,+ _runtime_type: &str,+ ) -> Result<Option<cua_sandbox_core::StopReason>, RuntimeError> {+ Ok(self.exits.lock().unwrap().get(name).copied())+ }async fn endpoints(&self, _name: &str) -> Result<LocalEndpoints, RuntimeError> {Ok(LocalEndpoints {host: "127.0.0.1".into(),@@ -113,6 +122,7 @@ async fn fixture() -> Fixture {started: Mutex::default(),deleted: Mutex::default(),running: Mutex::default(),+ exits: Mutex::default(),});let spaces = Spaces::builder().home(&home)@@ -247,6 +257,57 @@ async fn a_job_runs_once_in_a_fresh_space_that_is_deleted() {assert!(f.home.join("host/jobs/ledger.jsonl").is_file());}+/// A job's Space that is killed while its command runs ends the job at once,+/// with the reason, instead of the host waiting for a Space that is gone+/// (CUA-2967: a durable agent's driver whose container was killed was only+/// noticed by the 120 s stall watch).+#[tokio::test]+async fn a_job_whose_space_is_killed_ends_at_once_with_the_reason() {+ let f = fixture().await;+ seed_uv(&f.home);+ let jobs = f.server.jobs_for(&HostCaller::local()).await.unwrap();+ let mut long = spec("k1-1", &["sleep", "120000"]);+ long.timeout_secs = 300;+ let acks = jobs.run_batch(vec![long], "test").await;+ assert_eq!(acks[0].outcome, "accepted", "{acks:?}");+ // Wait until its Space runs, then have the engine report it killed+ // (the command's own stream is still open: the mock spacesd stays up).+ let mut name = String::new();+ for _ in 0..400 {+ if let Some(s) = f.rt.started.lock().unwrap().first() {+ name = s.name.clone();+ break;+ }+ tokio::time::sleep(Duration::from_millis(25)).await;+ }+ assert!(!name.is_empty(), "the job's Space never started");+ tokio::time::sleep(Duration::from_millis(500)).await;+ let killed = std::time::Instant::now();+ f.rt.exits.lock().unwrap().insert(+ name.clone(),+ cua_sandbox_core::StopReason {+ exit_code: 137,+ out_of_memory: false,+ },+ );+ let end = tokio::time::timeout(Duration::from_secs(20), ended(&jobs, "k1-1"))+ .await+ .expect("the job did not end after its Space was killed");+ assert!(+ killed.elapsed() < Duration::from_secs(15),+ "took {:?}",+ killed.elapsed()+ );+ assert_eq!(end.state, "error", "{end:?}");+ assert!(+ end.error.contains("Space stopped") && end.error.contains("killed"),+ "{}",+ end.error+ );+ // Its Space is removed like any job's.+ assert!(f.rt.deleted.lock().unwrap().contains(&name));+}+/// What a scheduler reads: `GetHostSpaces` carries the node status.#[tokio::test]async fn the_host_reports_its_node_status() {