Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(spaces): a durable driver whose Space container is killed is noticed in seconds, not by the stall limit (#6233)

ddupont0aebcf2b4a0a
diff --git a/docs/content/docs/spaces/guides/keep-an-agent-running.mdx b/docs/content/docs/spaces/guides/keep-an-agent-running.mdx
index 621b64b545..571a6f6ec8 100644
--- a/docs/content/docs/spaces/guides/keep-an-agent-running.mdx
+++ b/docs/content/docs/spaces/guides/keep-an-agent-running.mdx
@@ -68,6 +68,7 @@ cell, so read the times as rough).
|---|---|---|
| The agent process dies (`kill -9`) | The runner restarts it in place and resumes the session, in seconds | Yes, the recovery note lists what ran and what may have run halfway |
| The runner (the driver process) dies | Noticed at once; a new driver starts a new Space on the same machine, 40 to 80 s with Claude Code and Codex | Yes by the journal: the harness's own session files are gone with the old Space, so the recovery note carries the journal's summary instead of the conversation |
+| The driver's Space is killed or runs out of memory | The machine sees the Space stop within a few seconds and ends the driver; a new driver starts a new Space on the same machine, about 55 s from the kill to the resumed agent with Claude Code | Same as above |
| The driver's Space is removed, or the driver freezes | Noticed only by the stall limit (`--stall-secs`, default 120 s): about 140 to 170 s from the fault to the resumed agent | Same as above |
| The machine running the agent stops answering | After about 65 to 75 s the session records the lost machine and starts the next drive on **another machine** | **No.** The task starts from the beginning |
diff --git a/libs/cua/crates/cua-sandbox-core/src/sandbox.rs b/libs/cua/crates/cua-sandbox-core/src/sandbox.rs
index 34a066f2cf..8de0ee54b6 100644
--- a/libs/cua/crates/cua-sandbox-core/src/sandbox.rs
+++ b/libs/cua/crates/cua-sandbox-core/src/sandbox.rs
@@ -1825,6 +1825,19 @@ impl Sandboxes {
out
}
+ /// [`Self::note_unasked_stops`] for the one local sandbox `name`, looked
+ /// at now: `Some` (and recorded as stopped) when it was running and its
+ /// engine says it stopped without being asked to. A job's Space is
+ /// watched this way while its command runs.
+ pub async fn unasked_stop(&self, name: &str) -> Option<crate::StopReason> {
+ self.inner.local.as_ref()?;
+ let s = self.inner.state.load(name)?;
+ if provider_of(&s) != ProviderKind::Local || s.status() != "running" {
+ return None;
+ }
+ self.note_stop(name, s.runtime_type()).await
+ }
+
/// [`Self::note_unasked_stops`] for one sandbox.
async fn note_stop(&self, name: &str, runtime_type: &str) -> Option<crate::StopReason> {
let rt = self.inner.local.as_ref()?;
diff --git a/libs/cua/crates/cua-sandbox-core/tests/power.rs b/libs/cua/crates/cua-sandbox-core/tests/power.rs
index d947a41f90..68d9157bf5 100644
--- a/libs/cua/crates/cua-sandbox-core/tests/power.rs
+++ b/libs/cua/crates/cua-sandbox-core/tests/power.rs
@@ -361,6 +361,29 @@ async fn the_watch_finds_a_stop_nobody_listed() {
);
}
+#[tokio::test]
+async fn one_sandbox_can_be_looked_at_for_an_unasked_stop() {
+ let (_dir, rt, sbx) = env();
+ sbx.state().save(&local("box", "runc")).unwrap();
+ sbx.state().save(&local("other", "runc")).unwrap();
+ rt.set("box", InstanceStatus::Running);
+ rt.set("other", InstanceStatus::Running);
+ // Running: nothing to say, and nothing recorded.
+ assert_eq!(sbx.unasked_stop("box").await, None);
+ assert_eq!(sbx.unasked_stop("missing").await, None);
+ let killed = StopReason {
+ exit_code: 137,
+ out_of_memory: false,
+ };
+ rt.exit("box", killed);
+ // Only the one asked about is looked at, and it is recorded as stopped.
+ assert_eq!(sbx.unasked_stop("box").await, Some(killed));
+ assert_eq!(sbx.stop_reason("box"), Some(killed));
+ assert_eq!(sbx.stop_reason("other"), None);
+ // Already recorded as stopped: it is not an unasked stop again.
+ assert_eq!(sbx.unasked_stop("box").await, None);
+}
+
#[tokio::test]
async fn a_stop_that_was_asked_for_has_no_reason() {
let (_dir, rt, sbx) = env();
diff --git a/libs/cua/crates/cua-spaces/src/host_jobs.rs b/libs/cua/crates/cua-spaces/src/host_jobs.rs
index 24c6989d52..20b34e404b 100644
--- a/libs/cua/crates/cua-spaces/src/host_jobs.rs
+++ b/libs/cua/crates/cua-spaces/src/host_jobs.rs
@@ -62,6 +62,9 @@ const MACOS_MIN: (u32, u64) = (4, 8192);
/// Smallest macOS VM a member that asks for an accelerator gets.
const MACOS_GPU_MIN: (u32, u64) = (6, 16384);
+/// How often a running job's Space is looked at for having stopped on its own.
+const SPACE_WATCH_EVERY: Duration = Duration::from_secs(3);
+
/// Where a macOS member sees its directories: links without spaces to the
/// shared directories (`/Volumes/My Shared Files/<name>`), since a path with
/// a space breaks the `#!` line of every console script a Python
@@ -385,7 +388,7 @@ impl SpacesExecutor {
let job = &job;
out.create_ms = t0.elapsed().as_millis() as u32;
let t1 = Instant::now();
- let end = member::run(
+ let run = member::run(
MemberCommand {
client: &client,
job,
@@ -406,8 +409,21 @@ impl SpacesExecutor {
},
space,
cancel,
- )
- .await;
+ );
+ // A Space that stops on its own (killed, out of memory) ends the job
+ // now: left to the command's output stream, the host would retry
+ // reaching a Space that is gone for minutes before it said so.
+ let end = tokio::select! {
+ end = run => end,
+ why = self.space_stopped(id) => {
+ let note = format!("the job's Space stopped while its command ran: {why}");
+ jobdir::append(
+ &self.dirs.output_log(&job.job_id, job.attempt),
+ format!("cua: {note}\n").as_bytes(),
+ );
+ ExecEnd::Error(note)
+ }
+ };
drop(routed);
drop(services);
out.run_ms = t1.elapsed().as_millis() as u32;
@@ -420,6 +436,18 @@ impl SpacesExecutor {
Ok(end)
}
+ /// Resolves with why the job's Space `id` stopped without being asked
+ /// to, as soon as it did (looked at every [`SPACE_WATCH_EVERY`]). Never
+ /// resolves while it runs, and never for a Space that is not local.
+ async fn space_stopped(&self, id: &str) -> String {
+ loop {
+ tokio::time::sleep(SPACE_WATCH_EVERY).await;
+ if let Some(why) = self.spaces.unasked_stop_of(id).await {
+ return why;
+ }
+ }
+ }
+
/// Starts `job`'s host services ([`crate::job_services`]): their files
/// in the work directory, their environment, the hotspot that reaches
/// their ports, and the command wrapped in the port forwarder. Returns
diff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rs
index 8028b310f2..fd75977ca1 100644
--- a/libs/cua/crates/cua-spaces/src/spaces.rs
+++ b/libs/cua/crates/cua-spaces/src/spaces.rs
@@ -3429,6 +3429,21 @@ impl Spaces {
.unwrap_or_default()
}
+ /// Why the local Space `id` stopped without being asked to, looked at
+ /// now (its engine is asked, never started): `None` while it runs, for
+ /// a Space that is not local, or one that was stopped on purpose. Unlike
+ /// [`Self::stop_reason_of`] this does not wait for the next stop watch.
+ pub async fn unasked_stop_of(&self, id: &str) -> Option<String> {
+ let Ok(SpaceId::Local { name }) = SpaceId::parse(id) else {
+ return None;
+ };
+ self.inner
+ .sandboxes
+ .unasked_stop(&name)
+ .await
+ .map(|r| r.words())
+ }
+
/// [`Self::power_of`] for the sandbox `name` (a local Space's, or the
/// cloud sandbox behind a `relay:` Space).
fn sandbox_power(&self, name: &str) -> (String, String) {
diff --git a/libs/cua/crates/cua-spaces/tests/host_jobs.rs b/libs/cua/crates/cua-spaces/tests/host_jobs.rs
index ec9e55c35f..2f76c884de 100644
--- a/libs/cua/crates/cua-spaces/tests/host_jobs.rs
+++ b/libs/cua/crates/cua-spaces/tests/host_jobs.rs
@@ -29,6 +29,8 @@ struct FakeRuntime {
started: Mutex<Vec<LocalStartSpec>>,
deleted: Mutex<Vec<String>>,
running: Mutex<BTreeMap<String, bool>>,
+ /// Instances their engine says stopped on their own (a killed container).
+ exits: Mutex<BTreeMap<String, cua_sandbox_core::StopReason>>,
}
#[async_trait]
@@ -77,6 +79,13 @@ impl LocalRuntime for FakeRuntime {
self.running.lock().unwrap().remove(name);
Ok(())
}
+ async fn stop_reason(
+ &self,
+ name: &str,
+ _runtime_type: &str,
+ ) -> Result<Option<cua_sandbox_core::StopReason>, RuntimeError> {
+ Ok(self.exits.lock().unwrap().get(name).copied())
+ }
async fn endpoints(&self, _name: &str) -> Result<LocalEndpoints, RuntimeError> {
Ok(LocalEndpoints {
host: "127.0.0.1".into(),
@@ -113,6 +122,7 @@ async fn fixture() -> Fixture {
started: Mutex::default(),
deleted: Mutex::default(),
running: Mutex::default(),
+ exits: Mutex::default(),
});
let spaces = Spaces::builder()
.home(&home)
@@ -247,6 +257,57 @@ async fn a_job_runs_once_in_a_fresh_space_that_is_deleted() {
assert!(f.home.join("host/jobs/ledger.jsonl").is_file());
}
+/// A job's Space that is killed while its command runs ends the job at once,
+/// with the reason, instead of the host waiting for a Space that is gone
+/// (CUA-2967: a durable agent's driver whose container was killed was only
+/// noticed by the 120 s stall watch).
+#[tokio::test]
+async fn a_job_whose_space_is_killed_ends_at_once_with_the_reason() {
+ let f = fixture().await;
+ seed_uv(&f.home);
+ let jobs = f.server.jobs_for(&HostCaller::local()).await.unwrap();
+ let mut long = spec("k1-1", &["sleep", "120000"]);
+ long.timeout_secs = 300;
+ let acks = jobs.run_batch(vec![long], "test").await;
+ assert_eq!(acks[0].outcome, "accepted", "{acks:?}");
+ // Wait until its Space runs, then have the engine report it killed
+ // (the command's own stream is still open: the mock spacesd stays up).
+ let mut name = String::new();
+ for _ in 0..400 {
+ if let Some(s) = f.rt.started.lock().unwrap().first() {
+ name = s.name.clone();
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(25)).await;
+ }
+ assert!(!name.is_empty(), "the job's Space never started");
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ let killed = std::time::Instant::now();
+ f.rt.exits.lock().unwrap().insert(
+ name.clone(),
+ cua_sandbox_core::StopReason {
+ exit_code: 137,
+ out_of_memory: false,
+ },
+ );
+ let end = tokio::time::timeout(Duration::from_secs(20), ended(&jobs, "k1-1"))
+ .await
+ .expect("the job did not end after its Space was killed");
+ assert!(
+ killed.elapsed() < Duration::from_secs(15),
+ "took {:?}",
+ killed.elapsed()
+ );
+ assert_eq!(end.state, "error", "{end:?}");
+ assert!(
+ end.error.contains("Space stopped") && end.error.contains("killed"),
+ "{}",
+ end.error
+ );
+ // Its Space is removed like any job's.
+ assert!(f.rt.deleted.lock().unwrap().contains(&name));
+}
+
/// What a scheduler reads: `GetHostSpaces` carries the node status.
#[tokio::test]
async fn the_host_reports_its_node_status() {