Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(agents): a durable drive keeps its journal across a driver restart on the same machine (#6228)
ddupont7509a74060fc
diff --git a/docs/content/docs/spaces/guides/keep-an-agent-running.mdx b/docs/content/docs/spaces/guides/keep-an-agent-running.mdxindex 34b33f01c5..05b604844b 100644--- a/docs/content/docs/spaces/guides/keep-an-agent-running.mdx+++ b/docs/content/docs/spaces/guides/keep-an-agent-running.mdx@@ -45,7 +45,11 @@ schedule; those are on [named agents](/spaces/guides/agent-memory-and-identity)today.The session keeps a journal in its state directory (`CUA_SESSION_STATE_DIR`,-or `$CUA_JOB_DATA/session` when that is empty). It holds:+or `$CUA_JOB_DATA/session` when that is empty). `cua agent add --durable`+leaves it empty, so the journal sits in the run's data folder on the machine+that drives it: it outlives the driver's Space, and a driver that dies or stalls+is started again on the same machine with the same conversation. A new drive on+another machine does not find it and starts the task from the beginning. It holds:| File | What ||---|---|diff --git a/libs/cua/crates/cua-agents/src/durable.rs b/libs/cua/crates/cua-agents/src/durable.rsindex 9ef03cd81a..0487c99e5a 100644--- a/libs/cua/crates/cua-agents/src/durable.rs+++ b/libs/cua/crates/cua-agents/src/durable.rs@@ -101,8 +101,8 @@ pub struct DurableOptions {/// overrides it; the journal itself is the truth.pub resume: bool,/// Where the journal lives. In a Space: default the run directory. For a- /// driver spec: default `~/.cua/durable/<session>`, else empty (the- /// runner then uses `$CUA_JOB_DATA/session`).+ /// driver spec: default empty (the runner then uses+ /// `$CUA_JOB_DATA/session`, which outlives the driver's Space).pub state_dir: Option<String>,/// The session's name (for the default state directory and the journal).pub session: Option<String>,@@ -413,11 +413,14 @@ pub fn durable_driver_spec(for k in opts.env.keys() {crate::runs::valid_env_name(k)?;}- // Empty: the coordinator leaves CUA_SESSION_STATE_DIR empty and the- // runner uses $CUA_JOB_DATA/session.+ // Empty by default: the coordinator leaves CUA_SESSION_STATE_DIR empty and+ // the runner uses $CUA_JOB_DATA/session. The job data directory is per run+ // on each machine and the run id is the same for every drive, so the+ // journal outlives the driver's Space (a `~/.cua/...` default sat in the+ // Space's home and died with it: the task restarted from scratch).let state_dir = match (&d.state_dir, &d.session) {(Some(s), _) => s.clone(),- (None, Some(n)) if crate::runs::valid_home_name(n) => format!("~/.cua/durable/{n}"),+ (None, Some(n)) if crate::runs::valid_home_name(n) => String::new(),(None, Some(n)) => {return Err(Error::Invalid(format!("session name {n:?}: use 1-63 of a-z, 0-9, `.`, `_`, `-`"@@ -671,10 +674,10 @@ mod tests {"${CUA_MCP_0_0}");assert_eq!(run["durable"]["policy"], "ask-model");- assert_eq!(run["durable"]["stateDir"], "~/.cua/durable/nightly");+ assert_eq!(run["durable"]["stateDir"], "");assert_eq!(run["exitWhenIdle"], true);assert_eq!(run["prompt"], "fix the bug");- assert_eq!(spec.state_dir, "~/.cua/durable/nightly");+ assert_eq!(spec.state_dir, "");assert_eq!(spec.label, "claude-code");}diff --git a/libs/cua/crates/cua-cli/src/agent_session_cmd.rs b/libs/cua/crates/cua-cli/src/agent_session_cmd.rsindex bcdf9246bb..d9bf906562 100644--- a/libs/cua/crates/cua-cli/src/agent_session_cmd.rs+++ b/libs/cua/crates/cua-cli/src/agent_session_cmd.rs@@ -269,6 +269,10 @@ fn state_line(v: &Value) -> String {if let Some(n) = v["node"].as_str().filter(|n| !n.is_empty()) {s.push_str(&format!(", on {n}"));}+ // No machine has room for the driver yet.+ if let Some(w) = v["waiting"].as_str().filter(|w| !w.is_empty()) {+ s.push_str(&format!(", {w}"));+ }s}@@ -453,6 +457,19 @@ pub async fn rm(name: &str, json: bool, out: &mut dyn Write) -> Result<i32, CuaE#[cfg(test)]mod tests {+ #[test]+ fn a_waiting_driver_says_what_it_waits_for() {+ let v = json!({"name":"n","state":"running","epoch":2,"attempts":2,"max_attempts":5,+ "waiting":"waiting for a machine with 2 CPUs and 4 GB memory free"});+ let l = state_line(&v);+ assert!(+ l.ends_with(", waiting for a machine with 2 CPUs and 4 GB memory free"),+ "{l}"+ );+ let v = json!({"name":"n","state":"running","epoch":1,"attempts":1,"max_attempts":5});+ assert!(!state_line(&v).contains("waiting"));+ }+use super::*;use crate::agent_cmd::AgentCmd;use clap::Parser;@@ -604,7 +621,9 @@ mod tests {assert!(spec["env"].get("CUA_AGENT_PROMPT").is_none());assert_eq!(spec["env"]["ANTHROPIC_API_KEY"], "sk-ant-secret");assert!(!bundle.contains("sk-ant-secret"));- assert_eq!(spec["state_dir"], "~/.cua/durable/fix-it");+ // The journal stays in the job's data directory, which outlives the+ // driver's Space on the same machine (never `~/.cua/...` in the Space).+ assert_eq!(spec["state_dir"], "");// The harness installs and runs in the driver's Space: sized for it.assert_eq!(spec["cpus"], ACP_DRIVER_CPUS);assert_eq!(spec["memory_mb"], ACP_DRIVER_MEMORY_MB);diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rsindex 66e8b8d310..b58a2d9c07 100644--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs@@ -189,6 +189,8 @@ struct Queued {/// Not placed before this (a failed attempt's backoff).not_before: Instant,submitted_ms: i64,+ /// Why no machine took it at the last look (empty: not tried yet).+ waiting: String,}/// A gang run as the scheduler keeps it (its row is in the database).@@ -296,6 +298,10 @@ struct Sched {/// Start barriers to release with each host's next feed call: host ->/// job -> attempt.releases: HashMap<String, BTreeMap<i64, u32>>,+ /// Run -> the host it is placed on first when that host fits (a durable+ /// session's next drive: its journal is on the host that ran the last).+ /// A preference, never a pin: a host that is gone or full is passed over.+ prefer: HashMap<String, String>,}/// A member's start barrier and progress as its host last reported them.@@ -345,6 +351,7 @@ impl Sched {ready_at: now,not_before: now + delay,submitted_ms: f.submitted_ms,+ waiting: String::new(),},);self.queue.push_front(id);@@ -1197,6 +1204,7 @@ impl Coordinator {ready_at: now,not_before: now,submitted_ms: r.submitted_ms,+ waiting: String::new(),},);g.queue.push_back(r.id);@@ -1313,6 +1321,9 @@ impl Coordinator {r.short = r.min.max(1);}}+ for q in g.queued.values_mut() {+ q.waiting = "waiting for a machine that can take work".into();+ }return (vec![], vec![], vec![]);}if g.queue.is_empty() && !gangs_ready {@@ -1362,7 +1373,7 @@ impl Coordinator {// later job of the same shape is passed over at once, so jobs that// fit elsewhere (another OS, a pinned host, the elastic capacity)// behind thousands that do not are still placed.- let mut missed: std::collections::HashSet<NeedKey> = std::collections::HashSet::new();+ let mut missed: HashMap<NeedKey, String> = HashMap::new();while let Some(id) = g.queue.pop_front() {let Some(q) = g.queued.get(&id) else {continue;@@ -1375,7 +1386,10 @@ impl Coordinator {let (retries, max_attempts, run) = (q.retries, q.max_attempts, q.run.clone());let need = need_of(&q.spec);let key = need_key(&need);- if retries < max_attempts && missed.contains(&key) {+ if retries < max_attempts && missed.contains_key(&key) {+ if let Some(q) = g.queued.get_mut(&id) {+ q.waiting = missed[&key].clone();+ }deferred.push(id);skipped += 1;if skipped >= SKIP_MAX {@@ -1410,10 +1424,19 @@ impl Coordinator {continue;}}- let Some(&best) = placement::rank(&cands, &need, self.cfg.policy).first() else {+ let ranked = placement::rank(&cands, &need, self.cfg.policy);+ let preferred = g+ .prefer+ .get(&run)+ .and_then(|n| ranked.iter().copied().find(|&i| &cands[i].name == n));+ let Some(best) = preferred.or_else(|| ranked.first().copied()) else {deferred.push(id);misses += 1;- missed.insert(key);+ let why = waiting_reason(&need);+ if let Some(q) = g.queued.get_mut(&id) {+ q.waiting = why.clone();+ }+ missed.insert(key, why);continue;};misses = 0;@@ -1423,6 +1446,7 @@ impl Coordinator {c.pending_jobs += 1;let node = c.name.clone();let q = g.queued.remove(&id).expect("queued");+ g.prefer.remove(&q.run);let attempt = q.attempt + 1;let mut spec = (*q.spec).clone();spec.job_id = host_job_id(&self.cluster_id, id);@@ -2525,6 +2549,23 @@ impl Coordinator {g.inflight.values().any(|f| f.node == name)}+ /// Places `run` on `host` first when it fits (see `Sched::prefer`).+ pub(crate) fn prefer_host(&self, run: &str, host: &str) {+ if !host.is_empty() {+ let mut g = self.sched.lock().expect("sched");+ g.prefer.insert(run.to_string(), host.to_string());+ }+ }++ /// Why queued job `id` is not placed (empty: it is placed, or not yet tried).+ pub fn queued_waiting(&self, id: i64) -> String {+ let g = self.sched.lock().expect("sched");+ g.queued+ .get(&id)+ .map(|q| q.waiting.clone())+ .unwrap_or_default()+ }+/// Jobs waiting in memory and in flight.pub fn depth(&self) -> (usize, usize) {let g = self.sched.lock().expect("sched");@@ -2909,6 +2950,25 @@ fn need_key(n: &Need) -> NeedKey {}/// What a job needs of its host.+/// What a job that no machine took is waiting for, in plain words.+fn waiting_reason(need: &Need) -> String {+ if !need.on.is_empty() && need.on != "auto" {+ return format!("waiting for {} to have room", need.on);+ }+ let mut parts = vec![];+ if need.millicpus > 0 {+ let c = need.millicpus.div_ceil(1000);+ parts.push(format!("{c} CPU{}", if c == 1 { "" } else { "s" }));+ }+ if need.memory_mb > 0 {+ parts.push(format!("{} GB memory", need.memory_mb.div_ceil(1024)));+ }+ if parts.is_empty() {+ return "waiting for a machine with room".into();+ }+ format!("waiting for a machine with {} free", parts.join(" and "))+}+fn need_of(spec: &JobSpec) -> Need {Need {private_image: cua_cluster::placement::image_is_private(&spec.image),diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rsindex 8fc585a859..5483046063 100644--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs@@ -252,6 +252,10 @@ pub(crate) struct Row {max_attempts: u32,driver_job: i64,node: String,+ /// The machine of the previous drive (kept when `node` is cleared for+ /// the next one): the next drive starts there first, where the journal is.+ #[serde(default)]+ last_node: String,error: String,next_attempt_ms: i64,/// The Space lease's fencing token of the current epoch.@@ -317,6 +321,10 @@ pub struct SessionView {pub error: String,/// Seconds since the running driver last wrote output (stall watch on).pub last_output_age_secs: Option<i64>,+ /// Why the driver is not running yet: no machine has room for it (empty+ /// when it runs or has not been tried).+ #[serde(default, skip_serializing_if = "String::is_empty")]+ pub waiting: String,/// What it runs (no secrets).pub spec: SpecView,/// What happened, oldest first.@@ -343,6 +351,7 @@ impl Row {node: self.node.clone(),error: self.error.clone(),last_output_age_secs: None,+ waiting: String::new(),spec: SpecView {argv: self.spec.argv.clone(),env_keys: self.spec.env.keys().cloned().collect(),@@ -545,6 +554,9 @@ impl Coordinator {{v.node = j.node;}+ if r.state == sstate::RUNNING && r.driver_job > 0 {+ v.waiting = self.queued_waiting(r.driver_job);+ }if r.state == sstate::RUNNING&& let Some((job, _, at)) = self.session_watch@@ -1001,6 +1013,9 @@ impl Coordinator {next.attempts += 1;next.state = sstate::RUNNING.into();next.driver_job = 0;+ if !next.node.is_empty() {+ next.last_node = next.node.clone();+ }next.node.clear();next.error.clear();next.next_attempt_ms = 0;@@ -1062,6 +1077,9 @@ impl Coordinator {let run = format!("session-{}", r.name);let key = format!("session:{}:{}", r.name, r.epoch);let spec = driver_job(&r, &run, lease.as_ref())?;+ // The journal is in the data folder of the machine that ran the last+ // drive: start there again when that machine has room.+ self.prefer_host(&run, &r.last_node);let ids = self.submit(&run,diff --git a/libs/cua/crates/cua-spaces-cluster/tests/common/mod.rs b/libs/cua/crates/cua-spaces-cluster/tests/common/mod.rsindex 74139d3a22..a8502b94a9 100644--- a/libs/cua/crates/cua-spaces-cluster/tests/common/mod.rs+++ b/libs/cua/crates/cua-spaces-cluster/tests/common/mod.rs@@ -61,7 +61,7 @@ impl DriverRun {////// A session driver is `["driver", "<script>"]`; the script says what the/// driver does at each epoch: `1=exit:1;2=hang;3=exit:0` (`exit:N`, `hang`-/// until canceled, `sleep:<ms>` then exit 0; an epoch not listed hangs).+/// until canceled, `sleep:<ms>` then exit 0, `crashafter:<ms>` then exit 1; an epoch not listed hangs).////// argv: `["sleep", "<ms>"]`, `["exit", "<code>"]`, `["nospace"]` (the/// Space does not start), `["echo", "<text>"]`.@@ -180,6 +180,13 @@ impl JobExecutor for FakeExec {_ = tokio::time::sleep(Duration::from_millis(ms)) => ExecEnd::Exited(0),}}+ Some(("crashafter", ms)) => {+ let ms: u64 = ms.parse().unwrap_or(10);+ tokio::select! {+ _ = cancel.cancelled() => ExecEnd::Canceled,+ _ = tokio::time::sleep(Duration::from_millis(ms)) => ExecEnd::Exited(1),+ }+ }_ => {cancel.cancelled().await;ExecEnd::Canceleddiff --git a/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs b/libs/cua/crates/cua-spaces-cluster/tests/sessions.rsindex 2dc6a57ed6..cd3f4e95d4 100644--- a/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs+++ b/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs@@ -1113,3 +1113,86 @@ async fn an_unreachable_host_is_not_a_stall() {);lab.stop();}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn a_driver_that_fits_on_no_machine_says_what_it_waits_for() {+ let lab = Lab::new(&["a"]).await;+ let mut spec = driver("1=hang");+ // The test hosts have 8 CPUs: this driver fits on none of them.+ spec.cpus = 64.0;+ spec.memory_mb = 4096;+ lab.coord.session_start("big", spec, 0).await.unwrap();+ let v = lab+ .until("big", "a waiting reason", T, |v| !v.waiting.is_empty())+ .await;+ assert_eq!(v.state, sstate::RUNNING);+ assert!(+ v.waiting.contains("64 CPUs") && v.waiting.contains("4 GB memory"),+ "{}",+ v.waiting+ );+ assert_eq!(lab.starts_of(1), 0, "it must not have started");+ lab.stop();+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn a_driver_that_fits_has_no_waiting_reason() {+ let lab = Lab::new(&["a"]).await;+ lab.coord+ .session_start("small", driver("1=hang"), 0)+ .await+ .unwrap();+ wait_for("epoch 1 started", T, async || {+ (lab.starts_of(1) == 1).then_some(())+ })+ .await;+ let v = lab.coord.session_get("small").await.unwrap();+ assert!(v.waiting.is_empty(), "{}", v.waiting);+ lab.stop();+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn the_next_drive_starts_on_the_machine_that_ran_the_last_when_it_has_room() {+ // The journal is in the data folder of the machine that ran the last+ // drive. Spreading would send the next drive to the idle machine `b`;+ // the session goes back to `a`.+ let lab = Lab::new(&["a", "b"]).await;+ lab.coord+ .session_start("work", driver("1=crashafter:1500;2=hang"), 0)+ .await+ .unwrap();+ wait_for("epoch 1 started", T, async || {+ (lab.starts_of(1) == 1).then_some(())+ })+ .await;+ let first = lab.starts()[0].0;+ // Load the first machine so that, without the preference, the next+ // drive would go to the other one.+ let other = 1 - first;+ let names = ["a", "b"];+ for (i, host) in [(0, names[other]), (1, names[first]), (2, names[first])] {+ let mut s = driver("1=hang");+ s.on = host.into();+ lab.coord+ .session_start(&format!("filler{i}"), s, 0)+ .await+ .unwrap();+ }+ let v = lab+ .until("work", "epoch 2 running", Duration::from_secs(30), |v| {+ v.epoch == 2 && v.state == sstate::RUNNING+ })+ .await;+ wait_for("epoch 2 started", T, async || {+ (lab.starts_of(2) == 1).then_some(())+ })+ .await;+ let second = lab+ .starts()+ .iter()+ .find(|(_, e)| *e == 2)+ .map(|(i, _)| *i)+ .unwrap();+ assert_eq!(second, first, "{v:?}");+ lab.stop();+}