Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(agents): a durable drive keeps its journal across a driver restart on the same machine (#6228)

ddupont7509a74060fc
diff --git a/docs/content/docs/spaces/guides/keep-an-agent-running.mdx b/docs/content/docs/spaces/guides/keep-an-agent-running.mdx
index 34b33f01c5..05b604844b 100644
--- a/docs/content/docs/spaces/guides/keep-an-agent-running.mdx
+++ b/docs/content/docs/spaces/guides/keep-an-agent-running.mdx
@@ -45,7 +45,11 @@ schedule; those are on [named agents](/spaces/guides/agent-memory-and-identity)
today.
The session keeps a journal in its state directory (`CUA_SESSION_STATE_DIR`,
-or `$CUA_JOB_DATA/session` when that is empty). It holds:
+or `$CUA_JOB_DATA/session` when that is empty). `cua agent add --durable`
+leaves it empty, so the journal sits in the run's data folder on the machine
+that drives it: it outlives the driver's Space, and a driver that dies or stalls
+is started again on the same machine with the same conversation. A new drive on
+another machine does not find it and starts the task from the beginning. It holds:
| File | What |
|---|---|
diff --git a/libs/cua/crates/cua-agents/src/durable.rs b/libs/cua/crates/cua-agents/src/durable.rs
index 9ef03cd81a..0487c99e5a 100644
--- a/libs/cua/crates/cua-agents/src/durable.rs
+++ b/libs/cua/crates/cua-agents/src/durable.rs
@@ -101,8 +101,8 @@ pub struct DurableOptions {
/// overrides it; the journal itself is the truth.
pub resume: bool,
/// Where the journal lives. In a Space: default the run directory. For a
- /// driver spec: default `~/.cua/durable/<session>`, else empty (the
- /// runner then uses `$CUA_JOB_DATA/session`).
+ /// driver spec: default empty (the runner then uses
+ /// `$CUA_JOB_DATA/session`, which outlives the driver's Space).
pub state_dir: Option<String>,
/// The session's name (for the default state directory and the journal).
pub session: Option<String>,
@@ -413,11 +413,14 @@ pub fn durable_driver_spec(
for k in opts.env.keys() {
crate::runs::valid_env_name(k)?;
}
- // Empty: the coordinator leaves CUA_SESSION_STATE_DIR empty and the
- // runner uses $CUA_JOB_DATA/session.
+ // Empty by default: the coordinator leaves CUA_SESSION_STATE_DIR empty and
+ // the runner uses $CUA_JOB_DATA/session. The job data directory is per run
+ // on each machine and the run id is the same for every drive, so the
+ // journal outlives the driver's Space (a `~/.cua/...` default sat in the
+ // Space's home and died with it: the task restarted from scratch).
let state_dir = match (&d.state_dir, &d.session) {
(Some(s), _) => s.clone(),
- (None, Some(n)) if crate::runs::valid_home_name(n) => format!("~/.cua/durable/{n}"),
+ (None, Some(n)) if crate::runs::valid_home_name(n) => String::new(),
(None, Some(n)) => {
return Err(Error::Invalid(format!(
"session name {n:?}: use 1-63 of a-z, 0-9, `.`, `_`, `-`"
@@ -671,10 +674,10 @@ mod tests {
"${CUA_MCP_0_0}"
);
assert_eq!(run["durable"]["policy"], "ask-model");
- assert_eq!(run["durable"]["stateDir"], "~/.cua/durable/nightly");
+ assert_eq!(run["durable"]["stateDir"], "");
assert_eq!(run["exitWhenIdle"], true);
assert_eq!(run["prompt"], "fix the bug");
- assert_eq!(spec.state_dir, "~/.cua/durable/nightly");
+ assert_eq!(spec.state_dir, "");
assert_eq!(spec.label, "claude-code");
}
diff --git a/libs/cua/crates/cua-cli/src/agent_session_cmd.rs b/libs/cua/crates/cua-cli/src/agent_session_cmd.rs
index bcdf9246bb..d9bf906562 100644
--- a/libs/cua/crates/cua-cli/src/agent_session_cmd.rs
+++ b/libs/cua/crates/cua-cli/src/agent_session_cmd.rs
@@ -269,6 +269,10 @@ fn state_line(v: &Value) -> String {
if let Some(n) = v["node"].as_str().filter(|n| !n.is_empty()) {
s.push_str(&format!(", on {n}"));
}
+ // No machine has room for the driver yet.
+ if let Some(w) = v["waiting"].as_str().filter(|w| !w.is_empty()) {
+ s.push_str(&format!(", {w}"));
+ }
s
}
@@ -453,6 +457,19 @@ pub async fn rm(name: &str, json: bool, out: &mut dyn Write) -> Result<i32, CuaE
#[cfg(test)]
mod tests {
+ #[test]
+ fn a_waiting_driver_says_what_it_waits_for() {
+ let v = json!({"name":"n","state":"running","epoch":2,"attempts":2,"max_attempts":5,
+ "waiting":"waiting for a machine with 2 CPUs and 4 GB memory free"});
+ let l = state_line(&v);
+ assert!(
+ l.ends_with(", waiting for a machine with 2 CPUs and 4 GB memory free"),
+ "{l}"
+ );
+ let v = json!({"name":"n","state":"running","epoch":1,"attempts":1,"max_attempts":5});
+ assert!(!state_line(&v).contains("waiting"));
+ }
+
use super::*;
use crate::agent_cmd::AgentCmd;
use clap::Parser;
@@ -604,7 +621,9 @@ mod tests {
assert!(spec["env"].get("CUA_AGENT_PROMPT").is_none());
assert_eq!(spec["env"]["ANTHROPIC_API_KEY"], "sk-ant-secret");
assert!(!bundle.contains("sk-ant-secret"));
- assert_eq!(spec["state_dir"], "~/.cua/durable/fix-it");
+ // The journal stays in the job's data directory, which outlives the
+ // driver's Space on the same machine (never `~/.cua/...` in the Space).
+ assert_eq!(spec["state_dir"], "");
// The harness installs and runs in the driver's Space: sized for it.
assert_eq!(spec["cpus"], ACP_DRIVER_CPUS);
assert_eq!(spec["memory_mb"], ACP_DRIVER_MEMORY_MB);
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
index 66e8b8d310..b58a2d9c07 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
@@ -189,6 +189,8 @@ struct Queued {
/// Not placed before this (a failed attempt's backoff).
not_before: Instant,
submitted_ms: i64,
+ /// Why no machine took it at the last look (empty: not tried yet).
+ waiting: String,
}
/// A gang run as the scheduler keeps it (its row is in the database).
@@ -296,6 +298,10 @@ struct Sched {
/// Start barriers to release with each host's next feed call: host ->
/// job -> attempt.
releases: HashMap<String, BTreeMap<i64, u32>>,
+ /// Run -> the host it is placed on first when that host fits (a durable
+ /// session's next drive: its journal is on the host that ran the last).
+ /// A preference, never a pin: a host that is gone or full is passed over.
+ prefer: HashMap<String, String>,
}
/// A member's start barrier and progress as its host last reported them.
@@ -345,6 +351,7 @@ impl Sched {
ready_at: now,
not_before: now + delay,
submitted_ms: f.submitted_ms,
+ waiting: String::new(),
},
);
self.queue.push_front(id);
@@ -1197,6 +1204,7 @@ impl Coordinator {
ready_at: now,
not_before: now,
submitted_ms: r.submitted_ms,
+ waiting: String::new(),
},
);
g.queue.push_back(r.id);
@@ -1313,6 +1321,9 @@ impl Coordinator {
r.short = r.min.max(1);
}
}
+ for q in g.queued.values_mut() {
+ q.waiting = "waiting for a machine that can take work".into();
+ }
return (vec![], vec![], vec![]);
}
if g.queue.is_empty() && !gangs_ready {
@@ -1362,7 +1373,7 @@ impl Coordinator {
// later job of the same shape is passed over at once, so jobs that
// fit elsewhere (another OS, a pinned host, the elastic capacity)
// behind thousands that do not are still placed.
- let mut missed: std::collections::HashSet<NeedKey> = std::collections::HashSet::new();
+ let mut missed: HashMap<NeedKey, String> = HashMap::new();
while let Some(id) = g.queue.pop_front() {
let Some(q) = g.queued.get(&id) else {
continue;
@@ -1375,7 +1386,10 @@ impl Coordinator {
let (retries, max_attempts, run) = (q.retries, q.max_attempts, q.run.clone());
let need = need_of(&q.spec);
let key = need_key(&need);
- if retries < max_attempts && missed.contains(&key) {
+ if retries < max_attempts && missed.contains_key(&key) {
+ if let Some(q) = g.queued.get_mut(&id) {
+ q.waiting = missed[&key].clone();
+ }
deferred.push(id);
skipped += 1;
if skipped >= SKIP_MAX {
@@ -1410,10 +1424,19 @@ impl Coordinator {
continue;
}
}
- let Some(&best) = placement::rank(&cands, &need, self.cfg.policy).first() else {
+ let ranked = placement::rank(&cands, &need, self.cfg.policy);
+ let preferred = g
+ .prefer
+ .get(&run)
+ .and_then(|n| ranked.iter().copied().find(|&i| &cands[i].name == n));
+ let Some(best) = preferred.or_else(|| ranked.first().copied()) else {
deferred.push(id);
misses += 1;
- missed.insert(key);
+ let why = waiting_reason(&need);
+ if let Some(q) = g.queued.get_mut(&id) {
+ q.waiting = why.clone();
+ }
+ missed.insert(key, why);
continue;
};
misses = 0;
@@ -1423,6 +1446,7 @@ impl Coordinator {
c.pending_jobs += 1;
let node = c.name.clone();
let q = g.queued.remove(&id).expect("queued");
+ g.prefer.remove(&q.run);
let attempt = q.attempt + 1;
let mut spec = (*q.spec).clone();
spec.job_id = host_job_id(&self.cluster_id, id);
@@ -2525,6 +2549,23 @@ impl Coordinator {
g.inflight.values().any(|f| f.node == name)
}
+ /// Places `run` on `host` first when it fits (see `Sched::prefer`).
+ pub(crate) fn prefer_host(&self, run: &str, host: &str) {
+ if !host.is_empty() {
+ let mut g = self.sched.lock().expect("sched");
+ g.prefer.insert(run.to_string(), host.to_string());
+ }
+ }
+
+ /// Why queued job `id` is not placed (empty: it is placed, or not yet tried).
+ pub fn queued_waiting(&self, id: i64) -> String {
+ let g = self.sched.lock().expect("sched");
+ g.queued
+ .get(&id)
+ .map(|q| q.waiting.clone())
+ .unwrap_or_default()
+ }
+
/// Jobs waiting in memory and in flight.
pub fn depth(&self) -> (usize, usize) {
let g = self.sched.lock().expect("sched");
@@ -2909,6 +2950,25 @@ fn need_key(n: &Need) -> NeedKey {
}
/// What a job needs of its host.
+/// What a job that no machine took is waiting for, in plain words.
+fn waiting_reason(need: &Need) -> String {
+ if !need.on.is_empty() && need.on != "auto" {
+ return format!("waiting for {} to have room", need.on);
+ }
+ let mut parts = vec![];
+ if need.millicpus > 0 {
+ let c = need.millicpus.div_ceil(1000);
+ parts.push(format!("{c} CPU{}", if c == 1 { "" } else { "s" }));
+ }
+ if need.memory_mb > 0 {
+ parts.push(format!("{} GB memory", need.memory_mb.div_ceil(1024)));
+ }
+ if parts.is_empty() {
+ return "waiting for a machine with room".into();
+ }
+ format!("waiting for a machine with {} free", parts.join(" and "))
+}
+
fn need_of(spec: &JobSpec) -> Need {
Need {
private_image: cua_cluster::placement::image_is_private(&spec.image),
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs
index 8fc585a859..5483046063 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs
@@ -252,6 +252,10 @@ pub(crate) struct Row {
max_attempts: u32,
driver_job: i64,
node: String,
+ /// The machine of the previous drive (kept when `node` is cleared for
+ /// the next one): the next drive starts there first, where the journal is.
+ #[serde(default)]
+ last_node: String,
error: String,
next_attempt_ms: i64,
/// The Space lease's fencing token of the current epoch.
@@ -317,6 +321,10 @@ pub struct SessionView {
pub error: String,
/// Seconds since the running driver last wrote output (stall watch on).
pub last_output_age_secs: Option<i64>,
+ /// Why the driver is not running yet: no machine has room for it (empty
+ /// when it runs or has not been tried).
+ #[serde(default, skip_serializing_if = "String::is_empty")]
+ pub waiting: String,
/// What it runs (no secrets).
pub spec: SpecView,
/// What happened, oldest first.
@@ -343,6 +351,7 @@ impl Row {
node: self.node.clone(),
error: self.error.clone(),
last_output_age_secs: None,
+ waiting: String::new(),
spec: SpecView {
argv: self.spec.argv.clone(),
env_keys: self.spec.env.keys().cloned().collect(),
@@ -545,6 +554,9 @@ impl Coordinator {
{
v.node = j.node;
}
+ if r.state == sstate::RUNNING && r.driver_job > 0 {
+ v.waiting = self.queued_waiting(r.driver_job);
+ }
if r.state == sstate::RUNNING
&& let Some((job, _, at)) = self
.session_watch
@@ -1001,6 +1013,9 @@ impl Coordinator {
next.attempts += 1;
next.state = sstate::RUNNING.into();
next.driver_job = 0;
+ if !next.node.is_empty() {
+ next.last_node = next.node.clone();
+ }
next.node.clear();
next.error.clear();
next.next_attempt_ms = 0;
@@ -1062,6 +1077,9 @@ impl Coordinator {
let run = format!("session-{}", r.name);
let key = format!("session:{}:{}", r.name, r.epoch);
let spec = driver_job(&r, &run, lease.as_ref())?;
+ // The journal is in the data folder of the machine that ran the last
+ // drive: start there again when that machine has room.
+ self.prefer_host(&run, &r.last_node);
let ids = self
.submit(
&run,
diff --git a/libs/cua/crates/cua-spaces-cluster/tests/common/mod.rs b/libs/cua/crates/cua-spaces-cluster/tests/common/mod.rs
index 74139d3a22..a8502b94a9 100644
--- a/libs/cua/crates/cua-spaces-cluster/tests/common/mod.rs
+++ b/libs/cua/crates/cua-spaces-cluster/tests/common/mod.rs
@@ -61,7 +61,7 @@ impl DriverRun {
///
/// A session driver is `["driver", "<script>"]`; the script says what the
/// driver does at each epoch: `1=exit:1;2=hang;3=exit:0` (`exit:N`, `hang`
-/// until canceled, `sleep:<ms>` then exit 0; an epoch not listed hangs).
+/// until canceled, `sleep:<ms>` then exit 0, `crashafter:<ms>` then exit 1; an epoch not listed hangs).
///
/// argv: `["sleep", "<ms>"]`, `["exit", "<code>"]`, `["nospace"]` (the
/// Space does not start), `["echo", "<text>"]`.
@@ -180,6 +180,13 @@ impl JobExecutor for FakeExec {
_ = tokio::time::sleep(Duration::from_millis(ms)) => ExecEnd::Exited(0),
}
}
+ Some(("crashafter", ms)) => {
+ let ms: u64 = ms.parse().unwrap_or(10);
+ tokio::select! {
+ _ = cancel.cancelled() => ExecEnd::Canceled,
+ _ = tokio::time::sleep(Duration::from_millis(ms)) => ExecEnd::Exited(1),
+ }
+ }
_ => {
cancel.cancelled().await;
ExecEnd::Canceled
diff --git a/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs b/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs
index 2dc6a57ed6..cd3f4e95d4 100644
--- a/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs
+++ b/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs
@@ -1113,3 +1113,86 @@ async fn an_unreachable_host_is_not_a_stall() {
);
lab.stop();
}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_driver_that_fits_on_no_machine_says_what_it_waits_for() {
+ let lab = Lab::new(&["a"]).await;
+ let mut spec = driver("1=hang");
+ // The test hosts have 8 CPUs: this driver fits on none of them.
+ spec.cpus = 64.0;
+ spec.memory_mb = 4096;
+ lab.coord.session_start("big", spec, 0).await.unwrap();
+ let v = lab
+ .until("big", "a waiting reason", T, |v| !v.waiting.is_empty())
+ .await;
+ assert_eq!(v.state, sstate::RUNNING);
+ assert!(
+ v.waiting.contains("64 CPUs") && v.waiting.contains("4 GB memory"),
+ "{}",
+ v.waiting
+ );
+ assert_eq!(lab.starts_of(1), 0, "it must not have started");
+ lab.stop();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_driver_that_fits_has_no_waiting_reason() {
+ let lab = Lab::new(&["a"]).await;
+ lab.coord
+ .session_start("small", driver("1=hang"), 0)
+ .await
+ .unwrap();
+ wait_for("epoch 1 started", T, async || {
+ (lab.starts_of(1) == 1).then_some(())
+ })
+ .await;
+ let v = lab.coord.session_get("small").await.unwrap();
+ assert!(v.waiting.is_empty(), "{}", v.waiting);
+ lab.stop();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn the_next_drive_starts_on_the_machine_that_ran_the_last_when_it_has_room() {
+ // The journal is in the data folder of the machine that ran the last
+ // drive. Spreading would send the next drive to the idle machine `b`;
+ // the session goes back to `a`.
+ let lab = Lab::new(&["a", "b"]).await;
+ lab.coord
+ .session_start("work", driver("1=crashafter:1500;2=hang"), 0)
+ .await
+ .unwrap();
+ wait_for("epoch 1 started", T, async || {
+ (lab.starts_of(1) == 1).then_some(())
+ })
+ .await;
+ let first = lab.starts()[0].0;
+ // Load the first machine so that, without the preference, the next
+ // drive would go to the other one.
+ let other = 1 - first;
+ let names = ["a", "b"];
+ for (i, host) in [(0, names[other]), (1, names[first]), (2, names[first])] {
+ let mut s = driver("1=hang");
+ s.on = host.into();
+ lab.coord
+ .session_start(&format!("filler{i}"), s, 0)
+ .await
+ .unwrap();
+ }
+ let v = lab
+ .until("work", "epoch 2 running", Duration::from_secs(30), |v| {
+ v.epoch == 2 && v.state == sstate::RUNNING
+ })
+ .await;
+ wait_for("epoch 2 started", T, async || {
+ (lab.starts_of(2) == 1).then_some(())
+ })
+ .await;
+ let second = lab
+ .starts()
+ .iter()
+ .find(|(_, e)| *e == 2)
+ .map(|(i, _)| *i)
+ .unwrap();
+ assert_eq!(second, first, "{v:?}");
+ lab.stop();
+}