Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(agents): plain words for how many drives a durable agent used and why one failed (#6247)
ddupont321938ca5b14
diff --git a/libs/cua/crates/cua-cli/src/agent_session_cmd.rs b/libs/cua/crates/cua-cli/src/agent_session_cmd.rsindex e8a543f3ad..e78a549e60 100644--- a/libs/cua/crates/cua-cli/src/agent_session_cmd.rs+++ b/libs/cua/crates/cua-cli/src/agent_session_cmd.rs@@ -383,6 +383,9 @@ pub async fn status(name: &str, json: bool, out: &mut dyn Write) -> Result<i32,return Ok(0);}line(out, state_line(&v));+ if let Some(l) = drives_line(&v) {+ line(out, l);+ }if let Some(a) = v["last_output_age_secs"].as_i64() {line(out,@@ -396,28 +399,66 @@ pub async fn status(name: &str, json: bool, out: &mut dyn Write) -> Result<i32,line(out, format!("last problem: {e}"));}for h in v["history"].as_array().into_iter().flatten() {- line(- out,- format!(- " epoch {} drive {} {}{}{}",- h["epoch"],- h["attempt"],- h["event"].as_str().unwrap_or(""),- h["exit_code"]- .as_i64()- .map(|c| format!(" (exit {c})"))- .unwrap_or_default(),- h["node"]- .as_str()- .filter(|n| !n.is_empty())- .map(|n| format!(" on {n}"))- .unwrap_or_default(),- ),- );+ line(out, history_line(h));}Ok(0)}+/// How many drives a session has used and how many of them failed, in words+/// ("drive 2 of 5 so far, 1 failed"): a restart after a lost machine or a+/// crashed driver is a drive of its own, and this says which ones.+pub(crate) fn drives_line(v: &Value) -> Option<String> {+ let (used, max) = (v["attempts"].as_u64()?, v["max_attempts"].as_u64()?);+ let failed = v["history"]+ .as_array()+ .into_iter()+ .flatten()+ .filter(|h| matches!(h["event"].as_str(), Some("crashed" | "stalled" | "fenced")))+ .count();+ if used < 2 && failed == 0 {+ return None;+ }+ Some(format!(+ "drive {used} of {max} so far, {failed} failed ({} left)",+ max.saturating_sub(used)+ ))+}++/// One history entry of `cua agent status`. A drive that ended badly says why+/// (the driver's last words when the machine kept them), cut to one line.+pub(crate) fn history_line(h: &Value) -> String {+ let event = h["event"].as_str().unwrap_or("");+ let mut s = format!(+ " epoch {} drive {} {}{}{}",+ h["epoch"],+ h["attempt"],+ event,+ h["exit_code"]+ .as_i64()+ .map(|c| format!(" (exit {c})"))+ .unwrap_or_default(),+ h["node"]+ .as_str()+ .filter(|n| !n.is_empty())+ .map(|n| format!(" on {n}"))+ .unwrap_or_default(),+ );+ if matches!(event, "crashed" | "stalled" | "fenced") {+ let d = h["detail"]+ .as_str()+ .unwrap_or("")+ .split_whitespace()+ .collect::<Vec<_>>()+ .join(" ");+ if !d.is_empty() {+ let skip = d.chars().count().saturating_sub(200);+ let tail: String = d.chars().skip(skip).collect();+ s.push_str(&format!(": {}{tail}", if skip > 0 { "..." } else { "" }));+ }+ }+ s+}+/// `cua agent logs session:NAME [-f]`: the output of each epoch's driver in/// turn (a separator line where a new driver starts).pub async fn logs(@@ -498,6 +539,31 @@ pub async fn rm(name: &str, json: bool, out: &mut dyn Write) -> Result<i32, CuaE#[cfg(test)]mod tests {+ #[test]+ fn status_says_how_many_drives_failed_and_why() {+ let v = json!({"attempts": 2, "max_attempts": 5, "history": [+ {"event":"started"},{"event":"crashed"},{"event":"started"}]});+ assert_eq!(+ drives_line(&v).unwrap(),+ "drive 2 of 5 so far, 1 failed (3 left)"+ );+ // A first drive with nothing failed needs no line.+ let v = json!({"attempts": 1, "max_attempts": 5, "history": [{"event":"started"}]});+ assert!(drives_line(&v).is_none());+ // A crashed drive carries its reason, cut to one line from the end.+ let h = json!({"epoch":1,"attempt":1,"event":"crashed","exit_code":1,"node":"m1",+ "detail":"drive 1 of 5: the driver ended (failed), exit 1: Error: ENOSPC: no space left on device\n at write"});+ let l = history_line(&h);+ assert!(+ l.starts_with(" epoch 1 drive 1 crashed (exit 1) on m1: drive 1 of 5"),+ "{l}"+ );+ assert!(l.contains("ENOSPC") && !l.contains('\n'), "{l}");+ // A drive that started or finished has no detail.+ let h = json!({"epoch":1,"attempt":1,"event":"started","detail":"x"});+ assert_eq!(history_line(&h), " epoch 1 drive 1 started");+ }+#[test]fn a_waiting_driver_says_what_it_waits_for() {let v = json!({"name":"n","state":"running","epoch":2,"attempts":2,"max_attempts":5,diff --git a/libs/cua/crates/cua-cluster/src/node.rs b/libs/cua/crates/cua-cluster/src/node.rsindex 871a0256ba..9715b6bb90 100644--- a/libs/cua/crates/cua-cluster/src/node.rs+++ b/libs/cua/crates/cua-cluster/src/node.rs@@ -1496,11 +1496,23 @@ fn release(g: &mut Inner, spec: &JobSpec, kind: &str) {/// it never got to write.pub(crate) fn exit_reason(code: i32, stdout: &[u8], stderr: &[u8]) -> String {let last = [stderr, stdout].iter().find_map(|b| {- String::from_utf8_lossy(b)+ let text = String::from_utf8_lossy(b);+ let lines: Vec<&str> = text.lines().map(str::trim)- .rfind(|l| !l.is_empty())- .map(|l| l.chars().take(240).collect::<String>())+ .filter(|l| !l.is_empty())+ .collect();+ let last = lines.last()?;+ // Node ends an uncaught error with a bare `Node.js v24.1.0` line, and+ // the cause (`code: 'ENOSPC'`, a full disk) sits a few lines above+ // it: keep those with it.+ if last.starts_with("Node.js v") && lines.len() > 1 {+ let from = lines.len().saturating_sub(7);+ let joined = lines[from..].join(" ");+ let skip = joined.chars().count().saturating_sub(240);+ return Some(joined.chars().skip(skip).collect::<String>());+ }+ Some(last.chars().take(240).collect::<String>())});match last {Some(l) => format!("exited with code {code}: {l}"),@@ -1537,4 +1549,23 @@ mod exit_reason_tests {"exited with code 3: err");}++ /// A Node crash ends with `Node.js v24.1.0`; the reason keeps the lines+ /// above it, where a full disk says `ENOSPC` (CUA-3000).+ #[test]+ fn a_node_crash_keeps_the_lines_above_its_version_footer() {+ let err = b"node:fs:2397\n throw err;\nError: ENOSPC: no space left on device, write\n at writeSync (node:fs:2397:20) {\n errno: -28,\n code: 'ENOSPC',\n syscall: 'write'\n}\n\nNode.js v24.21.0\n";+ let r = super::exit_reason(1, b"", err);+ assert!(r.starts_with("exited with code 1: "), "{r}");+ assert!(+ r.contains("ENOSPC") && r.contains("Node.js v24.21.0"),+ "{r}"+ );+ assert!(!r.contains('\n'), "{r}");+ // A lone footer is still just the footer.+ assert_eq!(+ super::exit_reason(1, b"", b"Node.js v24.21.0\n"),+ "exited with code 1: Node.js v24.21.0"+ );+ }}diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rsindex b58a2d9c07..3b89767c39 100644--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs@@ -1407,10 +1407,7 @@ impl Coordinator {ended.push(Change::EndQueued {id,state: state::ERROR.into(),- error: format!(- "gave up after {} attempts were lost (hosts went down or could not run it)",- q.retries- ),+ error: lost_attempts_words(q.retries),at_ms: now_ms(),});Metrics::inc(&self.metrics.ended_other);@@ -1846,7 +1843,12 @@ impl Coordinator {g.land(&f);let mut c = complete(id, h, state::ERROR);if let Change::Complete { error, .. } = &mut c {- *error = format!("gave up after {} attempts: {}", f.retries + 1, h.error);+ *error = format!(+ "gave up after {} attempt{}: {}",+ f.retries + 1,+ if f.retries == 0 { "" } else { "s" },+ h.error+ );}changes.push(c);Metrics::inc(&self.metrics.ended_other);@@ -3016,6 +3018,33 @@ fn member_end(id: i64, attempt: u32, state: &str, error: String) -> Change {/// it stops: a bound on the work per round with a very long queue.const SKIP_MAX: usize = 200_000;+/// Why a job that was lost and queued again is given up, in plain words+/// ("1 attempt was lost", "3 attempts were lost").+fn lost_attempts_words(n: u32) -> String {+ format!(+ "{n} {} lost: the machine it ran on stopped answering or could not run it",+ if n == 1 {+ "attempt was"+ } else {+ "attempts were"+ }+ )+}++#[cfg(test)]+mod words_tests {+ use super::lost_attempts_words;++ #[test]+ fn the_give_up_message_has_the_right_plural_and_no_job_names() {+ assert_eq!(+ lost_attempts_words(1),+ "1 attempt was lost: the machine it ran on stopped answering or could not run it"+ );+ assert!(lost_attempts_words(3).starts_with("3 attempts were lost"));+ }+}+#[cfg(test)]mod tests {use super::*;diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rsindex 5483046063..eb864f3b22 100644--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs@@ -840,9 +840,9 @@ impl Coordinator {let mut n = r.clone();n.node = j.node.clone();let why = format!(- "epoch {}: driver job {} wrote no output for {} s (limit {limit} s): wedged, cancelled",- n.epoch,- j.id,+ "drive {} of {}: the driver wrote no output for {} s (limit {limit} s): wedged, cancelled",+ n.attempts,+ n.max_attempts,(now - changed) / 1000);if let Err(e) = self.cancel_job(j.id).await {@@ -874,7 +874,10 @@ impl Coordinator {r.node = j.node.clone();let code = j.exit_code;let why = {- let mut w = format!("epoch {}: driver job {} {}", r.epoch, j.id, j.state);+ let mut w = format!(+ "drive {} of {}: the driver ended ({})",+ r.attempts, r.max_attempts, j.state+ );if let Some(c) = code {w.push_str(&format!(", exit {c}"));}diff --git a/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs b/libs/cua/crates/cua-spaces-cluster/tests/sessions.rsindex cd3f4e95d4..ccd725e864 100644--- a/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs+++ b/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs@@ -310,6 +310,12 @@ async fn a_session_that_keeps_crashing_ends_failed_exhausted_with_a_clear_error(v.error);assert!(v.error.contains("exit 3"), "{}", v.error);+ // Plain words: the drive, not an internal job number.+ assert!(+ v.error.contains("drive 3 of 3: the driver ended") && !v.error.contains("driver job"),+ "{}",+ v.error+ );tokio::time::sleep(Duration::from_secs(5)).await;assert_eq!(lab.starts().len(), 3, "no fourth drive");assert_eq!(