Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(agents): plain words for how many drives a durable agent used and why one failed (#6247)

ddupont321938ca5b14
diff --git a/libs/cua/crates/cua-cli/src/agent_session_cmd.rs b/libs/cua/crates/cua-cli/src/agent_session_cmd.rs
index e8a543f3ad..e78a549e60 100644
--- a/libs/cua/crates/cua-cli/src/agent_session_cmd.rs
+++ b/libs/cua/crates/cua-cli/src/agent_session_cmd.rs
@@ -383,6 +383,9 @@ pub async fn status(name: &str, json: bool, out: &mut dyn Write) -> Result<i32,
return Ok(0);
}
line(out, state_line(&v));
+ if let Some(l) = drives_line(&v) {
+ line(out, l);
+ }
if let Some(a) = v["last_output_age_secs"].as_i64() {
line(
out,
@@ -396,28 +399,66 @@ pub async fn status(name: &str, json: bool, out: &mut dyn Write) -> Result<i32,
line(out, format!("last problem: {e}"));
}
for h in v["history"].as_array().into_iter().flatten() {
- line(
- out,
- format!(
- " epoch {} drive {} {}{}{}",
- h["epoch"],
- h["attempt"],
- h["event"].as_str().unwrap_or(""),
- h["exit_code"]
- .as_i64()
- .map(|c| format!(" (exit {c})"))
- .unwrap_or_default(),
- h["node"]
- .as_str()
- .filter(|n| !n.is_empty())
- .map(|n| format!(" on {n}"))
- .unwrap_or_default(),
- ),
- );
+ line(out, history_line(h));
}
Ok(0)
}
+/// How many drives a session has used and how many of them failed, in words
+/// ("drive 2 of 5 so far, 1 failed"): a restart after a lost machine or a
+/// crashed driver is a drive of its own, and this says which ones.
+pub(crate) fn drives_line(v: &Value) -> Option<String> {
+ let (used, max) = (v["attempts"].as_u64()?, v["max_attempts"].as_u64()?);
+ let failed = v["history"]
+ .as_array()
+ .into_iter()
+ .flatten()
+ .filter(|h| matches!(h["event"].as_str(), Some("crashed" | "stalled" | "fenced")))
+ .count();
+ if used < 2 && failed == 0 {
+ return None;
+ }
+ Some(format!(
+ "drive {used} of {max} so far, {failed} failed ({} left)",
+ max.saturating_sub(used)
+ ))
+}
+
+/// One history entry of `cua agent status`. A drive that ended badly says why
+/// (the driver's last words when the machine kept them), cut to one line.
+pub(crate) fn history_line(h: &Value) -> String {
+ let event = h["event"].as_str().unwrap_or("");
+ let mut s = format!(
+ " epoch {} drive {} {}{}{}",
+ h["epoch"],
+ h["attempt"],
+ event,
+ h["exit_code"]
+ .as_i64()
+ .map(|c| format!(" (exit {c})"))
+ .unwrap_or_default(),
+ h["node"]
+ .as_str()
+ .filter(|n| !n.is_empty())
+ .map(|n| format!(" on {n}"))
+ .unwrap_or_default(),
+ );
+ if matches!(event, "crashed" | "stalled" | "fenced") {
+ let d = h["detail"]
+ .as_str()
+ .unwrap_or("")
+ .split_whitespace()
+ .collect::<Vec<_>>()
+ .join(" ");
+ if !d.is_empty() {
+ let skip = d.chars().count().saturating_sub(200);
+ let tail: String = d.chars().skip(skip).collect();
+ s.push_str(&format!(": {}{tail}", if skip > 0 { "..." } else { "" }));
+ }
+ }
+ s
+}
+
/// `cua agent logs session:NAME [-f]`: the output of each epoch's driver in
/// turn (a separator line where a new driver starts).
pub async fn logs(
@@ -498,6 +539,31 @@ pub async fn rm(name: &str, json: bool, out: &mut dyn Write) -> Result<i32, CuaE
#[cfg(test)]
mod tests {
+ #[test]
+ fn status_says_how_many_drives_failed_and_why() {
+ let v = json!({"attempts": 2, "max_attempts": 5, "history": [
+ {"event":"started"},{"event":"crashed"},{"event":"started"}]});
+ assert_eq!(
+ drives_line(&v).unwrap(),
+ "drive 2 of 5 so far, 1 failed (3 left)"
+ );
+ // A first drive with nothing failed needs no line.
+ let v = json!({"attempts": 1, "max_attempts": 5, "history": [{"event":"started"}]});
+ assert!(drives_line(&v).is_none());
+ // A crashed drive carries its reason, cut to one line from the end.
+ let h = json!({"epoch":1,"attempt":1,"event":"crashed","exit_code":1,"node":"m1",
+ "detail":"drive 1 of 5: the driver ended (failed), exit 1: Error: ENOSPC: no space left on device\n at write"});
+ let l = history_line(&h);
+ assert!(
+ l.starts_with(" epoch 1 drive 1 crashed (exit 1) on m1: drive 1 of 5"),
+ "{l}"
+ );
+ assert!(l.contains("ENOSPC") && !l.contains('\n'), "{l}");
+ // A drive that started or finished has no detail.
+ let h = json!({"epoch":1,"attempt":1,"event":"started","detail":"x"});
+ assert_eq!(history_line(&h), " epoch 1 drive 1 started");
+ }
+
#[test]
fn a_waiting_driver_says_what_it_waits_for() {
let v = json!({"name":"n","state":"running","epoch":2,"attempts":2,"max_attempts":5,
diff --git a/libs/cua/crates/cua-cluster/src/node.rs b/libs/cua/crates/cua-cluster/src/node.rs
index 871a0256ba..9715b6bb90 100644
--- a/libs/cua/crates/cua-cluster/src/node.rs
+++ b/libs/cua/crates/cua-cluster/src/node.rs
@@ -1496,11 +1496,23 @@ fn release(g: &mut Inner, spec: &JobSpec, kind: &str) {
/// it never got to write.
pub(crate) fn exit_reason(code: i32, stdout: &[u8], stderr: &[u8]) -> String {
let last = [stderr, stdout].iter().find_map(|b| {
- String::from_utf8_lossy(b)
+ let text = String::from_utf8_lossy(b);
+ let lines: Vec<&str> = text
.lines()
.map(str::trim)
- .rfind(|l| !l.is_empty())
- .map(|l| l.chars().take(240).collect::<String>())
+ .filter(|l| !l.is_empty())
+ .collect();
+ let last = lines.last()?;
+ // Node ends an uncaught error with a bare `Node.js v24.1.0` line, and
+ // the cause (`code: 'ENOSPC'`, a full disk) sits a few lines above
+ // it: keep those with it.
+ if last.starts_with("Node.js v") && lines.len() > 1 {
+ let from = lines.len().saturating_sub(7);
+ let joined = lines[from..].join(" ");
+ let skip = joined.chars().count().saturating_sub(240);
+ return Some(joined.chars().skip(skip).collect::<String>());
+ }
+ Some(last.chars().take(240).collect::<String>())
});
match last {
Some(l) => format!("exited with code {code}: {l}"),
@@ -1537,4 +1549,23 @@ mod exit_reason_tests {
"exited with code 3: err"
);
}
+
+ /// A Node crash ends with `Node.js v24.1.0`; the reason keeps the lines
+ /// above it, where a full disk says `ENOSPC` (CUA-3000).
+ #[test]
+ fn a_node_crash_keeps_the_lines_above_its_version_footer() {
+ let err = b"node:fs:2397\n throw err;\nError: ENOSPC: no space left on device, write\n at writeSync (node:fs:2397:20) {\n errno: -28,\n code: 'ENOSPC',\n syscall: 'write'\n}\n\nNode.js v24.21.0\n";
+ let r = super::exit_reason(1, b"", err);
+ assert!(r.starts_with("exited with code 1: "), "{r}");
+ assert!(
+ r.contains("ENOSPC") && r.contains("Node.js v24.21.0"),
+ "{r}"
+ );
+ assert!(!r.contains('\n'), "{r}");
+ // A lone footer is still just the footer.
+ assert_eq!(
+ super::exit_reason(1, b"", b"Node.js v24.21.0\n"),
+ "exited with code 1: Node.js v24.21.0"
+ );
+ }
}
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
index b58a2d9c07..3b89767c39 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
@@ -1407,10 +1407,7 @@ impl Coordinator {
ended.push(Change::EndQueued {
id,
state: state::ERROR.into(),
- error: format!(
- "gave up after {} attempts were lost (hosts went down or could not run it)",
- q.retries
- ),
+ error: lost_attempts_words(q.retries),
at_ms: now_ms(),
});
Metrics::inc(&self.metrics.ended_other);
@@ -1846,7 +1843,12 @@ impl Coordinator {
g.land(&f);
let mut c = complete(id, h, state::ERROR);
if let Change::Complete { error, .. } = &mut c {
- *error = format!("gave up after {} attempts: {}", f.retries + 1, h.error);
+ *error = format!(
+ "gave up after {} attempt{}: {}",
+ f.retries + 1,
+ if f.retries == 0 { "" } else { "s" },
+ h.error
+ );
}
changes.push(c);
Metrics::inc(&self.metrics.ended_other);
@@ -3016,6 +3018,33 @@ fn member_end(id: i64, attempt: u32, state: &str, error: String) -> Change {
/// it stops: a bound on the work per round with a very long queue.
const SKIP_MAX: usize = 200_000;
+/// Why a job that was lost and queued again is given up, in plain words
+/// ("1 attempt was lost", "3 attempts were lost").
+fn lost_attempts_words(n: u32) -> String {
+ format!(
+ "{n} {} lost: the machine it ran on stopped answering or could not run it",
+ if n == 1 {
+ "attempt was"
+ } else {
+ "attempts were"
+ }
+ )
+}
+
+#[cfg(test)]
+mod words_tests {
+ use super::lost_attempts_words;
+
+ #[test]
+ fn the_give_up_message_has_the_right_plural_and_no_job_names() {
+ assert_eq!(
+ lost_attempts_words(1),
+ "1 attempt was lost: the machine it ran on stopped answering or could not run it"
+ );
+ assert!(lost_attempts_words(3).starts_with("3 attempts were lost"));
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs
index 5483046063..eb864f3b22 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs
@@ -840,9 +840,9 @@ impl Coordinator {
let mut n = r.clone();
n.node = j.node.clone();
let why = format!(
- "epoch {}: driver job {} wrote no output for {} s (limit {limit} s): wedged, cancelled",
- n.epoch,
- j.id,
+ "drive {} of {}: the driver wrote no output for {} s (limit {limit} s): wedged, cancelled",
+ n.attempts,
+ n.max_attempts,
(now - changed) / 1000
);
if let Err(e) = self.cancel_job(j.id).await {
@@ -874,7 +874,10 @@ impl Coordinator {
r.node = j.node.clone();
let code = j.exit_code;
let why = {
- let mut w = format!("epoch {}: driver job {} {}", r.epoch, j.id, j.state);
+ let mut w = format!(
+ "drive {} of {}: the driver ended ({})",
+ r.attempts, r.max_attempts, j.state
+ );
if let Some(c) = code {
w.push_str(&format!(", exit {c}"));
}
diff --git a/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs b/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs
index cd3f4e95d4..ccd725e864 100644
--- a/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs
+++ b/libs/cua/crates/cua-spaces-cluster/tests/sessions.rs
@@ -310,6 +310,12 @@ async fn a_session_that_keeps_crashing_ends_failed_exhausted_with_a_clear_error(
v.error
);
assert!(v.error.contains("exit 3"), "{}", v.error);
+ // Plain words: the drive, not an internal job number.
+ assert!(
+ v.error.contains("drive 3 of 3: the driver ended") && !v.error.contains("driver job"),
+ "{}",
+ v.error
+ );
tokio::time::sleep(Duration::from_secs(5)).await;
assert_eq!(lab.starts().len(), 3, "no fourth drive");
assert_eq!(