Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(telemetry): report how durable agents behave in the field (#6207)

ddupont514b2a8f0c0c
diff --git a/docs/content/docs/cua-sdk/concepts/telemetry-events.mdx b/docs/content/docs/cua-sdk/concepts/telemetry-events.mdx
index 7fb6e395f2..62fd5cc7c9 100644
--- a/docs/content/docs/cua-sdk/concepts/telemetry-events.mdx
+++ b/docs/content/docs/cua-sdk/concepts/telemetry-events.mdx
@@ -35,6 +35,7 @@ Every event carries: `telemetry_schema_version`, `event_version`, `product`
| `cua_agent_run_started` | `harness` (cua-agents id or `other`), `location`, `entry` (`spaces_tool`, `cli`, `sdk`, `persistent`, `routine`), `outcome`, `error_kind` | Agent runs started, and failed starts. |
| `cua_agent_run_completed` | `harness`, `location`, `entry`, `outcome`, `error_kind`, `duration_bucket` | First agent run, and how reliably runs finish. Once per run. |
| `cua_agent_join_phase` | `phase` (`posted`, `dispatch_picked`, `host_prepared`, `guest_spawned`, `install_done`, `runner_started`, `agent_spawned`, `acp_initialized`, `acp_authenticated`, `session_ready`, `turn_started`, `first_token`, `turn_ended` or `failed`), `harness`, `runtime`, `cold`, `ms_since_post`, `run` (a per-install salted hash, never the run id), `tool` (`install_done` only), `failure` (`failed` only: `auth`, `install`, `timeout` or `other`) | How long an agent takes to join a Space and answer, step by step, and where joins fail. Never the message, the Space or the account. |
+| `cua_durable_agent` | `phase` (`started`, `restarted`, `failed_over`, `exhausted`, `ended`, `space_recreated`, `refused`), `harness`, `placement` (`auto` or `host`, never which machine), `attempt`, `max_attempts`, `reason` (`process_exit`, `stall`, `driver_lost`, `host_down`, `fenced`, `permanent`, `other`, or `none`), `end` (`done`, `failed`, `stopped`, or `none`), `failure` (`exhausted`, `permanent`, `space_gone`, `setup_failed`, `lease_refused`, `epoch_refused`, `other`, or `none`), `failover`, `machines` (a count), `recovery_time_ms`, `interrupted_calls` (`0` when not known), `session` (a per-install salted hash, never the name) | How durable agents behave in the field: restarts and why, moves to another machine, how long recovery takes, how often attempts run out, how sessions end. Sent by the machine that keeps the account's sessions. |
| `cua_repo_cache` | `outcome` (`hit`, `miss` or `fallback`), `source` (`local` or `network`), `bytes_saved`, `seconds_saved`, `reason` (`none`, `disabled`, `no_git`, `unsupported_url`, `host_not_public`, `private`, `unreachable`, `too_big`, `cache_full`, `disk_full`, `verify_failed`, `fill_failed` or `other`) | Whether the repo cache makes repeat `git clone`s in Spaces faster, how often it can be used, and why it cannot. Counts and fixed words only: never a URL, a repository name or a path. |
| `cua_bench_run_completed` | `taskset` (cua-bench dataset or `custom`), `score_bucket`, `task_count`, `outcome` | Benchmark use and coarse scores. |
| `cua_daemon_started` | `mode`, `outcome` | Daemon startup reliability. |
diff --git a/docs/content/docs/cua-sdk/concepts/telemetry.mdx b/docs/content/docs/cua-sdk/concepts/telemetry.mdx
index c4e5e65ce5..9317a9a794 100644
--- a/docs/content/docs/cua-sdk/concepts/telemetry.mdx
+++ b/docs/content/docs/cua-sdk/concepts/telemetry.mdx
@@ -103,6 +103,21 @@ anonymous session per launch, so we can tell what share of launches end in a
crash. What is in a report, how it is cleaned, and what we use it for:
[Crash and error reports](/cua-sdk/concepts/telemetry-events#crash-and-error-reports).
+## Durable agents
+
+When you run an agent with `--durable`, or make a durable Space, the machine
+that keeps the account's sessions sends a few events about how it went: that
+it started, that it was restarted and why (the process exited, it printed
+nothing for too long, its machine stopped answering), that it moved to
+another machine, how long recovery took, that it ran out of attempts, and how
+it ended (done, failed or stopped). They hold fixed words and counts, and a
+random-looking hash of the session so the events of one session can be put
+together. They never hold the task, prompts, tool inputs or outputs, file
+paths, repository names, keys, your email, or the names of your machines,
+sessions or Spaces. They follow the same switch as everything above:
+`cua telemetry off`, `CUA_TELEMETRY=0` or `DO_NOT_TRACK=1` stops them. See
+`cua_durable_agent` in [the event list](/cua-sdk/concepts/telemetry-events).
+
## What we use it for
To answer how many people install Cua Spaces, finish the first run, sign in,
diff --git a/libs/cua/Cargo.lock b/libs/cua/Cargo.lock
index dcf38890c2..f7500f3e52 100644
--- a/libs/cua/Cargo.lock
+++ b/libs/cua/Cargo.lock
@@ -3275,6 +3275,7 @@ dependencies = [
"cua-host",
"cua-proto",
"cua-spaces",
+ "cua-telemetry",
"flate2",
"hex",
"hpke",
diff --git a/libs/cua/crates/cua-spaces-cluster/Cargo.toml b/libs/cua/crates/cua-spaces-cluster/Cargo.toml
index 6944cc4e02..66022ca913 100644
--- a/libs/cua/crates/cua-spaces-cluster/Cargo.toml
+++ b/libs/cua/crates/cua-spaces-cluster/Cargo.toml
@@ -22,6 +22,7 @@ ring = "0.17"
zeroize = "1"
cua-cluster.workspace = true
cua-host.workspace = true
+cua-telemetry.workspace = true
cua-proto = { workspace = true, features = ["client", "server", "serde"] }
hex.workspace = true
rand.workspace = true
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs
index 8cb21fb434..83a28c2897 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs
@@ -570,6 +570,13 @@ impl Coordinator {
self.audit("durable", "coordinator", &row.name, &detail)
.await;
let row = self.durable_save(row).await?;
+ if row.generation > 1 {
+ super::telemetry::space_recreated(
+ &row.name,
+ !from.is_empty() && from != row.node,
+ self.nodes().len() as u32,
+ );
+ }
self.durable_setup_spawn(&row);
Ok(row)
}
@@ -703,6 +710,7 @@ impl Coordinator {
.ok_or_else(|| CoordError::NotFound(format!("no durable Space {name}")))?;
let now = now_ms();
if r.leased(now) && r.holder != holder.trim() {
+ super::telemetry::lease_refused(name, self.nodes().len() as u32);
return Err(CoordError::Denied(format!(
"{name} is held by {} for {} s more",
r.holder,
@@ -736,6 +744,7 @@ impl Coordinator {
.filter(|r| r.state != state::DELETED)
.ok_or_else(|| CoordError::NotFound(format!("no durable Space {name}")))?;
if r.lease_token != token || !r.leased(now_ms()) {
+ super::telemetry::lease_refused(name, self.nodes().len() as u32);
return Err(CoordError::Denied(format!(
"the lease of {name} with token {token} is no longer held (it ran out or was taken); acquire it again"
)));
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
index 920c981e1e..5d9a88925b 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
@@ -39,6 +39,7 @@ pub mod metrics;
pub mod requests;
pub mod sessions;
pub mod store;
+mod telemetry;
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::path::PathBuf;
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs
index f69011b82c..8fc585a859 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions.rs
@@ -84,6 +84,8 @@ use std::time::Duration;
use rusqlite::{Connection, OptionalExtension, params};
use serde::{Deserialize, Serialize};
+mod telemetry;
+
use super::store::{self, NewJob, state as jstate};
use super::{CoordError, Coordinator, sanitize_name};
use cua_cluster::model::{JobSpec, now_ms};
@@ -232,6 +234,11 @@ pub struct Hist {
pub exit_code: Option<i32>,
/// More.
pub detail: String,
+ /// Why a drive ended badly, as a fixed word (`process_exit`, `stall`,
+ /// `host_down`, `driver_lost`, `fenced`); empty otherwise. Telemetry
+ /// reads it; it never holds free text.
+ #[serde(default, skip_serializing_if = "String::is_empty")]
+ pub cause: String,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@@ -254,6 +261,10 @@ pub(crate) struct Row {
history: Vec<Hist>,
created_ms: i64,
updated_ms: i64,
+ /// What [`Row::note`] recorded since this row was loaded: telemetry
+ /// reads it once the row is saved. Never stored.
+ #[serde(skip)]
+ fresh: Vec<Hist>,
}
/// What `spec` shows back: no environment values, no bundle.
@@ -351,7 +362,17 @@ impl Row {
}
fn note(&mut self, event: &str, exit_code: Option<i32>, detail: impl Into<String>) {
- self.history.push(Hist {
+ self.note_cause(event, exit_code, detail, "");
+ }
+
+ fn note_cause(
+ &mut self,
+ event: &str,
+ exit_code: Option<i32>,
+ detail: impl Into<String>,
+ cause: &str,
+ ) {
+ let h = Hist {
epoch: self.epoch,
attempt: self.attempts,
at_ms: now_ms(),
@@ -360,7 +381,10 @@ impl Row {
job: self.driver_job,
exit_code,
detail: detail.into(),
- });
+ cause: cause.into(),
+ };
+ self.fresh.push(h.clone());
+ self.history.push(h);
if self.history.len() > HISTORY_MAX {
let cut = self.history.len() - HISTORY_MAX;
self.history.drain(..cut);
@@ -441,8 +465,8 @@ pub(crate) fn remove_named(c: &Connection, name: &str) -> Result<bool, String> {
/// The driver of `r`'s epoch ended badly (`event`: `crashed`, `fenced` or
/// `stalled`): back off for the next drive, or give up at the cap.
-fn mark_crashed(r: &mut Row, event: &str, code: Option<i32>, why: String) {
- r.note(event, code, why.clone());
+fn mark_crashed(r: &mut Row, event: &str, code: Option<i32>, why: String, cause: &str) {
+ r.note_cause(event, code, why.clone(), cause);
if r.attempts >= r.max_attempts {
r.state = sstate::FAILED.into();
r.reason = reason::EXHAUSTED.into();
@@ -500,7 +524,12 @@ impl Coordinator {
}])
.await
.map_err(CoordError::Store)?;
- Ok(outs.first().is_some_and(|o| o.applied()).then_some(r))
+ let applied = outs.first().is_some_and(|o| o.applied());
+ if applied && !r.fresh.is_empty() {
+ let machines = self.nodes().len() as u32;
+ telemetry::session_changed(&r, machines);
+ }
+ Ok(applied.then_some(r))
}
/// A session with its driver's machine filled in.
@@ -807,7 +836,7 @@ impl Coordinator {
if let Err(e) = self.cancel_job(j.id).await {
tracing::warn!(session = %name, job = j.id, "cancelling a wedged driver: {e}");
}
- mark_crashed(&mut n, "stalled", None, why);
+ mark_crashed(&mut n, "stalled", None, why, "stall");
let state = n.state.clone();
let Some(n) = self.session_save(n, rev).await? else {
return self.session_reload(&name).await;
@@ -860,7 +889,22 @@ impl Coordinator {
r.note("permanent", code, why);
} else {
let fenced = code == Some(EXIT_FENCED);
- mark_crashed(&mut r, if fenced { "fenced" } else { "crashed" }, code, why);
+ let cause = if fenced {
+ "fenced"
+ } else if j.error.ends_with(" went down") {
+ "host_down"
+ } else if code.is_some() {
+ "process_exit"
+ } else {
+ "driver_lost"
+ };
+ mark_crashed(
+ &mut r,
+ if fenced { "fenced" } else { "crashed" },
+ code,
+ why,
+ cause,
+ );
}
let state = r.state.clone();
let Some(r) = self.session_save(r, rev).await? else {
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions/telemetry.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions/telemetry.rs
new file mode 100644
index 0000000000..7aa10d4a29
--- /dev/null
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/sessions/telemetry.rs
@@ -0,0 +1,295 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Telemetry for what just happened to a durable session: the history
+//! entries a save recorded become `cua_durable_agent` events (see
+//! [`super::super::telemetry`]).
+
+use super::{Hist, Row};
+use cua_telemetry::events::{self, DurableAgent, Event};
+
+/// Words for how a drive ended, by the history entry's event.
+fn is_drive_failure(event: &str) -> bool {
+ matches!(event, "crashed" | "stalled" | "fenced")
+}
+
+/// The machine the drive before `h` ran on, when known.
+fn previous_node<'a>(r: &'a Row, h: &Hist) -> Option<&'a str> {
+ r.history
+ .iter()
+ .rev()
+ .filter(|x| x.epoch < h.epoch && !x.node.is_empty())
+ .map(|x| x.node.as_str())
+ .next()
+}
+
+/// The failure that ended the drive before the one `h` started.
+fn previous_failure<'a>(r: &'a Row, h: &Hist) -> Option<&'a Hist> {
+ r.history
+ .iter()
+ .rev()
+ .find(|x| is_drive_failure(&x.event) && x.at_ms <= h.at_ms && x.epoch < h.epoch)
+}
+
+/// What a permanent failure was, from the recorded message. The message is
+/// only matched here, never sent.
+fn permanent_failure(error: &str) -> &'static str {
+ if error.contains(".cua/setup") {
+ "setup_failed"
+ } else if error.contains("no durable Space") {
+ "space_gone"
+ } else {
+ "permanent"
+ }
+}
+
+/// Sends the events for the history entries `r` recorded since it was
+/// loaded. `machines` is how many machines the coordinator knows.
+pub(super) fn session_changed(r: &Row, machines: u32) {
+ let t = cua_telemetry::global();
+ let Some(hash) = t.salted_hash("durable_session", &r.name) else {
+ return;
+ };
+ for e in session_events(r, machines, &hash) {
+ t.capture(e);
+ }
+}
+
+/// The events for the history entries `r` recorded since it was loaded.
+/// `session` is the salted hash of the session's name.
+pub(super) fn session_events(r: &Row, machines: u32, session: &str) -> Vec<Event> {
+ let mut out = Vec::new();
+ let mut push = |d: DurableAgent<'_>| out.extend(events::durable_agent(&d));
+ for h in &r.fresh {
+ let base = || DurableAgent {
+ harness: &r.spec.label,
+ on: &r.spec.on,
+ attempt: h.attempt,
+ max_attempts: r.max_attempts,
+ reason: "none",
+ end: "none",
+ failure: "none",
+ machines,
+ session,
+ ..Default::default()
+ };
+ // A drive that ended on another machine than the drive before it.
+ let ends_a_drive =
+ is_drive_failure(&h.event) || matches!(h.event.as_str(), "done" | "permanent");
+ if ends_a_drive
+ && !h.node.is_empty()
+ && h.attempt > 1
+ && previous_node(r, h).is_some_and(|n| n != h.node)
+ {
+ push(DurableAgent {
+ phase: "failed_over",
+ failover: true,
+ reason: if h.cause.is_empty() { "none" } else { &h.cause },
+ ..base()
+ });
+ }
+ match h.event.as_str() {
+ "started" if h.attempt <= 1 => push(DurableAgent {
+ phase: "started",
+ ..base()
+ }),
+ "started" => {
+ let prev = previous_failure(r, h);
+ push(DurableAgent {
+ phase: "restarted",
+ reason: prev
+ .map(|p| p.cause.as_str())
+ .filter(|c| !c.is_empty())
+ .unwrap_or("other"),
+ recovery_time_ms: prev.map_or(0, |p| (h.at_ms - p.at_ms).max(0) as u64),
+ ..base()
+ });
+ }
+ "fenced" => push(DurableAgent {
+ phase: "refused",
+ failure: "epoch_refused",
+ reason: "fenced",
+ ..base()
+ }),
+ "exhausted" => push(DurableAgent {
+ phase: "exhausted",
+ end: "failed",
+ failure: "exhausted",
+ reason: r
+ .history
+ .iter()
+ .rev()
+ .find(|x| is_drive_failure(&x.event))
+ .map(|x| x.cause.as_str())
+ .filter(|c| !c.is_empty())
+ .unwrap_or("other"),
+ ..base()
+ }),
+ "permanent" => push(DurableAgent {
+ phase: "ended",
+ end: "failed",
+ failure: permanent_failure(&r.error),
+ reason: "permanent",
+ ..base()
+ }),
+ "done" => push(DurableAgent {
+ phase: "ended",
+ end: "done",
+ ..base()
+ }),
+ "stopped" => push(DurableAgent {
+ phase: "ended",
+ end: "stopped",
+ ..base()
+ }),
+ _ => {}
+ }
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::coordinator::sessions::sstate;
+ use cua_telemetry::schema;
+
+ /// Text a person or a machine could put anywhere a session carries text.
+ const PII: &[&str] = &[
+ "alices-macbook-pro.local",
+ "/Users/alice/work/secret-repo",
+ "192.168.1.44",
+ "Fix the failing tests in acme-billing",
+ "sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+ ];
+
+ fn hostile_row() -> Row {
+ let mut r = Row {
+ name: "alice-nightly-acme".into(),
+ state: sstate::RUNNING.into(),
+ max_attempts: 5,
+ error: format!("{} {} {}", PII[0], PII[1], PII[5]),
+ ..Default::default()
+ };
+ r.spec.label = PII[2].into();
+ r.spec.on = PII[0].into();
+ r.spec.state_dir = PII[1].into();
+ r.spec.argv = vec!["node".into(), PII[4].into()];
+ r.spec.env.insert("KEY".into(), PII[5].into());
+ // drive 1 crashes on one machine, drive 2 runs on another and is
+ // fenced, then it runs out of attempts, and a later one finishes.
+ r.attempts = 1;
+ r.epoch = 1;
+ r.node = PII[0].into();
+ r.note("started", None, PII[4]);
+ r.note_cause("crashed", Some(137), PII[1], "process_exit");
+ r.attempts = 2;
+ r.epoch = 2;
+ r.node = PII[3].into();
+ r.note("started", None, "");
+ r.note_cause("fenced", Some(75), PII[2], "fenced");
+ r.note("exhausted", None, PII[5]);
+ r.note("permanent", Some(78), PII[1]);
+ r.note("done", Some(0), PII[4]);
+ r.note("stopped", None, PII[2]);
+ r
+ }
+
+ /// The payload of a session's events holds only vocabulary words, small
+ /// numbers and the session's hash: no free text, whatever the session
+ /// carries (its name, machine, label, error, driver command and
+ /// environment).
+ #[test]
+ fn events_carry_no_free_text() {
+ let r = hostile_row();
+ let hash = "0123456789abcdef";
+ let evs = session_events(&r, 3, hash);
+ assert!(evs.len() >= 7, "every phase is reported: {}", evs.len());
+ for e in &evs {
+ let spec = schema::EVENTS
+ .iter()
+ .find(|s| s.name == e.name)
+ .expect("declared event");
+ let json = serde_json::to_string(&e.props).unwrap();
+ for frag in [
+ "alice", "macbook", "secret", "acme", "example", "192.168", "sk-ant", "Users",
+ "failing",
+ ] {
+ assert!(!json.contains(frag), "{frag} leaked into {json}");
+ }
+ // Every property is declared and every value is in its
+ // vocabulary: the same check the client runs before queueing.
+ for (k, v) in &e.props {
+ let p = spec.props.iter().find(|p| p.name == k).unwrap_or_else(|| {
+ panic!("undeclared property {k}");
+ });
+ match p.kind {
+ schema::Kind::Enum(vocab) => {
+ assert!(vocab.contains(&v.as_str().unwrap()), "{k}={v}");
+ }
+ schema::Kind::Bool => assert!(v.is_boolean(), "{k}"),
+ schema::Kind::Count { max } => assert!(v.as_u64().unwrap() <= max, "{k}"),
+ schema::Kind::SaltedHash => assert_eq!(v, hash),
+ other => panic!("free-form property kind {other:?} on {k}"),
+ }
+ }
+ }
+ // The pinned placement is a word, never which machine.
+ assert!(evs.iter().all(|e| e.props["placement"] == "host"));
+ }
+
+ #[test]
+ fn a_restart_names_why_and_how_long_recovery_took() {
+ let mut r = Row {
+ name: "n".into(),
+ max_attempts: 5,
+ ..Default::default()
+ };
+ r.spec.label = "claude-code".into();
+ r.attempts = 1;
+ r.epoch = 1;
+ r.note("started", None, "");
+ r.node = "m1".into();
+ r.note_cause("stalled", None, "wedged", "stall");
+ r.history.last_mut().unwrap().at_ms = 1_000;
+ r.attempts = 2;
+ r.epoch = 2;
+ r.node = String::new();
+ r.note("started", None, "");
+ r.history.last_mut().unwrap().at_ms = 4_500;
+ r.fresh = r.history[2..].to_vec();
+ let evs = session_events(&r, 2, "0123456789abcdef");
+ assert_eq!(evs.len(), 1);
+ let p = &evs[0].props;
+ assert_eq!(p["phase"], "restarted");
+ assert_eq!(p["reason"], "stall");
+ assert_eq!(p["recovery_time_ms"], 3_500);
+ assert_eq!(p["attempt"], 2);
+ assert_eq!(p["harness"], "claude-code");
+ assert_eq!(p["placement"], "auto");
+ }
+
+ #[test]
+ fn a_drive_on_another_machine_is_a_failover() {
+ let mut r = Row {
+ name: "n".into(),
+ max_attempts: 5,
+ ..Default::default()
+ };
+ r.attempts = 1;
+ r.epoch = 1;
+ r.node = "m1".into();
+ r.note_cause("crashed", None, "x", "host_down");
+ r.attempts = 2;
+ r.epoch = 2;
+ r.node = "m2".into();
+ r.fresh.clear();
+ r.note("done", Some(0), "");
+ let evs = session_events(&r, 2, "0123456789abcdef");
+ let phases: Vec<_> = evs.iter().map(|e| e.props["phase"].clone()).collect();
+ assert_eq!(phases, ["failed_over", "ended"]);
+ assert_eq!(evs[0].props["failover"], true);
+ assert_eq!(evs[0].props["reason"], "none");
+ }
+}
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/telemetry.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/telemetry.rs
new file mode 100644
index 0000000000..a9e7a04c19
--- /dev/null
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/telemetry.rs
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Usage telemetry of durable sessions and durable Spaces: the one place the
+//! coordinator talks to `cua-telemetry`.
+//!
+//! What leaves the machine is the event `cua_durable_agent` and nothing else:
+//! fixed words, small counts and a per-install salted hash of the session so
+//! the events of one session can be put together. Never a session or Space
+//! name, a machine name, an address, a prompt, a path or an error message;
+//! `cua_telemetry::events::durable_agent` takes no free text, and
+//! `cua_telemetry::schema::validate` drops a payload that has any. It follows
+//! the install's telemetry setting like every other event (off with
+//! `DO_NOT_TRACK=1`, `CUA_TELEMETRY=0` or `cua telemetry off`).
+
+use cua_telemetry::events::{self, DurableAgent};
+
+/// Captures one `cua_durable_agent` event for the session or Space `name`
+/// (hashed here). Does nothing when telemetry is off.
+pub(super) fn emit(name: &str, build: impl FnOnce(&str) -> DurableAgent<'_>) {
+ let t = cua_telemetry::global();
+ let Some(hash) = t.salted_hash("durable_session", name) else {
+ return;
+ };
+ let d = build(&hash);
+ if let Some(e) = events::durable_agent(&d) {
+ t.capture(e);
+ }
+}
+
+/// A durable Space was placed again on a machine (the one it was on lost
+/// it); its disk starts fresh.
+pub(super) fn space_recreated(name: &str, failover: bool, machines: u32) {
+ emit(name, |session| DurableAgent {
+ phase: "space_recreated",
+ harness: "other",
+ on: "auto",
+ reason: "none",
+ end: "none",
+ failure: "none",
+ failover,
+ machines,
+ session,
+ ..Default::default()
+ });
+}
+
+/// A lease was refused: it is held by another holder, or the caller's token
+/// is no longer the current one.
+pub(super) fn lease_refused(name: &str, machines: u32) {
+ emit(name, |session| DurableAgent {
+ phase: "refused",
+ harness: "other",
+ on: "auto",
+ reason: "none",
+ end: "none",
+ failure: "lease_refused",
+ machines,
+ session,
+ ..Default::default()
+ });
+}
diff --git a/libs/cua/crates/cua-telemetry/src/events.rs b/libs/cua/crates/cua-telemetry/src/events.rs
index 9da5efe8a4..296b694f76 100644
--- a/libs/cua/crates/cua-telemetry/src/events.rs
+++ b/libs/cua/crates/cua-telemetry/src/events.rs
@@ -760,6 +760,72 @@ pub fn agent_run_completed(
.s("duration_bucket", duration_bucket(elapsed))
}
+/// What `cua_durable_agent` is built from. Every text field is a fixed word
+/// chosen from its vocabulary (anything else becomes the fallback), every
+/// number is bounded; no name, path, address or message is ever taken.
+#[derive(Debug, Clone, Default)]
+pub struct DurableAgent<'a> {
+ /// One of [`schema::DURABLE_PHASES`].
+ pub phase: &'a str,
+ /// A cua-agents harness id.
+ pub harness: &'a str,
+ /// The session's placement word (`auto`, anything else is `host`).
+ pub on: &'a str,
+ pub attempt: u32,
+ pub max_attempts: u32,
+ /// One of [`schema::DURABLE_REASONS`].
+ pub reason: &'a str,
+ /// One of [`schema::DURABLE_ENDS`].
+ pub end: &'a str,
+ /// One of [`schema::DURABLE_FAILURES`].
+ pub failure: &'a str,
+ pub failover: bool,
+ pub machines: u32,
+ pub recovery_time_ms: u64,
+ pub interrupted_calls: u32,
+ /// The salted hash of the session (see [`crate::Telemetry::salted_hash`]).
+ pub session: &'a str,
+}
+
+/// `cua_durable_agent`; None for a phase outside the vocabulary or a
+/// session id that is not a salted hash.
+pub fn durable_agent(d: &DurableAgent<'_>) -> Option<Event> {
+ let phase = schema::DURABLE_PHASES
+ .iter()
+ .copied()
+ .find(|p| *p == d.phase)?;
+ schema::is_salted_hash(d.session).then_some(())?;
+ Some(
+ Event::new(event::DURABLE_AGENT)
+ .s("phase", phase)
+ .s("harness", harness(d.harness))
+ .s(
+ "placement",
+ if matches!(d.on.trim(), "" | "auto") {
+ "auto"
+ } else {
+ "host"
+ },
+ )
+ .n("attempt", u64::from(d.attempt.min(100)))
+ .n("max_attempts", u64::from(d.max_attempts.min(100)))
+ .s("reason", pick(schema::DURABLE_REASONS, d.reason, "other"))
+ .s("end", pick(schema::DURABLE_ENDS, d.end, "none"))
+ .s(
+ "failure",
+ pick(schema::DURABLE_FAILURES, d.failure, "other"),
+ )
+ .b("failover", d.failover)
+ .n("machines", u64::from(d.machines.min(1000)))
+ .n("recovery_time_ms", d.recovery_time_ms.min(86_400_000))
+ .n(
+ "interrupted_calls",
+ u64::from(d.interrupted_calls.min(10_000)),
+ )
+ .owned("session", d.session.to_string()),
+ )
+}
+
/// A fixed cause word for a failed agent run, chosen from its recorded
/// error text: `no_api_key`, `provider_error`, `timeout`, `user_cancelled`,
/// else `AgentFailed`. The text is only matched, never kept or sent.
diff --git a/libs/cua/crates/cua-telemetry/src/schema.rs b/libs/cua/crates/cua-telemetry/src/schema.rs
index 915ccefbd5..e2cbdce78f 100644
--- a/libs/cua/crates/cua-telemetry/src/schema.rs
+++ b/libs/cua/crates/cua-telemetry/src/schema.rs
@@ -716,6 +716,43 @@ pub const JOIN_RUNTIMES: &[&str] = &[
/// What an agent's join failed on: a sign-in, an install, a timeout, or
/// anything else. `none` for a step that worked.
pub const JOIN_FAILURES: &[&str] = &["none", "auth", "install", "timeout", "other"];
+/// What happened to a durable agent session (`cua_durable_agent`).
+pub const DURABLE_PHASES: &[&str] = &[
+ "started",
+ "restarted",
+ "failed_over",
+ "exhausted",
+ "ended",
+ "space_recreated",
+ "refused",
+];
+/// Why a durable session's driver was started again, or could not be.
+pub const DURABLE_REASONS: &[&str] = &[
+ "none",
+ "process_exit",
+ "stall",
+ "driver_lost",
+ "host_down",
+ "fenced",
+ "permanent",
+ "other",
+];
+/// How a durable session ended.
+pub const DURABLE_ENDS: &[&str] = &["none", "done", "failed", "stopped"];
+/// Where a durable session was asked to run: any machine or one named one
+/// (never which).
+pub const DURABLE_PLACEMENTS: &[&str] = &["auto", "host"];
+/// What a durable session gave up on, as a fixed word (never the message).
+pub const DURABLE_FAILURES: &[&str] = &[
+ "none",
+ "exhausted",
+ "permanent",
+ "space_gone",
+ "setup_failed",
+ "lease_refused",
+ "epoch_refused",
+ "other",
+];
/// The tools an agent's start installs (`cua-agents` installables), `other`
/// for any other, `none` for a phase that is not an install.
pub const JOIN_TOOLS: &[&str] = &[
@@ -835,6 +872,8 @@ pub const AGENT_PRODUCTS: &[&str] = &[
"sdk_kotlin",
"spaces_app",
];
+/// The products whose process hosts a session coordinator (the daemon).
+pub const DURABLE_PRODUCTS: &[&str] = &["daemon", "cli"];
/// cua-bench dataset names (`cua_bench/registry.json`) or `custom`.
pub const TASKSETS: &[&str] = &[
"cua-bench-basic",
@@ -1003,6 +1042,11 @@ const PLAN: Prop = p(
// Events
// ---------------------------------------------------------------------------
+/// A per-install salted hash value (16 lowercase hex).
+pub fn is_salted_hash(s: &str) -> bool {
+ is_hex(s, 16)
+}
+
pub mod event {
pub const FIRST_RUN: &str = "cua_first_run";
pub const CLI_COMMAND: &str = "cua_cli_command";
@@ -1018,6 +1062,7 @@ pub mod event {
pub const AGENT_RUN_STARTED: &str = "cua_agent_run_started";
pub const AGENT_RUN_COMPLETED: &str = "cua_agent_run_completed";
pub const AGENT_JOIN_PHASE: &str = "cua_agent_join_phase";
+ pub const DURABLE_AGENT: &str = "cua_durable_agent";
pub const REPO_CACHE: &str = "cua_repo_cache";
pub const BENCH_RUN_COMPLETED: &str = "cua_bench_run_completed";
pub const DAEMON_STARTED: &str = "cua_daemon_started";
@@ -1490,6 +1535,77 @@ pub const EVENTS: &[EventSpec] = &[
],
purpose: "How long an agent takes to join a Space and answer, step by step, per agent and runtime, cold and warm; where joins fail.",
},
+ EventSpec {
+ name: event::DURABLE_AGENT,
+ version: 1,
+ tier: Tier::Usage,
+ sample_rate: 1.0,
+ products: DURABLE_PRODUCTS,
+ props: &[
+ p(
+ "phase",
+ Kind::Enum(DURABLE_PHASES),
+ "What happened: started, restarted, failed_over (a drive ran on another machine than the one before), exhausted (out of attempts), ended, space_recreated (a durable Space was placed again on another machine) or refused (a lease or epoch was refused).",
+ ),
+ AGENT_HARNESS,
+ p(
+ "placement",
+ Kind::Enum(DURABLE_PLACEMENTS),
+ "auto (any machine of the account) or host (pinned to one machine). Never which.",
+ ),
+ p(
+ "attempt",
+ Kind::Count { max: 100 },
+ "Which drive of the session this is (1 is the first); 0 when not about a drive.",
+ ),
+ p(
+ "max_attempts",
+ Kind::Count { max: 100 },
+ "Drives allowed in total.",
+ ),
+ p(
+ "reason",
+ Kind::Enum(DURABLE_REASONS),
+ "Why the driver was started again: process_exit, stall, driver_lost, host_down, fenced, permanent or other; none otherwise.",
+ ),
+ p(
+ "end",
+ Kind::Enum(DURABLE_ENDS),
+ "ended only: done, failed or stopped; none otherwise.",
+ ),
+ p(
+ "failure",
+ Kind::Enum(DURABLE_FAILURES),
+ "A fixed word for what it gave up on or was refused; none otherwise. Never the message.",
+ ),
+ p(
+ "failover",
+ Kind::Bool,
+ "This drive ran on another machine than the one before.",
+ ),
+ p(
+ "machines",
+ Kind::Count { max: 1000 },
+ "How many machines of the account were available to place it on; 0 when unknown.",
+ ),
+ p(
+ "recovery_time_ms",
+ Kind::Count { max: 86_400_000 },
+ "restarted, failed_over: milliseconds from the end of the driver that failed to the start of the next one (includes the wait before it); 0 otherwise.",
+ ),
+ p(
+ "interrupted_calls",
+ Kind::Count { max: 10_000 },
+ "How many tool calls may have run halfway when the driver stopped; 0 when not known.",
+ ),
+ p(
+ "session",
+ Kind::SaltedHash,
+ "A per-install salted hash of the session, so the events of one session can be put together. Not its name.",
+ ),
+ ],
+ purpose: "How durable agents behave in the field: starts, restarts and why, failover between machines, how long recovery takes, how often attempts run out, and how sessions end.",
+ },
EventSpec {
name: event::REPO_CACHE,
version: 1,
diff --git a/libs/cua/crates/cua-telemetry/tests/privacy.rs b/libs/cua/crates/cua-telemetry/tests/privacy.rs
index 4891a73131..b7bd6dcde0 100644
--- a/libs/cua/crates/cua-telemetry/tests/privacy.rs
+++ b/libs/cua/crates/cua-telemetry/tests/privacy.rs
@@ -198,6 +198,16 @@ fn all_builders(s: &str) -> Vec<Event> {
events::first_run_step("welcome", s, "none"),
events::space_wizard(s),
events::persistent_agent(s, Some(s), Outcome::Ok, Some(s)),
+ events::durable_agent(&DurableAgent {
+ phase: s,
+ session: "0123456789abcdef",
+ ..Default::default()
+ }),
+ events::durable_agent(&DurableAgent {
+ phase: "started",
+ session: s,
+ ..Default::default()
+ }),
events::share(s, s, Outcome::Ok, "none", "none"),
events::app_update(s, s, s),
events::device_enroll(s, Outcome::Ok),
@@ -227,6 +237,26 @@ fn all_builders(s: &str) -> Vec<Event> {
}
v.push(events::space_wizard("cancelled").unwrap());
v.push(events::persistent_agent("create", Some(s), Outcome::Error, Some(s)).unwrap());
+ for phase in schema::DURABLE_PHASES {
+ v.push(
+ events::durable_agent(&DurableAgent {
+ phase,
+ harness: s,
+ on: s,
+ attempt: u32::MAX,
+ max_attempts: u32::MAX,
+ reason: s,
+ end: s,
+ failure: s,
+ failover: true,
+ machines: u32::MAX,
+ recovery_time_ms: u64::MAX,
+ interrupted_calls: u32::MAX,
+ session: "0123456789abcdef",
+ })
+ .unwrap(),
+ );
+ }
v.push(events::share("share", s, Outcome::Ok, s, s).unwrap());
v.push(events::share("share", "viewer", Outcome::Error, s, s).unwrap());
v.push(events::app_update("installed", s, s).unwrap());