Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

telemetry: actionable error kinds for failed Space creates (#5405)

Francesco Bonacci11a17974aeda
diff --git a/apps/cua-spaces/core-wasm/Cargo.lock b/apps/cua-spaces/core-wasm/Cargo.lock
index 78a43b64bf..78593c3143 100644
--- a/apps/cua-spaces/core-wasm/Cargo.lock
+++ b/apps/cua-spaces/core-wasm/Cargo.lock
@@ -14,6 +14,10 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
+[[package]]
+name = "cua-create-failure"
+version = "0.1.0"
+
[[package]]
name = "cua-home"
version = "0.1.0"
@@ -26,6 +30,7 @@ dependencies = [
name = "cua-spaces-app-core"
version = "0.1.0"
dependencies = [
+ "cua-create-failure",
"cua-home",
"serde",
"serde_json",
diff --git a/apps/cua-spaces/src-tauri/Cargo.lock b/apps/cua-spaces/src-tauri/Cargo.lock
index 203ae383d5..4789e62415 100644
--- a/apps/cua-spaces/src-tauri/Cargo.lock
+++ b/apps/cua-spaces/src-tauri/Cargo.lock
@@ -1335,6 +1335,10 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "cua-create-failure"
+version = "0.1.0"
+
[[package]]
name = "cua-daemon"
version = "0.1.0"
@@ -1652,6 +1656,7 @@ dependencies = [
"chrono",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-fleet",
"cua-home",
"cua-host",
@@ -1738,6 +1743,7 @@ name = "cua-spaces-app-core"
version = "0.1.0"
dependencies = [
"async-trait",
+ "cua-create-failure",
"cua-home",
"cua-keyvault",
"cua-telemetry",
diff --git a/docs/content/docs/cua-sdk/concepts/meta.json b/docs/content/docs/cua-sdk/concepts/meta.json
index 7c50cc85a9..a19fc7f501 100644
--- a/docs/content/docs/cua-sdk/concepts/meta.json
+++ b/docs/content/docs/cua-sdk/concepts/meta.json
@@ -3,6 +3,7 @@
"pages": [
"how-sandboxes-work",
"telemetry",
- "telemetry-events"
+ "telemetry-events",
+ "telemetry-error-kinds"
]
}
diff --git a/docs/content/docs/cua-sdk/concepts/telemetry-error-kinds.mdx b/docs/content/docs/cua-sdk/concepts/telemetry-error-kinds.mdx
new file mode 100644
index 0000000000..a66f4a4d2f
--- /dev/null
+++ b/docs/content/docs/cua-sdk/concepts/telemetry-error-kinds.mdx
@@ -0,0 +1,30 @@
+---
+title: Create failure kinds
+description: The fixed error_kind words a failed sandbox or Space create sends, and what each means.
+---
+
+When a create fails, `cua_space_create`, `cua_sandbox_created` and
+`cua_host_space_provided` set `error_kind` to a fixed word for the cause. The
+program reads the error on your machine and sends only the word, never the
+error text.
+
+| `error_kind` | Cause |
+|---|---|
+| `image_pull_not_found` | The image is not in the registry. |
+| `image_pull_auth` | The registry refused the download. |
+| `image_pull_network` | The network failed or timed out during the download. |
+| `image_pull_rate_limited` | The registry rate-limited the download. |
+| `image_pull` | The download failed for another reason, or stopped partway. |
+| `insufficient_disk` | Not enough disk space. |
+| `runtime_missing`, `runtime_not_running`, `runtime_permission` | The local runtime (Docker, Lume, QEMU) is not installed, not running, or not allowed. |
+| `virtualization_unavailable` | The machine cannot run VMs (no Virtualization.framework, KVM or hardware virtualization). |
+| `vm_limit` | The Mac already runs the two macOS VMs Apple allows. |
+| `vm_boot_timeout` | The VM did not finish booting in time. |
+| `services_unreachable` | The Space booted but its services never answered. |
+| `local_network` | macOS Local Network privacy blocked the connection to the new VM. |
+| `quota_exceeded` | The account's quota, plan limit or credits ran out. |
+| `cancelled` | You cancelled the create. |
+
+Otherwise it is the error's type (for example `unauthenticated`, `timeout`,
+`runtime`), and `other` only when nothing names the cause. A failed create
+never reports `none`.
diff --git a/docs/content/docs/cua-sdk/concepts/telemetry-events.mdx b/docs/content/docs/cua-sdk/concepts/telemetry-events.mdx
index 85ea6f9d7e..4e370a821c 100644
--- a/docs/content/docs/cua-sdk/concepts/telemetry-events.mdx
+++ b/docs/content/docs/cua-sdk/concepts/telemetry-events.mdx
@@ -38,7 +38,7 @@ Every event carries: `telemetry_schema_version`, `event_version`, `product`
| `cua_onboarding_page` | `page` (`welcome`, `command_line`, `signin`, `agents`, `presentation`, `volume`, `this_machine`, `done`), `action` (`shown`, `completed`, `skipped`, `back`), `choice` (a fixed word such as `installed`, `signed_in`, `menu_bar_only`, `s3`, `host`, or `none`) | Which first-run page people leave on, and what they pick. |
| `cua_space_wizard` | `action` (`opened`, `cancelled`, `submitted`) | How often a New Space is started and abandoned. |
| `cua_space_create_started` | `location`, `guest_os`, `kind`, `gpu` (GPU acceleration turned on) | Which Spaces are started, and how often with GPU acceleration. |
-| `cua_space_create` | `location`, `guest_os`, `kind`, `outcome` (`ok`, `error`, or `cancelled` when Cancel was pressed while it was being created), `error_kind`, `failed_phase` (the phase it failed or was cancelled in, or `none`), `stalled`, `time_bucket` (to ready, the failure, or cancel finishing), `gpu` | Space create success, failure and cancel rates, time to ready, and which phase fails. |
+| `cua_space_create` | `location`, `guest_os`, `kind`, `outcome` (`ok`, `error`, or `cancelled` when Cancel was pressed while it was being created), `error_kind` ([kinds](/cua-sdk/concepts/telemetry-error-kinds)), `failed_phase` (the phase it failed or was cancelled in, or `none`), `stalled`, `time_bucket` (to ready, the failure, or cancel finishing), `gpu` | Space create success, failure and cancel rates, time to ready, and which phase fails. |
| `cua_volume_setup` | `surface` (`onboarding` or `settings`), `storage` (`this_mac`, `s3`, `later`), `add_to_finder`, `mount_method` (`fskit`, `nfs`, `fuse`, `none`, `unknown`), `outcome` | Cua Volume adoption. |
| `cua_persistent_agent` | `action` (`create`, `pause`, `resume`, `routine_add`, ...), `harness`, `outcome`, `error_kind` | Persistent agents and routines adoption. |
| `cua_host_setup` | `mode` (`relay` or `direct`), `profile` (`desktop`, `spare`, `both`), `outcome`, `error_kind`, `signed_in`, `token_state` (`provided`, `expired`, `missing`, `not_needed`; never the token), `failed_stage` (`none`, `preflight`, `token`, `register`, `download`, `service`), `http_status` (the status a server answered a failure with, `0` when none) | Machines set up for access, and where a failed setup stopped. |
diff --git a/examples/streaming/rust/Cargo.lock b/examples/streaming/rust/Cargo.lock
index 1c1b729608..639736c6fe 100644
--- a/examples/streaming/rust/Cargo.lock
+++ b/examples/streaming/rust/Cargo.lock
@@ -1307,6 +1307,10 @@ dependencies = [
"url",
]
+[[package]]
+name = "cua-create-failure"
+version = "0.1.0"
+
[[package]]
name = "cua-daemon"
version = "0.1.0"
@@ -1550,6 +1554,7 @@ dependencies = [
"cua-agent-setup",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-daemon",
"cua-fleet",
"cua-host",
@@ -1593,6 +1598,7 @@ dependencies = [
"chrono",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-fleet",
"cua-home",
"cua-host",
diff --git a/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock b/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock
index ea1119eb9a..5ac50c4621 100644
--- a/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock
+++ b/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock
@@ -2066,6 +2066,10 @@ dependencies = [
"url",
]
+[[package]]
+name = "cua-create-failure"
+version = "0.1.0"
+
[[package]]
name = "cua-cursor-motion"
version = "0.36.0"
@@ -2494,6 +2498,7 @@ dependencies = [
"cua-agent-setup",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-daemon",
"cua-fleet",
"cua-host",
@@ -2537,6 +2542,7 @@ dependencies = [
"chrono",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-fleet",
"cua-home",
"cua-host",
diff --git a/libs/cua/Cargo.lock b/libs/cua/Cargo.lock
index af57c34e1a..c151419026 100644
--- a/libs/cua/Cargo.lock
+++ b/libs/cua/Cargo.lock
@@ -2445,6 +2445,13 @@ dependencies = [
"url",
]
+[[package]]
+name = "cua-create-failure"
+version = "0.1.0"
+dependencies = [
+ "cua-telemetry",
+]
+
[[package]]
name = "cua-daemon"
version = "0.1.0"
@@ -2838,6 +2845,7 @@ dependencies = [
"cua-agent-setup",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-daemon",
"cua-fleet",
"cua-home",
@@ -2886,6 +2894,7 @@ dependencies = [
"chrono",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-fleet",
"cua-home",
"cua-host",
@@ -2928,6 +2937,7 @@ name = "cua-spaces-app-core"
version = "0.1.0"
dependencies = [
"async-trait",
+ "cua-create-failure",
"cua-home",
"cua-keyvault",
"cua-telemetry",
diff --git a/libs/cua/Cargo.toml b/libs/cua/Cargo.toml
index c470eb0bdf..595b4a94e7 100644
--- a/libs/cua/Cargo.toml
+++ b/libs/cua/Cargo.toml
@@ -60,6 +60,8 @@ members = [
"crates/cua-logging",
# Anonymous product telemetry.
"crates/cua-telemetry",
+ # A failed create's cause as a fixed telemetry word (no dependencies).
+ "crates/cua-create-failure",
"crates/cua-viewer",
# The HTML5 viewer (embedded in cua-spacesd at /viewer; standalone server).
"crates/cua-spacesd-html5",
@@ -176,6 +178,7 @@ cua-spaces-app-core = { path = "crates/cua-spaces-app-core", version = "0.1.0" }
cua-spaces-ext = { path = "crates/cua-spaces-ext", version = "0.1.0" }
cua-byoc = { path = "crates/cua-byoc", version = "0.1.0", default-features = false }
cua-telemetry = { path = "crates/cua-telemetry", version = "0.1.0" }
+cua-create-failure = { path = "crates/cua-create-failure", version = "0.1.0" }
cua-media-protocol = { path = "crates/cua-media-protocol" }
cua-media-transport = { path = "crates/cua-media-transport" }
cua-media-codec = { path = "crates/cua-media-codec", default-features = false }
diff --git a/libs/cua/crates/cua-create-failure/Cargo.toml b/libs/cua/crates/cua-create-failure/Cargo.toml
new file mode 100644
index 0000000000..4fc9b3ce3c
--- /dev/null
+++ b/libs/cua/crates/cua-create-failure/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "cua-create-failure"
+description = "Why a sandbox or Space create failed, as a fixed telemetry word (`image_pull_not_found`, `vm_boot_timeout`, ...) read from the error text inside the process; the text never leaves. No dependencies, so the Spaces app core's wasm build uses it too."
+version.workspace = true
+edition.workspace = true
+rust-version = "1.95"
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+publish = false
+
+[dev-dependencies]
+cua-telemetry.workspace = true
diff --git a/libs/cua/crates/cua-create-failure/src/lib.rs b/libs/cua/crates/cua-create-failure/src/lib.rs
new file mode 100644
index 0000000000..e8fdfd28a8
--- /dev/null
+++ b/libs/cua/crates/cua-create-failure/src/lib.rs
@@ -0,0 +1,573 @@
+//! Why a create failed, as a fixed word: the error text a failed create
+//! shows is read here, inside the process, and only a word from
+//! `cua_telemetry::schema::ERROR_KINDS` (or the caller's own enum case)
+//! comes out. The text itself never leaves.
+//!
+//! One mapping serves `cua_space_create` (the Spaces app core, also in its
+//! wasm build), `cua_sandbox_created` (the SDK) and
+//! `cua_host_space_provided` (a host's daemon), so the three events name
+//! the same cause the same way. Callers pass the result through
+//! `cua_telemetry::events::error_kind`, which keeps only schema words.
+
+/// Enum cases that say little on their own (compared lowercase, without
+/// underscores): the error text decides instead.
+const GENERIC: &[&str] = &[
+ "", "none", "other", "runtime", "env", "sandbox", "internal", "io", "fleet", "relay",
+];
+
+/// The kind for a failed create: a cause word when the text or the enum
+/// case names one, else the caller's enum case (`Runtime`, `Env`), else
+/// `other`. Never empty and never `none`.
+///
+/// - `message`: the error's text (read here, never returned).
+/// - `phase`: the create phase it failed in (`preparing`, `pulling`,
+/// `booting`, `waiting_for_services`, ...), or empty when not known.
+/// - `variant`: the error enum's case name (`InsufficientDisk`, `Runtime`)
+/// or its snake tag (`insufficient_disk`), or empty.
+/// - `local`: the create ran on a machine's own runtime (Docker, Lume,
+/// QEMU), so "X is not available" means that runtime is missing. False
+/// for a cloud create, where it means the account's provider.
+pub fn create_failure_kind<'v>(
+ message: &str,
+ phase: &str,
+ variant: &'v str,
+ local: bool,
+) -> &'v str {
+ let phase = phase.trim();
+ let variant = variant.trim();
+ let flat = flatten(variant);
+ match flat.as_str() {
+ "cloudcreditexhausted" | "creditexhausted" | "limitexceeded" => return "quota_exceeded",
+ // A specific case from the caller stays; a timeout is refined by
+ // the text and the phase.
+ f if f != "timeout" && !GENERIC.contains(&f) => return variant,
+ _ => {}
+ }
+ if let Some(kind) = from_text(message, phase, local) {
+ return kind;
+ }
+ match flat.as_str() {
+ "timeout" => timeout_in(phase),
+ "" | "none" => "other",
+ _ => variant,
+ }
+}
+
+fn flatten(variant: &str) -> String {
+ variant
+ .trim()
+ .chars()
+ .filter(|c| *c != '_')
+ .map(|c| c.to_ascii_lowercase())
+ .collect()
+}
+
+/// A timeout, by the phase it happened in.
+fn timeout_in(phase: &str) -> &'static str {
+ match phase {
+ "pulling" => "image_pull_network",
+ "booting" | "creating" => "vm_boot_timeout",
+ "waiting_for_services" | "connecting" => "services_unreachable",
+ _ => "timeout",
+ }
+}
+
+fn has(l: &str, any: &[&str]) -> bool {
+ any.iter().any(|n| l.contains(n))
+}
+
+/// The cause the text names, if it names one.
+fn from_text(message: &str, phase: &str, local: bool) -> Option<&'static str> {
+ let l = message.to_ascii_lowercase();
+ if l.trim().is_empty() {
+ return None;
+ }
+ // Cancel first: a cancelled create may carry any earlier text.
+ if has(
+ &l,
+ &[
+ "cancelled by user",
+ "canceled by user",
+ "create was cancelled",
+ ],
+ ) {
+ return Some("cancelled");
+ }
+ if has(
+ &l,
+ &[
+ "not enough disk space",
+ "insufficient disk",
+ "insufficient_disk",
+ "no space left on device",
+ "disk quota exceeded",
+ ],
+ ) {
+ return Some("insufficient_disk");
+ }
+ let vmnet = l.contains("192.168.64.");
+ if l.contains("local network access")
+ || (vmnet && has(&l, &["no route to host", "os error 65"]))
+ {
+ return Some("local_network");
+ }
+ if l.contains("virtualmachinelimitexceeded")
+ || (has(&l, &["virtual machine", "vms"])
+ && l.contains("limit")
+ && has(&l, &["exceed", "maximum", "reached"]))
+ {
+ return Some("vm_limit");
+ }
+ if has(
+ &l,
+ &[
+ "quota",
+ "credit exhausted",
+ "credits exhausted",
+ "out of credits",
+ "payment required",
+ "plan limit",
+ "seat limit",
+ "402 ",
+ ],
+ ) {
+ return Some("quota_exceeded");
+ }
+ if has(
+ &l,
+ &[
+ "virtualization is not available",
+ "virtualization is not supported",
+ "virtualization.framework",
+ "vz is not available",
+ "hypervisor framework",
+ "hypervisor is not available",
+ "/dev/kvm",
+ "kvm is not available",
+ "nested virtualization",
+ "vt-x",
+ "amd-v",
+ "hyper-v is not",
+ "whpx",
+ ],
+ ) {
+ return Some("virtualization_unavailable");
+ }
+ // The local runtime (Docker, Podman, Lume, QEMU).
+ if local {
+ if l.contains("permission denied") {
+ return Some("runtime_permission");
+ }
+ if l.contains(" is not running: ")
+ || has(
+ &l,
+ &[
+ "cannot connect to the docker daemon",
+ "is the docker daemon running",
+ "docker desktop is not running",
+ "lume is not running",
+ ],
+ )
+ {
+ return Some("runtime_not_running");
+ }
+ if (l.contains(" is not available: ") && !l.contains("network route"))
+ || (l.contains("qemu") && l.contains("not found on path"))
+ {
+ return Some("runtime_missing");
+ }
+ }
+ if let Some(kind) = image_pull(&l, phase) {
+ return Some(kind);
+ }
+ if l.contains("unauthenticated") {
+ return Some("unauthenticated");
+ }
+ // The Space booted but its service (cua-spacesd, the computer server)
+ // did not answer.
+ if l.contains("cua-spacesd did not answer within ")
+ || (phase == "waiting_for_services"
+ && has(
+ &l,
+ &[
+ "connection refused",
+ "connection reset",
+ "unreachable",
+ "no route to host",
+ "did not answer",
+ "health",
+ "probe",
+ ],
+ ))
+ {
+ return Some("services_unreachable");
+ }
+ let timed_out = has(&l, &["timed out", "timeout", "not ready after", "deadline"]);
+ if timed_out {
+ return Some(timeout_in(phase));
+ }
+ None
+}
+
+/// An image download failure split by why: the image is not there, the
+/// registry refused, the network failed, the registry rate-limited, or it
+/// stopped partway.
+fn image_pull(l: &str, phase: &str) -> Option<&'static str> {
+ let about_pull = phase == "pulling"
+ || has(
+ l,
+ &[
+ "container engine: pull ",
+ "registry: ",
+ "manifest unknown",
+ "pull access denied",
+ "image download",
+ "pull ghcr.io",
+ "pull docker.io",
+ "lume api 404",
+ "error pulling image",
+ "failed to pull",
+ ],
+ );
+ if !about_pull {
+ return None;
+ }
+ if has(
+ l,
+ &["toomanyrequests", "too many requests", "rate limit", " 429"],
+ ) {
+ return Some("image_pull_rate_limited");
+ }
+ if has(
+ l,
+ &[
+ "pull access denied",
+ "not authorized",
+ "unauthorized",
+ "authentication required",
+ "denied: ",
+ " 401",
+ " 403",
+ "forbidden",
+ ],
+ ) {
+ return Some("image_pull_auth");
+ }
+ if has(
+ l,
+ &[
+ "manifest unknown",
+ "name unknown",
+ "registry: not found",
+ "not found: manifest",
+ "repository does not exist",
+ "no such image",
+ "lume api 404",
+ " 404",
+ "no matching manifest",
+ "manifest in image index",
+ ],
+ ) {
+ return Some("image_pull_not_found");
+ }
+ if has(
+ l,
+ &[
+ "dns error",
+ "no such host",
+ "failed to lookup address",
+ "connection refused",
+ "connection reset",
+ "connection closed",
+ "error sending request",
+ "network is unreachable",
+ "tls handshake",
+ "certificate",
+ "i/o timeout",
+ "timed out",
+ "timeout",
+ "unexpected eof",
+ "broken pipe",
+ "temporary failure in name resolution",
+ ],
+ ) {
+ return Some("image_pull_network");
+ }
+ if l.contains("the image download stopped before it finished") {
+ return Some("image_pull");
+ }
+ // A pull that failed for a reason the text does not name.
+ (phase == "pulling"
+ || has(
+ l,
+ &["container engine: pull ", "registry: ", "failed to pull"],
+ ))
+ .then_some("image_pull")
+}
+
+/// Every cause word [`create_failure_kind`] can return from the text or
+/// the phase (the rest are the caller's enum cases).
+pub const TEXT_KINDS: &[&str] = &[
+ "cancelled",
+ "insufficient_disk",
+ "local_network",
+ "vm_limit",
+ "quota_exceeded",
+ "virtualization_unavailable",
+ "image_pull_rate_limited",
+ "image_pull_auth",
+ "image_pull_not_found",
+ "image_pull_network",
+ "image_pull",
+ "runtime_permission",
+ "runtime_not_running",
+ "runtime_missing",
+ "unauthenticated",
+ "services_unreachable",
+ "vm_boot_timeout",
+ "timeout",
+];
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn kind<'a>(message: &'a str, phase: &'a str, variant: &'a str) -> &'a str {
+ create_failure_kind(message, phase, variant, true)
+ }
+
+ #[test]
+ fn every_cause_word_is_in_the_telemetry_schema() {
+ for k in TEXT_KINDS.iter().chain(&["quota_exceeded", "other"]) {
+ assert!(
+ cua_telemetry::schema::ERROR_KINDS.contains(k),
+ "{k} missing from ERROR_KINDS"
+ );
+ }
+ }
+
+ #[test]
+ fn image_pulls_split_by_cause() {
+ let p = "pulling";
+ assert_eq!(
+ kind(
+ "container engine: pull ghcr.io/x/y:1: manifest unknown",
+ "",
+ "Runtime"
+ ),
+ "image_pull_not_found"
+ );
+ assert_eq!(
+ kind("registry: not found: ghcr.io/example/missing:1", p, ""),
+ "image_pull_not_found"
+ );
+ assert_eq!(
+ kind("container engine: pull x: pull access denied for x", "", ""),
+ "image_pull_auth"
+ );
+ assert_eq!(
+ kind("registry: Not authorized: ghcr.io/me/private", p, "Other"),
+ "image_pull_auth"
+ );
+ assert_eq!(
+ kind(
+ "container engine: pull x: toomanyrequests: slow down",
+ p,
+ ""
+ ),
+ "image_pull_rate_limited"
+ );
+ assert_eq!(
+ kind(
+ "registry: error sending request: dns error: no such host",
+ p,
+ ""
+ ),
+ "image_pull_network"
+ );
+ assert_eq!(kind("operation timed out", p, ""), "image_pull_network");
+ assert_eq!(
+ kind(
+ "The image download stopped before it finished (lume stopped). Try again; it picks up where it stopped.",
+ p,
+ "Runtime"
+ ),
+ "image_pull"
+ );
+ assert_eq!(kind("boom", p, ""), "image_pull");
+ }
+
+ #[test]
+ fn disk_network_limits_and_quota() {
+ assert_eq!(
+ kind(
+ "not enough disk space to pull x: it needs about 30.0 GB",
+ "preparing",
+ ""
+ ),
+ "insufficient_disk"
+ );
+ assert_eq!(
+ kind("write: No space left on device", "pulling", ""),
+ "insufficient_disk"
+ );
+ assert_eq!(
+ kind(
+ "Local Network access is not available: cannot reach the VM at 192.168.64.3 (No route to host)",
+ "waiting_for_services",
+ "Runtime"
+ ),
+ "local_network"
+ );
+ assert_eq!(
+ kind(
+ "connect 192.168.64.9:8000: os error 65",
+ "waiting_for_services",
+ ""
+ ),
+ "local_network"
+ );
+ assert_eq!(
+ kind("VZErrorDomain virtualMachineLimitExceeded", "booting", ""),
+ "vm_limit"
+ );
+ assert_eq!(
+ kind("sandbox quota reached for this org", "", "Fleet"),
+ "quota_exceeded"
+ );
+ assert_eq!(kind("", "", "cloud_credit_exhausted"), "quota_exceeded");
+ }
+
+ #[test]
+ fn runtime_causes() {
+ assert_eq!(
+ kind(
+ "local runtime: [local] a container engine is not available: no socket",
+ "preparing",
+ "Runtime"
+ ),
+ "runtime_missing"
+ );
+ assert_eq!(
+ kind(
+ "Docker is not running: start Docker Desktop",
+ "preparing",
+ "Runtime"
+ ),
+ "runtime_not_running"
+ );
+ assert_eq!(
+ kind(
+ "Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?",
+ "preparing",
+ ""
+ ),
+ "runtime_not_running"
+ );
+ assert_eq!(
+ kind(
+ "permission denied on unix:///var/run/docker.sock",
+ "preparing",
+ ""
+ ),
+ "runtime_permission"
+ );
+ assert_eq!(
+ kind(
+ "QEMU (qemu-system-x86_64, qemu-img) is not available: install it",
+ "pulling",
+ ""
+ ),
+ "runtime_missing"
+ );
+ assert_eq!(
+ kind(
+ "/dev/kvm is missing, so same-architecture guests run slowly",
+ "booting",
+ ""
+ ),
+ "virtualization_unavailable"
+ );
+ assert_eq!(
+ kind(
+ "Virtualization is not available on this Mac",
+ "preparing",
+ ""
+ ),
+ "virtualization_unavailable"
+ );
+ }
+
+ #[test]
+ fn a_cloud_provider_that_is_not_available_is_not_a_runtime() {
+ let k = create_failure_kind(
+ "cloud is not available: sign in",
+ "",
+ "ProviderNotConfigured",
+ false,
+ );
+ assert_eq!(k, "ProviderNotConfigured");
+ }
+
+ #[test]
+ fn timeouts_split_by_phase() {
+ assert_eq!(
+ kind(
+ "sandbox 'x' was not ready after 300s: no answer",
+ "booting",
+ ""
+ ),
+ "vm_boot_timeout"
+ );
+ assert_eq!(
+ kind(
+ "cua-spacesd did not answer within 120s",
+ "waiting_for_services",
+ ""
+ ),
+ "services_unreachable"
+ );
+ assert_eq!(
+ kind(
+ "probe http://10.0.0.2:8000/health: connection refused",
+ "waiting_for_services",
+ ""
+ ),
+ "services_unreachable"
+ );
+ assert_eq!(kind("timed out: waiting", "", "Timeout"), "timeout");
+ assert_eq!(kind("", "booting", "Timeout"), "vm_boot_timeout");
+ }
+
+ #[test]
+ fn enum_cases_and_fallbacks() {
+ assert_eq!(
+ kind("whatever", "preparing", "InsufficientDisk"),
+ "InsufficientDisk"
+ );
+ assert_eq!(kind("whatever", "", "LimitExceeded"), "quota_exceeded");
+ assert_eq!(
+ kind("the token was rejected: unauthenticated", "", ""),
+ "unauthenticated"
+ );
+ assert_eq!(
+ kind("create was cancelled by user", "pulling", ""),
+ "cancelled"
+ );
+ assert_eq!(kind("boom", "preparing", "Runtime"), "Runtime");
+ assert_eq!(kind("boom", "preparing", "Env"), "Env");
+ assert_eq!(kind("boom", "preparing", ""), "other");
+ assert_eq!(kind("", "", ""), "other");
+ // Never `none` on a failure.
+ assert_eq!(kind("boom", "", "none"), "other");
+ }
+
+ #[test]
+ fn the_message_never_comes_out() {
+ let secret =
+ "pull ghcr.io/me/private: token ghp_abc123 for [email protected] at /Users/maya";
+ let k = kind(secret, "pulling", "");
+ assert!(cua_telemetry::schema::ERROR_KINDS.contains(&k));
+ assert!(!k.contains("maya"));
+ }
+}
diff --git a/libs/cua/crates/cua-sdk/Cargo.toml b/libs/cua/crates/cua-sdk/Cargo.toml
index baa8f667ec..8d945cc264 100644
--- a/libs/cua/crates/cua-sdk/Cargo.toml
+++ b/libs/cua/crates/cua-sdk/Cargo.toml
@@ -63,6 +63,7 @@ cua-fleet.workspace = true
cua-image.workspace = true
# Anonymous usage telemetry.
cua-telemetry.workspace = true
+cua-create-failure.workspace = true
cua-sandbox-core.workspace = true
async-trait.workspace = true
cua-host = { workspace = true, optional = true }
diff --git a/libs/cua/crates/cua-sdk/src/native/telemetry.rs b/libs/cua/crates/cua-sdk/src/native/telemetry.rs
index e07178e8f1..19d48b4a73 100644
--- a/libs/cua/crates/cua-sdk/src/native/telemetry.rs
+++ b/libs/cua/crates/cua-sdk/src/native/telemetry.rs
@@ -252,18 +252,26 @@ pub(crate) fn variant<T>(r: &Result<T>) -> Option<&'static str> {
r.as_ref().err().map(CuaError::variant)
}
-/// A create's error kind: a local runtime failure split by cause
-/// (`runtime_missing`, `runtime_not_running`, `runtime_permission`,
-/// `image_pull`), else the variant. Only the bounded word leaves the
-/// process, never the message.
+/// A create's error kind: the cause the error's text names
+/// (`image_pull_not_found`, `runtime_not_running`, `vm_boot_timeout`, ...;
+/// [`cua_create_failure::create_failure_kind`]), else the variant. Only the
+/// bounded word leaves the process, never the message. A cloud create's
+/// "X is not available" is its provider, not a local runtime.
fn create_error_kind<T>(on: &str, r: &Result<T>) -> Option<&'static str> {
let e = r.as_ref().err()?;
- let local_runtime =
- on == "local" && matches!(e, CuaError::Runtime(_) | CuaError::ProviderNotConfigured(_));
- if local_runtime && let Some(kind) = cua_sandbox_core::runtime_failure_kind(&e.to_string()) {
- return Some(kind);
- }
- Some(e.variant())
+ let local = on == "local";
+ let variant = match e {
+ // A cloud account that is not set up stays provider_not_configured.
+ CuaError::ProviderNotConfigured(_) if !local => return Some(e.variant()),
+ CuaError::ProviderNotConfigured(_) => "Runtime",
+ _ => e.variant(),
+ };
+ Some(cua_create_failure::create_failure_kind(
+ &e.to_string(),
+ "",
+ variant,
+ local,
+ ))
}
fn outcome<T>(r: &Result<T>) -> Outcome {
@@ -461,6 +469,27 @@ mod tests {
),
Some("provider_not_configured")
);
+ assert_eq!(
+ kind(
+ "local",
+ CuaError::Runtime("container engine: pull ghcr.io/x/y:1: manifest unknown".into())
+ ),
+ Some("image_pull_not_found")
+ );
+ assert_eq!(
+ kind(
+ "local",
+ CuaError::Runtime("Docker is not running: start Docker Desktop".into())
+ ),
+ Some("runtime_not_running")
+ );
+ assert_eq!(
+ kind(
+ "cloud",
+ CuaError::Runtime("sandbox quota exceeded for this account".into())
+ ),
+ Some("quota_exceeded")
+ );
assert_eq!(create_error_kind::<()>("local", &Ok(())), None);
}
diff --git a/libs/cua/crates/cua-spaces-app-core/Cargo.toml b/libs/cua/crates/cua-spaces-app-core/Cargo.toml
index 1844a00a04..321b324746 100644
--- a/libs/cua/crates/cua-spaces-app-core/Cargo.toml
+++ b/libs/cua/crates/cua-spaces-app-core/Cargo.toml
@@ -24,6 +24,9 @@ telemetry = ["dep:cua-telemetry"]
[dependencies]
cua-home.workspace = true
+# A failed create's cause as a fixed telemetry word (no dependencies, so
+# also in the wasm build).
+cua-create-failure.workspace = true
serde.workspace = true
serde_json.workspace = true
cua-keyvault = { workspace = true, optional = true }
diff --git a/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs b/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs
index 00e104ffdc..45b9140219 100644
--- a/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs
+++ b/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs
@@ -849,15 +849,28 @@ pub fn failure_cause(raw: &str, phase: Option<&str>, gpu: bool) -> Option<Failur
}
}
-/// The error enum case name a failed create records in telemetry: what
-/// the shell sent, unless that is none or a generic case (`Runtime`,
-/// `Other`, as most are) and the text names a known cause
-/// ([`failure_cause`]).
-pub fn failure_variant<'a>(raw: &str, phase: Option<&str>, gpu: bool, shell: &'a str) -> &'a str {
- if !matches!(shell.trim(), "" | "Runtime" | "Other") {
- return shell;
- }
- failure_cause(raw, phase, gpu).map_or(shell, |c| c.variant())
+/// The `error_kind` word a failed create records in telemetry: the cause
+/// the SDK's text names (`image_pull_not_found`, `vm_boot_timeout`,
+/// `services_unreachable`, ...; [`cua_create_failure::create_failure_kind`]),
+/// else what the shell sent (`Runtime`, as most are), else `other`. Only
+/// the word leaves the process, never the text. `provider` is where it ran:
+/// a cloud create's "X is not available" is not this machine's runtime.
+pub fn failure_variant<'a>(
+ raw: &str,
+ phase: Option<&str>,
+ gpu: bool,
+ shell: &'a str,
+ provider: Option<SpaceProvider>,
+) -> &'a str {
+ if failure_cause(raw, phase, gpu) == Some(FailureCause::Gpu) {
+ return FailureCause::Gpu.variant();
+ }
+ cua_create_failure::create_failure_kind(
+ &crate::errors::plain_error(raw),
+ phase.unwrap_or(""),
+ shell,
+ provider != Some(SpaceProvider::Cloud),
+ )
}
/// [`failure_text`] for a create on one of your machines named `host_name`:
@@ -2541,8 +2554,8 @@ mod tests {
Try again; it picks up where it stopped."
);
assert_eq!(
- failure_variant(stopped, Some("pulling"), false, "Runtime"),
- "ImagePull"
+ failure_variant(stopped, Some("pulling"), false, "Runtime", None),
+ "image_pull"
);
// QEMU missing on a Windows PC: 0.9.2's after the pull, and the
// refusal before it. Not `other` in telemetry, and the SDK's words.
@@ -2558,8 +2571,8 @@ mod tests {
"{raw}"
);
assert_eq!(
- failure_variant(raw, Some("pulling"), false, "Other"),
- "RuntimeMissing"
+ failure_variant(raw, Some("pulling"), false, "Other", None),
+ "runtime_missing"
);
}
// Local Network privacy on the Mac that runs the VM.
diff --git a/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs b/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs
index b0de43a52f..287ffa41c8 100644
--- a/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs
+++ b/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs
@@ -547,13 +547,15 @@ pub fn creates(before: &CreatesState, action: &CreateAction, now_ms: i64) -> Vec
}
Some(p) if p.space_id.is_none() && (p.error.is_none() || stalled(p)) => {
// The shell's enum case is often `Runtime` or none: the
- // cause the app reads from the text (Local Network, Apple's
- // VM limit, disk) says more.
+ // cause the app reads from the text (an image that is not
+ // in the registry, Local Network, a VM that did not boot in
+ // time, disk) says more. Only the word is sent.
let variant = crate::spaces::creating::failure_variant(
error,
Some(&p.phase),
p.gpu,
error_variant,
+ Some(p.provider),
);
vec![create_event(p, "error", stalled(p), variant, now_ms)]
}
@@ -1301,14 +1303,19 @@ mod tests {
(
"runtime: Local Network access is not available: Cua Spaces cannot reach the VM at 192.168.64.5:3211 (No route to host (os error 65))",
"Runtime",
- "LocalNetwork",
+ "local_network",
),
(
"lume: The number of virtual machines exceeds the limit",
"",
- "VmLimit",
+ "vm_limit",
),
("something else at /Users/alice", "Runtime", "Runtime"),
+ (
+ "container engine: pull ghcr.io/trycua/linux:24.04: manifest unknown",
+ "",
+ "image_pull_not_found",
+ ),
// A specific case from the shell stays.
(
"cua-spacesd timed out",
diff --git a/libs/cua/crates/cua-spaces/Cargo.toml b/libs/cua/crates/cua-spaces/Cargo.toml
index dcaac664ae..8a6e1d3ac9 100644
--- a/libs/cua/crates/cua-spaces/Cargo.toml
+++ b/libs/cua/crates/cua-spaces/Cargo.toml
@@ -51,6 +51,7 @@ required-features = ["mcp-client"]
[dependencies]
# Host Spaces use counts (fixed words only).
cua-telemetry.workspace = true
+cua-create-failure.workspace = true
cua-spaces-contract = { path = "../cua-spaces-contract" }
cua-proto = { workspace = true, features = ["client", "serde"] }
cua-spacesd-client.workspace = true
diff --git a/libs/cua/crates/cua-spaces/src/host_spaces.rs b/libs/cua/crates/cua-spaces/src/host_spaces.rs
index 1028791cfe..57bb947929 100644
--- a/libs/cua/crates/cua-spaces/src/host_spaces.rs
+++ b/libs/cua/crates/cua-spaces/src/host_spaces.rs
@@ -829,7 +829,13 @@ impl HostSpacesServer {
} else {
cua_telemetry::Outcome::Error
},
- r.is_err().then_some("other"),
+ // The cause the error's text names (`image_pull_auth`,
+ // `insufficient_disk`, ...), else the error's tag; only the
+ // word is sent, never the text.
+ r.as_ref().err().map(|e| {
+ let message = e.to_string();
+ cua_create_failure::create_failure_kind(&message, "", e.tag(), true)
+ }),
));
r
}
diff --git a/libs/cua/crates/cua-telemetry/src/events.rs b/libs/cua/crates/cua-telemetry/src/events.rs
index b149c75dcc..3d4fffd7cd 100644
--- a/libs/cua/crates/cua-telemetry/src/events.rs
+++ b/libs/cua/crates/cua-telemetry/src/events.rs
@@ -895,7 +895,11 @@ pub fn space_create(s: &SpaceCreate<'_>, outcome: Outcome, elapsed: Duration) ->
};
let error_kind = match outcome {
Outcome::Error if s.stalled && s.error_variant.is_empty() => "timeout",
- Outcome::Error => error_kind(s.error_variant),
+ // A failure never reports `none`.
+ Outcome::Error => match error_kind(s.error_variant) {
+ "none" => "other",
+ k => k,
+ },
_ => "none",
};
Event::new(event::SPACE_CREATE)
@@ -1047,7 +1051,10 @@ pub fn host_space_provided(
.s("outcome", outcome.as_str())
.s(
"error_kind",
- error_variant.map(error_kind).unwrap_or("none"),
+ match (outcome, error_variant.map(error_kind)) {
+ (Outcome::Error, None | Some("none")) => "other",
+ (_, k) => k.unwrap_or("none"),
+ },
)
}
diff --git a/libs/cua/crates/cua-telemetry/src/schema.rs b/libs/cua/crates/cua-telemetry/src/schema.rs
index 90fd290664..ff241efab7 100644
--- a/libs/cua/crates/cua-telemetry/src/schema.rs
+++ b/libs/cua/crates/cua-telemetry/src/schema.rs
@@ -142,6 +142,23 @@ pub const ERROR_KINDS: &[&str] = &[
"runtime_not_running",
"runtime_permission",
"image_pull",
+ // A failed create split by cause (cua_space_create, cua_sandbox_created,
+ // cua_host_space_provided; `classify::create_failure_kind`): the image is
+ // not in the registry, the registry refused the credentials, the network
+ // failed during the download, the registry rate-limited it.
+ "image_pull_not_found",
+ "image_pull_auth",
+ "image_pull_network",
+ "image_pull_rate_limited",
+ // The VM did not finish booting in time; it booted but its services
+ // (cua-spacesd, the computer server) never answered.
+ "vm_boot_timeout",
+ "services_unreachable",
+ // The machine cannot run VMs (no Virtualization.framework, no KVM, no
+ // hardware virtualization).
+ "virtualization_unavailable",
+ // The account's sandbox quota, plan limit or credits ran out.
+ "quota_exceeded",
"env",
"http",
"unauthenticated",
@@ -560,7 +577,7 @@ const OUTCOME: Prop = p("outcome", Kind::Enum(OUTCOMES), "ok, error or cancelled
const ERROR_KIND: Prop = p(
"error_kind",
Kind::Enum(ERROR_KINDS),
- "Typed error category (the error enum variant); never the message.",
+ "Typed error category (the error enum variant, or a fixed cause word for a failed create); never the message.",
);
const DURATION: Prop = p(
"duration_bucket",
@@ -691,7 +708,7 @@ pub const EVENTS: &[EventSpec] = &[
},
EventSpec {
name: event::SANDBOX_CREATED,
- version: 1,
+ version: 2,
tier: Tier::Usage,
sample_rate: 1.0,
products: CLIENTS,
@@ -1078,7 +1095,7 @@ pub const EVENTS: &[EventSpec] = &[
},
EventSpec {
name: event::SPACE_CREATE,
- version: 2,
+ version: 3,
tier: Tier::Usage,
sample_rate: 1.0,
products: &["spaces_app"],
@@ -1200,7 +1217,7 @@ pub const EVENTS: &[EventSpec] = &[
},
EventSpec {
name: event::HOST_SPACE_PROVIDED,
- version: 1,
+ version: 2,
tier: Tier::Usage,
sample_rate: 1.0,
products: &["daemon"],
diff --git a/libs/cua/crates/cua-telemetry/tests/privacy.rs b/libs/cua/crates/cua-telemetry/tests/privacy.rs
index 20c91b6b16..26c37d65e5 100644
--- a/libs/cua/crates/cua-telemetry/tests/privacy.rs
+++ b/libs/cua/crates/cua-telemetry/tests/privacy.rs
@@ -992,7 +992,23 @@ fn activation_builders_keep_vocabulary_and_coarsen_the_rest() {
);
assert_eq!(disk.props["error_kind"], "insufficient_disk");
// The causes the Spaces app reads from a failure's text.
- for (variant, kind) in [("LocalNetwork", "local_network"), ("VmLimit", "vm_limit")] {
+ for (variant, kind) in [
+ ("LocalNetwork", "local_network"),
+ ("VmLimit", "vm_limit"),
+ // The cause words the app core sends as they are.
+ ("image_pull_not_found", "image_pull_not_found"),
+ ("image_pull_auth", "image_pull_auth"),
+ ("image_pull_network", "image_pull_network"),
+ ("image_pull_rate_limited", "image_pull_rate_limited"),
+ ("vm_boot_timeout", "vm_boot_timeout"),
+ ("services_unreachable", "services_unreachable"),
+ ("virtualization_unavailable", "virtualization_unavailable"),
+ ("quota_exceeded", "quota_exceeded"),
+ ("runtime_not_running", "runtime_not_running"),
+ // A failure never reports `none`.
+ ("none", "other"),
+ ("", "other"),
+ ] {
let e = events::space_create(
&events::SpaceCreate {
on: "local",
@@ -1423,3 +1439,13 @@ fn agent_run_end_follows_the_published_status() {
assert_eq!(events::error_kind("AgentFailed"), "agent_failed");
assert_eq!(events::error_kind("Agent"), "agent");
}
+
+#[test]
+fn a_failed_host_space_never_reports_no_error_kind() {
+ let e = events::host_space_provided("vm", "macos", Outcome::Error, None);
+ assert_eq!(e.props["error_kind"], "other");
+ let e = events::host_space_provided("vm", "macos", Outcome::Error, Some("image_pull_auth"));
+ assert_eq!(e.props["error_kind"], "image_pull_auth");
+ let ok = events::host_space_provided("vm", "macos", Outcome::Ok, None);
+ assert_eq!(ok.props["error_kind"], "none");
+}
diff --git a/samples/openkoalabot-example-tauri/src-tauri/Cargo.lock b/samples/openkoalabot-example-tauri/src-tauri/Cargo.lock
index 15417d64e6..b9a5fcf0a9 100644
--- a/samples/openkoalabot-example-tauri/src-tauri/Cargo.lock
+++ b/samples/openkoalabot-example-tauri/src-tauri/Cargo.lock
@@ -1326,6 +1326,10 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "cua-create-failure"
+version = "0.1.0"
+
[[package]]
name = "cua-daemon"
version = "0.1.0"
@@ -1642,6 +1646,7 @@ dependencies = [
"chrono",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-fleet",
"cua-home",
"cua-host",
diff --git a/tests/e2e/cua-sdk/rust/Cargo.lock b/tests/e2e/cua-sdk/rust/Cargo.lock
index 24e78df376..3639c5b703 100644
--- a/tests/e2e/cua-sdk/rust/Cargo.lock
+++ b/tests/e2e/cua-sdk/rust/Cargo.lock
@@ -1307,6 +1307,10 @@ dependencies = [
"url",
]
+[[package]]
+name = "cua-create-failure"
+version = "0.1.0"
+
[[package]]
name = "cua-daemon"
version = "0.1.0"
@@ -1550,6 +1554,7 @@ dependencies = [
"cua-agent-setup",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-daemon",
"cua-fleet",
"cua-host",
@@ -1604,6 +1609,7 @@ dependencies = [
"chrono",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-fleet",
"cua-home",
"cua-host",
diff --git a/tests/e2e/local-runtimes/Cargo.lock b/tests/e2e/local-runtimes/Cargo.lock
index 1c43f6097f..351b8cf747 100644
--- a/tests/e2e/local-runtimes/Cargo.lock
+++ b/tests/e2e/local-runtimes/Cargo.lock
@@ -765,6 +765,10 @@ dependencies = [
"url",
]
+[[package]]
+name = "cua-create-failure"
+version = "0.1.0"
+
[[package]]
name = "cua-daemon"
version = "0.1.0"
@@ -1022,6 +1026,7 @@ dependencies = [
"chrono",
"cua-agents",
"cua-auth",
+ "cua-create-failure",
"cua-fleet",
"cua-home",
"cua-host",