Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua): split the rest of create runtime errors and explain the macOS VM limit (CUA-1356) (#5243)

Francesco Bonacci3a4ec34a77ee
diff --git a/libs/cua/crates/cua-create-failure/src/lib.rs b/libs/cua/crates/cua-create-failure/src/lib.rs
index f49f939f4d..3dba370f10 100644
--- a/libs/cua/crates/cua-create-failure/src/lib.rs
+++ b/libs/cua/crates/cua-create-failure/src/lib.rs
@@ -141,6 +141,7 @@ fn from_text(message: &str, phase: &str, local: bool) -> Option<&'static str> {
return Some(local_network::DENIED);
}
if l.contains("virtualmachinelimitexceeded")
+ || l.contains("running macos guests reached")
|| (has(&l, &["virtual machine", "vms"])
&& l.contains("limit")
&& has(&l, &["exceed", "maximum", "reached"]))
@@ -203,7 +204,19 @@ fn from_text(message: &str, phase: &str, local: bool) -> Option<&'static str> {
if (l.contains(" is not available: ") && !l.contains("network route"))
|| (l.contains("qemu") && l.contains("not found on path"))
{
- return Some("runtime_missing");
+ // What is missing is the text before " is not available: "
+ // (behind any `local runtime: [local] `): QEMU, Lume, UEFI
+ // firmware or Homebrew is a VM runtime; anything else (the
+ // container engine, gVisor) is the container runtime.
+ let what = l
+ .split_once(" is not available: ")
+ .map_or(l.as_str(), |(w, _)| w);
+ let vm = has(what, &["qemu", "lume", "uefi firmware", "homebrew"]);
+ return Some(if vm {
+ "vm_runtime_missing"
+ } else {
+ "runtime_missing"
+ });
}
}
if let Some(kind) = image_pull(&l, phase) {
@@ -291,6 +304,19 @@ fn from_text(message: &str, phase: &str, local: bool) -> Option<&'static str> {
if timed_out {
return Some(timeout_in(phase));
}
+ // The rest of a local `runtime` failure, by cause (the same words as
+ // `cua_vmm::error::failure_kind`, which names them for the app).
+ if local {
+ if has(&l, &["exited right after start", "' exited (code "]) {
+ return Some("container_exited");
+ }
+ if has(&l, &["lume api ", "qmp: ", "could not launch "]) {
+ return Some("vm_error");
+ }
+ if has(&l, &["container engine: ", "did not answer ping"]) {
+ return Some("engine_error");
+ }
+ }
None
}
@@ -408,6 +434,10 @@ pub const TEXT_KINDS: &[&str] = &[
"runtime_permission",
"runtime_not_running",
"runtime_missing",
+ "vm_runtime_missing",
+ "container_exited",
+ "vm_error",
+ "engine_error",
"unauthenticated",
"out_of_memory",
"port_in_use",
@@ -426,6 +456,73 @@ mod tests {
create_failure_kind(message, phase, variant, true)
}
+ /// The rest of a local `runtime` failure is named by cause, and the
+ /// words agree with `cua_vmm::error::failure_kind` (the app's own).
+ #[test]
+ fn the_rest_of_a_local_runtime_failure_is_named_by_cause() {
+ for (message, want) in [
+ (
+ "local runtime: [local] QEMU is not available: download QEMU for Windows",
+ "vm_runtime_missing",
+ ),
+ (
+ "local runtime: [local] lume serve at http://127.0.0.1:7777 is not available: start it",
+ "vm_runtime_missing",
+ ),
+ (
+ "local runtime: [local] UEFI firmware for x86_64 is not available: install edk2",
+ "vm_runtime_missing",
+ ),
+ (
+ "local runtime: [local] a container engine is not available: no socket",
+ "runtime_missing",
+ ),
+ (
+ "local runtime: [local] gVisor (runsc) is not available: install it",
+ "runtime_missing",
+ ),
+ (
+ "local runtime: [local] container 'a' exited right after start (status Stopped)",
+ "container_exited",
+ ),
+ (
+ "local runtime: [local] container 'a' exited (code 1) before Tcp(8000) passed",
+ "container_exited",
+ ),
+ ("local runtime: [local] lume API 500: boom", "vm_error"),
+ ("local runtime: [local] qmp: closed", "vm_error"),
+ (
+ "local runtime: [local] container engine: create: conflict",
+ "engine_error",
+ ),
+ ] {
+ assert_eq!(
+ create_failure_kind(message, "", "Runtime", true),
+ want,
+ "{message}"
+ );
+ assert!(
+ cua_telemetry::schema::ERROR_KINDS.contains(&want),
+ "{want} missing from ERROR_KINDS"
+ );
+ }
+ // An earlier, more specific cause still wins over these.
+ assert_eq!(
+ create_failure_kind(
+ "local runtime: [local] container engine: pull ghcr.io/x/y:1: manifest unknown",
+ "pulling",
+ "Runtime",
+ true
+ ),
+ "image_pull_not_found"
+ );
+ assert_eq!(
+ create_failure_kind("lume API 500: boom", "", "Runtime", false),
+ "Runtime",
+ "a cloud create has no local runtime to name"
+ );
+ }
+
#[test]
fn every_cause_word_is_in_the_telemetry_schema() {
for k in TEXT_KINDS
@@ -623,6 +720,14 @@ mod tests {
kind("VZErrorDomain virtualMachineLimitExceeded", "booting", ""),
"vm_limit"
);
+ assert_eq!(
+ kind(
+ "lume API 409: VM start rejected: host limit of 2 running macOS guests reached",
+ "booting",
+ ""
+ ),
+ "vm_limit"
+ );
assert_eq!(
kind("sandbox quota reached for this org", "", "Fleet"),
"quota_exceeded"
@@ -692,7 +797,7 @@ mod tests {
"pulling",
""
),
- "runtime_missing"
+ "vm_runtime_missing"
);
assert_eq!(
kind(
diff --git a/libs/cua/crates/cua-create-failure/src/retry.rs b/libs/cua/crates/cua-create-failure/src/retry.rs
index 0212648e60..5a498d4dd8 100644
--- a/libs/cua/crates/cua-create-failure/src/retry.rs
+++ b/libs/cua/crates/cua-create-failure/src/retry.rs
@@ -159,6 +159,7 @@ const LASTING: &[&str] = &[
"insufficient_disk",
"runtime_permission",
"runtime_missing",
+ "vm_runtime_missing",
"runtime_not_running",
"unsupported",
"invalid_argument",
diff --git a/libs/cua/crates/cua-sdk/src/native/telemetry.rs b/libs/cua/crates/cua-sdk/src/native/telemetry.rs
index d64d94e2e7..354bfe87fe 100644
--- a/libs/cua/crates/cua-sdk/src/native/telemetry.rs
+++ b/libs/cua/crates/cua-sdk/src/native/telemetry.rs
@@ -677,6 +677,30 @@ mod tests {
kind("local", CuaError::Runtime("boom".into())),
Some("runtime")
);
+ // The rest of `runtime`, by cause (each a schema word, not `other`).
+ for (message, want) in [
+ (
+ "lume API 409: VM start rejected: host limit of 2 running macOS guests reached",
+ "vm_limit",
+ ),
+ (
+ "Local Network access is not available: allow it",
+ "local_network",
+ ),
+ (
+ "QEMU is not available: download QEMU for Windows",
+ "vm_runtime_missing",
+ ),
+ (
+ "container 'a' exited right after start (status Stopped)",
+ "container_exited",
+ ),
+ ("lume API 500: boom", "vm_error"),
+ ("container engine: create: conflict", "engine_error"),
+ ] {
+ let m = format!("local runtime: [local] {message}");
+ assert_eq!(kind("local", CuaError::Runtime(m)), Some(want), "{message}");
+ }
// A cloud without an account stays provider_not_configured.
assert_eq!(
kind(
diff --git a/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs b/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs
index 54c1ae1847..59d0d5d962 100644
--- a/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs
+++ b/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs
@@ -938,8 +938,11 @@ pub fn failure_cause(raw: &str, phase: Option<&str>, gpu: bool) -> Option<Failur
return Some(FailureCause::LocalNetwork);
}
// Virtualization.framework's `virtualMachineLimitExceeded` ("The number
- // of virtual machines exceeds the limit"): Apple's two macOS VMs per Mac.
+ // of virtual machines exceeds the limit"), or Lume's own refusal ("VM
+ // start rejected: host limit of 2 running macOS guests reached"):
+ // Apple's two macOS VMs per Mac.
if l.contains("virtualmachinelimitexceeded")
+ || l.contains("running macos guests reached")
|| ((l.contains("virtual machine") || l.contains("vms"))
&& l.contains("limit")
&& (l.contains("exceed") || l.contains("maximum") || l.contains("reached")))
@@ -1124,7 +1127,7 @@ pub fn fix_for(kind: &str, p: &PendingCreate) -> Option<CreateFix> {
),
"cua runtime doctor",
),
- "runtime_missing" | "runtime_permission" => copy(
+ "runtime_missing" | "vm_runtime_missing" | "runtime_permission" => copy(
format!(
"It failed twice for the same reason: Cua can't use a container engine or VM \
runtime{on}. Run `cua runtime doctor` in Terminal to see what to install or \
@@ -2180,6 +2183,7 @@ mod tests {
"insufficient_disk",
"runtime_not_running",
"runtime_missing",
+ "vm_runtime_missing",
"runtime_permission",
"virtualization_unavailable",
"vm_limit",
@@ -3192,9 +3196,11 @@ mod tests {
Some(FailureCause::RuntimeMissing),
"{raw}"
);
+ // QEMU is a VM runtime: its own word in telemetry, while the
+ // person still reads the same "runtime missing" message.
assert_eq!(
failure_variant(raw, Some("pulling"), false, "Other"),
- "runtime_missing"
+ "vm_runtime_missing"
);
}
// Local Network privacy on the Mac that runs the VM.
@@ -3225,6 +3231,7 @@ mod tests {
"lume API 500: The number of virtual machines exceeds the limit.",
"Error Domain=VZErrorDomain Code=6 virtualMachineLimitExceeded",
"booting: the maximum number of VMs has been reached (limit 2)",
+ "lume API 409: VM start rejected: host limit of 2 running macOS guests reached",
] {
assert!(
failure_text(raw, Some("booting"), Some(SpaceProvider::Local), false)
diff --git a/libs/cua/crates/cua-telemetry/src/events.rs b/libs/cua/crates/cua-telemetry/src/events.rs
index bb2c6a3f19..7e68d28139 100644
--- a/libs/cua/crates/cua-telemetry/src/events.rs
+++ b/libs/cua/crates/cua-telemetry/src/events.rs
@@ -1135,9 +1135,10 @@ pub fn failed_step(
"vm_never_started" => Some("start_vm"),
"vm_config_invalid" => Some("configure_vm"),
_ => match kind {
- "runtime_missing" | "runtime_not_running" | "runtime_permission" => {
- Some("start_runtime")
- }
+ "runtime_missing"
+ | "vm_runtime_missing"
+ | "runtime_not_running"
+ | "runtime_permission" => Some("start_runtime"),
"virtualization_unavailable" | "host_capacity" => Some("check_host"),
// The checks before anything starts refuse these in `preparing`.
"local_network" | "insufficient_disk" if phase == "preparing" => Some("check_host"),
diff --git a/libs/cua/crates/cua-telemetry/src/schema.rs b/libs/cua/crates/cua-telemetry/src/schema.rs
index 490a95cf93..8d7321e614 100644
--- a/libs/cua/crates/cua-telemetry/src/schema.rs
+++ b/libs/cua/crates/cua-telemetry/src/schema.rs
@@ -162,6 +162,12 @@ pub const ERROR_KINDS: &[&str] = &[
// Cua Cloud has ended: a cloud create is refused up front. Expected, so
// reports can leave it out of the failures that need a fix.
"cloud_ended",
+ // QEMU, Lume or UEFI firmware missing; a container that exited at
+ // start; QEMU or Lume refusing or failing; any other engine error.
+ "vm_runtime_missing",
+ "container_exited",
+ "vm_error",
+ "engine_error",
"env",
"http",
"unauthenticated",
diff --git a/libs/cua/crates/cua-vmm/src/error.rs b/libs/cua/crates/cua-vmm/src/error.rs
index 76ab2fd30d..dbbdebc669 100644
--- a/libs/cua/crates/cua-vmm/src/error.rs
+++ b/libs/cua/crates/cua-vmm/src/error.rs
@@ -113,9 +113,23 @@ pub enum VmmError {
Other(String),
}
+/// Lume's refusal at Apple's two running macOS guests per Mac (HTTP 409,
+/// "VM start rejected: host limit of 2 running macOS guests reached"), or
+/// Virtualization.framework's own `virtualMachineLimitExceeded`, as the
+/// Spaces app reads it. `lower` is the lowercased message.
+pub fn is_vm_limit(lower: &str) -> bool {
+ lower.contains("running macos guests reached")
+ || lower.contains("virtualmachinelimitexceeded")
+ || ((lower.contains("virtual machine") || lower.contains("vms"))
+ && lower.contains("limit")
+ && (lower.contains("exceed") || lower.contains("maximum") || lower.contains("reached")))
+}
+
/// Why a local create failed, as a bounded word for telemetry (never the
-/// message): `runtime_missing`, `runtime_not_running`,
-/// `runtime_permission`, `image_pull`, else `None`.
+/// message): `image_pull`, `runtime_permission`, `runtime_not_running`,
+/// `local_network`, `vm_limit`, `vm_runtime_missing` (QEMU, Lume or UEFI
+/// firmware), `runtime_missing` (container engine, gVisor), `container_exited`,
+/// `vm_error` (QEMU or Lume refused or failed), `engine_error`, else `None`.
pub fn failure_kind(message: &str) -> Option<&'static str> {
let m = message.to_ascii_lowercase();
if m.contains("container engine: pull ")
@@ -129,11 +143,31 @@ pub fn failure_kind(message: &str) -> Option<&'static str> {
Some("runtime_permission")
} else if m.contains(" is not running: ") {
Some("runtime_not_running")
- } else if m.contains(" is not available: ")
- && !m.contains("local network access")
+ } else if m.contains("local network access")
+ || (m.contains("192.168.64.")
+ && (m.contains("no route to host") || m.contains("os error 65")))
+ {
+ Some("local_network")
+ } else if is_vm_limit(&m) {
+ Some("vm_limit")
+ } else if let Some((what, _)) = m.split_once(" is not available: ")
&& !m.contains("network route")
{
- Some("runtime_missing")
+ // `what` is the missing piece (behind any `local runtime: [local] `).
+ let vm = ["qemu", "lume", "uefi firmware", "homebrew"]
+ .iter()
+ .any(|w| what.contains(w));
+ Some(if vm {
+ "vm_runtime_missing"
+ } else {
+ "runtime_missing"
+ })
+ } else if m.contains("exited right after start") || m.contains("' exited (code ") {
+ Some("container_exited")
+ } else if m.contains("lume api ") || m.contains("qmp: ") || m.contains("could not launch ") {
+ Some("vm_error")
+ } else if m.contains("container engine: ") || m.contains("did not answer ping") {
+ Some("engine_error")
} else {
None
}
@@ -187,7 +221,36 @@ mod tests {
assert_eq!(failure_kind(&wrap(stopped)), Some("image_pull"));
assert_eq!(failure_kind(&VmmError::other("boom").to_string()), None);
let lan = VmmError::missing("Local Network access", "click Allow");
- assert_eq!(failure_kind(&lan.to_string()), None);
+ assert_eq!(failure_kind(&wrap(lan)), Some("local_network"));
+ let unroutable =
+ VmmError::other("connect 192.168.64.7:8000: No route to host (os error 65)");
+ assert_eq!(failure_kind(&wrap(unroutable)), Some("local_network"));
+ let cap = VmmError::Lume {
+ status: 409,
+ message: "VM start rejected: host limit of 2 running macOS guests reached".into(),
+ };
+ assert_eq!(failure_kind(&wrap(cap)), Some("vm_limit"));
+ for missing in [
+ VmmError::missing("QEMU", "download QEMU for Windows"),
+ VmmError::missing("qemu-system-x86_64", "install QEMU"),
+ VmmError::missing("lume serve at http://127.0.0.1:7777", "start it"),
+ VmmError::missing("UEFI firmware for x86_64", "install edk2"),
+ ] {
+ assert_eq!(failure_kind(&wrap(missing)), Some("vm_runtime_missing"));
+ }
+ let gvisor = VmmError::missing("gVisor (runsc)", "install it");
+ assert_eq!(failure_kind(&wrap(gvisor)), Some("runtime_missing"));
+ let exited = VmmError::other("container 'a' exited (code 1) before Tcp(8000) passed");
+ assert_eq!(failure_kind(&wrap(exited)), Some("container_exited"));
+ let lume = VmmError::Lume {
+ status: 500,
+ message: "boom".into(),
+ };
+ assert_eq!(failure_kind(&wrap(lume)), Some("vm_error"));
+ let qmp = VmmError::Qmp("closed".into());
+ assert_eq!(failure_kind(&wrap(qmp)), Some("vm_error"));
+ let engine = VmmError::Engine("create: conflict".into());
+ assert_eq!(failure_kind(&wrap(engine)), Some("engine_error"));
}
/// The create-failure subclass each error the macOS and Linux VM start
diff --git a/libs/cua/crates/cua-vmm/src/lume/client.rs b/libs/cua/crates/cua-vmm/src/lume/client.rs
index 635064f820..b42717ff9d 100644
--- a/libs/cua/crates/cua-vmm/src/lume/client.rs
+++ b/libs/cua/crates/cua-vmm/src/lume/client.rs
@@ -272,6 +272,17 @@ impl LumeClient {
.unwrap_or(message);
return Err(VmmError::NotFound(name));
}
+ // Apple's license allows two running macOS VMs per Mac: say how to
+ // free one rather than only that Lume refused.
+ let message = if crate::error::is_vm_limit(&message.to_ascii_lowercase()) {
+ format!(
+ "{message}. This Mac is already running two macOS VMs, the most Apple allows at \
+ once: stop one (in Cua Spaces, or `cua sb ls` then `cua sb suspend <name>`) and \
+ try again"
+ )
+ } else {
+ message
+ };
Err(VmmError::Lume {
status: status.as_u16(),
message,
@@ -1000,6 +1011,19 @@ pub(crate) mod tests {
)
}
+ /// Lume's 409 at Apple's two macOS VMs says how to free one, and
+ /// telemetry reads it as `vm_limit`.
+ #[tokio::test]
+ async fn the_macos_vm_limit_says_how_to_free_one() {
+ let body =
+ r#"{"message":"VM start rejected: host limit of 2 running macOS guests reached"}"#;
+ let (lume, _) = scripted_lume(vec![(409, body.to_string())]).await;
+ let e = lume.get("b").await.unwrap_err().to_string();
+ assert!(e.starts_with("lume API 409: VM start rejected"), "{e}");
+ assert!(e.contains("`cua sb suspend <name>`"), "{e}");
+ assert_eq!(crate::error::failure_kind(&e), Some("vm_limit"));
+ }
+
#[tokio::test]
async fn a_pull_reports_lumes_bytes() {
let mut gets = vec![];