Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(spaces): name why a Space create failed, and wait for Local Network access on macOS (CUA-1368) (#4998)

Francesco Bonaccicd07939d962d
diff --git a/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs b/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs
index 7097f25cf3..7051e217e2 100644
--- a/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs
+++ b/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs
@@ -713,24 +713,53 @@ pub fn failure_text(
failure_text_on(raw, phase, provider, None, gpu)
}
-/// [`failure_text`] for a create on one of your machines named `host_name`:
-/// the causes that are about the Mac that runs the Space (Local Network
-/// access, Apple's two macOS VMs) name that machine, not this Mac.
-pub fn failure_text_on(
- raw: &str,
- phase: Option<&str>,
- provider: Option<SpaceProvider>,
- host_name: Option<&str>,
- gpu: bool,
-) -> String {
- let plain = crate::errors::plain_error(raw);
- let l = plain.to_ascii_lowercase();
+/// The cause of a failed create that the app knows how to explain, read
+/// from the SDK's error text ([`failure_cause`]).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FailureCause {
+ /// Not enough disk space for the image or the VM.
+ Disk,
+ /// macOS Local Network privacy keeps the app (or a host's service) from
+ /// reaching its new VM.
+ LocalNetwork,
+ /// Apple's two macOS VMs per Mac are already running.
+ VmLimit,
+ /// GPU acceleration was asked for and could not be turned on.
+ Gpu,
+ /// The Space's service, or a relay host, refused the sign-in.
+ Unauthenticated,
+ /// The Space booted but its service did not answer in time.
+ ServiceTimeout,
+ /// The image download did not finish in time.
+ PullTimeout,
+}
+
+impl FailureCause {
+ /// The error enum case name telemetry records for it (`error_kind`
+ /// `insufficient_disk`, `local_network`, `vm_limit`, ...): a fixed word,
+ /// never the message.
+ pub fn variant(self) -> &'static str {
+ match self {
+ Self::Disk => "InsufficientDisk",
+ Self::LocalNetwork => "LocalNetwork",
+ Self::VmLimit => "VmLimit",
+ Self::Gpu => "Unsupported",
+ Self::Unauthenticated => "Unauthenticated",
+ Self::ServiceTimeout | Self::PullTimeout => "Timeout",
+ }
+ }
+}
+
+/// Which known cause the SDK's `raw` error is, if any. `phase` is where it
+/// failed and `gpu` whether GPU acceleration was asked for, when known.
+pub fn failure_cause(raw: &str, phase: Option<&str>, gpu: bool) -> Option<FailureCause> {
+ let l = crate::errors::plain_error(raw).to_ascii_lowercase();
if l.contains("not enough disk space")
|| l.contains("insufficient disk")
|| l.contains("insufficient_disk")
|| l.contains("no space left on device")
{
- return disk_text(&plain);
+ return Some(FailureCause::Disk);
}
// macOS Local Network privacy: the process that runs the VM (this
// app's daemon, or a host's cua-spacesd service) may not reach the
@@ -741,12 +770,7 @@ pub fn failure_text_on(
if l.contains("local network access")
|| (vmnet && (l.contains("no route to host") || l.contains("os error 65")))
{
- let who = machine_subject(provider, host_name);
- return format!(
- "{who} can't reach its new VM because Cua doesn't have Local Network access there. \
- On that Mac, open System Settings > Privacy & Security > Local Network, turn on Cua \
- Spaces (or cua-spacesd), then try again."
- );
+ return Some(FailureCause::LocalNetwork);
}
// Virtualization.framework's `virtualMachineLimitExceeded` ("The number
// of virtual machines exceeds the limit"): Apple's two macOS VMs per Mac.
@@ -755,61 +779,108 @@ pub fn failure_text_on(
&& l.contains("limit")
&& (l.contains("exceed") || l.contains("maximum") || l.contains("reached")))
{
- return format!(
- "{} is already running two macOS VMs, the most Apple's macOS license allows at once. \
- Stop one, then try again; the image is downloaded now, so it is quicker.",
- machine_subject(provider, host_name)
- );
+ return Some(FailureCause::VmLimit);
}
if gpu && (l.contains("gpu") || l.contains("paravirtual")) {
- return format!(
- "GPU acceleration could not be turned on here ({plain}). Create the Space again \
- with GPU acceleration off."
- );
+ return Some(FailureCause::Gpu);
}
if l.contains("unauthenticated") {
- return if provider == Some(SpaceProvider::Relay) {
- "Your other Mac did not accept the sign-in: its Cua sign-in may have expired or \
- belong to another account. Open Cua Spaces on that Mac, sign in again, then try \
- again."
- .into()
- } else {
- format!(
- "The new Space's service did not accept the app's token ({plain}). Try again; if \
- it keeps happening, run `cua doctor` in Terminal."
- )
- };
+ return Some(FailureCause::Unauthenticated);
}
- let spacesd = "cua-spacesd did not answer within ";
let timed_out =
l.contains("timed out") || l.contains("timeout") || l.contains("not ready after");
- if let Some(at) = l.find(spacesd) {
- let token = l[at + spacesd.len()..]
- .split_whitespace()
- .next()
- .unwrap_or("");
- let within = debug_duration_secs(token)
- .filter(|s| *s >= 5.0)
- .map(|s| format!(" within {}", short_duration(s)));
- return match within {
- Some(within) => format!(
- "The Space booted, but its service did not answer{within}. The image is \
- downloaded now, so trying again usually works."
- ),
- None => "The Space booted, but the time allowed for creating it ran out before its \
- service answered. The image is downloaded now, so trying again usually \
- works."
- .into(),
- };
+ if l.contains("cua-spacesd did not answer within ") {
+ return Some(FailureCause::ServiceTimeout);
}
match phase {
- Some("waiting_for_services") if timed_out => "The Space booted, but its service did not \
- answer in time. The image is downloaded now, so trying again usually works."
- .into(),
- Some("pulling") if timed_out => "The image download did not finish in the time \
+ Some("waiting_for_services") if timed_out => Some(FailureCause::ServiceTimeout),
+ Some("pulling") if timed_out => Some(FailureCause::PullTimeout),
+ _ => None,
+ }
+}
+
+/// The error enum case name a failed create records in telemetry: what
+/// the shell sent, unless that is none or a generic case (`Runtime`,
+/// `Other`, as most are) and the text names a known cause
+/// ([`failure_cause`]).
+pub fn failure_variant<'a>(raw: &str, phase: Option<&str>, gpu: bool, shell: &'a str) -> &'a str {
+ if !matches!(shell.trim(), "" | "Runtime" | "Other") {
+ return shell;
+ }
+ failure_cause(raw, phase, gpu).map_or(shell, |c| c.variant())
+}
+
+/// [`failure_text`] for a create on one of your machines named `host_name`:
+/// the causes that are about the Mac that runs the Space (Local Network
+/// access, Apple's two macOS VMs) name that machine, not this Mac.
+pub fn failure_text_on(
+ raw: &str,
+ phase: Option<&str>,
+ provider: Option<SpaceProvider>,
+ host_name: Option<&str>,
+ gpu: bool,
+) -> String {
+ let plain = crate::errors::plain_error(raw);
+ let l = plain.to_ascii_lowercase();
+ match failure_cause(raw, phase, gpu) {
+ Some(FailureCause::Disk) => disk_text(&plain),
+ Some(FailureCause::LocalNetwork) => format!(
+ "{} can't reach its new VM because Cua doesn't have Local Network access there. \
+ On that Mac, open System Settings > Privacy & Security > Local Network, turn on Cua \
+ Spaces (or cua-spacesd), then try again.",
+ machine_subject(provider, host_name)
+ ),
+ Some(FailureCause::VmLimit) => format!(
+ "{} is already running two macOS VMs, the most Apple's macOS license allows at once. \
+ Stop one, then try again; the image is downloaded now, so it is quicker.",
+ machine_subject(provider, host_name)
+ ),
+ Some(FailureCause::Gpu) => format!(
+ "GPU acceleration could not be turned on here ({plain}). Create the Space again \
+ with GPU acceleration off."
+ ),
+ Some(FailureCause::Unauthenticated) => {
+ if provider == Some(SpaceProvider::Relay) {
+ "Your other Mac did not accept the sign-in: its Cua sign-in may have expired or \
+ belong to another account. Open Cua Spaces on that Mac, sign in again, then try \
+ again."
+ .into()
+ } else {
+ format!(
+ "The new Space's service did not accept the app's token ({plain}). Try again; \
+ if it keeps happening, run `cua doctor` in Terminal."
+ )
+ }
+ }
+ Some(FailureCause::ServiceTimeout) => {
+ let spacesd = "cua-spacesd did not answer within ";
+ let within = l.find(spacesd).map(|at| {
+ let token = l[at + spacesd.len()..]
+ .split_whitespace()
+ .next()
+ .unwrap_or("");
+ debug_duration_secs(token)
+ .filter(|s| *s >= 5.0)
+ .map(|s| format!(" within {}", short_duration(s)))
+ });
+ match within {
+ Some(Some(within)) => format!(
+ "The Space booted, but its service did not answer{within}. The image is \
+ downloaded now, so trying again usually works."
+ ),
+ Some(None) => "The Space booted, but the time allowed for creating it ran out \
+ before its service answered. The image is downloaded now, so \
+ trying again usually works."
+ .into(),
+ None => "The Space booted, but its service did not answer in time. The image \
+ is downloaded now, so trying again usually works."
+ .into(),
+ }
+ }
+ Some(FailureCause::PullTimeout) => "The image download did not finish in the time \
allowed. Check your internet connection, then try again."
.into(),
- _ => plain,
+ None => plain,
}
}
diff --git a/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs b/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs
index 5b1c64513e..4ede2d52cb 100644
--- a/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs
+++ b/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs
@@ -489,13 +489,24 @@ pub fn creates(before: &CreatesState, action: &CreateAction, now_ms: i64) -> Vec
_ => vec![],
},
CreateAction::Fail {
- id, error_variant, ..
+ id,
+ error,
+ error_variant,
} => match find(id) {
Some(p) if p.space_id.is_none() && p.cancelling => {
vec![create_event(p, "cancelled", false, "", now_ms)]
}
Some(p) if p.space_id.is_none() && (p.error.is_none() || stalled(p)) => {
- vec![create_event(p, "error", stalled(p), error_variant, now_ms)]
+ // The shell's enum case is often `Runtime` or none: the
+ // cause the app reads from the text (Local Network, Apple's
+ // VM limit, disk) says more.
+ let variant = crate::spaces::creating::failure_variant(
+ error,
+ Some(&p.phase),
+ p.gpu,
+ error_variant,
+ );
+ vec![create_event(p, "error", stalled(p), variant, now_ms)]
}
_ => vec![],
},
@@ -1163,6 +1174,40 @@ mod tests {
TelemetrySignal::SpaceCreate { failed_phase, stalled: false, outcome, error_variant, .. }
if failed_phase == "booting" && outcome == "error" && error_variant == "InsufficientDisk"
));
+ // A cause the app reads from the text replaces the shell's generic case.
+ for (error, shell, want) in [
+ (
+ "runtime: Local Network access is not available: Cua Spaces cannot reach the VM at 192.168.64.5:3211 (No route to host (os error 65))",
+ "Runtime",
+ "LocalNetwork",
+ ),
+ (
+ "lume: The number of virtual machines exceeds the limit",
+ "",
+ "VmLimit",
+ ),
+ ("something else at /Users/alice", "Runtime", "Runtime"),
+ // A specific case from the shell stays.
+ (
+ "cua-spacesd timed out",
+ "SpacesdNotAvailable",
+ "SpacesdNotAvailable",
+ ),
+ ] {
+ let failed = creates(
+ &st,
+ &CreateAction::Fail {
+ id: "pending:a".into(),
+ error: error.into(),
+ error_variant: shell.into(),
+ },
+ 5_000,
+ );
+ assert!(matches!(
+ &failed[0],
+ TelemetrySignal::SpaceCreate { error_variant, .. } if error_variant == want
+ ));
+ }
// A stalled row is counted when dismissed, not when it stalls.
let stall = crate::spaces::creating::stall_error("pulling", 900.0);
let st = CreatesState {
diff --git a/libs/cua/crates/cua-telemetry/src/schema.rs b/libs/cua/crates/cua-telemetry/src/schema.rs
index 8d74702dda..c30e014483 100644
--- a/libs/cua/crates/cua-telemetry/src/schema.rs
+++ b/libs/cua/crates/cua-telemetry/src/schema.rs
@@ -151,6 +151,11 @@ pub const ERROR_KINDS: &[&str] = &[
"internal",
"ambiguous_sandbox",
"insufficient_disk",
+ // A Spaces-app create the app explained from the error's text: macOS
+ // Local Network privacy blocked the new VM; Apple's two macOS VMs
+ // were already running.
+ "local_network",
+ "vm_limit",
"requires_cua_app",
"cancelled",
// An agent run: cua_spaces `Error::Agent`, a run that recorded a
diff --git a/libs/cua/crates/cua-telemetry/tests/privacy.rs b/libs/cua/crates/cua-telemetry/tests/privacy.rs
index 888526f512..6076a6d681 100644
--- a/libs/cua/crates/cua-telemetry/tests/privacy.rs
+++ b/libs/cua/crates/cua-telemetry/tests/privacy.rs
@@ -989,6 +989,23 @@ fn activation_builders_keep_vocabulary_and_coarsen_the_rest() {
d,
);
assert_eq!(disk.props["error_kind"], "insufficient_disk");
+ // The causes the Spaces app reads from a failure's text.
+ for (variant, kind) in [("LocalNetwork", "local_network"), ("VmLimit", "vm_limit")] {
+ let e = events::space_create(
+ &events::SpaceCreate {
+ on: "local",
+ guest_os: "macos",
+ kind: "vm",
+ last_phase: "waiting_for_services",
+ stalled: false,
+ gpu: false,
+ error_variant: variant,
+ },
+ Outcome::Error,
+ d,
+ );
+ assert_eq!(e.props["error_kind"], kind);
+ }
let stalled = events::space_create(
&events::SpaceCreate {
on: "local",
diff --git a/libs/cua/crates/cua-vmm/src/probe.rs b/libs/cua/crates/cua-vmm/src/probe.rs
index d43afda678..12115ce22a 100644
--- a/libs/cua/crates/cua-vmm/src/probe.rs
+++ b/libs/cua/crates/cua-vmm/src/probe.rs
@@ -18,9 +18,33 @@ const HOLD_OPEN: Duration = Duration::from_millis(400);
/// [`unreachable_error`] instead of waiting out its whole budget. A guest
/// that just got its address can be briefly unroutable; one that stays so
/// is blocked from this process (on macOS: Local Network privacy) and
-/// never becomes reachable by waiting.
+/// never becomes reachable by waiting, unless someone lifts the block
+/// ([`APP_UNREACHABLE_GRACE`]).
pub const UNREACHABLE_GRACE: Duration = Duration::from_secs(20);
+/// [`UNREACHABLE_GRACE`] for a process that runs from a macOS `.app` (the
+/// Spaces app's daemon): there the block lifts as soon as the person clicks
+/// Allow on the "find devices on local networks" prompt, which macOS shows
+/// on the first connection, during this very wait, or turns the app on in
+/// System Settings. With 20 s, a macOS Space create in the Spaces app could
+/// fail while the prompt was still on screen. A CLI or a background service
+/// has no prompt anyone answers, so it keeps the short grace.
+pub const APP_UNREACHABLE_GRACE: Duration = Duration::from_secs(120);
+
+/// The grace this process gets ([`UNREACHABLE_GRACE`],
+/// [`APP_UNREACHABLE_GRACE`]).
+pub fn unreachable_grace() -> Duration {
+ grace_for(std::env::current_exe().ok().as_deref())
+}
+
+fn grace_for(exe: Option<&std::path::Path>) -> Duration {
+ if cfg!(target_os = "macos") && exe.and_then(app_bundle).is_some() {
+ APP_UNREACHABLE_GRACE
+ } else {
+ UNREACHABLE_GRACE
+ }
+}
+
/// One probe attempt.
#[derive(Debug)]
pub enum Attempt {
@@ -54,8 +78,9 @@ pub async fn attempt(ep: &Endpoints, probe: &Probe) -> Result<Attempt> {
}
/// Poll every probe until all pass or `timeout` elapses. A probe whose
-/// every attempt fails with "no route to host" for [`UNREACHABLE_GRACE`]
-/// fails at once with [`unreachable_error`].
+/// every attempt fails with "no route to host" for [`unreachable_grace`]
+/// fails at once with [`unreachable_error`], and so does one that is still
+/// unreachable when `timeout` runs out.
pub async fn wait_all(
name: &str,
ep: &Endpoints,
@@ -63,6 +88,7 @@ pub async fn wait_all(
timeout: Duration,
) -> Result<()> {
let deadline = tokio::time::Instant::now() + timeout;
+ let grace = unreachable_grace();
for probe in probes {
let mut unreachable = UnreachableWatch::default();
loop {
@@ -74,7 +100,10 @@ pub async fn wait_all(
}
Attempt::NotYet => unreachable.reset(),
Attempt::Unreachable { addr, error } => {
- if unreachable.observe(now) >= UNREACHABLE_GRACE {
+ // Still unroutable when the budget ends: say why, not
+ // that nothing answered.
+ if unreachable.observe(now) >= grace || tokio::time::Instant::now() >= deadline
+ {
return Err(unreachable_error(&addr, &error));
}
}
@@ -410,6 +439,22 @@ mod tests {
assert_eq!(app_bundle(std::path::Path::new("/usr/local/bin/cua")), None);
}
+ #[test]
+ fn only_an_app_waits_long_for_local_network() {
+ let app = std::path::Path::new("/Applications/Cua Spaces.app/Contents/MacOS/cua");
+ let want = if cfg!(target_os = "macos") {
+ APP_UNREACHABLE_GRACE
+ } else {
+ UNREACHABLE_GRACE
+ };
+ assert_eq!(grace_for(Some(app)), want);
+ assert_eq!(
+ grace_for(Some(std::path::Path::new("/usr/local/bin/cua"))),
+ UNREACHABLE_GRACE
+ );
+ assert_eq!(grace_for(None), UNREACHABLE_GRACE);
+ }
+
#[test]
fn parses_status_lines() {
assert_eq!(parse_status(b"HTTP/1.1 204 No Content\r\n"), Some(204));