Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(spaces): name why a Space create failed, and wait for Local Network access on macOS (CUA-1368) (#4998)
Francesco Bonaccicd07939d962d
diff --git a/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs b/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rsindex 7097f25cf3..7051e217e2 100644--- a/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs+++ b/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs@@ -713,24 +713,53 @@ pub fn failure_text(failure_text_on(raw, phase, provider, None, gpu)}-/// [`failure_text`] for a create on one of your machines named `host_name`:-/// the causes that are about the Mac that runs the Space (Local Network-/// access, Apple's two macOS VMs) name that machine, not this Mac.-pub fn failure_text_on(- raw: &str,- phase: Option<&str>,- provider: Option<SpaceProvider>,- host_name: Option<&str>,- gpu: bool,-) -> String {- let plain = crate::errors::plain_error(raw);- let l = plain.to_ascii_lowercase();+/// The cause of a failed create that the app knows how to explain, read+/// from the SDK's error text ([`failure_cause`]).+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum FailureCause {+ /// Not enough disk space for the image or the VM.+ Disk,+ /// macOS Local Network privacy keeps the app (or a host's service) from+ /// reaching its new VM.+ LocalNetwork,+ /// Apple's two macOS VMs per Mac are already running.+ VmLimit,+ /// GPU acceleration was asked for and could not be turned on.+ Gpu,+ /// The Space's service, or a relay host, refused the sign-in.+ Unauthenticated,+ /// The Space booted but its service did not answer in time.+ ServiceTimeout,+ /// The image download did not finish in time.+ PullTimeout,+}++impl FailureCause {+ /// The error enum case name telemetry records for it (`error_kind`+ /// `insufficient_disk`, `local_network`, `vm_limit`, ...): a fixed word,+ /// never the message.+ pub fn variant(self) -> &'static str {+ match self {+ Self::Disk => "InsufficientDisk",+ Self::LocalNetwork => "LocalNetwork",+ Self::VmLimit => "VmLimit",+ Self::Gpu => "Unsupported",+ Self::Unauthenticated => "Unauthenticated",+ Self::ServiceTimeout | Self::PullTimeout => "Timeout",+ }+ }+}++/// Which known cause the SDK's `raw` error is, if any. `phase` is where it+/// failed and `gpu` whether GPU acceleration was asked for, when known.+pub fn failure_cause(raw: &str, phase: Option<&str>, gpu: bool) -> Option<FailureCause> {+ let l = crate::errors::plain_error(raw).to_ascii_lowercase();if l.contains("not enough disk space")|| l.contains("insufficient disk")|| l.contains("insufficient_disk")|| l.contains("no space left on device"){- return disk_text(&plain);+ return Some(FailureCause::Disk);}// macOS Local Network privacy: the process that runs the VM (this// app's daemon, or a host's cua-spacesd service) may not reach the@@ -741,12 +770,7 @@ pub fn failure_text_on(if l.contains("local network access")|| (vmnet && (l.contains("no route to host") || l.contains("os error 65"))){- let who = machine_subject(provider, host_name);- return format!(- "{who} can't reach its new VM because Cua doesn't have Local Network access there. \- On that Mac, open System Settings > Privacy & Security > Local Network, turn on Cua \- Spaces (or cua-spacesd), then try again."- );+ return Some(FailureCause::LocalNetwork);}// Virtualization.framework's `virtualMachineLimitExceeded` ("The number// of virtual machines exceeds the limit"): Apple's two macOS VMs per Mac.@@ -755,61 +779,108 @@ pub fn failure_text_on(&& l.contains("limit")&& (l.contains("exceed") || l.contains("maximum") || l.contains("reached"))){- return format!(- "{} is already running two macOS VMs, the most Apple's macOS license allows at once. \- Stop one, then try again; the image is downloaded now, so it is quicker.",- machine_subject(provider, host_name)- );+ return Some(FailureCause::VmLimit);}if gpu && (l.contains("gpu") || l.contains("paravirtual")) {- return format!(- "GPU acceleration could not be turned on here ({plain}). Create the Space again \- with GPU acceleration off."- );+ return Some(FailureCause::Gpu);}if l.contains("unauthenticated") {- return if provider == Some(SpaceProvider::Relay) {- "Your other Mac did not accept the sign-in: its Cua sign-in may have expired or \- belong to another account. Open Cua Spaces on that Mac, sign in again, then try \- again."- .into()- } else {- format!(- "The new Space's service did not accept the app's token ({plain}). Try again; if \- it keeps happening, run `cua doctor` in Terminal."- )- };+ return Some(FailureCause::Unauthenticated);}- let spacesd = "cua-spacesd did not answer within ";let timed_out =l.contains("timed out") || l.contains("timeout") || l.contains("not ready after");- if let Some(at) = l.find(spacesd) {- let token = l[at + spacesd.len()..]- .split_whitespace()- .next()- .unwrap_or("");- let within = debug_duration_secs(token)- .filter(|s| *s >= 5.0)- .map(|s| format!(" within {}", short_duration(s)));- return match within {- Some(within) => format!(- "The Space booted, but its service did not answer{within}. The image is \- downloaded now, so trying again usually works."- ),- None => "The Space booted, but the time allowed for creating it ran out before its \- service answered. The image is downloaded now, so trying again usually \- works."- .into(),- };+ if l.contains("cua-spacesd did not answer within ") {+ return Some(FailureCause::ServiceTimeout);}match phase {- Some("waiting_for_services") if timed_out => "The Space booted, but its service did not \- answer in time. The image is downloaded now, so trying again usually works."- .into(),- Some("pulling") if timed_out => "The image download did not finish in the time \+ Some("waiting_for_services") if timed_out => Some(FailureCause::ServiceTimeout),+ Some("pulling") if timed_out => Some(FailureCause::PullTimeout),+ _ => None,+ }+}++/// The error enum case name a failed create records in telemetry: what+/// the shell sent, unless that is none or a generic case (`Runtime`,+/// `Other`, as most are) and the text names a known cause+/// ([`failure_cause`]).+pub fn failure_variant<'a>(raw: &str, phase: Option<&str>, gpu: bool, shell: &'a str) -> &'a str {+ if !matches!(shell.trim(), "" | "Runtime" | "Other") {+ return shell;+ }+ failure_cause(raw, phase, gpu).map_or(shell, |c| c.variant())+}++/// [`failure_text`] for a create on one of your machines named `host_name`:+/// the causes that are about the Mac that runs the Space (Local Network+/// access, Apple's two macOS VMs) name that machine, not this Mac.+pub fn failure_text_on(+ raw: &str,+ phase: Option<&str>,+ provider: Option<SpaceProvider>,+ host_name: Option<&str>,+ gpu: bool,+) -> String {+ let plain = crate::errors::plain_error(raw);+ let l = plain.to_ascii_lowercase();+ match failure_cause(raw, phase, gpu) {+ Some(FailureCause::Disk) => disk_text(&plain),+ Some(FailureCause::LocalNetwork) => format!(+ "{} can't reach its new VM because Cua doesn't have Local Network access there. \+ On that Mac, open System Settings > Privacy & Security > Local Network, turn on Cua \+ Spaces (or cua-spacesd), then try again.",+ machine_subject(provider, host_name)+ ),+ Some(FailureCause::VmLimit) => format!(+ "{} is already running two macOS VMs, the most Apple's macOS license allows at once. \+ Stop one, then try again; the image is downloaded now, so it is quicker.",+ machine_subject(provider, host_name)+ ),+ Some(FailureCause::Gpu) => format!(+ "GPU acceleration could not be turned on here ({plain}). Create the Space again \+ with GPU acceleration off."+ ),+ Some(FailureCause::Unauthenticated) => {+ if provider == Some(SpaceProvider::Relay) {+ "Your other Mac did not accept the sign-in: its Cua sign-in may have expired or \+ belong to another account. Open Cua Spaces on that Mac, sign in again, then try \+ again."+ .into()+ } else {+ format!(+ "The new Space's service did not accept the app's token ({plain}). Try again; \+ if it keeps happening, run `cua doctor` in Terminal."+ )+ }+ }+ Some(FailureCause::ServiceTimeout) => {+ let spacesd = "cua-spacesd did not answer within ";+ let within = l.find(spacesd).map(|at| {+ let token = l[at + spacesd.len()..]+ .split_whitespace()+ .next()+ .unwrap_or("");+ debug_duration_secs(token)+ .filter(|s| *s >= 5.0)+ .map(|s| format!(" within {}", short_duration(s)))+ });+ match within {+ Some(Some(within)) => format!(+ "The Space booted, but its service did not answer{within}. The image is \+ downloaded now, so trying again usually works."+ ),+ Some(None) => "The Space booted, but the time allowed for creating it ran out \+ before its service answered. The image is downloaded now, so \+ trying again usually works."+ .into(),+ None => "The Space booted, but its service did not answer in time. The image \+ is downloaded now, so trying again usually works."+ .into(),+ }+ }+ Some(FailureCause::PullTimeout) => "The image download did not finish in the time \allowed. Check your internet connection, then try again.".into(),- _ => plain,+ None => plain,}}diff --git a/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs b/libs/cua/crates/cua-spaces-app-core/src/telemetry.rsindex 5b1c64513e..4ede2d52cb 100644--- a/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs+++ b/libs/cua/crates/cua-spaces-app-core/src/telemetry.rs@@ -489,13 +489,24 @@ pub fn creates(before: &CreatesState, action: &CreateAction, now_ms: i64) -> Vec_ => vec![],},CreateAction::Fail {- id, error_variant, ..+ id,+ error,+ error_variant,} => match find(id) {Some(p) if p.space_id.is_none() && p.cancelling => {vec![create_event(p, "cancelled", false, "", now_ms)]}Some(p) if p.space_id.is_none() && (p.error.is_none() || stalled(p)) => {- vec![create_event(p, "error", stalled(p), error_variant, now_ms)]+ // The shell's enum case is often `Runtime` or none: the+ // cause the app reads from the text (Local Network, Apple's+ // VM limit, disk) says more.+ let variant = crate::spaces::creating::failure_variant(+ error,+ Some(&p.phase),+ p.gpu,+ error_variant,+ );+ vec![create_event(p, "error", stalled(p), variant, now_ms)]}_ => vec![],},@@ -1163,6 +1174,40 @@ mod tests {TelemetrySignal::SpaceCreate { failed_phase, stalled: false, outcome, error_variant, .. }if failed_phase == "booting" && outcome == "error" && error_variant == "InsufficientDisk"));+ // A cause the app reads from the text replaces the shell's generic case.+ for (error, shell, want) in [+ (+ "runtime: Local Network access is not available: Cua Spaces cannot reach the VM at 192.168.64.5:3211 (No route to host (os error 65))",+ "Runtime",+ "LocalNetwork",+ ),+ (+ "lume: The number of virtual machines exceeds the limit",+ "",+ "VmLimit",+ ),+ ("something else at /Users/alice", "Runtime", "Runtime"),+ // A specific case from the shell stays.+ (+ "cua-spacesd timed out",+ "SpacesdNotAvailable",+ "SpacesdNotAvailable",+ ),+ ] {+ let failed = creates(+ &st,+ &CreateAction::Fail {+ id: "pending:a".into(),+ error: error.into(),+ error_variant: shell.into(),+ },+ 5_000,+ );+ assert!(matches!(+ &failed[0],+ TelemetrySignal::SpaceCreate { error_variant, .. } if error_variant == want+ ));+ }// A stalled row is counted when dismissed, not when it stalls.let stall = crate::spaces::creating::stall_error("pulling", 900.0);let st = CreatesState {diff --git a/libs/cua/crates/cua-telemetry/src/schema.rs b/libs/cua/crates/cua-telemetry/src/schema.rsindex 8d74702dda..c30e014483 100644--- a/libs/cua/crates/cua-telemetry/src/schema.rs+++ b/libs/cua/crates/cua-telemetry/src/schema.rs@@ -151,6 +151,11 @@ pub const ERROR_KINDS: &[&str] = &["internal","ambiguous_sandbox","insufficient_disk",+ // A Spaces-app create the app explained from the error's text: macOS+ // Local Network privacy blocked the new VM; Apple's two macOS VMs+ // were already running.+ "local_network",+ "vm_limit","requires_cua_app","cancelled",// An agent run: cua_spaces `Error::Agent`, a run that recorded adiff --git a/libs/cua/crates/cua-telemetry/tests/privacy.rs b/libs/cua/crates/cua-telemetry/tests/privacy.rsindex 888526f512..6076a6d681 100644--- a/libs/cua/crates/cua-telemetry/tests/privacy.rs+++ b/libs/cua/crates/cua-telemetry/tests/privacy.rs@@ -989,6 +989,23 @@ fn activation_builders_keep_vocabulary_and_coarsen_the_rest() {d,);assert_eq!(disk.props["error_kind"], "insufficient_disk");+ // The causes the Spaces app reads from a failure's text.+ for (variant, kind) in [("LocalNetwork", "local_network"), ("VmLimit", "vm_limit")] {+ let e = events::space_create(+ &events::SpaceCreate {+ on: "local",+ guest_os: "macos",+ kind: "vm",+ last_phase: "waiting_for_services",+ stalled: false,+ gpu: false,+ error_variant: variant,+ },+ Outcome::Error,+ d,+ );+ assert_eq!(e.props["error_kind"], kind);+ }let stalled = events::space_create(&events::SpaceCreate {on: "local",diff --git a/libs/cua/crates/cua-vmm/src/probe.rs b/libs/cua/crates/cua-vmm/src/probe.rsindex d43afda678..12115ce22a 100644--- a/libs/cua/crates/cua-vmm/src/probe.rs+++ b/libs/cua/crates/cua-vmm/src/probe.rs@@ -18,9 +18,33 @@ const HOLD_OPEN: Duration = Duration::from_millis(400);/// [`unreachable_error`] instead of waiting out its whole budget. A guest/// that just got its address can be briefly unroutable; one that stays so/// is blocked from this process (on macOS: Local Network privacy) and-/// never becomes reachable by waiting.+/// never becomes reachable by waiting, unless someone lifts the block+/// ([`APP_UNREACHABLE_GRACE`]).pub const UNREACHABLE_GRACE: Duration = Duration::from_secs(20);+/// [`UNREACHABLE_GRACE`] for a process that runs from a macOS `.app` (the+/// Spaces app's daemon): there the block lifts as soon as the person clicks+/// Allow on the "find devices on local networks" prompt, which macOS shows+/// on the first connection, during this very wait, or turns the app on in+/// System Settings. With 20 s, a macOS Space create in the Spaces app could+/// fail while the prompt was still on screen. A CLI or a background service+/// has no prompt anyone answers, so it keeps the short grace.+pub const APP_UNREACHABLE_GRACE: Duration = Duration::from_secs(120);++/// The grace this process gets ([`UNREACHABLE_GRACE`],+/// [`APP_UNREACHABLE_GRACE`]).+pub fn unreachable_grace() -> Duration {+ grace_for(std::env::current_exe().ok().as_deref())+}++fn grace_for(exe: Option<&std::path::Path>) -> Duration {+ if cfg!(target_os = "macos") && exe.and_then(app_bundle).is_some() {+ APP_UNREACHABLE_GRACE+ } else {+ UNREACHABLE_GRACE+ }+}+/// One probe attempt.#[derive(Debug)]pub enum Attempt {@@ -54,8 +78,9 @@ pub async fn attempt(ep: &Endpoints, probe: &Probe) -> Result<Attempt> {}/// Poll every probe until all pass or `timeout` elapses. A probe whose-/// every attempt fails with "no route to host" for [`UNREACHABLE_GRACE`]-/// fails at once with [`unreachable_error`].+/// every attempt fails with "no route to host" for [`unreachable_grace`]+/// fails at once with [`unreachable_error`], and so does one that is still+/// unreachable when `timeout` runs out.pub async fn wait_all(name: &str,ep: &Endpoints,@@ -63,6 +88,7 @@ pub async fn wait_all(timeout: Duration,) -> Result<()> {let deadline = tokio::time::Instant::now() + timeout;+ let grace = unreachable_grace();for probe in probes {let mut unreachable = UnreachableWatch::default();loop {@@ -74,7 +100,10 @@ pub async fn wait_all(}Attempt::NotYet => unreachable.reset(),Attempt::Unreachable { addr, error } => {- if unreachable.observe(now) >= UNREACHABLE_GRACE {+ // Still unroutable when the budget ends: say why, not+ // that nothing answered.+ if unreachable.observe(now) >= grace || tokio::time::Instant::now() >= deadline+ {return Err(unreachable_error(&addr, &error));}}@@ -410,6 +439,22 @@ mod tests {assert_eq!(app_bundle(std::path::Path::new("/usr/local/bin/cua")), None);}+ #[test]+ fn only_an_app_waits_long_for_local_network() {+ let app = std::path::Path::new("/Applications/Cua Spaces.app/Contents/MacOS/cua");+ let want = if cfg!(target_os = "macos") {+ APP_UNREACHABLE_GRACE+ } else {+ UNREACHABLE_GRACE+ };+ assert_eq!(grace_for(Some(app)), want);+ assert_eq!(+ grace_for(Some(std::path::Path::new("/usr/local/bin/cua"))),+ UNREACHABLE_GRACE+ );+ assert_eq!(grace_for(None), UNREACHABLE_GRACE);+ }+#[test]fn parses_status_lines() {assert_eq!(parse_status(b"HTTP/1.1 204 No Content\r\n"), Some(204));