Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
Reclaim failed creates' VMs and say what holds a full host (CUA-3139) (#6378)
cua-agents[bot]6ccb4fc9bd59
diff --git a/docs/content/docs/cua-sdk/concepts/telemetry-error-kinds.mdx b/docs/content/docs/cua-sdk/concepts/telemetry-error-kinds.mdxindex c64909796d..2a6572bcf7 100644--- a/docs/content/docs/cua-sdk/concepts/telemetry-error-kinds.mdx+++ b/docs/content/docs/cua-sdk/concepts/telemetry-error-kinds.mdx@@ -47,6 +47,8 @@ say.| `pull_interrupted` | The image download stopped partway. Another try resumes it. || `local_network_denied` | macOS Local Network privacy is off for the app or its service. || `local_network_vpn` | A VPN routes the VM's private network away. |+| `capacity_vms` | `host_capacity`: this machine had no room and Cua VMs it runs hold the capacity (the error names them so the person can stop them). A VM left running by an earlier failed attempt shows up here. |+| `capacity_other` | `host_capacity`: this machine had no room while no Cua VM runs there; other programs use its memory or CPUs, or its limits are too tight. |## Automatic retrydiff --git a/libs/cua/crates/cua-create-failure/src/retry.rs b/libs/cua/crates/cua-create-failure/src/retry.rsindex ffd41505c0..b5ed00c0ae 100644--- a/libs/cua/crates/cua-create-failure/src/retry.rs+++ b/libs/cua/crates/cua-create-failure/src/retry.rs@@ -40,6 +40,13 @@ pub enum Subclass {LocalNetworkDenied,/// A VPN routes the VM's private network away.LocalNetworkVpn,+ /// This machine had no room, and Cua VMs it runs hold the capacity (a+ /// refusal that names them, so the person can stop them). A VM left+ /// running by an earlier attempt shows up here.+ CapacityVms,+ /// This machine had no room while no Cua VM runs: other programs use+ /// its memory or CPUs, or its limits are set too tight.+ CapacityOther,}/// Every [`Subclass`] word, in order (`cua_telemetry::schema::ERROR_SUBCLASSES`).@@ -50,6 +57,8 @@ pub const SUBCLASSES: &[&str] = &["pull_interrupted","local_network_denied","local_network_vpn",+ "capacity_vms",+ "capacity_other",];impl Subclass {@@ -62,6 +71,8 @@ impl Subclass {Self::PullInterrupted => "pull_interrupted",Self::LocalNetworkDenied => "local_network_denied",Self::LocalNetworkVpn => "local_network_vpn",+ Self::CapacityVms => "capacity_vms",+ Self::CapacityOther => "capacity_other",}}@@ -73,6 +84,8 @@ impl Subclass {"pull_interrupted" => Self::PullInterrupted,"local_network_denied" => Self::LocalNetworkDenied,"local_network_vpn" => Self::LocalNetworkVpn,+ "capacity_vms" => Self::CapacityVms,+ "capacity_other" => Self::CapacityOther,_ => Self::None,}}@@ -133,6 +146,17 @@ pub fn subclass(kind: &str, message: &str) -> Subclass {"local_network_vpn" => return Subclass::LocalNetworkVpn,_ => {}}+ // No room on this machine: told apart by whether the refusal names Cua+ // VMs holding it (`cua_vmm::density::holders_note`). Both are lasting.+ if kind == "host_capacity" {+ if message.contains("Running now:") {+ return Subclass::CapacityVms;+ }+ if message.contains("No Cua VM is running") {+ return Subclass::CapacityOther;+ }+ return Subclass::None;+ }// A specific lasting cause stays itself: "permission denied" in a// runtime's start is a permission problem, not a passing one.if LASTING.contains(&kind.as_str()) {@@ -208,6 +232,23 @@ mod tests {}}+ #[test]+ fn no_room_says_whether_cua_vms_hold_the_capacity() {+ let held = "this host is at its VM memory limit: its VMs use 5.0 GiB. Running now: a (4.0 GiB, 2 vCPUs), b (1.0 GiB, 2 vCPUs). Stop or delete the ones you do not need";+ let (kind, sub, retry) = classify(held, "booting", "HostCapacity");+ assert_eq!(kind, "host_capacity");+ assert_eq!(sub, Subclass::CapacityVms);+ assert_eq!(retry, Retry::Never);+ let other = "this host has 1.0 GiB of memory available. No Cua VM is running on this machine, so other programs are using its memory or CPUs";+ let (kind, sub, retry) = classify(other, "booting", "HostCapacity");+ assert_eq!(kind, "host_capacity");+ assert_eq!(sub, Subclass::CapacityOther);+ assert_eq!(retry, Retry::Never);+ // Other refusals (file table, disk) say nothing finer.+ let (_, sub, _) = classify("out of open-file capacity", "booting", "HostCapacity");+ assert_eq!(sub, Subclass::None);+ }+#[test]fn passing_failures_retry_once() {let cases = [diff --git a/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs b/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rsindex 5db5ec65fd..54c1ae1847 100644--- a/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs+++ b/libs/cua/crates/cua-spaces-app-core/src/spaces/creating.rs@@ -1142,6 +1142,12 @@ pub fn fix_for(kind: &str, p: &PendingCreate) -> Option<CreateFix> {allows. Stop or delete one, then create it again.",machine)),+ "host_capacity" => help(format!(+ "It failed twice for the same reason: {} has no room for another VM right now. \+ Stop or delete a Space you are not using (the message names what is running), or \+ close other programs, then create it again.",+ machine+ )),"local_network" | "local_network_denied" | "local_network_vpn" => help(format!("It failed twice for the same reason: Cua can't reach the new VM{on}. Turn on \Local Network access for Cua Spaces in System Settings (or turn off a VPN), then \@@ -2177,6 +2183,7 @@ mod tests {"runtime_permission","virtualization_unavailable","vm_limit",+ "host_capacity","local_network","local_network_denied","local_network_vpn",diff --git a/libs/cua/crates/cua-spaces/src/creating.rs b/libs/cua/crates/cua-spaces/src/creating.rsindex 452d6650bf..010557645d 100644--- a/libs/cua/crates/cua-spaces/src/creating.rs+++ b/libs/cua/crates/cua-spaces/src/creating.rs@@ -281,14 +281,6 @@ pub(crate) fn record(made: Made) {let _ = CURRENT.try_with(|j| j.record(made));}-/// Whether the create this task runs generated its name (`false` outside-/// one).-pub(crate) fn generated_name() -> bool {- CURRENT- .try_with(|j| j.state.lock().unwrap_or_else(|e| e.into_inner()).generated)- .unwrap_or(false)-}-/// Records the token of the create this task runs.pub(crate) fn set_token(token: &str) {let _ = CURRENT.try_with(|j| j.set_token(token));diff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rsindex b2fccaaf8b..aeafdca45e 100644--- a/libs/cua/crates/cua-spaces/src/spaces.rs+++ b/libs/cua/crates/cua-spaces/src/spaces.rs@@ -2972,10 +2972,13 @@ impl Spaces {}if let Err(e) = created {// A start that failed after the instance existed (a readiness- // probe that never passed) leaves a running VM no registry- // lists: nothing would ever show or stop it. One under a name- // the caller chose stays to debug.- if fresh && crate::creating::generated_name() {+ // probe that never passed, a host with no room) leaves a VM no+ // registry lists: nothing would ever show or stop it, and it+ // would hold this machine's memory and CPUs against every retry.+ // `fresh` says the name had nothing before this create, so+ // whatever is under it is this attempt's: delete it, whoever+ // chose the name.+ if fresh {match self.inner.sandboxes.delete(&name).await {Ok(()) | Err(cua_sandbox_core::Error::NotFound(_)) => {}Err(d) => {diff --git a/libs/cua/crates/cua-spaces/tests/providers.rs b/libs/cua/crates/cua-spaces/tests/providers.rsindex 9c0de48f55..b35f5db5fd 100644--- a/libs/cua/crates/cua-spaces/tests/providers.rs+++ b/libs/cua/crates/cua-spaces/tests/providers.rs@@ -615,9 +615,9 @@ impl LocalRuntime for UnreadyRuntime {}/// A Space whose start fails after its instance exists leaves nothing-/// running that no registry lists: the instance of a generated name is-/// deleted, and the error says why. A name the caller chose is never-/// deleted (it may be an instance the caller already had).+/// running that no registry lists: the instance this create made is+/// deleted, and the error says why, whoever chose the name. A name that+/// already had an instance is never deleted (it may be one the caller had).#[tokio::test]async fn a_space_that_fails_to_start_leaves_no_instance() {let rt = Arc::new(UnreadyRuntime::default());@@ -642,14 +642,25 @@ async fn a_space_that_fails_to_start_leaves_no_instance() {let e = spaces.create(create(None)).await.unwrap_err();assert!(e.to_string().contains("Local Network access"), "{e}");let deleted = rt.deleted.lock().unwrap().clone();- assert_eq!(deleted.len(), 1, "{deleted:?}");- assert!(deleted[0].starts_with("space-"), "{deleted:?}");+ assert!(!deleted.is_empty(), "{deleted:?}");+ assert!(+ deleted.iter().all(|d| d.starts_with("space-")),+ "{deleted:?}"+ );assert!(rt.running.lock().unwrap().is_empty());assert!(spaces.list().unwrap().is_empty());+ // A name the caller chose is this attempt's too: a failed create must+ // not leave a VM holding the machine's capacity against the retry.spaces.create(create(Some("mine"))).await.unwrap_err();- assert_eq!(rt.deleted.lock().unwrap().len(), 1);- assert!(rt.running.lock().unwrap().contains_key("mine"));+ assert!(rt.deleted.lock().unwrap().iter().any(|d| d == "mine"));+ assert!(rt.running.lock().unwrap().is_empty());++ // But a name that already had an instance is left alone.+ rt.running.lock().unwrap().insert("theirs".into(), true);+ spaces.create(create(Some("theirs"))).await.unwrap_err();+ assert!(!rt.deleted.lock().unwrap().iter().any(|d| d == "theirs"));+ assert!(rt.running.lock().unwrap().contains_key("theirs"));}/// `cua sb resume` on a local Space returns once its cua-spacesd answers,diff --git a/libs/cua/crates/cua-telemetry/src/schema.rs b/libs/cua/crates/cua-telemetry/src/schema.rsindex c2a8dc3470..668247fb50 100644--- a/libs/cua/crates/cua-telemetry/src/schema.rs+++ b/libs/cua/crates/cua-telemetry/src/schema.rs@@ -509,7 +509,8 @@ pub const CREATE_PHASES: &[&str] = &[/// too coarse (`cua_create_failure::retry::Subclass`): a local runtime that/// did not start this time, Cua's service that did not take the create, an/// image download that stopped partway, macOS Local Network privacy off, a-/// VPN routing the VM's network away. `none` when there is nothing finer.+/// VPN routing the VM's network away, a machine with no room (held by Cua+/// VMs, or by other programs). `none` when there is nothing finer.pub const ERROR_SUBCLASSES: &[&str] = &["none","runtime_start",@@ -517,6 +518,8 @@ pub const ERROR_SUBCLASSES: &[&str] = &["pull_interrupted","local_network_denied","local_network_vpn",+ "capacity_vms",+ "capacity_other",];/// Time from create to ready (or to the failure), bucketed finely enough/// for median and p90 estimates.@@ -1917,7 +1920,7 @@ pub const EVENTS: &[EventSpec] = &[p("error_subclass",Kind::Enum(ERROR_SUBCLASSES),- "The finer cause of the failure (runtime_start, daemon_unresponsive, pull_interrupted, local_network_denied, local_network_vpn); on a retried create that succeeded, the cause of the failure it recovered from; none otherwise. Never the message.",+ "The finer cause of the failure (runtime_start, daemon_unresponsive, pull_interrupted, local_network_denied, local_network_vpn, capacity_vms, capacity_other); on a retried create that succeeded, the cause of the failure it recovered from; none otherwise. Never the message.",),],purpose: "Activation and reliability: Space create success, failure and cancel rates by kind, location and GPU, time to ready or to cancel, which phase fails or is cancelled, stall timeouts, and how often one automatic retry saves a create.",diff --git a/libs/cua/crates/cua-vmm/src/density.rs b/libs/cua/crates/cua-vmm/src/density.rsindex a8ad12f33d..9d2de0ce3d 100644--- a/libs/cua/crates/cua-vmm/src/density.rs+++ b/libs/cua/crates/cua-vmm/src/density.rs@@ -403,6 +403,77 @@ pub fn decide(usage: &Usage,pending: &[Pending],req: &Request,+) -> Result<Admit, Refusal> {+ decide_inner(cfg, usage, pending, req).map_err(|r| {+ // A VM that never fits (bigger than everything VMs may use) is not+ // held back by the others: nothing to stop.+ if r.0.contains(NEVER_FITS) {+ r+ } else {+ Refusal(format!("{} {}", r.0, holders_note(usage, pending, req)))+ }+ })+}++/// Marks a refusal of a VM bigger than the host's whole VM memory limit.+const NEVER_FITS: &str = "more than this host's VM memory limit";++/// Opens the sentence that names the VMs holding this host's capacity; its+/// presence is how a refusal is told from one where none of this runtime's+/// VMs is running ([`HOLDERS_NONE`]).+pub const HOLDERS_RUNNING: &str = "Running now:";+/// Said when no VM of this runtime runs: the host's capacity is held by+/// other programs, or the limits are set too tight for this machine.+pub const HOLDERS_NONE: &str = "No Cua VM is running";++/// What to stop or free, for a refusal: the VMs holding capacity, biggest+/// first, or that none is.+pub fn holders_note(usage: &Usage, pending: &[Pending], req: &Request) -> String {+ let mut vms: Vec<(&str, u64, u32)> = usage+ .vms+ .iter()+ .filter(|v| v.name != req.name)+ .map(|v| {+ let held = pending+ .iter()+ .find(|p| p.name == v.name)+ .map_or(v.private_mb, |p| p.expected_mb.max(v.private_mb));+ (v.name.as_str(), held, v.vcpus)+ })+ .collect();+ for p in pending {+ if p.name != req.name && !vms.iter().any(|(n, _, _)| *n == p.name) {+ vms.push((p.name.as_str(), p.expected_mb, p.vcpus));+ }+ }+ if vms.is_empty() {+ return format!(+ "{HOLDERS_NONE} on this machine, so other programs are using its memory or CPUs: close some, or raise the limits (CUA_VM_MAX_RSS_FRACTION, CUA_VM_MIN_AVAILABLE_MB)."+ );+ }+ vms.sort_by_key(|(_, mb, _)| std::cmp::Reverse(*mb));+ let shown: Vec<String> = vms+ .iter()+ .take(5)+ .map(|(n, mb, c)| format!("{n} ({}, {c} vCPUs)", gib(*mb)))+ .collect();+ let more = vms.len().saturating_sub(5);+ format!(+ "{HOLDERS_RUNNING} {}{}. Stop or delete the ones you do not need; `cua sb list` shows them.",+ shown.join(", "),+ if more > 0 {+ format!(" and {more} more")+ } else {+ String::new()+ }+ )+}++fn decide_inner(+ cfg: &DensityConfig,+ usage: &Usage,+ pending: &[Pending],+ req: &Request,) -> Result<Admit, Refusal> {// A shared RAM file already counted (mapped by a running VM, or reserved// by a pending one) costs nothing more.@@ -975,6 +1046,42 @@ mod tests {}}+ #[test]+ fn a_refusal_names_the_vms_holding_the_host() {+ let (cfg, mut u) = runner();+ // Two VMs left running (a failed create's leftover among them) hold+ // the 6.2 GiB VMs may use: the next 4 GiB VM is refused and the+ // message names them, biggest first.+ u.vms = vec![vm("leftover", 3_900, None), vm("mine", 2_000, None)];+ let e = decide(&cfg, &u, &[], &fresh("next", 4096)).unwrap_err().0;+ assert!(e.contains("VM memory limit"), "{e}");+ assert!(e.contains(HOLDERS_RUNNING), "{e}");+ assert!(+ e.find("leftover").unwrap() < e.find("mine").unwrap(),+ "biggest first: {e}"+ );+ assert!(!e.contains(HOLDERS_NONE), "{e}");+ }++ #[test]+ fn a_refusal_with_no_vm_running_says_other_programs_hold_it() {+ let (cfg, mut u) = runner();+ // Nothing of ours runs, yet only 1 GiB of memory is available.+ u.host = host(7_990, 1_000);+ let e = decide(&cfg, &u, &[], &fresh("first", 4096)).unwrap_err().0;+ assert!(e.contains(HOLDERS_NONE), "{e}");+ assert!(!e.contains(HOLDERS_RUNNING), "{e}");+ }++ #[test]+ fn a_vm_that_never_fits_is_not_blamed_on_the_others() {+ let (cfg, mut u) = runner();+ u.vms = vec![vm("other", 100, None)];+ let e = decide(&cfg, &u, &[], &fresh("huge", 20_000)).unwrap_err().0;+ assert!(e.contains("more than this host's VM memory limit"), "{e}");+ assert!(!e.contains(HOLDERS_RUNNING), "{e}");+ }+#[test]fn a_vm_under_the_cap_is_admitted_on_an_empty_host() {let (cfg, u) = runner();diff --git a/libs/cua/crates/cua-vmm/src/qemu/mod.rs b/libs/cua/crates/cua-vmm/src/qemu/mod.rsindex 32fad0023c..1d84c93d07 100644--- a/libs/cua/crates/cua-vmm/src/qemu/mod.rs+++ b/libs/cua/crates/cua-vmm/src/qemu/mod.rs@@ -2704,6 +2704,7 @@ impl Runtime for QemuRuntime {}}+ let mut launched = false;if !Self::is_running(&st) {// Linux guests always get SSH access: the caller's, the one this// instance already has, or a managed per-instance key.@@ -2781,6 +2782,7 @@ impl Runtime for QemuRuntime {st.last_start = Some("cold".into());self.save(&st)?;}+ launched = true;}drop(owner);let inst = self.instance(&st);@@ -2793,7 +2795,16 @@ impl Runtime for QemuRuntime {// Booted (or restored) and answering, or not going to: from here on// the VM counts what it really uses, not what it was expected to.self.release(&st.name);- ready?;+ if let Err(e) = ready {+ // A start that did not come up must not keep the VM it launched:+ // nothing lists it, and its memory and vCPUs would count against+ // every later create on this host until someone found it.+ if launched {+ let _ = self.force_stop(&mut st, Duration::from_secs(2)).await;+ let _ = self.save(&st);+ }+ return Err(e);+ }Ok(inst)}