Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(spaces): Windows Spaces on nested AWS KVM hosts (windows-kvm experiment, off by default) (#6358)

Francesco Bonacci6bdfdc539720
diff --git a/examples/streaming/rust/Cargo.lock b/examples/streaming/rust/Cargo.lock
index 5527e20059..1e48f094a4 100644
--- a/examples/streaming/rust/Cargo.lock
+++ b/examples/streaming/rust/Cargo.lock
@@ -1378,6 +1378,7 @@ dependencies = [
"cua-host",
"cua-sandbox-core",
"cua-spacesd-client",
+ "cua-vmm",
"humantime",
"rand 0.9.5",
"reqwest 0.12.28",
diff --git a/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock b/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock
index 13bfe744ad..0563856206 100644
--- a/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock
+++ b/libs/cua-spacesd/tests/spaces-e2e/Cargo.lock
@@ -2129,6 +2129,7 @@ dependencies = [
"cua-host",
"cua-sandbox-core",
"cua-spacesd-client",
+ "cua-vmm",
"humantime",
"rand 0.9.5",
"reqwest 0.12.28",
diff --git a/libs/cua/crates/cua-byoc/Cargo.toml b/libs/cua/crates/cua-byoc/Cargo.toml
index 05d9a96b0a..728fdcee26 100644
--- a/libs/cua/crates/cua-byoc/Cargo.toml
+++ b/libs/cua/crates/cua-byoc/Cargo.toml
@@ -57,11 +57,13 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-
# contrib providers.
cua-contrib = { workspace = true, optional = true }
+# The experiment switches (`windows-kvm`).
+cua-vmm.workspace = true
+
[dev-dependencies]
axum.workspace = true
url.workspace = true
# The NoCloud seed writer of the opt-in QEMU boot test (tests/boot_qemu.rs).
-cua-vmm.workspace = true
ring = "0.17"
cua-spaces.workspace = true
cua-proto.workspace = true
diff --git a/libs/cua/crates/cua-byoc/src/aws.rs b/libs/cua/crates/cua-byoc/src/aws.rs
index 407aa1f609..94c0f34603 100644
--- a/libs/cua/crates/cua-byoc/src/aws.rs
+++ b/libs/cua/crates/cua-byoc/src/aws.rs
@@ -94,7 +94,7 @@ pub const HOST_DISK_GB: u32 = 60;
/// Machine types offered to the elastic pool: name, arch, vCPUs, GiB,
/// on-demand Linux USD per hour in us-west-2 (AWS Price List, 2026-10-21),
-/// whether it has `/dev/kvm` (bare metal), and its GPU (empty: none).
+/// whether it has `/dev/kvm` directly (bare metal), and its GPU (empty: none).
pub const HOST_TYPES: &[(&str, &str, u32, u64, f64, bool, &str)] = &[
("t3.medium", "amd64", 2, 4, 0.0416, false, ""),
("t3.large", "amd64", 2, 8, 0.0832, false, ""),
@@ -804,6 +804,13 @@ impl Aws {
if let Some(g) = group {
req = req.security_group_ids(g);
}
+ if spec.nested_virtualization {
+ req = req.cpu_options(
+ CpuOptionsRequest::builder()
+ .nested_virtualization(NestedVirtualizationSpecification::Enabled)
+ .build(),
+ );
+ }
req
}
@@ -1523,24 +1530,32 @@ impl CloudApi for Aws {
let regions = self.capabilities(conn).regions;
let mut v: Vec<HostType> = HOST_TYPES
.iter()
- .map(|(name, arch, cpus, gb, usd, kvm, gpu)| HostType {
- name: (*name).into(),
- arch: (*arch).into(),
- vcpus: *cpus,
- memory_mb: gb * 1024,
- disk_gb: if gpu.is_empty() { HOST_DISK_GB } else { 100 },
- kvm: *kvm,
- gpu: (*gpu).into(),
- usd_per_hour: ((usd + extras_per_hour(HOST_DISK_GB)) * 10_000.0).round() / 10_000.0,
- regions: regions.clone(),
- boot_secs: if *kvm { 300 } else { 90 },
- // EC2's on-demand vCPU quotas: standard families, G, P.
- quota_pool: match name.chars().next() {
- Some('g') if !gpu.is_empty() => "g",
- Some('p') if !gpu.is_empty() => "p",
- _ => "standard",
+ .map(|(name, arch, cpus, gb, usd, bare_metal_kvm, gpu)| {
+ // Nested-KVM instances receive /dev/kvm only when Hosts::add
+ // enables it in the EC2 launch request. Advertise that
+ // capability here as well, or the elastic planner will never
+ // choose these cheaper x86 hosts for VM/microVM Spaces.
+ let kvm = *bare_metal_kvm || crate::hosts::nested_kvm("aws", name);
+ HostType {
+ name: (*name).into(),
+ arch: (*arch).into(),
+ vcpus: *cpus,
+ memory_mb: gb * 1024,
+ disk_gb: if gpu.is_empty() { HOST_DISK_GB } else { 100 },
+ kvm,
+ gpu: (*gpu).into(),
+ usd_per_hour: ((usd + extras_per_hour(HOST_DISK_GB)) * 10_000.0).round()
+ / 10_000.0,
+ regions: regions.clone(),
+ boot_secs: if kvm { 300 } else { 90 },
+ // EC2's on-demand vCPU quotas: standard families, G, P.
+ quota_pool: match name.chars().next() {
+ Some('g') if !gpu.is_empty() => "g",
+ Some('p') if !gpu.is_empty() => "p",
+ _ => "standard",
+ }
+ .into(),
}
- .into(),
})
.collect();
v.sort_by(|a, b| {
diff --git a/libs/cua/crates/cua-byoc/src/hosts.rs b/libs/cua/crates/cua-byoc/src/hosts.rs
index a6a20ac6f8..3e88d1e88e 100644
--- a/libs/cua/crates/cua-byoc/src/hosts.rs
+++ b/libs/cua/crates/cua-byoc/src/hosts.rs
@@ -220,6 +220,21 @@ pub fn supports_nested_virtualization(instance_type: &str) -> bool {
)
}
+/// Whether a host of `machine_type` on `provider` gets nested KVM: an AWS
+/// type that supports it, with the `windows-kvm` experiment on. Off (the
+/// default): none do, and hosts and the pool planner behave as before.
+pub fn nested_kvm(provider: &str, machine_type: &str) -> bool {
+ nested_kvm_when(
+ cua_vmm::experiments::enabled(cua_vmm::experiments::Gate::WindowsKvm),
+ provider,
+ machine_type,
+ )
+}
+
+fn nested_kvm_when(on: bool, provider: &str, machine_type: &str) -> bool {
+ on && provider == "aws" && supports_nested_virtualization(machine_type)
+}
+
fn sh_quote(s: &str) -> String {
format!("'{}'", s.replace('\'', r"'\''"))
}
@@ -975,8 +990,7 @@ impl Hosts {
name: &name,
relay: &relay.url,
provider: &req.provider,
- nested_virtualization: api.name() == "aws"
- && supports_nested_virtualization(&t.name),
+ nested_virtualization: nested_kvm(api.name(), &t.name),
env: &opts.env,
usd_per_hour: t.usd_per_hour,
ttl_secs: ttl,
@@ -996,7 +1010,7 @@ impl Hosts {
ttl_secs: ttl,
image_parameter: opts.image_parameter.clone(),
peer_ports: opts.peer_ports,
- nested_virtualization: api.name() == "aws" && supports_nested_virtualization(&t.name),
+ nested_virtualization: nested_kvm(api.name(), &t.name),
role: ROLE_HOST.into(),
region: region.clone(),
..Default::default()
@@ -1519,4 +1533,28 @@ mod tests {
sandbox.extra.remove(EXTRA_ROLE);
assert!(!is_host_record(&sandbox));
}
+
+ #[test]
+ fn nested_kvm_needs_aws_a_supported_type_and_the_experiment() {
+ assert!(supports_nested_virtualization("c7i.2xlarge"));
+ assert!(!supports_nested_virtualization("t3.large"));
+ assert!(nested_kvm_when(true, "aws", "c7i.2xlarge"));
+ // Experiment off (the default): never.
+ assert!(!nested_kvm_when(false, "aws", "c7i.2xlarge"));
+ assert!(!nested_kvm_when(true, "hetzner", "c7i.2xlarge"));
+ assert!(!nested_kvm_when(true, "aws", "t3.large"));
+ }
+
+ #[test]
+ fn qemu_installs_only_for_nested_hosts() {
+ let env: Vec<String> = vec![];
+ let src = Source::Release;
+ let mk = |nested| {
+ let mut b = boot(&src, &env);
+ b.nested_virtualization = nested;
+ user_data(&b)
+ };
+ assert!(mk(true).contains("qemu-system-x86 qemu-utils ovmf"));
+ assert!(!mk(false).contains("qemu-system-x86"));
+ }
}
diff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rs
index c386981338..b2fccaaf8b 100644
--- a/libs/cua/crates/cua-spaces/src/spaces.rs
+++ b/libs/cua/crates/cua-spaces/src/spaces.rs
@@ -2339,10 +2339,12 @@ impl Spaces {
}
let sink = opts.progress.take().map(|p| p.0);
let (key, on) = self.create_key(&mut opts)?;
- // A Mac cannot create a Windows VM locally yet. The selected host
- // may be a Linux or Windows machine that can run it, so do not apply
- // the coordinator's local runtime restriction to remote placement.
- if on == On::Local {
+ // A Mac cannot create a Windows VM locally yet. With the `windows-kvm`
+ // experiment on, the selected remote host may be a Linux or Windows
+ // machine that can run it, so the coordinator's local restriction
+ // applies only to local placement. Off: it applies everywhere.
+ if on == On::Local || !cua_vmm::experiments::enabled(cua_vmm::experiments::Gate::WindowsKvm)
+ {
refuse_windows_on_mac(opts.image.as_deref(), cfg!(target_os = "macos"))?;
}
// The record is written before the Space connects, and the volume
diff --git a/libs/cua/crates/cua-vmm/src/experiments.rs b/libs/cua/crates/cua-vmm/src/experiments.rs
index b441ef297b..57d7fc66c8 100644
--- a/libs/cua/crates/cua-vmm/src/experiments.rs
+++ b/libs/cua/crates/cua-vmm/src/experiments.rs
@@ -24,17 +24,21 @@ pub enum Gate {
/// Child Spaces: a Space makes Spaces of its own through a channel only
/// it can reach, inside a budget for its tree (`cua spaces children`).
ChildSpaces,
+ /// Windows on KVM: Windows Spaces from a Mac coordinator on a Cua-managed
+ /// AWS x86 pool host with nested KVM.
+ WindowsKvm,
}
impl Gate {
/// Every experiment, in the order they are listed.
- pub const ALL: &'static [Gate] = &[Gate::SecureSpaces, Gate::ChildSpaces];
+ pub const ALL: &'static [Gate] = &[Gate::SecureSpaces, Gate::ChildSpaces, Gate::WindowsKvm];
/// The stable name (`secure-spaces`): the file name, the CLI word.
pub fn name(self) -> &'static str {
match self {
Gate::SecureSpaces => "secure-spaces",
Gate::ChildSpaces => "child-spaces",
+ Gate::WindowsKvm => "windows-kvm",
}
}
@@ -43,6 +47,7 @@ impl Gate {
match self {
Gate::SecureSpaces => "secure_spaces",
Gate::ChildSpaces => "child_spaces",
+ Gate::WindowsKvm => "windows_kvm",
}
}
@@ -57,6 +62,7 @@ impl Gate {
match self {
Gate::SecureSpaces => "Secure Spaces",
Gate::ChildSpaces => "Child Spaces",
+ Gate::WindowsKvm => "Windows on KVM",
}
}
@@ -69,6 +75,9 @@ impl Gate {
Gate::ChildSpaces => {
"A Space can make Spaces of its own, inside a budget, and keep a command running."
}
+ Gate::WindowsKvm => {
+ "Windows Spaces from a Mac run on a Cua-managed cloud host with nested virtualization."
+ }
}
}
@@ -193,6 +202,21 @@ mod tests {
assert_eq!(super::switch(home.path(), g), Some(false));
}
+ #[test]
+ fn windows_kvm_is_a_named_experiment_and_off_by_default() {
+ let home = tempfile::tempdir().unwrap();
+ assert_eq!(Gate::parse("windows_kvm"), Some(Gate::WindowsKvm));
+ assert_eq!(Gate::WindowsKvm.name(), "windows-kvm");
+ assert!(Gate::ALL.contains(&Gate::WindowsKvm));
+ assert!(
+ !super::enabled_in(home.path(), Gate::WindowsKvm)
+ || super::env_says(
+ std::env::var(EXPERIMENTS_ENV).ok().as_deref(),
+ Gate::WindowsKvm
+ ) == Some(true)
+ );
+ }
+
#[test]
fn the_environment_list_names_experiments_on_or_off() {
let g = Gate::SecureSpaces;
diff --git a/tests/e2e/cua-sdk/rust/Cargo.lock b/tests/e2e/cua-sdk/rust/Cargo.lock
index 76b95753e0..04b7f621de 100644
--- a/tests/e2e/cua-sdk/rust/Cargo.lock
+++ b/tests/e2e/cua-sdk/rust/Cargo.lock
@@ -1378,6 +1378,7 @@ dependencies = [
"cua-host",
"cua-sandbox-core",
"cua-spacesd-client",
+ "cua-vmm",
"humantime",
"rand 0.9.5",
"reqwest 0.12.28",