Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cluster): place durable Spaces on the most reliable machine (#6279)
dduponted887bb55f0e
diff --git a/libs/cua/crates/cua-cluster/src/lib.rs b/libs/cua/crates/cua-cluster/src/lib.rsindex 123c599e25..a36fc7337c 100644--- a/libs/cua/crates/cua-cluster/src/lib.rs+++ b/libs/cua/crates/cua-cluster/src/lib.rs@@ -42,6 +42,8 @@ pub mod model;pub mod node;pub mod peers;pub mod placement;+pub mod power;+pub mod reliability;pub mod sysinfo;pub use model::{HostJobState, JobSpec};diff --git a/libs/cua/crates/cua-cluster/src/placement.rs b/libs/cua/crates/cua-cluster/src/placement.rsindex 01e7b478b7..1b5955c489 100644--- a/libs/cua/crates/cua-cluster/src/placement.rs+++ b/libs/cua/crates/cua-cluster/src/placement.rs@@ -16,6 +16,11 @@ pub enum Policy {/// The most loaded host that still fits first: packs work, so other/// hosts stay idle (and can sleep or be removed).Pack,+ /// A durable drive: the most reliable machine first (always on, then a+ /// laptop on power, then one on battery; never the laptop the user is+ /// closing), spread by free capacity among equals. Capacity still gates:+ /// a machine that does not fit is not chosen.+ Durable,}impl Policy {@@ -24,6 +29,7 @@ impl Policy {match s.trim().to_ascii_lowercase().as_str() {"" | "spread" => Some(Self::Spread),"pack" | "binpack" => Some(Self::Pack),+ "durable" => Some(Self::Durable),_ => None,}}@@ -99,6 +105,10 @@ pub struct Candidate {/// load average). Only lowers its score, among hosts that fit; it never/// makes work misfit.pub busy: f64,+ /// What the machine says about its own reliability ([`crate::reliability`]),+ /// when it does; [`Policy::Durable`] scores a machine that does not say as+ /// [`crate::reliability::UNKNOWN`].+ pub reliability: Option<crate::reliability::Facts>,}/// How far elastic (not yet running) capacity scores below every host that@@ -239,6 +249,12 @@ impl Candidate {));}if !need.kind.is_empty() && !s.kinds.contains(&need.kind) {+ if need.kind == "microvm" {+ return Some(format!(+ "{} does not create microvm Spaces (they need KVM on Linux or Apple Containerization on macOS 26; a gVisor container works there)",+ self.name+ ));+ }return Some(format!("{} does not create {} Spaces",self.name, need.kind@@ -292,6 +308,13 @@ impl Candidate {None}+ /// How reliable this machine says it is (or [`crate::reliability::UNKNOWN`]).+ pub fn reliability_score(&self) -> f64 {+ self.reliability+ .as_ref()+ .map_or(crate::reliability::UNKNOWN, crate::reliability::score) as f64+ }+/// Whether a ready warm-pool member of `need`'s shape waits here.pub fn warm_for(&self, need: &Need) -> bool {!need.shape.is_empty()@@ -343,6 +366,11 @@ impl Candidate {let mut s = match policy {Policy::Spread => free_after,Policy::Pack => 1.0 - free_after,+ // Reliability in whole classes (1000 apart) dominates; free+ // capacity (0..1) only orders machines of one class.+ Policy::Durable => {+ self.reliability_score() / crate::reliability::STEP as f64 + free_after+ }};// A host already compressing or swapping goes after the others.if self.measured().is_some_and(|m| m.memory_pressure == 1) {@@ -409,6 +437,13 @@ pub fn rank(cands: &[Candidate], need: &Need, policy: Policy) -> Vec<usize> {scored.into_iter().map(|(i, _)| i).collect()}+/// Why each machine that was not chosen was skipped, in plain words (the+/// ones that fit are not listed): "Mac mini is arm64, not amd64", "box has+/// no KVM ...". Used to tell a user where their durable agent did not go.+pub fn skipped(cands: &[Candidate], need: &Need) -> Vec<String> {+ cands.iter().filter_map(|c| c.misfit(need)).collect()+}+/// Why nothing fits, for people: each host's reason.pub fn why_nothing_fits(cands: &[Candidate], need: &Need) -> String {if cands.is_empty() {@@ -945,4 +980,186 @@ mod tests {assert!(m.starts_with("no machine has room for a 2-CPU, 4 GiB Space now (this machine: "));assert!(m.contains("box: 0 of 16 CPUs free"), "{m}");}++ // ---- durable drives follow reliability ---------------------------------++ use crate::reliability::Facts;++ fn machine(name: &str, cpu_free: u32, f: Option<Facts>) -> Candidate {+ Candidate {+ reliability: f,+ ..host(name, cpu_free, 16384)+ }+ }++ fn laptop(on_battery: bool, user_active: bool) -> Facts {+ Facts {+ has_battery: true,+ on_battery,+ battery_pct: Some(90),+ user_active,+ ..Default::default()+ }+ }++ #[test]+ fn a_durable_drive_goes_to_the_machine_least_likely_to_stop() {+ let cands = [+ // The laptop the user is typing on, and the emptiest machine.+ machine("macbook", 16000, Some(laptop(false, true))),+ machine("studio", 8000, Some(Facts::default())),+ machine("old-air", 15000, Some(laptop(true, false))),+ ];+ let n = need(2000, 4096);+ // Spread alone would pick the emptiest (the laptop in use).+ let spread = rank(&cands, &n, Policy::Spread);+ assert_eq!(cands[spread[0]].name, "macbook");+ let durable = rank(&cands, &n, Policy::Durable);+ let names: Vec<_> = durable.iter().map(|i| cands[*i].name.as_str()).collect();+ assert_eq!(names, ["studio", "old-air", "macbook"]);+ }++ #[test]+ fn capacity_still_gates_a_durable_drive() {+ let cands = [+ machine("studio", 500, Some(Facts::default())), // no room+ machine("macbook", 16000, Some(laptop(false, false))),+ ];+ let d = rank(&cands, &need(2000, 4096), Policy::Durable);+ assert_eq!(d.len(), 1);+ assert_eq!(cands[d[0]].name, "macbook");+ }++ #[test]+ fn the_only_machine_is_used_even_if_it_is_the_laptop_in_use() {+ let cands = [machine("macbook", 16000, Some(laptop(true, true)))];+ assert_eq!(rank(&cands, &need(2000, 4096), Policy::Durable), vec![0]);+ }++ #[test]+ fn a_machine_that_does_not_say_scores_between_plugged_and_battery() {+ let cands = [+ machine("unknown", 16000, None),+ machine("plugged", 16000, Some(laptop(false, false))),+ machine("battery", 16000, Some(laptop(true, false))),+ ];+ let d = rank(&cands, &need(2000, 4096), Policy::Durable);+ let names: Vec<_> = d.iter().map(|i| cands[*i].name.as_str()).collect();+ assert_eq!(names, ["plugged", "unknown", "battery"]);+ assert_eq!(Policy::parse("durable"), Some(Policy::Durable));+ }++ // ---- mixed architectures, KVM on and off, BYOC linked or not ---------++ fn node(+ name: &str,+ arch: &str,+ os: &str,+ kinds: &[&str],+ elastic: bool,+ f: Facts,+ ) -> Candidate {+ Candidate {+ name: name.into(),+ ready: true,+ os: os.into(),+ elastic,+ reliability: Some(f),+ status: Some(pb::HostNodeStatus {+ millicpus_total: 8000,+ millicpus_free: 8000,+ memory_total_mb: 16384,+ memory_free_mb: 16384,+ arch: arch.into(),+ kinds: kinds.iter().map(|k| k.to_string()).collect(),+ runtimes: vec!["gvisor".into(), "runc".into()],+ ..Default::default()+ }),+ ..Default::default()+ }+ }++ #[test]+ fn a_mixed_cluster_places_durable_work_by_arch_kvm_and_kind_and_says_why() {+ let always_on = Facts::default();+ let cands = [+ node(+ "mac-studio",+ "aarch64",+ "macos",+ &["container", "vm", "microvm"],+ false,+ always_on.clone(),+ ),+ node(+ "linux-box",+ "x86_64",+ "linux",+ &["container"],+ false,+ always_on.clone(),+ ),+ node(+ "kvm-box",+ "x86_64",+ "linux",+ &["container", "microvm"],+ false,+ Facts {+ has_battery: false,+ ..always_on.clone()+ },+ ),+ node(+ "cloud-arm",+ "aarch64",+ "linux",+ &["container", "microvm"],+ true,+ always_on.clone(),+ ),+ node(+ "macbook",+ "aarch64",+ "macos",+ &["container", "vm", "microvm"],+ false,+ laptop(false, true),+ ),+ ];+ // A microVM wants KVM or Containerization: the box without it is+ // skipped with the reason, and the elastic cloud machine goes last.+ let mut need = need(2000, 4096);+ need.kind = "microvm".into();+ let order = rank(&cands, &need, Policy::Durable);+ let names: Vec<_> = order.iter().map(|i| cands[*i].name.as_str()).collect();+ assert_eq!(&names[..2], ["kvm-box", "mac-studio"], "{names:?}");+ assert!(!names.contains(&"linux-box"));+ let why = skipped(&cands, &need);+ assert!(+ why.iter()+ .any(|w| w.contains("linux-box") && w.contains("KVM") && w.contains("gVisor")),+ "{why:?}"+ );+ // An amd64-only workload skips the arm machines and says so.+ let mut amd = need.clone();+ amd.kind = "container".into();+ amd.arch = "amd64".into();+ let order = rank(&cands, &amd, Policy::Durable);+ let names: Vec<_> = order.iter().map(|i| cands[*i].name.as_str()).collect();+ assert_eq!(names, ["kvm-box", "linux-box"], "{names:?}");+ let why = skipped(&cands, &amd);+ assert!(+ why.iter()+ .any(|w| w.contains("mac-studio") && w.contains("amd64")),+ "{why:?}"+ );+ // No arch asked: every machine that fits is a candidate, the most+ // reliable always-on machines first and the laptop in use last.+ amd.arch = String::new();+ let order = rank(&cands, &amd, Policy::Durable);+ let names: Vec<_> = order.iter().map(|i| cands[*i].name.as_str()).collect();+ assert_eq!(names.first().copied(), Some("kvm-box"));+ assert_eq!(names.last().copied(), Some("macbook"));+ }}diff --git a/libs/cua/crates/cua-cluster/src/power.rs b/libs/cua/crates/cua-cluster/src/power.rsnew file mode 100644index 0000000000..ab617b728e--- /dev/null+++ b/libs/cua/crates/cua-cluster/src/power.rs@@ -0,0 +1,150 @@+//! Keeping this machine awake while a durable run needs it, with a reason+//! anyone can see (`pmset -g assertions` on a Mac lists it by name).+//!+//! An idle-sleep assertion stops the machine sleeping because nobody touched+//! it. It does not stop a laptop sleeping when its lid closes, unless it is+//! plugged in with an external display (clamshell mode). Callers that tell+//! the user what will happen must say so ([`LID_NOTE`]).++use std::sync::Mutex;++/// What to tell the user when the only machine is a laptop.+pub const LID_NOTE: &str =+ "Closing the lid still puts a laptop to sleep unless it is plugged in with a display attached.";++static REASONS: Mutex<Vec<(u64, String)>> = Mutex::new(Vec::new());+static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);++/// The reasons this process currently keeps the machine awake.+pub fn active_reasons() -> Vec<String> {+ REASONS+ .lock()+ .expect("reasons")+ .iter()+ .map(|(_, r)| r.clone())+ .collect()+}++/// Holds the machine awake until dropped.+#[derive(Debug)]+pub struct SleepGuard {+ id: u64,+ #[allow(dead_code)]+ os: Os,+}++impl SleepGuard {+ /// Keeps the machine from sleeping on idle for `reason` (shown to the+ /// user: "Cua: the durable agent ada is running").+ pub fn hold(reason: &str) -> SleepGuard {+ let id = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);+ REASONS+ .lock()+ .expect("reasons")+ .push((id, reason.to_string()));+ SleepGuard {+ id,+ os: Os::acquire(reason),+ }+ }+}++impl Drop for SleepGuard {+ fn drop(&mut self) {+ REASONS+ .lock()+ .expect("reasons")+ .retain(|(i, _)| *i != self.id);+ }+}++#[cfg(target_os = "macos")]+mod imp {+ use std::ffi::{CString, c_char, c_void};++ #[link(name = "IOKit", kind = "framework")]+ unsafe extern "C" {+ fn IOPMAssertionCreateWithName(+ kind: *const c_void,+ level: u32,+ name: *const c_void,+ id: *mut u32,+ ) -> i32;+ fn IOPMAssertionRelease(id: u32) -> i32;+ }++ #[link(name = "CoreFoundation", kind = "framework")]+ unsafe extern "C" {+ fn CFStringCreateWithCString(+ alloc: *const c_void,+ s: *const c_char,+ encoding: u32,+ ) -> *const c_void;+ fn CFRelease(cf: *const c_void);+ }++ const UTF8: u32 = 0x0800_0100;+ const LEVEL_ON: u32 = 255;++ #[derive(Debug)]+ pub struct Os(Option<u32>);++ fn cf(s: &str) -> *const c_void {+ let c = CString::new(s.replace('\0', " ")).unwrap_or_default();+ // SAFETY: a valid C string; the result is released by the caller.+ unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) }+ }++ impl Os {+ pub fn acquire(reason: &str) -> Os {+ let (kind, name) = (cf("PreventUserIdleSystemSleep"), cf(reason));+ let mut id = 0u32;+ // SAFETY: valid CFStrings and a valid out pointer.+ let r = unsafe { IOPMAssertionCreateWithName(kind, LEVEL_ON, name, &mut id) };+ // SAFETY: both were created above and are not used again.+ unsafe {+ CFRelease(kind);+ CFRelease(name);+ }+ Os((r == 0).then_some(id))+ }+ }++ impl Drop for Os {+ fn drop(&mut self) {+ if let Some(id) = self.0 {+ // SAFETY: an assertion id this process created.+ unsafe { IOPMAssertionRelease(id) };+ }+ }+ }+}++#[cfg(not(target_os = "macos"))]+mod imp {+ #[derive(Debug)]+ pub struct Os;++ impl Os {+ pub fn acquire(_reason: &str) -> Os {+ Os+ }+ }+}++use imp::Os;++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn a_guard_shows_its_reason_until_dropped() {+ let before = active_reasons().len();+ let g = SleepGuard::hold("Cua: the durable agent ada is running");+ assert!(active_reasons().contains(&"Cua: the durable agent ada is running".to_string()));+ assert_eq!(active_reasons().len(), before + 1);+ drop(g);+ assert_eq!(active_reasons().len(), before);+ }+}diff --git a/libs/cua/crates/cua-cluster/src/reliability.rs b/libs/cua/crates/cua-cluster/src/reliability.rsnew file mode 100644index 0000000000..5bf9a128a8--- /dev/null+++ b/libs/cua/crates/cua-cluster/src/reliability.rs@@ -0,0 +1,435 @@+//! How much a machine can be trusted to keep running a durable drive.+//!+//! The question a user never wants to answer: "which of my machines should+//! run this agent?" The answer is the machine least likely to stop: a desktop+//! that is always on, then a laptop on power, then a laptop on battery; never+//! the laptop the user is about to close. Each machine measures a few facts+//! about itself ([`Facts`], [`probe`]) and turns them into one number+//! ([`score`]) that placement ([`crate::placement::Policy::Durable`]) and the+//! Volume's choice of where to keep its copies both use, so a durable drive+//! and its files end up on the same machine.+//!+//! The number is a ladder of classes 1000 apart with small terms inside a+//! class (less than one class in total), so a small change never moves a+//! drive: only a different kind of machine, or the same machine changing+//! state (unplugged, lid about to close), does.++use serde::{Deserialize, Serialize};++/// A class step: scores this far apart are a different kind of machine.+pub const STEP: u64 = 1000;++/// What the machine's network is.+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum Link {+ #[default]+ Unknown,+ Wired,+ Wifi,+ /// A phone's hotspot or another metered link.+ Metered,+}++/// What a machine knows about itself.+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]+pub struct Facts {+ /// A laptop (it has a battery and a lid).+ pub has_battery: bool,+ /// Running on battery right now.+ pub on_battery: bool,+ /// Battery charge, 0 to 100, when there is a battery.+ #[serde(default)]+ pub battery_pct: Option<u8>,+ /// A durable run holds a sleep assertion here.+ #[serde(default)]+ pub sleep_prevented: bool,+ /// The user is working at this machine (input in the last 5 minutes).+ #[serde(default)]+ pub user_active: bool,+ /// Seconds since boot.+ #[serde(default)]+ pub uptime_secs: u64,+ /// Free disk where the volume lives, in GB.+ #[serde(default)]+ pub free_disk_gb: u64,+ #[serde(default)]+ pub link: Link,+ /// Times its peers lost it recently (from the Volume's failure detector).+ #[serde(default)]+ pub recent_flaps: u32,+ /// A dedicated host (`cua host setup --dedicated`, a cloud machine added+ /// to the account): always on by construction.+ #[serde(default)]+ pub dedicated: bool,+}++impl Facts {+ /// Whether this machine sleeps when its lid closes.+ pub fn sleep_prone(&self) -> bool {+ self.has_battery && !self.dedicated+ }+}++/// The reliability of a machine: bigger keeps the drive. See the module+/// docs for the ladder.+pub fn score(f: &Facts) -> u64 {+ let class: i64 = if f.dedicated || !f.has_battery {+ 9000+ } else if !f.on_battery {+ 5000+ } else if f.battery_pct.is_some_and(|p| p < 20) {+ 1000+ } else {+ 2000+ };+ let mut s = class;+ // A laptop the user is sitting at is the one about to be closed; a+ // desktop they work at only loses a tie.+ if f.user_active {+ s -= if f.sleep_prone() { 3500 } else { 300 };+ }+ s += i64::from(f.sleep_prevented) * 400;+ s += match f.link {+ Link::Wired => 250,+ Link::Wifi | Link::Unknown => 0,+ Link::Metered => -250,+ };+ s -= i64::from(f.recent_flaps.min(5)) * 60;+ s += (f.uptime_secs / 86_400).min(10) as i64 * 10;+ s += match f.free_disk_gb {+ 0 => 0,+ 1..=19 => 0,+ 20..=99 => 25,+ _ => 50,+ };+ s.max(0) as u64+}++/// What a machine we know nothing about scores: between a laptop on power+/// and a laptop on battery, so a known good desktop wins and a known weak+/// laptop loses.+pub const UNKNOWN: u64 = 4000;++/// A sentence for the user about why this machine would (not) keep a drive.+pub fn describe(f: &Facts) -> String {+ if f.dedicated {+ "always on".into()+ } else if !f.has_battery {+ if f.user_active {+ "a desktop you are using".into()+ } else {+ "a desktop".into()+ }+ } else if f.user_active && f.sleep_prone() {+ "a laptop you are using".into()+ } else if f.on_battery {+ "a laptop on battery".into()+ } else {+ "a laptop on power".into()+ }+}++/// Reads this machine's facts (best effort; unknown things stay default).+// Per-OS blocks fill different fields, so a struct initializer does not fit.+#[allow(clippy::field_reassign_with_default)]+pub fn probe() -> Facts {+ let mut f = Facts::default();+ #[cfg(target_os = "macos")]+ {+ if let Some(text) = run("/usr/bin/pmset", &["-g", "batt"]) {+ let (has, on_battery, pct) = parse_pmset_batt(&text);+ f.has_battery = has;+ f.on_battery = on_battery;+ f.battery_pct = pct;+ }+ if let Some(text) = run("/usr/sbin/ioreg", &["-c", "IOHIDSystem", "-d", "4"]) {+ f.user_active = parse_hid_idle_secs(&text).is_some_and(|s| s < 300);+ }+ if let Some(text) = run("/usr/sbin/sysctl", &["-n", "kern.boottime"])+ && let Some(boot) = parse_boottime(&text)+ {+ f.uptime_secs = now_secs().saturating_sub(boot);+ }+ f.link = macos_link();+ }+ #[cfg(target_os = "linux")]+ {+ let (has, on_battery, pct) = linux_power();+ f.has_battery = has;+ f.on_battery = on_battery;+ f.battery_pct = pct;+ if let Ok(t) = std::fs::read_to_string("/proc/uptime") {+ f.uptime_secs = t+ .split_whitespace()+ .next()+ .and_then(|s| s.parse::<f64>().ok())+ .map_or(0, |s| s as u64);+ }+ }+ f.free_disk_gb = free_disk_gb().unwrap_or(0);+ f+}++#[cfg(target_os = "macos")]+fn now_secs() -> u64 {+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .map_or(0, |d| d.as_secs())+}++#[cfg(target_os = "macos")]+fn run(bin: &str, args: &[&str]) -> Option<String> {+ let out = std::process::Command::new(bin).args(args).output().ok()?;+ out.status+ .success()+ .then(|| String::from_utf8_lossy(&out.stdout).into_owned())+}++/// `pmset -g batt`: (has a battery, running on battery, charge).+pub fn parse_pmset_batt(text: &str) -> (bool, bool, Option<u8>) {+ let on_battery = text.contains("'Battery Power'");+ let mut has = false;+ let mut pct = None;+ for line in text.lines() {+ if line.contains("InternalBattery") {+ has = true;+ pct = line+ .split('\t')+ .nth(1)+ .or(Some(line))+ .and_then(|l| l.split('%').next())+ .and_then(|l| l.split_whitespace().last())+ .and_then(|n| n.parse::<u8>().ok());+ }+ }+ (has, has && on_battery, pct)+}++/// `ioreg -c IOHIDSystem`: seconds since the last keyboard or mouse input.+pub fn parse_hid_idle_secs(text: &str) -> Option<u64> {+ let line = text.lines().find(|l| l.contains("HIDIdleTime"))?;+ let ns: u64 = line.rsplit('=').next()?.trim().parse().ok()?;+ Some(ns / 1_000_000_000)+}++/// `sysctl -n kern.boottime`: `{ sec = 1700000000, usec = 0 } ...`.+pub fn parse_boottime(text: &str) -> Option<u64> {+ let rest = text.split("sec =").nth(1)?;+ rest.trim_start()+ .split(|c: char| !c.is_ascii_digit())+ .next()?+ .parse()+ .ok()+}++#[cfg(target_os = "macos")]+fn macos_link() -> Link {+ let Some(route) = run("/sbin/route", &["-n", "get", "default"]) else {+ return Link::Unknown;+ };+ let Some(iface) = route.lines().find_map(|l| {+ l.trim()+ .strip_prefix("interface:")+ .map(|s| s.trim().to_string())+ }) else {+ return Link::Unknown;+ };+ let Some(ports) = run("/usr/sbin/networksetup", &["-listallhardwareports"]) else {+ return Link::Unknown;+ };+ link_of_port(&ports, &iface)+}++/// Which kind of port `iface` is, from `networksetup -listallhardwareports`.+pub fn link_of_port(ports: &str, iface: &str) -> Link {+ let mut name = "";+ for line in ports.lines() {+ if let Some(n) = line.strip_prefix("Hardware Port:") {+ name = n.trim();+ }+ if let Some(d) = line.strip_prefix("Device:")+ && d.trim() == iface+ {+ let n = name.to_ascii_lowercase();+ return if n.contains("wi-fi") || n.contains("airport") {+ Link::Wifi+ } else if n.contains("iphone")+ || n.contains("ipad")+ || n.contains("usb") && n.contains("tether")+ {+ Link::Metered+ } else if n.contains("ethernet") || n.contains("thunderbolt") || n.contains("lan") {+ Link::Wired+ } else {+ Link::Unknown+ };+ }+ }+ Link::Unknown+}++#[cfg(target_os = "linux")]+fn linux_power() -> (bool, bool, Option<u8>) {+ let mut has = false;+ let mut on_battery = false;+ let mut pct = None;+ if let Ok(rd) = std::fs::read_dir("/sys/class/power_supply") {+ for e in rd.flatten() {+ let p = e.path();+ let kind = std::fs::read_to_string(p.join("type")).unwrap_or_default();+ if kind.trim() == "Battery" {+ has = true;+ let st = std::fs::read_to_string(p.join("status")).unwrap_or_default();+ if st.trim() == "Discharging" {+ on_battery = true;+ }+ pct = std::fs::read_to_string(p.join("capacity"))+ .ok()+ .and_then(|c| c.trim().parse().ok());+ }+ }+ }+ (has, on_battery, pct)+}++#[cfg(unix)]+fn free_disk_gb() -> Option<u64> {+ let home = std::env::var_os("HOME").map(std::path::PathBuf::from)?;+ let c = std::ffi::CString::new(home.to_string_lossy().as_bytes()).ok()?;+ // SAFETY: an all-zero statvfs is a valid out value.+ let mut st: libc::statvfs = unsafe { std::mem::zeroed() };+ // SAFETY: `c` is a valid C string and `st` a valid out pointer.+ if unsafe { libc::statvfs(c.as_ptr(), &mut st) } == 0 {+ Some((st.f_bavail as u64).saturating_mul(st.f_frsize as u64) >> 30)+ } else {+ None+ }+}++#[cfg(not(unix))]+fn free_disk_gb() -> Option<u64> {+ None+}++#[cfg(test)]+mod tests {+ use super::*;++ fn laptop(on_battery: bool) -> Facts {+ Facts {+ has_battery: true,+ on_battery,+ battery_pct: Some(80),+ ..Default::default()+ }+ }++ #[test]+ fn a_desktop_beats_a_plugged_laptop_beats_a_laptop_on_battery() {+ let desktop = Facts::default();+ let plugged = laptop(false);+ let battery = laptop(true);+ assert!(score(&desktop) >= score(&plugged) + STEP);+ assert!(score(&plugged) >= score(&battery) + STEP);+ let low = Facts {+ battery_pct: Some(10),+ ..laptop(true)+ };+ assert!(score(&battery) >= score(&low) + STEP / 2);+ }++ #[test]+ fn the_laptop_the_user_is_using_loses_but_a_desktop_only_loses_a_tie() {+ let mine = Facts {+ user_active: true,+ ..laptop(false)+ };+ let away = laptop(true);+ assert!(+ score(&away) > score(&mine),+ "the laptop in use is the one about to close"+ );+ let desk_used = Facts {+ user_active: true,+ ..Default::default()+ };+ assert!(score(&desk_used) > score(&laptop(false)));+ assert!(score(&desk_used) < score(&Facts::default()));+ }++ #[test]+ fn small_terms_never_cross_a_class() {+ // The best a laptop on battery can do is below the worst a plugged+ // laptop does.+ let best_battery = Facts {+ sleep_prevented: true,+ link: Link::Wired,+ uptime_secs: 30 * 86_400,+ free_disk_gb: 500,+ ..laptop(true)+ };+ let worst_plugged = Facts {+ link: Link::Metered,+ recent_flaps: 9,+ ..laptop(false)+ };+ assert!(score(&best_battery) < score(&worst_plugged));+ let best_plugged = Facts {+ sleep_prevented: true,+ link: Link::Wired,+ uptime_secs: 30 * 86_400,+ free_disk_gb: 500,+ ..laptop(false)+ };+ let worst_desktop = Facts {+ link: Link::Metered,+ recent_flaps: 9,+ ..Default::default()+ };+ assert!(score(&best_plugged) < score(&worst_desktop));+ // An unknown machine sits between plugged and battery.+ assert!(score(&laptop(true)) < UNKNOWN && UNKNOWN < score(&laptop(false)));+ }++ #[test]+ fn a_dedicated_host_is_always_a_desktop() {+ let f = Facts {+ dedicated: true,+ has_battery: true,+ on_battery: true,+ ..Default::default()+ };+ assert_eq!(score(&f), 9000);+ assert!(!f.sleep_prone());+ }++ #[test]+ fn pmset_and_ioreg_and_sysctl_output_parse() {+ let ac = "Now drawing from 'AC Power'\n -InternalBattery-0 (id=4653155)\t100%; charged; 0:00 remaining present: true\n";+ assert_eq!(parse_pmset_batt(ac), (true, false, Some(100)));+ let bat = "Now drawing from 'Battery Power'\n -InternalBattery-0 (id=1)\t43%; discharging; 3:12 remaining present: true\n";+ assert_eq!(parse_pmset_batt(bat), (true, true, Some(43)));+ let desk = "Now drawing from 'AC Power'\n";+ assert_eq!(parse_pmset_batt(desk), (false, false, None));+ let hid = " | | \"HIDIdleTime\" = 125000000000\n";+ assert_eq!(parse_hid_idle_secs(hid), Some(125));+ assert_eq!(+ parse_boottime("{ sec = 1700000000, usec = 123 } Tue Nov 14 22:13:20 2023"),+ Some(1_700_000_000)+ );+ let ports = "Hardware Port: Wi-Fi\nDevice: en0\nEthernet Address: x\n\nHardware Port: Thunderbolt Ethernet Slot 1\nDevice: en5\n\nHardware Port: iPhone USB\nDevice: en7\n";+ assert_eq!(link_of_port(ports, "en0"), Link::Wifi);+ assert_eq!(link_of_port(ports, "en5"), Link::Wired);+ assert_eq!(link_of_port(ports, "en7"), Link::Metered);+ assert_eq!(link_of_port(ports, "en9"), Link::Unknown);+ }++ #[test]+ fn this_machine_can_be_probed() {+ let f = probe();+ // Whatever it is, it scores and describes itself.+ assert!(!describe(&f).is_empty());+ let _ = score(&f);+ }+}diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rsindex 83a28c2897..8e4db943b2 100644--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs@@ -66,6 +66,10 @@ pub struct DurableSpec {/// No desktop (`SpaceCreate::headless`): kept by every copy.#[serde(default)]pub headless: bool,+ /// CPU architecture every copy must run on (`amd64`, `arm64`); empty:+ /// any (the default: a durable Space may move between architectures).+ #[serde(default)]+ pub arch: String,}/// What a Space's `.cua/setup` did.@@ -535,6 +539,8 @@ impl Coordinator {cpus: row.spec.cpus,memory_mb: row.spec.memory_mb,headless: row.spec.headless,+ durable: true,+ arch: row.spec.arch.clone(),};match self.place_space_excluding(req, &exclude).await {Ok(p) => {diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rsindex 3b89767c39..4eb75704f9 100644--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs@@ -88,6 +88,25 @@ pub struct Config {/// Runs a durable Space's `.cua/setup` after each placement (the first/// copy and every re-placement from scratch). `None`: no setup step.pub space_setup: Option<durable::SetupHook>,+ /// What each machine says about its own reliability (a laptop on+ /// battery, a desktop that is always on), by machine name. A durable+ /// Space goes to the most reliable machine that has room+ /// ([`Policy::Durable`]); a machine that says nothing scores as+ /// unknown. `None`: nothing is known (capacity decides).+ pub reliability: Option<ReliabilityHook>,+}++/// Looks up a machine's [`cua_cluster::reliability::Facts`] by name.+#[derive(Clone)]+pub struct ReliabilityHook(pub Arc<FactsOf>);++/// Facts of a machine by name.+pub type FactsOf = dyn Fn(&str) -> Option<cua_cluster::reliability::Facts> + Send + Sync;++impl std::fmt::Debug for ReliabilityHook {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str("ReliabilityHook")+ }}impl Config {@@ -105,6 +124,7 @@ impl Config {default_max_attempts: 5,session_tick: Duration::from_secs(2),space_setup: None,+ reliability: None,}}}@@ -442,6 +462,9 @@ pub struct PlacedSpace {pub local_space: String,/// Its kind.pub kind: String,+ /// Machines that were not chosen and why, in plain words ("linux-box+ /// cannot run microVM Spaces ...", "mac-studio is aarch64, not amd64").+ pub skipped: Vec<String>,}/// What `place_space` needs.@@ -468,6 +491,14 @@ pub struct SpaceRequest {/// No desktop (`CreateHostSpaceRequest.headless`).#[serde(default)]pub headless: bool,+ /// A durable Space: placed on the most reliable machine that has room.+ #[serde(default)]+ pub durable: bool,+ /// CPU architecture the Space must run on (`amd64`, `arm64`); empty: any+ /// machine (a headless image is multi-arch; the workspace's built+ /// programs are not, which a takeover across architectures says).+ #[serde(default)]+ pub arch: String,}/// The coordinator.@@ -2680,7 +2711,7 @@ impl Coordinator {} else {String::new()},- arch: String::new(),+ arch: req.arch.clone(),on: String::new(),shape: cua_cluster::placement::warm_key(&req.image,@@ -2722,6 +2753,11 @@ impl Coordinator {elastic: false,pinned: n.row.state == "pinned",provider: n.row.provider.clone(),+ reliability: self+ .cfg+ .reliability+ .as_ref()+ .and_then(|h| (h.0)(&n.row.name)),..Default::default()},(n.link.clone(), n.row.direct.clone()),@@ -2729,7 +2765,12 @@ impl Coordinator {}).unzip()};- let order = placement::rank(&cands, &need, self.cfg.policy);+ let policy = if req.durable {+ Policy::Durable+ } else {+ self.cfg.policy+ };+ let order = placement::rank(&cands, &need, policy);if cands.is_empty() {let relayed = {let g = self.sched.lock().expect("sched");@@ -2750,6 +2791,7 @@ impl Coordinator {placement::why_nothing_fits(&cands, &need))));}+ let skipped = placement::skipped(&cands, &need);let mut errors = Vec::new();for i in order {let (link, direct) = &links[i];@@ -2782,6 +2824,7 @@ impl Coordinator {token: String::new(),local_space: space.local_space,kind: space.kind,+ skipped: skipped.clone(),});}Ok(_) if via_relay => {@@ -2795,6 +2838,7 @@ impl Coordinator {token: space.direct_token,local_space: space.local_space,kind: space.kind,+ skipped: skipped.clone(),});}Ok(_) => errors.push(format!("{}: created no direct port", cands[i].name)),