Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cluster): place durable Spaces on the most reliable machine (#6279)

dduponted887bb55f0e
diff --git a/libs/cua/crates/cua-cluster/src/lib.rs b/libs/cua/crates/cua-cluster/src/lib.rs
index 123c599e25..a36fc7337c 100644
--- a/libs/cua/crates/cua-cluster/src/lib.rs
+++ b/libs/cua/crates/cua-cluster/src/lib.rs
@@ -42,6 +42,8 @@ pub mod model;
pub mod node;
pub mod peers;
pub mod placement;
+pub mod power;
+pub mod reliability;
pub mod sysinfo;
pub use model::{HostJobState, JobSpec};
diff --git a/libs/cua/crates/cua-cluster/src/placement.rs b/libs/cua/crates/cua-cluster/src/placement.rs
index 01e7b478b7..1b5955c489 100644
--- a/libs/cua/crates/cua-cluster/src/placement.rs
+++ b/libs/cua/crates/cua-cluster/src/placement.rs
@@ -16,6 +16,11 @@ pub enum Policy {
/// The most loaded host that still fits first: packs work, so other
/// hosts stay idle (and can sleep or be removed).
Pack,
+ /// A durable drive: the most reliable machine first (always on, then a
+ /// laptop on power, then one on battery; never the laptop the user is
+ /// closing), spread by free capacity among equals. Capacity still gates:
+ /// a machine that does not fit is not chosen.
+ Durable,
}
impl Policy {
@@ -24,6 +29,7 @@ impl Policy {
match s.trim().to_ascii_lowercase().as_str() {
"" | "spread" => Some(Self::Spread),
"pack" | "binpack" => Some(Self::Pack),
+ "durable" => Some(Self::Durable),
_ => None,
}
}
@@ -99,6 +105,10 @@ pub struct Candidate {
/// load average). Only lowers its score, among hosts that fit; it never
/// makes work misfit.
pub busy: f64,
+ /// What the machine says about its own reliability ([`crate::reliability`]),
+ /// when it does; [`Policy::Durable`] scores a machine that does not say as
+ /// [`crate::reliability::UNKNOWN`].
+ pub reliability: Option<crate::reliability::Facts>,
}
/// How far elastic (not yet running) capacity scores below every host that
@@ -239,6 +249,12 @@ impl Candidate {
));
}
if !need.kind.is_empty() && !s.kinds.contains(&need.kind) {
+ if need.kind == "microvm" {
+ return Some(format!(
+ "{} does not create microvm Spaces (they need KVM on Linux or Apple Containerization on macOS 26; a gVisor container works there)",
+ self.name
+ ));
+ }
return Some(format!(
"{} does not create {} Spaces",
self.name, need.kind
@@ -292,6 +308,13 @@ impl Candidate {
None
}
+ /// How reliable this machine says it is (or [`crate::reliability::UNKNOWN`]).
+ pub fn reliability_score(&self) -> f64 {
+ self.reliability
+ .as_ref()
+ .map_or(crate::reliability::UNKNOWN, crate::reliability::score) as f64
+ }
+
/// Whether a ready warm-pool member of `need`'s shape waits here.
pub fn warm_for(&self, need: &Need) -> bool {
!need.shape.is_empty()
@@ -343,6 +366,11 @@ impl Candidate {
let mut s = match policy {
Policy::Spread => free_after,
Policy::Pack => 1.0 - free_after,
+ // Reliability in whole classes (1000 apart) dominates; free
+ // capacity (0..1) only orders machines of one class.
+ Policy::Durable => {
+ self.reliability_score() / crate::reliability::STEP as f64 + free_after
+ }
};
// A host already compressing or swapping goes after the others.
if self.measured().is_some_and(|m| m.memory_pressure == 1) {
@@ -409,6 +437,13 @@ pub fn rank(cands: &[Candidate], need: &Need, policy: Policy) -> Vec<usize> {
scored.into_iter().map(|(i, _)| i).collect()
}
+/// Why each machine that was not chosen was skipped, in plain words (the
+/// ones that fit are not listed): "Mac mini is arm64, not amd64", "box has
+/// no KVM ...". Used to tell a user where their durable agent did not go.
+pub fn skipped(cands: &[Candidate], need: &Need) -> Vec<String> {
+ cands.iter().filter_map(|c| c.misfit(need)).collect()
+}
+
/// Why nothing fits, for people: each host's reason.
pub fn why_nothing_fits(cands: &[Candidate], need: &Need) -> String {
if cands.is_empty() {
@@ -945,4 +980,186 @@ mod tests {
assert!(m.starts_with("no machine has room for a 2-CPU, 4 GiB Space now (this machine: "));
assert!(m.contains("box: 0 of 16 CPUs free"), "{m}");
}
+
+ // ---- durable drives follow reliability ---------------------------------
+
+ use crate::reliability::Facts;
+
+ fn machine(name: &str, cpu_free: u32, f: Option<Facts>) -> Candidate {
+ Candidate {
+ reliability: f,
+ ..host(name, cpu_free, 16384)
+ }
+ }
+
+ fn laptop(on_battery: bool, user_active: bool) -> Facts {
+ Facts {
+ has_battery: true,
+ on_battery,
+ battery_pct: Some(90),
+ user_active,
+ ..Default::default()
+ }
+ }
+
+ #[test]
+ fn a_durable_drive_goes_to_the_machine_least_likely_to_stop() {
+ let cands = [
+ // The laptop the user is typing on, and the emptiest machine.
+ machine("macbook", 16000, Some(laptop(false, true))),
+ machine("studio", 8000, Some(Facts::default())),
+ machine("old-air", 15000, Some(laptop(true, false))),
+ ];
+ let n = need(2000, 4096);
+ // Spread alone would pick the emptiest (the laptop in use).
+ let spread = rank(&cands, &n, Policy::Spread);
+ assert_eq!(cands[spread[0]].name, "macbook");
+ let durable = rank(&cands, &n, Policy::Durable);
+ let names: Vec<_> = durable.iter().map(|i| cands[*i].name.as_str()).collect();
+ assert_eq!(names, ["studio", "old-air", "macbook"]);
+ }
+
+ #[test]
+ fn capacity_still_gates_a_durable_drive() {
+ let cands = [
+ machine("studio", 500, Some(Facts::default())), // no room
+ machine("macbook", 16000, Some(laptop(false, false))),
+ ];
+ let d = rank(&cands, &need(2000, 4096), Policy::Durable);
+ assert_eq!(d.len(), 1);
+ assert_eq!(cands[d[0]].name, "macbook");
+ }
+
+ #[test]
+ fn the_only_machine_is_used_even_if_it_is_the_laptop_in_use() {
+ let cands = [machine("macbook", 16000, Some(laptop(true, true)))];
+ assert_eq!(rank(&cands, &need(2000, 4096), Policy::Durable), vec![0]);
+ }
+
+ #[test]
+ fn a_machine_that_does_not_say_scores_between_plugged_and_battery() {
+ let cands = [
+ machine("unknown", 16000, None),
+ machine("plugged", 16000, Some(laptop(false, false))),
+ machine("battery", 16000, Some(laptop(true, false))),
+ ];
+ let d = rank(&cands, &need(2000, 4096), Policy::Durable);
+ let names: Vec<_> = d.iter().map(|i| cands[*i].name.as_str()).collect();
+ assert_eq!(names, ["plugged", "unknown", "battery"]);
+ assert_eq!(Policy::parse("durable"), Some(Policy::Durable));
+ }
+
+ // ---- mixed architectures, KVM on and off, BYOC linked or not ---------
+
+ fn node(
+ name: &str,
+ arch: &str,
+ os: &str,
+ kinds: &[&str],
+ elastic: bool,
+ f: Facts,
+ ) -> Candidate {
+ Candidate {
+ name: name.into(),
+ ready: true,
+ os: os.into(),
+ elastic,
+ reliability: Some(f),
+ status: Some(pb::HostNodeStatus {
+ millicpus_total: 8000,
+ millicpus_free: 8000,
+ memory_total_mb: 16384,
+ memory_free_mb: 16384,
+ arch: arch.into(),
+ kinds: kinds.iter().map(|k| k.to_string()).collect(),
+ runtimes: vec!["gvisor".into(), "runc".into()],
+ ..Default::default()
+ }),
+ ..Default::default()
+ }
+ }
+
+ #[test]
+ fn a_mixed_cluster_places_durable_work_by_arch_kvm_and_kind_and_says_why() {
+ let always_on = Facts::default();
+ let cands = [
+ node(
+ "mac-studio",
+ "aarch64",
+ "macos",
+ &["container", "vm", "microvm"],
+ false,
+ always_on.clone(),
+ ),
+ node(
+ "linux-box",
+ "x86_64",
+ "linux",
+ &["container"],
+ false,
+ always_on.clone(),
+ ),
+ node(
+ "kvm-box",
+ "x86_64",
+ "linux",
+ &["container", "microvm"],
+ false,
+ Facts {
+ has_battery: false,
+ ..always_on.clone()
+ },
+ ),
+ node(
+ "cloud-arm",
+ "aarch64",
+ "linux",
+ &["container", "microvm"],
+ true,
+ always_on.clone(),
+ ),
+ node(
+ "macbook",
+ "aarch64",
+ "macos",
+ &["container", "vm", "microvm"],
+ false,
+ laptop(false, true),
+ ),
+ ];
+ // A microVM wants KVM or Containerization: the box without it is
+ // skipped with the reason, and the elastic cloud machine goes last.
+ let mut need = need(2000, 4096);
+ need.kind = "microvm".into();
+ let order = rank(&cands, &need, Policy::Durable);
+ let names: Vec<_> = order.iter().map(|i| cands[*i].name.as_str()).collect();
+ assert_eq!(&names[..2], ["kvm-box", "mac-studio"], "{names:?}");
+ assert!(!names.contains(&"linux-box"));
+ let why = skipped(&cands, &need);
+ assert!(
+ why.iter()
+ .any(|w| w.contains("linux-box") && w.contains("KVM") && w.contains("gVisor")),
+ "{why:?}"
+ );
+ // An amd64-only workload skips the arm machines and says so.
+ let mut amd = need.clone();
+ amd.kind = "container".into();
+ amd.arch = "amd64".into();
+ let order = rank(&cands, &amd, Policy::Durable);
+ let names: Vec<_> = order.iter().map(|i| cands[*i].name.as_str()).collect();
+ assert_eq!(names, ["kvm-box", "linux-box"], "{names:?}");
+ let why = skipped(&cands, &amd);
+ assert!(
+ why.iter()
+ .any(|w| w.contains("mac-studio") && w.contains("amd64")),
+ "{why:?}"
+ );
+ // No arch asked: every machine that fits is a candidate, the most
+ // reliable always-on machines first and the laptop in use last.
+ amd.arch = String::new();
+ let order = rank(&cands, &amd, Policy::Durable);
+ let names: Vec<_> = order.iter().map(|i| cands[*i].name.as_str()).collect();
+ assert_eq!(names.first().copied(), Some("kvm-box"));
+ assert_eq!(names.last().copied(), Some("macbook"));
+ }
}
diff --git a/libs/cua/crates/cua-cluster/src/power.rs b/libs/cua/crates/cua-cluster/src/power.rs
new file mode 100644
index 0000000000..ab617b728e
--- /dev/null
+++ b/libs/cua/crates/cua-cluster/src/power.rs
@@ -0,0 +1,150 @@
+//! Keeping this machine awake while a durable run needs it, with a reason
+//! anyone can see (`pmset -g assertions` on a Mac lists it by name).
+//!
+//! An idle-sleep assertion stops the machine sleeping because nobody touched
+//! it. It does not stop a laptop sleeping when its lid closes, unless it is
+//! plugged in with an external display (clamshell mode). Callers that tell
+//! the user what will happen must say so ([`LID_NOTE`]).
+
+use std::sync::Mutex;
+
+/// What to tell the user when the only machine is a laptop.
+pub const LID_NOTE: &str =
+ "Closing the lid still puts a laptop to sleep unless it is plugged in with a display attached.";
+
+static REASONS: Mutex<Vec<(u64, String)>> = Mutex::new(Vec::new());
+static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
+
+/// The reasons this process currently keeps the machine awake.
+pub fn active_reasons() -> Vec<String> {
+ REASONS
+ .lock()
+ .expect("reasons")
+ .iter()
+ .map(|(_, r)| r.clone())
+ .collect()
+}
+
+/// Holds the machine awake until dropped.
+#[derive(Debug)]
+pub struct SleepGuard {
+ id: u64,
+ #[allow(dead_code)]
+ os: Os,
+}
+
+impl SleepGuard {
+ /// Keeps the machine from sleeping on idle for `reason` (shown to the
+ /// user: "Cua: the durable agent ada is running").
+ pub fn hold(reason: &str) -> SleepGuard {
+ let id = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ REASONS
+ .lock()
+ .expect("reasons")
+ .push((id, reason.to_string()));
+ SleepGuard {
+ id,
+ os: Os::acquire(reason),
+ }
+ }
+}
+
+impl Drop for SleepGuard {
+ fn drop(&mut self) {
+ REASONS
+ .lock()
+ .expect("reasons")
+ .retain(|(i, _)| *i != self.id);
+ }
+}
+
+#[cfg(target_os = "macos")]
+mod imp {
+ use std::ffi::{CString, c_char, c_void};
+
+ #[link(name = "IOKit", kind = "framework")]
+ unsafe extern "C" {
+ fn IOPMAssertionCreateWithName(
+ kind: *const c_void,
+ level: u32,
+ name: *const c_void,
+ id: *mut u32,
+ ) -> i32;
+ fn IOPMAssertionRelease(id: u32) -> i32;
+ }
+
+ #[link(name = "CoreFoundation", kind = "framework")]
+ unsafe extern "C" {
+ fn CFStringCreateWithCString(
+ alloc: *const c_void,
+ s: *const c_char,
+ encoding: u32,
+ ) -> *const c_void;
+ fn CFRelease(cf: *const c_void);
+ }
+
+ const UTF8: u32 = 0x0800_0100;
+ const LEVEL_ON: u32 = 255;
+
+ #[derive(Debug)]
+ pub struct Os(Option<u32>);
+
+ fn cf(s: &str) -> *const c_void {
+ let c = CString::new(s.replace('\0', " ")).unwrap_or_default();
+ // SAFETY: a valid C string; the result is released by the caller.
+ unsafe { CFStringCreateWithCString(std::ptr::null(), c.as_ptr(), UTF8) }
+ }
+
+ impl Os {
+ pub fn acquire(reason: &str) -> Os {
+ let (kind, name) = (cf("PreventUserIdleSystemSleep"), cf(reason));
+ let mut id = 0u32;
+ // SAFETY: valid CFStrings and a valid out pointer.
+ let r = unsafe { IOPMAssertionCreateWithName(kind, LEVEL_ON, name, &mut id) };
+ // SAFETY: both were created above and are not used again.
+ unsafe {
+ CFRelease(kind);
+ CFRelease(name);
+ }
+ Os((r == 0).then_some(id))
+ }
+ }
+
+ impl Drop for Os {
+ fn drop(&mut self) {
+ if let Some(id) = self.0 {
+ // SAFETY: an assertion id this process created.
+ unsafe { IOPMAssertionRelease(id) };
+ }
+ }
+ }
+}
+
+#[cfg(not(target_os = "macos"))]
+mod imp {
+ #[derive(Debug)]
+ pub struct Os;
+
+ impl Os {
+ pub fn acquire(_reason: &str) -> Os {
+ Os
+ }
+ }
+}
+
+use imp::Os;
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn a_guard_shows_its_reason_until_dropped() {
+ let before = active_reasons().len();
+ let g = SleepGuard::hold("Cua: the durable agent ada is running");
+ assert!(active_reasons().contains(&"Cua: the durable agent ada is running".to_string()));
+ assert_eq!(active_reasons().len(), before + 1);
+ drop(g);
+ assert_eq!(active_reasons().len(), before);
+ }
+}
diff --git a/libs/cua/crates/cua-cluster/src/reliability.rs b/libs/cua/crates/cua-cluster/src/reliability.rs
new file mode 100644
index 0000000000..5bf9a128a8
--- /dev/null
+++ b/libs/cua/crates/cua-cluster/src/reliability.rs
@@ -0,0 +1,435 @@
+//! How much a machine can be trusted to keep running a durable drive.
+//!
+//! The question a user never wants to answer: "which of my machines should
+//! run this agent?" The answer is the machine least likely to stop: a desktop
+//! that is always on, then a laptop on power, then a laptop on battery; never
+//! the laptop the user is about to close. Each machine measures a few facts
+//! about itself ([`Facts`], [`probe`]) and turns them into one number
+//! ([`score`]) that placement ([`crate::placement::Policy::Durable`]) and the
+//! Volume's choice of where to keep its copies both use, so a durable drive
+//! and its files end up on the same machine.
+//!
+//! The number is a ladder of classes 1000 apart with small terms inside a
+//! class (less than one class in total), so a small change never moves a
+//! drive: only a different kind of machine, or the same machine changing
+//! state (unplugged, lid about to close), does.
+
+use serde::{Deserialize, Serialize};
+
+/// A class step: scores this far apart are a different kind of machine.
+pub const STEP: u64 = 1000;
+
+/// What the machine's network is.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum Link {
+ #[default]
+ Unknown,
+ Wired,
+ Wifi,
+ /// A phone's hotspot or another metered link.
+ Metered,
+}
+
+/// What a machine knows about itself.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Facts {
+ /// A laptop (it has a battery and a lid).
+ pub has_battery: bool,
+ /// Running on battery right now.
+ pub on_battery: bool,
+ /// Battery charge, 0 to 100, when there is a battery.
+ #[serde(default)]
+ pub battery_pct: Option<u8>,
+ /// A durable run holds a sleep assertion here.
+ #[serde(default)]
+ pub sleep_prevented: bool,
+ /// The user is working at this machine (input in the last 5 minutes).
+ #[serde(default)]
+ pub user_active: bool,
+ /// Seconds since boot.
+ #[serde(default)]
+ pub uptime_secs: u64,
+ /// Free disk where the volume lives, in GB.
+ #[serde(default)]
+ pub free_disk_gb: u64,
+ #[serde(default)]
+ pub link: Link,
+ /// Times its peers lost it recently (from the Volume's failure detector).
+ #[serde(default)]
+ pub recent_flaps: u32,
+ /// A dedicated host (`cua host setup --dedicated`, a cloud machine added
+ /// to the account): always on by construction.
+ #[serde(default)]
+ pub dedicated: bool,
+}
+
+impl Facts {
+ /// Whether this machine sleeps when its lid closes.
+ pub fn sleep_prone(&self) -> bool {
+ self.has_battery && !self.dedicated
+ }
+}
+
+/// The reliability of a machine: bigger keeps the drive. See the module
+/// docs for the ladder.
+pub fn score(f: &Facts) -> u64 {
+ let class: i64 = if f.dedicated || !f.has_battery {
+ 9000
+ } else if !f.on_battery {
+ 5000
+ } else if f.battery_pct.is_some_and(|p| p < 20) {
+ 1000
+ } else {
+ 2000
+ };
+ let mut s = class;
+ // A laptop the user is sitting at is the one about to be closed; a
+ // desktop they work at only loses a tie.
+ if f.user_active {
+ s -= if f.sleep_prone() { 3500 } else { 300 };
+ }
+ s += i64::from(f.sleep_prevented) * 400;
+ s += match f.link {
+ Link::Wired => 250,
+ Link::Wifi | Link::Unknown => 0,
+ Link::Metered => -250,
+ };
+ s -= i64::from(f.recent_flaps.min(5)) * 60;
+ s += (f.uptime_secs / 86_400).min(10) as i64 * 10;
+ s += match f.free_disk_gb {
+ 0 => 0,
+ 1..=19 => 0,
+ 20..=99 => 25,
+ _ => 50,
+ };
+ s.max(0) as u64
+}
+
+/// What a machine we know nothing about scores: between a laptop on power
+/// and a laptop on battery, so a known good desktop wins and a known weak
+/// laptop loses.
+pub const UNKNOWN: u64 = 4000;
+
+/// A sentence for the user about why this machine would (not) keep a drive.
+pub fn describe(f: &Facts) -> String {
+ if f.dedicated {
+ "always on".into()
+ } else if !f.has_battery {
+ if f.user_active {
+ "a desktop you are using".into()
+ } else {
+ "a desktop".into()
+ }
+ } else if f.user_active && f.sleep_prone() {
+ "a laptop you are using".into()
+ } else if f.on_battery {
+ "a laptop on battery".into()
+ } else {
+ "a laptop on power".into()
+ }
+}
+
+/// Reads this machine's facts (best effort; unknown things stay default).
+// Per-OS blocks fill different fields, so a struct initializer does not fit.
+#[allow(clippy::field_reassign_with_default)]
+pub fn probe() -> Facts {
+ let mut f = Facts::default();
+ #[cfg(target_os = "macos")]
+ {
+ if let Some(text) = run("/usr/bin/pmset", &["-g", "batt"]) {
+ let (has, on_battery, pct) = parse_pmset_batt(&text);
+ f.has_battery = has;
+ f.on_battery = on_battery;
+ f.battery_pct = pct;
+ }
+ if let Some(text) = run("/usr/sbin/ioreg", &["-c", "IOHIDSystem", "-d", "4"]) {
+ f.user_active = parse_hid_idle_secs(&text).is_some_and(|s| s < 300);
+ }
+ if let Some(text) = run("/usr/sbin/sysctl", &["-n", "kern.boottime"])
+ && let Some(boot) = parse_boottime(&text)
+ {
+ f.uptime_secs = now_secs().saturating_sub(boot);
+ }
+ f.link = macos_link();
+ }
+ #[cfg(target_os = "linux")]
+ {
+ let (has, on_battery, pct) = linux_power();
+ f.has_battery = has;
+ f.on_battery = on_battery;
+ f.battery_pct = pct;
+ if let Ok(t) = std::fs::read_to_string("/proc/uptime") {
+ f.uptime_secs = t
+ .split_whitespace()
+ .next()
+ .and_then(|s| s.parse::<f64>().ok())
+ .map_or(0, |s| s as u64);
+ }
+ }
+ f.free_disk_gb = free_disk_gb().unwrap_or(0);
+ f
+}
+
+#[cfg(target_os = "macos")]
+fn now_secs() -> u64 {
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map_or(0, |d| d.as_secs())
+}
+
+#[cfg(target_os = "macos")]
+fn run(bin: &str, args: &[&str]) -> Option<String> {
+ let out = std::process::Command::new(bin).args(args).output().ok()?;
+ out.status
+ .success()
+ .then(|| String::from_utf8_lossy(&out.stdout).into_owned())
+}
+
+/// `pmset -g batt`: (has a battery, running on battery, charge).
+pub fn parse_pmset_batt(text: &str) -> (bool, bool, Option<u8>) {
+ let on_battery = text.contains("'Battery Power'");
+ let mut has = false;
+ let mut pct = None;
+ for line in text.lines() {
+ if line.contains("InternalBattery") {
+ has = true;
+ pct = line
+ .split('\t')
+ .nth(1)
+ .or(Some(line))
+ .and_then(|l| l.split('%').next())
+ .and_then(|l| l.split_whitespace().last())
+ .and_then(|n| n.parse::<u8>().ok());
+ }
+ }
+ (has, has && on_battery, pct)
+}
+
+/// `ioreg -c IOHIDSystem`: seconds since the last keyboard or mouse input.
+pub fn parse_hid_idle_secs(text: &str) -> Option<u64> {
+ let line = text.lines().find(|l| l.contains("HIDIdleTime"))?;
+ let ns: u64 = line.rsplit('=').next()?.trim().parse().ok()?;
+ Some(ns / 1_000_000_000)
+}
+
+/// `sysctl -n kern.boottime`: `{ sec = 1700000000, usec = 0 } ...`.
+pub fn parse_boottime(text: &str) -> Option<u64> {
+ let rest = text.split("sec =").nth(1)?;
+ rest.trim_start()
+ .split(|c: char| !c.is_ascii_digit())
+ .next()?
+ .parse()
+ .ok()
+}
+
+#[cfg(target_os = "macos")]
+fn macos_link() -> Link {
+ let Some(route) = run("/sbin/route", &["-n", "get", "default"]) else {
+ return Link::Unknown;
+ };
+ let Some(iface) = route.lines().find_map(|l| {
+ l.trim()
+ .strip_prefix("interface:")
+ .map(|s| s.trim().to_string())
+ }) else {
+ return Link::Unknown;
+ };
+ let Some(ports) = run("/usr/sbin/networksetup", &["-listallhardwareports"]) else {
+ return Link::Unknown;
+ };
+ link_of_port(&ports, &iface)
+}
+
+/// Which kind of port `iface` is, from `networksetup -listallhardwareports`.
+pub fn link_of_port(ports: &str, iface: &str) -> Link {
+ let mut name = "";
+ for line in ports.lines() {
+ if let Some(n) = line.strip_prefix("Hardware Port:") {
+ name = n.trim();
+ }
+ if let Some(d) = line.strip_prefix("Device:")
+ && d.trim() == iface
+ {
+ let n = name.to_ascii_lowercase();
+ return if n.contains("wi-fi") || n.contains("airport") {
+ Link::Wifi
+ } else if n.contains("iphone")
+ || n.contains("ipad")
+ || n.contains("usb") && n.contains("tether")
+ {
+ Link::Metered
+ } else if n.contains("ethernet") || n.contains("thunderbolt") || n.contains("lan") {
+ Link::Wired
+ } else {
+ Link::Unknown
+ };
+ }
+ }
+ Link::Unknown
+}
+
+#[cfg(target_os = "linux")]
+fn linux_power() -> (bool, bool, Option<u8>) {
+ let mut has = false;
+ let mut on_battery = false;
+ let mut pct = None;
+ if let Ok(rd) = std::fs::read_dir("/sys/class/power_supply") {
+ for e in rd.flatten() {
+ let p = e.path();
+ let kind = std::fs::read_to_string(p.join("type")).unwrap_or_default();
+ if kind.trim() == "Battery" {
+ has = true;
+ let st = std::fs::read_to_string(p.join("status")).unwrap_or_default();
+ if st.trim() == "Discharging" {
+ on_battery = true;
+ }
+ pct = std::fs::read_to_string(p.join("capacity"))
+ .ok()
+ .and_then(|c| c.trim().parse().ok());
+ }
+ }
+ }
+ (has, on_battery, pct)
+}
+
+#[cfg(unix)]
+fn free_disk_gb() -> Option<u64> {
+ let home = std::env::var_os("HOME").map(std::path::PathBuf::from)?;
+ let c = std::ffi::CString::new(home.to_string_lossy().as_bytes()).ok()?;
+ // SAFETY: an all-zero statvfs is a valid out value.
+ let mut st: libc::statvfs = unsafe { std::mem::zeroed() };
+ // SAFETY: `c` is a valid C string and `st` a valid out pointer.
+ if unsafe { libc::statvfs(c.as_ptr(), &mut st) } == 0 {
+ Some((st.f_bavail as u64).saturating_mul(st.f_frsize as u64) >> 30)
+ } else {
+ None
+ }
+}
+
+#[cfg(not(unix))]
+fn free_disk_gb() -> Option<u64> {
+ None
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn laptop(on_battery: bool) -> Facts {
+ Facts {
+ has_battery: true,
+ on_battery,
+ battery_pct: Some(80),
+ ..Default::default()
+ }
+ }
+
+ #[test]
+ fn a_desktop_beats_a_plugged_laptop_beats_a_laptop_on_battery() {
+ let desktop = Facts::default();
+ let plugged = laptop(false);
+ let battery = laptop(true);
+ assert!(score(&desktop) >= score(&plugged) + STEP);
+ assert!(score(&plugged) >= score(&battery) + STEP);
+ let low = Facts {
+ battery_pct: Some(10),
+ ..laptop(true)
+ };
+ assert!(score(&battery) >= score(&low) + STEP / 2);
+ }
+
+ #[test]
+ fn the_laptop_the_user_is_using_loses_but_a_desktop_only_loses_a_tie() {
+ let mine = Facts {
+ user_active: true,
+ ..laptop(false)
+ };
+ let away = laptop(true);
+ assert!(
+ score(&away) > score(&mine),
+ "the laptop in use is the one about to close"
+ );
+ let desk_used = Facts {
+ user_active: true,
+ ..Default::default()
+ };
+ assert!(score(&desk_used) > score(&laptop(false)));
+ assert!(score(&desk_used) < score(&Facts::default()));
+ }
+
+ #[test]
+ fn small_terms_never_cross_a_class() {
+ // The best a laptop on battery can do is below the worst a plugged
+ // laptop does.
+ let best_battery = Facts {
+ sleep_prevented: true,
+ link: Link::Wired,
+ uptime_secs: 30 * 86_400,
+ free_disk_gb: 500,
+ ..laptop(true)
+ };
+ let worst_plugged = Facts {
+ link: Link::Metered,
+ recent_flaps: 9,
+ ..laptop(false)
+ };
+ assert!(score(&best_battery) < score(&worst_plugged));
+ let best_plugged = Facts {
+ sleep_prevented: true,
+ link: Link::Wired,
+ uptime_secs: 30 * 86_400,
+ free_disk_gb: 500,
+ ..laptop(false)
+ };
+ let worst_desktop = Facts {
+ link: Link::Metered,
+ recent_flaps: 9,
+ ..Default::default()
+ };
+ assert!(score(&best_plugged) < score(&worst_desktop));
+ // An unknown machine sits between plugged and battery.
+ assert!(score(&laptop(true)) < UNKNOWN && UNKNOWN < score(&laptop(false)));
+ }
+
+ #[test]
+ fn a_dedicated_host_is_always_a_desktop() {
+ let f = Facts {
+ dedicated: true,
+ has_battery: true,
+ on_battery: true,
+ ..Default::default()
+ };
+ assert_eq!(score(&f), 9000);
+ assert!(!f.sleep_prone());
+ }
+
+ #[test]
+ fn pmset_and_ioreg_and_sysctl_output_parse() {
+ let ac = "Now drawing from 'AC Power'\n -InternalBattery-0 (id=4653155)\t100%; charged; 0:00 remaining present: true\n";
+ assert_eq!(parse_pmset_batt(ac), (true, false, Some(100)));
+ let bat = "Now drawing from 'Battery Power'\n -InternalBattery-0 (id=1)\t43%; discharging; 3:12 remaining present: true\n";
+ assert_eq!(parse_pmset_batt(bat), (true, true, Some(43)));
+ let desk = "Now drawing from 'AC Power'\n";
+ assert_eq!(parse_pmset_batt(desk), (false, false, None));
+ let hid = " | | \"HIDIdleTime\" = 125000000000\n";
+ assert_eq!(parse_hid_idle_secs(hid), Some(125));
+ assert_eq!(
+ parse_boottime("{ sec = 1700000000, usec = 123 } Tue Nov 14 22:13:20 2023"),
+ Some(1_700_000_000)
+ );
+ let ports = "Hardware Port: Wi-Fi\nDevice: en0\nEthernet Address: x\n\nHardware Port: Thunderbolt Ethernet Slot 1\nDevice: en5\n\nHardware Port: iPhone USB\nDevice: en7\n";
+ assert_eq!(link_of_port(ports, "en0"), Link::Wifi);
+ assert_eq!(link_of_port(ports, "en5"), Link::Wired);
+ assert_eq!(link_of_port(ports, "en7"), Link::Metered);
+ assert_eq!(link_of_port(ports, "en9"), Link::Unknown);
+ }
+
+ #[test]
+ fn this_machine_can_be_probed() {
+ let f = probe();
+ // Whatever it is, it scores and describes itself.
+ assert!(!describe(&f).is_empty());
+ let _ = score(&f);
+ }
+}
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs
index 83a28c2897..8e4db943b2 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/durable.rs
@@ -66,6 +66,10 @@ pub struct DurableSpec {
/// No desktop (`SpaceCreate::headless`): kept by every copy.
#[serde(default)]
pub headless: bool,
+ /// CPU architecture every copy must run on (`amd64`, `arm64`); empty:
+ /// any (the default: a durable Space may move between architectures).
+ #[serde(default)]
+ pub arch: String,
}
/// What a Space's `.cua/setup` did.
@@ -535,6 +539,8 @@ impl Coordinator {
cpus: row.spec.cpus,
memory_mb: row.spec.memory_mb,
headless: row.spec.headless,
+ durable: true,
+ arch: row.spec.arch.clone(),
};
match self.place_space_excluding(req, &exclude).await {
Ok(p) => {
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
index 3b89767c39..4eb75704f9 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
@@ -88,6 +88,25 @@ pub struct Config {
/// Runs a durable Space's `.cua/setup` after each placement (the first
/// copy and every re-placement from scratch). `None`: no setup step.
pub space_setup: Option<durable::SetupHook>,
+ /// What each machine says about its own reliability (a laptop on
+ /// battery, a desktop that is always on), by machine name. A durable
+ /// Space goes to the most reliable machine that has room
+ /// ([`Policy::Durable`]); a machine that says nothing scores as
+ /// unknown. `None`: nothing is known (capacity decides).
+ pub reliability: Option<ReliabilityHook>,
+}
+
+/// Looks up a machine's [`cua_cluster::reliability::Facts`] by name.
+#[derive(Clone)]
+pub struct ReliabilityHook(pub Arc<FactsOf>);
+
+/// Facts of a machine by name.
+pub type FactsOf = dyn Fn(&str) -> Option<cua_cluster::reliability::Facts> + Send + Sync;
+
+impl std::fmt::Debug for ReliabilityHook {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str("ReliabilityHook")
+ }
}
impl Config {
@@ -105,6 +124,7 @@ impl Config {
default_max_attempts: 5,
session_tick: Duration::from_secs(2),
space_setup: None,
+ reliability: None,
}
}
}
@@ -442,6 +462,9 @@ pub struct PlacedSpace {
pub local_space: String,
/// Its kind.
pub kind: String,
+ /// Machines that were not chosen and why, in plain words ("linux-box
+ /// cannot run microVM Spaces ...", "mac-studio is aarch64, not amd64").
+ pub skipped: Vec<String>,
}
/// What `place_space` needs.
@@ -468,6 +491,14 @@ pub struct SpaceRequest {
/// No desktop (`CreateHostSpaceRequest.headless`).
#[serde(default)]
pub headless: bool,
+ /// A durable Space: placed on the most reliable machine that has room.
+ #[serde(default)]
+ pub durable: bool,
+ /// CPU architecture the Space must run on (`amd64`, `arm64`); empty: any
+ /// machine (a headless image is multi-arch; the workspace's built
+ /// programs are not, which a takeover across architectures says).
+ #[serde(default)]
+ pub arch: String,
}
/// The coordinator.
@@ -2680,7 +2711,7 @@ impl Coordinator {
} else {
String::new()
},
- arch: String::new(),
+ arch: req.arch.clone(),
on: String::new(),
shape: cua_cluster::placement::warm_key(
&req.image,
@@ -2722,6 +2753,11 @@ impl Coordinator {
elastic: false,
pinned: n.row.state == "pinned",
provider: n.row.provider.clone(),
+ reliability: self
+ .cfg
+ .reliability
+ .as_ref()
+ .and_then(|h| (h.0)(&n.row.name)),
..Default::default()
},
(n.link.clone(), n.row.direct.clone()),
@@ -2729,7 +2765,12 @@ impl Coordinator {
})
.unzip()
};
- let order = placement::rank(&cands, &need, self.cfg.policy);
+ let policy = if req.durable {
+ Policy::Durable
+ } else {
+ self.cfg.policy
+ };
+ let order = placement::rank(&cands, &need, policy);
if cands.is_empty() {
let relayed = {
let g = self.sched.lock().expect("sched");
@@ -2750,6 +2791,7 @@ impl Coordinator {
placement::why_nothing_fits(&cands, &need)
)));
}
+ let skipped = placement::skipped(&cands, &need);
let mut errors = Vec::new();
for i in order {
let (link, direct) = &links[i];
@@ -2782,6 +2824,7 @@ impl Coordinator {
token: String::new(),
local_space: space.local_space,
kind: space.kind,
+ skipped: skipped.clone(),
});
}
Ok(_) if via_relay => {
@@ -2795,6 +2838,7 @@ impl Coordinator {
token: space.direct_token,
local_space: space.local_space,
kind: space.kind,
+ skipped: skipped.clone(),
});
}
Ok(_) => errors.push(format!("{}: created no direct port", cands[i].name)),