Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
perf(cua-spaces): dial a slow computer once, not once a poll (#5908)
ddupont45a0cb1106ef
diff --git a/.github/workflows/ci-cua-sdk.yml b/.github/workflows/ci-cua-sdk.ymlindex 4098ac167b..078c8958d8 100644--- a/.github/workflows/ci-cua-sdk.yml+++ b/.github/workflows/ci-cua-sdk.yml@@ -226,6 +226,11 @@ jobs:run: |cargo test --locked -p cua-mesh --lib --test mesh -- --test-threads=4cargo test --locked -p cua-mesh --test scale -- --test-threads=1 --nocapture+ - name: Test the Space list at scale and under chaos (budgets)+ # 20 computers x 10 Spaces with computers off, slow, lossy, hung or+ # half-open: each scenario asserts wall-clock budgets, so they run+ # one at a time.+ run: cargo test --locked -p cua-spaces --test scale -- --test-threads=1 --nocapture- name: Test (embedded, daemon over UDS and loopback, CLI, cluster, Cua Spaces extensions)run: >cargo test --locked --no-fail-fast -p cua-sdk -p cua-daemon -p cua-cli -p cua-spacesd-clientdiff --git a/libs/cua/crates/cua-host/src/testing.rs b/libs/cua/crates/cua-host/src/testing.rsindex 39ec75bcb6..38e35404be 100644--- a/libs/cua/crates/cua-host/src/testing.rs+++ b/libs/cua/crates/cua-host/src/testing.rs@@ -78,6 +78,9 @@ struct State_ {/// Every machine read answers 503, as the proxy in front of the/// relay does while it has no backend.reads_down: bool,+ /// `/m/<id>` of a machine listed offline answers 502 at once, as the+ /// relay does once past its start-up grace.+ refuse_offline: bool,/// A relay that predates meta patches: the key is ignored.ignore_meta_patches: bool,/// A relay (or its control plane) that refuses to change who a computer@@ -973,6 +976,18 @@ async fn proxied(if let Some(r) = refused {return r;}+ {+ let st = s.lock().unwrap();+ let offline = uri+ .path()+ .strip_prefix("/m/")+ .and_then(|rest| rest.split('/').next())+ .and_then(|id| st.machines.get(id))+ .is_some_and(|r| !r.machine.online);+ if st.refuse_offline && offline {+ return err(StatusCode::BAD_GATEWAY, "the computer is not connected");+ }+ }if let Some(target) = target&& headers.get(axum::http::header::CONTENT_TYPE)@@ -1080,7 +1095,16 @@ async fn forward(.and_then(|rest| rest.find('/').map(|i| &rest[i..])).unwrap_or(path);let url = format!("{}{path}", target.trim_end_matches('/'));- let mut req = reqwest::Client::new().request(method, url).body(payload);+ // One client for every forward: a client per request read the system+ // proxy settings each time, which serialized a test's many connects.+ static HTTP: std::sync::OnceLock<reqwest::Client> = std::sync::OnceLock::new();+ let http = HTTP.get_or_init(|| {+ reqwest::Client::builder()+ .no_proxy()+ .build()+ .expect("fake relay client")+ });+ let mut req = http.request(method, url).body(payload);for (k, v) in headers {if k != axum::http::header::HOST && k != axum::http::header::CONTENT_LENGTH {req = req.header(k, v);@@ -1238,6 +1262,12 @@ impl FakeRelay {.extend(statuses.iter().copied());}+ /// While `on`, a request to a computer listed offline answers 502 at+ /// once, as the relay does past its start-up grace.+ pub fn set_refuse_offline(&self, on: bool) {+ self.state.lock().unwrap().refuse_offline = on;+ }+/// While `down`, every machine read answers 503 "no available server"/// (a relay deploy, as its proxy answers).pub fn set_reads_down(&self, down: bool) {diff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rsindex 85b4b7b6a7..c32ddca518 100644--- a/libs/cua/crates/cua-spaces/src/spaces.rs+++ b/libs/cua/crates/cua-spaces/src/spaces.rs@@ -173,6 +173,20 @@ impl SpaceCreated {}/// Options for a local Space (see [`Spaces::create`]).+/// [`Spaces::space`]'s per-Space turn, and the last failure under it.+#[derive(Default)]+struct ConnectGate {+ turn: Arc<tokio::sync::Mutex<()>>,+ failed: std::sync::Mutex<Option<FailedConnect>>,+}++#[derive(Clone)]+struct FailedConnect {+ at: std::time::Instant,+ presence: crate::dial_backoff::Presence,+ reason: String,+}+#[derive(Clone, Debug, Default)]pub(crate) struct LocalProvision {/// Image. Default: the canonical Linux image.@@ -343,6 +357,7 @@ impl SpacesBuilder {probe_timeout: self.probe_timeout.unwrap_or(DEFAULT_PROBE_TIMEOUT),download_dir,connections: tokio::sync::Mutex::new(HashMap::new()),+ connecting: Default::default(),dial_backoff: Default::default(),mesh: crate::mesh::MeshClients::default(),#[cfg(feature = "spaces-hotspot")]@@ -549,6 +564,9 @@ pub(crate) struct Inner {probe_timeout: Duration,pub(crate) download_dir: PathBuf,connections: tokio::sync::Mutex<HashMap<String, Space>>,+ /// One connect at a time per Space; callers that arrive meanwhile take+ /// its outcome instead of dialing again.+ connecting: std::sync::Mutex<HashMap<String, Arc<ConnectGate>>>,/// Mesh nodes for `relay:` Spaces ([`crate::mesh`])./// Your computers that did not answer, and when to dial them again.pub(crate) dial_backoff: crate::dial_backoff::DialBackoff,@@ -1472,6 +1490,68 @@ impl Spaces {pub async fn space(&self, space: &str) -> Result<Space> {let id = self.resolve_now(space).await?;let key = id.to_string();+ if let Some(s) = self.cached_connection(&key).await {+ return Ok(s);+ }+ // One connect per Space at a time, run on its own: an app polling+ // the list gives up on a computer that does not answer and asks+ // again before the attempt ended. The attempt goes on (a caller's+ // deadline does not cancel it), and its outcome answers whoever+ // asked meanwhile, so a slow computer is dialed once, not once a+ // poll.+ let gate = self+ .inner+ .connecting+ .lock()+ .expect("connecting")+ .entry(key.clone())+ .or_default()+ .clone();+ let arrived = std::time::Instant::now();+ let me = self.clone();+ let attempt = tokio::spawn(async move {+ let _turn = gate.turn.clone().lock_owned().await;+ if let Some(s) = me.inner.connections.lock().await.get(&key).cloned() {+ return Ok(s);+ }+ // What the relay says about the computer now: one that came+ // back (online again, or a new session) is dialed afresh.+ let presence = match &id {+ SpaceId::Relay { machine_id } => me.relay_presence(machine_id),+ _ => None,+ };+ if let Some(f) = gate.failed.lock().expect("gate").clone()+ && f.at >= arrived+ && f.presence == presence+ {+ return Err(Error::SpacesdNotAvailable {+ space: key,+ reason: f.reason,+ });+ }+ let r = me.connect_space(id, key).await;+ if let Err(e) = &r {+ let reason = match e {+ Error::SpacesdNotAvailable { reason, .. } => reason.clone(),+ e => e.to_string(),+ };+ *gate.failed.lock().expect("gate") = Some(FailedConnect {+ at: std::time::Instant::now(),+ presence,+ reason,+ });+ }+ r+ });+ attempt+ .await+ .map_err(|e| Error::Cancelled(format!("the connect to this Space ended: {e}")))?+ }++ /// The open connection to `key`, unless its route went stale or it can+ /// move to the mesh now (then it is dropped, to connect again).+ async fn cached_connection(&self, key: &str) -> Option<Space> {+ let key = key.to_string();// Bound first: the cache's guard must be gone before the lock is// taken again below.let cached = self.inner.connections.lock().await.get(&key).cloned();@@ -1490,7 +1570,7 @@ impl Spaces {.upgrade(&key, crate::mesh::UPGRADE_CHECK).await{- None => return Ok(s),+ None => return Some(s),Some(reason) => {tracing::info!(space = %key, %reason, "moving to the mesh");self.inner.connections.lock().await.remove(&key);@@ -1502,6 +1582,19 @@ impl Spaces {}}}+ None+ }++ /// [`Spaces::space`] behind a type-erased future: the routed Space's+ /// connect awaits its host's, and the erased type ends the recursion.+ fn space_of_host(+ &self,+ host: String,+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Space>> + Send + '_>> {+ Box::pin(async move { self.space(&host).await })+ }++ async fn connect_space(&self, id: SpaceId, key: String) -> Result<Space> {let name = match (&id, self.inner.registry.get(&key)?) {(_, Some(record)) => Some(record.name),(SpaceId::Relay { machine_id }, None) => Some(@@ -1668,7 +1761,7 @@ impl Spaces {// Provided by a host and reached through it: a forward of// its loopback port over the host's own channel.let (machine, port) = crate::id::routed_host(authority).expect("routed");- let host = Box::pin(self.space(&format!("relay:{machine}"))).await?;+ let host = self.space_of_host(format!("relay:{machine}")).await?;let fwd = host.spacesd()?.forward_tcp(cua_spacesd_client::tunnel::ForwardOptions::new(port))diff --git a/libs/cua/crates/cua-spaces/tests/scale.rs b/libs/cua/crates/cua-spaces/tests/scale.rsnew file mode 100644index 0000000000..456fa05669--- /dev/null+++ b/libs/cua/crates/cua-spaces/tests/scale.rs@@ -0,0 +1,558 @@+// SPDX-License-Identifier: MIT+// Copyright (c) 2026 Cua AI, Inc.++//! The Space list at scale, and under chaos: an account with many+//! computers, each with many Spaces, some of them offline, slow, lossy or+//! unresponsive.+//!+//! Each scenario does what the apps do after a launch or a daemon restart:+//! read this machine's Spaces, read the whole list (the relay directory),+//! then reach every Space at once with the app's per-Space deadline. It+//! asserts the budgets:+//!+//! - this machine's Spaces list within [`LOCAL_BUDGET`], and the list is+//! never empty while they exist;+//! - every Space on a computer that answers is reached within+//! [`ONLINE_BUDGET`];+//! - every Space on an offline or unresponsive computer is marked within+//! [`OFFLINE_BUDGET`], and never holds up the rest;+//! - a computer that comes back is reachable within [`RECOVERY_BUDGET`];+//! - a computer that keeps failing is not dialed again on every poll.+//!+//! Sizes: `CUA_SCALE_HOSTS` (default 20) computers with `CUA_SCALE_SPACES`+//! (default 10) Spaces each. Each scenario prints a `SCALE` line.++use cua_host::relay::RegisterRequest;+use cua_host::testing::FakeRelay;+use cua_spaces::relay::{AccountTokens, StaticToken};+use cua_spaces::{RelayAccount, Spaces};+use cua_spacesd_client::testing::{Fault, FaultProxy, MockAuth, MockServer};+use std::collections::BTreeSet;+use std::sync::Arc;+use std::time::{Duration, Instant};++const LOCAL_BUDGET: Duration = Duration::from_millis(200);+const LIST_BUDGET: Duration = Duration::from_secs(2);+const ONLINE_BUDGET: Duration = Duration::from_secs(2);+const OFFLINE_BUDGET: Duration = Duration::from_secs(3);+const RECOVERY_BUDGET: Duration = Duration::from_secs(3);+// The app gives up on one Space after this; the connect goes on, and the+// next read picks up its outcome.+const APP_PROBE: Duration = Duration::from_millis(2500);+/// This machine's own Spaces.+const LOCAL_SPACES: usize = 10;+const TOKEN: &str = "ada-token";++fn size(var: &str, default: usize) -> usize {+ std::env::var(var)+ .ok()+ .and_then(|v| v.parse().ok())+ .unwrap_or(default)+}++/// How a computer behaves in a scenario.+#[derive(Debug, Clone, Copy, PartialEq)]+enum Health {+ Online,+ /// Switched off: the relay lists it offline.+ Offline,+ /// The relay lists it online; its link has this fault.+ Faulty(Fault),+}++impl Health {+ /// Its Spaces are expected to answer.+ fn answers(self) -> bool {+ match self {+ Health::Online => true,+ Health::Offline => false,+ Health::Faulty(f) => matches!(+ f,+ Fault::Pass | Fault::Latency(_) | Fault::Bandwidth(_) | Fault::Lossy { .. }+ ),+ }+ }+}++struct Computer {+ id: String,+ spaces: Vec<String>,+ proxy: FaultProxy,+ _env: MockServer,+}++struct World {+ _dir: tempfile::TempDir,+ home: std::path::PathBuf,+ relay: FakeRelay,+ /// In front of the relay: its faults are the relay's.+ relay_link: FaultProxy,+ computers: Vec<Computer>,+ local: Vec<String>,+ _local_env: MockServer,+ _local_links: Vec<FaultProxy>,+}++fn machine_id(n: usize) -> String {+ format!("{n:016x}")+}++impl World {+ async fn new(hosts: usize, per_host: usize) -> Self {+ let relay = FakeRelay::start().await;+ relay.add_account(TOKEN, "ada", Some("[email protected]"));+ relay.set_refuse_offline(true);+ let relay_link =+ FaultProxy::start(relay.url.trim_start_matches("http://").parse().unwrap()).await;+ let client = cua_host::RelayClient::new(&relay.url).unwrap();+ let mut computers = Vec::new();+ let mut n = 1;+ for h in 0..hosts {+ let env = MockServer::start(MockAuth {+ token: None,+ gateway: None,+ prefix: None,+ })+ .await;+ let proxy =+ FaultProxy::start(env.url().trim_start_matches("http://").parse().unwrap()).await;+ let id = machine_id(n);+ n += 1;+ client+ .register(+ TOKEN,+ &RegisterRequest {+ id: id.clone(),+ name: format!("computer-{h}"),+ allow: vec![],+ host: None,+ meta: Default::default(),+ },+ )+ .await+ .unwrap();+ let mut spaces = Vec::new();+ for s in 0..per_host {+ let sid = machine_id(n);+ n += 1;+ client+ .register(+ TOKEN,+ &RegisterRequest {+ id: sid.clone(),+ name: format!("space-{h}-{s}"),+ allow: vec![],+ host: Some(id.clone()),+ meta: Default::default(),+ },+ )+ .await+ .unwrap();+ spaces.push(sid);+ }+ computers.push(Computer {+ id,+ spaces,+ proxy,+ _env: env,+ });+ }+ let dir = tempfile::tempdir().unwrap();+ let home = dir.path().join(".cua");+ // This machine's own Spaces: in its registry, read without the+ // network.+ let local_env = MockServer::start(MockAuth {+ token: None,+ gateway: None,+ prefix: None,+ })+ .await;+ let spaces = Spaces::builder().home(&home).build();+ let mut local = Vec::new();+ let mut local_links = Vec::new();+ for i in 0..LOCAL_SPACES {+ // One address each (a Space is registered by its address).+ let link = FaultProxy::start(+ local_env+ .url()+ .trim_start_matches("http://")+ .parse()+ .unwrap(),+ )+ .await;+ let info = spaces+ .add(&link.url(), None, Some(format!("local-{i}")))+ .await+ .unwrap();+ local.push(info.id);+ local_links.push(link);+ }+ let mut w = Self {+ _dir: dir,+ home,+ relay,+ relay_link,+ computers,+ local,+ _local_env: local_env,+ _local_links: local_links,+ };+ for i in 0..w.computers.len() {+ w.set(i, Health::Online);+ }+ w+ }++ /// Puts computer `i` (and its Spaces) in `health`.+ fn set(&mut self, i: usize, health: Health) {+ let c = &self.computers[i];+ let online = health != Health::Offline;+ for id in std::iter::once(&c.id).chain(c.spaces.iter()) {+ self.relay.set_online(id, online, "1.0.0");+ if online {+ // Idempotent enough: the newest tunnel is the one used.+ self.relay.tunnel(id, &c.proxy.url());+ }+ }+ c.proxy.set(match health {+ Health::Faulty(f) => f,+ _ => Fault::Pass,+ });+ }++ /// Computer `i` and its Spaces connect to the relay again (a new+ /// session).+ fn reconnect(&self, i: usize) {+ let c = &self.computers[i];+ let at = std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap()+ .as_nanos() as u64;+ for id in std::iter::once(&c.id).chain(c.spaces.iter()) {+ self.relay.set_session(id, true, Some(at));+ }+ }++ fn spaces_with(&self, tokens: Arc<dyn AccountTokens>) -> Spaces {+ Spaces::builder()+ .home(&self.home)+ .relay(RelayAccount::new(self.relay_link.url(), tokens))+ .build()+ }++ fn spaces(&self) -> Spaces {+ self.spaces_with(Arc::new(StaticToken(TOKEN.into())))+ }++ fn remote_ids(&self) -> BTreeSet<String> {+ self.computers+ .iter()+ .flat_map(|c| c.spaces.iter().map(|s| format!("relay:{s}")))+ .collect()+ }+}++#[derive(Default, Debug)]+struct Run {+ local: Duration,+ list: Duration,+ listed: usize,+ online_max: Duration,+ offline_max: Duration,+ online_failed: Vec<String>,+ offline_answered: Vec<String>,+}++/// What an app does after a launch: this machine's Spaces, the whole list,+/// then every Space at once (each bounded by the app's deadline).+async fn app_poll(w: &World, spaces: &Spaces, health: &[Health]) -> Run {+ let mut run = Run::default();+ let t = Instant::now();+ let local = spaces.list().unwrap();+ run.local = t.elapsed();+ let mine: BTreeSet<_> = local.iter().map(|i| i.id.clone()).collect();+ for id in &w.local {+ assert!(mine.contains(id), "this machine's Space {id} is not listed");+ }+ let t = Instant::now();+ let listed = tokio::time::timeout(Duration::from_secs(60), spaces.list_all()).await;+ run.list = t.elapsed();+ let listed = match listed {+ Ok(Ok(l)) => l,+ // The relay or the sign-in failed: the list still has this+ // machine's Spaces (what the daemon must show).+ _ => spaces.list().unwrap(),+ };+ run.listed = listed.len();+ let start = Instant::now();+ let mut tasks = Vec::new();+ for (c, h) in w.computers.iter().zip(health) {+ for s in &c.spaces {+ let id = format!("relay:{s}");+ let spaces = spaces.clone();+ let answers = h.answers();+ tasks.push(tokio::spawn(async move {+ let r = tokio::time::timeout(APP_PROBE, spaces.space(&id)).await;+ let ok = matches!(r, Ok(Ok(_)));+ let why = match r {+ Ok(Ok(_)) => String::new(),+ Ok(Err(e)) => e.to_string(),+ Err(_) => "the app's deadline".into(),+ };+ (format!("{id}: {why}"), answers, ok, start.elapsed())+ }));+ }+ }+ for t in tasks {+ let (id, answers, ok, took) = t.await.unwrap();+ match (answers, ok) {+ (true, true) => run.online_max = run.online_max.max(took),+ (true, false) => run.online_failed.push(format!("{id} after {took:?}")),+ (false, true) => run.offline_answered.push(id),+ (false, false) => run.offline_max = run.offline_max.max(took),+ }+ }+ run+}++fn check(name: &str, run: &Run, expect_listed: usize) {+ eprintln!(+ "SCALE {name}: local={:?} list={:?} listed={} online_max={:?} offline_max={:?} online_failed={} offline_answered={}",+ run.local,+ run.list,+ run.listed,+ run.online_max,+ run.offline_max,+ run.online_failed.len(),+ run.offline_answered.len()+ );+ assert!(+ run.local <= LOCAL_BUDGET,+ "{name}: local list took {:?}",+ run.local+ );+ assert!(+ run.listed >= expect_listed,+ "{name}: listed {} < {expect_listed}",+ run.listed+ );+ assert!(+ run.list <= LIST_BUDGET,+ "{name}: the list took {:?}",+ run.list+ );+ assert!(+ run.online_failed.is_empty(),+ "{name}: Spaces that answer failed: {:?}",+ &run.online_failed[..run.online_failed.len().min(5)]+ );+ assert!(+ run.offline_answered.is_empty(),+ "{name}: {:?}",+ run.offline_answered+ );+ assert!(+ run.online_max <= ONLINE_BUDGET,+ "{name}: slowest reachable Space took {:?}",+ run.online_max+ );+ assert!(+ run.offline_max <= OFFLINE_BUDGET,+ "{name}: slowest unreachable Space was marked after {:?}",+ run.offline_max+ );+}++async fn world() -> (World, usize) {+ let hosts = size("CUA_SCALE_HOSTS", 20);+ let per = size("CUA_SCALE_SPACES", 10);+ let w = World::new(hosts, per).await;+ let n = hosts * (per + 1) + LOCAL_SPACES;+ (w, n)+}++/// `fraction` of the computers (the first ones) switched off.+async fn down_scenario(name: &str, down: usize) {+ let (mut w, n) = world().await;+ let mut health = vec![Health::Online; w.computers.len()];+ for (i, h) in health.iter_mut().enumerate().take(down) {+ *h = Health::Offline;+ w.set(i, *h);+ }+ let spaces = w.spaces();+ let run = app_poll(&w, &spaces, &health).await;+ check(name, &run, n);+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn scale_all_online() {+ down_scenario("all_online", 0).await;+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn scale_one_down() {+ down_scenario("one_down", 1).await;+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn scale_quarter_down() {+ let hosts = size("CUA_SCALE_HOSTS", 20);+ down_scenario("25pct_down", hosts / 4).await;+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn scale_half_down() {+ let hosts = size("CUA_SCALE_HOSTS", 20);+ down_scenario("50pct_down", hosts / 2).await;+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn scale_ninety_down() {+ let hosts = size("CUA_SCALE_HOSTS", 20);+ down_scenario("90pct_down", hosts * 9 / 10).await;+}++/// Slow, thin and lossy links answer within budget; unresponsive and+/// half-open computers are marked within budget and hold up nothing.+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn scale_slow_and_unresponsive_computers() {+ let (mut w, n) = world().await;+ let faults = [+ Fault::Latency(Duration::from_millis(150)),+ Fault::Latency(Duration::from_millis(300)),+ Fault::Bandwidth(256 * 1024),+ Fault::Lossy {+ p: 0.05,+ stall: Duration::from_millis(200),+ },+ Fault::Blackhole,+ Fault::Blackhole,+ Fault::HalfOpen,+ Fault::Reset,+ ];+ let mut health = vec![Health::Online; w.computers.len()];+ for (i, f) in faults.iter().enumerate().take(w.computers.len()) {+ health[i] = Health::Faulty(*f);+ w.set(i, health[i]);+ }+ let spaces = w.spaces();+ let run = app_poll(&w, &spaces, &health).await;+ check("slow_and_unresponsive", &run, n);+}++/// Computers come back: reachable on the next poll, with no ghost and no+/// duplicate rows; one that keeps failing is not dialed on every poll.+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn scale_recovery_and_backoff() {+ let (mut w, n) = world().await;+ let hosts = w.computers.len();+ let mut health = vec![Health::Online; hosts];+ // Half off, two unresponsive.+ for (i, h) in health.iter_mut().enumerate().take(hosts / 2) {+ *h = Health::Offline;+ w.set(i, *h);+ }+ for (i, h) in health.iter_mut().enumerate().skip(hosts / 2).take(2) {+ *h = Health::Faulty(Fault::Blackhole);+ w.set(i, *h);+ }+ let spaces = w.spaces();+ let run = app_poll(&w, &spaces, &health).await;+ check("recovery_before", &run, n);++ // Polled again and again: the blackholed computers are not dialed on+ // every poll (backoff), and the offline ones never are.+ let dials_before: u64 = (hosts / 2..(hosts / 2 + 2).min(hosts))+ .map(|i| w.computers[i].proxy.accepted())+ .sum();+ let proxied_before = w.relay.proxied().len();+ for _ in 0..5 {+ let run = app_poll(&w, &spaces, &health).await;+ check("recovery_repoll", &run, n);+ }+ let dials_after: u64 = (hosts / 2..(hosts / 2 + 2).min(hosts))+ .map(|i| w.computers[i].proxy.accepted())+ .sum();+ let offline_spaces = (hosts / 2) * w.computers[0].spaces.len();+ let proxied = w.relay.proxied().len() - proxied_before;+ eprintln!(+ "SCALE backoff: blackholed dials over 5 polls={} relay requests={proxied}",+ dials_after - dials_before+ );+ assert_eq!(+ dials_after - dials_before,+ 0,+ "unresponsive computers were dialed again inside their backoff"+ );+ assert!(+ proxied < offline_spaces,+ "offline Spaces were dialed through the relay on every poll ({proxied} requests)"+ );++ // Everything comes back: each computer reconnects to the relay (a new+ // session), as one that wakes up or boots does.+ for (i, h) in health.iter_mut().enumerate() {+ *h = Health::Online;+ w.set(i, *h);+ w.reconnect(i);+ }+ let t = Instant::now();+ let run = app_poll(&w, &spaces, &health).await;+ let took = t.elapsed();+ check("recovered", &run, n);+ assert!(took <= RECOVERY_BUDGET, "recovery took {took:?}");+ let listed = spaces.list_all().await.unwrap();+ let ids: BTreeSet<_> = listed.iter().map(|i| i.id.clone()).collect();+ assert_eq!(ids.len(), listed.len(), "duplicate rows");+ for id in w.remote_ids() {+ assert!(ids.contains(&id), "{id} missing after recovery");+ }++ // A computer removed from the account leaves the list (no ghost).+ let gone = w.computers[0].spaces[0].clone();+ cua_host::RelayClient::new(&w.relay.url)+ .unwrap()+ .delete(TOKEN, &gone)+ .await+ .unwrap();+ // Past the directory's cache age.+ tokio::time::sleep(Duration::from_millis(50)).await;+ let listed = spaces.list_all().await.unwrap();+ assert!(+ !listed.iter().any(|i| i.id == format!("relay:{gone}")),+ "a removed Space is still listed"+ );+}++/// A daemon restart in the middle of a sync: the new process lists this+/// machine's Spaces at once and converges with no duplicates.+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn scale_restart_mid_sync() {+ let (mut w, n) = world().await;+ let hosts = w.computers.len();+ let mut health = vec![Health::Online; hosts];+ for (i, h) in health.iter_mut().enumerate().take(hosts / 4) {+ *h = Health::Offline;+ w.set(i, *h);+ }+ let first = w.spaces();+ let polling = {+ let first = first.clone();+ let ids: Vec<String> = w.remote_ids().into_iter().collect();+ tokio::spawn(async move {+ let _ = first.list_all().await;+ for id in ids {+ let _ = tokio::time::timeout(APP_PROBE, first.space(&id)).await;+ }+ })+ };+ tokio::time::sleep(Duration::from_millis(100)).await;+ polling.abort();+ drop(first);+ let second = w.spaces();+ let run = app_poll(&w, &second, &health).await;+ check("restart_mid_sync", &run, n);+ let listed = second.list_all().await.unwrap();+ let ids: BTreeSet<_> = listed.iter().map(|i| i.id.clone()).collect();+ assert_eq!(ids.len(), listed.len(), "duplicate rows after a restart");+}diff --git a/libs/cua/crates/cua-spacesd-client/src/testing.rs b/libs/cua/crates/cua-spacesd-client/src/testing.rsindex f2160fc274..75a7f9df1d 100644--- a/libs/cua/crates/cua-spacesd-client/src/testing.rs+++ b/libs/cua/crates/cua-spacesd-client/src/testing.rs@@ -3402,3 +3402,154 @@ impl CutProxy {format!("http://{}", self.addr)}}++// --------------------------------------------------------------- FaultProxy++/// What a [`FaultProxy`] does to the traffic through it. Switched at run+/// time with [`FaultProxy::set`]; live connections follow the new fault from+/// their next chunk.+#[derive(Debug, Clone, Copy, PartialEq)]+pub enum Fault {+ /// Carries everything.+ Pass,+ /// Holds every chunk, both ways, this long (a slow, far link).+ Latency(Duration),+ /// At most this many bytes per second, both ways (a thin link).+ Bandwidth(u64),+ /// Each chunk, with probability `p`, is held for `stall` first (a lossy+ /// link: TCP waits out a retransmission).+ Lossy {+ /// Probability, 0 to 1.+ p: f64,+ /// How long a lost chunk waits.+ stall: Duration,+ },+ /// Accepts connections and never carries or answers anything (a+ /// computer that went to sleep, a black-holed route).+ Blackhole,+ /// Accepts connections and closes them at once.+ Reset,+ /// Carries requests to the server but never one byte back (a half-open+ /// connection).+ HalfOpen,+}++/// A loopback TCP proxy that injects a [`Fault`] in front of `upstream`,+/// for chaos tests: slow, thin, lossy, unresponsive or half-open links.+pub struct FaultProxy {+ /// Proxy address.+ pub addr: SocketAddr,+ fault: Arc<Mutex<Fault>>,+ accepted: Arc<AtomicU64>,+}++impl FaultProxy {+ /// Proxies to `upstream`, carrying everything until [`Self::set`].+ pub async fn start(upstream: SocketAddr) -> Self {+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let fault = Arc::new(Mutex::new(Fault::Pass));+ let accepted = Arc::new(AtomicU64::new(0));+ let (f, n) = (fault.clone(), accepted.clone());+ tokio::spawn(async move {+ loop {+ let Ok((client, _)) = listener.accept().await else {+ return;+ };+ n.fetch_add(1, Ordering::SeqCst);+ let f = f.clone();+ tokio::spawn(async move {+ let now = *f.lock().unwrap();+ match now {+ Fault::Reset => return,+ Fault::Blackhole => {+ // Held open, never read or answered, until the+ // client gives up.+ let mut client = client;+ let mut sink = [0u8; 1];+ use tokio::io::AsyncReadExt;+ loop {+ tokio::time::sleep(Duration::from_secs(3600)).await;+ if client.read(&mut sink).await.unwrap_or(0) == 0 {+ return;+ }+ }+ }+ _ => {}+ }+ let Ok(server) = TcpStream::connect(upstream).await else {+ return;+ };+ let _ = client.set_nodelay(true);+ let _ = server.set_nodelay(true);+ let (cr, cw) = client.into_split();+ let (sr, sw) = server.into_split();+ tokio::select! {+ _ = Self::pipe(cr, sw, f.clone(), false) => {}+ _ = Self::pipe(sr, cw, f.clone(), true) => {}+ }+ });+ }+ });+ Self {+ addr,+ fault,+ accepted,+ }+ }++ async fn pipe(+ mut from: tokio::net::tcp::OwnedReadHalf,+ mut to: tokio::net::tcp::OwnedWriteHalf,+ fault: Arc<Mutex<Fault>>,+ downstream: bool,+ ) -> std::io::Result<()> {+ use tokio::io::{AsyncReadExt, AsyncWriteExt};+ let mut buf = vec![0u8; 16 * 1024];+ loop {+ let n = from.read(&mut buf).await?;+ if n == 0 {+ return Ok(());+ }+ let now = *fault.lock().unwrap();+ match now {+ Fault::Pass => {}+ Fault::Latency(d) => tokio::time::sleep(d).await,+ Fault::Bandwidth(bps) => {+ let secs = n as f64 / bps.max(1) as f64;+ tokio::time::sleep(Duration::from_secs_f64(secs)).await;+ }+ Fault::Lossy { p, stall } => {+ if rand::random::<f64>() < p {+ tokio::time::sleep(stall).await;+ }+ }+ // Took effect after this connection opened: it goes quiet.+ Fault::Blackhole | Fault::Reset => {+ if now == Fault::Reset {+ return Ok(());+ }+ std::future::pending::<()>().await;+ }+ Fault::HalfOpen if downstream => continue,+ Fault::HalfOpen => {}+ }+ to.write_all(&buf[..n]).await?;+ }+ }++ /// Switches the fault.+ pub fn set(&self, fault: Fault) {+ *self.fault.lock().unwrap() = fault;+ }++ /// Connections accepted so far (a retry storm shows here).+ pub fn accepted(&self) -> u64 {+ self.accepted.load(Ordering::SeqCst)+ }++ /// `http://127.0.0.1:<port>`.+ pub fn url(&self) -> String {+ format!("http://{}", self.addr)+ }+}