Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

perf(cua-spaces): dial a slow computer once, not once a poll (#5908)

ddupont45a0cb1106ef
diff --git a/.github/workflows/ci-cua-sdk.yml b/.github/workflows/ci-cua-sdk.yml
index 4098ac167b..078c8958d8 100644
--- a/.github/workflows/ci-cua-sdk.yml
+++ b/.github/workflows/ci-cua-sdk.yml
@@ -226,6 +226,11 @@ jobs:
run: |
cargo test --locked -p cua-mesh --lib --test mesh -- --test-threads=4
cargo test --locked -p cua-mesh --test scale -- --test-threads=1 --nocapture
+ - name: Test the Space list at scale and under chaos (budgets)
+ # 20 computers x 10 Spaces with computers off, slow, lossy, hung or
+ # half-open: each scenario asserts wall-clock budgets, so they run
+ # one at a time.
+ run: cargo test --locked -p cua-spaces --test scale -- --test-threads=1 --nocapture
- name: Test (embedded, daemon over UDS and loopback, CLI, cluster, Cua Spaces extensions)
run: >
cargo test --locked --no-fail-fast -p cua-sdk -p cua-daemon -p cua-cli -p cua-spacesd-client
diff --git a/libs/cua/crates/cua-host/src/testing.rs b/libs/cua/crates/cua-host/src/testing.rs
index 39ec75bcb6..38e35404be 100644
--- a/libs/cua/crates/cua-host/src/testing.rs
+++ b/libs/cua/crates/cua-host/src/testing.rs
@@ -78,6 +78,9 @@ struct State_ {
/// Every machine read answers 503, as the proxy in front of the
/// relay does while it has no backend.
reads_down: bool,
+ /// `/m/<id>` of a machine listed offline answers 502 at once, as the
+ /// relay does once past its start-up grace.
+ refuse_offline: bool,
/// A relay that predates meta patches: the key is ignored.
ignore_meta_patches: bool,
/// A relay (or its control plane) that refuses to change who a computer
@@ -973,6 +976,18 @@ async fn proxied(
if let Some(r) = refused {
return r;
}
+ {
+ let st = s.lock().unwrap();
+ let offline = uri
+ .path()
+ .strip_prefix("/m/")
+ .and_then(|rest| rest.split('/').next())
+ .and_then(|id| st.machines.get(id))
+ .is_some_and(|r| !r.machine.online);
+ if st.refuse_offline && offline {
+ return err(StatusCode::BAD_GATEWAY, "the computer is not connected");
+ }
+ }
if let Some(target) = target
&& headers
.get(axum::http::header::CONTENT_TYPE)
@@ -1080,7 +1095,16 @@ async fn forward(
.and_then(|rest| rest.find('/').map(|i| &rest[i..]))
.unwrap_or(path);
let url = format!("{}{path}", target.trim_end_matches('/'));
- let mut req = reqwest::Client::new().request(method, url).body(payload);
+ // One client for every forward: a client per request read the system
+ // proxy settings each time, which serialized a test's many connects.
+ static HTTP: std::sync::OnceLock<reqwest::Client> = std::sync::OnceLock::new();
+ let http = HTTP.get_or_init(|| {
+ reqwest::Client::builder()
+ .no_proxy()
+ .build()
+ .expect("fake relay client")
+ });
+ let mut req = http.request(method, url).body(payload);
for (k, v) in headers {
if k != axum::http::header::HOST && k != axum::http::header::CONTENT_LENGTH {
req = req.header(k, v);
@@ -1238,6 +1262,12 @@ impl FakeRelay {
.extend(statuses.iter().copied());
}
+ /// While `on`, a request to a computer listed offline answers 502 at
+ /// once, as the relay does past its start-up grace.
+ pub fn set_refuse_offline(&self, on: bool) {
+ self.state.lock().unwrap().refuse_offline = on;
+ }
+
/// While `down`, every machine read answers 503 "no available server"
/// (a relay deploy, as its proxy answers).
pub fn set_reads_down(&self, down: bool) {
diff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rs
index 85b4b7b6a7..c32ddca518 100644
--- a/libs/cua/crates/cua-spaces/src/spaces.rs
+++ b/libs/cua/crates/cua-spaces/src/spaces.rs
@@ -173,6 +173,20 @@ impl SpaceCreated {
}
/// Options for a local Space (see [`Spaces::create`]).
+/// [`Spaces::space`]'s per-Space turn, and the last failure under it.
+#[derive(Default)]
+struct ConnectGate {
+ turn: Arc<tokio::sync::Mutex<()>>,
+ failed: std::sync::Mutex<Option<FailedConnect>>,
+}
+
+#[derive(Clone)]
+struct FailedConnect {
+ at: std::time::Instant,
+ presence: crate::dial_backoff::Presence,
+ reason: String,
+}
+
#[derive(Clone, Debug, Default)]
pub(crate) struct LocalProvision {
/// Image. Default: the canonical Linux image.
@@ -343,6 +357,7 @@ impl SpacesBuilder {
probe_timeout: self.probe_timeout.unwrap_or(DEFAULT_PROBE_TIMEOUT),
download_dir,
connections: tokio::sync::Mutex::new(HashMap::new()),
+ connecting: Default::default(),
dial_backoff: Default::default(),
mesh: crate::mesh::MeshClients::default(),
#[cfg(feature = "spaces-hotspot")]
@@ -549,6 +564,9 @@ pub(crate) struct Inner {
probe_timeout: Duration,
pub(crate) download_dir: PathBuf,
connections: tokio::sync::Mutex<HashMap<String, Space>>,
+ /// One connect at a time per Space; callers that arrive meanwhile take
+ /// its outcome instead of dialing again.
+ connecting: std::sync::Mutex<HashMap<String, Arc<ConnectGate>>>,
/// Mesh nodes for `relay:` Spaces ([`crate::mesh`]).
/// Your computers that did not answer, and when to dial them again.
pub(crate) dial_backoff: crate::dial_backoff::DialBackoff,
@@ -1472,6 +1490,68 @@ impl Spaces {
pub async fn space(&self, space: &str) -> Result<Space> {
let id = self.resolve_now(space).await?;
let key = id.to_string();
+ if let Some(s) = self.cached_connection(&key).await {
+ return Ok(s);
+ }
+ // One connect per Space at a time, run on its own: an app polling
+ // the list gives up on a computer that does not answer and asks
+ // again before the attempt ended. The attempt goes on (a caller's
+ // deadline does not cancel it), and its outcome answers whoever
+ // asked meanwhile, so a slow computer is dialed once, not once a
+ // poll.
+ let gate = self
+ .inner
+ .connecting
+ .lock()
+ .expect("connecting")
+ .entry(key.clone())
+ .or_default()
+ .clone();
+ let arrived = std::time::Instant::now();
+ let me = self.clone();
+ let attempt = tokio::spawn(async move {
+ let _turn = gate.turn.clone().lock_owned().await;
+ if let Some(s) = me.inner.connections.lock().await.get(&key).cloned() {
+ return Ok(s);
+ }
+ // What the relay says about the computer now: one that came
+ // back (online again, or a new session) is dialed afresh.
+ let presence = match &id {
+ SpaceId::Relay { machine_id } => me.relay_presence(machine_id),
+ _ => None,
+ };
+ if let Some(f) = gate.failed.lock().expect("gate").clone()
+ && f.at >= arrived
+ && f.presence == presence
+ {
+ return Err(Error::SpacesdNotAvailable {
+ space: key,
+ reason: f.reason,
+ });
+ }
+ let r = me.connect_space(id, key).await;
+ if let Err(e) = &r {
+ let reason = match e {
+ Error::SpacesdNotAvailable { reason, .. } => reason.clone(),
+ e => e.to_string(),
+ };
+ *gate.failed.lock().expect("gate") = Some(FailedConnect {
+ at: std::time::Instant::now(),
+ presence,
+ reason,
+ });
+ }
+ r
+ });
+ attempt
+ .await
+ .map_err(|e| Error::Cancelled(format!("the connect to this Space ended: {e}")))?
+ }
+
+ /// The open connection to `key`, unless its route went stale or it can
+ /// move to the mesh now (then it is dropped, to connect again).
+ async fn cached_connection(&self, key: &str) -> Option<Space> {
+ let key = key.to_string();
// Bound first: the cache's guard must be gone before the lock is
// taken again below.
let cached = self.inner.connections.lock().await.get(&key).cloned();
@@ -1490,7 +1570,7 @@ impl Spaces {
.upgrade(&key, crate::mesh::UPGRADE_CHECK)
.await
{
- None => return Ok(s),
+ None => return Some(s),
Some(reason) => {
tracing::info!(space = %key, %reason, "moving to the mesh");
self.inner.connections.lock().await.remove(&key);
@@ -1502,6 +1582,19 @@ impl Spaces {
}
}
}
+ None
+ }
+
+ /// [`Spaces::space`] behind a type-erased future: the routed Space's
+ /// connect awaits its host's, and the erased type ends the recursion.
+ fn space_of_host(
+ &self,
+ host: String,
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Space>> + Send + '_>> {
+ Box::pin(async move { self.space(&host).await })
+ }
+
+ async fn connect_space(&self, id: SpaceId, key: String) -> Result<Space> {
let name = match (&id, self.inner.registry.get(&key)?) {
(_, Some(record)) => Some(record.name),
(SpaceId::Relay { machine_id }, None) => Some(
@@ -1668,7 +1761,7 @@ impl Spaces {
// Provided by a host and reached through it: a forward of
// its loopback port over the host's own channel.
let (machine, port) = crate::id::routed_host(authority).expect("routed");
- let host = Box::pin(self.space(&format!("relay:{machine}"))).await?;
+ let host = self.space_of_host(format!("relay:{machine}")).await?;
let fwd = host
.spacesd()?
.forward_tcp(cua_spacesd_client::tunnel::ForwardOptions::new(port))
diff --git a/libs/cua/crates/cua-spaces/tests/scale.rs b/libs/cua/crates/cua-spaces/tests/scale.rs
new file mode 100644
index 0000000000..456fa05669
--- /dev/null
+++ b/libs/cua/crates/cua-spaces/tests/scale.rs
@@ -0,0 +1,558 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! The Space list at scale, and under chaos: an account with many
+//! computers, each with many Spaces, some of them offline, slow, lossy or
+//! unresponsive.
+//!
+//! Each scenario does what the apps do after a launch or a daemon restart:
+//! read this machine's Spaces, read the whole list (the relay directory),
+//! then reach every Space at once with the app's per-Space deadline. It
+//! asserts the budgets:
+//!
+//! - this machine's Spaces list within [`LOCAL_BUDGET`], and the list is
+//! never empty while they exist;
+//! - every Space on a computer that answers is reached within
+//! [`ONLINE_BUDGET`];
+//! - every Space on an offline or unresponsive computer is marked within
+//! [`OFFLINE_BUDGET`], and never holds up the rest;
+//! - a computer that comes back is reachable within [`RECOVERY_BUDGET`];
+//! - a computer that keeps failing is not dialed again on every poll.
+//!
+//! Sizes: `CUA_SCALE_HOSTS` (default 20) computers with `CUA_SCALE_SPACES`
+//! (default 10) Spaces each. Each scenario prints a `SCALE` line.
+
+use cua_host::relay::RegisterRequest;
+use cua_host::testing::FakeRelay;
+use cua_spaces::relay::{AccountTokens, StaticToken};
+use cua_spaces::{RelayAccount, Spaces};
+use cua_spacesd_client::testing::{Fault, FaultProxy, MockAuth, MockServer};
+use std::collections::BTreeSet;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+const LOCAL_BUDGET: Duration = Duration::from_millis(200);
+const LIST_BUDGET: Duration = Duration::from_secs(2);
+const ONLINE_BUDGET: Duration = Duration::from_secs(2);
+const OFFLINE_BUDGET: Duration = Duration::from_secs(3);
+const RECOVERY_BUDGET: Duration = Duration::from_secs(3);
+// The app gives up on one Space after this; the connect goes on, and the
+// next read picks up its outcome.
+const APP_PROBE: Duration = Duration::from_millis(2500);
+/// This machine's own Spaces.
+const LOCAL_SPACES: usize = 10;
+const TOKEN: &str = "ada-token";
+
+fn size(var: &str, default: usize) -> usize {
+ std::env::var(var)
+ .ok()
+ .and_then(|v| v.parse().ok())
+ .unwrap_or(default)
+}
+
+/// How a computer behaves in a scenario.
+#[derive(Debug, Clone, Copy, PartialEq)]
+enum Health {
+ Online,
+ /// Switched off: the relay lists it offline.
+ Offline,
+ /// The relay lists it online; its link has this fault.
+ Faulty(Fault),
+}
+
+impl Health {
+ /// Its Spaces are expected to answer.
+ fn answers(self) -> bool {
+ match self {
+ Health::Online => true,
+ Health::Offline => false,
+ Health::Faulty(f) => matches!(
+ f,
+ Fault::Pass | Fault::Latency(_) | Fault::Bandwidth(_) | Fault::Lossy { .. }
+ ),
+ }
+ }
+}
+
+struct Computer {
+ id: String,
+ spaces: Vec<String>,
+ proxy: FaultProxy,
+ _env: MockServer,
+}
+
+struct World {
+ _dir: tempfile::TempDir,
+ home: std::path::PathBuf,
+ relay: FakeRelay,
+ /// In front of the relay: its faults are the relay's.
+ relay_link: FaultProxy,
+ computers: Vec<Computer>,
+ local: Vec<String>,
+ _local_env: MockServer,
+ _local_links: Vec<FaultProxy>,
+}
+
+fn machine_id(n: usize) -> String {
+ format!("{n:016x}")
+}
+
+impl World {
+ async fn new(hosts: usize, per_host: usize) -> Self {
+ let relay = FakeRelay::start().await;
+ relay.add_account(TOKEN, "ada", Some("[email protected]"));
+ relay.set_refuse_offline(true);
+ let relay_link =
+ FaultProxy::start(relay.url.trim_start_matches("http://").parse().unwrap()).await;
+ let client = cua_host::RelayClient::new(&relay.url).unwrap();
+ let mut computers = Vec::new();
+ let mut n = 1;
+ for h in 0..hosts {
+ let env = MockServer::start(MockAuth {
+ token: None,
+ gateway: None,
+ prefix: None,
+ })
+ .await;
+ let proxy =
+ FaultProxy::start(env.url().trim_start_matches("http://").parse().unwrap()).await;
+ let id = machine_id(n);
+ n += 1;
+ client
+ .register(
+ TOKEN,
+ &RegisterRequest {
+ id: id.clone(),
+ name: format!("computer-{h}"),
+ allow: vec![],
+ host: None,
+ meta: Default::default(),
+ },
+ )
+ .await
+ .unwrap();
+ let mut spaces = Vec::new();
+ for s in 0..per_host {
+ let sid = machine_id(n);
+ n += 1;
+ client
+ .register(
+ TOKEN,
+ &RegisterRequest {
+ id: sid.clone(),
+ name: format!("space-{h}-{s}"),
+ allow: vec![],
+ host: Some(id.clone()),
+ meta: Default::default(),
+ },
+ )
+ .await
+ .unwrap();
+ spaces.push(sid);
+ }
+ computers.push(Computer {
+ id,
+ spaces,
+ proxy,
+ _env: env,
+ });
+ }
+ let dir = tempfile::tempdir().unwrap();
+ let home = dir.path().join(".cua");
+ // This machine's own Spaces: in its registry, read without the
+ // network.
+ let local_env = MockServer::start(MockAuth {
+ token: None,
+ gateway: None,
+ prefix: None,
+ })
+ .await;
+ let spaces = Spaces::builder().home(&home).build();
+ let mut local = Vec::new();
+ let mut local_links = Vec::new();
+ for i in 0..LOCAL_SPACES {
+ // One address each (a Space is registered by its address).
+ let link = FaultProxy::start(
+ local_env
+ .url()
+ .trim_start_matches("http://")
+ .parse()
+ .unwrap(),
+ )
+ .await;
+ let info = spaces
+ .add(&link.url(), None, Some(format!("local-{i}")))
+ .await
+ .unwrap();
+ local.push(info.id);
+ local_links.push(link);
+ }
+ let mut w = Self {
+ _dir: dir,
+ home,
+ relay,
+ relay_link,
+ computers,
+ local,
+ _local_env: local_env,
+ _local_links: local_links,
+ };
+ for i in 0..w.computers.len() {
+ w.set(i, Health::Online);
+ }
+ w
+ }
+
+ /// Puts computer `i` (and its Spaces) in `health`.
+ fn set(&mut self, i: usize, health: Health) {
+ let c = &self.computers[i];
+ let online = health != Health::Offline;
+ for id in std::iter::once(&c.id).chain(c.spaces.iter()) {
+ self.relay.set_online(id, online, "1.0.0");
+ if online {
+ // Idempotent enough: the newest tunnel is the one used.
+ self.relay.tunnel(id, &c.proxy.url());
+ }
+ }
+ c.proxy.set(match health {
+ Health::Faulty(f) => f,
+ _ => Fault::Pass,
+ });
+ }
+
+ /// Computer `i` and its Spaces connect to the relay again (a new
+ /// session).
+ fn reconnect(&self, i: usize) {
+ let c = &self.computers[i];
+ let at = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos() as u64;
+ for id in std::iter::once(&c.id).chain(c.spaces.iter()) {
+ self.relay.set_session(id, true, Some(at));
+ }
+ }
+
+ fn spaces_with(&self, tokens: Arc<dyn AccountTokens>) -> Spaces {
+ Spaces::builder()
+ .home(&self.home)
+ .relay(RelayAccount::new(self.relay_link.url(), tokens))
+ .build()
+ }
+
+ fn spaces(&self) -> Spaces {
+ self.spaces_with(Arc::new(StaticToken(TOKEN.into())))
+ }
+
+ fn remote_ids(&self) -> BTreeSet<String> {
+ self.computers
+ .iter()
+ .flat_map(|c| c.spaces.iter().map(|s| format!("relay:{s}")))
+ .collect()
+ }
+}
+
+#[derive(Default, Debug)]
+struct Run {
+ local: Duration,
+ list: Duration,
+ listed: usize,
+ online_max: Duration,
+ offline_max: Duration,
+ online_failed: Vec<String>,
+ offline_answered: Vec<String>,
+}
+
+/// What an app does after a launch: this machine's Spaces, the whole list,
+/// then every Space at once (each bounded by the app's deadline).
+async fn app_poll(w: &World, spaces: &Spaces, health: &[Health]) -> Run {
+ let mut run = Run::default();
+ let t = Instant::now();
+ let local = spaces.list().unwrap();
+ run.local = t.elapsed();
+ let mine: BTreeSet<_> = local.iter().map(|i| i.id.clone()).collect();
+ for id in &w.local {
+ assert!(mine.contains(id), "this machine's Space {id} is not listed");
+ }
+ let t = Instant::now();
+ let listed = tokio::time::timeout(Duration::from_secs(60), spaces.list_all()).await;
+ run.list = t.elapsed();
+ let listed = match listed {
+ Ok(Ok(l)) => l,
+ // The relay or the sign-in failed: the list still has this
+ // machine's Spaces (what the daemon must show).
+ _ => spaces.list().unwrap(),
+ };
+ run.listed = listed.len();
+ let start = Instant::now();
+ let mut tasks = Vec::new();
+ for (c, h) in w.computers.iter().zip(health) {
+ for s in &c.spaces {
+ let id = format!("relay:{s}");
+ let spaces = spaces.clone();
+ let answers = h.answers();
+ tasks.push(tokio::spawn(async move {
+ let r = tokio::time::timeout(APP_PROBE, spaces.space(&id)).await;
+ let ok = matches!(r, Ok(Ok(_)));
+ let why = match r {
+ Ok(Ok(_)) => String::new(),
+ Ok(Err(e)) => e.to_string(),
+ Err(_) => "the app's deadline".into(),
+ };
+ (format!("{id}: {why}"), answers, ok, start.elapsed())
+ }));
+ }
+ }
+ for t in tasks {
+ let (id, answers, ok, took) = t.await.unwrap();
+ match (answers, ok) {
+ (true, true) => run.online_max = run.online_max.max(took),
+ (true, false) => run.online_failed.push(format!("{id} after {took:?}")),
+ (false, true) => run.offline_answered.push(id),
+ (false, false) => run.offline_max = run.offline_max.max(took),
+ }
+ }
+ run
+}
+
+fn check(name: &str, run: &Run, expect_listed: usize) {
+ eprintln!(
+ "SCALE {name}: local={:?} list={:?} listed={} online_max={:?} offline_max={:?} online_failed={} offline_answered={}",
+ run.local,
+ run.list,
+ run.listed,
+ run.online_max,
+ run.offline_max,
+ run.online_failed.len(),
+ run.offline_answered.len()
+ );
+ assert!(
+ run.local <= LOCAL_BUDGET,
+ "{name}: local list took {:?}",
+ run.local
+ );
+ assert!(
+ run.listed >= expect_listed,
+ "{name}: listed {} < {expect_listed}",
+ run.listed
+ );
+ assert!(
+ run.list <= LIST_BUDGET,
+ "{name}: the list took {:?}",
+ run.list
+ );
+ assert!(
+ run.online_failed.is_empty(),
+ "{name}: Spaces that answer failed: {:?}",
+ &run.online_failed[..run.online_failed.len().min(5)]
+ );
+ assert!(
+ run.offline_answered.is_empty(),
+ "{name}: {:?}",
+ run.offline_answered
+ );
+ assert!(
+ run.online_max <= ONLINE_BUDGET,
+ "{name}: slowest reachable Space took {:?}",
+ run.online_max
+ );
+ assert!(
+ run.offline_max <= OFFLINE_BUDGET,
+ "{name}: slowest unreachable Space was marked after {:?}",
+ run.offline_max
+ );
+}
+
+async fn world() -> (World, usize) {
+ let hosts = size("CUA_SCALE_HOSTS", 20);
+ let per = size("CUA_SCALE_SPACES", 10);
+ let w = World::new(hosts, per).await;
+ let n = hosts * (per + 1) + LOCAL_SPACES;
+ (w, n)
+}
+
+/// `fraction` of the computers (the first ones) switched off.
+async fn down_scenario(name: &str, down: usize) {
+ let (mut w, n) = world().await;
+ let mut health = vec![Health::Online; w.computers.len()];
+ for (i, h) in health.iter_mut().enumerate().take(down) {
+ *h = Health::Offline;
+ w.set(i, *h);
+ }
+ let spaces = w.spaces();
+ let run = app_poll(&w, &spaces, &health).await;
+ check(name, &run, n);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn scale_all_online() {
+ down_scenario("all_online", 0).await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn scale_one_down() {
+ down_scenario("one_down", 1).await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn scale_quarter_down() {
+ let hosts = size("CUA_SCALE_HOSTS", 20);
+ down_scenario("25pct_down", hosts / 4).await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn scale_half_down() {
+ let hosts = size("CUA_SCALE_HOSTS", 20);
+ down_scenario("50pct_down", hosts / 2).await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn scale_ninety_down() {
+ let hosts = size("CUA_SCALE_HOSTS", 20);
+ down_scenario("90pct_down", hosts * 9 / 10).await;
+}
+
+/// Slow, thin and lossy links answer within budget; unresponsive and
+/// half-open computers are marked within budget and hold up nothing.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn scale_slow_and_unresponsive_computers() {
+ let (mut w, n) = world().await;
+ let faults = [
+ Fault::Latency(Duration::from_millis(150)),
+ Fault::Latency(Duration::from_millis(300)),
+ Fault::Bandwidth(256 * 1024),
+ Fault::Lossy {
+ p: 0.05,
+ stall: Duration::from_millis(200),
+ },
+ Fault::Blackhole,
+ Fault::Blackhole,
+ Fault::HalfOpen,
+ Fault::Reset,
+ ];
+ let mut health = vec![Health::Online; w.computers.len()];
+ for (i, f) in faults.iter().enumerate().take(w.computers.len()) {
+ health[i] = Health::Faulty(*f);
+ w.set(i, health[i]);
+ }
+ let spaces = w.spaces();
+ let run = app_poll(&w, &spaces, &health).await;
+ check("slow_and_unresponsive", &run, n);
+}
+
+/// Computers come back: reachable on the next poll, with no ghost and no
+/// duplicate rows; one that keeps failing is not dialed on every poll.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn scale_recovery_and_backoff() {
+ let (mut w, n) = world().await;
+ let hosts = w.computers.len();
+ let mut health = vec![Health::Online; hosts];
+ // Half off, two unresponsive.
+ for (i, h) in health.iter_mut().enumerate().take(hosts / 2) {
+ *h = Health::Offline;
+ w.set(i, *h);
+ }
+ for (i, h) in health.iter_mut().enumerate().skip(hosts / 2).take(2) {
+ *h = Health::Faulty(Fault::Blackhole);
+ w.set(i, *h);
+ }
+ let spaces = w.spaces();
+ let run = app_poll(&w, &spaces, &health).await;
+ check("recovery_before", &run, n);
+
+ // Polled again and again: the blackholed computers are not dialed on
+ // every poll (backoff), and the offline ones never are.
+ let dials_before: u64 = (hosts / 2..(hosts / 2 + 2).min(hosts))
+ .map(|i| w.computers[i].proxy.accepted())
+ .sum();
+ let proxied_before = w.relay.proxied().len();
+ for _ in 0..5 {
+ let run = app_poll(&w, &spaces, &health).await;
+ check("recovery_repoll", &run, n);
+ }
+ let dials_after: u64 = (hosts / 2..(hosts / 2 + 2).min(hosts))
+ .map(|i| w.computers[i].proxy.accepted())
+ .sum();
+ let offline_spaces = (hosts / 2) * w.computers[0].spaces.len();
+ let proxied = w.relay.proxied().len() - proxied_before;
+ eprintln!(
+ "SCALE backoff: blackholed dials over 5 polls={} relay requests={proxied}",
+ dials_after - dials_before
+ );
+ assert_eq!(
+ dials_after - dials_before,
+ 0,
+ "unresponsive computers were dialed again inside their backoff"
+ );
+ assert!(
+ proxied < offline_spaces,
+ "offline Spaces were dialed through the relay on every poll ({proxied} requests)"
+ );
+
+ // Everything comes back: each computer reconnects to the relay (a new
+ // session), as one that wakes up or boots does.
+ for (i, h) in health.iter_mut().enumerate() {
+ *h = Health::Online;
+ w.set(i, *h);
+ w.reconnect(i);
+ }
+ let t = Instant::now();
+ let run = app_poll(&w, &spaces, &health).await;
+ let took = t.elapsed();
+ check("recovered", &run, n);
+ assert!(took <= RECOVERY_BUDGET, "recovery took {took:?}");
+ let listed = spaces.list_all().await.unwrap();
+ let ids: BTreeSet<_> = listed.iter().map(|i| i.id.clone()).collect();
+ assert_eq!(ids.len(), listed.len(), "duplicate rows");
+ for id in w.remote_ids() {
+ assert!(ids.contains(&id), "{id} missing after recovery");
+ }
+
+ // A computer removed from the account leaves the list (no ghost).
+ let gone = w.computers[0].spaces[0].clone();
+ cua_host::RelayClient::new(&w.relay.url)
+ .unwrap()
+ .delete(TOKEN, &gone)
+ .await
+ .unwrap();
+ // Past the directory's cache age.
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ let listed = spaces.list_all().await.unwrap();
+ assert!(
+ !listed.iter().any(|i| i.id == format!("relay:{gone}")),
+ "a removed Space is still listed"
+ );
+}
+
+/// A daemon restart in the middle of a sync: the new process lists this
+/// machine's Spaces at once and converges with no duplicates.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn scale_restart_mid_sync() {
+ let (mut w, n) = world().await;
+ let hosts = w.computers.len();
+ let mut health = vec![Health::Online; hosts];
+ for (i, h) in health.iter_mut().enumerate().take(hosts / 4) {
+ *h = Health::Offline;
+ w.set(i, *h);
+ }
+ let first = w.spaces();
+ let polling = {
+ let first = first.clone();
+ let ids: Vec<String> = w.remote_ids().into_iter().collect();
+ tokio::spawn(async move {
+ let _ = first.list_all().await;
+ for id in ids {
+ let _ = tokio::time::timeout(APP_PROBE, first.space(&id)).await;
+ }
+ })
+ };
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ polling.abort();
+ drop(first);
+ let second = w.spaces();
+ let run = app_poll(&w, &second, &health).await;
+ check("restart_mid_sync", &run, n);
+ let listed = second.list_all().await.unwrap();
+ let ids: BTreeSet<_> = listed.iter().map(|i| i.id.clone()).collect();
+ assert_eq!(ids.len(), listed.len(), "duplicate rows after a restart");
+}
diff --git a/libs/cua/crates/cua-spacesd-client/src/testing.rs b/libs/cua/crates/cua-spacesd-client/src/testing.rs
index f2160fc274..75a7f9df1d 100644
--- a/libs/cua/crates/cua-spacesd-client/src/testing.rs
+++ b/libs/cua/crates/cua-spacesd-client/src/testing.rs
@@ -3402,3 +3402,154 @@ impl CutProxy {
format!("http://{}", self.addr)
}
}
+
+// --------------------------------------------------------------- FaultProxy
+
+/// What a [`FaultProxy`] does to the traffic through it. Switched at run
+/// time with [`FaultProxy::set`]; live connections follow the new fault from
+/// their next chunk.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum Fault {
+ /// Carries everything.
+ Pass,
+ /// Holds every chunk, both ways, this long (a slow, far link).
+ Latency(Duration),
+ /// At most this many bytes per second, both ways (a thin link).
+ Bandwidth(u64),
+ /// Each chunk, with probability `p`, is held for `stall` first (a lossy
+ /// link: TCP waits out a retransmission).
+ Lossy {
+ /// Probability, 0 to 1.
+ p: f64,
+ /// How long a lost chunk waits.
+ stall: Duration,
+ },
+ /// Accepts connections and never carries or answers anything (a
+ /// computer that went to sleep, a black-holed route).
+ Blackhole,
+ /// Accepts connections and closes them at once.
+ Reset,
+ /// Carries requests to the server but never one byte back (a half-open
+ /// connection).
+ HalfOpen,
+}
+
+/// A loopback TCP proxy that injects a [`Fault`] in front of `upstream`,
+/// for chaos tests: slow, thin, lossy, unresponsive or half-open links.
+pub struct FaultProxy {
+ /// Proxy address.
+ pub addr: SocketAddr,
+ fault: Arc<Mutex<Fault>>,
+ accepted: Arc<AtomicU64>,
+}
+
+impl FaultProxy {
+ /// Proxies to `upstream`, carrying everything until [`Self::set`].
+ pub async fn start(upstream: SocketAddr) -> Self {
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let fault = Arc::new(Mutex::new(Fault::Pass));
+ let accepted = Arc::new(AtomicU64::new(0));
+ let (f, n) = (fault.clone(), accepted.clone());
+ tokio::spawn(async move {
+ loop {
+ let Ok((client, _)) = listener.accept().await else {
+ return;
+ };
+ n.fetch_add(1, Ordering::SeqCst);
+ let f = f.clone();
+ tokio::spawn(async move {
+ let now = *f.lock().unwrap();
+ match now {
+ Fault::Reset => return,
+ Fault::Blackhole => {
+ // Held open, never read or answered, until the
+ // client gives up.
+ let mut client = client;
+ let mut sink = [0u8; 1];
+ use tokio::io::AsyncReadExt;
+ loop {
+ tokio::time::sleep(Duration::from_secs(3600)).await;
+ if client.read(&mut sink).await.unwrap_or(0) == 0 {
+ return;
+ }
+ }
+ }
+ _ => {}
+ }
+ let Ok(server) = TcpStream::connect(upstream).await else {
+ return;
+ };
+ let _ = client.set_nodelay(true);
+ let _ = server.set_nodelay(true);
+ let (cr, cw) = client.into_split();
+ let (sr, sw) = server.into_split();
+ tokio::select! {
+ _ = Self::pipe(cr, sw, f.clone(), false) => {}
+ _ = Self::pipe(sr, cw, f.clone(), true) => {}
+ }
+ });
+ }
+ });
+ Self {
+ addr,
+ fault,
+ accepted,
+ }
+ }
+
+ async fn pipe(
+ mut from: tokio::net::tcp::OwnedReadHalf,
+ mut to: tokio::net::tcp::OwnedWriteHalf,
+ fault: Arc<Mutex<Fault>>,
+ downstream: bool,
+ ) -> std::io::Result<()> {
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
+ let mut buf = vec![0u8; 16 * 1024];
+ loop {
+ let n = from.read(&mut buf).await?;
+ if n == 0 {
+ return Ok(());
+ }
+ let now = *fault.lock().unwrap();
+ match now {
+ Fault::Pass => {}
+ Fault::Latency(d) => tokio::time::sleep(d).await,
+ Fault::Bandwidth(bps) => {
+ let secs = n as f64 / bps.max(1) as f64;
+ tokio::time::sleep(Duration::from_secs_f64(secs)).await;
+ }
+ Fault::Lossy { p, stall } => {
+ if rand::random::<f64>() < p {
+ tokio::time::sleep(stall).await;
+ }
+ }
+ // Took effect after this connection opened: it goes quiet.
+ Fault::Blackhole | Fault::Reset => {
+ if now == Fault::Reset {
+ return Ok(());
+ }
+ std::future::pending::<()>().await;
+ }
+ Fault::HalfOpen if downstream => continue,
+ Fault::HalfOpen => {}
+ }
+ to.write_all(&buf[..n]).await?;
+ }
+ }
+
+ /// Switches the fault.
+ pub fn set(&self, fault: Fault) {
+ *self.fault.lock().unwrap() = fault;
+ }
+
+ /// Connections accepted so far (a retry storm shows here).
+ pub fn accepted(&self) -> u64 {
+ self.accepted.load(Ordering::SeqCst)
+ }
+
+ /// `http://127.0.0.1:<port>`.
+ pub fn url(&self) -> String {
+ format!("http://{}", self.addr)
+ }
+}