Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-spaces): publish running Spaces on the relay after a late cua-spacesd (#6581)

ddupont2ee22216370d
diff --git a/libs/cua/crates/cua-spaces/src/share.rs b/libs/cua/crates/cua-spaces/src/share.rs
index b0ddce796d..a9e87a2b6b 100644
--- a/libs/cua/crates/cua-spaces/src/share.rs
+++ b/libs/cua/crates/cua-spaces/src/share.rs
@@ -162,6 +162,42 @@ struct Attached {
}
mod publish;
+
+/// Spaces with a background re-attach running (one each).
+static REATTACHING: std::sync::LazyLock<std::sync::Mutex<std::collections::BTreeSet<String>>> =
+ std::sync::LazyLock::new(Default::default);
+
+static REATTACH_STEP_MS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(2000);
+
+fn reattach_step() -> std::time::Duration {
+ std::time::Duration::from_millis(REATTACH_STEP_MS.load(std::sync::atomic::Ordering::Relaxed))
+}
+
+/// Shortens the pauses between background re-attach and publish retries
+/// for a Space whose cua-spacesd is still waking (tests only).
+#[doc(hidden)]
+pub fn set_waking_retry_ms(ms: u64) {
+ REATTACH_STEP_MS.store(ms, std::sync::atomic::Ordering::Relaxed);
+ publish::set_waking_retry(std::time::Duration::from_millis(ms.saturating_mul(5)));
+}
+
+/// Whether the relay row names `host` as its host (always true without a
+/// host to name).
+pub(crate) fn host_matches(m: &cua_host::Machine, host: Option<&str>) -> bool {
+ host.is_none_or(|h| m.host.as_deref() == Some(h))
+}
+
+/// A Space whose image runs cua-spacesd, which is not answering yet.
+pub(crate) fn waking_error(space: &str) -> Error {
+ Error::SpacesdNotAvailable {
+ space: space.into(),
+ reason: "its cua-spacesd is not answering yet (the Space is still starting)".into(),
+ }
+}
+
+pub(crate) fn is_waking(e: &Error) -> bool {
+ matches!(e, Error::SpacesdNotAvailable { .. })
+}
pub use publish::{
CHANGED as PUBLISH_CHANGED, Off, PublishReport, PublishStatus, breaker_wait, jitter,
};
@@ -379,6 +415,16 @@ impl Spaces {
.collect()
}
+ /// This computer's own relay machine id when it is set up as a host
+ /// that is sharing (the id its Spaces name as their host).
+ pub(crate) fn own_host_id(&self) -> Option<String> {
+ let c = cua_host::Host::new(self.home_dir()).config().ok()??;
+ (c.sharing && !c.paused_signed_out)
+ .then_some(c.machine_id)
+ .flatten()
+ .filter(|m| !m.is_empty())
+ }
+
/// The relay machines Cua listed itself for Spaces on this computer
/// ([`publish`]): the Space is listed once, as the local one.
pub(crate) fn auto_published_machines(&self) -> std::collections::HashSet<String> {
@@ -548,17 +594,29 @@ impl Spaces {
}
return Ok(machine_id.clone());
}
- let s = self.space(&canonical).await?;
+ let mut s = self.space(&canonical).await?;
+ if !s.has_spacesd() && self.record_expects_spacesd(&id) {
+ // Its image runs cua-spacesd (it answered before): the Space is
+ // still waking, not unsupported. Dial once more now; if it still
+ // does not answer, say so (a later pass tries again).
+ self.drop_connection(&id).await;
+ s = self.space(&canonical).await?;
+ if !s.has_spacesd() {
+ return Err(waking_error(&canonical));
+ }
+ }
s.require(RELAY_ATTACH_FEATURE)?;
let all = self.attached();
let machine = match all.get(&canonical) {
Some(a) => a.machine.clone(),
None => format!("space-{:016x}", rand::random::<u64>()),
};
+ let host_id = self.own_host_id();
if wait
&& let Ok(m) = client.machine(&token, &machine).await
&& m.online
&& m.role == "owner"
+ && host_matches(&m, host_id.as_deref())
{
return Ok(machine);
}
@@ -583,7 +641,10 @@ impl Spaces {
id: machine.clone(),
name,
allow: vec![],
- host: None,
+ // This computer provides the Space, so the phone and
+ // the owner's other devices read its thread through
+ // this computer's daemon.
+ host: host_id.clone(),
meta,
},
)
@@ -995,9 +1056,46 @@ impl Spaces {
if !self.attached().contains_key(&canonical) {
return;
}
- if let Err(e) = self.relay_register(&canonical).await {
+ let first = self.relay_register(&canonical).await;
+ let Err(e) = first else { return };
+ if !is_waking(&e) {
tracing::warn!(space = %canonical, error = %e, "re-attaching to the relay after a boot failed");
+ return;
}
+ // Its cua-spacesd is not answering yet (a boot takes a while): keep
+ // trying in the background, one task per Space, with growing
+ // pauses, for a bounded time. The publish pass picks it up after.
+ if !REATTACHING
+ .lock()
+ .expect("reattaching")
+ .insert(canonical.clone())
+ {
+ return;
+ }
+ tracing::info!(space = %canonical, "cua-spacesd is not answering yet; re-attaching to the relay when it does");
+ let this = self.clone();
+ tokio::spawn(async move {
+ let step = reattach_step();
+ let mut pause = step;
+ let deadline = std::time::Instant::now() + step * 90;
+ loop {
+ tokio::time::sleep(pause).await;
+ if !this.attached().contains_key(&canonical) {
+ break;
+ }
+ match this.relay_register(&canonical).await {
+ Ok(_) => break,
+ Err(e) if is_waking(&e) && std::time::Instant::now() < deadline => {
+ pause = (pause * 2).min(step * 15);
+ }
+ Err(e) => {
+ tracing::warn!(space = %canonical, error = %e, "re-attaching to the relay after a boot failed");
+ break;
+ }
+ }
+ }
+ REATTACHING.lock().expect("reattaching").remove(&canonical);
+ });
}
pub async fn relay_register(&self, space: &str) -> Result<RelayRegistration> {
diff --git a/libs/cua/crates/cua-spaces/src/share/publish.rs b/libs/cua/crates/cua-spaces/src/share/publish.rs
index dd63df35e5..47145a3513 100644
--- a/libs/cua/crates/cua-spaces/src/share/publish.rs
+++ b/libs/cua/crates/cua-spaces/src/share/publish.rs
@@ -40,6 +40,12 @@ pub static CHANGED: tokio::sync::Notify = tokio::sync::Notify::const_new();
/// How long a Space that could not be listed is left alone.
const RETRY_AFTER: Duration = Duration::from_secs(60);
+/// How long a Space whose cua-spacesd is still waking is left alone.
+static WAKING_RETRY_MS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(10_000);
+
+pub(super) fn set_waking_retry(d: Duration) {
+ WAKING_RETRY_MS.store(d.as_millis() as u64, std::sync::atomic::Ordering::Relaxed);
+}
/// Calls to the relay in flight at once.
const AT_ONCE: usize = 4;
/// Most registrations and removals one pass starts; the rest wait for the
@@ -601,7 +607,9 @@ impl Spaces {
}
}
- // On: Spaces that are not online and not on their way.
+ // On: Spaces that are not online and not on their way (or whose row
+ // does not name this computer as host yet: it is registered again).
+ let host_id = self.own_host_id();
let now = Instant::now();
let due: Vec<String> = want
.iter()
@@ -610,7 +618,9 @@ impl Spaces {
let online = all
.get(id)
.and_then(|a| rows.get(&a.machine))
- .is_some_and(|m| m.online && m.role == "owner");
+ .is_some_and(|m| {
+ m.online && m.role == "owner" && super::host_matches(m, host_id.as_deref())
+ });
let (waiting, dialing) = memo(&home, |m| {
(
m.retry_at.get(id).is_some_and(|t| *t > now),
@@ -714,8 +724,15 @@ impl Spaces {
relay_error = Some((e, wait));
report.failed.push((id, msg));
} else {
+ let after = if super::is_waking(&e) {
+ Duration::from_millis(
+ WAKING_RETRY_MS.load(std::sync::atomic::Ordering::Relaxed),
+ )
+ } else {
+ RETRY_AFTER
+ };
memo(&home, |m| {
- m.retry_at.insert(id.clone(), Instant::now() + RETRY_AFTER)
+ m.retry_at.insert(id.clone(), Instant::now() + after)
});
report.failed.push((id, e.to_string()));
}
@@ -738,7 +755,13 @@ impl Spaces {
report.more = false;
}
None => {
- let finished = !over && !skipped && !young && report.failed.is_empty();
+ // A Space waiting for its retry is not finished: the next
+ // passes must come back to it, not rest until the verify.
+ let retrying = memo(&home, |m| {
+ want.iter()
+ .any(|i| m.retry_at.get(&i.id).is_some_and(|t| *t > Instant::now()))
+ });
+ let finished = !over && !skipped && !young && !retrying && report.failed.is_empty();
memo(&home, |m| {
m.breaker = Breaker::default();
m.dialing.retain(|_, t| *t > Instant::now());
diff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rs
index 7a7f9ab23c..8064f88e2f 100644
--- a/libs/cua/crates/cua-spaces/src/spaces.rs
+++ b/libs/cua/crates/cua-spaces/src/spaces.rs
@@ -1759,6 +1759,17 @@ impl Spaces {
.map_err(|e| Error::Cancelled(format!("the connect to this Space ended: {e}")))?
}
+ /// Whether the registry remembers cua-spacesd answering for `id`: its
+ /// image runs one, so a connection without it is a Space still waking.
+ pub(crate) fn record_expects_spacesd(&self, id: &SpaceId) -> bool {
+ self.inner
+ .registry
+ .get(&id.to_string())
+ .ok()
+ .flatten()
+ .is_some_and(|r| !r.spacesd_version.is_empty() || !r.features.is_empty())
+ }
+
/// The open connection to `key`, unless its route went stale or it can
/// move to the mesh now (then it is dropped, to connect again).
async fn cached_connection(&self, key: &str) -> Option<Space> {
@@ -2164,7 +2175,20 @@ impl Spaces {
};
let mut unavailable: Option<(String, String)> = None;
let env = match options {
- None => None,
+ None => {
+ // No spacesd port is published. For a Space this computer
+ // knows runs cua-spacesd (it answered before), that is a
+ // runtime still bringing the Space back (a restart
+ // publishes the ports a moment later), not an image
+ // without it: mark it so the next use dials again.
+ if matches!(id, SpaceId::Local { .. }) && self.record_expects_spacesd(id) {
+ unavailable = Some((
+ String::new(),
+ "the Space's cua-spacesd port is not published yet".into(),
+ ));
+ }
+ None
+ }
Some(options) => match SpacesdClient::connect(options.probe_timeout(probe)).await {
Ok(env) => Some(env),
Err(cua_spacesd_client::Error::SpacesdNotAvailable { endpoint, reason }) => {
diff --git a/libs/cua/crates/cua-spaces/src/thread_route.rs b/libs/cua/crates/cua-spaces/src/thread_route.rs
index 3432fd7d67..b9fc59aaf2 100644
--- a/libs/cua/crates/cua-spaces/src/thread_route.rs
+++ b/libs/cua/crates/cua-spaces/src/thread_route.rs
@@ -385,6 +385,18 @@ impl crate::host_spaces::HostSpacesServer {
&& (p.relay_machine == machine || (caller.space.is_none() && p.local_space == id))
}) {
Some(p) => p.local_space.clone(),
+ // A Space of this computer that Cua listed on the account
+ // itself (it names this computer as host): its thread is here,
+ // under its local id.
+ None if !machine.is_empty()
+ && let Some((local, _)) = self
+ .spaces()
+ .attached_machines()
+ .into_iter()
+ .find(|(_, m)| *m == machine) =>
+ {
+ local
+ }
// The owner, from another of their devices: any of their Spaces.
None if caller.is_owner() && !id.is_empty() => id.to_string(),
None => {
diff --git a/libs/cua/crates/cua-spaces/tests/relay_publish.rs b/libs/cua/crates/cua-spaces/tests/relay_publish.rs
index 5ab2d17fee..552b80ffa7 100644
--- a/libs/cua/crates/cua-spaces/tests/relay_publish.rs
+++ b/libs/cua/crates/cua-spaces/tests/relay_publish.rs
@@ -962,3 +962,159 @@ async fn a_failing_relay_is_told_and_forgotten_when_listing_works() {
let st = other.relay_publish_status(&w.spaces.list().unwrap()).await;
assert_eq!((st.listed, st.not_listed.len()), (1, 0), "{st:?}");
}
+
+impl World {
+ /// The runtime publishes no port for `name`'s cua-spacesd (a restarted
+ /// runtime brings the ports back a moment after the Space), or again.
+ fn ports(&self, name: &str, port: Option<u16>) {
+ let mut m = self.rt.spacesd.lock().unwrap();
+ match port {
+ Some(p) => m.insert(name.into(), p),
+ None => m.remove(name),
+ };
+ }
+
+ fn port_of(&self, name: &str) -> u16 {
+ self.rt.spacesd.lock().unwrap()[name]
+ }
+
+ /// A daemon restart: a new `Spaces` over the same home and state, with
+ /// no connection cached.
+ fn restarted(&self) -> Spaces {
+ Spaces::builder()
+ .home(self.home.path())
+ .sandboxes(
+ Sandboxes::builder()
+ .local(self.rt.clone())
+ .state_dir(self.state.path())
+ .build(),
+ )
+ .relay(RelayAccount::new(&self.relay.url, self.account.clone()))
+ .build()
+ }
+}
+
+/// A Space that comes back after a runtime restart before its cua-spacesd
+/// answers is not "unsupported": it is listed once the driver answers, with
+/// no help, and the retries are paced, not a hot loop.
+#[tokio::test]
+async fn a_space_whose_spacesd_wakes_late_is_listed_when_it_does() {
+ let w = World::new().await;
+ cua_spaces::share::set_waking_retry_ms(100);
+ w.create("late", "linux", Kind::Container).await;
+ let port = w.port_of("late");
+ // The runtime and the daemon restarted; the port is not back yet.
+ w.ports("late", None);
+ let spaces = w.restarted();
+
+ let r = spaces.publish_to_relay().await.unwrap();
+ assert!(r.published.is_empty(), "{r:?}");
+ assert_eq!(r.failed.len(), 1, "{r:?}");
+ assert!(
+ r.failed[0].1.contains("not answering yet"),
+ "waking, not unsupported: {r:?}"
+ );
+ assert!(!r.failed[0].1.contains("runs no cua-spacesd"), "{r:?}");
+ // Bounded: an immediate second pass leaves it alone (paced retry).
+ let again = spaces.publish_to_relay().await.unwrap();
+ assert!(
+ again.failed.is_empty() && again.published.is_empty(),
+ "{again:?}"
+ );
+
+ // The driver answers: a later pass lists it.
+ w.ports("late", Some(port));
+ let mut listed = false;
+ for _ in 0..40 {
+ tokio::time::sleep(Duration::from_millis(150)).await;
+ let r = spaces.publish_to_relay().await.unwrap();
+ if !r.published.is_empty() {
+ listed = true;
+ break;
+ }
+ }
+ assert!(listed, "never listed");
+ assert_eq!(w.rows().await.len(), 1);
+ assert!(w.attached("late"));
+}
+
+/// Registering a Space whose cua-spacesd is not up yet says it is waking
+/// (not that its image has none), and works as soon as it answers.
+#[tokio::test]
+async fn registering_a_waking_space_is_not_an_unsupported_one() {
+ let w = World::new().await;
+ cua_spaces::share::set_waking_retry_ms(100);
+ let id = w.create("wake", "linux", Kind::Container).await;
+ let port = w.port_of("wake");
+ w.ports("wake", None);
+ let spaces = w.restarted();
+ let e = spaces.relay_register(&id).await.unwrap_err();
+ assert!(
+ matches!(e, cua_spaces::Error::SpacesdNotAvailable { .. }),
+ "{e:?}"
+ );
+ w.ports("wake", Some(port));
+ spaces.relay_register(&id).await.unwrap();
+ assert!(w.attached("wake"));
+}
+
+/// A computer set up as a host: its relay machine exists on the account and
+/// its config names it.
+async fn set_up_host(w: &World, id: &str) {
+ cua_host::RelayClient::new(&w.relay.url)
+ .unwrap()
+ .register(
+ "ada-token",
+ &cua_host::relay::RegisterRequest {
+ id: id.into(),
+ name: "Studio".into(),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+ let dir = w.home.path().join("host");
+ std::fs::create_dir_all(&dir).unwrap();
+ std::fs::write(
+ dir.join("config.json"),
+ serde_json::json!({
+ "mode": "relay", "name": "Studio", "runner": "process",
+ "driver_bin": "/nonexistent", "sharing": true, "machine_id": id,
+ })
+ .to_string(),
+ )
+ .unwrap();
+}
+
+/// A Space this computer lists names this computer as its host, so the
+/// phone reads its thread through the computer's daemon; a row registered
+/// before that (by hand, without a host) gets the host on the next pass,
+/// and a Space re-registered after a restart keeps it.
+#[tokio::test]
+async fn a_listed_space_names_this_computer_as_its_host() {
+ let w = World::new().await;
+ let id = w.create("hosted", "linux", Kind::Container).await;
+ let old = w.create("old", "linux", Kind::Container).await;
+ // Registered by hand before the computer was a host: no host on the row.
+ w.spaces.relay_register(&old).await.unwrap();
+ assert!(w.rows().await[0].host.is_none());
+ set_up_host(&w, "host-mac").await;
+
+ w.spaces.publish_to_relay().await.unwrap();
+ let rows = w.rows().await;
+ assert_eq!(rows.len(), 2, "{rows:#?}");
+ for m in &rows {
+ assert_eq!(m.host.as_deref(), Some("host-mac"), "{m:#?}");
+ }
+ assert!(w.attached("old"), "repaired in place, driver on the relay");
+
+ // A restart re-registers the same machine: the host stays.
+ let manual = w.spaces.relay_register(&id).await.unwrap();
+ let row = w
+ .rows()
+ .await
+ .into_iter()
+ .find(|m| m.id == manual.machine)
+ .unwrap();
+ assert_eq!(row.host.as_deref(), Some("host-mac"));
+}