Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-spaces): publish running Spaces on the relay after a late cua-spacesd (#6581)
ddupont2ee22216370d
diff --git a/libs/cua/crates/cua-spaces/src/share.rs b/libs/cua/crates/cua-spaces/src/share.rsindex b0ddce796d..a9e87a2b6b 100644--- a/libs/cua/crates/cua-spaces/src/share.rs+++ b/libs/cua/crates/cua-spaces/src/share.rs@@ -162,6 +162,42 @@ struct Attached {}mod publish;++/// Spaces with a background re-attach running (one each).+static REATTACHING: std::sync::LazyLock<std::sync::Mutex<std::collections::BTreeSet<String>>> =+ std::sync::LazyLock::new(Default::default);++static REATTACH_STEP_MS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(2000);++fn reattach_step() -> std::time::Duration {+ std::time::Duration::from_millis(REATTACH_STEP_MS.load(std::sync::atomic::Ordering::Relaxed))+}++/// Shortens the pauses between background re-attach and publish retries+/// for a Space whose cua-spacesd is still waking (tests only).+#[doc(hidden)]+pub fn set_waking_retry_ms(ms: u64) {+ REATTACH_STEP_MS.store(ms, std::sync::atomic::Ordering::Relaxed);+ publish::set_waking_retry(std::time::Duration::from_millis(ms.saturating_mul(5)));+}++/// Whether the relay row names `host` as its host (always true without a+/// host to name).+pub(crate) fn host_matches(m: &cua_host::Machine, host: Option<&str>) -> bool {+ host.is_none_or(|h| m.host.as_deref() == Some(h))+}++/// A Space whose image runs cua-spacesd, which is not answering yet.+pub(crate) fn waking_error(space: &str) -> Error {+ Error::SpacesdNotAvailable {+ space: space.into(),+ reason: "its cua-spacesd is not answering yet (the Space is still starting)".into(),+ }+}++pub(crate) fn is_waking(e: &Error) -> bool {+ matches!(e, Error::SpacesdNotAvailable { .. })+}pub use publish::{CHANGED as PUBLISH_CHANGED, Off, PublishReport, PublishStatus, breaker_wait, jitter,};@@ -379,6 +415,16 @@ impl Spaces {.collect()}+ /// This computer's own relay machine id when it is set up as a host+ /// that is sharing (the id its Spaces name as their host).+ pub(crate) fn own_host_id(&self) -> Option<String> {+ let c = cua_host::Host::new(self.home_dir()).config().ok()??;+ (c.sharing && !c.paused_signed_out)+ .then_some(c.machine_id)+ .flatten()+ .filter(|m| !m.is_empty())+ }+/// The relay machines Cua listed itself for Spaces on this computer/// ([`publish`]): the Space is listed once, as the local one.pub(crate) fn auto_published_machines(&self) -> std::collections::HashSet<String> {@@ -548,17 +594,29 @@ impl Spaces {}return Ok(machine_id.clone());}- let s = self.space(&canonical).await?;+ let mut s = self.space(&canonical).await?;+ if !s.has_spacesd() && self.record_expects_spacesd(&id) {+ // Its image runs cua-spacesd (it answered before): the Space is+ // still waking, not unsupported. Dial once more now; if it still+ // does not answer, say so (a later pass tries again).+ self.drop_connection(&id).await;+ s = self.space(&canonical).await?;+ if !s.has_spacesd() {+ return Err(waking_error(&canonical));+ }+ }s.require(RELAY_ATTACH_FEATURE)?;let all = self.attached();let machine = match all.get(&canonical) {Some(a) => a.machine.clone(),None => format!("space-{:016x}", rand::random::<u64>()),};+ let host_id = self.own_host_id();if wait&& let Ok(m) = client.machine(&token, &machine).await&& m.online&& m.role == "owner"+ && host_matches(&m, host_id.as_deref()){return Ok(machine);}@@ -583,7 +641,10 @@ impl Spaces {id: machine.clone(),name,allow: vec![],- host: None,+ // This computer provides the Space, so the phone and+ // the owner's other devices read its thread through+ // this computer's daemon.+ host: host_id.clone(),meta,},)@@ -995,9 +1056,46 @@ impl Spaces {if !self.attached().contains_key(&canonical) {return;}- if let Err(e) = self.relay_register(&canonical).await {+ let first = self.relay_register(&canonical).await;+ let Err(e) = first else { return };+ if !is_waking(&e) {tracing::warn!(space = %canonical, error = %e, "re-attaching to the relay after a boot failed");+ return;}+ // Its cua-spacesd is not answering yet (a boot takes a while): keep+ // trying in the background, one task per Space, with growing+ // pauses, for a bounded time. The publish pass picks it up after.+ if !REATTACHING+ .lock()+ .expect("reattaching")+ .insert(canonical.clone())+ {+ return;+ }+ tracing::info!(space = %canonical, "cua-spacesd is not answering yet; re-attaching to the relay when it does");+ let this = self.clone();+ tokio::spawn(async move {+ let step = reattach_step();+ let mut pause = step;+ let deadline = std::time::Instant::now() + step * 90;+ loop {+ tokio::time::sleep(pause).await;+ if !this.attached().contains_key(&canonical) {+ break;+ }+ match this.relay_register(&canonical).await {+ Ok(_) => break,+ Err(e) if is_waking(&e) && std::time::Instant::now() < deadline => {+ pause = (pause * 2).min(step * 15);+ }+ Err(e) => {+ tracing::warn!(space = %canonical, error = %e, "re-attaching to the relay after a boot failed");+ break;+ }+ }+ }+ REATTACHING.lock().expect("reattaching").remove(&canonical);+ });}pub async fn relay_register(&self, space: &str) -> Result<RelayRegistration> {diff --git a/libs/cua/crates/cua-spaces/src/share/publish.rs b/libs/cua/crates/cua-spaces/src/share/publish.rsindex dd63df35e5..47145a3513 100644--- a/libs/cua/crates/cua-spaces/src/share/publish.rs+++ b/libs/cua/crates/cua-spaces/src/share/publish.rs@@ -40,6 +40,12 @@ pub static CHANGED: tokio::sync::Notify = tokio::sync::Notify::const_new();/// How long a Space that could not be listed is left alone.const RETRY_AFTER: Duration = Duration::from_secs(60);+/// How long a Space whose cua-spacesd is still waking is left alone.+static WAKING_RETRY_MS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(10_000);++pub(super) fn set_waking_retry(d: Duration) {+ WAKING_RETRY_MS.store(d.as_millis() as u64, std::sync::atomic::Ordering::Relaxed);+}/// Calls to the relay in flight at once.const AT_ONCE: usize = 4;/// Most registrations and removals one pass starts; the rest wait for the@@ -601,7 +607,9 @@ impl Spaces {}}- // On: Spaces that are not online and not on their way.+ // On: Spaces that are not online and not on their way (or whose row+ // does not name this computer as host yet: it is registered again).+ let host_id = self.own_host_id();let now = Instant::now();let due: Vec<String> = want.iter()@@ -610,7 +618,9 @@ impl Spaces {let online = all.get(id).and_then(|a| rows.get(&a.machine))- .is_some_and(|m| m.online && m.role == "owner");+ .is_some_and(|m| {+ m.online && m.role == "owner" && super::host_matches(m, host_id.as_deref())+ });let (waiting, dialing) = memo(&home, |m| {(m.retry_at.get(id).is_some_and(|t| *t > now),@@ -714,8 +724,15 @@ impl Spaces {relay_error = Some((e, wait));report.failed.push((id, msg));} else {+ let after = if super::is_waking(&e) {+ Duration::from_millis(+ WAKING_RETRY_MS.load(std::sync::atomic::Ordering::Relaxed),+ )+ } else {+ RETRY_AFTER+ };memo(&home, |m| {- m.retry_at.insert(id.clone(), Instant::now() + RETRY_AFTER)+ m.retry_at.insert(id.clone(), Instant::now() + after)});report.failed.push((id, e.to_string()));}@@ -738,7 +755,13 @@ impl Spaces {report.more = false;}None => {- let finished = !over && !skipped && !young && report.failed.is_empty();+ // A Space waiting for its retry is not finished: the next+ // passes must come back to it, not rest until the verify.+ let retrying = memo(&home, |m| {+ want.iter()+ .any(|i| m.retry_at.get(&i.id).is_some_and(|t| *t > Instant::now()))+ });+ let finished = !over && !skipped && !young && !retrying && report.failed.is_empty();memo(&home, |m| {m.breaker = Breaker::default();m.dialing.retain(|_, t| *t > Instant::now());diff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rsindex 7a7f9ab23c..8064f88e2f 100644--- a/libs/cua/crates/cua-spaces/src/spaces.rs+++ b/libs/cua/crates/cua-spaces/src/spaces.rs@@ -1759,6 +1759,17 @@ impl Spaces {.map_err(|e| Error::Cancelled(format!("the connect to this Space ended: {e}")))?}+ /// Whether the registry remembers cua-spacesd answering for `id`: its+ /// image runs one, so a connection without it is a Space still waking.+ pub(crate) fn record_expects_spacesd(&self, id: &SpaceId) -> bool {+ self.inner+ .registry+ .get(&id.to_string())+ .ok()+ .flatten()+ .is_some_and(|r| !r.spacesd_version.is_empty() || !r.features.is_empty())+ }+/// The open connection to `key`, unless its route went stale or it can/// move to the mesh now (then it is dropped, to connect again).async fn cached_connection(&self, key: &str) -> Option<Space> {@@ -2164,7 +2175,20 @@ impl Spaces {};let mut unavailable: Option<(String, String)> = None;let env = match options {- None => None,+ None => {+ // No spacesd port is published. For a Space this computer+ // knows runs cua-spacesd (it answered before), that is a+ // runtime still bringing the Space back (a restart+ // publishes the ports a moment later), not an image+ // without it: mark it so the next use dials again.+ if matches!(id, SpaceId::Local { .. }) && self.record_expects_spacesd(id) {+ unavailable = Some((+ String::new(),+ "the Space's cua-spacesd port is not published yet".into(),+ ));+ }+ None+ }Some(options) => match SpacesdClient::connect(options.probe_timeout(probe)).await {Ok(env) => Some(env),Err(cua_spacesd_client::Error::SpacesdNotAvailable { endpoint, reason }) => {diff --git a/libs/cua/crates/cua-spaces/src/thread_route.rs b/libs/cua/crates/cua-spaces/src/thread_route.rsindex 3432fd7d67..b9fc59aaf2 100644--- a/libs/cua/crates/cua-spaces/src/thread_route.rs+++ b/libs/cua/crates/cua-spaces/src/thread_route.rs@@ -385,6 +385,18 @@ impl crate::host_spaces::HostSpacesServer {&& (p.relay_machine == machine || (caller.space.is_none() && p.local_space == id))}) {Some(p) => p.local_space.clone(),+ // A Space of this computer that Cua listed on the account+ // itself (it names this computer as host): its thread is here,+ // under its local id.+ None if !machine.is_empty()+ && let Some((local, _)) = self+ .spaces()+ .attached_machines()+ .into_iter()+ .find(|(_, m)| *m == machine) =>+ {+ local+ }// The owner, from another of their devices: any of their Spaces.None if caller.is_owner() && !id.is_empty() => id.to_string(),None => {diff --git a/libs/cua/crates/cua-spaces/tests/relay_publish.rs b/libs/cua/crates/cua-spaces/tests/relay_publish.rsindex 5ab2d17fee..552b80ffa7 100644--- a/libs/cua/crates/cua-spaces/tests/relay_publish.rs+++ b/libs/cua/crates/cua-spaces/tests/relay_publish.rs@@ -962,3 +962,159 @@ async fn a_failing_relay_is_told_and_forgotten_when_listing_works() {let st = other.relay_publish_status(&w.spaces.list().unwrap()).await;assert_eq!((st.listed, st.not_listed.len()), (1, 0), "{st:?}");}++impl World {+ /// The runtime publishes no port for `name`'s cua-spacesd (a restarted+ /// runtime brings the ports back a moment after the Space), or again.+ fn ports(&self, name: &str, port: Option<u16>) {+ let mut m = self.rt.spacesd.lock().unwrap();+ match port {+ Some(p) => m.insert(name.into(), p),+ None => m.remove(name),+ };+ }++ fn port_of(&self, name: &str) -> u16 {+ self.rt.spacesd.lock().unwrap()[name]+ }++ /// A daemon restart: a new `Spaces` over the same home and state, with+ /// no connection cached.+ fn restarted(&self) -> Spaces {+ Spaces::builder()+ .home(self.home.path())+ .sandboxes(+ Sandboxes::builder()+ .local(self.rt.clone())+ .state_dir(self.state.path())+ .build(),+ )+ .relay(RelayAccount::new(&self.relay.url, self.account.clone()))+ .build()+ }+}++/// A Space that comes back after a runtime restart before its cua-spacesd+/// answers is not "unsupported": it is listed once the driver answers, with+/// no help, and the retries are paced, not a hot loop.+#[tokio::test]+async fn a_space_whose_spacesd_wakes_late_is_listed_when_it_does() {+ let w = World::new().await;+ cua_spaces::share::set_waking_retry_ms(100);+ w.create("late", "linux", Kind::Container).await;+ let port = w.port_of("late");+ // The runtime and the daemon restarted; the port is not back yet.+ w.ports("late", None);+ let spaces = w.restarted();++ let r = spaces.publish_to_relay().await.unwrap();+ assert!(r.published.is_empty(), "{r:?}");+ assert_eq!(r.failed.len(), 1, "{r:?}");+ assert!(+ r.failed[0].1.contains("not answering yet"),+ "waking, not unsupported: {r:?}"+ );+ assert!(!r.failed[0].1.contains("runs no cua-spacesd"), "{r:?}");+ // Bounded: an immediate second pass leaves it alone (paced retry).+ let again = spaces.publish_to_relay().await.unwrap();+ assert!(+ again.failed.is_empty() && again.published.is_empty(),+ "{again:?}"+ );++ // The driver answers: a later pass lists it.+ w.ports("late", Some(port));+ let mut listed = false;+ for _ in 0..40 {+ tokio::time::sleep(Duration::from_millis(150)).await;+ let r = spaces.publish_to_relay().await.unwrap();+ if !r.published.is_empty() {+ listed = true;+ break;+ }+ }+ assert!(listed, "never listed");+ assert_eq!(w.rows().await.len(), 1);+ assert!(w.attached("late"));+}++/// Registering a Space whose cua-spacesd is not up yet says it is waking+/// (not that its image has none), and works as soon as it answers.+#[tokio::test]+async fn registering_a_waking_space_is_not_an_unsupported_one() {+ let w = World::new().await;+ cua_spaces::share::set_waking_retry_ms(100);+ let id = w.create("wake", "linux", Kind::Container).await;+ let port = w.port_of("wake");+ w.ports("wake", None);+ let spaces = w.restarted();+ let e = spaces.relay_register(&id).await.unwrap_err();+ assert!(+ matches!(e, cua_spaces::Error::SpacesdNotAvailable { .. }),+ "{e:?}"+ );+ w.ports("wake", Some(port));+ spaces.relay_register(&id).await.unwrap();+ assert!(w.attached("wake"));+}++/// A computer set up as a host: its relay machine exists on the account and+/// its config names it.+async fn set_up_host(w: &World, id: &str) {+ cua_host::RelayClient::new(&w.relay.url)+ .unwrap()+ .register(+ "ada-token",+ &cua_host::relay::RegisterRequest {+ id: id.into(),+ name: "Studio".into(),+ ..Default::default()+ },+ )+ .await+ .unwrap();+ let dir = w.home.path().join("host");+ std::fs::create_dir_all(&dir).unwrap();+ std::fs::write(+ dir.join("config.json"),+ serde_json::json!({+ "mode": "relay", "name": "Studio", "runner": "process",+ "driver_bin": "/nonexistent", "sharing": true, "machine_id": id,+ })+ .to_string(),+ )+ .unwrap();+}++/// A Space this computer lists names this computer as its host, so the+/// phone reads its thread through the computer's daemon; a row registered+/// before that (by hand, without a host) gets the host on the next pass,+/// and a Space re-registered after a restart keeps it.+#[tokio::test]+async fn a_listed_space_names_this_computer_as_its_host() {+ let w = World::new().await;+ let id = w.create("hosted", "linux", Kind::Container).await;+ let old = w.create("old", "linux", Kind::Container).await;+ // Registered by hand before the computer was a host: no host on the row.+ w.spaces.relay_register(&old).await.unwrap();+ assert!(w.rows().await[0].host.is_none());+ set_up_host(&w, "host-mac").await;++ w.spaces.publish_to_relay().await.unwrap();+ let rows = w.rows().await;+ assert_eq!(rows.len(), 2, "{rows:#?}");+ for m in &rows {+ assert_eq!(m.host.as_deref(), Some("host-mac"), "{m:#?}");+ }+ assert!(w.attached("old"), "repaired in place, driver on the relay");++ // A restart re-registers the same machine: the host stays.+ let manual = w.spaces.relay_register(&id).await.unwrap();+ let row = w+ .rows()+ .await+ .into_iter()+ .find(|m| m.id == manual.machine)+ .unwrap();+ assert_eq!(row.host.as_deref(), Some("host-mac"));+}