Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(spaces): reattach Cua Volume after a daemon restart; keep the cua keychain unlocked (#4458)

ddupont9eb7edbfc7c7
diff --git a/docs/content/docs/spaces/guides/cua-volume.mdx b/docs/content/docs/spaces/guides/cua-volume.mdx
index 1693caa73c..b3799b28be 100644
--- a/docs/content/docs/spaces/guides/cua-volume.mdx
+++ b/docs/content/docs/spaces/guides/cua-volume.mdx
@@ -191,6 +191,25 @@ The mount goes away when the Space detaches the volume, when its session is
revoked and when the guest's cua-spacesd stops. `cua-spacesd doctor` checks
the mount end to end (`volume.mount`).
+### When the guest's cua-spacesd restarts
+
+An update, a crash or a reboot restarts the guest's cua-spacesd, which holds
+the volume's connection and the network-sharing (hotspot) connection. You do
+not need to reconnect the Space:
+
+- On a clean stop (`SIGTERM`) the daemon unmounts the volume before it exits,
+ so no dead mount is left behind.
+- When it starts, the daemon unmounts anything a previous daemon left at the
+ mount path (a crash, a force-quit) before it does anything else.
+- Your machine notices the connection closed, or that the new daemon reports
+ the volume detached, and mounts the volume again, then restarts the hotspot
+ if one was on. It retries with a growing wait (2 s, then 4 s, up to a
+ minute) while the daemon is still coming up, and keeps a persistent agent's
+ view on the volume.
+- A mount whose server is gone never hangs a program for long: the NFS mount
+ is `soft`, interruptible and times out in seconds, and the system marks it
+ dead and unmounts it (`deadtimeout`).
+
## Sync across devices
Two machines using the same bucket see each other's changes within
diff --git a/docs/content/docs/spaces/guides/teleport-an-app.mdx b/docs/content/docs/spaces/guides/teleport-an-app.mdx
index 39154f4e51..fd4178156a 100644
--- a/docs/content/docs/spaces/guides/teleport-an-app.mdx
+++ b/docs/content/docs/spaces/guides/teleport-an-app.mdx
@@ -106,6 +106,28 @@ For a web UI, `@trycua/cua/teleport` has the headless picker state machine,
`<cua-teleport-picker>` and `<cua-teleport-drop>` web components and
drag-payload helpers over a `TeleportHost` you provide.
+## The `cua` keychain on macOS Spaces
+
+Chrome and other Chromium browsers keep their cookie key ("Safe Storage") in
+the Keychain. Session teleport installs that key into the Space's login
+keychain and pre-authorizes the browser. When the login keychain's password
+is not one the Space knows, cua-spacesd keeps the key in its own keychain,
+`~/Library/Keychains/cua.keychain-db`, instead. It behaves like this so that
+no password dialog ever appears in a Space nobody is watching:
+
+- The `cua` keychain is on the keychain search list, after the login keychain.
+ It is not the default keychain, so system services such as Spotlight do not
+ reach for it.
+- cua-spacesd unlocks it when it starts and again every 20 seconds, so it is
+ open after a reboot and within seconds of a wake. The password comes from
+ the Space's protected configuration (`CUA_ENV_LOGIN_KEYCHAIN_PW`,
+ `~/.cua/spacesd/keychain-password`, `/etc/cua/keychain-password`, then the
+ image's account password).
+- It never auto-locks: no lock after a timeout and none on sleep
+ (`security set-keychain-settings` without `-l` or `-u`).
+- Most Spaces never create it: the Cua macOS image gives the login keychain
+ the account password, so the key goes there.
+
## Session teleport
To move only an app's signed-in state (scope `tabs` or `full`) into a Space,
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/server.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/server.rs
index 01569cf626..3dc77a37b7 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/server.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/server.rs
@@ -137,6 +137,7 @@ impl ServerBuilder {
let fs = FsState::new(ctx.clone());
let tunnel = TunnelState::new(ctx.clone());
let volume = VolumeShared::new(ctx.clone());
+ let volume_state = volume.clone();
let teleport = TeleportServiceImpl::new(ctx.clone(), receiver);
let mut core_features = vec![
// Unix openpty; Windows ConPTY.
@@ -346,7 +347,12 @@ impl ServerBuilder {
let app = http.fallback_service(grpc).layer(cors_layer()).layer(
axum::middleware::from_fn_with_state(ctx.clone(), crate::browser_guard::guard),
);
- Server { ctx, app, manifest }
+ Server {
+ ctx,
+ app,
+ manifest,
+ volume: volume_state,
+ }
}
}
@@ -392,6 +398,7 @@ pub struct Server {
ctx: ServerContext,
app: Router,
manifest: RouteManifest,
+ volume: VolumeShared,
}
impl Server {
@@ -421,6 +428,14 @@ impl Server {
if crate::telemetry::spawn_reporter(&self.ctx.config().data_dir) {
tracing::info!("aggregate telemetry on (CUA_SPACESD_TELEMETRY)");
}
+ // A mount left by a previous daemon that died without unmounting
+ // answers nothing; clear it before anything touches the path.
+ self.volume.clear_stale_mount().await;
+ // The `cua` keychain (Chrome's fallback home for its Safe Storage
+ // key) must never be locked: whatever touches it while it is
+ // raises a password dialog nobody can answer.
+ #[cfg(target_os = "macos")]
+ spawn_keychain_keeper(self.ctx.shutdown_token());
let shutdown = self.ctx.shutdown_token();
let grace = self.ctx.config().shutdown_grace;
let keepalive = self.ctx.config().keepalive_interval;
@@ -499,6 +514,9 @@ impl Server {
{
tracing::warn!(?grace, "grace period elapsed with connections still open");
}
+ // Unmount the volume before the process exits: a daemon that leaves
+ // its NFS mount behind leaves a dead mount that hangs `ls`.
+ self.volume.finish().await;
Ok(())
}
}
@@ -527,3 +545,23 @@ pub async fn spawn_local(server: Server) -> std::io::Result<SocketAddr> {
});
Ok(addr)
}
+
+/// Unlocks the `cua` keychain now and keeps it unlocked (after a reboot, a
+/// wake, a timeout): see `cua_spacesd_teleport::keychain::keep_cua_keychain_unlocked`.
+#[cfg(target_os = "macos")]
+fn spawn_keychain_keeper(shutdown: tokio_util::sync::CancellationToken) {
+ tokio::spawn(async move {
+ let host = cua_spacesd_teleport::default_host();
+ loop {
+ let h = host.clone();
+ let _ = tokio::task::spawn_blocking(move || {
+ cua_spacesd_teleport::keychain::keep_cua_keychain_unlocked(&*h)
+ })
+ .await;
+ tokio::select! {
+ _ = tokio::time::sleep(cua_spacesd_teleport::keychain::KEEPER_INTERVAL) => {}
+ _ = shutdown.cancelled() => return,
+ }
+ }
+ });
+}
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs
index 72a0a35197..a07d74738b 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs
@@ -178,6 +178,8 @@ impl Session {
pub struct VolumeShared {
ctx: ServerContext,
current: Arc<Mutex<Option<Arc<Session>>>>,
+ /// Held while a detach unmounts, so shutdown can wait for it.
+ unmounting: Arc<tokio::sync::Mutex<()>>,
}
impl VolumeShared {
@@ -186,6 +188,7 @@ impl VolumeShared {
let state = Self {
ctx,
current: Arc::default(),
+ unmounting: Arc::default(),
};
if let Ok(handle) = tokio::runtime::Handle::try_current() {
let mut revoked = state.ctx.session_revocations();
@@ -212,6 +215,7 @@ impl VolumeShared {
/// Unmounts and forgets the session (`id` must match when given).
/// Returns whether one was attached.
async fn detach(&self, id: Option<&str>) -> bool {
+ let _unmounting = self.unmounting.lock().await;
let session = {
let mut slot = self.current.lock().expect("volume");
match slot.as_ref() {
@@ -230,6 +234,116 @@ impl VolumeShared {
tracing::info!(volume = %session.id, "volume detached");
true
}
+
+ /// Shutdown: unmounts whatever is mounted and waits for an unmount
+ /// already under way (bounded, so a stuck umount cannot hold the
+ /// process).
+ pub async fn finish(&self) {
+ let _ = tokio::time::timeout(SHUTDOWN_UNMOUNT_TIMEOUT, self.detach(None)).await;
+ }
+
+ /// Start-up: unmounts a mount a previous daemon left at the volume path
+ /// (it exited without unmounting; nothing serves it, so every access
+ /// hangs until the NFS client gives up). Never blocks start-up for long.
+ pub async fn clear_stale_mount(&self) {
+ let _ = tokio::time::timeout(Duration::from_secs(30), clear_stale_mounts()).await;
+ }
+}
+
+/// How long shutdown waits for the volume to unmount.
+const SHUTDOWN_UNMOUNT_TIMEOUT: Duration = Duration::from_secs(25);
+
+/// The paths a volume may have been mounted at.
+fn stale_candidates() -> Vec<PathBuf> {
+ if cfg!(windows) {
+ return vec![];
+ }
+ let home = std::env::var_os("HOME")
+ .map(PathBuf::from)
+ .unwrap_or_else(|| PathBuf::from("/tmp"));
+ let mut v = vec![home.join("Cua Volume")];
+ if cfg!(target_os = "linux") {
+ v.push(PathBuf::from("/volume"));
+ }
+ v
+}
+
+/// Whether `table` (macOS `mount` output or Linux `/proc/self/mountinfo`)
+/// lists `path` as a mount point. Compares text only: stat-ing a dead mount
+/// is what hangs.
+fn listed_in_mount_table(table: &str, path: &Path, linux: bool) -> bool {
+ let raw = path.to_string_lossy().to_string();
+ // macOS lists /var, /tmp and /etc as /private/...
+ let private = format!("/private{raw}");
+ table.lines().any(|l| {
+ if linux {
+ l.split(' ')
+ .nth(4)
+ .is_some_and(|m| m.replace("\\040", " ") == raw)
+ } else {
+ [&raw, &private]
+ .iter()
+ .any(|p| l.contains(&format!(" on {p} (")))
+ }
+ })
+}
+
+async fn mount_table() -> String {
+ if cfg!(target_os = "linux") {
+ tokio::fs::read_to_string("/proc/self/mountinfo")
+ .await
+ .unwrap_or_default()
+ } else {
+ // `mount` only reads the kernel's table; it does not stat mounts.
+ match tokio::time::timeout(
+ Duration::from_secs(10),
+ tokio::process::Command::new("/sbin/mount").output(),
+ )
+ .await
+ {
+ Ok(Ok(o)) => String::from_utf8_lossy(&o.stdout).into_owned(),
+ _ => String::new(),
+ }
+ }
+}
+
+async fn clear_stale_mounts() {
+ let table = mount_table().await;
+ for path in stale_candidates() {
+ if !listed_in_mount_table(&table, &path, cfg!(target_os = "linux")) {
+ continue;
+ }
+ // Only a mount that does not answer is stale: a host's own live
+ // mount at the same path (a machine that is both a host and a Space)
+ // is left alone. A dead mount's stat hangs until the NFS timeouts
+ // give up, so it runs on a thread with a deadline.
+ let probe = path.clone();
+ let alive = tokio::time::timeout(
+ Duration::from_secs(5),
+ tokio::task::spawn_blocking(move || std::fs::metadata(&probe).is_ok()),
+ )
+ .await;
+ if matches!(alive, Ok(Ok(true))) {
+ continue;
+ }
+ let p = path.to_string_lossy().to_string();
+ tracing::warn!(path = %p, "clearing a stale volume mount left by a previous daemon");
+ let r = if cfg!(target_os = "linux") {
+ // Lazy: a dead FUSE/NFS mount may refuse a plain unmount.
+ // SAFETY: geteuid never fails.
+ let root = unsafe { libc::geteuid() } == 0;
+ if root {
+ run("umount", &["-l", &p]).await
+ } else {
+ run("sudo", &["-n", "umount", "-l", &p]).await
+ }
+ } else {
+ run("/sbin/umount", &["-f", &p]).await
+ };
+ if let Err(e) = r {
+ tracing::warn!(path = %p, error = %e, "stale volume mount did not unmount");
+ }
+ }
}
async fn run(cmd: &str, args: &[&str]) -> Result<(), String> {
@@ -320,7 +434,7 @@ async fn mount(session: &Arc<Session>) -> Result<(), String> {
let _ = run("/sbin/umount", &["-f", &p]).await;
}
let opts = format!(
- "port={port},mountport={port},vers=3,tcp,nolocks,locallocks,soft,timeo=50,retrans=2,intr,noowners,rsize=1048576,wsize=1048576,actimeo=1",
+ "port={port},mountport={port},vers=3,tcp,nolocks,locallocks,soft,timeo=50,retrans=2,deadtimeout=30,intr,noowners,rsize=1048576,wsize=1048576,actimeo=1",
port = session.port
);
run("/sbin/mount_nfs", &["-o", &opts, "127.0.0.1:/", &p]).await
@@ -696,6 +810,24 @@ pub async fn volume_ws(
mod tests {
use super::*;
+ #[test]
+ fn a_listed_mount_is_found_without_stat() {
+ let mac = "127.0.0.1:/ on /Users/a/Cua Volume (nfs, nodev, nosuid, mounted by a)\n\
+ /dev/disk3s1 on / (apfs, local)";
+ let p = Path::new("/Users/a/Cua Volume");
+ assert!(listed_in_mount_table(mac, p, false));
+ assert!(!listed_in_mount_table(
+ mac,
+ Path::new("/Users/b/Cua Volume"),
+ false
+ ));
+ let linux =
+ "36 25 0:32 / /volume rw - fuse none rw\n40 25 0:33 / /mnt/my\\040vol rw - tmpfs";
+ assert!(listed_in_mount_table(linux, Path::new("/volume"), true));
+ assert!(listed_in_mount_table(linux, Path::new("/mnt/my vol"), true));
+ assert!(!listed_in_mount_table(linux, Path::new("/vol"), true));
+ }
+
#[test]
fn the_backend_and_path_follow_the_guest() {
let b = backend();
diff --git a/libs/cua-spacesd/crates/cua-spacesd-teleport/src/keychain.rs b/libs/cua-spacesd/crates/cua-spacesd-teleport/src/keychain.rs
index 05c1ae6d43..2c40d6daac 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-teleport/src/keychain.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-teleport/src/keychain.rs
@@ -157,8 +157,9 @@ pub fn install_generic(host: &dyn HostEffects, item: &KeychainItem) -> std::io::
/// of the passwords this guest can know opens it (the login keychain of an
/// image whose autologin password file is wrong has a password nobody knows),
/// the item moves to a keychain spacesd owns (`cua.keychain-db`, a password
-/// it knows), which becomes the default and first on the search list, where
-/// the write succeeds and the app reads the key silently.
+/// it knows), which joins the search list (never the default) and is kept
+/// unlocked by [`keep_cua_keychain_unlocked`], where the write succeeds and the
+/// app reads the key silently.
pub fn install_generic_noted(
host: &dyn HostEffects,
item: &KeychainItem,
@@ -527,9 +528,80 @@ pub(crate) fn trusted_app_for(service: &str) -> Option<&'static str> {
}
}
+/// Where spacesd's own keychain lives.
+#[cfg(target_os = "macos")]
+fn cua_keychain_path(host: &dyn HostEffects) -> Option<std::path::PathBuf> {
+ Some(host.home_dir()?.join("Library/Keychains/cua.keychain-db"))
+}
+
+/// Unlocks `path` with the first of `passwords` it takes and gives it no
+/// auto-lock: `set-keychain-settings` without `-l` (lock on sleep) or `-u`
+/// (lock after a timeout) leaves it open until logout. Whether it opened.
+#[cfg(target_os = "macos")]
+fn unlock_and_unlimit(host: &dyn HostEffects, path: &str, passwords: &[String]) -> bool {
+ let mut opened = false;
+ for pw in passwords {
+ let ok = host
+ .run(
+ &HostCommand::new(EffectKind::KeychainWrite, "security")
+ .args(["unlock-keychain", "-p", pw, path])
+ .timeout(std::time::Duration::from_secs(20)),
+ )
+ .map(|o| o.success)
+ .unwrap_or(false);
+ if ok {
+ opened = true;
+ break;
+ }
+ }
+ if opened {
+ let _ = host.run(
+ &HostCommand::new(EffectKind::KeychainWrite, "security")
+ .args(["set-keychain-settings", path])
+ .timeout(std::time::Duration::from_secs(20)),
+ );
+ }
+ opened
+}
+
+/// Unlocks spacesd's `cua` keychain (the fallback home of Chrome's Safe
+/// Storage key when the login keychain's password is unknown) and keeps it
+/// from locking. A locked keychain on the search list makes whatever touches
+/// it raise "<App> wants to use the 'cua' keychain. Please enter the
+/// keychain password.", which nobody can answer in an unattended Space. It
+/// locks again when the Space reboots (the lock is not persistent) and a
+/// keychain created with the defaults locks on sleep and after a timeout, so
+/// spacesd calls this at start-up and then every [`KEEPER_INTERVAL`], which
+/// also covers wake from sleep. The password is the one from spacesd's
+/// protected config (`CUA_ENV_LOGIN_KEYCHAIN_PW`, `~/.cua/spacesd/keychain-password`,
+/// `/etc/cua/keychain-password`, then the image's `lume`). Does nothing when
+/// the keychain does not exist (the login keychain took the password, the
+/// usual case) or off macOS. Returns whether the keychain is now open.
+pub fn keep_cua_keychain_unlocked(host: &dyn HostEffects) -> bool {
+ #[cfg(target_os = "macos")]
+ {
+ let Some(path) = cua_keychain_path(host) else {
+ return false;
+ };
+ if !path.exists() {
+ return false;
+ }
+ unlock_and_unlimit(host, &path.to_string_lossy(), &keychain_passwords(host))
+ }
+ #[cfg(not(target_os = "macos"))]
+ {
+ let _ = host;
+ false
+ }
+}
+
+/// How often [`keep_cua_keychain_unlocked`] runs in the daemon: the longest
+/// the keychain stays locked after a wake.
+pub const KEEPER_INTERVAL: std::time::Duration = std::time::Duration::from_secs(20);
+
/// Makes sure a keychain whose password this guest knows exists, is unlocked,
-/// first on the search list and the default, and returns its path. `None`
-/// when it cannot be created.
+/// never auto-locks and is on the search list (not the default), and returns
+/// its path. `None` when it cannot be created or opened.
#[cfg(target_os = "macos")]
fn adopt_known_keychain(host: &dyn HostEffects) -> Option<String> {
let home = host.home_dir()?;
@@ -551,15 +623,18 @@ fn adopt_known_keychain(host: &dyn HostEffects) -> Option<String> {
{
return None;
}
- sec(vec![
- "unlock-keychain".into(),
- "-p".into(),
- pw,
- path_str.clone(),
- ]);
- sec(vec!["set-keychain-settings".into(), path_str.clone()]);
- // First on the search list (apps read through it), the rest kept.
- let mut list = vec![path_str.clone()];
+ // Open and never auto-lock (see [`keep_cua_keychain_unlocked`]).
+ if !unlock_and_unlimit(host, &path_str, std::slice::from_ref(&pw)) {
+ return None;
+ }
+ // On the search list (apps read through it), AFTER the login keychain
+ // and the rest, which stay as they were. It is deliberately NOT the
+ // default keychain: system services (Spotlight's mdworker above all)
+ // reach for the default, and any moment it is locked they raise
+ // "<Process> wants to use the 'cua' keychain", a dialog nothing in an
+ // unattended Space can answer. Chrome finds its Safe Storage key
+ // through the search list.
+ let mut list: Vec<String> = Vec::new();
if let Ok(o) = host.run(
&HostCommand::new(EffectKind::KeychainRead, "security").args([
"list-keychains",
@@ -574,22 +649,17 @@ fn adopt_known_keychain(host: &dyn HostEffects) -> Option<String> {
}
}
}
- let mut args = vec![
- "list-keychains".to_string(),
- "-d".into(),
- "user".into(),
- "-s".into(),
- ];
- args.extend(list);
- sec(args);
- // Default too: some apps resolve only the default keychain.
- sec(vec![
- "default-keychain".into(),
- "-d".into(),
- "user".into(),
- "-s".into(),
- path_str.clone(),
- ]);
+ if !list.contains(&path_str) {
+ list.push(path_str.clone());
+ let mut args = vec![
+ "list-keychains".to_string(),
+ "-d".into(),
+ "user".into(),
+ "-s".into(),
+ ];
+ args.extend(list);
+ sec(args);
+ }
Some(path_str)
}
@@ -941,13 +1011,28 @@ mod tests {
),
"{writes:?}"
);
+ // On the search list, but never made the default (system services
+ // would reach for it and prompt whenever it is locked).
+ assert!(
+ writes
+ .iter()
+ .all(|c| c.args.first().map(String::as_str) != Some("default-keychain")),
+ "{writes:?}"
+ );
assert!(
writes.iter().any(
- |c| c.args.first().map(String::as_str) == Some("default-keychain")
+ |c| c.args.first().map(String::as_str) == Some("list-keychains")
&& c.args.last().map(String::as_str) == Some(known)
),
"{writes:?}"
);
+ // Open, with no lock-on-sleep and no timeout (no -l, -u or -t).
+ assert!(
+ writes.iter().any(|c| c.args.first().map(String::as_str)
+ == Some("set-keychain-settings")
+ && c.args == ["set-keychain-settings", known]),
+ "{writes:?}"
+ );
// Added twice (login, then the known keychain), authorized on the second.
assert_eq!(writes.iter().filter(|c| is_add_command(c)).count(), 2);
let last = partition_calls(&host).pop().unwrap();
@@ -955,6 +1040,61 @@ mod tests {
assert!(last.args[2].contains("teamid:EQHXZ8M8AV"));
}
+ /// The `cua` keychain is reopened (and told never to lock) by the
+ /// keeper, with the first password that takes; a Space whose login
+ /// keychain took the password has no `cua` keychain and nothing happens.
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn the_keeper_unlocks_the_cua_keychain_and_never_auto_locks_it() {
+ let dir = std::env::temp_dir().join(format!("cua-keeper-{}", std::process::id()));
+ let kc = dir.join("Library/Keychains");
+ std::fs::create_dir_all(&kc).unwrap();
+ let path = kc.join("cua.keychain-db");
+ let none = FakeHost::new().with_home(&dir);
+ assert!(!keep_cua_keychain_unlocked(&none), "no keychain, no work");
+ assert!(none.calls().is_empty());
+
+ std::fs::write(&path, b"").unwrap();
+ // The first password (an env one nobody set) is skipped: only `lume` opens it.
+ let host = FakeHost::new().with_home(&dir).with_responder(|c| {
+ Ok(
+ if c.args.first().map(String::as_str) == Some("unlock-keychain")
+ && c.args[2] != "lume"
+ {
+ HostOutput::failed()
+ } else {
+ HostOutput::ok("")
+ },
+ )
+ });
+ assert!(keep_cua_keychain_unlocked(&host));
+ let calls = host.calls_of(EffectKind::KeychainWrite);
+ let p = path.to_string_lossy().into_owned();
+ assert_eq!(
+ calls.last().unwrap().args,
+ ["set-keychain-settings".to_string(), p.clone()],
+ "no -l / -u / -t: no lock on sleep, no timeout"
+ );
+ assert!(calls.iter().any(|c| c.args
+ == [
+ "unlock-keychain".to_string(),
+ "-p".into(),
+ "lume".into(),
+ p.clone()
+ ]));
+
+ // A password that never takes: still tries, reports closed, sets nothing.
+ let host = FakeHost::new()
+ .with_home(&dir)
+ .with_responder(|_| Ok(HostOutput::failed()));
+ assert!(!keep_cua_keychain_unlocked(&host));
+ assert!(host
+ .calls()
+ .iter()
+ .all(|c| c.args.first().map(String::as_str) != Some("set-keychain-settings")));
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
/// When no password works anywhere, the teleport says so in words the
/// user can act on.
#[cfg(target_os = "macos")]
diff --git a/libs/cua/crates/cua-spaces-ext/src/drive_runtime.rs b/libs/cua/crates/cua-spaces-ext/src/drive_runtime.rs
index c385bc7397..ec72f3763a 100644
--- a/libs/cua/crates/cua-spaces-ext/src/drive_runtime.rs
+++ b/libs/cua/crates/cua-spaces-ext/src/drive_runtime.rs
@@ -88,6 +88,9 @@ pub(crate) struct DriveRuntime {
/// Why a Space that connected has no volume (the attach failed, or its
/// image has no volume), until it mounts or the Space goes.
unavailable: std::sync::Mutex<BTreeMap<String, String>>,
+ /// Per Space, the task that re-mounts the volume when the guest daemon
+ /// restarts (see [`spawn_attach`]).
+ watchers: std::sync::Mutex<BTreeMap<String, tokio::task::JoinHandle<()>>>,
pub(crate) volume_auto: bool,
}
@@ -98,6 +101,7 @@ impl Default for DriveRuntime {
service: tokio::sync::OnceCell::new(),
volumes: tokio::sync::Mutex::new(BTreeMap::new()),
unavailable: std::sync::Mutex::new(BTreeMap::new()),
+ watchers: std::sync::Mutex::new(BTreeMap::new()),
volume_auto: true,
}
}
@@ -228,6 +232,7 @@ impl SpacesDrive for Spaces {
let Ok(ext) = self.drive_extension() else {
return;
};
+ ext.runtime.stop_watchers();
let all: Vec<Mounted> = std::mem::take(&mut *ext.runtime.volumes.lock().await)
.into_values()
.collect();
@@ -386,8 +391,22 @@ impl SpacesDrive for Spaces {
}
impl DriveRuntime {
+ fn stop_watcher(&self, id: &str) {
+ if let Some(t) = self.watchers.lock().expect("volume watchers").remove(id) {
+ t.abort();
+ }
+ }
+
+ pub(crate) fn stop_watchers(&self) {
+ let all = std::mem::take(&mut *self.watchers.lock().expect("volume watchers"));
+ for t in all.into_values() {
+ t.abort();
+ }
+ }
+
/// Unmounts `id`'s volume; whether there was one.
pub(crate) async fn detach(&self, id: &str) -> bool {
+ self.stop_watcher(id);
self.note_unavailable(id, None);
let m = self.volumes.lock().await.remove(id);
match m {
@@ -411,22 +430,115 @@ pub(crate) fn spawn_attach(spaces: &Spaces, id: &str) {
if !(id.starts_with("local:") || id.starts_with("cloud:")) {
return;
}
- let spaces = spaces.clone();
+ let Ok(ext) = spaces.drive_extension() else {
+ return;
+ };
let key = id.to_string();
- tokio::spawn(async move {
- // Bounded: the Space is ready either way; a volume that cannot
- // mount says why (`volume_sync_status`, `cua volume status`).
- let why = match tokio::time::timeout(ATTACH_TIMEOUT, spaces.volume_attach(&key)).await {
- Ok(Ok(_)) => return,
- Ok(Err(e)) => format!("Cua Volume not mounted: {e}"),
- Err(_) => format!(
- "Cua Volume not mounted: no answer within {} s",
- ATTACH_TIMEOUT.as_secs()
- ),
- };
- tracing::warn!(space = %key, "{why}");
- if let Ok(ext) = spaces.drive_extension() {
- ext.runtime.note_unavailable(&key, Some(why));
+ let task = tokio::spawn(keep_attached(spaces.clone(), key.clone()));
+ // A fresh connection replaces the old watcher.
+ if let Some(old) = ext
+ .runtime
+ .watchers
+ .lock()
+ .expect("volume watchers")
+ .insert(key, task)
+ {
+ old.abort();
+ }
+}
+
+/// Mounts the Space's volume and keeps it mounted: when the guest daemon
+/// restarts (an update, a crash, a reboot) its mount and its half of the
+/// socket are gone, so the volume is mounted again, with a backoff while the
+/// daemon is still coming up. Ends when the Space is dropped or its volume
+/// detached (both abort this task) or when the guest has no mount backend.
+async fn keep_attached(spaces: Spaces, key: String) {
+ use cua_spaces::reattach::{Attempt, Policy, supervise};
+ let health = {
+ let (spaces, key) = (spaces.clone(), key.clone());
+ move || {
+ let (spaces, key) = (spaces.clone(), key.clone());
+ async move { volume_health(&spaces, &key).await }
}
- });
+ };
+ let reattach = {
+ let (spaces, key) = (spaces.clone(), key.clone());
+ move || {
+ let (spaces, key) = (spaces.clone(), key.clone());
+ async move {
+ // Bounded: the Space is ready either way; a volume that
+ // cannot mount says why (`volume_sync_status`,
+ // `cua volume status`).
+ let r = tokio::time::timeout(ATTACH_TIMEOUT, remount(&spaces, &key)).await;
+ let why = match r {
+ Ok(Ok(Some(_))) => return Ok(Attempt::Attached),
+ Ok(Ok(None)) => return Ok(Attempt::Unsupported),
+ Ok(Err(e)) => format!("Cua Volume not mounted: {e}"),
+ Err(_) => format!(
+ "Cua Volume not mounted: no answer within {} s",
+ ATTACH_TIMEOUT.as_secs()
+ ),
+ };
+ if let Ok(ext) = spaces.drive_extension() {
+ ext.runtime.note_unavailable(&key, Some(why.clone()));
+ }
+ Err(why)
+ }
+ }
+ };
+ supervise(Policy::default(), health, reattach).await;
+}
+
+/// Whether the Space's volume is mounted and served: this side's socket is
+/// open and the guest still reports its mount.
+async fn volume_health(spaces: &Spaces, key: &str) -> cua_spaces::reattach::Health {
+ use cua_spaces::reattach::Health;
+ let Ok(ext) = spaces.drive_extension() else {
+ return Health::Gone;
+ };
+ let live = ext
+ .runtime
+ .volumes
+ .lock()
+ .await
+ .get(key)
+ .map(|m| m.attachment.is_live());
+ match live {
+ None | Some(false) => Health::Down,
+ Some(true) => match spaces.space(key).await {
+ // A restarted daemon answers `detached`: it knows nothing of the
+ // socket this side still holds.
+ Ok(s) => match s.volume_status().await {
+ Ok(st) if st.state == "detached" || st.state == "error" => Health::Down,
+ _ => Health::Healthy,
+ },
+ Err(_) => Health::Healthy,
+ },
+ }
+}
+
+/// Mounts again: drops what is left of the old attachment (keeping a
+/// persistent agent's view on it), then attaches.
+async fn remount(spaces: &Spaces, key: &str) -> Result<Option<VolumeInfo>> {
+ let ext = spaces.drive_extension()?;
+ let holder = {
+ let mut volumes = ext.runtime.volumes.lock().await;
+ match volumes.remove(key) {
+ Some(old) => {
+ let holder = old.holder.clone();
+ drop(volumes);
+ old.detach().await;
+ holder
+ }
+ None => None,
+ }
+ };
+ let info = spaces.volume_attach(key).await?;
+ if let (Some(agent), Some(_)) = (holder, &info) {
+ let _ = spaces.volume_for_agent(key, &agent).await;
+ }
+ if info.is_some() {
+ tracing::info!(space = %key, "Cua Volume mounted again");
+ }
+ Ok(info)
}
diff --git a/libs/cua/crates/cua-spaces/Cargo.toml b/libs/cua/crates/cua-spaces/Cargo.toml
index 5441c63f0c..cb55f0a81b 100644
--- a/libs/cua/crates/cua-spaces/Cargo.toml
+++ b/libs/cua/crates/cua-spaces/Cargo.toml
@@ -101,5 +101,6 @@ cua-fleet = { workspace = true, features = ["testing"] }
cua-proto = { workspace = true, features = ["client", "server", "serde"] }
tonic = { workspace = true, default-features = false, features = ["codegen", "server", "router", "channel"] }
tokio-stream = { workspace = true, features = ["net"] }
+tokio = { workspace = true, features = ["test-util"] }
tempfile.workspace = true
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] }
diff --git a/libs/cua/crates/cua-spaces/src/hotspot.rs b/libs/cua/crates/cua-spaces/src/hotspot.rs
index 74ce9aa2e0..e39ece2187 100644
--- a/libs/cua/crates/cua-spaces/src/hotspot.rs
+++ b/libs/cua/crates/cua-spaces/src/hotspot.rs
@@ -280,7 +280,7 @@ impl crate::Spaces {
if let Some(old) = self.inner.hotspots.lock().await.remove(&key) {
let _ = old.stop().await;
}
- let hotspot = s.start_hotspot(options).await?;
+ let hotspot = s.start_hotspot(options.clone()).await?;
let status = HotspotStatus {
space: key.clone(),
state: "waiting_for_peer".into(),
@@ -291,10 +291,90 @@ impl crate::Spaces {
bytes_in: 0,
served_here: true,
};
- self.inner.hotspots.lock().await.insert(key, hotspot);
+ let id = hotspot.id().to_string();
+ self.inner
+ .hotspots
+ .lock()
+ .await
+ .insert(key.clone(), hotspot);
+ self.supervise_hotspot(key, id, options);
Ok(status)
}
+ /// Re-starts the hotspot of `key` when its guest daemon restarts (the
+ /// socket closes and the new daemon has no hotspot), until it is
+ /// stopped or replaced.
+ fn supervise_hotspot(&self, key: String, id: String, options: HotspotOptions) {
+ use crate::reattach::{Attempt, Health, Policy, supervise};
+ let spaces = self.clone();
+ // The hotspot this supervisor last saw; another id means a caller
+ // replaced it (and started its own supervisor).
+ let mine = Arc::new(std::sync::Mutex::new(Some(id)));
+ tokio::spawn(async move {
+ let health = {
+ let (spaces, key, mine) = (spaces.clone(), key.clone(), mine.clone());
+ move || {
+ let (spaces, key, mine) = (spaces.clone(), key.clone(), mine.clone());
+ async move {
+ let (id, running) = {
+ let map = spaces.inner.hotspots.lock().await;
+ match map.get(&key) {
+ Some(h) => (h.id().to_string(), h.is_running()),
+ None => return Health::Gone,
+ }
+ };
+ {
+ let mut m = mine.lock().expect("hotspot id");
+ match m.as_deref() {
+ None => *m = Some(id.clone()),
+ Some(seen) if seen != id => return Health::Gone,
+ _ => {}
+ }
+ }
+ if !running {
+ return Health::Down;
+ }
+ // The socket is open; does the guest still have its
+ // half? A restarted daemon reports no hotspot.
+ let Ok(space) = spaces.space(&key).await else {
+ return Health::Healthy;
+ };
+ match space.hotspot_status().await {
+ Ok(s) if s.state == "stopped" || s.hotspot_id != id => Health::Down,
+ _ => Health::Healthy,
+ }
+ }
+ }
+ };
+ let reattach = {
+ let (spaces, key, mine) = (spaces.clone(), key.clone(), mine.clone());
+ move || {
+ let (spaces, key, mine, options) =
+ (spaces.clone(), key.clone(), mine.clone(), options.clone());
+ async move {
+ let space = spaces.space(&key).await.map_err(|e| e.to_string())?;
+ // Hold the map across the start so a stop cannot slip in.
+ let mut map = spaces.inner.hotspots.lock().await;
+ if !map.contains_key(&key) {
+ return Ok(Attempt::Unsupported);
+ }
+ let fresh = space
+ .start_hotspot(options)
+ .await
+ .map_err(|e| e.to_string())?;
+ tracing::info!(space = %key, "hotspot re-attached after the guest daemon restarted");
+ *mine.lock().expect("hotspot id") = Some(fresh.id().to_string());
+ if let Some(old) = map.insert(key, fresh) {
+ drop(old);
+ }
+ Ok(Attempt::Attached)
+ }
+ }
+ };
+ supervise(Policy::default(), health, reattach).await;
+ });
+ }
+
/// Stops the hotspot of one Space, or of every Space when `None`.
/// Returns the Spaces that were stopped.
pub async fn hotspot_stop(&self, space: Option<&str>) -> Result<Vec<String>> {
diff --git a/libs/cua/crates/cua-spaces/src/lib.rs b/libs/cua/crates/cua-spaces/src/lib.rs
index d066080099..6af853b2bd 100644
--- a/libs/cua/crates/cua-spaces/src/lib.rs
+++ b/libs/cua/crates/cua-spaces/src/lib.rs
@@ -96,6 +96,7 @@ pub mod install_cache;
pub mod mcp;
#[cfg(feature = "spaces-presence")]
pub mod presence;
+pub mod reattach;
/// Sharing a Space with other accounts, to watch or to edit, through the
/// relay.
#[cfg(feature = "spaces-agents")]
diff --git a/libs/cua/crates/cua-spaces/src/reattach.rs b/libs/cua/crates/cua-spaces/src/reattach.rs
new file mode 100644
index 0000000000..663ac3c048
--- /dev/null
+++ b/libs/cua/crates/cua-spaces/src/reattach.rs
@@ -0,0 +1,234 @@
+//! Keeping a host-served link to a Space alive across guest daemon restarts.
+//!
+//! The Cua Volume and the network hotspot are served from this host over a
+//! socket the guest's `cua-spacesd` holds. When that daemon restarts (an
+//! update, a crash, a reboot) the socket closes, the guest drops the mount
+//! and the proxy, and nothing on this side starts them again. [`supervise`]
+//! watches one such link and re-creates it, backing off while the guest is
+//! still coming up.
+
+use std::future::Future;
+use std::time::Duration;
+
+/// What a link looks like right now.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Health {
+ /// Served and the guest agrees.
+ Healthy,
+ /// The socket closed, or the guest lost its half (a restarted daemon).
+ Down,
+ /// Nobody wants it any more (stopped, released, replaced): stop
+ /// supervising.
+ Gone,
+}
+
+/// What a re-attach attempt did.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Attempt {
+ /// Attached; keep watching.
+ Attached,
+ /// Nothing to attach (the guest has no such backend): stop supervising.
+ Unsupported,
+}
+
+/// Timing of [`supervise`].
+#[derive(Clone, Copy, Debug)]
+pub struct Policy {
+ /// How often the link is checked.
+ pub poll: Duration,
+ /// The wait after the first failed attempt.
+ pub initial_backoff: Duration,
+ /// The wait never grows past this.
+ pub max_backoff: Duration,
+}
+
+impl Default for Policy {
+ fn default() -> Self {
+ Self {
+ poll: Duration::from_secs(3),
+ initial_backoff: Duration::from_secs(2),
+ max_backoff: Duration::from_secs(60),
+ }
+ }
+}
+
+/// Exponential backoff between failed attempts.
+#[derive(Clone, Copy, Debug)]
+pub struct Backoff {
+ initial: Duration,
+ max: Duration,
+ next: Duration,
+}
+
+impl Backoff {
+ pub fn new(policy: &Policy) -> Self {
+ Self {
+ initial: policy.initial_backoff,
+ max: policy.max_backoff,
+ next: policy.initial_backoff,
+ }
+ }
+
+ /// The wait after a failure; the one after that doubles, up to the cap.
+ pub fn fail(&mut self) -> Duration {
+ let wait = self.next;
+ self.next = (self.next * 2).min(self.max);
+ wait
+ }
+
+ /// An attempt worked (or the link was healthy): start over.
+ pub fn reset(&mut self) {
+ self.next = self.initial;
+ }
+}
+
+/// Checks a link every `policy.poll` and calls `reattach` while it is
+/// [`Health::Down`], waiting out an exponential backoff after each failure.
+/// The first check is immediate, so a link that was never made is made at
+/// once. Returns when `health` says [`Health::Gone`] or `reattach` says
+/// [`Attempt::Unsupported`].
+pub async fn supervise<H, HF, R, RF>(policy: Policy, mut health: H, mut reattach: R)
+where
+ H: FnMut() -> HF,
+ HF: Future<Output = Health>,
+ R: FnMut() -> RF,
+ RF: Future<Output = Result<Attempt, String>>,
+{
+ let mut backoff = Backoff::new(&policy);
+ let mut retry_at: Option<tokio::time::Instant> = None;
+ loop {
+ match health().await {
+ Health::Gone => return,
+ Health::Healthy => {
+ backoff.reset();
+ retry_at = None;
+ }
+ Health::Down => {
+ if retry_at.is_none_or(|t| tokio::time::Instant::now() >= t) {
+ match reattach().await {
+ Ok(Attempt::Attached) => {
+ backoff.reset();
+ retry_at = None;
+ }
+ Ok(Attempt::Unsupported) => return,
+ Err(why) => {
+ let wait = backoff.fail();
+ tracing::warn!(error = %why, retry_in = ?wait, "re-attach failed");
+ retry_at = Some(tokio::time::Instant::now() + wait);
+ }
+ }
+ }
+ }
+ }
+ tokio::time::sleep(policy.poll).await;
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::Arc;
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ fn policy() -> Policy {
+ Policy {
+ poll: Duration::from_secs(1),
+ initial_backoff: Duration::from_secs(2),
+ max_backoff: Duration::from_secs(8),
+ }
+ }
+
+ #[test]
+ fn backoff_doubles_to_the_cap_and_resets() {
+ let mut b = Backoff::new(&policy());
+ let waits: Vec<_> = (0..5).map(|_| b.fail().as_secs()).collect();
+ assert_eq!(waits, [2, 4, 8, 8, 8]);
+ b.reset();
+ assert_eq!(b.fail().as_secs(), 2);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn a_link_that_goes_down_is_reattached_once() {
+ // Healthy for two polls, down for one, then healthy again.
+ let polls = Arc::new(AtomicUsize::new(0));
+ let attaches = Arc::new(AtomicUsize::new(0));
+ let (p, a) = (polls.clone(), attaches.clone());
+ supervise(
+ policy(),
+ move || {
+ let n = p.fetch_add(1, Ordering::SeqCst);
+ let a = a.load(Ordering::SeqCst);
+ async move {
+ match n {
+ 0 | 1 => Health::Healthy,
+ _ if a == 0 => Health::Down,
+ 2..=5 => Health::Healthy,
+ _ => Health::Gone,
+ }
+ }
+ },
+ {
+ let a = attaches.clone();
+ move || {
+ a.fetch_add(1, Ordering::SeqCst);
+ async { Ok(Attempt::Attached) }
+ }
+ },
+ )
+ .await;
+ assert_eq!(attaches.load(Ordering::SeqCst), 1);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn failures_back_off_and_a_guest_that_returns_is_attached() {
+ let start = tokio::time::Instant::now();
+ let times = Arc::new(std::sync::Mutex::new(Vec::new()));
+ let t = times.clone();
+ supervise(
+ policy(),
+ || async { Health::Down },
+ move || {
+ let t = t.clone();
+ async move {
+ let mut t = t.lock().unwrap();
+ t.push(start.elapsed().as_secs());
+ // The guest daemon is back on the fourth try.
+ if t.len() < 4 {
+ Err("connection refused".to_string())
+ } else {
+ Ok(Attempt::Unsupported)
+ }
+ }
+ },
+ )
+ .await;
+ let t = times.lock().unwrap().clone();
+ assert_eq!(t.len(), 4);
+ // Immediate, then waits of 2 s, 4 s, 8 s (rounded up to the 1 s poll).
+ assert_eq!(t[0], 0);
+ assert!((2..=3).contains(&(t[1] - t[0])), "{t:?}");
+ assert!((4..=5).contains(&(t[2] - t[1])), "{t:?}");
+ assert!((8..=9).contains(&(t[3] - t[2])), "{t:?}");
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn a_healthy_link_is_left_alone_and_gone_stops() {
+ let polls = Arc::new(AtomicUsize::new(0));
+ let p = polls.clone();
+ supervise(
+ policy(),
+ move || {
+ let n = p.fetch_add(1, Ordering::SeqCst);
+ async move {
+ if n < 10 {
+ Health::Healthy
+ } else {
+ Health::Gone
+ }
+ }
+ },
+ || async { panic!("a healthy link must not be re-attached") },
+ )
+ .await;
+ }
+}