Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(relay): hand over to the next relay without downtime (#5320)

ddupont3c8ab1808bfa
diff --git a/libs/cua-spacesd/crates/cua-relay/README.md b/libs/cua-spacesd/crates/cua-relay/README.md
index c442ec728e..4ff826f788 100644
--- a/libs/cua-spacesd/crates/cua-relay/README.md
+++ b/libs/cua-spacesd/crates/cua-relay/README.md
@@ -262,12 +262,30 @@ register machines: `CUA_RELAY_TOKENS`/`CUA_RELAY_TOKEN_FILE` or
| `CUA_RELAY_MACHINE_WINDOW_BYTES` | `67108864` (64 MiB) | yamux's per-machine connection receive window (S7); must be `>=` `machine-max-streams * 256 KiB`. |
| `CUA_RELAY_MAX_GLOBAL_STREAMS` | `4096` | Concurrent client streams across every machine at once: the relay-wide memory budget (S7). Tune to the pod's memory limit. |
| `CUA_RELAY_IDLE_SECS` | `90` | Seconds without a heartbeat before a machine is dropped (min 5). |
-| `CUA_RELAY_SHUTDOWN_GRACE_SECS` | `10` | Seconds open requests get after SIGTERM / SIGINT. |
+| `CUA_RELAY_SHUTDOWN_GRACE_SECS` | `10` | Seconds open requests get after SIGTERM / SIGINT. Machine tunnels are closed at once, so they reconnect to the next relay. |
+| `CUA_RELAY_HEALTH_LISTEN` | unset | Probe address (`0.0.0.0:8081`) answering `GET /healthz` from start, standby included. Point the Kubernetes probes here for rolling deploys. |
+| `CUA_RELAY_RECONNECT_GRACE_SECS` | `15` | For this long after start, a request for a known machine that is not connected yet waits for it to reconnect instead of failing. `0`: fail at once. |
| `RUST_LOG` | `info` | Log filter. |
Connected machines live in the process, and the directory is a local file:
-run one replica per hostname and replace it (Recreate) rather than rolling
-it.
+run one replica per hostname.
+
+**Deploys without downtime.** One relay at a time owns the state directory:
+it holds an exclusive `flock` on `<state dir>/.relay.lock` (the kernel drops
+it when the process exits, so it is never stale). A second relay on the same
+volume binds its ports and waits in standby: its main port queues
+connections (the listen backlog) instead of refusing them, and
+`CUA_RELAY_HEALTH_LISTEN` answers probes. When the first relay gets SIGTERM
+it closes every machine tunnel, lets open requests finish and exits; the
+second takes the lock, loads the state and serves the queued requests and
+reconnecting machines. On Kubernetes: a rolling update with `maxSurge: 1,
+maxUnavailable: 0`, the new pod on the old pod's node (a ReadWriteOnce
+volume mounts on one node, in any number of its pods), probes on the health
+port and a short `preStop` sleep so the ingress stops sending to the old pod
+before it stops serving. See `deploy/relay.k8s.yaml` and `src/handover.rs`.
+Switch an existing Recreate deployment over in two steps: first deploy this
+version with Recreate (the old version takes no lock), then the rolling
+update.
## Run and test
diff --git a/libs/cua-spacesd/crates/cua-relay/deploy/relay.k8s.yaml b/libs/cua-spacesd/crates/cua-relay/deploy/relay.k8s.yaml
index 5d04d6f396..2b490292be 100644
--- a/libs/cua-spacesd/crates/cua-relay/deploy/relay.k8s.yaml
+++ b/libs/cua-spacesd/crates/cua-relay/deploy/relay.k8s.yaml
@@ -6,7 +6,10 @@
# reconnects to whichever replica it lands on, so run one replica per
# hostname (or route /relay/v1/connect and /m/<id> with consistent hashing
# on the id). Account mode also keeps its directory on a ReadWriteOnce
-# volume, so replace the pod (Recreate) instead of rolling it.
+# volume. Deploys still overlap: the new pod starts on the same node, waits
+# in standby until the old one exits (a lock in the state directory) and
+# then takes over, with requests queued rather than refused (README,
+# "Deploys without downtime").
#
# Account mode (cua.ai logins, `cua host setup`): set CUA_RELAY_OIDC_ISSUER
# (+ audience), and keep the machine directory and the assertion signing key
@@ -26,18 +29,34 @@ metadata:
name: cua-relay
spec:
replicas: 1
- strategy: { type: Recreate }
+ strategy:
+ type: RollingUpdate
+ rollingUpdate: { maxSurge: 1, maxUnavailable: 0 }
selector:
matchLabels: { app: cua-relay }
template:
metadata:
labels: { app: cua-relay }
spec:
+ # The next pod lands on the node that has the volume mounted.
+ affinity:
+ podAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector: { matchLabels: { app: cua-relay } }
+ topologyKey: kubernetes.io/hostname
+ terminationGracePeriodSeconds: 20
containers:
- name: cua-relay
image: ghcr.io/trycua/cua-relay:latest
- ports: [{ containerPort: 8080, name: http }]
+ ports:
+ - { containerPort: 8080, name: http }
+ - { containerPort: 8081, name: health }
+ lifecycle:
+ # Keep serving while the ingress drops this pod.
+ preStop: { sleep: { seconds: 3 } }
env:
+ - { name: CUA_RELAY_HEALTH_LISTEN, value: "0.0.0.0:8081" }
+ - { name: CUA_RELAY_SHUTDOWN_GRACE_SECS, value: "3" }
- { name: CUA_RELAY_HOST_DOMAIN, value: relay.example.com }
- name: CUA_RELAY_TOKENS
valueFrom: { secretKeyRef: { name: cua-relay, key: tokens } }
@@ -50,9 +69,10 @@ spec:
- { name: CUA_RELAY_SIGNING_KEY_FILE, value: /var/lib/cua-relay/signing.key }
volumeMounts: [{ name: state, mountPath: /var/lib/cua-relay }]
readinessProbe:
- httpGet: { path: /healthz, port: http }
+ httpGet: { path: /healthz, port: health }
+ periodSeconds: 2
livenessProbe:
- httpGet: { path: /healthz, port: http }
+ httpGet: { path: /healthz, port: health }
periodSeconds: 20
resources:
requests: { cpu: 100m, memory: 64Mi }
diff --git a/libs/cua-spacesd/crates/cua-relay/src/handover.rs b/libs/cua-spacesd/crates/cua-relay/src/handover.rs
new file mode 100644
index 0000000000..d794ae5c44
--- /dev/null
+++ b/libs/cua-spacesd/crates/cua-relay/src/handover.rs
@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Deploys without downtime (CUA-1774): one relay at a time owns the state
+//! directory, and the next one waits beside it.
+//!
+//! The relay keeps its state in files it loads at start and rewrites as it
+//! goes, so two processes must never use the same state directory at once.
+//! [`StateLock`] enforces that with an exclusive `flock` on
+//! `<state dir>/.relay.lock`. The kernel drops the lock when the process
+//! exits, however it exits, so a crash never leaves it stale.
+//!
+//! A rolling deploy then hands over like this:
+//!
+//! 1. The new relay starts on the same node (same volume), binds its port
+//! and finds the lock taken. It waits in **standby**: it does not
+//! accept connections yet, so the kernel queues them (the listen
+//! backlog) instead of anyone seeing an error, and its health port
+//! answers, so Kubernetes routes to it and stops the old relay.
+//! 2. The old relay gets SIGTERM, closes every machine tunnel so they
+//! reconnect at once, lets open requests finish
+//! (`CUA_RELAY_SHUTDOWN_GRACE_SECS`) and exits.
+//! 3. The lock is free: the new relay loads the state the old one left and
+//! starts accepting, queued requests and reconnecting machines first.
+//! For a short while after start, a request for a known machine that
+//! is not back yet waits for it (`CUA_RELAY_RECONNECT_GRACE_SECS`)
+//! instead of failing.
+
+use std::fs::{File, OpenOptions, TryLockError};
+use std::path::{Path, PathBuf};
+
+use axum::{http::StatusCode, routing::get, Router};
+
+/// The lock file's name in the state directory.
+pub const LOCK_FILE: &str = ".relay.lock";
+
+/// The lock file for state directory `dir`.
+pub fn lock_path(dir: &Path) -> PathBuf {
+ dir.join(LOCK_FILE)
+}
+
+/// Exclusive ownership of a state directory, held until dropped or until
+/// the process exits.
+#[derive(Debug)]
+pub struct StateLock {
+ _file: File,
+ path: PathBuf,
+}
+
+impl StateLock {
+ fn open(path: &Path) -> std::io::Result<File> {
+ if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
+ std::fs::create_dir_all(dir)?;
+ }
+ let mut options = OpenOptions::new();
+ options.read(true).write(true).create(true).truncate(false);
+ #[cfg(unix)]
+ std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
+ options.open(path)
+ }
+
+ /// Takes the lock if no one holds it; `None` when another process does.
+ pub fn try_acquire(path: &Path) -> std::io::Result<Option<Self>> {
+ let file = Self::open(path)?;
+ match file.try_lock() {
+ Ok(()) => Ok(Some(Self {
+ _file: file,
+ path: path.to_path_buf(),
+ })),
+ Err(TryLockError::WouldBlock) => Ok(None),
+ Err(TryLockError::Error(e)) => Err(e),
+ }
+ }
+
+ /// Takes the lock, waiting (standby) for as long as another process
+ /// holds it.
+ pub async fn acquire(path: &Path) -> std::io::Result<Self> {
+ if let Some(lock) = Self::try_acquire(path)? {
+ return Ok(lock);
+ }
+ tracing::info!(
+ lock = %path.display(),
+ "standby: another relay owns the state directory; waiting for it to exit"
+ );
+ let started = std::time::Instant::now();
+ let owned = path.to_path_buf();
+ let lock = tokio::task::spawn_blocking(move || {
+ let file = Self::open(&owned)?;
+ file.lock()?;
+ Ok::<_, std::io::Error>(Self {
+ _file: file,
+ path: owned,
+ })
+ })
+ .await
+ .map_err(std::io::Error::other)??;
+ tracing::info!(
+ waited_ms = started.elapsed().as_millis() as u64,
+ "took over the state directory from the previous relay"
+ );
+ Ok(lock)
+ }
+
+ /// The lock file.
+ pub fn path(&self) -> &Path {
+ &self.path
+ }
+}
+
+/// Serves `GET /healthz` (204) on `listener` until the process exits: the
+/// probe port, answering in standby too, when the main port does not
+/// accept yet.
+pub async fn serve_health(listener: tokio::net::TcpListener) -> std::io::Result<()> {
+ let router = Router::new().route("/healthz", get(|| async { StatusCode::NO_CONTENT }));
+ axum::serve(listener, router).await
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn one_owner_at_a_time() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = lock_path(dir.path());
+ let first = StateLock::try_acquire(&path).unwrap().expect("free");
+ assert!(StateLock::try_acquire(&path).unwrap().is_none());
+ drop(first);
+ assert!(StateLock::try_acquire(&path).unwrap().is_some());
+ }
+
+ #[tokio::test]
+ async fn standby_takes_over_when_released() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = lock_path(dir.path());
+ let first = StateLock::try_acquire(&path).unwrap().expect("free");
+ let waiter = tokio::spawn({
+ let path = path.clone();
+ async move { StateLock::acquire(&path).await }
+ });
+ tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+ assert!(!waiter.is_finished(), "took the lock while it was held");
+ drop(first);
+ let second = tokio::time::timeout(std::time::Duration::from_secs(5), waiter)
+ .await
+ .expect("took over")
+ .unwrap()
+ .unwrap();
+ assert_eq!(second.path(), path);
+ }
+}
diff --git a/libs/cua-spacesd/crates/cua-relay/src/lib.rs b/libs/cua-spacesd/crates/cua-relay/src/lib.rs
index 6e2783605a..7efd37bed0 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/lib.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/lib.rs
@@ -29,6 +29,7 @@ pub mod client;
pub mod device_api;
pub mod devices;
pub mod directory;
+pub mod handover;
pub mod mux;
pub mod oidc;
pub mod server;
diff --git a/libs/cua-spacesd/crates/cua-relay/src/main.rs b/libs/cua-spacesd/crates/cua-relay/src/main.rs
index 877311299e..c909b40072 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/main.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/main.rs
@@ -122,6 +122,15 @@ struct Args {
/// Devices and audit log file (default: `<state file>.devices.json`).
#[arg(long, env = "CUA_RELAY_DEVICES_FILE")]
devices_file: Option<std::path::PathBuf>,
+ /// Probe address answering `GET /healthz` from the moment the relay
+ /// starts, standby included (e.g. `0.0.0.0:8081`). See
+ /// `cua_relay::handover`.
+ #[arg(long, env = "CUA_RELAY_HEALTH_LISTEN")]
+ health_listen: Option<SocketAddr>,
+ /// Seconds after start during which a request for a known machine that
+ /// is not connected yet waits for it to reconnect instead of failing.
+ #[arg(long, env = "CUA_RELAY_RECONNECT_GRACE_SECS", default_value_t = 15)]
+ reconnect_grace_secs: u64,
/// Seconds to let open requests finish after SIGTERM / SIGINT before
/// exiting (machines and clients reconnect on their own).
#[arg(long, env = "CUA_RELAY_SHUTDOWN_GRACE_SECS", default_value_t = 10)]
@@ -158,6 +167,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
)
.init();
let args = Args::parse();
+ // Ports first: from here on connections queue (the listen backlog)
+ // instead of being refused, while this relay waits its turn below.
+ let listener = tokio::net::TcpListener::bind(args.listen).await?;
+ if let Some(addr) = args.health_listen {
+ let health = tokio::net::TcpListener::bind(addr).await?;
+ tracing::info!(%addr, "health probe listening");
+ tokio::spawn(async move {
+ if let Err(e) = cua_relay::handover::serve_health(health).await {
+ tracing::error!(%e, "health probe stopped");
+ }
+ });
+ }
let mut tokens: Vec<String> = args
.tokens
.iter()
@@ -188,12 +209,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if tokens.is_empty() && oidc.is_none() {
return Err("no way to register machines: set CUA_RELAY_TOKENS / --token-file (static) or CUA_RELAY_OIDC_ISSUER (accounts)".into());
}
- let signing_key = match &args.signing_key_file {
- Some(path) => Some(std::sync::Arc::new(
- cua_relay::assertion::RelayKey::load_or_create(path)?,
- )),
- None => None,
- };
let device_enrollment = match args.device_enrollment.trim().to_ascii_lowercase().as_str() {
"on" | "true" | "1" | "yes" => true,
"off" | "false" | "0" | "no" => false,
@@ -204,6 +219,29 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if !device_enrollment && oidc.is_some() {
tracing::warn!("device enrollment is off: an account token alone reaches machines");
}
+ // One relay per state directory (CUA-1774): wait here, in standby, until
+ // the relay before this one exits. Every config error above surfaces
+ // before this point, so a bad deploy never takes over.
+ let state_dir = args
+ .state_file
+ .as_ref()
+ .or(args.devices_file.as_ref())
+ .map(|p| match p.parent() {
+ Some(d) if !d.as_os_str().is_empty() => d.to_path_buf(),
+ _ => std::path::PathBuf::from("."),
+ });
+ let _state_lock = match &state_dir {
+ Some(dir) => Some(
+ cua_relay::handover::StateLock::acquire(&cua_relay::handover::lock_path(dir)).await?,
+ ),
+ None => None,
+ };
+ let signing_key = match &args.signing_key_file {
+ Some(path) => Some(std::sync::Arc::new(
+ cua_relay::assertion::RelayKey::load_or_create(path)?,
+ )),
+ None => None,
+ };
let relay = Relay::try_new(RelayConfig {
tokens,
max_machines_per_token: args.max_machines_per_token,
@@ -229,8 +267,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
..cua_relay::devices::DevicePolicy::default()
},
devices_file: args.devices_file,
+ reconnect_grace: Duration::from_secs(args.reconnect_grace_secs),
})?;
- let listener = tokio::net::TcpListener::bind(args.listen).await?;
tracing::info!(listen = %args.listen, "cua-relay listening");
let (stop_tx, mut stop_rx) = tokio::sync::watch::channel(false);
let serve = relay.serve(listener, async move {
diff --git a/libs/cua-spacesd/crates/cua-relay/src/server.rs b/libs/cua-spacesd/crates/cua-relay/src/server.rs
index 8989133edd..207e60e69b 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/server.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/server.rs
@@ -114,6 +114,11 @@ pub struct RelayConfig {
/// yamux's per-machine connection receive window: the most one
/// machine's connection buffers across all of its streams (S7).
pub machine_window_bytes: usize,
+ /// For this long after start, a request for a known machine that is
+ /// not connected waits for it to reconnect instead of failing: the
+ /// machines of the relay this one took over from are on their way.
+ /// Zero: fail at once.
+ pub reconnect_grace: Duration,
}
impl Default for RelayConfig {
@@ -146,6 +151,7 @@ impl Default for RelayConfig {
max_global_streams: 4096,
machine_max_streams: crate::mux::MAX_STREAMS,
machine_window_bytes: crate::mux::DEFAULT_WINDOW_BYTES,
+ reconnect_grace: Duration::ZERO,
}
}
}
@@ -250,6 +256,8 @@ pub struct Relay {
/// Open client streams across every machine (S7): the global memory
/// budget's live counter.
pub(crate) global_streams: Arc<AtomicU64>,
+ /// When this relay started (the reconnect grace counts from it).
+ started: Instant,
}
/// One row of `/relay/v1/machines`.
@@ -304,6 +312,7 @@ impl Relay {
devices: Arc::new(devices),
key,
global_streams: Arc::default(),
+ started: Instant::now(),
})
}
@@ -369,6 +378,23 @@ impl Relay {
self.machines.lock().expect("machines").get(id).cloned()
}
+ /// Within the reconnect grace after start, waits for the known machine
+ /// `id` to connect (it was connected to the relay this one replaced and
+ /// is reconnecting); `None` once the grace is over or for an unknown id.
+ pub(crate) async fn await_reconnect(&self, id: &str) -> Option<Arc<Machine>> {
+ let until = self.started + self.config.reconnect_grace;
+ if Instant::now() >= until || self.directory.get(id).is_none() {
+ return None;
+ }
+ while Instant::now() < until {
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ if let Some(machine) = self.machine(id) {
+ return Some(machine);
+ }
+ }
+ None
+ }
+
/// Drops machine `id` and cuts its clients.
pub(crate) fn drop_machine(&self, id: &str) {
if let Some(m) = self.machines.lock().expect("machines").remove(id) {
@@ -433,12 +459,21 @@ impl Relay {
listener: tokio::net::TcpListener,
shutdown: impl std::future::Future<Output = ()> + Send + 'static,
) -> std::io::Result<()> {
+ let handle = self.clone();
axum::serve(
listener,
self.router()
.into_make_service_with_connect_info::<SocketAddr>(),
)
- .with_graceful_shutdown(shutdown)
+ .with_graceful_shutdown(async move {
+ shutdown.await;
+ // Machines reconnect at once, to the relay that replaces this
+ // one, instead of holding the shutdown open until the process
+ // exits.
+ let machines = handle.machine_ids().len();
+ handle.disconnect_all();
+ tracing::info!(machines, "closed machine connections");
+ })
.await
}
@@ -1033,6 +1068,10 @@ async fn proxy(State(relay): State<Relay>, mut request: Request) -> Response {
"the relay forwards only requests carrying a spacesd credential",
);
}
+ let machine = match machine {
+ Some(machine) => Some(machine),
+ None => relay.await_reconnect(&id).await,
+ };
let Some(machine) = machine else {
return refuse(
request.headers(),
diff --git a/libs/cua-spacesd/crates/cua-relay/tests/handover.rs b/libs/cua-spacesd/crates/cua-relay/tests/handover.rs
new file mode 100644
index 0000000000..56f2fc6c7d
--- /dev/null
+++ b/libs/cua-spacesd/crates/cua-relay/tests/handover.rs
@@ -0,0 +1,180 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Deploys without downtime (CUA-1774): a stopping relay closes its machine
+//! tunnels at once so they move to the next relay, and a relay that just
+//! started holds requests for known machines until they are back instead
+//! of failing them.
+
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use cua_relay::oidc::testing::FakeIssuer;
+use cua_relay::oidc::{OidcConfig, OidcValidator};
+use cua_relay::server::{Relay, RelayConfig};
+use serde_json::{json, Value};
+use tokio_util::sync::CancellationToken;
+
+const ISSUER: &str = "https://auth.test/realms/cua";
+
+async fn start(
+ issuer: &FakeIssuer,
+ reconnect_grace: Duration,
+) -> (
+ Relay,
+ String,
+ tokio::sync::oneshot::Sender<()>,
+ tokio::task::JoinHandle<()>,
+) {
+ let relay = Relay::new(RelayConfig {
+ oidc: Some(Arc::new(OidcValidator::with_jwks(
+ OidcConfig::new(ISSUER),
+ issuer.jwks(),
+ ))),
+ device_enrollment: false,
+ reconnect_grace,
+ ..RelayConfig::default()
+ });
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let base = format!("http://{}", listener.local_addr().unwrap());
+ let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>();
+ let serving = relay.clone();
+ let task = tokio::spawn(async move {
+ let _ = serving
+ .serve(listener, async move {
+ let _ = stop_rx.await;
+ })
+ .await;
+ });
+ (relay, base, stop_tx, task)
+}
+
+/// A stand-in spacesd: answers every request 200.
+async fn local_machine() -> std::net::SocketAddr {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let app = axum::Router::new().fallback(|| async { "machine says hi" });
+ tokio::spawn(async move {
+ let _ = axum::serve(listener, app).await;
+ });
+ addr
+}
+
+async fn register(base: &str, account: &str, id: &str) -> String {
+ let r = reqwest::Client::new()
+ .post(format!("{base}/v1/machines"))
+ .bearer_auth(account)
+ .json(&json!({"id": id, "name": id}))
+ .send()
+ .await
+ .unwrap();
+ assert!(r.status().is_success(), "{}", r.status());
+ let v: Value = r.json().await.unwrap();
+ v["machine_token"].as_str().unwrap().to_owned()
+}
+
+fn join(base: &str, id: &str, token: &str, local: std::net::SocketAddr) -> CancellationToken {
+ let mut config = cua_relay::client::JoinConfig::new(
+ base.replace("http://", "ws://"),
+ token.to_owned(),
+ id.to_owned(),
+ local,
+ );
+ config.heartbeat = Duration::from_secs(1);
+ let stop = CancellationToken::new();
+ tokio::spawn(cua_relay::client::run(config, stop.clone()));
+ stop
+}
+
+async fn wait_connected(relay: &Relay, id: &str) {
+ for _ in 0..200 {
+ if relay.machine_ids().iter().any(|m| m == id) {
+ return;
+ }
+ tokio::time::sleep(Duration::from_millis(25)).await;
+ }
+ panic!("{id} did not connect");
+}
+
+async fn get_machine(base: &str, id: &str, account: &str) -> reqwest::StatusCode {
+ reqwest::Client::new()
+ .get(format!("{base}/m/{id}/hello"))
+ .bearer_auth(account)
+ .timeout(Duration::from_secs(20))
+ .send()
+ .await
+ .unwrap()
+ .status()
+}
+
+#[tokio::test]
+async fn a_new_relay_holds_requests_until_the_machine_is_back() {
+ let issuer = FakeIssuer::new(ISSUER);
+ let ada = issuer.token("ada", Some("[email protected]"), "cua-relay", 600);
+ let (relay, base, _stop, _task) = start(&issuer, Duration::from_secs(10)).await;
+ let id = format!("m{}", uuid::Uuid::new_v4().simple());
+ let token = register(&base, &ada, &id).await;
+ let local = local_machine().await;
+
+ // The request arrives before the machine has reconnected...
+ let request = tokio::spawn({
+ let (base, id, ada) = (base.clone(), id.clone(), ada.clone());
+ async move { get_machine(&base, &id, &ada).await }
+ });
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ assert!(
+ !request.is_finished(),
+ "answered before the machine was back"
+ );
+ // ...and is answered once it has.
+ let _machine = join(&base, &id, &token, local);
+ let status = tokio::time::timeout(Duration::from_secs(10), request)
+ .await
+ .expect("held past the machine's return")
+ .unwrap();
+ assert_eq!(status, reqwest::StatusCode::OK);
+ wait_connected(&relay, &id).await;
+
+ // An id the relay does not know is refused at once.
+ let started = Instant::now();
+ let status = get_machine(&base, "nosuchmachine", &ada).await;
+ assert!(
+ status.is_client_error() || status.is_server_error(),
+ "{status}"
+ );
+ assert!(started.elapsed() < Duration::from_secs(2));
+}
+
+#[tokio::test]
+async fn without_a_grace_an_offline_machine_fails_at_once() {
+ let issuer = FakeIssuer::new(ISSUER);
+ let ada = issuer.token("ada", Some("[email protected]"), "cua-relay", 600);
+ let (_relay, base, _stop, _task) = start(&issuer, Duration::ZERO).await;
+ let id = format!("m{}", uuid::Uuid::new_v4().simple());
+ register(&base, &ada, &id).await;
+ let started = Instant::now();
+ assert_eq!(
+ get_machine(&base, &id, &ada).await,
+ reqwest::StatusCode::BAD_GATEWAY
+ );
+ assert!(started.elapsed() < Duration::from_secs(2));
+}
+
+#[tokio::test]
+async fn a_stopping_relay_closes_machine_tunnels_at_once() {
+ let issuer = FakeIssuer::new(ISSUER);
+ let ada = issuer.token("ada", Some("[email protected]"), "cua-relay", 600);
+ let (relay, base, stop, task) = start(&issuer, Duration::ZERO).await;
+ let id = format!("m{}", uuid::Uuid::new_v4().simple());
+ let token = register(&base, &ada, &id).await;
+ let machine = join(&base, &id, &token, local_machine().await);
+ wait_connected(&relay, &id).await;
+
+ let _ = stop.send(());
+ tokio::time::timeout(Duration::from_secs(5), task)
+ .await
+ .expect("the open tunnel held the shutdown")
+ .unwrap();
+ assert!(relay.machine_ids().is_empty(), "the tunnel was left open");
+ machine.cancel();
+}
diff --git a/libs/cua-spacesd/crates/cua-relay/tests/handover_process.rs b/libs/cua-spacesd/crates/cua-relay/tests/handover_process.rs
new file mode 100644
index 0000000000..9126d8a1f3
--- /dev/null
+++ b/libs/cua-spacesd/crates/cua-relay/tests/handover_process.rs
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Two relay processes on one state directory (a rolling deploy, CUA-1774):
+//! the second waits in standby, its health port answering and its main
+//! port holding requests, and takes over the moment the first exits.
+
+#![cfg(unix)]
+
+use std::io::{Read as _, Write as _};
+use std::net::{TcpListener, TcpStream};
+use std::path::Path;
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+fn free_port() -> u16 {
+ TcpListener::bind("127.0.0.1:0")
+ .unwrap()
+ .local_addr()
+ .unwrap()
+ .port()
+}
+
+fn start(state: &Path, port: u16, health: u16) -> Child {
+ Command::new(env!("CARGO_BIN_EXE_cua-relay"))
+ .env("CUA_RELAY_LISTEN", format!("127.0.0.1:{port}"))
+ .env("CUA_RELAY_HEALTH_LISTEN", format!("127.0.0.1:{health}"))
+ .env("CUA_RELAY_TOKENS", "handover-test-token")
+ .env("CUA_RELAY_STATE_FILE", state.join("machines.json"))
+ .env("CUA_RELAY_SHUTDOWN_GRACE_SECS", "2")
+ .env_remove("CUA_RELAY_OIDC_ISSUER")
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .spawn()
+ .expect("spawn cua-relay")
+}
+
+/// The status line of `GET path`, or `None` without an answer in `wait`.
+fn get(port: u16, path: &str, wait: Duration) -> Option<String> {
+ let mut s = TcpStream::connect(("127.0.0.1", port)).ok()?;
+ s.set_read_timeout(Some(wait)).ok()?;
+ s.write_all(
+ format!("GET {path} HTTP/1.1\r\nhost: relay\r\nconnection: close\r\n\r\n").as_bytes(),
+ )
+ .ok()?;
+ let mut out = String::new();
+ s.read_to_string(&mut out).ok()?;
+ out.lines().next().map(str::to_owned)
+}
+
+fn wait_for(port: u16) {
+ let deadline = Instant::now() + Duration::from_secs(20);
+ while Instant::now() < deadline {
+ if get(port, "/healthz", Duration::from_secs(1)).is_some() {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(50));
+ }
+ panic!("nothing answered on port {port}");
+}
+
+fn sigterm(child: &Child) {
+ assert!(Command::new("kill")
+ .args(["-TERM", &child.id().to_string()])
+ .status()
+ .expect("kill")
+ .success());
+}
+
+#[test]
+fn the_next_relay_waits_then_serves_what_queued() {
+ let state = tempfile::tempdir().unwrap();
+ let (a_port, a_health) = (free_port(), free_port());
+ let mut a = start(state.path(), a_port, a_health);
+ wait_for(a_port);
+
+ let (b_port, b_health) = (free_port(), free_port());
+ let mut b = start(state.path(), b_port, b_health);
+ wait_for(b_health);
+ // Standby: the probe port answers, the main port holds requests.
+ assert!(
+ get(b_port, "/healthz", Duration::from_millis(500)).is_none(),
+ "the second relay served while the first still owned the state"
+ );
+ let held = std::thread::spawn(move || {
+ let started = Instant::now();
+ (
+ get(b_port, "/v1/info", Duration::from_secs(15)),
+ started.elapsed(),
+ )
+ });
+ std::thread::sleep(Duration::from_millis(300));
+
+ let stopped = Instant::now();
+ sigterm(&a);
+ let (status, _) = held.join().unwrap();
+ let after_stop = stopped.elapsed();
+ let status = status.expect("the held request was dropped");
+ assert!(status.contains(" 200 "), "{status}");
+ assert!(
+ after_stop < Duration::from_secs(5),
+ "took over {after_stop:?} after the first relay was stopped"
+ );
+ assert!(a.wait().unwrap().success());
+ assert!(get(b_health, "/healthz", Duration::from_secs(2))
+ .unwrap()
+ .contains(" 204 "));
+ sigterm(&b);
+ assert!(b.wait().unwrap().success());
+}