Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(mesh): honour 429 Retry-After, send a client version, retire a taken-over identity; relay holds refused connects (CUA-2362) (#6016)

cua-agents[bot]87650e144354
diff --git a/libs/cua-spacesd/crates/cua-relay/src/mesh.rs b/libs/cua-spacesd/crates/cua-relay/src/mesh.rs
index 198ed21097..777bc4ae48 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/mesh.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/mesh.rs
@@ -96,6 +96,8 @@ const DENY_FOR: Duration = Duration::from_secs(30);
/// relay: the node's own backoff keeps well under this).
const MAX_CONNECTS_PER_WINDOW: usize = 10;
const CONNECT_WINDOW: Duration = Duration::from_secs(60);
+/// How long a refused connect is held open before the refusal is sent.
+const LIMITED_HOLD_S: u64 = 5;
/// A node that keeps being refused is logged on its first refusal, then once
/// per this interval with the count since (a client stuck in a reconnect loop
/// would otherwise flood the logs).
@@ -1162,9 +1164,15 @@ pub(crate) async fn connect(
Ok(o) => o,
Err(r) => return r,
};
+ // The client's own name and version (cua-mesh/x.y.z; older clients send
+ // none), for the logs of a loop.
+ let agent = headers
+ .get(header::USER_AGENT)
+ .and_then(|v| v.to_str().ok())
+ .map_or_else(|| "none".to_owned(), |ua| ua.chars().take(80).collect());
upgrade
.max_message_size(proto::MAX_RELAY_PACKET + 64)
- .on_upgrade(move |socket| async move { session(relay, owner, socket).await })
+ .on_upgrade(move |socket| async move { session(relay, owner, agent, socket).await })
}
#[allow(clippy::result_large_err)]
@@ -1250,7 +1258,7 @@ async fn authenticate(relay: &Relay, headers: &HeaderMap) -> Result<Owner, Respo
}
}
-async fn session(relay: Relay, owner: Owner, socket: WebSocket) {
+async fn session(relay: Relay, owner: Owner, agent: String, socket: WebSocket) {
let (mut sink, mut stream) = socket.split();
let server = SecretKey::generate();
let nonce = hex::encode(rand::random::<[u8; 16]>());
@@ -1323,17 +1331,22 @@ async fn session(relay: Relay, owner: Owner, socket: WebSocket) {
if let Err(wait) = mesh.admit(&node_key, Instant::now()) {
match mesh.limited_log(&node_key, Instant::now()) {
LimitedLog::First => {
- tracing::warn!(node = %node_key.short(), ?kind, retry_after_s = wait, "mesh node connects too often");
+ tracing::warn!(node = %node_key.short(), ?kind, %agent, retry_after_s = wait, "mesh node connects too often");
}
LimitedLog::Summary(refused) => {
tracing::warn!(
- node = %node_key.short(), ?kind, refused,
+ node = %node_key.short(), ?kind, %agent, refused,
interval_s = LIMITED_LOG_INTERVAL.as_secs(),
"mesh node still connects too often"
);
}
LimitedLog::Quiet => {}
}
+ // Clients before the 0.16 backoff ignore the retry-after and come
+ // straight back: holding the socket open for a few seconds first
+ // slows them to a fraction of the pace, and costs the relay a
+ // parked task.
+ tokio::time::sleep(Duration::from_secs(wait.min(LIMITED_HOLD_S))).await;
let _ = sink
.send(close(
proto::CLOSE_RATE_LIMITED,
@@ -1463,7 +1476,7 @@ async fn session(relay: Relay, owner: Owner, socket: WebSocket) {
// The same node reconnected: the newer connection wins. A live old
// connection means a second process holds this node key; it is told
// so, and a node that keeps hearing it stops reconnecting.
- tracing::info!(node = %node_key.short(), "mesh node replaced its older connection");
+ tracing::info!(node = %node_key.short(), %agent, "mesh node replaced its older connection");
let _ = old.tx.try_send(close(
proto::CLOSE_REPLACED,
"replaced: another connection with the same node key took over",
diff --git a/libs/cua/crates/cua-mesh/src/coord.rs b/libs/cua/crates/cua-mesh/src/coord.rs
index cdce06b1ff..980415651e 100644
--- a/libs/cua/crates/cua-mesh/src/coord.rs
+++ b/libs/cua/crates/cua-mesh/src/coord.rs
@@ -96,6 +96,10 @@ pub(crate) struct ControlConfig {
pub ephemeral: bool,
/// Divides the reconnect wait (tests).
pub reconnect_speedup: u32,
+ /// The node identity file, when the identity is persistent: replaced by
+ /// a new identity when other connections keep taking this one over (see
+ /// [`MAX_TAKEOVERS`]).
+ pub identity_path: Option<std::path::PathBuf>,
}
impl Control {
@@ -274,6 +278,21 @@ struct Ended {
/// The coordinator ended the session (a close or a dropped
/// connection after the welcome), as opposed to a failure on this side.
by_relay: bool,
+ /// The `Retry-After` of an HTTP 429 that refused the connection.
+ http_retry_after: Option<Duration>,
+}
+
+/// The seconds of a `Retry-After` header (the delay form; a date is ignored),
+/// capped so a bad header cannot park a node for long.
+fn retry_after_header(headers: &http::HeaderMap) -> Option<Duration> {
+ let secs: u64 = headers
+ .get(http::header::RETRY_AFTER)?
+ .to_str()
+ .ok()?
+ .trim()
+ .parse()
+ .ok()?;
+ Some(Duration::from_secs(secs.min(3600)))
}
impl Ended {
@@ -297,6 +316,9 @@ impl Ended {
}
fn retry_after(&self) -> Option<Duration> {
+ if let Some(wait) = self.http_retry_after {
+ return Some(wait);
+ }
let (code, reason) = self.close.as_ref()?;
(*code == proto::CLOSE_RATE_LIMITED)
.then(|| proto::parse_retry_after(reason))
@@ -312,6 +334,17 @@ const MAX_TAKEOVERS: u32 = 3;
const TAKEN_OVER: &str = "another copy of Spaces keeps taking over this computer's connection \
(two copies share one identity). Quit the other copy, then reopen Spaces here.";
+/// Removes the identity file at `path` if it still holds `identity`.
+fn retire_identity(path: &std::path::Path, identity: &NodeIdentity) {
+ let ours = std::fs::read_to_string(path)
+ .ok()
+ .and_then(|raw| NodeIdentity::from_json(&raw).ok())
+ .is_some_and(|on_disk| on_disk.public() == identity.public());
+ if ours && let Err(e) = std::fs::remove_file(path) {
+ tracing::warn!(error = %e, "could not retire the mesh identity");
+ }
+}
+
pub(crate) async fn run(
cfg: ControlConfig,
control: Arc<Control>,
@@ -382,6 +415,12 @@ pub(crate) async fn run(
);
if gave_up {
tracing::error!("{TAKEN_OVER} Not reconnecting.");
+ // A copy of this identity file on another computer (a cloned
+ // disk or image) would keep fighting for the key, so the next
+ // node on this computer starts with a new one.
+ if let Some(path) = &cfg.identity_path {
+ retire_identity(path, &identity);
+ }
return;
}
tokio::select! {
@@ -413,6 +452,14 @@ async fn session(
value.set_sensitive(true);
request.headers_mut().insert(name, value);
}
+ // The version in the relay's logs: who loops is then one grep. A
+ // credential provider may set its own.
+ if !request.headers().contains_key(http::header::USER_AGENT) {
+ request.headers_mut().insert(
+ http::header::USER_AGENT,
+ http::HeaderValue::from_static(concat!("cua-mesh/", env!("CARGO_PKG_VERSION"))),
+ );
+ }
// An explicit TLS setup (ring, the web PKI roots): never the
// process-wide default provider, which a host process may leave
// ambiguous. No Nagle: relayed packets and control frames are small.
@@ -426,6 +473,10 @@ async fn session(
Ok(Ok(v)) => v,
Ok(Err(tokio_tungstenite::tungstenite::Error::Http(resp))) => {
let status = resp.status();
+ if status.as_u16() == 429 {
+ // Too many connections: wait what the relay says, at least.
+ ended.http_retry_after = retry_after_header(resp.headers());
+ }
let body = resp
.body()
.as_ref()
@@ -704,6 +755,44 @@ mod tests {
assert!(bare.by_relay && bare.close.is_none());
}
+ #[test]
+ fn an_http_429_retry_after_is_honoured_and_capped() {
+ let mut h = http::HeaderMap::new();
+ assert_eq!(retry_after_header(&h), None);
+ h.insert(http::header::RETRY_AFTER, "42".parse().unwrap());
+ assert_eq!(retry_after_header(&h), Some(Duration::from_secs(42)));
+ h.insert(http::header::RETRY_AFTER, "999999".parse().unwrap());
+ assert_eq!(retry_after_header(&h), Some(Duration::from_secs(3600)));
+ h.insert(
+ http::header::RETRY_AFTER,
+ "Wed, 21 Oct 2026 07:28:00 GMT".parse().unwrap(),
+ );
+ assert_eq!(retry_after_header(&h), None);
+ let ended = Ended {
+ http_retry_after: Some(Duration::from_secs(9)),
+ ..Ended::default()
+ };
+ assert_eq!(ended.retry_after(), Some(Duration::from_secs(9)));
+ }
+
+ #[test]
+ fn a_node_that_gave_up_retires_its_own_identity_file_only() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("client.json");
+ let mine = NodeIdentity::load_or_create(&path).unwrap();
+ // Someone else's identity on disk is left alone.
+ let other = NodeIdentity::generate().unwrap();
+ retire_identity(&path, &other);
+ assert!(path.exists());
+ retire_identity(&path, &mine);
+ assert!(!path.exists());
+ // The next start gets a new key.
+ assert_ne!(
+ NodeIdentity::load_or_create(&path).unwrap().public(),
+ mine.public()
+ );
+ }
+
#[test]
fn a_rate_limit_close_says_how_long_to_wait() {
let mut ended = Ended::default();
diff --git a/libs/cua/crates/cua-mesh/src/key.rs b/libs/cua/crates/cua-mesh/src/key.rs
index ad009a878e..abd34fa3cb 100644
--- a/libs/cua/crates/cua-mesh/src/key.rs
+++ b/libs/cua/crates/cua-mesh/src/key.rs
@@ -204,7 +204,7 @@ impl NodeIdentity {
write_private(path, &json)
}
- fn from_json(raw: &str) -> Result<Self> {
+ pub(crate) fn from_json(raw: &str) -> Result<Self> {
let s: StoredIdentity =
serde_json::from_str(raw).map_err(|e| Error::Protocol(e.to_string()))?;
let d = |v: &str| {
diff --git a/libs/cua/crates/cua-mesh/src/node.rs b/libs/cua/crates/cua-mesh/src/node.rs
index f100cc18ce..5f50fe507c 100644
--- a/libs/cua/crates/cua-mesh/src/node.rs
+++ b/libs/cua/crates/cua-mesh/src/node.rs
@@ -344,6 +344,7 @@ impl Mesh {
approved,
ephemeral: cfg.identity_path.is_none(),
reconnect_speedup: cfg.reconnect_speedup.max(1),
+ identity_path: cfg.identity_path.clone(),
},
control,
engine.clone(),