Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(mesh): honour 429 Retry-After, send a client version, retire a taken-over identity; relay holds refused connects (CUA-2362) (#6016)
cua-agents[bot]87650e144354
diff --git a/libs/cua-spacesd/crates/cua-relay/src/mesh.rs b/libs/cua-spacesd/crates/cua-relay/src/mesh.rsindex 198ed21097..777bc4ae48 100644--- a/libs/cua-spacesd/crates/cua-relay/src/mesh.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/mesh.rs@@ -96,6 +96,8 @@ const DENY_FOR: Duration = Duration::from_secs(30);/// relay: the node's own backoff keeps well under this).const MAX_CONNECTS_PER_WINDOW: usize = 10;const CONNECT_WINDOW: Duration = Duration::from_secs(60);+/// How long a refused connect is held open before the refusal is sent.+const LIMITED_HOLD_S: u64 = 5;/// A node that keeps being refused is logged on its first refusal, then once/// per this interval with the count since (a client stuck in a reconnect loop/// would otherwise flood the logs).@@ -1162,9 +1164,15 @@ pub(crate) async fn connect(Ok(o) => o,Err(r) => return r,};+ // The client's own name and version (cua-mesh/x.y.z; older clients send+ // none), for the logs of a loop.+ let agent = headers+ .get(header::USER_AGENT)+ .and_then(|v| v.to_str().ok())+ .map_or_else(|| "none".to_owned(), |ua| ua.chars().take(80).collect());upgrade.max_message_size(proto::MAX_RELAY_PACKET + 64)- .on_upgrade(move |socket| async move { session(relay, owner, socket).await })+ .on_upgrade(move |socket| async move { session(relay, owner, agent, socket).await })}#[allow(clippy::result_large_err)]@@ -1250,7 +1258,7 @@ async fn authenticate(relay: &Relay, headers: &HeaderMap) -> Result<Owner, Respo}}-async fn session(relay: Relay, owner: Owner, socket: WebSocket) {+async fn session(relay: Relay, owner: Owner, agent: String, socket: WebSocket) {let (mut sink, mut stream) = socket.split();let server = SecretKey::generate();let nonce = hex::encode(rand::random::<[u8; 16]>());@@ -1323,17 +1331,22 @@ async fn session(relay: Relay, owner: Owner, socket: WebSocket) {if let Err(wait) = mesh.admit(&node_key, Instant::now()) {match mesh.limited_log(&node_key, Instant::now()) {LimitedLog::First => {- tracing::warn!(node = %node_key.short(), ?kind, retry_after_s = wait, "mesh node connects too often");+ tracing::warn!(node = %node_key.short(), ?kind, %agent, retry_after_s = wait, "mesh node connects too often");}LimitedLog::Summary(refused) => {tracing::warn!(- node = %node_key.short(), ?kind, refused,+ node = %node_key.short(), ?kind, %agent, refused,interval_s = LIMITED_LOG_INTERVAL.as_secs(),"mesh node still connects too often");}LimitedLog::Quiet => {}}+ // Clients before the 0.16 backoff ignore the retry-after and come+ // straight back: holding the socket open for a few seconds first+ // slows them to a fraction of the pace, and costs the relay a+ // parked task.+ tokio::time::sleep(Duration::from_secs(wait.min(LIMITED_HOLD_S))).await;let _ = sink.send(close(proto::CLOSE_RATE_LIMITED,@@ -1463,7 +1476,7 @@ async fn session(relay: Relay, owner: Owner, socket: WebSocket) {// The same node reconnected: the newer connection wins. A live old// connection means a second process holds this node key; it is told// so, and a node that keeps hearing it stops reconnecting.- tracing::info!(node = %node_key.short(), "mesh node replaced its older connection");+ tracing::info!(node = %node_key.short(), %agent, "mesh node replaced its older connection");let _ = old.tx.try_send(close(proto::CLOSE_REPLACED,"replaced: another connection with the same node key took over",diff --git a/libs/cua/crates/cua-mesh/src/coord.rs b/libs/cua/crates/cua-mesh/src/coord.rsindex cdce06b1ff..980415651e 100644--- a/libs/cua/crates/cua-mesh/src/coord.rs+++ b/libs/cua/crates/cua-mesh/src/coord.rs@@ -96,6 +96,10 @@ pub(crate) struct ControlConfig {pub ephemeral: bool,/// Divides the reconnect wait (tests).pub reconnect_speedup: u32,+ /// The node identity file, when the identity is persistent: replaced by+ /// a new identity when other connections keep taking this one over (see+ /// [`MAX_TAKEOVERS`]).+ pub identity_path: Option<std::path::PathBuf>,}impl Control {@@ -274,6 +278,21 @@ struct Ended {/// The coordinator ended the session (a close or a dropped/// connection after the welcome), as opposed to a failure on this side.by_relay: bool,+ /// The `Retry-After` of an HTTP 429 that refused the connection.+ http_retry_after: Option<Duration>,+}++/// The seconds of a `Retry-After` header (the delay form; a date is ignored),+/// capped so a bad header cannot park a node for long.+fn retry_after_header(headers: &http::HeaderMap) -> Option<Duration> {+ let secs: u64 = headers+ .get(http::header::RETRY_AFTER)?+ .to_str()+ .ok()?+ .trim()+ .parse()+ .ok()?;+ Some(Duration::from_secs(secs.min(3600)))}impl Ended {@@ -297,6 +316,9 @@ impl Ended {}fn retry_after(&self) -> Option<Duration> {+ if let Some(wait) = self.http_retry_after {+ return Some(wait);+ }let (code, reason) = self.close.as_ref()?;(*code == proto::CLOSE_RATE_LIMITED).then(|| proto::parse_retry_after(reason))@@ -312,6 +334,17 @@ const MAX_TAKEOVERS: u32 = 3;const TAKEN_OVER: &str = "another copy of Spaces keeps taking over this computer's connection \(two copies share one identity). Quit the other copy, then reopen Spaces here.";+/// Removes the identity file at `path` if it still holds `identity`.+fn retire_identity(path: &std::path::Path, identity: &NodeIdentity) {+ let ours = std::fs::read_to_string(path)+ .ok()+ .and_then(|raw| NodeIdentity::from_json(&raw).ok())+ .is_some_and(|on_disk| on_disk.public() == identity.public());+ if ours && let Err(e) = std::fs::remove_file(path) {+ tracing::warn!(error = %e, "could not retire the mesh identity");+ }+}+pub(crate) async fn run(cfg: ControlConfig,control: Arc<Control>,@@ -382,6 +415,12 @@ pub(crate) async fn run();if gave_up {tracing::error!("{TAKEN_OVER} Not reconnecting.");+ // A copy of this identity file on another computer (a cloned+ // disk or image) would keep fighting for the key, so the next+ // node on this computer starts with a new one.+ if let Some(path) = &cfg.identity_path {+ retire_identity(path, &identity);+ }return;}tokio::select! {@@ -413,6 +452,14 @@ async fn session(value.set_sensitive(true);request.headers_mut().insert(name, value);}+ // The version in the relay's logs: who loops is then one grep. A+ // credential provider may set its own.+ if !request.headers().contains_key(http::header::USER_AGENT) {+ request.headers_mut().insert(+ http::header::USER_AGENT,+ http::HeaderValue::from_static(concat!("cua-mesh/", env!("CARGO_PKG_VERSION"))),+ );+ }// An explicit TLS setup (ring, the web PKI roots): never the// process-wide default provider, which a host process may leave// ambiguous. No Nagle: relayed packets and control frames are small.@@ -426,6 +473,10 @@ async fn session(Ok(Ok(v)) => v,Ok(Err(tokio_tungstenite::tungstenite::Error::Http(resp))) => {let status = resp.status();+ if status.as_u16() == 429 {+ // Too many connections: wait what the relay says, at least.+ ended.http_retry_after = retry_after_header(resp.headers());+ }let body = resp.body().as_ref()@@ -704,6 +755,44 @@ mod tests {assert!(bare.by_relay && bare.close.is_none());}+ #[test]+ fn an_http_429_retry_after_is_honoured_and_capped() {+ let mut h = http::HeaderMap::new();+ assert_eq!(retry_after_header(&h), None);+ h.insert(http::header::RETRY_AFTER, "42".parse().unwrap());+ assert_eq!(retry_after_header(&h), Some(Duration::from_secs(42)));+ h.insert(http::header::RETRY_AFTER, "999999".parse().unwrap());+ assert_eq!(retry_after_header(&h), Some(Duration::from_secs(3600)));+ h.insert(+ http::header::RETRY_AFTER,+ "Wed, 21 Oct 2026 07:28:00 GMT".parse().unwrap(),+ );+ assert_eq!(retry_after_header(&h), None);+ let ended = Ended {+ http_retry_after: Some(Duration::from_secs(9)),+ ..Ended::default()+ };+ assert_eq!(ended.retry_after(), Some(Duration::from_secs(9)));+ }++ #[test]+ fn a_node_that_gave_up_retires_its_own_identity_file_only() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("client.json");+ let mine = NodeIdentity::load_or_create(&path).unwrap();+ // Someone else's identity on disk is left alone.+ let other = NodeIdentity::generate().unwrap();+ retire_identity(&path, &other);+ assert!(path.exists());+ retire_identity(&path, &mine);+ assert!(!path.exists());+ // The next start gets a new key.+ assert_ne!(+ NodeIdentity::load_or_create(&path).unwrap().public(),+ mine.public()+ );+ }+#[test]fn a_rate_limit_close_says_how_long_to_wait() {let mut ended = Ended::default();diff --git a/libs/cua/crates/cua-mesh/src/key.rs b/libs/cua/crates/cua-mesh/src/key.rsindex ad009a878e..abd34fa3cb 100644--- a/libs/cua/crates/cua-mesh/src/key.rs+++ b/libs/cua/crates/cua-mesh/src/key.rs@@ -204,7 +204,7 @@ impl NodeIdentity {write_private(path, &json)}- fn from_json(raw: &str) -> Result<Self> {+ pub(crate) fn from_json(raw: &str) -> Result<Self> {let s: StoredIdentity =serde_json::from_str(raw).map_err(|e| Error::Protocol(e.to_string()))?;let d = |v: &str| {diff --git a/libs/cua/crates/cua-mesh/src/node.rs b/libs/cua/crates/cua-mesh/src/node.rsindex f100cc18ce..5f50fe507c 100644--- a/libs/cua/crates/cua-mesh/src/node.rs+++ b/libs/cua/crates/cua-mesh/src/node.rs@@ -344,6 +344,7 @@ impl Mesh {approved,ephemeral: cfg.identity_path.is_none(),reconnect_speedup: cfg.reconnect_speedup.max(1),+ identity_path: cfg.identity_path.clone(),},control,engine.clone(),