Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-mesh): a client that hangs up mid-dial, a connection burst and a forward's UDP flows no longer hold descriptors (CUA-2159) (#6475)

ddupont2cac516f93eb
diff --git a/libs/cua/crates/cua-mesh/src/node.rs b/libs/cua/crates/cua-mesh/src/node.rs
index 496de0153b..8265f605cb 100644
--- a/libs/cua/crates/cua-mesh/src/node.rs
+++ b/libs/cua/crates/cua-mesh/src/node.rs
@@ -11,7 +11,7 @@ use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, Instant};
use serde::Serialize;
-use tokio::net::{TcpListener, UdpSocket};
+use tokio::net::{TcpListener, TcpStream, UdpSocket};
use tokio::sync::mpsc;
use crate::coord::{self, AuthProvider, Control, ControlConfig, ControlState, PairVerifier};
@@ -119,6 +119,8 @@ pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(20);
/// How long a failed dial answers for its peer: callers that waited for it,
/// and callers arriving right after, fail at once instead of each dialing
/// (and holding its local connection) for another full timeout.
+/// Connections one [`Forward`] serves at once; more are closed on accept.
+const MAX_FORWARD_CONNECTIONS: usize = 256;
const FAILED_DIAL_HOLD: Duration = Duration::from_secs(5);
/// The machine's hostname, for display.
@@ -846,6 +848,7 @@ impl Mesh {
// its local socket while it waited for a peer that never answered.
let stop = coord::tokio_util_cancel::Token::new();
let stop2 = stop.clone();
+ let inflight = Arc::new(tokio::sync::Semaphore::new(MAX_FORWARD_CONNECTIONS));
let tcp_task = tokio::spawn(async move {
loop {
let conn = match tcp.accept().await {
@@ -858,13 +861,28 @@ impl Mesh {
continue;
}
};
+ // A forward serves a bounded number of connections at once:
+ // past it a new one is closed at once (the client retries)
+ // instead of piling up sockets behind a peer that is slow.
+ let Ok(permit) = inflight.clone().try_acquire_owned() else {
+ tracing::debug!("mesh forward at its connection limit");
+ continue;
+ };
let me = me.clone();
let service = service.clone();
let session = session2.clone();
let stop = stop2.clone();
tokio::spawn(async move {
+ let _permit = permit;
let serve = async {
- match me.open_stream(&peer, &service, session).await {
+ // The client hanging up while the stream is still
+ // being dialled ends the connection now, not when
+ // the dial gives up (up to the connect timeout).
+ let opened = tokio::select! {
+ r = me.open_stream(&peer, &service, session) => r,
+ () = client_hung_up(&conn) => return,
+ };
+ match opened {
Ok(stream) => quic::splice(conn, stream).await,
// The process or the node is going away (a short
// `cua` run exiting): nothing to warn about.
@@ -1020,6 +1038,21 @@ impl Mesh {
}
}
+/// Resolves when the peer of `conn` closed its end (EOF or reset). Bytes it
+/// sent first are left unread for the splice.
+async fn client_hung_up(conn: &TcpStream) {
+ loop {
+ match conn.ready(tokio::io::Interest::READABLE).await {
+ Ok(r) if r.is_read_closed() => return,
+ Ok(_) => {}
+ Err(_) => return,
+ }
+ // Readable with data (not EOF): look again shortly; the bytes stay
+ // queued for the splice.
+ tokio::time::sleep(Duration::from_millis(250)).await;
+ }
+}
+
/// Whether a forward's failure only means this node is stopping: it was
/// shut down, or the QUIC endpoint is (the runtime is ending).
fn stopping(e: &Error, shut_down: bool) -> bool {
@@ -1349,9 +1382,14 @@ type Flows = HashMap<SocketAddr, (Arc<VSocket>, tokio::time::Instant)>;
async fn udp_client_forward(mesh: Mesh, local: Arc<UdpSocket>, peer: PublicKey) {
// One mesh socket per local source, so replies find their way back.
let flows: Arc<Mutex<Flows>> = Arc::default();
+ // The per-flow readers hold the local socket: they end with this task
+ // (the forward being dropped), not 180 s later.
+ let mut readers = tokio::task::JoinSet::new();
let mut buf = vec![0u8; 65_535];
loop {
+ while readers.try_join_next().is_some() {}
let Ok((n, src)) = local.recv_from(&mut buf).await else {
+ tokio::time::sleep(Duration::from_millis(100)).await;
continue;
};
let Some(info) = mesh
@@ -1380,7 +1418,7 @@ async fn udp_client_forward(mesh: Mesh, local: Arc<UdpSocket>, peer: PublicKey)
let back = local.clone();
let reader = s.clone();
let flows = flows.clone();
- tokio::spawn(async move {
+ readers.spawn(async move {
loop {
let r =
tokio::time::timeout(Duration::from_secs(180), reader.recv_from())
diff --git a/libs/cua/crates/cua-mesh/tests/forward_fds_hangup.rs b/libs/cua/crates/cua-mesh/tests/forward_fds_hangup.rs
new file mode 100644
index 0000000000..ace914817a
--- /dev/null
+++ b/libs/cua/crates/cua-mesh/tests/forward_fds_hangup.rs
@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! The other ways a loopback forward can hold a descriptor after its user
+//! is gone (CUA-2159 follow-up; `forward_fds.rs` covers the probe loop):
+//! a client that hangs up while its stream is still being dialled (the
+//! connection must not wait out the dial), a burst of clients past the
+//! forward's in-flight bound, and the UDP socket and flows of a forward
+//! that carried datagrams (they used to outlive the forward by 180 s).
+//!
+//! One test per binary: it counts this process's descriptors.
+
+#![cfg(unix)]
+
+use std::time::Duration;
+
+use cua_mesh::testing::FakeCoordinator;
+use cua_mesh::{Mesh, MeshConfig, Mode, NodeKind, PublicKey, static_auth};
+use tokio::io::AsyncWriteExt as _;
+use tokio::net::TcpStream;
+
+/// Probe cycles: enough that a leak of one descriptor each stands out.
+const CYCLES: usize = 40;
+/// Descriptors the runtime may open or close on its own meanwhile.
+const SLACK: usize = 6;
+
+fn open_fds() -> usize {
+ std::fs::read_dir("/dev/fd").map(|d| d.count()).unwrap_or(0)
+}
+
+async fn node(coord: &FakeCoordinator, kind: NodeKind, dir: &std::path::Path, name: &str) -> Mesh {
+ let mut headers = Vec::new();
+ if kind == NodeKind::Machine {
+ headers.push(("x-cua-machine-id".to_owned(), format!("machine-{name}")));
+ }
+ let mut cfg = MeshConfig::new(
+ coord.url.clone(),
+ kind,
+ Some(dir.join(format!("{name}.json"))),
+ static_auth(headers),
+ );
+ cfg.name = name.to_owned();
+ cfg.stun = vec![coord.stun.to_string()];
+ // Only the relay carries packets, so turning it off silences the peer.
+ cfg.relay_only = true;
+ cfg.connect_timeout = Duration::from_secs(30);
+ let m = Mesh::start(cfg).await.expect("start");
+ m.wait_ready(Duration::from_secs(10)).await.expect("ready");
+ m
+}
+
+async fn wait_peer(m: &Mesh, key: &PublicKey) {
+ for _ in 0..200 {
+ if m.peers().iter().any(|p| &p.node_key == key) {
+ return;
+ }
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ panic!("peer never appeared");
+}
+
+/// One probe: connect, send a request, give up after `wait`.
+async fn probe(addr: std::net::SocketAddr, wait: Duration) {
+ let mut c = TcpStream::connect(addr).await.expect("connect");
+ let _ = c
+ .write_all(b"GET /v1/info HTTP/1.1\r\nhost: probe\r\n\r\n")
+ .await;
+ tokio::time::sleep(wait).await;
+}
+
+/// Polls until the count is back within `SLACK` of `base`, or `within`.
+async fn settle(base: usize, within: Duration) -> usize {
+ let deadline = tokio::time::Instant::now() + within;
+ loop {
+ let n = open_fds();
+ if n <= base + SLACK || tokio::time::Instant::now() >= deadline {
+ return n;
+ }
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn clients_that_hang_up_and_udp_flows_do_not_outlive_the_forward() {
+ let dir = tempfile::tempdir().unwrap();
+ let coord = FakeCoordinator::start(Mode::Account).await;
+ let machine = node(&coord, NodeKind::Machine, dir.path(), "m").await;
+ let client = node(&coord, NodeKind::Client, dir.path(), "c").await;
+ let key = machine.node_key();
+ wait_peer(&client, &key).await;
+ coord.set_relay(false);
+
+ {
+ let fwd = client.forward(key, "spacesd", None).await.unwrap();
+ probe(fwd.local, Duration::from_millis(50)).await;
+ udp_burst(fwd.local, 2).await;
+ }
+ let base = settle(open_fds(), Duration::from_secs(3)).await;
+
+ // 1. Clients hang up while the (30 s) dial to the silent peer is still
+ // running: their sockets close within moments, not after the dial.
+ let fwd = client.forward(key, "spacesd", None).await.unwrap();
+ for _ in 0..CYCLES {
+ probe(fwd.local, Duration::from_millis(10)).await;
+ }
+ let after = settle(base + 2, Duration::from_secs(3)).await;
+ assert!(
+ after <= base + 2 + SLACK,
+ "hung-up clients kept their sockets while the dial ran: {base} before {CYCLES} probes, {after} after"
+ );
+
+ // 2. More simultaneous clients than the forward serves: the rest are
+ // refused at once, so the count stays bounded.
+ let mut held = Vec::new();
+ for _ in 0..(CYCLES * 20) {
+ if let Ok(c) = TcpStream::connect(fwd.local).await {
+ held.push(c);
+ }
+ }
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ let ours = held.len();
+ // The forward's side of each accepted connection is in this process too.
+ let peak = open_fds();
+ assert!(
+ peak < base + ours + 300,
+ "a burst of {ours} clients opened {} descriptors on the forward's side",
+ peak - base - ours
+ );
+ drop(held);
+ drop(fwd);
+
+ // 3. Datagrams through a forward: its UDP socket and flows end with it.
+ for _ in 0..CYCLES / 2 {
+ let fwd = client.forward(key, "spacesd", None).await.unwrap();
+ udp_burst(fwd.local, 3).await;
+ drop(fwd);
+ }
+ let after_udp = settle(base, Duration::from_secs(3)).await;
+ assert!(
+ after_udp <= base + SLACK,
+ "forwards that carried datagrams kept descriptors: {base} before, {after_udp} after"
+ );
+ machine.shutdown();
+ client.shutdown();
+}
+
+/// Sends datagrams to the forward's UDP port from `flows` source ports.
+async fn udp_burst(addr: std::net::SocketAddr, flows: usize) {
+ for _ in 0..flows {
+ let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
+ let _ = s.send_to(b"hello", addr).await;
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+}