Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-mesh): a failed dial no longer answers for a peer that came online since (#6479)

injaneityebf96e5ff17a
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/tests/mesh_gang.rs b/libs/cua-spacesd/crates/cua-spacesd-server/tests/mesh_gang.rs
index 8a38b18f1e..fd574f6f54 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-server/tests/mesh_gang.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/tests/mesh_gang.rs
@@ -186,14 +186,17 @@ fn free_port() -> u16 {
.port()
}
+/// Waits for `key` to be in `m`'s map and online: a refusal asserted
+/// below must come from the host, not from a dial to a peer the map still
+/// lists offline (its first map can arrive before the host's node joins).
async fn in_map(m: &Mesh, key: &cua_mesh::PublicKey) {
for _ in 0..200 {
- if m.peers().iter().any(|p| &p.node_key == key) {
+ if m.peers().iter().any(|p| &p.node_key == key && p.online) {
return;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
- panic!("peer never appeared");
+ panic!("peer never appeared online");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
diff --git a/libs/cua/crates/cua-mesh/src/node.rs b/libs/cua/crates/cua-mesh/src/node.rs
index 9a03002fdc..9fa58b8738 100644
--- a/libs/cua/crates/cua-mesh/src/node.rs
+++ b/libs/cua/crates/cua-mesh/src/node.rs
@@ -341,8 +341,9 @@ struct Inner {
/// Connections peers opened to us, so a peer leaving the map is cut.
inbound: Mutex<Vec<(PublicKey, quinn::Connection)>>,
connect_timeout: Duration,
- /// The last failed dial per peer: when, and why.
- failed_dials: Mutex<HashMap<PublicKey, (Instant, String)>>,
+ /// The last failed dial per peer: when, whether the map listed the
+ /// peer online then, and why.
+ failed_dials: Mutex<HashMap<PublicKey, (Instant, bool, String)>>,
}
/// One node on the mesh. Cheap to clone.
@@ -667,7 +668,7 @@ impl Mesh {
return Ok(c);
}
// A dial that just failed answers the callers that waited for it.
- if let Some(why) = self.recent_failure(&peer.node_key) {
+ if let Some(why) = self.recent_failure(peer) {
return Err(Error::Unreachable(why));
}
self.inner.engine.want(&peer.node_key);
@@ -727,7 +728,7 @@ impl Mesh {
.failed_dials
.lock()
.expect("failed dials")
- .insert(peer.node_key, (Instant::now(), e.to_string()));
+ .insert(peer.node_key, (Instant::now(), peer.online, e.to_string()));
}
Err(e)
}
@@ -745,13 +746,14 @@ impl Mesh {
}
/// Why the last dial to `peer` failed, when it failed less than
- /// [`FAILED_DIAL_HOLD`] ago.
- fn recent_failure(&self, peer: &PublicKey) -> Option<String> {
+ /// [`FAILED_DIAL_HOLD`] ago and the map still lists it as it did then
+ /// (a peer that came online since gets a fresh dial).
+ fn recent_failure(&self, peer: &PeerInfo) -> Option<String> {
let failed = self.inner.failed_dials.lock().expect("failed dials");
failed
- .get(peer)
- .filter(|(at, _)| at.elapsed() < FAILED_DIAL_HOLD)
- .map(|(_, why)| why.clone())
+ .get(&peer.node_key)
+ .filter(|(at, online, _)| at.elapsed() < FAILED_DIAL_HOLD && *online == peer.online)
+ .map(|(_, _, why)| why.clone())
}
/// Why this node can no longer use the mesh, while it holds: its
diff --git a/libs/cua/crates/cua-mesh/tests/mesh.rs b/libs/cua/crates/cua-mesh/tests/mesh.rs
index be7b9b0bab..87bb649768 100644
--- a/libs/cua/crates/cua-mesh/tests/mesh.rs
+++ b/libs/cua/crates/cua-mesh/tests/mesh.rs
@@ -930,3 +930,48 @@ async fn a_node_dropped_for_other_reasons_keeps_reconnecting() {
}
assert!(client.control_state().denied.is_none());
}
+
+/// A dial to a machine the map lists offline fails, but once the map lists
+/// it online again the next stream dials it at once, instead of answering
+/// with the old failure for a few seconds.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_machine_back_online_is_dialed_again_at_once() {
+ let dir = tempfile::tempdir().unwrap();
+ let coord = FakeCoordinator::start(Mode::Account).await;
+ let machine = node(&coord, NodeKind::Machine, dir.path(), "m", false).await;
+ let key = machine.node_key();
+ let client = node(&coord, NodeKind::Client, dir.path(), "c", false).await;
+ wait_peer(&client, &key).await;
+ machine.shutdown();
+ drop(machine);
+ let online = |c: &Mesh| c.machine("machine-m").is_some_and(|p| p.online);
+ for _ in 0..100 {
+ if !online(&client) {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ assert!(!online(&client), "the map never listed the machine offline");
+ // The dial times out (only that is remembered) and starts the hold.
+ let failed_at = std::time::Instant::now();
+ assert!(client.open_stream(&key, "echo", None).await.is_err());
+
+ // The same machine (same identity) comes back.
+ let machine = node(&coord, NodeKind::Machine, dir.path(), "m", false).await;
+ assert_eq!(machine.node_key(), key);
+ serve_echo(&machine);
+ for _ in 0..100 {
+ if online(&client) {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ assert!(online(&client), "the map never listed the machine online");
+ // Inside the hold, or this test passes without the fix.
+ assert!(failed_at.elapsed() < Duration::from_secs(5));
+ let mut s = client
+ .open_stream(&key, "echo", None)
+ .await
+ .expect("a machine back online is dialed again");
+ roundtrip(&mut s, 1024).await;
+}