Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(mesh): report each machine's path and pin it, with plain reasons (#6278)

ddupont4ece1e9161aa
diff --git a/libs/cua/crates/cua-mesh/src/engine.rs b/libs/cua/crates/cua-mesh/src/engine.rs
index 0dade8d682..6c45beb108 100644
--- a/libs/cua/crates/cua-mesh/src/engine.rs
+++ b/libs/cua/crates/cua-mesh/src/engine.rs
@@ -18,7 +18,7 @@
use std::collections::HashMap;
use std::io;
use std::net::{IpAddr, Ipv6Addr, SocketAddr};
-use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
@@ -63,6 +63,116 @@ impl std::fmt::Display for Path {
}
}
+/// Which path traffic to a peer should take (`cua mesh status --prefer`,
+/// `CUA_MESH_PREFER`).
+///
+/// - `auto` (the default): the fastest answering direct path (the local
+/// network or across the internet), the relay while none answers;
+/// - `lan`: a path on the local network whenever one answers, even when a
+/// path across the internet is faster; otherwise as `auto`;
+/// - `direct`: as `auto` (direct first, the relay only while no direct path
+/// answers), named so a pin reads plainly;
+/// - `relay`: the relay always (end-to-end encrypted all the same).
+///
+/// Whatever the pin, traffic never stops: a pinned path that goes away
+/// falls back to the next one, and comes back when it answers again.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Prefer {
+ /// Fastest first.
+ #[default]
+ Auto,
+ /// The local network first.
+ Lan,
+ /// Direct first.
+ Direct,
+ /// The relay only.
+ Relay,
+}
+
+impl Prefer {
+ /// The environment variable that sets the pin at start.
+ pub const ENV: &'static str = "CUA_MESH_PREFER";
+
+ /// Its word.
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Prefer::Auto => "auto",
+ Prefer::Lan => "lan",
+ Prefer::Direct => "direct",
+ Prefer::Relay => "relay",
+ }
+ }
+
+ /// Parses its word.
+ pub fn parse(word: &str) -> Option<Prefer> {
+ match word.trim().to_ascii_lowercase().as_str() {
+ "" | "auto" => Some(Prefer::Auto),
+ "lan" | "local" => Some(Prefer::Lan),
+ "direct" => Some(Prefer::Direct),
+ "relay" => Some(Prefer::Relay),
+ _ => None,
+ }
+ }
+
+ /// `CUA_MESH_PREFER`, else auto.
+ pub fn from_env() -> Prefer {
+ std::env::var(Self::ENV)
+ .ok()
+ .and_then(|v| Prefer::parse(&v))
+ .unwrap_or_default()
+ }
+
+ fn to_u8(self) -> u8 {
+ self as u8
+ }
+
+ fn from_u8(v: u8) -> Prefer {
+ match v {
+ 1 => Prefer::Lan,
+ 2 => Prefer::Direct,
+ 3 => Prefer::Relay,
+ _ => Prefer::Auto,
+ }
+ }
+}
+
+/// An endpoint on the local network: a private, link-local or unique-local
+/// address (the two machines see each other without the internet).
+pub fn is_lan(addr: &SocketAddr) -> bool {
+ match addr.ip() {
+ IpAddr::V4(v4) => v4.is_private() || v4.is_link_local(),
+ IpAddr::V6(v6) => {
+ let s = v6.segments()[0];
+ (s & 0xfe00) == 0xfc00 || (s & 0xffc0) == 0xfe80
+ }
+ }
+}
+
+/// Whether a pong from `new` (round trip `rtt`) should replace the current
+/// best direct path `best` (round trip `brtt`), given the pin. `stale`: the
+/// current path missed its last round or a ping went unanswered.
+pub(crate) fn should_switch(
+ prefer: Prefer,
+ best: SocketAddr,
+ brtt: Duration,
+ new: SocketAddr,
+ rtt: Duration,
+ stale: bool,
+) -> bool {
+ if stale {
+ return true;
+ }
+ if prefer == Prefer::Lan {
+ match (is_lan(&best), is_lan(&new)) {
+ (false, true) => return true,
+ (true, false) => return false,
+ _ => {}
+ }
+ }
+ rtt.as_secs_f64() < brtt.as_secs_f64() * 0.8
+}
+
#[derive(Debug, Default)]
struct Candidate {
last_ping: Option<Instant>,
@@ -180,6 +290,8 @@ pub(crate) struct Engine {
disco_kick: Notify,
/// Never use direct paths (tests and `CUA_MESH_RELAY_ONLY`).
relay_only: AtomicBool,
+ /// [`Prefer`], as its `u8`.
+ prefer: AtomicU8,
peers_changed: tokio::sync::watch::Sender<u64>,
}
@@ -205,6 +317,7 @@ impl Engine {
local_endpoints: RwLock::new(Vec::new()),
disco_kick: Notify::new(),
relay_only: AtomicBool::new(relay_only),
+ prefer: AtomicU8::new(Prefer::Auto.to_u8()),
peers_changed,
});
let out: Arc<dyn Outbound> = engine.clone();
@@ -293,6 +406,29 @@ impl Engine {
}
}
+ /// The path pin ([`Prefer`]).
+ pub(crate) fn prefer(&self) -> Prefer {
+ Prefer::from_u8(self.prefer.load(Ordering::Relaxed))
+ }
+
+ /// Pins the path: `relay` keeps traffic on the relay; leaving it lets
+ /// discovery find direct paths again; any change re-selects at once.
+ pub(crate) fn set_prefer(&self, prefer: Prefer) {
+ let was = self.prefer();
+ self.prefer.store(prefer.to_u8(), Ordering::SeqCst);
+ if prefer == Prefer::Relay {
+ self.set_relay_only(true);
+ } else if was == Prefer::Relay {
+ self.relay_only.store(false, Ordering::SeqCst);
+ }
+ self.forget_paths();
+ }
+
+ /// Whether relay-only is on (a pin or `CUA_MESH_RELAY_ONLY`).
+ pub(crate) fn relay_only(&self) -> bool {
+ self.relay_only.load(Ordering::Relaxed)
+ }
+
pub(crate) fn set_local_endpoints(&self, endpoints: Vec<SocketAddr>) {
let changed = {
let mut eps = self.local_endpoints.write().expect("endpoints");
@@ -673,14 +809,15 @@ impl Engine {
// The current path leaves a ping unanswered well past
// its round trip (the peer moved: a new NAT mapping
// or network), it missed its last ping round, or
- // this one is clearly faster.
+ // this one is clearly faster (or on the local
+ // network, when pinned to it).
let overdue = brtt * 2 + Duration::from_millis(100);
let best_silent = ps.pending.values().any(|(a, t)| {
*a == best && *t > at && now.duration_since(*t) > overdue
});
- best_silent
- || now.duration_since(at) > PING_INTERVAL + Duration::from_secs(1)
- || rtt.as_secs_f64() < brtt.as_secs_f64() * 0.8
+ let stale = best_silent
+ || now.duration_since(at) > PING_INTERVAL + Duration::from_secs(1);
+ should_switch(self.prefer(), best, brtt, addr, rtt, stale)
}
};
if switch {
@@ -986,3 +1123,58 @@ async fn maintenance(engine: std::sync::Weak<Engine>) {
e.discover();
}
}
+
+#[cfg(test)]
+mod prefer_tests {
+ use super::*;
+
+ fn a(s: &str) -> SocketAddr {
+ s.parse().unwrap()
+ }
+
+ #[test]
+ fn local_network_endpoints_are_lan() {
+ assert!(is_lan(&a("192.168.1.4:41641")));
+ assert!(is_lan(&a("10.0.0.2:1")));
+ assert!(is_lan(&a("169.254.3.3:1")));
+ assert!(is_lan(&a("[fd00::1]:1")));
+ assert!(is_lan(&a("[fe80::1]:1")));
+ assert!(!is_lan(&a("203.0.113.9:41641")));
+ assert!(!is_lan(&a("[2001:db8::1]:1")));
+ }
+
+ #[test]
+ fn auto_takes_the_clearly_faster_path() {
+ let ms = Duration::from_millis;
+ let lan = a("192.168.1.4:1");
+ let wan = a("203.0.113.9:1");
+ assert!(should_switch(Prefer::Auto, lan, ms(10), wan, ms(5), false));
+ assert!(!should_switch(Prefer::Auto, lan, ms(10), wan, ms(9), false));
+ assert!(
+ should_switch(Prefer::Auto, lan, ms(1), wan, ms(50), true),
+ "stale moves on"
+ );
+ }
+
+ #[test]
+ fn a_lan_pin_keeps_the_local_network_unless_it_goes_away() {
+ let ms = Duration::from_millis;
+ let lan = a("192.168.1.4:1");
+ let wan = a("203.0.113.9:1");
+ assert!(!should_switch(Prefer::Lan, lan, ms(10), wan, ms(1), false));
+ assert!(should_switch(Prefer::Lan, wan, ms(1), lan, ms(10), false));
+ assert!(
+ should_switch(Prefer::Lan, lan, ms(10), wan, ms(1), true),
+ "never stuck"
+ );
+ }
+
+ #[test]
+ fn prefer_words_round_trip() {
+ for p in [Prefer::Auto, Prefer::Lan, Prefer::Direct, Prefer::Relay] {
+ assert_eq!(Prefer::parse(p.as_str()), Some(p));
+ assert_eq!(Prefer::from_u8(p.to_u8()), p);
+ }
+ assert_eq!(Prefer::parse("nope"), None);
+ }
+}
diff --git a/libs/cua/crates/cua-mesh/src/gang.rs b/libs/cua/crates/cua-mesh/src/gang.rs
index 4e38b9e8e0..0e85107cf2 100644
--- a/libs/cua/crates/cua-mesh/src/gang.rs
+++ b/libs/cua/crates/cua-mesh/src/gang.rs
@@ -37,6 +37,16 @@
//! `bind` (a private or loopback address of this machine; default
//! loopback) is where the forward listens.
//!
+//! The same socket answers how traffic reaches each other machine and
+//! takes a path pin (`cua mesh status`, `--prefer`):
+//!
+//! ```text
+//! -> {"status": {}}
+//! <- {"prefer": "auto", "paths": [{"machine": "<id>", "name": "studio-mac", "kind": "lan", ...}]}
+//! -> {"prefer": {"prefer": "lan"}}
+//! <- {"prefer": "lan"}
+//! ```
+//!
//! A forward lives until the requester closes its control connection
//! (or the node stops).
@@ -280,10 +290,60 @@ pub enum ControlRequest {
#[serde(default, skip_serializing_if = "Option::is_none")]
bind: Option<std::net::IpAddr>,
},
+ /// How traffic reaches each other machine now.
+ Status {},
+ /// Pins the path to every machine ([`crate::engine::Prefer`]).
+ Prefer {
+ /// The pin.
+ prefer: crate::engine::Prefer,
+ },
+}
+
+/// How traffic reaches one other machine ([`ControlRequest::Status`]).
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct MachinePath {
+ /// Its machine id.
+ #[serde(default)]
+ pub machine: String,
+ /// Its name.
+ pub name: String,
+ /// Online at the coordinator.
+ pub online: bool,
+ /// `lan`, `direct` or `relay`.
+ pub kind: String,
+ /// `direct <addr>` or `relay`.
+ pub path: String,
+ /// The direct path's round trip.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub rtt_ms: Option<f64>,
+ /// Why this path, in plain words.
+ pub why: String,
+ /// Path switches so far.
+ #[serde(default)]
+ pub switches: u64,
+}
+
+/// The machines of `status`, as [`MachinePath`]s.
+pub fn machine_paths(status: &crate::node::Status) -> Vec<MachinePath> {
+ status
+ .peers
+ .iter()
+ .filter(|p| p.kind == proto::NodeKind::Machine)
+ .map(|p| MachinePath {
+ machine: p.machine_id.clone().unwrap_or_default(),
+ name: p.name.clone(),
+ online: p.online,
+ kind: p.path_kind.clone(),
+ path: p.path.clone(),
+ rtt_ms: p.rtt_ms,
+ why: p.why.clone(),
+ switches: p.path_switches,
+ })
+ .collect()
}
/// A control reply.
-#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct ControlReply {
/// Where to connect.
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -297,6 +357,12 @@ pub struct ControlReply {
/// Why there is no forward.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
+ /// The path pin (status and prefer requests).
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub prefer: Option<crate::engine::Prefer>,
+ /// How traffic reaches each other machine (status requests).
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub paths: Option<Vec<MachinePath>>,
}
/// `direct` or `relayed`.
@@ -383,6 +449,45 @@ impl ControlClient {
(None, None) => Err("mesh control: an empty answer".into()),
}
}
+
+ /// One request, one reply.
+ async fn ask(&mut self, req: &ControlRequest) -> Result<ControlReply, String> {
+ use tokio::io::{AsyncBufReadExt as _, AsyncWriteExt as _};
+ let mut line = serde_json::to_string(req).map_err(|e| e.to_string())?;
+ line.push('\n');
+ self.stream
+ .get_mut()
+ .write_all(line.as_bytes())
+ .await
+ .map_err(|e| format!("mesh control: {e}"))?;
+ let mut reply = String::new();
+ let n = tokio::time::timeout(Duration::from_secs(10), self.stream.read_line(&mut reply))
+ .await
+ .map_err(|_| "mesh control: no answer".to_string())?
+ .map_err(|e| format!("mesh control: {e}"))?;
+ if n == 0 {
+ return Err("mesh control: the mesh node closed the connection".into());
+ }
+ let reply: ControlReply =
+ serde_json::from_str(reply.trim()).map_err(|e| format!("mesh control: {e}"))?;
+ match reply.error {
+ Some(e) => Err(e),
+ None => Ok(reply),
+ }
+ }
+
+ /// How traffic reaches each other machine now, and the pin.
+ pub async fn status(&mut self) -> Result<(crate::engine::Prefer, Vec<MachinePath>), String> {
+ let r = self.ask(&ControlRequest::Status {}).await?;
+ Ok((r.prefer.unwrap_or_default(), r.paths.unwrap_or_default()))
+ }
+
+ /// Pins the path to every machine (`auto` lifts the pin).
+ pub async fn set_prefer(&mut self, prefer: crate::engine::Prefer) -> Result<(), String> {
+ self.ask(&ControlRequest::Prefer { prefer })
+ .await
+ .map(|_| ())
+ }
}
/// Serves the control socket at `path` for `mesh` until the returned task
@@ -462,6 +567,22 @@ async fn control_session(mesh: Mesh, stream: tokio::net::UnixStream) {
..Default::default()
},
},
+ Ok(ControlRequest::Status {}) => {
+ let status = mesh.status();
+ ControlReply {
+ prefer: Some(status.prefer),
+ paths: Some(machine_paths(&status)),
+ ..Default::default()
+ }
+ }
+ Ok(ControlRequest::Prefer { prefer }) => {
+ mesh.set_prefer(prefer);
+ tracing::info!(prefer = prefer.as_str(), "mesh path pin");
+ ControlReply {
+ prefer: Some(prefer),
+ ..Default::default()
+ }
+ }
Err(e) => ControlReply {
error: Some(format!("bad request: {e}")),
..Default::default()
@@ -519,6 +640,7 @@ pub async fn forward_on(
path: path.as_ref().map(|p| path_word(p).to_owned()),
peer: Some(peer.name.clone()),
error: None,
+ ..ControlReply::default()
};
Ok((fwd, reply))
}
diff --git a/libs/cua/crates/cua-mesh/src/lib.rs b/libs/cua/crates/cua-mesh/src/lib.rs
index dad6c8a722..aee5ad2d82 100644
--- a/libs/cua/crates/cua-mesh/src/lib.rs
+++ b/libs/cua/crates/cua-mesh/src/lib.rs
@@ -50,7 +50,7 @@ pub mod vnet;
pub mod testing;
pub use coord::{AuthFuture, AuthProvider, ControlState, PairVerifier};
-pub use engine::Path;
+pub use engine::{Path, Prefer};
pub use key::{NodeIdentity, PublicKey, SecretKey};
pub use node::{Forward, IncomingStream, MEDIA_UDP_PORT, Mesh, MeshConfig, PeerStatus, Status};
pub use policy::Policy;
diff --git a/libs/cua/crates/cua-mesh/src/node.rs b/libs/cua/crates/cua-mesh/src/node.rs
index 5f50fe507c..23b96ceaf7 100644
--- a/libs/cua/crates/cua-mesh/src/node.rs
+++ b/libs/cua/crates/cua-mesh/src/node.rs
@@ -234,6 +234,48 @@ pub struct PeerStatus {
pub rx_relay: u64,
/// Path switches so far.
pub path_switches: u64,
+ /// `lan`, `direct` or `relay`.
+ pub path_kind: String,
+ /// Why this path, in plain words ("Fastest path: the local network,
+ /// 2 ms").
+ pub why: String,
+}
+
+/// Why traffic to a peer takes `path`, in plain words, and its kind.
+pub fn path_why(
+ path: &crate::engine::Path,
+ rtt_ms: Option<f64>,
+ prefer: crate::engine::Prefer,
+ relay_only: bool,
+) -> (String, String) {
+ use crate::engine::{Path, Prefer, is_lan};
+ let ms = rtt_ms.map(|m| format!(", {m:.0} ms")).unwrap_or_default();
+ match path {
+ Path::Relay if prefer == Prefer::Relay || relay_only => (
+ "relay".into(),
+ "Pinned to the relay (end-to-end encrypted)".into(),
+ ),
+ Path::Relay => (
+ "relay".into(),
+ "No direct path answered yet; the relay carries it (end-to-end encrypted)".into(),
+ ),
+ Path::Direct(a) if is_lan(a) => (
+ "lan".into(),
+ if prefer == Prefer::Lan {
+ format!("Pinned to the local network{ms}")
+ } else {
+ format!("Fastest path: the local network{ms}")
+ },
+ ),
+ Path::Direct(_) => (
+ "direct".into(),
+ if prefer == Prefer::Lan {
+ format!("No local network path answered; direct across the internet{ms}")
+ } else {
+ format!("Fastest path: direct across the internet{ms}")
+ },
+ ),
+ }
}
/// The node's status.
@@ -253,6 +295,8 @@ pub struct Status {
pub peers: Vec<PeerStatus>,
/// Last coordinator error.
pub last_error: Option<String>,
+ /// The path pin (`auto` unless set).
+ pub prefer: crate::engine::Prefer,
}
struct Inner {
@@ -304,6 +348,10 @@ impl Mesh {
let engine = Engine::new(identity.clone(), cfg.listen_port, cfg.relay_only)
.await
.map_err(|e| Error::Io(format!("udp: {e}")))?;
+ let prefer = crate::engine::Prefer::from_env();
+ if prefer != crate::engine::Prefer::Auto {
+ engine.set_prefer(prefer);
+ }
let control = Control::new();
let approved = match cfg.kind {
NodeKind::Machine => Some(Arc::new(Approved::load(cfg.approved_path.clone()))),
@@ -526,6 +574,8 @@ impl Mesh {
/// Status for displays and measurements.
pub fn status(&self) -> Status {
let state = self.inner.control.state.borrow().clone();
+ let prefer = self.inner.engine.prefer();
+ let relay_only = self.inner.engine.relay_only();
let peers = self
.inner
.engine
@@ -534,20 +584,25 @@ impl Mesh {
.map(|p| {
let info = p.info.read().expect("info").clone();
use std::sync::atomic::Ordering::Relaxed;
+ let path = p.path();
+ let rtt_ms = p.rtt().map(|d| d.as_secs_f64() * 1000.0);
+ let (path_kind, why) = path_why(&path, rtt_ms, prefer, relay_only);
PeerStatus {
node_key: info.node_key,
name: info.name.clone(),
machine_id: info.machine_id.clone(),
kind: info.kind,
online: info.online,
- path: p.path().to_string(),
- rtt_ms: p.rtt().map(|d| d.as_secs_f64() * 1000.0),
+ path: path.to_string(),
+ rtt_ms,
handshake_age_s: p.handshake_age().map(|d| d.as_secs_f64()),
tx_direct: p.counters.tx_direct.load(Relaxed),
tx_relay: p.counters.tx_relay.load(Relaxed),
rx_direct: p.counters.rx_direct.load(Relaxed),
rx_relay: p.counters.rx_relay.load(Relaxed),
path_switches: p.path_switches(),
+ path_kind,
+ why,
}
})
.collect();
@@ -559,6 +614,7 @@ impl Mesh {
endpoints: self.inner.control_endpoints(),
peers,
last_error: state.last_error,
+ prefer,
}
}
@@ -863,6 +919,17 @@ impl Mesh {
self.inner.engine.set_relay_only(on);
}
+ /// Pins the path to every peer ([`crate::engine::Prefer`]); `auto`
+ /// lifts the pin.
+ pub fn set_prefer(&self, prefer: crate::engine::Prefer) {
+ self.inner.engine.set_prefer(prefer);
+ }
+
+ /// The path pin.
+ pub fn prefer(&self) -> crate::engine::Prefer {
+ self.inner.engine.prefer()
+ }
+
/// Stops the node.
pub fn shutdown(&self) {
self.inner.shutdown.cancel();
diff --git a/libs/cua/crates/cua-mesh/tests/mesh.rs b/libs/cua/crates/cua-mesh/tests/mesh.rs
index 7bdecbab2a..34d87f8432 100644
--- a/libs/cua/crates/cua-mesh/tests/mesh.rs
+++ b/libs/cua/crates/cua-mesh/tests/mesh.rs
@@ -738,6 +738,36 @@ async fn the_control_socket_makes_forwards_that_live_with_the_connection() {
server.abort();
}
+/// The control socket reports how traffic reaches each other machine and
+/// takes a path pin; lifting the pin goes back to automatic.
+#[cfg(unix)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn the_control_socket_reports_paths_and_takes_a_pin() {
+ let dir = tempfile::tempdir().unwrap();
+ let coord = FakeCoordinator::start(Mode::Account).await;
+ coord.set_machine_peering(true);
+ let a = node(&coord, NodeKind::Machine, dir.path(), "a", false).await;
+ let _b = node(&coord, NodeKind::Machine, dir.path(), "b", false).await;
+ let sock = dir.path().join("host").join(cua_mesh::gang::CONTROL_SOCKET);
+ let server = cua_mesh::gang::serve_control(a.clone(), &sock).unwrap();
+ let mut ctl = cua_mesh::gang::ControlClient::connect(&sock).await.unwrap();
+
+ let (prefer, _paths) = ctl.status().await.unwrap();
+ assert_eq!(prefer, cua_mesh::Prefer::Auto);
+
+ ctl.set_prefer(cua_mesh::Prefer::Relay).await.unwrap();
+ assert_eq!(a.prefer(), cua_mesh::Prefer::Relay);
+ let (prefer, paths) = ctl.status().await.unwrap();
+ assert_eq!(prefer, cua_mesh::Prefer::Relay);
+ for p in paths {
+ assert_ne!(p.kind, "lan", "a relay pin keeps traffic off direct paths");
+ }
+
+ ctl.set_prefer(cua_mesh::Prefer::Auto).await.unwrap();
+ assert_eq!(a.prefer(), cua_mesh::Prefer::Auto);
+ server.abort();
+}
+
/// Bulk rank traffic (CUA-1783): 256 MiB one way through a machine's
/// loopback forward, spliced on the other machine to a local TCP sink.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]