Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(spaces): stop the pool re-adding a direct host under a new name every 30 s (#6206)

gracemsp-elandiscd395c30df3c
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
index 5d9a88925b..66e8b8d310 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs
@@ -2519,6 +2519,12 @@ impl Coordinator {
self.sched.lock().expect("sched").nodes.contains_key(name)
}
+ /// Whether a job is running on host `name` now.
+ pub fn node_busy(&self, name: &str) -> bool {
+ let g = self.sched.lock().expect("sched");
+ g.inflight.values().any(|f| f.node == name)
+ }
+
/// Jobs waiting in memory and in flight.
pub fn depth(&self) -> (usize, usize) {
let g = self.sched.lock().expect("sched");
diff --git a/libs/cua/crates/cua-spaces-cluster/src/pool/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/pool/mod.rs
index cc900536ac..88083c6a57 100644
--- a/libs/cua/crates/cua-spaces-cluster/src/pool/mod.rs
+++ b/libs/cua/crates/cua-spaces-cluster/src/pool/mod.rs
@@ -1129,23 +1129,42 @@ fn relay_key(url: &str) -> String {
/// Brings a coordinator's machines in line with `machines`: new ones are
/// added (named after themselves, uniquely; a machine keeps the name it
-/// had, matched by its relay id), changed ones updated, and ones no longer
-/// listed drained (their jobs finish; nothing new goes there) and, still
-/// unlisted the next time, removed (relay machines). `own`: also
-/// keeps this machine's runner as listed.
+/// had, matched by its relay id, else its direct address, else its
+/// name as stored), changed ones updated, and ones no longer listed
+/// drained (their jobs finish; nothing new goes there) and, still
+/// unlisted the next time, removed (relay and direct machines). `own`:
+/// also keeps this machine's runner as listed.
async fn apply_machines(coord: &Coordinator, links: &Links, machines: Vec<PoolMachine>, own: bool) {
let existing = coord.node_rows();
let mut seen = Vec::new();
for m in machines {
let mut row = m.row;
- let prior = existing.iter().find(|r| {
- (!row.relay_machine.is_empty() && r.relay_machine == row.relay_machine)
- || (row.relay_machine.is_empty() && r.name == row.name)
- });
+ // Rows are stored under the sanitized name, so a listed name is
+ // compared as it would be stored ("ExampleHost" is "examplehost").
+ let wanted = crate::coordinator::sanitize_name(&row.name);
+ let same = |r: &NodeRow| {
+ if !row.relay_machine.is_empty() {
+ r.relay_machine == row.relay_machine
+ } else if row.via == "direct" {
+ // By its address, else by name: the same host at a new
+ // address keeps its row (and its name) rather than being
+ // added as a second one.
+ r.via == "direct"
+ && ((!row.direct.is_empty() && r.direct == row.direct) || r.name == wanted)
+ } else {
+ r.relay_machine.is_empty() && r.name == wanted
+ }
+ };
+ // Several rows for one machine (left by an older agent): keep the
+ // one under its own name, else a ready one, else the newest; the
+ // others are unlisted, so they drain and then leave.
+ let prior = existing
+ .iter()
+ .filter(|r| same(r) && !seen.contains(&r.name))
+ .min_by_key(|r| (r.name != wanted, r.state != "ready", -r.added_ms));
row.name = match prior {
Some(p) => p.name.clone(),
None => {
- let wanted = crate::coordinator::sanitize_name(&row.name);
let mut name = wanted.clone();
let mut k = 2;
while existing.iter().any(|r| r.name == name) || seen.contains(&name) {
@@ -1191,11 +1210,20 @@ async fn apply_machines(coord: &Coordinator, links: &Links, machines: Vec<PoolMa
}
if r.state == "ready" {
let _ = coord.drain(&r.name, true).await;
- } else if r.via == "relay" && r.state == "draining" {
- // Gone from the account at two listings in a row (removed,
- // or no longer providing Spaces): it leaves the pool, so its
- // name is free again and `cua machines` stops showing it.
- let _ = coord.remove_node(&r.name).await;
+ } else if (r.via == "relay" || r.via == "direct") && r.state == "draining" {
+ // Gone from the account (or the direct hosts) at two listings
+ // in a row (removed, or no longer providing Spaces): it leaves
+ // the pool, so its name is free again, the coordinator stops
+ // polling it and `cua machines` stops showing it.
+ // A direct host's jobs are not given up on a listing that may
+ // be short (the file read mid-edit): it stays, draining, until
+ // none runs there.
+ if r.via == "direct" && coord.node_busy(&r.name) {
+ continue;
+ }
+ if coord.remove_node(&r.name).await.is_ok() && r.via == "direct" {
+ links.lock().expect("links").remove(&r.name);
+ }
}
}
}
@@ -1381,6 +1409,273 @@ mod tests {
assert_eq!(coord.node_rows()[0].state, "ready");
}
+ fn direct(dir: &Path, name: &str, addr: &str) -> PoolMachine {
+ let mut m = machine(dir, name, addr);
+ m.row.relay_machine.clear();
+ m.row.direct = addr.into();
+ m.row.via = "direct".into();
+ m
+ }
+
+ /// A direct host whose name is not stored as listed ("ExampleHost" is
+ /// stored as "examplehost") keeps its one row across listings, rather
+ /// than being added again under a new name each time while the old
+ /// rows drain and are polled forever.
+ #[tokio::test]
+ async fn a_direct_host_keeps_its_one_row_across_listings() {
+ let dir = tempfile::tempdir().unwrap();
+ let links: Links = Arc::default();
+ let coord = Coordinator::start(
+ crate::coordinator::Config::new(dir.path().join("c")),
+ Some(factory(&links, None)),
+ )
+ .await
+ .unwrap();
+ for _ in 0..5 {
+ apply_machines(
+ &coord,
+ &links,
+ vec![
+ direct(dir.path(), "ExampleHost", "192.0.2.10:3211"),
+ direct(dir.path(), "Example Worker", "192.0.2.20:3211"),
+ ],
+ true,
+ )
+ .await;
+ }
+ let mut rows: Vec<_> = coord
+ .node_rows()
+ .into_iter()
+ .map(|r| (r.name, r.direct, r.state))
+ .collect();
+ rows.sort();
+ assert_eq!(
+ rows,
+ vec![
+ (
+ "example-worker".into(),
+ "192.0.2.20:3211".into(),
+ "ready".into()
+ ),
+ (
+ "examplehost".into(),
+ "192.0.2.10:3211".into(),
+ "ready".into()
+ ),
+ ]
+ );
+ }
+
+ /// The rows an older agent left for one direct host (one per listing,
+ /// all but the newest draining) shrink back to one under the host's
+ /// own name; a direct host no longer listed drains, then leaves.
+ #[tokio::test]
+ async fn duplicate_direct_rows_collapse_and_an_unlisted_direct_host_leaves() {
+ let dir = tempfile::tempdir().unwrap();
+ let links: Links = Arc::default();
+ let coord = Coordinator::start(
+ crate::coordinator::Config::new(dir.path().join("c")),
+ Some(factory(&links, None)),
+ )
+ .await
+ .unwrap();
+ let host = direct(dir.path(), "ExampleHost", "192.0.2.10:3211");
+ for (i, name) in ["examplehost", "examplehost-2", "examplehost-3"]
+ .iter()
+ .enumerate()
+ {
+ let mut row = host.row.clone();
+ row.name = name.to_string();
+ row.state = if i == 2 { "ready" } else { "draining" }.into();
+ row.added_ms = 1000 + i as i64;
+ links
+ .lock()
+ .unwrap()
+ .insert(row.name.clone(), host.link.clone());
+ coord.add_node(row).await.unwrap();
+ }
+ let listed = || vec![direct(dir.path(), "ExampleHost", "192.0.2.10:3211")];
+ apply_machines(&coord, &links, listed(), true).await;
+ apply_machines(&coord, &links, listed(), true).await;
+ let rows = coord.node_rows();
+ assert_eq!(rows.len(), 1, "{rows:?}");
+ assert_eq!(
+ (rows[0].name.as_str(), rows[0].state.as_str()),
+ ("examplehost", "ready")
+ );
+ apply_machines(&coord, &links, vec![], true).await;
+ assert_eq!(coord.node_rows()[0].state, "draining");
+ apply_machines(&coord, &links, vec![], true).await;
+ assert!(coord.node_rows().is_empty(), "{:?}", coord.node_rows());
+ }
+
+ async fn pool(dir: &Path) -> (Arc<Coordinator>, Links) {
+ let links: Links = Arc::default();
+ let coord = Coordinator::start(
+ crate::coordinator::Config::new(dir.join("c")),
+ Some(factory(&links, None)),
+ )
+ .await
+ .unwrap();
+ (coord, links)
+ }
+
+ /// The same direct host as `base`, listed under another spelling.
+ fn direct_like(base: &PoolMachine, name: &str) -> PoolMachine {
+ let mut m = base.clone();
+ m.row.name = name.into();
+ m
+ }
+
+ fn names(coord: &Coordinator) -> Vec<String> {
+ let mut n: Vec<String> = coord.node_rows().into_iter().map(|r| r.name).collect();
+ n.sort();
+ n
+ }
+
+ #[tokio::test]
+ async fn a_direct_host_with_capitals_or_spaces_keeps_one_row_over_many_passes() {
+ let dir = tempfile::tempdir().unwrap();
+ let (coord, links) = pool(dir.path()).await;
+ for _ in 0..12 {
+ apply_machines(
+ &coord,
+ &links,
+ vec![
+ direct(dir.path(), "ExampleHost", "100.1.1.1:7000"),
+ direct(dir.path(), "My Mac Pro", "100.1.1.2:7000"),
+ ],
+ true,
+ )
+ .await;
+ }
+ assert_eq!(names(&coord), ["examplehost", "my-mac-pro"]);
+ assert!(coord.node_rows().iter().all(|r| r.state == "ready"));
+ }
+
+ #[tokio::test]
+ async fn a_direct_host_is_matched_by_its_address_when_its_name_changes() {
+ let dir = tempfile::tempdir().unwrap();
+ let (coord, links) = pool(dir.path()).await;
+ apply_machines(
+ &coord,
+ &links,
+ vec![direct(dir.path(), "old", "100.1.1.1:7000")],
+ true,
+ )
+ .await;
+ apply_machines(
+ &coord,
+ &links,
+ vec![direct(dir.path(), "New Name", "100.1.1.1:7000")],
+ true,
+ )
+ .await;
+ assert_eq!(names(&coord), ["old"]);
+ // The same name at a new address is the same host.
+ apply_machines(
+ &coord,
+ &links,
+ vec![direct(dir.path(), "old", "100.9.9.9:7000")],
+ true,
+ )
+ .await;
+ let rows = coord.node_rows();
+ assert_eq!(rows.len(), 1);
+ assert_eq!(rows[0].direct, "100.9.9.9:7000");
+ }
+
+ #[tokio::test]
+ async fn rows_left_by_the_rename_bug_heal_to_the_one_canonical_row() {
+ let dir = tempfile::tempdir().unwrap();
+ let (coord, links) = pool(dir.path()).await;
+ // What an older build left: the original row draining, a copy per pass.
+ let base = direct(dir.path(), "examplehost", "100.1.1.1:7000");
+ for i in 0..2000u32 {
+ let mut row = base.row.clone();
+ row.name = if i == 0 {
+ "examplehost".into()
+ } else {
+ format!("examplehost-{}", i + 1)
+ };
+ row.state = if i == 1999 { "ready" } else { "draining" }.into();
+ row.added_ms = 1000 + i64::from(i);
+ links
+ .lock()
+ .unwrap()
+ .insert(row.name.clone(), base.link.clone());
+ coord.add_node(row).await.unwrap();
+ }
+ let other = direct(dir.path(), "other", "100.2.2.2:7000");
+ let mut gone = other.row.clone();
+ gone.name = "gone".into();
+ gone.direct = "100.3.3.3:7000".into();
+ gone.state = "draining".into();
+ links
+ .lock()
+ .unwrap()
+ .insert(gone.name.clone(), base.link.clone());
+ coord.add_node(gone).await.unwrap();
+ let listed = || vec![direct_like(&base, "ExampleHost"), other.clone()];
+ // One pass drops every draining copy and the unlisted host; the
+ // ready copy drains, and leaves at the next.
+ apply_machines(&coord, &links, listed(), true).await;
+ assert!(coord.node_rows().len() <= 3, "{}", coord.node_rows().len());
+ apply_machines(&coord, &links, listed(), true).await;
+ assert_eq!(names(&coord), ["examplehost", "other"]);
+ let kept = coord
+ .node_rows()
+ .into_iter()
+ .find(|r| r.name == "examplehost")
+ .unwrap();
+ assert_eq!((kept.state.as_str(), kept.added_ms), ("ready", 1000));
+ apply_machines(&coord, &links, listed(), true).await;
+ assert_eq!(names(&coord), ["examplehost", "other"]);
+ }
+
+ #[tokio::test]
+ async fn a_direct_host_gone_from_the_list_drains_then_leaves() {
+ let dir = tempfile::tempdir().unwrap();
+ let (coord, links) = pool(dir.path()).await;
+ apply_machines(
+ &coord,
+ &links,
+ vec![direct(dir.path(), "a", "100.1.1.1:7000")],
+ true,
+ )
+ .await;
+ apply_machines(&coord, &links, vec![], true).await;
+ assert_eq!(coord.node_rows()[0].state, "draining");
+ apply_machines(&coord, &links, vec![], true).await;
+ assert!(coord.node_rows().is_empty());
+ assert!(links.lock().unwrap().is_empty());
+ }
+
+ #[tokio::test]
+ async fn relay_rows_are_not_touched_by_direct_matching() {
+ let dir = tempfile::tempdir().unwrap();
+ let (coord, links) = pool(dir.path()).await;
+ for _ in 0..5 {
+ apply_machines(
+ &coord,
+ &links,
+ vec![
+ machine(dir.path(), "AWS 1", "m1"),
+ direct(dir.path(), "AWS 1", "100.1.1.1:7000"),
+ ],
+ false,
+ )
+ .await;
+ }
+ assert_eq!(names(&coord), ["aws-1", "aws-1-2"]);
+ let vias: Vec<_> = coord
+ .node_rows()
+ .into_iter()
+ .map(|r| (r.name, r.via))
+ .collect();
+ assert!(vias.contains(&("aws-1".into(), "relay".into())));
+ }
+
#[test]
fn lease_levels_read_as_the_same_role_to_older_agents() {
// Older agents compare against PRIORITY_PROVIDER only.