Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(spaces): stop the pool re-adding a direct host under a new name every 30 s (#6206)
gracemsp-elandiscd395c30df3c
diff --git a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rsindex 5d9a88925b..66e8b8d310 100644--- a/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs+++ b/libs/cua/crates/cua-spaces-cluster/src/coordinator/mod.rs@@ -2519,6 +2519,12 @@ impl Coordinator {self.sched.lock().expect("sched").nodes.contains_key(name)}+ /// Whether a job is running on host `name` now.+ pub fn node_busy(&self, name: &str) -> bool {+ let g = self.sched.lock().expect("sched");+ g.inflight.values().any(|f| f.node == name)+ }+/// Jobs waiting in memory and in flight.pub fn depth(&self) -> (usize, usize) {let g = self.sched.lock().expect("sched");diff --git a/libs/cua/crates/cua-spaces-cluster/src/pool/mod.rs b/libs/cua/crates/cua-spaces-cluster/src/pool/mod.rsindex cc900536ac..88083c6a57 100644--- a/libs/cua/crates/cua-spaces-cluster/src/pool/mod.rs+++ b/libs/cua/crates/cua-spaces-cluster/src/pool/mod.rs@@ -1129,23 +1129,42 @@ fn relay_key(url: &str) -> String {/// Brings a coordinator's machines in line with `machines`: new ones are/// added (named after themselves, uniquely; a machine keeps the name it-/// had, matched by its relay id), changed ones updated, and ones no longer-/// listed drained (their jobs finish; nothing new goes there) and, still-/// unlisted the next time, removed (relay machines). `own`: also-/// keeps this machine's runner as listed.+/// had, matched by its relay id, else its direct address, else its+/// name as stored), changed ones updated, and ones no longer listed+/// drained (their jobs finish; nothing new goes there) and, still+/// unlisted the next time, removed (relay and direct machines). `own`:+/// also keeps this machine's runner as listed.async fn apply_machines(coord: &Coordinator, links: &Links, machines: Vec<PoolMachine>, own: bool) {let existing = coord.node_rows();let mut seen = Vec::new();for m in machines {let mut row = m.row;- let prior = existing.iter().find(|r| {- (!row.relay_machine.is_empty() && r.relay_machine == row.relay_machine)- || (row.relay_machine.is_empty() && r.name == row.name)- });+ // Rows are stored under the sanitized name, so a listed name is+ // compared as it would be stored ("ExampleHost" is "examplehost").+ let wanted = crate::coordinator::sanitize_name(&row.name);+ let same = |r: &NodeRow| {+ if !row.relay_machine.is_empty() {+ r.relay_machine == row.relay_machine+ } else if row.via == "direct" {+ // By its address, else by name: the same host at a new+ // address keeps its row (and its name) rather than being+ // added as a second one.+ r.via == "direct"+ && ((!row.direct.is_empty() && r.direct == row.direct) || r.name == wanted)+ } else {+ r.relay_machine.is_empty() && r.name == wanted+ }+ };+ // Several rows for one machine (left by an older agent): keep the+ // one under its own name, else a ready one, else the newest; the+ // others are unlisted, so they drain and then leave.+ let prior = existing+ .iter()+ .filter(|r| same(r) && !seen.contains(&r.name))+ .min_by_key(|r| (r.name != wanted, r.state != "ready", -r.added_ms));row.name = match prior {Some(p) => p.name.clone(),None => {- let wanted = crate::coordinator::sanitize_name(&row.name);let mut name = wanted.clone();let mut k = 2;while existing.iter().any(|r| r.name == name) || seen.contains(&name) {@@ -1191,11 +1210,20 @@ async fn apply_machines(coord: &Coordinator, links: &Links, machines: Vec<PoolMa}if r.state == "ready" {let _ = coord.drain(&r.name, true).await;- } else if r.via == "relay" && r.state == "draining" {- // Gone from the account at two listings in a row (removed,- // or no longer providing Spaces): it leaves the pool, so its- // name is free again and `cua machines` stops showing it.- let _ = coord.remove_node(&r.name).await;+ } else if (r.via == "relay" || r.via == "direct") && r.state == "draining" {+ // Gone from the account (or the direct hosts) at two listings+ // in a row (removed, or no longer providing Spaces): it leaves+ // the pool, so its name is free again, the coordinator stops+ // polling it and `cua machines` stops showing it.+ // A direct host's jobs are not given up on a listing that may+ // be short (the file read mid-edit): it stays, draining, until+ // none runs there.+ if r.via == "direct" && coord.node_busy(&r.name) {+ continue;+ }+ if coord.remove_node(&r.name).await.is_ok() && r.via == "direct" {+ links.lock().expect("links").remove(&r.name);+ }}}}@@ -1381,6 +1409,273 @@ mod tests {assert_eq!(coord.node_rows()[0].state, "ready");}+ fn direct(dir: &Path, name: &str, addr: &str) -> PoolMachine {+ let mut m = machine(dir, name, addr);+ m.row.relay_machine.clear();+ m.row.direct = addr.into();+ m.row.via = "direct".into();+ m+ }++ /// A direct host whose name is not stored as listed ("ExampleHost" is+ /// stored as "examplehost") keeps its one row across listings, rather+ /// than being added again under a new name each time while the old+ /// rows drain and are polled forever.+ #[tokio::test]+ async fn a_direct_host_keeps_its_one_row_across_listings() {+ let dir = tempfile::tempdir().unwrap();+ let links: Links = Arc::default();+ let coord = Coordinator::start(+ crate::coordinator::Config::new(dir.path().join("c")),+ Some(factory(&links, None)),+ )+ .await+ .unwrap();+ for _ in 0..5 {+ apply_machines(+ &coord,+ &links,+ vec![+ direct(dir.path(), "ExampleHost", "192.0.2.10:3211"),+ direct(dir.path(), "Example Worker", "192.0.2.20:3211"),+ ],+ true,+ )+ .await;+ }+ let mut rows: Vec<_> = coord+ .node_rows()+ .into_iter()+ .map(|r| (r.name, r.direct, r.state))+ .collect();+ rows.sort();+ assert_eq!(+ rows,+ vec![+ (+ "example-worker".into(),+ "192.0.2.20:3211".into(),+ "ready".into()+ ),+ (+ "examplehost".into(),+ "192.0.2.10:3211".into(),+ "ready".into()+ ),+ ]+ );+ }++ /// The rows an older agent left for one direct host (one per listing,+ /// all but the newest draining) shrink back to one under the host's+ /// own name; a direct host no longer listed drains, then leaves.+ #[tokio::test]+ async fn duplicate_direct_rows_collapse_and_an_unlisted_direct_host_leaves() {+ let dir = tempfile::tempdir().unwrap();+ let links: Links = Arc::default();+ let coord = Coordinator::start(+ crate::coordinator::Config::new(dir.path().join("c")),+ Some(factory(&links, None)),+ )+ .await+ .unwrap();+ let host = direct(dir.path(), "ExampleHost", "192.0.2.10:3211");+ for (i, name) in ["examplehost", "examplehost-2", "examplehost-3"]+ .iter()+ .enumerate()+ {+ let mut row = host.row.clone();+ row.name = name.to_string();+ row.state = if i == 2 { "ready" } else { "draining" }.into();+ row.added_ms = 1000 + i as i64;+ links+ .lock()+ .unwrap()+ .insert(row.name.clone(), host.link.clone());+ coord.add_node(row).await.unwrap();+ }+ let listed = || vec![direct(dir.path(), "ExampleHost", "192.0.2.10:3211")];+ apply_machines(&coord, &links, listed(), true).await;+ apply_machines(&coord, &links, listed(), true).await;+ let rows = coord.node_rows();+ assert_eq!(rows.len(), 1, "{rows:?}");+ assert_eq!(+ (rows[0].name.as_str(), rows[0].state.as_str()),+ ("examplehost", "ready")+ );+ apply_machines(&coord, &links, vec![], true).await;+ assert_eq!(coord.node_rows()[0].state, "draining");+ apply_machines(&coord, &links, vec![], true).await;+ assert!(coord.node_rows().is_empty(), "{:?}", coord.node_rows());+ }++ async fn pool(dir: &Path) -> (Arc<Coordinator>, Links) {+ let links: Links = Arc::default();+ let coord = Coordinator::start(+ crate::coordinator::Config::new(dir.join("c")),+ Some(factory(&links, None)),+ )+ .await+ .unwrap();+ (coord, links)+ }++ /// The same direct host as `base`, listed under another spelling.+ fn direct_like(base: &PoolMachine, name: &str) -> PoolMachine {+ let mut m = base.clone();+ m.row.name = name.into();+ m+ }++ fn names(coord: &Coordinator) -> Vec<String> {+ let mut n: Vec<String> = coord.node_rows().into_iter().map(|r| r.name).collect();+ n.sort();+ n+ }++ #[tokio::test]+ async fn a_direct_host_with_capitals_or_spaces_keeps_one_row_over_many_passes() {+ let dir = tempfile::tempdir().unwrap();+ let (coord, links) = pool(dir.path()).await;+ for _ in 0..12 {+ apply_machines(+ &coord,+ &links,+ vec![+ direct(dir.path(), "ExampleHost", "100.1.1.1:7000"),+ direct(dir.path(), "My Mac Pro", "100.1.1.2:7000"),+ ],+ true,+ )+ .await;+ }+ assert_eq!(names(&coord), ["examplehost", "my-mac-pro"]);+ assert!(coord.node_rows().iter().all(|r| r.state == "ready"));+ }++ #[tokio::test]+ async fn a_direct_host_is_matched_by_its_address_when_its_name_changes() {+ let dir = tempfile::tempdir().unwrap();+ let (coord, links) = pool(dir.path()).await;+ apply_machines(+ &coord,+ &links,+ vec![direct(dir.path(), "old", "100.1.1.1:7000")],+ true,+ )+ .await;+ apply_machines(+ &coord,+ &links,+ vec![direct(dir.path(), "New Name", "100.1.1.1:7000")],+ true,+ )+ .await;+ assert_eq!(names(&coord), ["old"]);+ // The same name at a new address is the same host.+ apply_machines(+ &coord,+ &links,+ vec![direct(dir.path(), "old", "100.9.9.9:7000")],+ true,+ )+ .await;+ let rows = coord.node_rows();+ assert_eq!(rows.len(), 1);+ assert_eq!(rows[0].direct, "100.9.9.9:7000");+ }++ #[tokio::test]+ async fn rows_left_by_the_rename_bug_heal_to_the_one_canonical_row() {+ let dir = tempfile::tempdir().unwrap();+ let (coord, links) = pool(dir.path()).await;+ // What an older build left: the original row draining, a copy per pass.+ let base = direct(dir.path(), "examplehost", "100.1.1.1:7000");+ for i in 0..2000u32 {+ let mut row = base.row.clone();+ row.name = if i == 0 {+ "examplehost".into()+ } else {+ format!("examplehost-{}", i + 1)+ };+ row.state = if i == 1999 { "ready" } else { "draining" }.into();+ row.added_ms = 1000 + i64::from(i);+ links+ .lock()+ .unwrap()+ .insert(row.name.clone(), base.link.clone());+ coord.add_node(row).await.unwrap();+ }+ let other = direct(dir.path(), "other", "100.2.2.2:7000");+ let mut gone = other.row.clone();+ gone.name = "gone".into();+ gone.direct = "100.3.3.3:7000".into();+ gone.state = "draining".into();+ links+ .lock()+ .unwrap()+ .insert(gone.name.clone(), base.link.clone());+ coord.add_node(gone).await.unwrap();+ let listed = || vec![direct_like(&base, "ExampleHost"), other.clone()];+ // One pass drops every draining copy and the unlisted host; the+ // ready copy drains, and leaves at the next.+ apply_machines(&coord, &links, listed(), true).await;+ assert!(coord.node_rows().len() <= 3, "{}", coord.node_rows().len());+ apply_machines(&coord, &links, listed(), true).await;+ assert_eq!(names(&coord), ["examplehost", "other"]);+ let kept = coord+ .node_rows()+ .into_iter()+ .find(|r| r.name == "examplehost")+ .unwrap();+ assert_eq!((kept.state.as_str(), kept.added_ms), ("ready", 1000));+ apply_machines(&coord, &links, listed(), true).await;+ assert_eq!(names(&coord), ["examplehost", "other"]);+ }++ #[tokio::test]+ async fn a_direct_host_gone_from_the_list_drains_then_leaves() {+ let dir = tempfile::tempdir().unwrap();+ let (coord, links) = pool(dir.path()).await;+ apply_machines(+ &coord,+ &links,+ vec![direct(dir.path(), "a", "100.1.1.1:7000")],+ true,+ )+ .await;+ apply_machines(&coord, &links, vec![], true).await;+ assert_eq!(coord.node_rows()[0].state, "draining");+ apply_machines(&coord, &links, vec![], true).await;+ assert!(coord.node_rows().is_empty());+ assert!(links.lock().unwrap().is_empty());+ }++ #[tokio::test]+ async fn relay_rows_are_not_touched_by_direct_matching() {+ let dir = tempfile::tempdir().unwrap();+ let (coord, links) = pool(dir.path()).await;+ for _ in 0..5 {+ apply_machines(+ &coord,+ &links,+ vec![+ machine(dir.path(), "AWS 1", "m1"),+ direct(dir.path(), "AWS 1", "100.1.1.1:7000"),+ ],+ false,+ )+ .await;+ }+ assert_eq!(names(&coord), ["aws-1", "aws-1-2"]);+ let vias: Vec<_> = coord+ .node_rows()+ .into_iter()+ .map(|r| (r.name, r.via))+ .collect();+ assert!(vias.contains(&("aws-1".into(), "relay".into())));+ }+#[test]fn lease_levels_read_as_the_same_role_to_older_agents() {// Older agents compare against PRIORITY_PROVIDER only.