Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(spaces): a new message and an agent's tool calls no longer wait on a slow Space (#6619)

ddupontb2e8df864277
diff --git a/libs/cua/crates/cua-spaces-ext/src/persistent/members.rs b/libs/cua/crates/cua-spaces-ext/src/persistent/members.rs
index 142a975e33..ae7b1f9f0a 100644
--- a/libs/cua/crates/cua-spaces-ext/src/persistent/members.rs
+++ b/libs/cua/crates/cua-spaces-ext/src/persistent/members.rs
@@ -743,6 +743,9 @@ impl Persistent {
if self.detach_dispatch {
for rec in members {
let Some(guard) = super::phases::ServingGuard::take(&self.dir, &rec.name) else {
+ if store_has_queued(&self.thread_store(&rec.space), &rec.name) {
+ tracing::info!(agent = %rec.name, "a queued task waits: the member is still being served");
+ }
continue; // still being served by an earlier pass
};
let me = self.clone();
@@ -2820,3 +2823,15 @@ fn clip_words(s: &str, n: usize) -> String {
out.push('\u{2026}');
out
}
+
+/// Whether `name` has a task queued in `store` (for the log of a wait).
+fn store_has_queued(store: &ThreadStore, name: &str) -> bool {
+ store
+ .fold()
+ .map(|f| {
+ f.open_tasks(Some(name))
+ .iter()
+ .any(|t| t.state == TaskState::Queued)
+ })
+ .unwrap_or(false)
+}
diff --git a/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs b/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs
index b976394a73..4839d603b1 100644
--- a/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs
+++ b/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs
@@ -517,6 +517,9 @@ pub struct Persistent {
/// The supervisor serves each member's queued tasks on a task of its
/// own instead of inside its pass (see [`Persistent::detached`]).
detach_dispatch: bool,
+ /// Errors of the per-agent passes that ran on their own tasks (detached),
+ /// handed to the next pass's report.
+ detached_errors: Arc<std::sync::Mutex<Vec<String>>>,
}
/// What each working session has said so far this turn, by `<space>/<name>#<session>`; shared by every
@@ -568,6 +571,7 @@ impl Persistent {
owner,
admins,
detach_dispatch: false,
+ detached_errors: Arc::default(),
}
}
@@ -1789,27 +1793,21 @@ impl Persistent {
// A Space someone stopped or suspended is left alone: its durable
// members wait for it to run again instead of waking it.
let off = self.spaces_off();
- for rec in records
- .iter()
- .filter(|r| !r.paused && !(r.policy.durable && off.contains(&r.space)))
- .flat_map(|r| r.session_views())
- .filter(|r| r.run_id.is_some())
- {
- let name = match rec.k() {
- 1 => rec.name.clone(),
- k => format!("{} session {k}", rec.name),
- };
- let view = rec.clone();
- match tokio::time::timeout(AGENT_TICK_BUDGET, self.tick_agent(rec, &mut report)).await {
- Ok(Ok(())) => {}
- Ok(Err(e)) => {
- let msg = e.to_string();
- let _ = self.update_in(&view, |r| r.last_error = Some(msg.clone()));
- report.errors.push(format!("{name}: {msg}"));
- }
- Err(_) => report
- .errors
- .push(format!("{name}: supervisor pass timed out")),
+ if self.detach_dispatch {
+ // A posted message is handed to its member first, ahead of
+ // anything below that can wait on a Space; each session is read
+ // on a task of its own (see `serve_sessions`).
+ self.dispatch(&mut report).await;
+ self.serve_sessions(&records, &off, &mut report);
+ } else {
+ let live_sessions: Vec<AgentRecord> = records
+ .iter()
+ .filter(|r| !r.paused && !(r.policy.durable && off.contains(&r.space)))
+ .flat_map(|r| r.session_views())
+ .filter(|r| r.run_id.is_some())
+ .collect();
+ for rec in live_sessions {
+ self.tick_one(rec, &mut report).await;
}
}
// Durable members whose run is lost with their Space.
@@ -1829,16 +1827,128 @@ impl Persistent {
}
}
self.tick_setups(&mut report).await;
- self.tick_standby(&mut report).await;
+ self.tick_standby(&mut report, false).await;
// Thread tasks for members that are free (after the turns that
// ended above closed theirs).
- self.dispatch(&mut report).await;
+ if !self.detach_dispatch {
+ self.dispatch(&mut report).await;
+ }
self.tick_watched(&mut report).await;
self.tick_watched_jobs(&mut report).await;
self.tick_mirror(&mut report).await;
report
}
+ /// Starts a task for every live session whose last one has finished:
+ /// each is read on its own, so a Space that does not answer (a minute,
+ /// at worst) holds up only its own agent. While an agent talks to its
+ /// bridge (its tool list at start, then every tool call) its session is
+ /// read every quarter second instead of every pass, so a call is
+ /// answered at once; ten quiet seconds go back to the pass.
+ fn serve_sessions(
+ &self,
+ records: &[AgentRecord],
+ off: &std::collections::BTreeSet<String>,
+ report: &mut TickReport,
+ ) {
+ report.errors.append(
+ &mut self
+ .detached_errors
+ .lock()
+ .unwrap_or_else(|p| p.into_inner()),
+ );
+ let live_sessions = records
+ .iter()
+ .filter(|r| !r.paused && !(r.policy.durable && off.contains(&r.space)))
+ .flat_map(|r| r.session_views())
+ .filter(|r| r.run_id.is_some());
+ for rec in live_sessions {
+ let key = format!("{}#{}\u{0}tick", rec.name, rec.k());
+ let Some(guard) = phases::ServingGuard::take(&self.dir, &key) else {
+ continue; // the earlier pass for it is still running
+ };
+ let me = self.clone();
+ tokio::spawn(async move {
+ let _guard = guard;
+ // While the agent is talking to its bridge (its tool list at
+ // start, then every tool call) the session is read every
+ // quarter second instead of every pass, so a call is
+ // answered at once; ten quiet seconds go back to the pass.
+ let (name, k) = (rec.name.clone(), rec.k());
+ let mut rec = rec;
+ let mut fast_until: Option<Instant> = None;
+ loop {
+ let mut r = TickReport::default();
+ me.tick_one(rec.clone(), &mut r).await;
+ if r.bridged > 0 {
+ fast_until = Some(Instant::now() + Duration::from_secs(10));
+ }
+ if !r.errors.is_empty() {
+ me.detached_errors
+ .lock()
+ .unwrap_or_else(|p| p.into_inner())
+ .append(&mut r.errors);
+ break;
+ }
+ if !fast_until.is_some_and(|t| Instant::now() < t) {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(250)).await;
+ match me.get(&name) {
+ Ok(fresh) if !fresh.paused && fresh.session(k).run_id.is_some() => {
+ rec = fresh.session(k);
+ }
+ _ => break,
+ }
+ }
+ });
+ }
+ }
+
+ /// One session's pass, within its budget.
+ async fn tick_one(&self, rec: AgentRecord, report: &mut TickReport) {
+ let name = match rec.k() {
+ 1 => rec.name.clone(),
+ k => format!("{} session {k}", rec.name),
+ };
+ let view = rec.clone();
+ match tokio::time::timeout(AGENT_TICK_BUDGET, self.tick_agent(rec, report)).await {
+ Ok(Ok(())) => {}
+ Ok(Err(e)) => {
+ let msg = e.to_string();
+ let _ = self.update_in(&view, |r| r.last_error = Some(msg.clone()));
+ report.errors.push(format!("{name}: {msg}"));
+ }
+ Err(_) => report
+ .errors
+ .push(format!("{name}: supervisor pass timed out")),
+ }
+ }
+
+ /// Hands the queued tasks to their members now, outside a pass (only the
+ /// process holding the supervisor lock does).
+ pub async fn dispatch_now(&self) {
+ if !matches!(self.claim_supervisor().await, Ok(true)) {
+ return;
+ }
+ let mut report = TickReport::default();
+ // A new member is settled (no `.cua/setup` hook) and its standby run
+ // started here, ahead of the pass, which can be held up by a Space
+ // that does not answer; a member with a setup hook is left to the pass.
+ let _ = tokio::time::timeout(Duration::from_secs(10), self.tick_setups(&mut report)).await;
+ self.tick_standby(&mut report, true).await;
+ self.dispatch(&mut report).await;
+ if let (Ok(records), off) = (self.list(), self.spaces_off()) {
+ self.serve_sessions(&records, &off, &mut report);
+ }
+ if !report.errors.is_empty() {
+ self.detached_errors
+ .lock()
+ .unwrap_or_else(|p| p.into_inner())
+ .append(&mut report.errors);
+ }
+ }
+
/// The Spaces of this machine that are stopped or suspended now.
fn spaces_off(&self) -> std::collections::BTreeSet<String> {
self.spaces
@@ -1856,7 +1966,7 @@ impl Persistent {
/// whose run is gone): the agent process, its handshake and its session,
/// no prompt. A message later finds a session ready. A start that fails
/// is tried again after half a minute.
- async fn tick_standby(&self, _report: &mut TickReport) {
+ async fn tick_standby(&self, _report: &mut TickReport, fresh_only: bool) {
let Ok(records) = self.list() else { return };
let off = self.spaces_off();
let wanted: Vec<AgentRecord> = records
@@ -1871,6 +1981,7 @@ impl Persistent {
&& r.setup != durable::setup::PENDING
&& r.setup != durable::setup::FAILED
&& !identity::is_external(r)
+ && (!fresh_only || r.run_id.is_none())
})
.collect();
for rec in wanted {
@@ -1913,12 +2024,16 @@ impl Persistent {
let me = self.clone();
tokio::spawn(async move {
let _guard = guard;
+ let t0 = Instant::now();
+ tracing::info!(agent = %rec.name, "standby run starting");
match Box::pin(me.start_session(&rec, "", "standby", None)).await {
Ok(d) => {
me.live.lock().await.standby_retry.remove(&rec.name);
- tracing::info!(agent = %rec.name, run = %d.run_id, "standby run started");
+ tracing::info!(agent = %rec.name, run = %d.run_id, ms = t0.elapsed().as_millis() as u64, "standby run started");
+ }
+ Err(e) => {
+ tracing::warn!(agent = %rec.name, ms = t0.elapsed().as_millis() as u64, "standby run did not start: {e}")
}
- Err(e) => tracing::warn!(agent = %rec.name, "standby run did not start: {e}"),
}
});
}
@@ -2388,7 +2503,23 @@ pub trait SpacesPersistent {
// is the longest the supervisor sleeps.
let p = p.detached();
let wake = crate::thread::supervisor_wake();
+ // The dispatcher has its own task: a message is handed to its agent
+ // the moment it is posted (in this process by a wake, by another
+ // process, such as the app, within half a second), whatever the
+ // supervisor's pass is waiting on.
+ let d = p.clone();
+ let dispatch_wake = crate::thread::dispatch_wake();
+ let dispatcher = tokio::spawn(async move {
+ loop {
+ tokio::select! {
+ () = dispatch_wake.notified() => {}
+ () = tokio::time::sleep(Duration::from_millis(500)) => {}
+ }
+ d.dispatch_now().await;
+ }
+ });
tokio::spawn(async move {
+ let _dispatcher = AbortOnDrop(dispatcher);
let mut said = ErrorThrottle::default();
loop {
let r = p.tick().await;
@@ -2406,6 +2537,15 @@ pub trait SpacesPersistent {
}
}
+/// Aborts its task when dropped (the supervisor's, when it is aborted).
+struct AbortOnDrop(tokio::task::JoinHandle<()>);
+
+impl Drop for AbortOnDrop {
+ fn drop(&mut self) {
+ self.0.abort();
+ }
+}
+
impl SpacesPersistent for Spaces {
fn persistent(&self) -> Persistent {
let ext = self
diff --git a/libs/cua/crates/cua-spaces-ext/src/thread/mod.rs b/libs/cua/crates/cua-spaces-ext/src/thread/mod.rs
index 2c07b41a5d..00f047ba6d 100644
--- a/libs/cua/crates/cua-spaces-ext/src/thread/mod.rs
+++ b/libs/cua/crates/cua-spaces-ext/src/thread/mod.rs
@@ -848,6 +848,14 @@ pub fn supervisor_wake() -> Arc<tokio::sync::Notify> {
W.get_or_init(Default::default).clone()
}
+/// Wakes the dispatcher of this process alone (it runs apart from the
+/// supervisor's pass, so a pass held up by a slow Space never delays a
+/// new message reaching its agent).
+pub fn dispatch_wake() -> Arc<tokio::sync::Notify> {
+ static W: OnceLock<Arc<tokio::sync::Notify>> = OnceLock::new();
+ W.get_or_init(Default::default).clone()
+}
+
fn fold_cache() -> &'static Mutex<HashMap<PathBuf, Fold>> {
static CACHE: OnceLock<Mutex<HashMap<PathBuf, Fold>>> = OnceLock::new();
CACHE.get_or_init(Default::default)
@@ -1078,6 +1086,7 @@ impl ThreadStore {
self.notifier().notify_waiters();
if wakes {
supervisor_wake().notify_one();
+ dispatch_wake().notify_one();
}
if let Some(hook) = APPEND_HOOK.get() {
hook(self.space(), &out);