Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(spacesd): volume attach no longer deadlocks a 1 vCPU Space (#6565)

ddupont8a708abf6a60
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/mod.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/mod.rs
index 23f3305b3b..9cc0995585 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/mod.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/mod.rs
@@ -4,6 +4,7 @@
//! `FilesystemService`: metadata, directory operations, watches, chunked
//! reads and writes, resumable uploads and signed URLs.
+pub mod offworker;
pub mod paths;
pub mod runas;
pub mod watch;
@@ -247,13 +248,17 @@ impl FilesystemServiceImpl {
"a bulk read needs a length within a slot and a slot of the lane",
));
}
- let file = std::fs::File::open(path).map_err(|e| io_status(&e, path))?;
+ // Opened (and closed) on a blocking thread: the path can be on the
+ // Cua Volume, which this runtime serves itself.
let (offset, length) = (body.offset, body.length);
- let (staged, sha) =
- tokio::task::spawn_blocking(move || disk.stage(&file, offset, length, slot - 1))
- .await
- .map_err(|e| crate::error::internal(e.to_string()))?
- .map_err(|e| crate::error::internal(format!("bulk lane: {e}")))?;
+ let open_path = path.to_path_buf();
+ let (staged, sha) = tokio::task::spawn_blocking(move || {
+ let file = std::fs::File::open(&open_path).map_err(|e| io_status(&e, &open_path))?;
+ disk.stage(&file, offset, length, slot - 1)
+ .map_err(|e| crate::error::internal(format!("bulk lane: {e}")))
+ })
+ .await
+ .map_err(|e| crate::error::internal(e.to_string()))??;
// Like the streamed read: the file's metadata in the namespace asked for.
let namespace = xattr::namespace(&body.xattr_namespace)?;
let entry = entry_info_with_metadata(path, meta, &namespace, true);
@@ -646,7 +651,12 @@ impl FilesystemService for FilesystemServiceImpl {
remote_mode(body.mode)
});
}
- builder.create(&path).map_err(|e| io_status(&e, &path))?;
+ // On a blocking thread: the path can be on the Cua Volume.
+ let made = path.clone();
+ tokio::task::spawn_blocking(move || builder.create(&made))
+ .await
+ .map_err(|e| crate::error::internal(e.to_string()))?
+ .map_err(|e| io_status(&e, &path))?;
Ok(Response::new(MakeDirResponse {
entry: Some(stat(&path, true, xattr::DEFAULT_NAMESPACE).await?),
created: true,
@@ -971,9 +981,11 @@ impl FilesystemService for FilesystemServiceImpl {
if body.bulk_slot != 0 {
return self.read_file_bulk(&path, &meta, &body).await;
}
- let mut file = tokio::fs::File::open(&path)
- .await
- .map_err(|e| io_status(&e, &path))?;
+ let mut file = offworker::OffWorker::new(
+ tokio::fs::File::open(&path)
+ .await
+ .map_err(|e| io_status(&e, &path))?,
+ );
if body.offset > 0 {
file.seek(std::io::SeekFrom::Start(body.offset))
.await
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/offworker.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/offworker.rs
new file mode 100644
index 0000000000..8e91ce6e90
--- /dev/null
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/offworker.rs
@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Values that must be dropped on a blocking thread.
+//!
+//! Closing a file is a syscall, and on the Cua Volume (`/volume`, a FUSE
+//! mount served by the volume helper, which dials the volume listener this
+//! very runtime runs) it waits for an answer that only this runtime can
+//! produce. Dropping a `tokio::fs::File` on an async worker closes it right
+//! there: with one worker (a 1 vCPU Space) the worker waits for a task it
+//! alone could run, and cua-spacesd stops answering. [`OffWorker`] closes
+//! on the blocking pool instead.
+
+use std::ops::{Deref, DerefMut};
+
+/// A value dropped on the blocking pool (when a runtime is running).
+pub struct OffWorker<T: Send + 'static>(Option<T>);
+
+impl<T: Send + 'static> OffWorker<T> {
+ /// Wraps `value`.
+ pub fn new(value: T) -> Self {
+ Self(Some(value))
+ }
+
+ /// [`OffWorker::close`] without consuming: the value is gone afterwards
+ /// (a later deref panics), so call it last.
+ pub async fn close_in_place(&mut self) {
+ if let Some(value) = self.0.take() {
+ let _ = tokio::task::spawn_blocking(move || drop(value)).await;
+ }
+ }
+
+ /// Drops the value on the blocking pool and waits for it (use before
+ /// something that needs the file closed, such as a rename).
+ pub async fn close(mut self) {
+ if let Some(value) = self.0.take() {
+ let _ = tokio::task::spawn_blocking(move || drop(value)).await;
+ }
+ }
+}
+
+impl<T: Send + 'static> Deref for OffWorker<T> {
+ type Target = T;
+ fn deref(&self) -> &T {
+ self.0.as_ref().expect("value present until closed")
+ }
+}
+
+impl<T: Send + 'static> DerefMut for OffWorker<T> {
+ fn deref_mut(&mut self) -> &mut T {
+ self.0.as_mut().expect("value present until closed")
+ }
+}
+
+impl<T: Send + 'static> Drop for OffWorker<T> {
+ fn drop(&mut self) {
+ if let Some(value) = self.0.take() {
+ match tokio::runtime::Handle::try_current() {
+ Ok(handle) => {
+ // Detached: nothing waits for the close.
+ drop(handle.spawn_blocking(move || drop(value)));
+ }
+ Err(_) => drop(value),
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::time::Duration;
+
+ /// A value whose drop waits for a task of the runtime it is dropped
+ /// from: dropped on the only worker, it would wait forever.
+ struct WaitsForRuntime(std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<bool>);
+
+ impl Drop for WaitsForRuntime {
+ fn drop(&mut self) {
+ let answered = self.0.recv_timeout(Duration::from_secs(10)).is_ok();
+ let _ = self.1.send(answered);
+ }
+ }
+
+ fn rig() -> (
+ WaitsForRuntime,
+ std::sync::mpsc::Sender<()>,
+ std::sync::mpsc::Receiver<bool>,
+ ) {
+ let (answer_tx, answer_rx) = std::sync::mpsc::channel();
+ let (done_tx, done_rx) = std::sync::mpsc::channel();
+ (WaitsForRuntime(answer_rx, done_tx), answer_tx, done_rx)
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+ async fn a_dropped_value_is_closed_off_the_only_worker() {
+ let (value, answer, done) = rig();
+ let guard = OffWorker::new(value);
+ // The task that answers the close runs on the same, only worker.
+ let server = tokio::spawn(async move {
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ answer.send(()).unwrap();
+ });
+ drop(guard);
+ server.await.unwrap();
+ assert!(done.recv_timeout(Duration::from_secs(10)).unwrap());
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+ async fn close_waits_for_the_drop_off_the_only_worker() {
+ let (value, answer, done) = rig();
+ let guard = OffWorker::new(value);
+ let server = tokio::spawn(async move {
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ answer.send(()).unwrap();
+ });
+ guard.close().await;
+ server.await.unwrap();
+ assert!(done.try_recv().unwrap());
+ }
+}
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/write.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/write.rs
index 821dcbbe18..bfc3b479da 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/write.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/write.rs
@@ -14,6 +14,7 @@ use tonic::{Code, Status};
use crate::error::{io_status, status, StatusBuilder};
use crate::util::{hex_digest, random_id};
+use super::offworker::OffWorker;
use super::paths::entry_info;
use super::xattr::{self, Attributes};
@@ -54,7 +55,7 @@ impl WriteTarget {
pub struct AtomicWriter {
target: WriteTarget,
tmp: Option<PathBuf>,
- file: tokio::fs::File,
+ file: OffWorker<tokio::fs::File>,
hasher: Sha256,
written: u64,
}
@@ -162,20 +163,25 @@ impl AtomicWriter {
} else {
target.permissions
});
- let file = options
- .open(&path)
- .await
- .map_err(|e| io_status(&e, &path))?;
+ let file = OffWorker::new(
+ options
+ .open(&path)
+ .await
+ .map_err(|e| io_status(&e, &path))?,
+ );
if !target.attributes.values.is_empty() {
#[cfg(unix)]
let stored = {
use std::os::fd::AsRawFd as _;
- xattr::set_on_fd(file.as_raw_fd(), &target.attributes)
+ let (fd, attributes) = (file.as_raw_fd(), target.attributes.clone());
+ tokio::task::spawn_blocking(move || xattr::set_on_fd(fd, &attributes))
+ .await
+ .unwrap_or_else(|e| Err(crate::error::internal(e.to_string())))
};
#[cfg(not(unix))]
let stored = xattr::set_on_fd(0, &target.attributes);
if let Err(error) = stored {
- drop(file);
+ file.close().await;
if let Some(tmp) = &tmp {
let _ = tokio::fs::remove_file(tmp).await;
}
@@ -251,8 +257,10 @@ impl AtomicWriter {
// The open file's own metadata, taken before it is renamed: what the
// entry reports if the destination cannot be looked up right after.
let published_meta = self.file.metadata().await.ok();
+ // Closed on a blocking thread (the destination can be on the Cua
+ // Volume), before the rename.
+ self.file.close_in_place().await;
if let Some(tmp) = self.tmp.take() {
- drop(self.file);
let published = match self.target.mode {
WriteMode::CreateNew => {
// link(2) fails if the destination appeared meanwhile.
@@ -286,13 +294,17 @@ impl AtomicWriter {
.await
.map_err(|e| io_status(&e, &dest))?;
let mut entry = entry_info(&dest, &meta);
- entry.metadata = xattr::read(&dest, &self.target.attributes.namespace, true);
+ let (read_path, namespace) = (dest.clone(), self.target.attributes.namespace.clone());
+ entry.metadata =
+ tokio::task::spawn_blocking(move || xattr::read(&read_path, &namespace, true))
+ .await
+ .unwrap_or_default();
Ok((entry, digest))
}
/// Discards the temporary file.
- pub async fn abort(self) {
- drop(self.file);
+ pub async fn abort(mut self) {
+ self.file.close_in_place().await;
if let Some(tmp) = self.tmp {
let _ = tokio::fs::remove_file(tmp).await;
}
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs
index ab6d1e64a3..33381bd760 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs
@@ -116,6 +116,29 @@ fn which(name: &str) -> Option<PathBuf> {
})
}
+/// Runs blocking filesystem work off the async workers.
+///
+/// A path under `/volume` is served by this very process (the FUSE helper
+/// dials the volume listener this runtime runs), so a `stat` or `mkdir` on it
+/// from a worker waits for a task only that worker could run. With one
+/// worker (a 1 vCPU Space) that is a deadlock: cua-spacesd keeps accepting
+/// connections and never answers one. Everything that can touch the mount
+/// goes through here.
+async fn off_runtime<T: Send + 'static>(
+ work: impl FnOnce() -> T + Send + 'static,
+) -> Result<T, String> {
+ tokio::task::spawn_blocking(work)
+ .await
+ .map_err(|e| format!("blocking filesystem call: {e}"))
+}
+
+/// [`default_mount_path`] on a blocking thread (it stats `/volume`).
+pub async fn default_mount_path_async() -> PathBuf {
+ off_runtime(default_mount_path)
+ .await
+ .unwrap_or_else(|_| PathBuf::from("/volume"))
+}
+
/// Where the volume mounts by default.
pub fn default_mount_path() -> PathBuf {
if cfg!(windows) {
@@ -510,7 +533,11 @@ async fn mount(session: &Arc<Session>) -> Result<(), String> {
*session.windows.lock().expect("windows") = Some(m);
return Ok(());
}
- std::fs::create_dir_all(path).map_err(|e| format!("{}: {e}", path.display()))?;
+ let dir = path.clone();
+ off_runtime(move || {
+ std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))
+ })
+ .await??;
let p = path.to_string_lossy().to_string();
match session.backend {
"nfs" => {
@@ -625,7 +652,8 @@ async fn unmount(session: &Arc<Session>, client_attached: bool) {
}
// A home-folder mount point goes away with the mount; /volume stays.
if path != Path::new("/volume") {
- let _ = std::fs::remove_dir(path);
+ let dir = path.clone();
+ let _ = off_runtime(move || std::fs::remove_dir(dir)).await;
}
}
@@ -706,7 +734,7 @@ impl VolumeService for VolumeServiceImpl {
let backend = backend()
.map_err(|reason| status(Code::FailedPrecondition, ErrorReason::Unspecified, reason))?;
let mut mount_path = if body.mount_path.is_empty() {
- default_mount_path()
+ default_mount_path_async().await
} else {
PathBuf::from(&body.mount_path)
};
@@ -949,6 +977,29 @@ fn running_as_root() -> bool {
mod tests {
use super::*;
+ /// A filesystem call that waits on a task of this runtime (a `stat` of
+ /// `/volume`, whose FUSE server is a task of this process) must not hold
+ /// the runtime's only worker: with one worker it would wait forever.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+ async fn blocking_mount_work_leaves_the_only_worker_free() {
+ let (tx, rx) = std::sync::mpsc::channel::<()>();
+ // Answers the "filesystem call" from a task of the same runtime.
+ let server = tokio::spawn(async move {
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ tx.send(()).unwrap();
+ });
+ let stat = off_runtime(move || rx.recv_timeout(Duration::from_secs(10)).is_ok())
+ .await
+ .unwrap();
+ server.await.unwrap();
+ assert!(stat, "the call was answered by the runtime's own task");
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+ async fn the_default_mount_path_is_the_blocking_one() {
+ assert_eq!(default_mount_path_async().await, default_mount_path());
+ }
+
#[test]
fn a_listed_mount_is_found_without_stat() {
let mac = "127.0.0.1:/ on /Users/a/Cua Volume (nfs, nodev, nosuid, mounted by a)\n\
diff --git a/libs/cua-spacesd/crates/cua-spacesd/src/main.rs b/libs/cua-spacesd/crates/cua-spacesd/src/main.rs
index e63960613a..02f4f52147 100644
--- a/libs/cua-spacesd/crates/cua-spacesd/src/main.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd/src/main.rs
@@ -483,7 +483,12 @@ fn main() {
/// Runs a daemon command on a fresh multi-threaded runtime.
fn run(command: Command) -> Result<Exit, BoxError> {
+ // At least two workers: a Space with one vCPU would otherwise run the
+ // whole server on one thread, and any blocking call that waits on this
+ // process itself (the Cua Volume is served from here) would stop it.
+ let workers = std::thread::available_parallelism().map_or(2, |n| n.get().max(2));
let runtime = tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(workers)
.enable_all()
.build()
.expect("tokio runtime");
diff --git a/libs/cua/crates/cua-agents/src/harness.rs b/libs/cua/crates/cua-agents/src/harness.rs
index 5906cab323..3a766dabd9 100644
--- a/libs/cua/crates/cua-agents/src/harness.rs
+++ b/libs/cua/crates/cua-agents/src/harness.rs
@@ -626,7 +626,22 @@ pub fn memory_dir(h: &Harness) -> Option<&'static str> {
/// state is `<guest home>/.cua/agents/codex-state/<name>`; a home elsewhere
/// (an explicit `home_dir`) keeps it in its own `.codex-sqlite` sibling.
pub fn codex_sqlite_dir(home: &str) -> String {
+ codex_sqlite_dir_in(home, "")
+}
+
+/// [`codex_sqlite_dir`] for a run whose directory is `run_dir`
+/// (`<guest home>/.cua/agents/<run>`). A home on a Space's mounted Volume
+/// (`/volume/agents/<name>`, where `volume_for_agent` puts it) names no
+/// guest home of its own; the run directory does.
+pub fn codex_sqlite_dir_in(home: &str, run_dir: &str) -> String {
let home = home.trim_end_matches('/');
+ if let Some(name) = home.strip_prefix("/volume/agents/")
+ && !name.contains('/')
+ && !name.is_empty()
+ && let Some((guest_home, _)) = run_dir.trim_end_matches('/').rsplit_once("/.cua/agents/")
+ {
+ return format!("{guest_home}/.cua/agents/codex-state/{name}");
+ }
// `cua-volume` in a Space; on a Mac the Volume is mounted at
// `~/Cua Volume`.
for marker in ["/cua-volume/agents/", "/Cua Volume/agents/"] {
@@ -771,7 +786,7 @@ pub fn launch_with_config(
// config and memories stay on the Volume. `sqlite_home` is a
// top-level key, so it goes before any table, and it is written on
// every start with the rest of the generated config.
- if let Some(dir) = home.map(codex_sqlite_dir) {
+ if let Some(dir) = home.map(|h| codex_sqlite_dir_in(h, run_dir)) {
toml.insert_str(0, &format!("sqlite_home = {dir:?}\n\n"));
l.prelude.push(format!("mkdir -p {}", crate::quote(&dir)));
env(&mut l, "CODEX_SQLITE_HOME", dir);
@@ -1215,6 +1230,26 @@ mod tests {
codex_sqlite_dir("/Users/ada/Cua Volume/agents/codex"),
"/Users/ada/.cua/agents/codex-state/codex"
);
+ // A home on the Space's mounted Volume (`/volume/agents/<name>`) keeps
+ // the state in the guest home the run directory names, never on the
+ // Volume (SQLite cannot map shared memory there).
+ assert_eq!(
+ codex_sqlite_dir_in("/volume/agents/ada", "/home/cua/.cua/agents/run-1"),
+ "/home/cua/.cua/agents/codex-state/ada"
+ );
+ let x = launch_with_home(
+ harness("openai-codex").unwrap(),
+ None,
+ None,
+ "/home/cua/.cua/agents/run-1",
+ "/w",
+ Some("/volume/agents/ada"),
+ )
+ .unwrap();
+ assert!(x.env.contains(&(
+ "CODEX_SQLITE_HOME".to_string(),
+ "/home/cua/.cua/agents/codex-state/ada".to_string()
+ )));
// Not the home's own `codex/` (CODEX_HOME), which is on the Volume.
let home = "/home/cua/cua-volume/agents/ada";
assert!(!codex_sqlite_dir(home).starts_with(home));
diff --git a/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs b/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs
index 118d214515..b976394a73 100644
--- a/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs
+++ b/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs
@@ -2341,6 +2341,33 @@ impl RoutineRunner for Runner {
}
}
+/// How long the supervisor stays quiet about an error it already logged.
+const ERROR_REPEAT: Duration = Duration::from_secs(300);
+
+/// Logs each supervisor error once per [`ERROR_REPEAT`]: a stopped Space
+/// makes every pass (about once a second) fail the same way for every agent
+/// in it, which used to fill the daemon log with ~9 lines a second.
+#[derive(Default)]
+struct ErrorThrottle {
+ seen: HashMap<String, Instant>,
+}
+
+impl ErrorThrottle {
+ fn should_say(&mut self, error: &str, now: Instant) -> bool {
+ if self.seen.len() > 1024 {
+ self.seen
+ .retain(|_, at| now.duration_since(*at) < ERROR_REPEAT);
+ }
+ match self.seen.get(error) {
+ Some(at) if now.duration_since(*at) < ERROR_REPEAT => false,
+ _ => {
+ self.seen.insert(error.to_string(), now);
+ true
+ }
+ }
+ }
+}
+
/// Persistent agents on a [`Spaces`] runtime that registered the
/// [`crate::DriveExtension`].
pub trait SpacesPersistent {
@@ -2362,10 +2389,13 @@ pub trait SpacesPersistent {
let p = p.detached();
let wake = crate::thread::supervisor_wake();
tokio::spawn(async move {
+ let mut said = ErrorThrottle::default();
loop {
let r = p.tick().await;
for e in &r.errors {
- tracing::warn!("persistent agents: {e}");
+ if said.should_say(e, Instant::now()) {
+ tracing::warn!("persistent agents: {e}");
+ }
}
tokio::select! {
() = tokio::time::sleep(every) => {}
@@ -2460,6 +2490,20 @@ pub fn parse_schedule(
mod tests {
use super::*;
+ #[test]
+ fn a_repeated_supervisor_error_is_logged_once_per_window() {
+ let mut t = ErrorThrottle::default();
+ let t0 = Instant::now();
+ assert!(t.should_say("sandbox a is Stopped", t0));
+ assert!(!t.should_say("sandbox a is Stopped", t0 + Duration::from_secs(1)));
+ assert!(t.should_say("sandbox b is Stopped", t0 + Duration::from_secs(1)));
+ assert!(!t.should_say(
+ "sandbox a is Stopped",
+ t0 + ERROR_REPEAT - Duration::from_secs(1)
+ ));
+ assert!(t.should_say("sandbox a is Stopped", t0 + ERROR_REPEAT));
+ }
+
// An events.jsonl line as the runner writes it.
fn ev(kind: &str, turn: u32, text: Option<&str>) -> ag::AgentEvent {
let line =