Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(spacesd): volume attach no longer deadlocks a 1 vCPU Space (#6565)
ddupont8a708abf6a60
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/mod.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/mod.rsindex 23f3305b3b..9cc0995585 100644--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/mod.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/mod.rs@@ -4,6 +4,7 @@//! `FilesystemService`: metadata, directory operations, watches, chunked//! reads and writes, resumable uploads and signed URLs.+pub mod offworker;pub mod paths;pub mod runas;pub mod watch;@@ -247,13 +248,17 @@ impl FilesystemServiceImpl {"a bulk read needs a length within a slot and a slot of the lane",));}- let file = std::fs::File::open(path).map_err(|e| io_status(&e, path))?;+ // Opened (and closed) on a blocking thread: the path can be on the+ // Cua Volume, which this runtime serves itself.let (offset, length) = (body.offset, body.length);- let (staged, sha) =- tokio::task::spawn_blocking(move || disk.stage(&file, offset, length, slot - 1))- .await- .map_err(|e| crate::error::internal(e.to_string()))?- .map_err(|e| crate::error::internal(format!("bulk lane: {e}")))?;+ let open_path = path.to_path_buf();+ let (staged, sha) = tokio::task::spawn_blocking(move || {+ let file = std::fs::File::open(&open_path).map_err(|e| io_status(&e, &open_path))?;+ disk.stage(&file, offset, length, slot - 1)+ .map_err(|e| crate::error::internal(format!("bulk lane: {e}")))+ })+ .await+ .map_err(|e| crate::error::internal(e.to_string()))??;// Like the streamed read: the file's metadata in the namespace asked for.let namespace = xattr::namespace(&body.xattr_namespace)?;let entry = entry_info_with_metadata(path, meta, &namespace, true);@@ -646,7 +651,12 @@ impl FilesystemService for FilesystemServiceImpl {remote_mode(body.mode)});}- builder.create(&path).map_err(|e| io_status(&e, &path))?;+ // On a blocking thread: the path can be on the Cua Volume.+ let made = path.clone();+ tokio::task::spawn_blocking(move || builder.create(&made))+ .await+ .map_err(|e| crate::error::internal(e.to_string()))?+ .map_err(|e| io_status(&e, &path))?;Ok(Response::new(MakeDirResponse {entry: Some(stat(&path, true, xattr::DEFAULT_NAMESPACE).await?),created: true,@@ -971,9 +981,11 @@ impl FilesystemService for FilesystemServiceImpl {if body.bulk_slot != 0 {return self.read_file_bulk(&path, &meta, &body).await;}- let mut file = tokio::fs::File::open(&path)- .await- .map_err(|e| io_status(&e, &path))?;+ let mut file = offworker::OffWorker::new(+ tokio::fs::File::open(&path)+ .await+ .map_err(|e| io_status(&e, &path))?,+ );if body.offset > 0 {file.seek(std::io::SeekFrom::Start(body.offset)).awaitdiff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/offworker.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/offworker.rsnew file mode 100644index 0000000000..8e91ce6e90--- /dev/null+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/offworker.rs@@ -0,0 +1,121 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Values that must be dropped on a blocking thread.+//!+//! Closing a file is a syscall, and on the Cua Volume (`/volume`, a FUSE+//! mount served by the volume helper, which dials the volume listener this+//! very runtime runs) it waits for an answer that only this runtime can+//! produce. Dropping a `tokio::fs::File` on an async worker closes it right+//! there: with one worker (a 1 vCPU Space) the worker waits for a task it+//! alone could run, and cua-spacesd stops answering. [`OffWorker`] closes+//! on the blocking pool instead.++use std::ops::{Deref, DerefMut};++/// A value dropped on the blocking pool (when a runtime is running).+pub struct OffWorker<T: Send + 'static>(Option<T>);++impl<T: Send + 'static> OffWorker<T> {+ /// Wraps `value`.+ pub fn new(value: T) -> Self {+ Self(Some(value))+ }++ /// [`OffWorker::close`] without consuming: the value is gone afterwards+ /// (a later deref panics), so call it last.+ pub async fn close_in_place(&mut self) {+ if let Some(value) = self.0.take() {+ let _ = tokio::task::spawn_blocking(move || drop(value)).await;+ }+ }++ /// Drops the value on the blocking pool and waits for it (use before+ /// something that needs the file closed, such as a rename).+ pub async fn close(mut self) {+ if let Some(value) = self.0.take() {+ let _ = tokio::task::spawn_blocking(move || drop(value)).await;+ }+ }+}++impl<T: Send + 'static> Deref for OffWorker<T> {+ type Target = T;+ fn deref(&self) -> &T {+ self.0.as_ref().expect("value present until closed")+ }+}++impl<T: Send + 'static> DerefMut for OffWorker<T> {+ fn deref_mut(&mut self) -> &mut T {+ self.0.as_mut().expect("value present until closed")+ }+}++impl<T: Send + 'static> Drop for OffWorker<T> {+ fn drop(&mut self) {+ if let Some(value) = self.0.take() {+ match tokio::runtime::Handle::try_current() {+ Ok(handle) => {+ // Detached: nothing waits for the close.+ drop(handle.spawn_blocking(move || drop(value)));+ }+ Err(_) => drop(value),+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::time::Duration;++ /// A value whose drop waits for a task of the runtime it is dropped+ /// from: dropped on the only worker, it would wait forever.+ struct WaitsForRuntime(std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<bool>);++ impl Drop for WaitsForRuntime {+ fn drop(&mut self) {+ let answered = self.0.recv_timeout(Duration::from_secs(10)).is_ok();+ let _ = self.1.send(answered);+ }+ }++ fn rig() -> (+ WaitsForRuntime,+ std::sync::mpsc::Sender<()>,+ std::sync::mpsc::Receiver<bool>,+ ) {+ let (answer_tx, answer_rx) = std::sync::mpsc::channel();+ let (done_tx, done_rx) = std::sync::mpsc::channel();+ (WaitsForRuntime(answer_rx, done_tx), answer_tx, done_rx)+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 1)]+ async fn a_dropped_value_is_closed_off_the_only_worker() {+ let (value, answer, done) = rig();+ let guard = OffWorker::new(value);+ // The task that answers the close runs on the same, only worker.+ let server = tokio::spawn(async move {+ tokio::time::sleep(Duration::from_millis(50)).await;+ answer.send(()).unwrap();+ });+ drop(guard);+ server.await.unwrap();+ assert!(done.recv_timeout(Duration::from_secs(10)).unwrap());+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 1)]+ async fn close_waits_for_the_drop_off_the_only_worker() {+ let (value, answer, done) = rig();+ let guard = OffWorker::new(value);+ let server = tokio::spawn(async move {+ tokio::time::sleep(Duration::from_millis(50)).await;+ answer.send(()).unwrap();+ });+ guard.close().await;+ server.await.unwrap();+ assert!(done.try_recv().unwrap());+ }+}diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/write.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/write.rsindex 821dcbbe18..bfc3b479da 100644--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/write.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/filesystem/write.rs@@ -14,6 +14,7 @@ use tonic::{Code, Status};use crate::error::{io_status, status, StatusBuilder};use crate::util::{hex_digest, random_id};+use super::offworker::OffWorker;use super::paths::entry_info;use super::xattr::{self, Attributes};@@ -54,7 +55,7 @@ impl WriteTarget {pub struct AtomicWriter {target: WriteTarget,tmp: Option<PathBuf>,- file: tokio::fs::File,+ file: OffWorker<tokio::fs::File>,hasher: Sha256,written: u64,}@@ -162,20 +163,25 @@ impl AtomicWriter {} else {target.permissions});- let file = options- .open(&path)- .await- .map_err(|e| io_status(&e, &path))?;+ let file = OffWorker::new(+ options+ .open(&path)+ .await+ .map_err(|e| io_status(&e, &path))?,+ );if !target.attributes.values.is_empty() {#[cfg(unix)]let stored = {use std::os::fd::AsRawFd as _;- xattr::set_on_fd(file.as_raw_fd(), &target.attributes)+ let (fd, attributes) = (file.as_raw_fd(), target.attributes.clone());+ tokio::task::spawn_blocking(move || xattr::set_on_fd(fd, &attributes))+ .await+ .unwrap_or_else(|e| Err(crate::error::internal(e.to_string())))};#[cfg(not(unix))]let stored = xattr::set_on_fd(0, &target.attributes);if let Err(error) = stored {- drop(file);+ file.close().await;if let Some(tmp) = &tmp {let _ = tokio::fs::remove_file(tmp).await;}@@ -251,8 +257,10 @@ impl AtomicWriter {// The open file's own metadata, taken before it is renamed: what the// entry reports if the destination cannot be looked up right after.let published_meta = self.file.metadata().await.ok();+ // Closed on a blocking thread (the destination can be on the Cua+ // Volume), before the rename.+ self.file.close_in_place().await;if let Some(tmp) = self.tmp.take() {- drop(self.file);let published = match self.target.mode {WriteMode::CreateNew => {// link(2) fails if the destination appeared meanwhile.@@ -286,13 +294,17 @@ impl AtomicWriter {.await.map_err(|e| io_status(&e, &dest))?;let mut entry = entry_info(&dest, &meta);- entry.metadata = xattr::read(&dest, &self.target.attributes.namespace, true);+ let (read_path, namespace) = (dest.clone(), self.target.attributes.namespace.clone());+ entry.metadata =+ tokio::task::spawn_blocking(move || xattr::read(&read_path, &namespace, true))+ .await+ .unwrap_or_default();Ok((entry, digest))}/// Discards the temporary file.- pub async fn abort(self) {- drop(self.file);+ pub async fn abort(mut self) {+ self.file.close_in_place().await;if let Some(tmp) = self.tmp {let _ = tokio::fs::remove_file(tmp).await;}diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rsindex ab6d1e64a3..33381bd760 100644--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/services/volume.rs@@ -116,6 +116,29 @@ fn which(name: &str) -> Option<PathBuf> {})}+/// Runs blocking filesystem work off the async workers.+///+/// A path under `/volume` is served by this very process (the FUSE helper+/// dials the volume listener this runtime runs), so a `stat` or `mkdir` on it+/// from a worker waits for a task only that worker could run. With one+/// worker (a 1 vCPU Space) that is a deadlock: cua-spacesd keeps accepting+/// connections and never answers one. Everything that can touch the mount+/// goes through here.+async fn off_runtime<T: Send + 'static>(+ work: impl FnOnce() -> T + Send + 'static,+) -> Result<T, String> {+ tokio::task::spawn_blocking(work)+ .await+ .map_err(|e| format!("blocking filesystem call: {e}"))+}++/// [`default_mount_path`] on a blocking thread (it stats `/volume`).+pub async fn default_mount_path_async() -> PathBuf {+ off_runtime(default_mount_path)+ .await+ .unwrap_or_else(|_| PathBuf::from("/volume"))+}+/// Where the volume mounts by default.pub fn default_mount_path() -> PathBuf {if cfg!(windows) {@@ -510,7 +533,11 @@ async fn mount(session: &Arc<Session>) -> Result<(), String> {*session.windows.lock().expect("windows") = Some(m);return Ok(());}- std::fs::create_dir_all(path).map_err(|e| format!("{}: {e}", path.display()))?;+ let dir = path.clone();+ off_runtime(move || {+ std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))+ })+ .await??;let p = path.to_string_lossy().to_string();match session.backend {"nfs" => {@@ -625,7 +652,8 @@ async fn unmount(session: &Arc<Session>, client_attached: bool) {}// A home-folder mount point goes away with the mount; /volume stays.if path != Path::new("/volume") {- let _ = std::fs::remove_dir(path);+ let dir = path.clone();+ let _ = off_runtime(move || std::fs::remove_dir(dir)).await;}}@@ -706,7 +734,7 @@ impl VolumeService for VolumeServiceImpl {let backend = backend().map_err(|reason| status(Code::FailedPrecondition, ErrorReason::Unspecified, reason))?;let mut mount_path = if body.mount_path.is_empty() {- default_mount_path()+ default_mount_path_async().await} else {PathBuf::from(&body.mount_path)};@@ -949,6 +977,29 @@ fn running_as_root() -> bool {mod tests {use super::*;+ /// A filesystem call that waits on a task of this runtime (a `stat` of+ /// `/volume`, whose FUSE server is a task of this process) must not hold+ /// the runtime's only worker: with one worker it would wait forever.+ #[tokio::test(flavor = "multi_thread", worker_threads = 1)]+ async fn blocking_mount_work_leaves_the_only_worker_free() {+ let (tx, rx) = std::sync::mpsc::channel::<()>();+ // Answers the "filesystem call" from a task of the same runtime.+ let server = tokio::spawn(async move {+ tokio::time::sleep(Duration::from_millis(50)).await;+ tx.send(()).unwrap();+ });+ let stat = off_runtime(move || rx.recv_timeout(Duration::from_secs(10)).is_ok())+ .await+ .unwrap();+ server.await.unwrap();+ assert!(stat, "the call was answered by the runtime's own task");+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 1)]+ async fn the_default_mount_path_is_the_blocking_one() {+ assert_eq!(default_mount_path_async().await, default_mount_path());+ }+#[test]fn a_listed_mount_is_found_without_stat() {let mac = "127.0.0.1:/ on /Users/a/Cua Volume (nfs, nodev, nosuid, mounted by a)\n\diff --git a/libs/cua-spacesd/crates/cua-spacesd/src/main.rs b/libs/cua-spacesd/crates/cua-spacesd/src/main.rsindex e63960613a..02f4f52147 100644--- a/libs/cua-spacesd/crates/cua-spacesd/src/main.rs+++ b/libs/cua-spacesd/crates/cua-spacesd/src/main.rs@@ -483,7 +483,12 @@ fn main() {/// Runs a daemon command on a fresh multi-threaded runtime.fn run(command: Command) -> Result<Exit, BoxError> {+ // At least two workers: a Space with one vCPU would otherwise run the+ // whole server on one thread, and any blocking call that waits on this+ // process itself (the Cua Volume is served from here) would stop it.+ let workers = std::thread::available_parallelism().map_or(2, |n| n.get().max(2));let runtime = tokio::runtime::Builder::new_multi_thread()+ .worker_threads(workers).enable_all().build().expect("tokio runtime");diff --git a/libs/cua/crates/cua-agents/src/harness.rs b/libs/cua/crates/cua-agents/src/harness.rsindex 5906cab323..3a766dabd9 100644--- a/libs/cua/crates/cua-agents/src/harness.rs+++ b/libs/cua/crates/cua-agents/src/harness.rs@@ -626,7 +626,22 @@ pub fn memory_dir(h: &Harness) -> Option<&'static str> {/// state is `<guest home>/.cua/agents/codex-state/<name>`; a home elsewhere/// (an explicit `home_dir`) keeps it in its own `.codex-sqlite` sibling.pub fn codex_sqlite_dir(home: &str) -> String {+ codex_sqlite_dir_in(home, "")+}++/// [`codex_sqlite_dir`] for a run whose directory is `run_dir`+/// (`<guest home>/.cua/agents/<run>`). A home on a Space's mounted Volume+/// (`/volume/agents/<name>`, where `volume_for_agent` puts it) names no+/// guest home of its own; the run directory does.+pub fn codex_sqlite_dir_in(home: &str, run_dir: &str) -> String {let home = home.trim_end_matches('/');+ if let Some(name) = home.strip_prefix("/volume/agents/")+ && !name.contains('/')+ && !name.is_empty()+ && let Some((guest_home, _)) = run_dir.trim_end_matches('/').rsplit_once("/.cua/agents/")+ {+ return format!("{guest_home}/.cua/agents/codex-state/{name}");+ }// `cua-volume` in a Space; on a Mac the Volume is mounted at// `~/Cua Volume`.for marker in ["/cua-volume/agents/", "/Cua Volume/agents/"] {@@ -771,7 +786,7 @@ pub fn launch_with_config(// config and memories stay on the Volume. `sqlite_home` is a// top-level key, so it goes before any table, and it is written on// every start with the rest of the generated config.- if let Some(dir) = home.map(codex_sqlite_dir) {+ if let Some(dir) = home.map(|h| codex_sqlite_dir_in(h, run_dir)) {toml.insert_str(0, &format!("sqlite_home = {dir:?}\n\n"));l.prelude.push(format!("mkdir -p {}", crate::quote(&dir)));env(&mut l, "CODEX_SQLITE_HOME", dir);@@ -1215,6 +1230,26 @@ mod tests {codex_sqlite_dir("/Users/ada/Cua Volume/agents/codex"),"/Users/ada/.cua/agents/codex-state/codex");+ // A home on the Space's mounted Volume (`/volume/agents/<name>`) keeps+ // the state in the guest home the run directory names, never on the+ // Volume (SQLite cannot map shared memory there).+ assert_eq!(+ codex_sqlite_dir_in("/volume/agents/ada", "/home/cua/.cua/agents/run-1"),+ "/home/cua/.cua/agents/codex-state/ada"+ );+ let x = launch_with_home(+ harness("openai-codex").unwrap(),+ None,+ None,+ "/home/cua/.cua/agents/run-1",+ "/w",+ Some("/volume/agents/ada"),+ )+ .unwrap();+ assert!(x.env.contains(&(+ "CODEX_SQLITE_HOME".to_string(),+ "/home/cua/.cua/agents/codex-state/ada".to_string()+ )));// Not the home's own `codex/` (CODEX_HOME), which is on the Volume.let home = "/home/cua/cua-volume/agents/ada";assert!(!codex_sqlite_dir(home).starts_with(home));diff --git a/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs b/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rsindex 118d214515..b976394a73 100644--- a/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs+++ b/libs/cua/crates/cua-spaces-ext/src/persistent/mod.rs@@ -2341,6 +2341,33 @@ impl RoutineRunner for Runner {}}+/// How long the supervisor stays quiet about an error it already logged.+const ERROR_REPEAT: Duration = Duration::from_secs(300);++/// Logs each supervisor error once per [`ERROR_REPEAT`]: a stopped Space+/// makes every pass (about once a second) fail the same way for every agent+/// in it, which used to fill the daemon log with ~9 lines a second.+#[derive(Default)]+struct ErrorThrottle {+ seen: HashMap<String, Instant>,+}++impl ErrorThrottle {+ fn should_say(&mut self, error: &str, now: Instant) -> bool {+ if self.seen.len() > 1024 {+ self.seen+ .retain(|_, at| now.duration_since(*at) < ERROR_REPEAT);+ }+ match self.seen.get(error) {+ Some(at) if now.duration_since(*at) < ERROR_REPEAT => false,+ _ => {+ self.seen.insert(error.to_string(), now);+ true+ }+ }+ }+}+/// Persistent agents on a [`Spaces`] runtime that registered the/// [`crate::DriveExtension`].pub trait SpacesPersistent {@@ -2362,10 +2389,13 @@ pub trait SpacesPersistent {let p = p.detached();let wake = crate::thread::supervisor_wake();tokio::spawn(async move {+ let mut said = ErrorThrottle::default();loop {let r = p.tick().await;for e in &r.errors {- tracing::warn!("persistent agents: {e}");+ if said.should_say(e, Instant::now()) {+ tracing::warn!("persistent agents: {e}");+ }}tokio::select! {() = tokio::time::sleep(every) => {}@@ -2460,6 +2490,20 @@ pub fn parse_schedule(mod tests {use super::*;+ #[test]+ fn a_repeated_supervisor_error_is_logged_once_per_window() {+ let mut t = ErrorThrottle::default();+ let t0 = Instant::now();+ assert!(t.should_say("sandbox a is Stopped", t0));+ assert!(!t.should_say("sandbox a is Stopped", t0 + Duration::from_secs(1)));+ assert!(t.should_say("sandbox b is Stopped", t0 + Duration::from_secs(1)));+ assert!(!t.should_say(+ "sandbox a is Stopped",+ t0 + ERROR_REPEAT - Duration::from_secs(1)+ ));+ assert!(t.should_say("sandbox a is Stopped", t0 + ERROR_REPEAT));+ }+// An events.jsonl line as the runner writes it.fn ev(kind: &str, turn: u32, text: Option<&str>) -> ag::AgentEvent {let line =