Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(spaces): don't count the image pull against the macOS readiness timeout (#4622)
Francesco Bonacci033b980cd2d1
diff --git a/docs/content/docs/cua-cli/reference/cli/sandbox.mdx b/docs/content/docs/cua-cli/reference/cli/sandbox.mdxindex b4f2e8631f..8716ecd66d 100644--- a/docs/content/docs/cua-cli/reference/cli/sandbox.mdx+++ b/docs/content/docs/cua-cli/reference/cli/sandbox.mdx@@ -63,7 +63,7 @@ cua sandbox create [OPTIONS] [IMAGE] [-- <COMMAND>...]| <span id="cua-sandbox-create--disk"></span>`--disk` | string | | | Disk size (accepted; the image decides today). || <span id="cua-sandbox-create--port"></span>`--port` | string | | | Expose a guest port: NAME=PORT (a named service) or PORT (repeatable). Alias: `--service`. || <span id="cua-sandbox-create--wait"></span>`--wait` | string | | | Wait until ready: tcp:SERVICE, http:SERVICE/path, a guest port (tcp:PORT, http:PORT/path), or `desktop` (the desktop session takes input: display, window manager and cua-driver) (repeatable). Alias: `--wait-for`. |-| <span id="cua-sandbox-create--ready-timeout"></span>`--ready-timeout` | integer | | | Readiness budget in seconds, the wait for the image's cua-spacesd included (default 600, with at most 120 of it for cua-spacesd). |+| <span id="cua-sandbox-create--ready-timeout"></span>`--ready-timeout` | integer | | | Readiness budget in seconds, counted once the image is downloaded, the wait for the image's cua-spacesd included (default 600, plus up to 120 for cua-spacesd). || <span id="cua-sandbox-create--overlay"></span>`--overlay` | string | | | Inject a freshly built binary once the sandbox is up: NAME=PATH (`cua-driver`, `cua-spacesd`) or NAME=PATH:GUEST_PATH (repeatable). It replaces the guest file atomically, is recorded with its sha256 (checked by `cua doctor`) and what runs it is restarted. Needs cua-spacesd in the image. || <span id="cua-sandbox-create--env"></span>`--env` | string | | | Environment KEY=VALUE for the sandbox (repeatable). || <span id="cua-sandbox-create--sidecar"></span>`--sidecar` | string | | | A sidecar container, reachable from the sandbox at its name (it reaches the sandbox at `main`): IMAGE[,port=PORT][,env=K=V][,name=NAME] (repeatable; `port=` and `env=` repeat too). Local containers and every cloud sandbox; locally it needs `--runtime runc` where gVisor would run. With sidecars the service names main, sidecars and sc are reserved. |diff --git a/docs/content/docs/cua-sdk/reference/rust/cua-sandbox-core.mdx b/docs/content/docs/cua-sdk/reference/rust/cua-sandbox-core.mdxindex 3fcfa48739..7197e7574b 100644--- a/docs/content/docs/cua-sdk/reference/rust/cua-sandbox-core.mdx+++ b/docs/content/docs/cua-sdk/reference/rust/cua-sandbox-core.mdx@@ -1579,7 +1579,7 @@ pub const MACOS_FLEET_UNSUPPORTED: &str = /* ... */;pub const MISSING: &str = "missing";/// Image-string prefix naming a Fleet pool ...pub const POOL_PREFIX: &str = "pool:";-/// How long `Sandboxes::create` waits, at ...+/// How long `Sandboxes::create` waits for ...pub const SPACESD_READY_TIMEOUT: Duration = _;// mod byocdiff --git a/libs/cua/crates/cua-cli/src/sandbox.rs b/libs/cua/crates/cua-cli/src/sandbox.rsindex e3de454fe1..ba16f53b5e 100644--- a/libs/cua/crates/cua-cli/src/sandbox.rs+++ b/libs/cua/crates/cua-cli/src/sandbox.rs@@ -132,8 +132,9 @@ pub struct CreateArgs {/// input: display, window manager and cua-driver) (repeatable).#[arg(long = "wait", visible_alias = "wait-for", value_parser = parse_wait)]wait: Vec<(WaitTarget, Option<String>)>,- /// Readiness budget in seconds, the wait for the image's cua-spacesd- /// included (default 600, with at most 120 of it for cua-spacesd).+ /// Readiness budget in seconds, counted once the image is downloaded,+ /// the wait for the image's cua-spacesd included (default 600, plus up+ /// to 120 for cua-spacesd).#[arg(long)]ready_timeout: Option<u32>,/// Inject a freshly built binary once the sandbox is up:diff --git a/libs/cua/crates/cua-sandbox-core/src/sandbox.rs b/libs/cua/crates/cua-sandbox-core/src/sandbox.rsindex b0577a51f5..0f7d7c8d47 100644--- a/libs/cua/crates/cua-sandbox-core/src/sandbox.rs+++ b/libs/cua/crates/cua-sandbox-core/src/sandbox.rs@@ -40,8 +40,8 @@ use std::{};use tokio::net::{TcpListener, TcpStream};-/// How long [`Sandboxes::create`] waits, at most, for the cua-spacesd of an-/// image that declares one to answer `Health` (within `ready_timeout`) when+/// How long [`Sandboxes::create`] waits for the cua-spacesd of an image+/// that declares one to answer `Health`, once the guest's ports answer, when/// the caller did not choose the budget ([`CreateOptions::ready_timeout_given`]);/// a chosen `ready_timeout` governs that wait too.pub const SPACESD_READY_TIMEOUT: Duration = Duration::from_secs(120);@@ -245,7 +245,10 @@ pub struct CreateOptions {pub services: BTreeMap<String, u16>,/// Readiness probes.pub wait_for: Vec<Probe>,- /// Total readiness budget, the cua-spacesd wait included.+ /// Readiness budget, counted from when the image is present (the image+ /// download is not counted: it has its own stall detection). A chosen+ /// budget includes the cua-spacesd wait; otherwise that wait gets+ /// [`SPACESD_READY_TIMEOUT`] (at most this budget) on top.pub ready_timeout: Duration,/// Whether the caller chose `ready_timeout` (`cua sb create/// --ready-timeout`, the SDK's `ready_timeout_ms`). Chosen, it bounds@@ -1042,26 +1045,36 @@ impl Sandboxes {}_ => false,};- let sandbox = match options.provider {- ProviderKind::Fleet => self.create_fleet(&options).await?,- ProviderKind::Local => self.create_local(&options).await?,- ProviderKind::Contrib => self.create_contrib(&options).await?,- ProviderKind::Direct => {- return Err(Error::InvalidArgument(+ // The readiness budget starts once the image is present (the first+ // report after the pull): a large download has its own stall+ // detection and does not use up the time the guest has to boot.+ let (sandbox, image_ready) = crate::progress::past(Phase::Pulling, async {+ match options.provider {+ ProviderKind::Fleet => self.create_fleet(&options).await,+ ProviderKind::Local => self.create_local(&options).await,+ ProviderKind::Contrib => self.create_contrib(&options).await,+ ProviderKind::Direct => Err(Error::InvalidArgument("direct sandboxes are not created; use Sandboxes::connect_url".into(),- ));+ )),}- };- let remaining = options.ready_timeout.saturating_sub(started.elapsed());+ })+ .await;+ let sandbox = sandbox?;+ let clock = image_ready.unwrap_or(started);if !options.wait_for.is_empty() {report(Progress::phase(Phase::WaitingForServices));}- let mut ready = sandbox.wait_ready(&options.wait_for, remaining).await;+ let mut ready = sandbox+ .wait_ready(+ &options.wait_for,+ readiness_left(options.ready_timeout, clock),+ )+ .await;// The image declares cua-spacesd: ready includes it answering// `Health` (bounded), so the first guest call does not race its// start. Images that do not declare it are never waited on.if ready.is_ok() && sandbox.declares_spacesd(&options) {- let remaining = options.ready_timeout.saturating_sub(started.elapsed());+ let remaining = readiness_left(options.ready_timeout, clock);report(Progress::phase(Phase::WaitingForServices).detail("cua-spacesd"));ready = sandbox.wait_spacesd(spacesd_budget(&options, remaining))@@ -4043,14 +4056,21 @@ fn unpublished_port_message(name: &str, port: u16, why: &str) -> String {)}-/// How long create waits for cua-spacesd, given `remaining` of the-/// readiness budget: all of it when the caller chose the budget, else at-/// most [`SPACESD_READY_TIMEOUT`].+/// What is left of a `ready_timeout` budget that started at `clock`.+fn readiness_left(ready_timeout: Duration, clock: Instant) -> Duration {+ ready_timeout.saturating_sub(clock.elapsed())+}++/// How long create waits for cua-spacesd once the guest's ports answer,+/// given `remaining` of the readiness budget: all of it when the caller+/// chose the budget, else [`SPACESD_READY_TIMEOUT`] (never more than the+/// whole budget) however much is left, so a slow boot does not leave the+/// daemon no time to answer.fn spacesd_budget(options: &CreateOptions, remaining: Duration) -> Duration {if options.ready_timeout_given {remaining} else {- remaining.min(SPACESD_READY_TIMEOUT)+ SPACESD_READY_TIMEOUT.min(options.ready_timeout)}}@@ -4193,15 +4213,42 @@ mod placement_tests {let mut o = opts(ProviderKind::Local, "macos:26");let left = Duration::from_secs(500);assert_eq!(spacesd_budget(&o, left), SPACESD_READY_TIMEOUT);- assert_eq!(- spacesd_budget(&o, Duration::from_secs(30)),- Duration::from_secs(30)- );o.ready_timeout = Duration::from_secs(900);o.ready_timeout_given = true;assert_eq!(spacesd_budget(&o, left), left);}+ #[test]+ fn the_default_spacesd_wait_is_never_cut_short_by_a_slow_boot() {+ let o = opts(ProviderKind::Local, "macos:26");+ for left in [Duration::ZERO, Duration::from_secs(30)] {+ assert_eq!(spacesd_budget(&o, left), SPACESD_READY_TIMEOUT);+ }+ // A budget shorter than the spacesd wait still bounds it.+ let mut o = o;+ o.ready_timeout = Duration::from_secs(2);+ assert_eq!(spacesd_budget(&o, Duration::ZERO), Duration::from_secs(2));+ }++ #[test]+ fn the_readiness_budget_counts_from_its_clock() {+ let budget = Duration::from_secs(600);+ // The clock that starts once the image is present leaves nearly+ // all of it.+ let left = readiness_left(budget, Instant::now());+ assert!(left > Duration::from_secs(590), "{left:?}");+ // A clock that started 9 minutes earlier (the old start, before a+ // macOS image pull) leaves about a minute.+ if let Some(before_pull) = Instant::now().checked_sub(Duration::from_secs(540)) {+ let left = readiness_left(budget, before_pull);+ assert!(left <= Duration::from_secs(60), "{left:?}");+ }+ // Spent: nothing, never negative.+ if let Some(long_ago) = Instant::now().checked_sub(Duration::from_secs(900)) {+ assert_eq!(readiness_left(budget, long_ago), Duration::ZERO);+ }+ }+#[test]fn kind_and_runtime_fold_into_the_provider_options() {let mut o = opts(ProviderKind::Local, "python:3.12-slim");diff --git a/libs/cua/crates/cua-sandbox-core/tests/readiness_clock.rs b/libs/cua/crates/cua-sandbox-core/tests/readiness_clock.rsnew file mode 100644index 0000000000..2e32cb0d7e--- /dev/null+++ b/libs/cua/crates/cua-sandbox-core/tests/readiness_clock.rs@@ -0,0 +1,138 @@+//! The readiness budget of a create is counted from when the image is+//! present: a pull longer than the whole budget does not fail a guest that+//! then comes up in time, and a guest that does not come up in time still+//! fails. The runtime is a fake that reports its phases and says when its+//! guest port listens; nothing is pulled or booted.++use async_trait::async_trait;+use cua_sandbox_core::progress::{Phase, Progress, report};+use cua_sandbox_core::{+ CreateOptions, Error, InstanceStatus, LocalEndpoints, LocalInstance, LocalRuntime,+ LocalStartSpec, LocalSummary, Probe, ProviderKind, RuntimeError, RuntimeResult, Sandboxes,+};+use std::collections::HashMap;+use std::sync::{Arc, Mutex};+use std::time::{Duration, Instant};++const PORT: u16 = 8080;++/// "Pulls" for `pull`, then its guest port listens `boot` after the start+/// returns.+struct SlowPull {+ pull: Duration,+ boot: Duration,+ listening_at: Mutex<HashMap<String, Instant>>,+}++impl SlowPull {+ fn new(pull: Duration, boot: Duration) -> Arc<Self> {+ Arc::new(Self {+ pull,+ boot,+ listening_at: Mutex::default(),+ })+ }++ fn instance(name: &str) -> LocalInstance {+ LocalInstance {+ name: name.into(),+ backend: "fake".into(),+ status: InstanceStatus::Running,+ endpoints: LocalEndpoints {+ host: "127.0.0.1".into(),+ ports: [(PORT, PORT)].into(),+ ..Default::default()+ },+ }+ }+}++#[async_trait]+impl LocalRuntime for SlowPull {+ fn backend(&self) -> String {+ "fake".into()+ }+ async fn start(&self, spec: &LocalStartSpec) -> RuntimeResult<LocalInstance> {+ report(Progress::phase(Phase::Pulling));+ tokio::time::sleep(self.pull).await;+ report(Progress::phase(Phase::Creating));+ report(Progress::phase(Phase::Booting));+ self.listening_at+ .lock()+ .unwrap()+ .insert(spec.name.clone(), Instant::now() + self.boot);+ Ok(Self::instance(&spec.name))+ }+ async fn guest_tcp_listening(&self, name: &str, port: u16) -> RuntimeResult<Option<bool>> {+ let at = self.listening_at.lock().unwrap().get(name).copied();+ Ok(Some(+ port == PORT && at.is_some_and(|at| Instant::now() >= at),+ ))+ }+ async fn stop(&self, _: &str) -> RuntimeResult<()> {+ Ok(())+ }+ async fn resume(&self, name: &str) -> RuntimeResult<LocalInstance> {+ Ok(Self::instance(name))+ }+ async fn list(&self) -> RuntimeResult<Vec<LocalSummary>> {+ Ok(vec![])+ }+ async fn status(&self, name: &str) -> RuntimeResult<InstanceStatus> {+ if self.listening_at.lock().unwrap().contains_key(name) {+ Ok(InstanceStatus::Running)+ } else {+ Err(RuntimeError::NotFound(name.into()))+ }+ }+ async fn delete(&self, name: &str) -> RuntimeResult<()> {+ self.listening_at.lock().unwrap().remove(name);+ Ok(())+ }+ async fn endpoints(&self, _: &str) -> RuntimeResult<LocalEndpoints> {+ Ok(Self::instance("").endpoints)+ }+}++fn options(budget: Duration) -> CreateOptions {+ let mut o = CreateOptions::new(ProviderKind::Local, "docker.io/library/python:3.12-slim")+ .name("readiness-clock")+ .wait_for(Probe::Tcp(PORT));+ o.ports = vec![PORT];+ o.ready_timeout = budget;+ o.ready_timeout_given = true;+ o+}++#[tokio::test]+async fn a_pull_longer_than_the_budget_does_not_count_against_it() {+ let dir = tempfile::tempdir().unwrap();+ // The pull takes twice the budget; the guest then listens well within it.+ let rt = SlowPull::new(Duration::from_millis(1600), Duration::from_millis(300));+ let sbx = Sandboxes::builder().local(rt).state_dir(dir.path()).build();+ let sandbox = sbx+ .create(options(Duration::from_millis(800)))+ .await+ .expect("the guest came up within the budget once the image was present");+ assert_eq!(sandbox.name(), "readiness-clock");+}++#[tokio::test]+async fn a_guest_that_does_not_come_up_in_the_budget_still_fails() {+ let dir = tempfile::tempdir().unwrap();+ let rt = SlowPull::new(Duration::from_millis(50), Duration::from_secs(30));+ let sbx = Sandboxes::builder()+ .local(rt.clone())+ .state_dir(dir.path())+ .build();+ let started = Instant::now();+ let err = sbx+ .create(options(Duration::from_millis(600)))+ .await+ .expect_err("the port never listened within the budget");+ assert!(matches!(err, Error::Timeout(_)), "{err}");+ assert!(+ started.elapsed() < Duration::from_secs(10),+ "bounded by the budget"+ );+}diff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rsindex e41e531511..16dd82d132 100644--- a/libs/cua/crates/cua-spaces/src/spaces.rs+++ b/libs/cua/crates/cua-spaces/src/spaces.rs@@ -2168,14 +2168,16 @@ impl Spaces {if expect_env {create = create.wait_for(Probe::Tcp(cua_proto::SPACESD_DEFAULT_PORT));}- let started = tokio::time::Instant::now();+ let started = std::time::Instant::now();// Cancel-safe: dropped (a cancelled create), it stops and deletes// what it made (its own token never fires; the create's does).- let created = self- .inner- .sandboxes- .create_cancellable(create, cua_sandbox_core::CancellationToken::new())- .await;+ let (created, image_ready) = cua_sandbox_core::progress::past(+ cua_sandbox_core::progress::Phase::Pulling,+ self.inner+ .sandboxes+ .create_cancellable(create, cua_sandbox_core::CancellationToken::new()),+ )+ .await;if let Err(e) = created {// A start that failed after the instance existed (a readiness// probe that never passed) leaves a running VM no registry@@ -2197,14 +2199,7 @@ impl Spaces {token: Some(token),..Default::default()};- // A chosen timeout governs the whole wait; the default budget caps- // the handshake at SPACESD_READY_TIMEOUT.- let mut remaining = timeout- .saturating_sub(started.elapsed())- .max(Duration::from_secs(10));- if opts.timeout.is_none() {- remaining = remaining.min(cua_sandbox_core::SPACESD_READY_TIMEOUT);- }+ let remaining = connect_budget(opts.timeout, image_ready.unwrap_or(started));cua_sandbox_core::progress::report(cua_sandbox_core::progress::Progress::phase(cua_sandbox_core::progress::Phase::Connecting,));@@ -2799,6 +2794,20 @@ impl Drop for UndoOnDrop {}}+/// How long a new local Space's cua-spacesd has to accept the connection+/// once its sandbox is ready. A chosen `timeout` is counted from `clock`+/// (when the image was present), with at least 10 s left; the default gives+/// the handshake [`cua_sandbox_core::SPACESD_READY_TIMEOUT`] however long+/// the boot took.+fn connect_budget(timeout: Option<Duration>, clock: std::time::Instant) -> Duration {+ match timeout {+ Some(t) => t+ .saturating_sub(clock.elapsed())+ .max(Duration::from_secs(10)),+ None => cua_sandbox_core::SPACESD_READY_TIMEOUT,+ }+}+fn strip_runtime_prefix(image: &str) -> &str {for prefix in ["container:", "docker:", "vm:", "disk:", "lume:"] {if let Some(rest) = image.strip_prefix(prefix) {@@ -2897,6 +2906,31 @@ fn thumbnail_failure(f: &crate::thumbnails::CaptureFailure) -> Error {mod tests {use super::*;+ #[test]+ fn the_default_connect_budget_does_not_depend_on_the_boot() {+ let now = std::time::Instant::now();+ assert_eq!(+ connect_budget(None, now),+ cua_sandbox_core::SPACESD_READY_TIMEOUT+ );+ if let Some(long_ago) = now.checked_sub(Duration::from_secs(900)) {+ assert_eq!(+ connect_budget(None, long_ago),+ cua_sandbox_core::SPACESD_READY_TIMEOUT+ );+ }+ }++ #[test]+ fn a_chosen_connect_budget_counts_from_the_image_being_present() {+ let t = Duration::from_secs(600);+ let left = connect_budget(Some(t), std::time::Instant::now());+ assert!(left > Duration::from_secs(590), "{left:?}");+ if let Some(spent) = std::time::Instant::now().checked_sub(Duration::from_secs(900)) {+ assert_eq!(connect_budget(Some(t), spent), Duration::from_secs(10));+ }+ }+#[test]fn a_pinned_cloud_image_carries_its_digest_a_tag_none() {let d = format!("sha256:{}", "a".repeat(64));diff --git a/libs/cua/crates/cua-vmm/src/progress.rs b/libs/cua/crates/cua-vmm/src/progress.rsindex 4df4a2ae4f..cbb258357a 100644--- a/libs/cua/crates/cua-vmm/src/progress.rs+++ b/libs/cua/crates/cua-vmm/src/progress.rs@@ -245,6 +245,30 @@ pub fn report(progress: Progress) {let _ = SINK.try_with(|sink| sink(&progress));}+/// Runs `fut`, passing its reports on to the current scope's sink, and+/// returns when it first reported a phase after `phase` (`None`: it never+/// did). A create's readiness budget is measured from the first report+/// after [`Phase::Pulling`], so the image download does not count against+/// it.+pub async fn past<F: Future>(phase: Phase, fut: F) -> (F::Output, Option<std::time::Instant>) {+ let seen = Arc::new(std::sync::Mutex::new(None));+ let outer = current();+ let mark = seen.clone();+ let sink: Sink = Arc::new(move |p: &Progress| {+ if p.phase.rank() > phase.rank() {+ mark.lock()+ .unwrap_or_else(|e| e.into_inner())+ .get_or_insert_with(std::time::Instant::now);+ }+ if let Some(outer) = &outer {+ outer(p);+ }+ });+ let out = scope(sink, fut).await;+ let at = *seen.lock().unwrap_or_else(|e| e.into_inner());+ (out, at)+}+/// Turns a stream of byte counts into [`Transfer`] reports: at most one per/// [`Meter::INTERVAL`] (the first and the last always pass), with a rate/// smoothed over the recent samples so the time left does not jump around.@@ -424,6 +448,52 @@ mod tests {assert_eq!(seen[0].fraction, Some(0.5));}+ #[tokio::test]+ async fn past_marks_the_first_later_phase_and_forwards_everything() {+ let seen = Arc::new(Mutex::new(Vec::new()));+ let s = seen.clone();+ let sink: Sink = Arc::new(move |p: &Progress| s.lock().unwrap().push(p.phase));+ let (before, after, at) = scope(sink, async {+ let before = std::time::Instant::now();+ let ((), at) = past(Phase::Pulling, async {+ report(Progress::phase(Phase::Preparing));+ report(Progress::phase(Phase::Pulling));+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;+ report(Progress::phase(Phase::Creating));+ report(Progress::phase(Phase::Booting));+ })+ .await;+ (before, std::time::Instant::now(), at)+ })+ .await;+ let at = at.expect("a phase after pulling was reported");+ assert!(+ at >= before + std::time::Duration::from_millis(20),+ "after the pull"+ );+ assert!(at <= after);+ assert_eq!(+ *seen.lock().unwrap(),+ [+ Phase::Preparing,+ Phase::Pulling,+ Phase::Creating,+ Phase::Booting+ ],+ "the outer scope still sees every report"+ );+ }++ #[tokio::test]+ async fn past_is_none_without_a_later_phase_and_works_outside_a_scope() {+ let ((), at) = past(Phase::Pulling, async {+ report(Progress::phase(Phase::Preparing));+ report(Progress::phase(Phase::Pulling));+ })+ .await;+ assert_eq!(at, None);+ }+#[test]fn bytes_fold_layers() {let mut b = Bytes::default();diff --git a/scripts/docs-generators/cli-specs/cua.json b/scripts/docs-generators/cli-specs/cua.jsonindex 80cc469cea..b9bad0cfb1 100644--- a/scripts/docs-generators/cli-specs/cua.json+++ b/scripts/docs-generators/cli-specs/cua.json@@ -2420,7 +2420,7 @@"aliases": [],"default_value": null,"env": null,- "help": "Readiness budget in seconds, the wait for the image's cua-spacesd included (default 600, with at most 120 of it for cua-spacesd)",+ "help": "Readiness budget in seconds, counted once the image is downloaded, the wait for the image's cua-spacesd included (default 600, plus up to 120 for cua-spacesd)","hidden": false,"is_optional": true,"multiple_values": false,