Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-cli): bound process start and Space lookup so a silent relay stream fails instead of hanging (#6601)

injaneityc6129be4b8ed
diff --git a/docs/content/docs/cua-sdk/reference/rust/cua-spacesd-client.mdx b/docs/content/docs/cua-sdk/reference/rust/cua-spacesd-client.mdx
index a799eac6d2..c0e422a108 100644
--- a/docs/content/docs/cua-sdk/reference/rust/cua-spacesd-client.mdx
+++ b/docs/content/docs/cua-sdk/reference/rust/cua-spacesd-client.mdx
@@ -98,6 +98,7 @@ pub struct ConnectOptions {
pub gateway: Option<GatewayAuth>,
pub connect_timeout: Duration,
pub probe_timeout: Duration,
+ pub start_timeout: Duration,
pub keepalive: Keepalive,
pub retry: RetryPolicy,
pub probe: bool,
@@ -113,6 +114,7 @@ impl ConnectOptions {
fn retry(self, retry: RetryPolicy) -> Self;
fn keepalive(self, keepalive: Keepalive) -> Self;
fn probe_timeout(self, timeout: Duration) -> Self;
+ fn start_timeout(self, timeout: Duration) -> Self;
fn header(self, name: impl Into<String>, value: impl Into<String>) -> Self;
fn probe(self, probe: bool) -> Self;
}
@@ -1374,6 +1376,8 @@ pub const MAX_HTTP_REQUEST_BYTES: usize = _;
pub const MAX_MESSAGE_BYTES: usize = _;
/// Capability: `ProcessConfig.user` (and ...
pub const PROCESS_RUN_AS_FEATURE: &str = "process.run_as";
+/// How long a start waits for ...
+pub const PROCESS_START_DEADLINE: Duration = _;
/// Capability: file calls accept `run_as` ...
pub const RUN_AS_FEATURE: &str = "fs.run_as";
/// Capability name a spacesd advertises ...
diff --git a/libs/cua/crates/cua-cli/src/sandbox.rs b/libs/cua/crates/cua-cli/src/sandbox.rs
index 2978da2acc..470a1c976a 100644
--- a/libs/cua/crates/cua-cli/src/sandbox.rs
+++ b/libs/cua/crates/cua-cli/src/sandbox.rs
@@ -2847,6 +2847,27 @@ where
}
}
+/// How long finding and dialing a Space may take in all. Longer than
+/// [`RELAY_START_WAIT`], so a Space that is still starting says so first.
+const SPACE_LOOKUP_DEADLINE: std::time::Duration = std::time::Duration::from_secs(60);
+
+/// `lookup` within `budget`, or a timeout that names the Space: a relay
+/// stream that goes silent fails instead of hanging the command.
+async fn within_lookup_deadline<T>(
+ name: &str,
+ budget: std::time::Duration,
+ lookup: impl std::future::Future<Output = Result<T, CuaError>>,
+) -> Result<T, CuaError> {
+ tokio::time::timeout(budget, lookup)
+ .await
+ .unwrap_or_else(|_| {
+ Err(CuaError::Timeout(format!(
+ "{name} did not answer within {}s; check that it is online: cua spaces ls",
+ budget.as_secs()
+ )))
+ })
+}
+
/// A spacesd client for a named sandbox or a relay Space. Direct sandboxes
/// use the remembered token (or `CUA_ENV_TOKEN`). A relay Space opens
/// through the Spaces API; the sandboxes API does not attach those machines.
@@ -2878,19 +2899,23 @@ pub async fn env_of(cua: &Cua, name: &str) -> Result<Arc<cua_sdk::SpacesdClient>
crate::host::daemon_relay_account(&crate::util::cua_home())
});
let spaces = cua.spaces();
- // NotFound before dial, and the snapshot has the name space() dials with.
- spaces.resolve(id.clone()).await?;
- // Right after `cua spaces start` the machine is listed but its
- // spacesd is not connected yet: wait for it (bounded), then say so.
- return wait_while_starting(
- &id,
- RELAY_START_WAIT,
- std::time::Duration::from_millis(500),
- || async { spaces.space(id.clone()).await?.spacesd() },
- )
+ return within_lookup_deadline(&id, SPACE_LOOKUP_DEADLINE, async {
+ // NotFound before dial, and the snapshot has the name space() dials with.
+ spaces.resolve(id.clone()).await?;
+ // Right after `cua spaces start` the machine is listed but its
+ // spacesd is not connected yet: wait for it (bounded), then say so.
+ wait_while_starting(
+ &id,
+ RELAY_START_WAIT,
+ std::time::Duration::from_millis(500),
+ || async { spaces.space(id.clone()).await?.spacesd() },
+ )
+ .await
+ })
.await;
}
- if let Some(space) = space_for(cua, name).await? {
+ let space = within_lookup_deadline(name, SPACE_LOOKUP_DEADLINE, space_for(cua, name)).await?;
+ if let Some(space) = space {
return space.spacesd();
}
let sbx = cua.sandboxes();
@@ -4377,6 +4402,28 @@ mod tests {
assert!(matches!(e, CuaError::NotFound(_)));
}
+ #[tokio::test]
+ async fn a_space_lookup_that_never_answers_times_out_with_its_name() {
+ let err = within_lookup_deadline::<()>(
+ "relay:a",
+ std::time::Duration::from_millis(20),
+ std::future::pending(),
+ )
+ .await
+ .unwrap_err();
+ assert!(matches!(err, CuaError::Timeout(_)), "{err:?}");
+ assert!(err.to_string().contains("relay:a did not answer"), "{err}");
+ // An answer within the deadline, error or not, passes through.
+ let e =
+ within_lookup_deadline::<()>("relay:a", std::time::Duration::from_secs(30), async {
+ Err(CuaError::NotFound("gone".into()))
+ })
+ .await
+ .unwrap_err();
+ assert!(matches!(e, CuaError::NotFound(_)));
+ assert!(SPACE_LOOKUP_DEADLINE > RELAY_START_WAIT);
+ }
+
#[test]
fn exec_takes_a_time_limit_before_the_command() {
let exec = |args: &[&str]| {
diff --git a/libs/cua/crates/cua-spacesd-client/src/client.rs b/libs/cua/crates/cua-spacesd-client/src/client.rs
index b66a8302f8..32965bb40d 100644
--- a/libs/cua/crates/cua-spacesd-client/src/client.rs
+++ b/libs/cua/crates/cua-spacesd-client/src/client.rs
@@ -108,6 +108,9 @@ pub struct ConnectOptions {
pub connect_timeout: Duration,
/// Timeout of the `GetCapabilities` probe.
pub probe_timeout: Duration,
+ /// How long a process start waits for `ProcessStart`
+ /// ([`crate::process::PROCESS_START_DEADLINE`] by default).
+ pub start_timeout: Duration,
/// Connection keepalive.
pub keepalive: Keepalive,
/// Reconnect / retry policy.
@@ -131,6 +134,7 @@ impl ConnectOptions {
gateway: None,
connect_timeout: Duration::from_secs(10),
probe_timeout: Duration::from_secs(15),
+ start_timeout: crate::process::PROCESS_START_DEADLINE,
keepalive: Keepalive::default(),
retry: RetryPolicy::default(),
probe: true,
@@ -186,6 +190,12 @@ impl ConnectOptions {
self
}
+ /// Sets how long a process start waits for `ProcessStart`.
+ pub fn start_timeout(mut self, timeout: Duration) -> Self {
+ self.start_timeout = timeout;
+ self
+ }
+
/// Adds a header sent on every request.
pub fn header(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.headers.push((name.into(), value.into()));
diff --git a/libs/cua/crates/cua-spacesd-client/src/lib.rs b/libs/cua/crates/cua-spacesd-client/src/lib.rs
index ffc262023c..8b33cdeca8 100644
--- a/libs/cua/crates/cua-spacesd-client/src/lib.rs
+++ b/libs/cua/crates/cua-spacesd-client/src/lib.rs
@@ -67,8 +67,8 @@ pub use http::{
DEFAULT_HTTP_RESPONSE_LIMIT, DEFAULT_HTTP_TIMEOUT, HttpCall, HttpReply, MAX_HTTP_REQUEST_BYTES,
};
pub use process::{
- Command, DEFAULT_PROCESS_KEEPALIVE, ExitStatus, Output, PROCESS_RUN_AS_FEATURE, ProcessEvent,
- ProcessHandle, ProcessRef, Replay, RunAsMode,
+ Command, DEFAULT_PROCESS_KEEPALIVE, ExitStatus, Output, PROCESS_RUN_AS_FEATURE,
+ PROCESS_START_DEADLINE, ProcessEvent, ProcessHandle, ProcessRef, Replay, RunAsMode,
};
pub use record::{RecordOptions, Recorded, ScreenRecording, find_ffmpeg};
pub use transport::{
diff --git a/libs/cua/crates/cua-spacesd-client/src/process.rs b/libs/cua/crates/cua-spacesd-client/src/process.rs
index db86621e13..c504be3e32 100644
--- a/libs/cua/crates/cua-spacesd-client/src/process.rs
+++ b/libs/cua/crates/cua-spacesd-client/src/process.rs
@@ -23,6 +23,11 @@ use tonic::Streaming;
/// Default server-side keepalive interval requested for process streams.
pub const DEFAULT_PROCESS_KEEPALIVE: Duration = Duration::from_secs(15);
+/// How long a start waits for `ProcessStart` (or the start's failure). A
+/// stream that stays silent past it fails with [`Error::Timeout`] instead of
+/// hanging: the server answers a start at once, so silence means the link
+/// or the Space is gone.
+pub const PROCESS_START_DEADLINE: Duration = Duration::from_secs(30);
/// Read-ahead bound: messages already received are taken off the HTTP/2
/// connection up to this many bytes, so a caller that consumes events more
@@ -952,12 +957,18 @@ impl SpacesdClient {
async fn start(&self, cmd: &Command) -> Result<ProcessHandle> {
let cmd = cmd.clone();
let keepalive = cmd.keepalive.unwrap_or(DEFAULT_PROCESS_KEEPALIVE);
- let mut stream = self
- .process()
- .start_process(cmd.to_request())
- .await?
- .into_inner();
- let first = stream.message().await?;
+ let deadline = self.inner.options.start_timeout;
+ let (stream, first) = tokio::time::timeout(deadline, async {
+ let mut stream = self
+ .process()
+ .start_process(cmd.to_request())
+ .await?
+ .into_inner();
+ let first = stream.message().await?;
+ Ok::<_, Error>((stream, first))
+ })
+ .await
+ .map_err(|_| Error::Timeout(deadline))??;
let start = match first.and_then(|m| m.event).and_then(|e| e.event) {
Some(Event::Start(s)) => s,
Some(Event::End(end)) => {
diff --git a/libs/cua/crates/cua-spacesd-client/src/testing.rs b/libs/cua/crates/cua-spacesd-client/src/testing.rs
index 18f0ded523..d6f913060f 100644
--- a/libs/cua/crates/cua-spacesd-client/src/testing.rs
+++ b/libs/cua/crates/cua-spacesd-client/src/testing.rs
@@ -313,6 +313,9 @@ pub struct MockState {
/// Report pointer input as delivered in the foreground without moving
/// the pointer (input that reached nothing).
pub pointer_not_moved: AtomicBool,
+ /// Accept `StartProcess` and never send a message on its stream (a
+ /// relay stream that went silent).
+ pub silent_start: AtomicBool,
/// The `desktop` component `Health` reports (none: an older spacesd).
pub desktop_health: Mutex<Option<pb::ComponentHealth>>,
/// Extra supported features reported by `GetCapabilities` (for clients
@@ -1593,6 +1596,11 @@ impl ProcessService for Svc {
) -> Result<tonic::Response<Self::StartProcessStream>, Status> {
let req = req.into_inner();
self.0.started.lock().unwrap().push(req.clone());
+ if self.0.silent_start.load(Ordering::Relaxed) {
+ return Ok(tonic::Response::new(Box::pin(
+ futures_util::stream::pending(),
+ )));
+ }
let cfg = req.config.clone().unwrap_or_default();
if cfg.command == "sudo" {
self.0.sudo_runs.fetch_add(1, Ordering::Relaxed);
diff --git a/libs/cua/crates/cua-spacesd-client/tests/client.rs b/libs/cua/crates/cua-spacesd-client/tests/client.rs
index 6a7ceb053d..9440ac51bb 100644
--- a/libs/cua/crates/cua-spacesd-client/tests/client.rs
+++ b/libs/cua/crates/cua-spacesd-client/tests/client.rs
@@ -103,6 +103,7 @@ both_transports!(
capabilities_and_transport,
run_collects_output_and_exit_codes,
run_timeout,
+ a_silent_start_times_out,
spawn_reattach_by_tag_with_scrollback,
stdin_roundtrip,
small_upload_download_and_typed_errors,
@@ -164,6 +165,32 @@ async fn run_timeout(t: Transport) {
assert!(out.status.timed_out);
}
+/// A start whose stream never answers (a relay that went quiet) fails with
+/// a timeout instead of hanging, with or without a command timeout.
+async fn a_silent_start_times_out(t: Transport) {
+ let srv = server().await;
+ let opts = ConnectOptions::parse(&srv.url())
+ .unwrap()
+ .token(TOKEN)
+ .transport(pref(t))
+ .start_timeout(Duration::from_millis(300));
+ let env = SpacesdClient::connect(opts).await.expect("connect");
+ srv.state.silent_start.store(true, Ordering::Relaxed);
+ let started = Instant::now();
+ let err = env.run(Command::shell("echo hi")).await.unwrap_err();
+ assert!(
+ matches!(err, Error::Timeout(d) if d == Duration::from_millis(300)),
+ "{err:?}"
+ );
+ let err = env.spawn(Command::shell("echo hi")).await.unwrap_err();
+ assert!(matches!(err, Error::Timeout(_)), "{err:?}");
+ assert!(started.elapsed() < Duration::from_secs(10));
+ // The deadline covers the start only: a reply that comes ends normally.
+ srv.state.silent_start.store(false, Ordering::Relaxed);
+ let out = env.run(Command::shell("echo hi")).await.unwrap();
+ assert_eq!(out.stdout_str(), "hi\n");
+}
+
async fn spawn_reattach_by_tag_with_scrollback(t: Transport) {
let srv = server().await;
let env = connect_to(&srv.url(), t).await;