Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(e2e): Python/TS fork-batch no longer writes to the real ~/.cua; batched forks keep their first error (CUA-3092) (#6312)

cua-agents[bot]662432d9ecdd
diff --git a/libs/cua/crates/cua-spaces/src/fork_batch.rs b/libs/cua/crates/cua-spaces/src/fork_batch.rs
index 59cf153078..a304128b27 100644
--- a/libs/cua/crates/cua-spaces/src/fork_batch.rs
+++ b/libs/cua/crates/cua-spaces/src/fork_batch.rs
@@ -52,6 +52,15 @@ fn no_host_message(reason: &str, cpus: u32) -> String {
m
}
+/// `why`, followed by why the fork's earlier attempts failed: a retry that
+/// finds no machine left would otherwise hide the error that started it.
+fn with_earlier(why: String, earlier: &[String]) -> String {
+ if earlier.is_empty() {
+ return why;
+ }
+ format!("{why} (earlier attempts: {})", earlier.join("; "))
+}
+
/// The machine the batch runs on, in reports.
pub const THIS_MACHINE: &str = "local";
@@ -314,6 +323,9 @@ struct Item {
source: String,
name: String,
attempt: u32,
+ /// Why each earlier attempt failed (`host: error`), so a fork that
+ /// ends with no machine left still says what went wrong first.
+ earlier: Vec<String>,
}
/// A step done once per batch (a checkpoint's copy to a machine, its
@@ -379,6 +391,7 @@ pub async fn run(
source,
name,
attempt: 1,
+ earlier: Vec::new(),
})
.collect();
let mut ok = 0u32;
@@ -392,7 +405,7 @@ pub async fn run(
source: it.source.clone(),
name: it.name.clone(),
host: host.to_string(),
- error: why,
+ error: with_earlier(why, &it.earlier),
error_code: f.code().into(),
attempt: it.attempt,
elapsed_ms: started.elapsed().as_millis() as u64,
@@ -594,8 +607,11 @@ pub async fn run(
}
if f.retried() && round <= spec.retries {
tracing::info!(fork = %it.name, %host, "placing the fork again: {why}");
+ let mut earlier = it.earlier;
+ earlier.push(format!("{host}: {why}"));
retry.push(Item {
attempt: it.attempt + 1,
+ earlier,
..it
});
} else {
@@ -1380,6 +1396,30 @@ mod tests {
assert!(its.iter().all(|i| i.error_code == "host_failed"), "{its:?}");
}
+ /// The only machine fails its forks: the retry finds no machine left,
+ /// and each fork's error still says why the first attempt failed.
+ #[tokio::test]
+ async fn the_first_error_survives_a_retry_with_no_machine_left() {
+ let only = Arc::new(Fake {
+ name: "mac-b".into(),
+ holds: vec!["w0".into()],
+ mem_gb: 64,
+ dies_after: Some(0),
+ ..Default::default()
+ });
+ let (sum, events, _) = go(vec![only], spec(&["w0"], 2)).await;
+ assert_eq!((sum.ok, sum.failed), (0, 2), "{sum:?}");
+ for i in items(&events) {
+ assert_eq!(i.error_code, "no_host", "{i:?}");
+ assert!(i.error.contains("mac-b is not ready"), "{i:?}");
+ assert!(
+ i.error.contains("earlier attempts: mac-b:")
+ && i.error.contains("connection refused"),
+ "{i:?}"
+ );
+ }
+ }
+
/// One fork that fails on its own is reported alone; the rest of the
/// batch is ready.
#[tokio::test]
diff --git a/libs/cua/crates/cua-spaces/src/spaces.rs b/libs/cua/crates/cua-spaces/src/spaces.rs
index fd75977ca1..5495efcbd3 100644
--- a/libs/cua/crates/cua-spaces/src/spaces.rs
+++ b/libs/cua/crates/cua-spaces/src/spaces.rs
@@ -2977,11 +2977,17 @@ impl Spaces {
return Err(e);
}
};
- self.store(&space, credential)?;
- if let Some(config) = built_from.as_deref().and_then(|b| self.build_config(b)) {
- space.set_image_config(config.clone());
- self.record_image_config(&id.to_string(), &config)?;
- }
+ // The fork is booted: a failure to register it must not leave it
+ // running for a retry to duplicate.
+ let registered = (|| -> Result<()> {
+ self.store(&space, credential)?;
+ if let Some(config) = built_from.as_deref().and_then(|b| self.build_config(b)) {
+ space.set_image_config(config.clone());
+ self.record_image_config(&id.to_string(), &config)?;
+ }
+ Ok(())
+ })();
+ teardown_on_error(registered, "fork", &name, sandbox.delete()).await?;
// A fork starts with its source Space's labels and egress policy (a
// checkpoint source has none).
let _ = self.inherit_meta(
@@ -3867,8 +3873,55 @@ fn refuse_windows_on_mac(image: Option<&str>, host_is_mac: bool) -> Result<()> {
Ok(())
}
+/// Passes `result` through; on an error runs `teardown` (best effort),
+/// logging its failure without replacing the original error.
+pub(crate) async fn teardown_on_error<T, E, D>(
+ result: Result<T>,
+ what: &str,
+ name: &str,
+ teardown: D,
+) -> Result<T>
+where
+ E: std::fmt::Display,
+ D: std::future::Future<Output = std::result::Result<(), E>>,
+{
+ if result.is_err()
+ && let Err(e) = teardown.await
+ {
+ tracing::warn!(error = %e, %what, %name, "teardown after failed registration also failed");
+ }
+ result
+}
+
#[cfg(test)]
mod tests {
+ #[tokio::test]
+ async fn a_failed_registration_tears_the_fork_down_and_keeps_its_error() {
+ use std::sync::atomic::{AtomicU32, Ordering::SeqCst};
+ let deleted = AtomicU32::new(0);
+ let td = |fail: bool| {
+ let deleted = &deleted;
+ async move {
+ deleted.fetch_add(1, SeqCst);
+ if fail { Err("delete refused") } else { Ok(()) }
+ }
+ };
+ let ok = super::teardown_on_error(Ok(7), "fork", "f", td(false)).await;
+ assert_eq!(ok.unwrap(), 7);
+ assert_eq!(deleted.load(SeqCst), 0, "a registered fork stays");
+ for fail in [false, true] {
+ let err = super::teardown_on_error::<(), _, _>(
+ Err(Error::invalid("register failed")),
+ "fork",
+ "f",
+ td(fail),
+ )
+ .await
+ .unwrap_err();
+ assert!(err.to_string().contains("register failed"), "{err}");
+ }
+ assert_eq!(deleted.load(SeqCst), 2);
+ }
#[test]
fn only_an_unplaced_create_overflows_into_your_clouds() {
diff --git a/tests/e2e/cua-sdk/python/conftest.py b/tests/e2e/cua-sdk/python/conftest.py
index 4341b9607b..66a9dc315c 100644
--- a/tests/e2e/cua-sdk/python/conftest.py
+++ b/tests/e2e/cua-sdk/python/conftest.py
@@ -160,4 +160,8 @@ def fixtures():
def local_cua(tmp_path):
import cua
- return cua.embedded(state_dir=str(tmp_path / "state"))
+ # Its own Spaces registry too: the default is the real ~/.cua, which the
+ # cua-home guard refuses in a test (a fork's registration fails there).
+ return cua.embedded(
+ state_dir=str(tmp_path / "state"), spaces_home=str(tmp_path / "spaces")
+ )
diff --git a/tests/e2e/cua-sdk/python/test_daemon_vs_embedded.py b/tests/e2e/cua-sdk/python/test_daemon_vs_embedded.py
index 19b530d435..8564a12317 100644
--- a/tests/e2e/cua-sdk/python/test_daemon_vs_embedded.py
+++ b/tests/e2e/cua-sdk/python/test_daemon_vs_embedded.py
@@ -133,7 +133,11 @@ def _shared_sandbox(daemon: e2e.Daemon, target: dict) -> None:
def _compare(target: dict, tmp_path) -> None:
embedded = e2e.run_async(
- script(cua.embedded(state_dir=str(tmp_path / "emb")), target, "emb"),
+ script(
+ cua.embedded(state_dir=str(tmp_path / "emb"), spaces_home=str(tmp_path / "emb-spaces")),
+ target,
+ "emb",
+ ),
timeout=900,
)
with e2e.cua_daemon() as d:
diff --git a/tests/e2e/cua-sdk/python/test_desktop.py b/tests/e2e/cua-sdk/python/test_desktop.py
index e146bc4840..92ac088ead 100644
--- a/tests/e2e/cua-sdk/python/test_desktop.py
+++ b/tests/e2e/cua-sdk/python/test_desktop.py
@@ -35,7 +35,10 @@ class Desktop:
@pytest.fixture(scope="module")
def local_desktop(tmp_path_factory):
e2e.require_image(e2e.desktop_image())
- c = cua.embedded(state_dir=str(tmp_path_factory.mktemp("desk")))
+ c = cua.embedded(
+ state_dir=str(tmp_path_factory.mktemp("desk")),
+ spaces_home=str(tmp_path_factory.mktemp("desk-spaces")),
+ )
token = secrets.token_hex(16)
sb = e2e.run_async(
c.sandboxes().create(
diff --git a/tests/e2e/cua-sdk/python/test_images.py b/tests/e2e/cua-sdk/python/test_images.py
index 8fe0cf3dcd..630c17dbd8 100644
--- a/tests/e2e/cua-sdk/python/test_images.py
+++ b/tests/e2e/cua-sdk/python/test_images.py
@@ -219,7 +219,9 @@ def test_cli_build_push_boot_vm(local_cua, tmp_path):
# A fresh runtime: the QEMU disk resolver reads CUA_INSECURE_REGISTRIES
# when the runtime is created, not per pull.
- fresh = cua.embedded(state_dir=str(tmp_path / "boot-state"))
+ fresh = cua.embedded(
+ state_dir=str(tmp_path / "boot-state"), spaces_home=str(tmp_path / "boot-spaces")
+ )
async def boot():
sb = await fresh.sandboxes().create(
diff --git a/tests/e2e/cua-sdk/typescript/lib.mjs b/tests/e2e/cua-sdk/typescript/lib.mjs
index f763c81433..e7169b0b62 100644
--- a/tests/e2e/cua-sdk/typescript/lib.mjs
+++ b/tests/e2e/cua-sdk/typescript/lib.mjs
@@ -460,6 +460,9 @@ export async function mcpInitialize(addr, token) {
return msg.result
}
-export const embeddedLocal = (dir) => cua.embedded({ stateDir: dir })
+// Its own Spaces registry too: the default is the real ~/.cua, which the
+// cua-home guard refuses in a test (a fork's registration fails there).
+export const embeddedLocal = (dir) => cua.embedded({ stateDir: dir, spacesHome: tmpSpaces() })
export const tmpState = () => mkdtempSync(join(process.env.TMPDIR ?? "/tmp", `cua-e2e-${RUN}-state-`))
+export const tmpSpaces = () => mkdtempSync(join(process.env.TMPDIR ?? "/tmp", `cua-e2e-${RUN}-spaces-`))
export { execFileSync }
diff --git a/tests/e2e/cua-sdk/typescript/unified.test.mjs b/tests/e2e/cua-sdk/typescript/unified.test.mjs
index d335319b10..85e235f63d 100644
--- a/tests/e2e/cua-sdk/typescript/unified.test.mjs
+++ b/tests/e2e/cua-sdk/typescript/unified.test.mjs
@@ -72,7 +72,7 @@ async function urlsAndForward(sb, port) {
}
e.e2eTest("unified-sandbox-api", "hermetic", "cloud create fails (Cua Cloud has ended) + daemon public URL", async () => {
- const c = cua.embedded({ stateDir: e.tmpState() })
+ const c = e.embeddedLocal(e.tmpState())
// A cloud create (command and services) fails before anything is created.
await e.assertCloudEnded(c, {
command: ["python", "/srv.py"],