Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(relay,spacesd): a removed computer deletes its Spaces when it next comes online (CUA-3479) (#6532)
cua-agents[bot]e7e78c8217b1
diff --git a/apps/cua-spaces-web/src/__tests__/spaces-tree.test.tsx b/apps/cua-spaces-web/src/__tests__/spaces-tree.test.tsxindex 9d5f1eb12d..22eb05ded5 100644--- a/apps/cua-spaces-web/src/__tests__/spaces-tree.test.tsx+++ b/apps/cua-spaces-web/src/__tests__/spaces-tree.test.tsx@@ -191,17 +191,15 @@ describe.skipIf(!wasmBuilt)("Spaces as a tree (Labs)", () => {expect(calls.some(([op]) => op === "spaces.delete")).toBe(false);expect(calls.find(([op]) => op === "machines.remove")![1]).not.toHaveProperty("deleteSpaces", true);- // Ticked: each Space is deleted through the computer first, then the computer goes with them.+ // Ticked: the computer goes with its Spaces in one call; the relay keeps them for the computer to delete when it's back.calls.length = 0;dialog = await open();fireEvent.click(q("[data-tree-also-delete]")!);- // Ticked, it says what really happens: the Spaces leave the account, their files stay on the offline computer.- expect(dialog.textContent).toContain("The 2 Spaces are deleted from your account and won't come back. Their files stay on Linux box, which is offline, until you delete them there.");+ expect(dialog.textContent).toContain("The 2 Spaces leave your account now and won't come back. Linux box deletes them, with their files, the next time it comes online.");expect(q("[data-tree-remove-confirm]")?.textContent).toBe("Remove and delete");fireEvent.click(q("[data-tree-remove-confirm]")!);- await waitFor(() => expect(calls.map(([op]) => op)).toEqual(["spaces.delete", "spaces.delete", "machines.remove"]));- expect(calls.slice(0, 2).map(([, a]) => a.spaceId).sort()).toEqual(["relay:linux-box/ubuntu-build", "relay:linux-box/windows-legacy"]);- expect(calls[2]![1]).toMatchObject({ machine: "linux-box", deleteSpaces: true });+ await waitFor(() => expect(calls.map(([op]) => op)).toEqual(["machines.remove"]));+ expect(calls[0]![1]).toMatchObject({ machine: "linux-box", deleteSpaces: true });});});diff --git a/apps/cua-spaces-web/src/components/spaces/spaces-tree-page.tsx b/apps/cua-spaces-web/src/components/spaces/spaces-tree-page.tsxindex bb4dbaf09a..833731f729 100644--- a/apps/cua-spaces-web/src/components/spaces/spaces-tree-page.tsx+++ b/apps/cua-spaces-web/src/components/spaces/spaces-tree-page.tsx@@ -49,13 +49,16 @@ export function removeText(? "It leaves your Spaces list, and your team can't use it any more.": `It leaves your Spaces list, and your team can't use it any more. The ${plural(n, "Space")} made on it leave${n === 1 ? "s" : ""} the list with it:`;const back = n === 0 ? "" : n === 1 ? " Add it again and it comes back." : ` Add it again and all ${n} come back.`;- // Stopgap (CUA-3413): a computer can only be removed while it is offline, so its Spaces can't be deleted on it. Ticked,- // they leave your account and their files stay on the computer until it deletes them. Say exactly that.- const them = n === 1 ? "Its files stay" : "Their files stay";+ // A computer is removed while it is offline (CUA-3479): ticked, its Spaces leave the account now, and the relay keeps+ // them as pending deletions that the computer carries out, files included, the next time it comes online.+ const deleted =+ n === 1+ ? `The Space leaves your account now and won't come back. ${node.name} deletes it, with its files, the next time it comes online.`+ : `The ${n} Spaces leave your account now and won't come back. ${node.name} deletes them, with their files, the next time it comes online.`;return {title: `Remove ${node.name}?`,lead,- keep: alsoDelete ? `${n === 1 ? "The Space is" : `The ${n} Spaces are`} deleted from your account and won't come back. ${them} on ${node.name}, which is offline, until you delete them there.` : `Nothing is deleted from ${node.name}.${back}`,+ keep: alsoDelete ? deleted : `Nothing is deleted from ${node.name}.${back}`,box: n === 0 ? null : `Also delete the ${n === 1 ? "Space" : `${n} Spaces`}`,confirm: alsoDelete ? `Remove and delete` : `Remove ${node.name}`,};@@ -293,11 +296,11 @@ export function SpacesTreePage() {/*** f9: removing a computer names each Space made on it and any task that- * stops. They are only detached unless "Also delete" is ticked: then each is- * deleted through the computer first, and the relay forgets them with it.+ * stops. They are only detached unless "Also delete" is ticked: then the+ * relay forgets them at once and the computer deletes them, files included,+ * the next time it comes online (pending deletions, CUA-3479).*/export function RemoveComputerDialog({ node, onClose }: { node: TreeNode; onClose: () => void }) {- const { deleteSpace } = useSpaces();const machines = useMachines();const [alsoDelete, setAlsoDelete] = useState(false);const [busy, setBusy] = useState(false);@@ -306,18 +309,6 @@ export function RemoveComputerDialog({ node, onClose }: { node: TreeNode; onClosconst run = async () => {setBusy(true);try {- if (alsoDelete) {- for (const c of node.children) {- if (!c.space) continue;- try {- await deleteSpace(c.space.id);- } catch (e) {- // A Space that couldn't be deleted stops here: the computer stays, so nothing is half-removed.- toastError(`Couldn't delete ${c.name}, so ${node.name} wasn't removed`)(e);- return;- }- }- }await machines.remove(r?.machine ?? node.machine!.id, r?.device ?? null, alsoDelete);toast(`Removed ${node.name}`, { type: "success" });onClose();diff --git a/docs/content/docs/spaces/guides/add-a-computer.mdx b/docs/content/docs/spaces/guides/add-a-computer.mdxindex 738b577095..3662227dd0 100644--- a/docs/content/docs/spaces/guides/add-a-computer.mdx+++ b/docs/content/docs/spaces/guides/add-a-computer.mdx@@ -168,8 +168,10 @@ cua devices revoke dev_0123456789abcdef01234567A computer that is gone: `cua spaces rm <name>` removes it from your relay directory and revokes its sign-in.The Spaces it provided stay on your account, hidden, and come back unchanged if the computer is added-again under the same id. `cua spaces rm <name> --delete-spaces` deletes them too, and their files on-that computer when it can be reached. (`cua machines rm` still works for one more release and says so.)+again under the same id. `cua spaces rm <name> --delete-spaces` deletes them instead: they leave your+account at once, and the computer deletes them, files included, the next time it comes online. If it+never comes back within 90 days, the request expires; adding the computer again before then doesn't+bring those Spaces back. (`cua machines rm` still works for one more release and says so.)In the app, the **Computers** tab holds all of it. A computer that asks to join waits under**Waiting for you** with **Add computer** and **Not me** (Add computer asks for Touch ID or your logindiff --git a/libs/cua-spacesd/crates/cua-relay/README.md b/libs/cua-spacesd/crates/cua-relay/README.mdindex b13bef15b3..04bc4ea817 100644--- a/libs/cua-spacesd/crates/cua-relay/README.md+++ b/libs/cua-spacesd/crates/cua-relay/README.md@@ -211,8 +211,25 @@ Removing a computer (`DELETE /v1/machines/{id}`) keeps the Spaces it providedconnections are refused (403, "its computer was removed from the account")until the computer registers again with the same id.`DELETE /v1/machines/{id}?spaces=delete` (owner) forgets them too and cuts-their connections; deleting their files is the client's job, through the-computer, before it removes it.+their connections. A computer can be removed while it is offline, so the+relay first keeps a pending deletion for it (the Spaces' ids and names, who+asked, the computer's token hash), and the computer deletes them, files+included, the next time it dials the relay:++- `GET /v1/machines/{id}/pending-deletions` lists them (`{computer, spaces:+ [{id, name}], requested_at, expires_at}`; empty `spaces` when none).+- `POST /v1/machines/{id}/pending-deletions/{space}/ack` marks one deleted+ (idempotent); the last ack clears the record.+- Only that computer's machine token is accepted: the one it had when it was+ removed, or its new one if it was added again with the same id. Another+ computer's token or an account token gets 403, an unknown token 401.+- Adding the computer again keeps the record (deleted means deleted), and a+ pending Space id cannot be registered again (410) until it is acked.+- A record expires 90 days after the (latest) request, so a computer that+ never comes back does not pin it forever; erasing the account drops it.++See `src/pending_deletions.rs`; the computer side is+`cua_spacesd_server::pending_deletions`.Each account has a default Space and a fallback order, so the apps, CLI,phone and agents agree where to work: `GET`/`PUT /v1/spaces/settings` and@@ -557,6 +574,7 @@ Everything the relay keeps lives next to the state file, on one volume:| Pools | `<state>.pool/<account>/` | `log.jsonl` and `terminal.txt` appended (fsynced) per append; `lease.json`, `keys.json`, `snapshot.bin` replaced whole. A failed append is cut back (no torn line) and answered `500 storage`. || Pairing codes | `<state>.pairing.json` | Compact JSON, rewritten (streamed) per mint, spend or revoke. A failed write refuses the mint or spend (`500`); the code stays as it was. || Space settings | `<state>.spaces.json` | Compact JSON of each account's default Space and fallback, rewritten (streamed) per change. A failed write refuses the change (`500`). |+| Pending deletions | `<state>.pending-deletions.json` | Compact JSON of removed computers' Spaces still to delete on them, rewritten (streamed) per removal and ack; records older than 90 days are dropped. A failed write refuses the removal (`500`, nothing removed). || Grants | `<state>.grants.json` | Compact JSON of who reached each machine and whose access was revoked, rewritten (streamed) on a change. A failed write is not rolled back (a revoke holds in memory, so old tickets stay refused) and is retried with the next authorization. |Every replaced file goes through a temp file (`.<name>.<uuid>.tmp`); startupdiff --git a/libs/cua-spacesd/crates/cua-relay/src/account_erase.rs b/libs/cua-spacesd/crates/cua-relay/src/account_erase.rsindex 355bd8f58f..aa2709a6a6 100644--- a/libs/cua-spacesd/crates/cua-relay/src/account_erase.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/account_erase.rs@@ -15,7 +15,8 @@//! accounts' chains are separate and still verify), pairing codes;//! - **teams** (leave, delete the ones solely owned, scrub invites and the//! audit log), who-reached-what rows (`grants`), usage and queued-//! analytics events, the account's default Space settings, the cached+//! analytics events, the account's default Space settings and its removed+//! computers' pending Space deletions, the cached//! plan, cached team-access answers, mesh nodes and pairings, and the//! account's pool.//!@@ -239,6 +240,10 @@ fn run(relay: &Relay, sub: &str, given: &[String]) -> Result<serde_json::Value,.pool.erase_account(sub).map_err(|e| failed("pool", e))?;+ let pending_deletions = relay+ .pending_deletions+ .erase_account(sub)+ .map_err(|e| failed("pending_deletions", e))?;Ok(serde_json::json!({"machines": machine_ids.len(),@@ -251,6 +256,7 @@ fn run(relay: &Relay, sub: &str, given: &[String]) -> Result<serde_json::Value,"plan_cache": plan_cache,"pool": pool,"spaces_settings": spaces_settings,+ "pending_deletions": pending_deletions,"teams": teams,"grants": grants,"posthog_distinct_id": crate::meter::posthog_distinct_id(sub),diff --git a/libs/cua-spacesd/crates/cua-relay/src/api.rs b/libs/cua-spacesd/crates/cua-relay/src/api.rsindex 8e62eddeae..c04ec7a641 100644--- a/libs/cua-spacesd/crates/cua-relay/src/api.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/api.rs@@ -12,7 +12,8 @@//! listed and refused (403, "its computer was removed from the account")//! until the computer registers again with the same id.//! `DELETE ...?spaces=delete` (owner, not a computer token) also removes-//! them and cuts their connections.+//! them and cuts their connections; the computer deletes them the next+//! time it dials the relay ([`crate::pending_deletions`]).//! - `POST /v1/machines/{id}/stop-sharing` | `start-sharing` (owner or the//! machine itself): cut every client and refuse new ones, or resume.//! - `POST /v1/machines/{id}/leave` (anyone it is shared with, never the@@ -449,6 +450,14 @@ async fn register("computer id is connected with a relay token",);}+ // Deleted means deleted: a Space its owner deleted with its computer is+ // not registered again before that computer has deleted it.+ if relay.pending_deletions.is_pending_space(&body.id) {+ return error(+ StatusCode::GONE,+ "this Space was deleted; its computer deletes it the next time it comes online",+ );+ }let existing = relay.directory.get(&body.id);let existed = existing.is_some();// Registering a new machine needs only the account (hosting never@@ -975,8 +984,9 @@ pub const DETACHED_MESSAGE: &str = "its computer was removed from the account";/// (records whose `host` is this id) are kept but detached: not listed, not/// reachable, back unchanged when the computer registers again with the/// same id. `?spaces=delete` also removes those records (the computer's-/// owner only) and cuts their connections; deleting their files is the-/// client's job, through the computer, before it removes it.+/// owner only) and cuts their connections, and records them as pending+/// deletions ([`crate::pending_deletions`]) first: the computer deletes+/// them, files included, the next time it dials the relay.async fn delete_one(State(relay): State<Relay>,Path(id): Path<String>,@@ -998,7 +1008,7 @@ async fn delete_one(if let Err(r) = caller.phone_needs_recent_sign_in(&relay) {return r;}- let (_, role) = match authorize(&caller, &relay, &id) {+ let (record, role) = match authorize(&caller, &relay, &id) {Ok(v) => v,Err(r) => return r,};@@ -1011,6 +1021,32 @@ async fn delete_one("a computer token cannot delete its Spaces; use your account",);}+ // The intent is durable before any record goes: a failed write refuses+ // the removal, so no Space is forgotten without its delete pending.+ if delete_spaces {+ let spaces = relay+ .directory+ .children_of(&id)+ .into_iter()+ .map(|c| crate::pending_deletions::PendingSpace {+ id: c.id,+ name: c.name,+ })+ .collect();+ if let Err(e) = relay.pending_deletions.record(+ &id,+ &record.owner.id,+ &record.token_sha256,+ caller.actor().or_else(|| Some(record.owner.id.clone())),+ spaces,+ ) {+ tracing::error!(machine = %id, error = %e, "pending deletions not recorded");+ return error(+ StatusCode::INTERNAL_SERVER_ERROR,+ "could not record the Spaces to delete; nothing was removed",+ );+ }+ }match relay.directory.remove(&id) {Ok(removed) => {relay.drop_machine(&id);diff --git a/libs/cua-spacesd/crates/cua-relay/src/client.rs b/libs/cua-spacesd/crates/cua-relay/src/client.rsindex 2833b02bd5..76d650eb5d 100644--- a/libs/cua-spacesd/crates/cua-relay/src/client.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/client.rs@@ -104,6 +104,10 @@ pub struct JoinConfig {/// every call). A build that does not must say `false`: the relay then sends/// it nothing of a phone's.pub phone_support: bool,+ /// Notified before every attempt to dial the relay (the first one and+ /// each reconnect): the machine reads its pending deletions then (see+ /// [`pending_deletions`]).+ pub dialing: Option<Arc<tokio::sync::Notify>>,}impl JoinConfig {@@ -126,6 +130,7 @@ impl JoinConfig {relay_token_file: None,account: Arc::default(),phone_support: true,+ dialing: None,}}}@@ -210,6 +215,9 @@ pub enum SessionEnd {pub async fn run(config: JoinConfig, shutdown: CancellationToken) {let mut attempt = 0u32;loop {+ if let Some(dialing) = &config.dialing {+ dialing.notify_one();+ }let started = Instant::now();let end = session(&config, &shutdown).await;match &end {@@ -330,6 +338,112 @@ pub async fn session(config: &JoinConfig, shutdown: &CancellationToken) -> Sessiend}+/// The relay's HTTP base for its REST routes (`ws(s)://` becomes+/// `http(s)://`; no trailing slash).+pub fn http_base(relay_url: &str) -> Result<String, String> {+ let mut url = url::Url::parse(relay_url).map_err(|_| "invalid relay URL".to_owned())?;+ let scheme = match url.scheme() {+ "wss" | "https" => "https",+ "ws" | "http" => "http",+ _ => return Err("unsupported relay scheme".into()),+ };+ url.set_scheme(scheme)+ .map_err(|_| "cannot set scheme".to_owned())?;+ Ok(url.to_string().trim_end_matches('/').to_owned())+}++/// What went wrong asking the relay for this machine's pending deletions.+#[derive(Debug)]+pub enum PendingError {+ /// The relay does not know this token (nothing is pending for a+ /// removed computer, or the token is wrong): ask again much later.+ Unknown(String),+ /// Anything else (network, a relay error, an old relay without the+ /// route): ask again soon.+ Failed(String),+}++impl std::fmt::Display for PendingError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ PendingError::Unknown(m) | PendingError::Failed(m) => f.write_str(m),+ }+ }+}++fn pending_http() -> reqwest::Client {+ reqwest::Client::builder()+ .timeout(Duration::from_secs(20))+ .build()+ .unwrap_or_default()+}++async fn pending_error(r: reqwest::Response) -> PendingError {+ let status = r.status();+ let body = r.text().await.unwrap_or_default();+ let message = serde_json::from_str::<serde_json::Value>(&body)+ .ok()+ .and_then(|v| v["error"].as_str().map(str::to_owned))+ .unwrap_or(body);+ let text = format!("relay answered {status}: {message}");+ if status == http::StatusCode::UNAUTHORIZED {+ PendingError::Unknown(text)+ } else {+ PendingError::Failed(text)+ }+}++/// The Spaces this machine's owner deleted while it was removed from the+/// account ([`crate::pending_deletions`]), read with its machine token.+pub async fn pending_deletions(+ relay_url: &str,+ machine_id: &str,+ token: &str,+) -> Result<Vec<crate::pending_deletions::PendingSpace>, PendingError> {+ #[derive(serde::Deserialize)]+ struct Answer {+ #[serde(default)]+ spaces: Vec<crate::pending_deletions::PendingSpace>,+ }+ let base = http_base(relay_url).map_err(PendingError::Failed)?;+ let r = pending_http()+ .get(format!("{base}/v1/machines/{machine_id}/pending-deletions"))+ .bearer_auth(token)+ .send()+ .await+ .map_err(|e| PendingError::Failed(e.to_string()))?;+ if !r.status().is_success() {+ return Err(pending_error(r).await);+ }+ r.json::<Answer>()+ .await+ .map(|a| a.spaces)+ .map_err(|e| PendingError::Failed(e.to_string()))+}++/// Tells the relay this machine deleted `space` (one of its pending+/// deletions). Idempotent.+pub async fn ack_pending_deletion(+ relay_url: &str,+ machine_id: &str,+ token: &str,+ space: &str,+) -> Result<(), PendingError> {+ let base = http_base(relay_url).map_err(PendingError::Failed)?;+ let r = pending_http()+ .post(format!(+ "{base}/v1/machines/{machine_id}/pending-deletions/{space}/ack"+ ))+ .bearer_auth(token)+ .send()+ .await+ .map_err(|e| PendingError::Failed(e.to_string()))?;+ if !r.status().is_success() {+ return Err(pending_error(r).await);+ }+ Ok(())+}+/// Records the relay keys and owner from the 101 response. With pinned/// keys, the relay must present one of them (else it is not the relay this/// machine was set up with, and the session is refused).@@ -454,6 +568,23 @@ mod tests {assert!(backoff(20, cap) >= Duration::from_secs(15));}+ #[test]+ fn http_base_turns_the_relay_url_into_its_rest_base() {+ assert_eq!(+ http_base("wss://relay.cua.ai").unwrap(),+ "https://relay.cua.ai"+ );+ assert_eq!(+ http_base("ws://127.0.0.1:9/").unwrap(),+ "http://127.0.0.1:9"+ );+ assert_eq!(+ http_base("https://r.example/base/").unwrap(),+ "https://r.example/base"+ );+ assert!(http_base("ftp://x").is_err());+ }+#[test]fn the_first_reconnect_after_a_session_is_spread_and_later_ones_back_off() {let cap = Duration::from_secs(30);diff --git a/libs/cua-spacesd/crates/cua-relay/src/directory.rs b/libs/cua-spacesd/crates/cua-relay/src/directory.rsindex f7e5e4a181..0e8ea4dd4b 100644--- a/libs/cua-spacesd/crates/cua-relay/src/directory.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/directory.rs@@ -219,7 +219,7 @@ pub struct Directory {storage: std::sync::Arc<Storage>,}-fn hash_token(token: &str) -> String {+pub(crate) fn hash_token(token: &str) -> String {hex::encode(ring::digest::digest(&ring::digest::SHA256, token.as_bytes()).as_ref())}diff --git a/libs/cua-spacesd/crates/cua-relay/src/lib.rs b/libs/cua-spacesd/crates/cua-relay/src/lib.rsindex 9733d341a7..087c46c451 100644--- a/libs/cua-spacesd/crates/cua-relay/src/lib.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/lib.rs@@ -49,6 +49,7 @@ pub mod mesh;pub mod mux;pub mod oidc;pub mod pairing;+pub mod pending_deletions;pub mod phone;pub mod pool;pub mod proxies;diff --git a/libs/cua-spacesd/crates/cua-relay/src/pending_deletions.rs b/libs/cua-spacesd/crates/cua-relay/src/pending_deletions.rsnew file mode 100644index 0000000000..a084c9a55a--- /dev/null+++ b/libs/cua-spacesd/crates/cua-relay/src/pending_deletions.rs@@ -0,0 +1,463 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Pending deletions: the Spaces of a removed computer that its owner asked+//! to delete (`DELETE /v1/machines/{id}?spaces=delete`, see [`crate::api`]).+//!+//! A computer can be removed while it is offline, so the relay cannot+//! delete its Spaces on it. The relay forgets their records at once (they+//! leave every list), and keeps a pending-deletion record per computer:+//! the Spaces' ids and names, who asked, and the computer's token hash at+//! that moment. The computer (its cua-spacesd) reads the record whenever it+//! dials the relay, deletes each listed Space through its cua daemon, and+//! acks each one; the last ack clears the record.+//!+//! - `GET /v1/machines/{id}/pending-deletions`: `{computer, spaces: [{id,+//! name}], requested_at, expires_at}` (an empty `spaces` when nothing is+//! pending).+//! - `POST /v1/machines/{id}/pending-deletions/{space}/ack`: `204`;+//! idempotent (a Space that is not pending is already done).+//!+//! Only that computer's machine token may call them: the token it had when+//! it was removed (the record keeps its hash), or the one it got when it was+//! added again with the same id. Another computer's token gets `403`, an+//! account token `403`, an unknown token `401`.+//!+//! Deleted means deleted: adding the computer again keeps the record, and+//! registering a pending Space id again is refused (`410`).+//!+//! A record expires [`TTL_SECS`] (90 days) after the request: a computer+//! that never comes back must not pin it forever. It persists in+//! `<state>.pending-deletions.json` (atomic replace, private file mode) and+//! is erased with the account ([`crate::account_erase`]).++// Handlers short-circuit with ready-made responses.+#![allow(clippy::result_large_err)]++use std::collections::BTreeMap;+use std::path::PathBuf;+use std::sync::{Arc, Mutex};++use axum::extract::{Path, State};+use axum::http::{header, HeaderMap, StatusCode};+use axum::response::{IntoResponse, Response};+use axum::routing::{get, post};+use axum::{Json, Router};+use serde::{Deserialize, Serialize};++use crate::api::error;+use crate::assertion::{now_secs, remove_stale_temps, write_private_with};+use crate::directory::{hash_token, MACHINE_TOKEN_PREFIX};+use crate::server::{bearer, Relay};+use crate::storage::{store, Storage};++/// How long a pending-deletion record lives: 90 days.+pub const TTL_SECS: u64 = 90 * 86_400;++/// One Space to delete.+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct PendingSpace {+ /// Its relay machine id (what the computer's daemon knows it by).+ pub id: String,+ /// Its name when it was deleted (for people and logs).+ #[serde(default)]+ pub name: String,+}++/// A removed computer's Spaces to delete.+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct Tombstone {+ /// The owning account.+ pub owner: String,+ /// SHA-256 (hex) of the computer's machine token when it was removed.+ pub token_sha256: String,+ /// Who asked (the device or account that removed it), if known.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub requested_by: Option<String>,+ /// Unix seconds of the (latest) request.+ pub requested_at: u64,+ /// What is left to delete.+ pub spaces: Vec<PendingSpace>,+}++impl Tombstone {+ fn expired(&self, now: u64) -> bool {+ now >= self.requested_at.saturating_add(TTL_SECS)+ }+}++/// Every computer's pending deletions, persisted as JSON (atomic replace)+/// when a file is configured.+#[derive(Debug, Default)]+pub struct PendingDeletions {+ path: Option<PathBuf>,+ computers: Mutex<BTreeMap<String, Tombstone>>,+ storage: Arc<Storage>,+}++impl PendingDeletions {+ /// In memory only.+ pub fn in_memory() -> Self {+ Self::default()+ }++ /// Loads (or starts) the store persisted at `path`, reporting writes to+ /// `storage`. Expired records are dropped (on the next write).+ pub fn open_with(path: PathBuf, storage: Arc<Storage>) -> std::io::Result<Self> {+ remove_stale_temps(&path)?;+ let mut computers: BTreeMap<String, Tombstone> = match std::fs::read(&path) {+ Ok(raw) => serde_json::from_slice(&raw)+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?,+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(),+ Err(e) => return Err(e),+ };+ let now = now_secs();+ computers.retain(|_, t| !t.expired(now));+ Ok(Self {+ path: Some(path),+ computers: Mutex::new(computers),+ storage,+ })+ }++ /// Writes `next` and swaps it in; on a failed write nothing changes.+ fn commit(+ &self,+ computers: &mut BTreeMap<String, Tombstone>,+ next: BTreeMap<String, Tombstone>,+ ) -> Result<(), String> {+ if let Some(path) = &self.path {+ let written = (|| {+ if let Some(parent) = path.parent() {+ std::fs::create_dir_all(parent)?;+ }+ write_private_with(path, |out| {+ serde_json::to_writer(&mut *out, &next).map_err(std::io::Error::other)+ })+ })();+ self.storage.record(store::PENDING_DELETIONS, &written);+ written.map_err(|e| e.to_string())?;+ }+ *computers = next;+ Ok(())+ }++ /// Records that `computer` (owned by `owner`, whose token hash was+ /// `token_sha256`) must delete `spaces`. A record already there gains+ /// them (and the newest token hash); nothing to delete records nothing.+ pub fn record(+ &self,+ computer: &str,+ owner: &str,+ token_sha256: &str,+ requested_by: Option<String>,+ spaces: Vec<PendingSpace>,+ ) -> Result<(), String> {+ self.record_at(+ computer,+ owner,+ token_sha256,+ requested_by,+ spaces,+ now_secs(),+ )+ }++ /// [`Self::record`] at `now`.+ pub fn record_at(+ &self,+ computer: &str,+ owner: &str,+ token_sha256: &str,+ requested_by: Option<String>,+ spaces: Vec<PendingSpace>,+ now: u64,+ ) -> Result<(), String> {+ if spaces.is_empty() {+ return Ok(());+ }+ let mut computers = self.computers.lock().expect("pending deletions");+ let mut next = computers.clone();+ next.retain(|_, t| !t.expired(now));+ let entry = next+ .entry(computer.to_owned())+ .or_insert_with(|| Tombstone {+ owner: owner.to_owned(),+ token_sha256: String::new(),+ requested_by: None,+ requested_at: now,+ spaces: Vec::new(),+ });+ entry.owner = owner.to_owned();+ entry.token_sha256 = token_sha256.to_owned();+ entry.requested_by = requested_by;+ entry.requested_at = now;+ for s in spaces {+ if !entry.spaces.iter().any(|e| e.id == s.id) {+ entry.spaces.push(s);+ }+ }+ self.commit(&mut computers, next)+ }++ /// `computer`'s record, unless there is none or it expired.+ pub fn get(&self, computer: &str) -> Option<Tombstone> {+ self.get_at(computer, now_secs())+ }++ /// [`Self::get`] at `now`.+ pub fn get_at(&self, computer: &str, now: u64) -> Option<Tombstone> {+ self.computers+ .lock()+ .expect("pending deletions")+ .get(computer)+ .filter(|t| !t.expired(now))+ .cloned()+ }++ /// Whether `space` is waiting to be deleted on some computer (its id+ /// may not be registered again meanwhile).+ pub fn is_pending_space(&self, space: &str) -> bool {+ let now = now_secs();+ self.computers+ .lock()+ .expect("pending deletions")+ .values()+ .any(|t| !t.expired(now) && t.spaces.iter().any(|s| s.id == space))+ }++ /// The computer whose record names `token_sha256`, if any (and not+ /// expired).+ fn computer_for_token(&self, token_sha256: &str) -> Option<String> {+ use subtle::ConstantTimeEq as _;+ let now = now_secs();+ self.computers+ .lock()+ .expect("pending deletions")+ .iter()+ .find(|(_, t)| {+ !t.expired(now)+ && !t.token_sha256.is_empty()+ && bool::from(t.token_sha256.as_bytes().ct_eq(token_sha256.as_bytes()))+ })+ .map(|(id, _)| id.clone())+ }++ /// `computer` deleted `space`: it leaves the record, and the last one+ /// clears it. Returns whether it was pending.+ pub fn ack(&self, computer: &str, space: &str) -> Result<bool, String> {+ self.ack_at(computer, space, now_secs())+ }++ /// [`Self::ack`] at a given time (tests).+ pub fn ack_at(&self, computer: &str, space: &str, now: u64) -> Result<bool, String> {+ let mut computers = self.computers.lock().expect("pending deletions");+ let mut next = computers.clone();+ next.retain(|_, t| !t.expired(now));+ let Some(t) = next.get_mut(computer) else {+ return Ok(false);+ };+ let before = t.spaces.len();+ t.spaces.retain(|s| s.id != space);+ let was = t.spaces.len() != before;+ if t.spaces.is_empty() {+ next.remove(computer);+ }+ if !was && next.len() == computers.len() {+ return Ok(false);+ }+ self.commit(&mut computers, next)?;+ Ok(was)+ }++ /// Account deletion: forgets every record of `account`'s computers.+ /// Returns how many there were. The file is rewritten every call, so a+ /// retry after a failed write finishes the job.+ pub fn erase_account(&self, account: &str) -> Result<usize, String> {+ let mut computers = self.computers.lock().expect("pending deletions");+ let mut next = computers.clone();+ let before = next.len();+ next.retain(|_, t| t.owner != account);+ let gone = before - next.len();+ self.commit(&mut computers, next)?;+ Ok(gone)+ }+}++pub(crate) fn routes() -> Router<Relay> {+ Router::new()+ .route("/v1/machines/{id}/pending-deletions", get(list))+ .route("/v1/machines/{id}/pending-deletions/{space}/ack", post(ack))+}++/// Lets only computer `id`'s own machine token through: the one it had+/// when removed (kept in the record), or its current one.+fn authorize(relay: &Relay, headers: &HeaderMap, id: &str) -> Result<(), Response> {+ let Some(token) = bearer(headers, header::AUTHORIZATION.as_str()) else {+ return Err(error(StatusCode::UNAUTHORIZED, "missing computer token"));+ };+ if !token.starts_with(MACHINE_TOKEN_PREFIX) {+ return Err(error(+ StatusCode::FORBIDDEN,+ "only the computer itself reads its pending deletions",+ ));+ }+ let hashed = hash_token(token);+ if let Some(t) = relay.pending_deletions.get(id) {+ use subtle::ConstantTimeEq as _;+ if bool::from(t.token_sha256.as_bytes().ct_eq(hashed.as_bytes())) {+ return Ok(());+ }+ }+ let other = match relay.directory.machine_for_token(token) {+ Some(r) if r.id == id => return Ok(()),+ Some(_) => true,+ None => relay+ .pending_deletions+ .computer_for_token(&hashed)+ .is_some(),+ };+ if other {+ return Err(error(+ StatusCode::FORBIDDEN,+ "computer token belongs to another computer id",+ ));+ }+ Err(error(StatusCode::UNAUTHORIZED, "invalid relay token"))+}++/// What `GET .../pending-deletions` answers.+#[derive(Debug, Serialize)]+struct View {+ computer: String,+ spaces: Vec<PendingSpace>,+ #[serde(skip_serializing_if = "Option::is_none")]+ requested_at: Option<u64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ expires_at: Option<u64>,+}++async fn list(State(relay): State<Relay>, Path(id): Path<String>, headers: HeaderMap) -> Response {+ if let Err(r) = authorize(&relay, &headers, &id) {+ return r;+ }+ let t = relay.pending_deletions.get(&id);+ Json(View {+ computer: id,+ requested_at: t.as_ref().map(|t| t.requested_at),+ expires_at: t.as_ref().map(|t| t.requested_at.saturating_add(TTL_SECS)),+ spaces: t.map(|t| t.spaces).unwrap_or_default(),+ })+ .into_response()+}++async fn ack(+ State(relay): State<Relay>,+ Path((id, space)): Path<(String, String)>,+ headers: HeaderMap,+) -> Response {+ if let Err(r) = authorize(&relay, &headers, &id) {+ return r;+ }+ let space = space.strip_prefix("relay:").unwrap_or(&space);+ match relay.pending_deletions.ack(&id, space) {+ Ok(was) => {+ if was {+ tracing::info!(computer = %id, space, "pending Space deletion acknowledged");+ }+ StatusCode::NO_CONTENT.into_response()+ }+ Err(e) => error(StatusCode::INTERNAL_SERVER_ERROR, e),+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn space(id: &str) -> PendingSpace {+ PendingSpace {+ id: id.into(),+ name: format!("{id}-name"),+ }+ }++ #[test]+ fn records_merge_ack_clears_and_expire() {+ let p = PendingDeletions::in_memory();+ p.record_at("c1", "ada", "h1", None, vec![], 100).unwrap();+ assert!(+ p.get_at("c1", 100).is_none(),+ "nothing to delete records nothing"+ );++ p.record_at(+ "c1",+ "ada",+ "h1",+ Some("dev".into()),+ vec![space("s1")],+ 100,+ )+ .unwrap();+ p.record_at("c1", "ada", "h2", None, vec![space("s1"), space("s2")], 200)+ .unwrap();+ let t = p.get_at("c1", 200).unwrap();+ assert_eq!(t.spaces, vec![space("s1"), space("s2")]);+ assert_eq!((t.token_sha256.as_str(), t.requested_at), ("h2", 200));++ // Expiry counts from the latest request.+ assert!(p.get_at("c1", 200 + TTL_SECS - 1).is_some());+ assert!(p.get_at("c1", 200 + TTL_SECS).is_none());++ assert!(p.ack_at("c1", "s1", 200).unwrap());+ assert!(!p.ack_at("c1", "s1", 200).unwrap(), "idempotent");+ assert!(!p.ack_at("c1", "nope", 200).unwrap());+ assert!(!p.ack_at("c9", "s2", 200).unwrap());+ assert!(p.ack_at("c1", "s2", 200).unwrap());+ assert!(+ p.computers.lock().unwrap().is_empty(),+ "the last ack clears it"+ );+ }++ #[test]+ fn persists_drops_expired_on_load_and_erases_by_account() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("state.pending-deletions.json");+ let p = PendingDeletions::open_with(path.clone(), Arc::default()).unwrap();+ let now = now_secs();+ p.record_at(+ "old",+ "ada",+ "h0",+ None,+ vec![space("s0")],+ now - TTL_SECS - 1,+ )+ .unwrap();+ p.record_at("c1", "ada", "h1", None, vec![space("s1")], now)+ .unwrap();+ p.record_at("c2", "bob", "h2", None, vec![space("s2")], now)+ .unwrap();+ drop(p);+ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt as _;+ let mode = std::fs::metadata(&path).unwrap().permissions().mode();+ assert_eq!(mode & 0o077, 0, "private file");+ }++ let p = PendingDeletions::open_with(path.clone(), Arc::default()).unwrap();+ assert!(p.get("old").is_none(), "expired on load");+ assert!(p.get("c1").is_some() && p.get("c2").is_some());+ assert!(p.is_pending_space("s1") && !p.is_pending_space("s0"));+ assert_eq!(p.computer_for_token("h2").as_deref(), Some("c2"));++ assert_eq!(p.erase_account("ada").unwrap(), 1);+ assert_eq!(p.erase_account("ada").unwrap(), 0);+ drop(p);+ let p = PendingDeletions::open_with(path, Arc::default()).unwrap();+ assert!(p.get("c1").is_none() && p.get("c2").is_some());+ }+}diff --git a/libs/cua-spacesd/crates/cua-relay/src/server.rs b/libs/cua-spacesd/crates/cua-relay/src/server.rsindex a199c69d8a..a03c21b24a 100644--- a/libs/cua-spacesd/crates/cua-relay/src/server.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/server.rs@@ -411,6 +411,8 @@ pub struct Relay {pub(crate) pairing: Arc<crate::pairing::PairingStore>,/// Each account's default Space and fallback order.pub(crate) spaces: Arc<crate::spaces::SpacesSettings>,+ /// Removed computers' Spaces still to delete on them.+ pub(crate) pending_deletions: Arc<crate::pending_deletions::PendingDeletions>,pub(crate) key: Arc<RelayKey>,/// Open client streams across every machine (S7): the global memory/// budget's live counter.@@ -538,6 +540,13 @@ impl Relay {)?,None => crate::spaces::SpacesSettings::in_memory(),};+ let pending_deletions = match &config.state_file {+ Some(path) => crate::pending_deletions::PendingDeletions::open_with(+ path.with_extension("pending-deletions.json"),+ storage.clone(),+ )?,+ None => crate::pending_deletions::PendingDeletions::in_memory(),+ };let key = config.signing_key.clone()@@ -593,6 +602,7 @@ impl Relay {grants: Arc::new(grants),pairing: Arc::new(pairing),spaces: Arc::new(spaces),+ pending_deletions: Arc::new(pending_deletions),key,global_streams: Arc::default(),mesh,@@ -650,6 +660,7 @@ impl Relay {.merge(self.hosted_routes()).merge(crate::pairing::routes()).merge(crate::spaces::routes())+ .merge(crate::pending_deletions::routes()).merge(crate::tcp_shares::routes()).merge(if self.share_web_enabled() {crate::share_web::routes()diff --git a/libs/cua-spacesd/crates/cua-relay/src/storage.rs b/libs/cua-spacesd/crates/cua-relay/src/storage.rsindex 586475deb2..6c7ba1413e 100644--- a/libs/cua-spacesd/crates/cua-relay/src/storage.rs+++ b/libs/cua-spacesd/crates/cua-relay/src/storage.rs@@ -4,7 +4,7 @@//! The health of the relay's state volume (CUA-1297).//!//! Every state store (devices, audit log, device sessions, machine-//! directory, mesh registry, pool, usage, teams, pairing codes, grants, Space settings)+//! directory, mesh registry, pool, usage, teams, pairing codes, grants, Space settings, pending deletions)//! reports each write here. A failed write//! is counted, logged with a stable line (`state persist failed`, field//! `alert = "state_persist_failed"`) and remembered as the store's last@@ -71,9 +71,23 @@ pub mod store {pub const GRANTS: &str = "grants";/// `<state>.spaces.json`: each account's default Space settings.pub const SPACES: &str = "spaces";+ /// `<state>.pending-deletions.json`: removed computers' Spaces still to+ /// delete on them.+ pub const PENDING_DELETIONS: &str = "pending_deletions";/// Every store, in report order.- pub const ALL: [&str; 11] = [- DEVICES, AUDIT, SESSIONS, DIRECTORY, MESH, POOL, USAGE, TEAMS, PAIRING, GRANTS, SPACES,+ pub const ALL: [&str; 12] = [+ DEVICES,+ AUDIT,+ SESSIONS,+ DIRECTORY,+ MESH,+ POOL,+ USAGE,+ TEAMS,+ PAIRING,+ GRANTS,+ SPACES,+ PENDING_DELETIONS,];}diff --git a/libs/cua-spacesd/crates/cua-relay/tests/account_erase.rs b/libs/cua-spacesd/crates/cua-relay/tests/account_erase.rsindex 85b10a68b8..8b8674ed8a 100644--- a/libs/cua-spacesd/crates/cua-relay/tests/account_erase.rs+++ b/libs/cua-spacesd/crates/cua-relay/tests/account_erase.rs@@ -429,6 +429,7 @@ async fn an_unknown_account_is_a_200_with_zero_counts() {"mesh_nodes","teams","grants",+ "pending_deletions",] {assert_eq!(v[k], 0, "{k}: {v}");}@@ -716,6 +717,7 @@ async fn it_erases_one_account_everywhere_and_nobody_elses_and_survives_a_restar"mesh_nodes","teams","grants",+ "pending_deletions",] {assert_eq!(again[k], 0, "{k}: {again}");}@@ -758,6 +760,62 @@ async fn it_erases_one_account_everywhere_and_nobody_elses_and_survives_a_restar}#[cfg(unix)]+#[tokio::test]+async fn erasing_an_account_drops_its_removed_computers_pending_deletions() {+ let env = Env::start().await;+ let ada = env.enroll(ADA).await;+ let register = |id: String, host: Option<String>| {+ let http = env.http.clone();+ let base = env.base.clone();+ let ada = ada.clone();+ async move {+ let r = http+ .post(format!("{base}/v1/machines"))+ .bearer_auth(&ada.token)+ .header(SESSION, &ada.session)+ .json(&json!({"id": id, "name": id, "host": host}))+ .send()+ .await+ .unwrap();+ assert_eq!(r.status(), StatusCode::CREATED);+ r.json::<Value>().await.unwrap()["machine_token"]+ .as_str()+ .unwrap()+ .to_owned()+ }+ };+ let host = format!("host-{}", &uuid::Uuid::new_v4().simple().to_string()[..16]);+ let token = register(host.clone(), None).await;+ register(format!("{host}-kid"), Some(host.clone())).await;+ let r = env+ .http+ .delete(format!("{}/v1/machines/{host}?spaces=delete", env.base))+ .bearer_auth(&ada.token)+ .header(SESSION, &ada.session)+ .send()+ .await+ .unwrap();+ assert_eq!(r.status(), StatusCode::NO_CONTENT);+ let pending = || {+ env.http+ .get(format!("{}/v1/machines/{host}/pending-deletions", env.base))+ .bearer_auth(&token)+ .send()+ };+ assert_eq!(pending().await.unwrap().status(), StatusCode::OK);++ let (s, v) = env.erase(ADA, &[]).await;+ assert_eq!(s, StatusCode::OK, "{v}");+ assert_eq!(v["pending_deletions"], 1, "{v}");+ assert_eq!(+ pending().await.unwrap().status(),+ StatusCode::UNAUTHORIZED,+ "nothing of hers is left"+ );+ let (_, again) = env.erase(ADA, &[]).await;+ assert_eq!(again["pending_deletions"], 0, "{again}");+}+#[tokio::test]async fn a_failed_write_answers_500_and_the_same_call_again_finishes() {let env = Env::start().await;diff --git a/libs/cua-spacesd/crates/cua-relay/tests/pending_deletions.rs b/libs/cua-spacesd/crates/cua-relay/tests/pending_deletions.rsnew file mode 100644index 0000000000..8215fb5835--- /dev/null+++ b/libs/cua-spacesd/crates/cua-relay/tests/pending_deletions.rs@@ -0,0 +1,359 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Pending deletions (`GET /v1/machines/{id}/pending-deletions`, `POST+//! .../{space}/ack`): removing a computer with `?spaces=delete` records its+//! Spaces for it to delete when it is next online. Only that computer's+//! token reads and acks them; adding it again keeps them, and a deleted+//! Space's id cannot be registered again until it is acked.++use std::path::Path;+use std::sync::Arc;++use cua_relay::devices::DevicePolicy;+use cua_relay::oidc::testing::FakeIssuer;+use cua_relay::oidc::{OidcConfig, OidcValidator};+use cua_relay::server::{Relay, RelayConfig};+use reqwest::{Method, StatusCode};+use serde_json::{json, Value};++const ISSUER: &str = "https://auth.test/realms/cua";++fn config(issuer: &FakeIssuer, dir: &Path) -> RelayConfig {+ RelayConfig {+ oidc: Some(Arc::new(OidcValidator::with_jwks(+ OidcConfig::new(ISSUER),+ issuer.jwks(),+ ))),+ device_policy: DevicePolicy {+ grace_secs: 14 * 86_400,+ ..DevicePolicy::default()+ },+ state_file: Some(dir.join("machines.json")),+ ..RelayConfig::default()+ }+}++async fn serve(relay: &Relay) -> String {+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let base = format!("http://{}", listener.local_addr().unwrap());+ let relay = relay.clone();+ tokio::spawn(async move {+ let _ = relay.serve(listener, std::future::pending()).await;+ });+ base+}++struct Env {+ base: String,+ issuer: FakeIssuer,+ dir: tempfile::TempDir,+ ada: String,+ bob: String,+}++impl Env {+ async fn start() -> Env {+ let issuer = FakeIssuer::new(ISSUER);+ let dir = tempfile::tempdir().unwrap();+ let relay = Relay::new(config(&issuer, dir.path()));+ let base = serve(&relay).await;+ let ada = issuer.token("ada-sub", Some("[email protected]"), "cua-relay", 3000);+ let bob = issuer.token("bob-sub", Some("[email protected]"), "cua-relay", 3000);+ Env {+ base,+ issuer,+ dir,+ ada,+ bob,+ }+ }++ /// A second relay on the same state files (a restart).+ async fn restart(&self) -> String {+ serve(&Relay::new(config(&self.issuer, self.dir.path()))).await+ }++ async fn call_on(+ base: &str,+ method: Method,+ path: &str,+ token: Option<&str>,+ body: Option<Value>,+ ) -> (StatusCode, Value) {+ let mut r = reqwest::Client::new().request(method, format!("{base}{path}"));+ if let Some(t) = token {+ r = r.bearer_auth(t);+ }+ if let Some(b) = body {+ r = r.json(&b);+ }+ let r = r.send().await.unwrap();+ let status = r.status();+ let text = r.text().await.unwrap_or_default();+ (+ status,+ serde_json::from_str(&text).unwrap_or(Value::String(text)),+ )+ }++ async fn call(+ &self,+ method: Method,+ path: &str,+ token: Option<&str>,+ body: Option<Value>,+ ) -> (StatusCode, Value) {+ Self::call_on(&self.base, method, path, token, body).await+ }++ /// Registers a computer (or, with `host`, a Space on it); returns its+ /// machine token.+ async fn try_register(&self, token: &str, id: &str, host: Option<&str>) -> (StatusCode, Value) {+ self.call(+ Method::POST,+ "/v1/machines",+ Some(token),+ Some(json!({"id": id, "name": format!("{id}-name"), "host": host})),+ )+ .await+ }++ async fn register(&self, token: &str, id: &str, host: Option<&str>) -> String {+ let (s, v) = self.try_register(token, id, host).await;+ assert!(s.is_success(), "register {id}: {s} {v}");+ v["machine_token"].as_str().unwrap().to_owned()+ }++ fn new_id(&self) -> String {+ format!("m{}", uuid::Uuid::new_v4().simple())+ }++ async fn listed(&self, token: &str) -> Vec<String> {+ let (s, v) = self+ .call(Method::GET, "/v1/machines", Some(token), None)+ .await;+ assert_eq!(s, StatusCode::OK, "{v}");+ v["machines"]+ .as_array()+ .unwrap()+ .iter()+ .map(|m| m["id"].as_str().unwrap().to_owned())+ .collect()+ }++ async fn remove(&self, computer: &str, delete: bool) {+ let q = if delete { "?spaces=delete" } else { "" };+ let (s, v) = self+ .call(+ Method::DELETE,+ &format!("/v1/machines/{computer}{q}"),+ Some(&self.ada),+ None,+ )+ .await;+ assert_eq!(s, StatusCode::NO_CONTENT, "{v}");+ }++ async fn pending(&self, computer: &str, token: Option<&str>) -> (StatusCode, Value) {+ self.call(+ Method::GET,+ &format!("/v1/machines/{computer}/pending-deletions"),+ token,+ None,+ )+ .await+ }++ async fn ack(&self, computer: &str, space: &str, token: &str) -> StatusCode {+ self.call(+ Method::POST,+ &format!("/v1/machines/{computer}/pending-deletions/{space}/ack"),+ Some(token),+ None,+ )+ .await+ .0+ }+}++fn ids(v: &Value) -> Vec<String> {+ let mut out: Vec<String> = v["spaces"]+ .as_array()+ .unwrap()+ .iter()+ .map(|s| s["id"].as_str().unwrap().to_owned())+ .collect();+ out.sort();+ out+}++#[tokio::test]+async fn removing_with_delete_records_the_spaces_for_the_computer_alone() {+ let env = Env::start().await;+ let host = env.new_id();+ let host_token = env.register(&env.ada, &host, None).await;+ let kid1 = env.new_id();+ let kid1_token = env.register(&env.ada, &kid1, Some(&host)).await;+ let kid2 = env.new_id();+ env.register(&env.ada, &kid2, Some(&host)).await;+ let other = env.new_id();+ let other_token = env.register(&env.ada, &other, None).await;+ let bobs = env.new_id();+ let bob_token = env.register(&env.bob, &bobs, None).await;++ // Nothing pending yet: the computer reads an empty list.+ let (s, v) = env.pending(&host, Some(&host_token)).await;+ assert_eq!(s, StatusCode::OK, "{v}");+ assert_eq!(v["spaces"], json!([]));+ assert!(v.get("expires_at").is_none(), "{v}");++ env.remove(&host, true).await;+ // The Spaces leave the account at once.+ assert_eq!(env.listed(&env.ada).await, vec![other.clone()]);++ // The removed computer's token still reads its own record.+ let (s, v) = env.pending(&host, Some(&host_token)).await;+ assert_eq!(s, StatusCode::OK, "{v}");+ let mut want = vec![kid1.clone(), kid2.clone()];+ want.sort();+ assert_eq!(ids(&v), want);+ assert_eq!(v["computer"], host.as_str());+ let names: Vec<&str> = v["spaces"]+ .as_array()+ .unwrap()+ .iter()+ .map(|s| s["name"].as_str().unwrap())+ .collect();+ assert!(names.contains(&format!("{kid1}-name").as_str()), "{v}");+ let at = v["requested_at"].as_u64().unwrap();+ assert_eq!(v["expires_at"].as_u64().unwrap(), at + 90 * 86_400);++ // No one else does.+ let (s, v) = env.pending(&host, Some(&other_token)).await;+ assert_eq!(s, StatusCode::FORBIDDEN, "another computer of hers: {v}");+ let (s, _) = env.pending(&host, Some(&bob_token)).await;+ assert_eq!(s, StatusCode::FORBIDDEN, "another account's computer");+ let (s, _) = env.pending(&host, Some(&env.ada)).await;+ assert_eq!(s, StatusCode::FORBIDDEN, "the owner's account token");+ let (s, _) = env.pending(&host, None).await;+ assert_eq!(s, StatusCode::UNAUTHORIZED);+ let (s, _) = env.pending(&host, Some("cmt_not-a-real-token")).await;+ assert_eq!(s, StatusCode::UNAUTHORIZED);+ let (s, _) = env.pending(&host, Some(&kid1_token)).await;+ assert_eq!(+ s,+ StatusCode::UNAUTHORIZED,+ "a deleted Space's own token is gone"+ );+ assert_eq!(+ env.ack(&host, &kid1, &other_token).await,+ StatusCode::FORBIDDEN+ );+ // ... and the computer's token reads only its own id.+ let (s, _) = env.pending(&other, Some(&host_token)).await;+ assert_eq!(s, StatusCode::FORBIDDEN);++ // It survives a restart.+ let restarted = env.restart().await;+ let (s, v) = Env::call_on(+ &restarted,+ Method::GET,+ &format!("/v1/machines/{host}/pending-deletions"),+ Some(&host_token),+ None,+ )+ .await;+ assert_eq!((s, ids(&v)), (StatusCode::OK, want.clone()));++ // Acks are idempotent; the last one clears the record.+ assert_eq!(+ env.ack(&host, &kid1, &host_token).await,+ StatusCode::NO_CONTENT+ );+ assert_eq!(+ env.ack(&host, &kid1, &host_token).await,+ StatusCode::NO_CONTENT+ );+ let (_, v) = env.pending(&host, Some(&host_token)).await;+ assert_eq!(ids(&v), vec![kid2.clone()]);+ assert_eq!(+ env.ack(&host, &format!("relay:{kid2}"), &host_token).await,+ StatusCode::NO_CONTENT+ );+ // Cleared: the removed computer's token now means nothing here.+ let (s, _) = env.pending(&host, Some(&host_token)).await;+ assert_eq!(s, StatusCode::UNAUTHORIZED);+ let raw =+ std::fs::read_to_string(env.dir.path().join("machines.pending-deletions.json")).unwrap();+ assert_eq!(raw.trim(), "{}");+}++#[tokio::test]+async fn removing_without_delete_records_nothing() {+ let env = Env::start().await;+ let host = env.new_id();+ let host_token = env.register(&env.ada, &host, None).await;+ let kid = env.new_id();+ env.register(&env.ada, &kid, Some(&host)).await;+ env.remove(&host, false).await;+ let (s, _) = env.pending(&host, Some(&host_token)).await;+ assert_eq!(s, StatusCode::UNAUTHORIZED);+ // A detached Space may come back: its id is not refused.+ env.register(&env.ada, &host, None).await;+ assert!(env.listed(&env.ada).await.contains(&kid));+}++#[tokio::test]+async fn adding_the_computer_again_keeps_the_deletions_and_refuses_the_ids() {+ let env = Env::start().await;+ let host = env.new_id();+ let old_token = env.register(&env.ada, &host, None).await;+ let kid = env.new_id();+ env.register(&env.ada, &kid, Some(&host)).await;+ env.remove(&host, true).await;++ // `cua host setup` adds it again with the same id and a new token.+ let new_token = env.register(&env.ada, &host, None).await;+ assert_ne!(new_token, old_token);+ for token in [&new_token, &old_token] {+ let (s, v) = env.pending(&host, Some(token)).await;+ assert_eq!((s, ids(&v)), (StatusCode::OK, vec![kid.clone()]), "{v}");+ }+ assert!(+ !env.listed(&env.ada).await.contains(&kid),+ "deleted means deleted"+ );++ // The deleted Space cannot be registered again meanwhile.+ let (s, v) = env.try_register(&env.ada, &kid, Some(&host)).await;+ assert_eq!(s, StatusCode::GONE, "{v}");+ assert!(v["error"].as_str().unwrap().contains("deleted"), "{v}");++ // Once the computer deleted it, the id is free again.+ assert_eq!(+ env.ack(&host, &kid, &new_token).await,+ StatusCode::NO_CONTENT+ );+ let (s, v) = env.pending(&host, Some(&new_token)).await;+ assert_eq!((s, v["spaces"].clone()), (StatusCode::OK, json!([])));+ env.register(&env.ada, &kid, Some(&host)).await;+}++#[tokio::test]+async fn a_second_removal_adds_to_the_record() {+ let env = Env::start().await;+ let host = env.new_id();+ env.register(&env.ada, &host, None).await;+ let kid1 = env.new_id();+ env.register(&env.ada, &kid1, Some(&host)).await;+ env.remove(&host, true).await;+ let token = env.register(&env.ada, &host, None).await;+ let kid2 = env.new_id();+ env.register(&env.ada, &kid2, Some(&host)).await;+ env.remove(&host, true).await;+ let (_, v) = env.pending(&host, Some(&token)).await;+ let mut want = vec![kid1, kid2];+ want.sort();+ assert_eq!(ids(&v), want);+}diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/host_spaces.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/host_spaces.rsindex 44e945fb8b..7f1ebb4b52 100644--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/host_spaces.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/host_spaces.rs@@ -165,7 +165,7 @@ mod routed_tests {}}-fn read_policy(path: &Path) -> Result<HostPolicy, Status> {+pub(crate) fn read_policy(path: &Path) -> Result<HostPolicy, Status> {let raw = std::fs::read(path).map_err(|e| {status(Code::FailedPrecondition,@@ -484,7 +484,9 @@ fn caller_json_of<T>(req: &Request<T>) -> Result<String, Status> {#[cfg(unix)]static STARTING: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());-async fn daemon_channel(daemon: &SpacesDaemon) -> Result<tonic::transport::Channel, Status> {+pub(crate) async fn daemon_channel(+ daemon: &SpacesDaemon,+) -> Result<tonic::transport::Channel, Status> {#[cfg(unix)]{let socket = PathBuf::from(&daemon.socket);diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/lib.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/lib.rsindex 7c228d0111..7946737195 100644--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/lib.rs+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/lib.rs@@ -22,6 +22,7 @@ pub mod http;pub mod identity;pub mod mesh_host;pub mod peer;+pub mod pending_deletions;pub mod phone_control;pub mod privilege;pub mod process;diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/pending_deletions.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/pending_deletions.rsnew file mode 100644index 0000000000..9d41109ad8--- /dev/null+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/pending_deletions.rs@@ -0,0 +1,261 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! Deleting the Spaces this computer's owner deleted while it was offline.+//!+//! Removing a computer from the account with "Also delete its Spaces"+//! (`DELETE /v1/machines/{id}?spaces=delete`) works while the computer is+//! offline: the relay forgets the Spaces at once and keeps a pending+//! deletion for the computer ([`cua_relay::pending_deletions`]). Every time+//! this host dials the relay ([`cua_relay::client::JoinConfig::dialing`]),+//! it reads that list with its machine token, deletes each listed Space+//! through its cua daemon (`DeleteHostSpace`, else `DeleteCloudSpace`: the+//! sandbox, its files and, for a cloud Space, its cloud resources) as this+//! machine's owner, and acks each one.+//!+//! - It deletes only Spaces on the list, by their relay id.+//! - A Space that is not here any more counts as deleted and is acked.+//! - A delete that fails is not acked: the next sweep tries again.+//! - A computer added again (`cua host setup` after a removal) keeps its+//! pending deletions, and the relay refuses to register those Spaces again+//! until they are acked, so they never come back (deleted means deleted).++use std::path::PathBuf;+use std::sync::Arc;+use std::time::Duration;++use cua_proto::env::v1::host_spaces_service_client::HostSpacesServiceClient;+use cua_proto::env::v1::{DeleteCloudSpaceRequest, DeleteHostSpaceRequest};+use cua_relay::client::{ack_pending_deletion, pending_deletions, PendingError};+use tokio::time::Instant;+use tokio_util::sync::CancellationToken;+use tonic::{Code, Request};++use crate::host_spaces::{daemon_channel, read_policy};++/// How long one Space's delete may take (a VM's disk can be large).+const DELETE_TIMEOUT: Duration = Duration::from_secs(10 * 60);++/// Deletes one Space on this machine.+#[tonic::async_trait]+pub trait SpaceDeleter: Send + Sync {+ /// Deletes `space` (a relay machine id). `Ok(true)`: deleted;+ /// `Ok(false)`: not on this machine (nothing to delete); `Err`: not+ /// deleted, try again later.+ async fn delete(&self, space: &str) -> Result<bool, String>;+}++/// Deletes through the host's cua daemon named in the host policy.+pub struct DaemonDeleter {+ policy_file: PathBuf,+}++impl DaemonDeleter {+ /// A deleter reading the host policy at `policy_file` on every call.+ pub fn new(policy_file: impl Into<PathBuf>) -> Self {+ Self {+ policy_file: policy_file.into(),+ }+ }+}++#[tonic::async_trait]+impl SpaceDeleter for DaemonDeleter {+ async fn delete(&self, space: &str) -> Result<bool, String> {+ let policy = read_policy(&self.policy_file).map_err(|s| s.message().to_owned())?;+ let daemon = policy+ .spaces_daemon+ .ok_or("this host's policy names no cua daemon; run `cua host setup` on it again")?;+ let channel = daemon_channel(&daemon)+ .await+ .map_err(|s| s.message().to_owned())?;+ // No caller metadata: the call is this machine's own (its owner's).+ let mut client = HostSpacesServiceClient::new(channel);+ let mut req = Request::new(DeleteHostSpaceRequest {+ space: space.to_owned(),+ });+ req.set_timeout(DELETE_TIMEOUT);+ match client.delete_host_space(req).await {+ Ok(_) => return Ok(true),+ Err(s) if s.code() == Code::NotFound => {}+ Err(s) => return Err(s.message().to_owned()),+ }+ // Not a Space this host provides: maybe one in the owner's cloud+ // that this computer created.+ let mut req = Request::new(DeleteCloudSpaceRequest {+ space: space.to_owned(),+ });+ req.set_timeout(DELETE_TIMEOUT);+ match client.delete_cloud_space(req).await {+ Ok(_) => Ok(true),+ Err(s) if s.code() == Code::NotFound => Ok(false),+ Err(s) => Err(s.message().to_owned()),+ }+ }+}++/// What one sweep did.+#[derive(Debug, Default, Clone, PartialEq, Eq)]+pub struct SweepReport {+ /// Deleted here and acked.+ pub deleted: Vec<String>,+ /// Not on this machine (already gone) and acked.+ pub absent: Vec<String>,+ /// Not deleted or not acked, with why: the next sweep tries again.+ pub failed: Vec<(String, String)>,+}++/// Reads this machine's pending deletions, deletes each listed Space with+/// `deleter` and acks it. Fails only when the list cannot be read.+pub async fn sweep(+ relay_url: &str,+ machine_id: &str,+ token: &str,+ deleter: &dyn SpaceDeleter,+) -> Result<SweepReport, PendingError> {+ let mut report = SweepReport::default();+ for space in pending_deletions(relay_url, machine_id, token).await? {+ // Only a plain relay id names a Space here; anything else is never+ // passed on (it could name something that is not on the list).+ if !cua_relay::valid_machine_id(&space.id) {+ report+ .failed+ .push((space.id.clone(), "not a Space id".into()));+ continue;+ }+ let here = match deleter.delete(&space.id).await {+ Ok(here) => here,+ Err(e) => {+ tracing::warn!(space = %space.id, name = %space.name, error = %e, "pending Space deletion failed; retrying later");+ report.failed.push((space.id, e));+ continue;+ }+ };+ if let Err(e) = ack_pending_deletion(relay_url, machine_id, token, &space.id).await {+ report.failed.push((space.id, format!("ack: {e}")));+ continue;+ }+ tracing::info!(space = %space.id, name = %space.name, here, "deleted a Space its owner deleted while this computer was removed");+ if here {+ report.deleted.push(space.id);+ } else {+ report.absent.push(space.id);+ }+ }+ Ok(report)+}++/// How the background sweeper runs.+pub struct PendingDeletionsConfig {+ /// The relay (`ws(s)://` or `http(s)://`).+ pub relay_url: String,+ /// This machine's id.+ pub machine_id: String,+ /// The machine token.+ pub relay_token: String,+ /// Re-read before every sweep when readable and non-empty (setup+ /// rotates it under a running service).+ pub relay_token_file: Option<PathBuf>,+ /// Notified on every relay dial ([`cua_relay::client::JoinConfig::dialing`]).+ pub dialing: Arc<tokio::sync::Notify>,+ /// Least time between two sweeps.+ pub min_interval: Duration,+ /// After a sweep with failures: try again this soon.+ pub retry_interval: Duration,+ /// After a clean sweep: look again this often even without a dial.+ pub idle_interval: Duration,+ /// After the relay did not know the token (nothing pending for a+ /// removed computer): ask again this late, unless the token changes.+ pub unknown_interval: Duration,+}++impl PendingDeletionsConfig {+ /// Defaults for everything but the relay, id, token and dial signal.+ pub fn new(+ relay_url: String,+ machine_id: String,+ relay_token: String,+ dialing: Arc<tokio::sync::Notify>,+ ) -> Self {+ Self {+ relay_url,+ machine_id,+ relay_token,+ relay_token_file: None,+ dialing,+ min_interval: Duration::from_secs(60),+ retry_interval: Duration::from_secs(5 * 60),+ idle_interval: Duration::from_secs(6 * 60 * 60),+ unknown_interval: Duration::from_secs(60 * 60),+ }+ }++ fn token(&self) -> String {+ self.relay_token_file+ .as_deref()+ .and_then(|p| std::fs::read_to_string(p).ok())+ .map(|t| t.trim().to_owned())+ .filter(|t| !t.is_empty())+ .unwrap_or_else(|| self.relay_token.clone())+ }+}++/// Sweeps at start and on each relay dial (at most once per+/// `min_interval`), until `shutdown`.+pub async fn run(+ config: PendingDeletionsConfig,+ deleter: Arc<dyn SpaceDeleter>,+ shutdown: CancellationToken,+) {+ let mut due = Instant::now();+ let mut not_before = Instant::now();+ // The token the relay last did not know: a new one is asked at once.+ let mut unknown: Option<String> = None;+ loop {+ tokio::select! {+ _ = shutdown.cancelled() => return,+ _ = tokio::time::sleep_until(due) => {}+ _ = config.dialing.notified() => {+ if unknown.as_deref().is_some_and(|t| t != config.token()) {+ unknown = None;+ not_before = Instant::now();+ }+ if unknown.is_none() {+ due = due.min(not_before.max(Instant::now()));+ }+ continue;+ }+ }+ let token = config.token();+ let started = Instant::now();+ not_before = started + config.min_interval;+ due = match sweep(&config.relay_url, &config.machine_id, &token, &*deleter).await {+ Ok(r) => {+ unknown = None;+ if !r.deleted.is_empty() || !r.absent.is_empty() || !r.failed.is_empty() {+ tracing::info!(+ deleted = r.deleted.len(),+ absent = r.absent.len(),+ failed = r.failed.len(),+ "pending Space deletions swept"+ );+ }+ if r.failed.is_empty() {+ started + config.idle_interval+ } else {+ started + config.retry_interval.max(config.min_interval)+ }+ }+ Err(PendingError::Unknown(e)) => {+ tracing::debug!(error = %e, "no pending Space deletions for this token");+ unknown = Some(token);+ not_before = started + config.unknown_interval;+ not_before+ }+ Err(PendingError::Failed(e)) => {+ tracing::info!(error = %e, "pending Space deletions not read; retrying later");+ started + config.retry_interval.max(config.min_interval)+ }+ };+ }+}diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/tests/pending_deletions.rs b/libs/cua-spacesd/crates/cua-spacesd-server/tests/pending_deletions.rsnew file mode 100644index 0000000000..c91c07f50b--- /dev/null+++ b/libs/cua-spacesd/crates/cua-spacesd-server/tests/pending_deletions.rs@@ -0,0 +1,508 @@+// SPDX-License-Identifier: FSL-1.1-MIT+// Copyright (c) 2026 Cua AI, Inc.++//! A host deletes the Spaces its owner deleted while it was removed from+//! the account (`DELETE /v1/machines/{id}?spaces=delete` on the relay), the+//! next time it dials the relay: the real relay in process (a fake OIDC+//! issuer) and a fake cua daemon on a Unix socket.+//!+//! - Only the listed Spaces are deleted; each is acked; a failed delete is+//! retried on the next sweep; a Space already gone is acked.+//! - End to end: an offline computer is removed with delete, comes back,+//! and its Spaces are deleted and acked; one not on the list stays.+//! - Added again with the same id (a new token): the deletions still run+//! first, and the deleted Space ids cannot be registered meanwhile.++#![cfg(unix)]++use std::collections::BTreeSet;+use std::path::Path;+use std::sync::{Arc, Mutex};+use std::time::Duration;++use cua_proto::env::v1::host_spaces_service_server::{HostSpacesService, HostSpacesServiceServer};+use cua_proto::env::v1::*;+use cua_relay::devices::DevicePolicy;+use cua_relay::oidc::testing::FakeIssuer;+use cua_relay::oidc::{OidcConfig, OidcValidator};+use cua_relay::server::{Relay, RelayConfig};+use cua_spacesd_server::host_spaces::CALLER_METADATA;+use cua_spacesd_server::pending_deletions::{+ run, sweep, DaemonDeleter, PendingDeletionsConfig, SpaceDeleter,+};+use serde_json::{json, Value};+use tokio_util::sync::CancellationToken;++const ISSUER: &str = "https://auth.test/realms/cua";++struct Env {+ base: String,+ ada: String,+ _issuer: FakeIssuer,+ _dir: tempfile::TempDir,+}++impl Env {+ async fn start() -> Env {+ let issuer = FakeIssuer::new(ISSUER);+ let dir = tempfile::tempdir().unwrap();+ let relay = Relay::new(RelayConfig {+ oidc: Some(Arc::new(OidcValidator::with_jwks(+ OidcConfig::new(ISSUER),+ issuer.jwks(),+ ))),+ device_policy: DevicePolicy {+ grace_secs: 14 * 86_400,+ ..DevicePolicy::default()+ },+ state_file: Some(dir.path().join("machines.json")),+ ..RelayConfig::default()+ });+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let base = format!("http://{}", listener.local_addr().unwrap());+ tokio::spawn(async move {+ let _ = relay.serve(listener, std::future::pending()).await;+ });+ let ada = issuer.token("ada-sub", Some("[email protected]"), "cua-relay", 3000);+ Env {+ base,+ ada,+ _issuer: issuer,+ _dir: dir,+ }+ }++ async fn call(+ &self,+ method: reqwest::Method,+ path: &str,+ token: &str,+ body: Option<Value>,+ ) -> (reqwest::StatusCode, Value) {+ let mut r = reqwest::Client::new()+ .request(method, format!("{}{path}", self.base))+ .bearer_auth(token);+ if let Some(b) = body {+ r = r.json(&b);+ }+ let r = r.send().await.unwrap();+ let s = r.status();+ let text = r.text().await.unwrap_or_default();+ (+ s,+ serde_json::from_str(&text).unwrap_or(Value::String(text)),+ )+ }++ async fn try_register(&self, id: &str, host: Option<&str>) -> (reqwest::StatusCode, Value) {+ self.call(+ reqwest::Method::POST,+ "/v1/machines",+ &self.ada,+ Some(json!({"id": id, "name": id, "host": host})),+ )+ .await+ }++ async fn register(&self, id: &str, host: Option<&str>) -> String {+ let (s, v) = self.try_register(id, host).await;+ assert!(s.is_success(), "{s} {v}");+ v["machine_token"].as_str().unwrap().to_owned()+ }++ async fn remove_with_delete(&self, computer: &str) {+ let (s, v) = self+ .call(+ reqwest::Method::DELETE,+ &format!("/v1/machines/{computer}?spaces=delete"),+ &self.ada,+ None,+ )+ .await;+ assert_eq!(s, reqwest::StatusCode::NO_CONTENT, "{v}");+ }++ /// The Space ids still pending for `computer` (`None`: the record is+ /// gone and the token means nothing to the relay).+ async fn pending(&self, computer: &str, token: &str) -> Option<BTreeSet<String>> {+ let (s, v) = self+ .call(+ reqwest::Method::GET,+ &format!("/v1/machines/{computer}/pending-deletions"),+ token,+ None,+ )+ .await;+ if s == reqwest::StatusCode::UNAUTHORIZED {+ return None;+ }+ assert_eq!(s, reqwest::StatusCode::OK, "{v}");+ Some(+ v["spaces"]+ .as_array()+ .unwrap()+ .iter()+ .map(|s| s["id"].as_str().unwrap().to_owned())+ .collect(),+ )+ }++ async fn listed(&self) -> Vec<String> {+ let (_, v) = self+ .call(reqwest::Method::GET, "/v1/machines", &self.ada, None)+ .await;+ v["machines"]+ .as_array()+ .unwrap()+ .iter()+ .map(|m| m["id"].as_str().unwrap().to_owned())+ .collect()+ }+}++fn new_id(tag: &str) -> String {+ format!("{tag}{}", uuid::Uuid::new_v4().simple())+}++/// Local Spaces by relay id; the ones in `fail_once` fail their first+/// delete.+#[derive(Default)]+struct FakeSpaces {+ local: Mutex<BTreeSet<String>>,+ fail_once: Mutex<BTreeSet<String>>,+ asked: Mutex<Vec<String>>,+}++impl FakeSpaces {+ fn with(local: &[&str]) -> Arc<Self> {+ let s = Self::default();+ s.local+ .lock()+ .unwrap()+ .extend(local.iter().map(|s| s.to_string()));+ Arc::new(s)+ }++ fn local(&self) -> BTreeSet<String> {+ self.local.lock().unwrap().clone()+ }++ fn delete(&self, space: &str) -> Result<bool, String> {+ self.asked.lock().unwrap().push(space.to_owned());+ if self.fail_once.lock().unwrap().remove(space) {+ return Err("the VM is busy".into());+ }+ Ok(self.local.lock().unwrap().remove(space))+ }+}++#[tonic::async_trait]+impl SpaceDeleter for FakeSpaces {+ async fn delete(&self, space: &str) -> Result<bool, String> {+ FakeSpaces::delete(self, space)+ }+}++#[tokio::test]+async fn a_sweep_deletes_only_the_listed_spaces_and_retries_a_failure() {+ let env = Env::start().await;+ let host = new_id("m");+ let token = env.register(&host, None).await;+ let (a, b, gone) = (new_id("space-"), new_id("space-"), new_id("space-"));+ for kid in [&a, &b, &gone] {+ env.register(kid, Some(&host)).await;+ }+ let spaces = FakeSpaces::with(&[&a, &b, "space-not-on-the-list"]);+ spaces.fail_once.lock().unwrap().insert(b.clone());++ // Nothing pending yet: nothing is touched.+ let r = sweep(&env.base, &host, &token, &*spaces).await.unwrap();+ assert_eq!(r, Default::default());++ env.remove_with_delete(&host).await;+ let r = sweep(&env.base, &host, &token, &*spaces).await.unwrap();+ assert_eq!(r.deleted, vec![a.clone()]);+ assert_eq!(r.absent, vec![gone.clone()], "already gone: acked");+ assert_eq!(r.failed.len(), 1);+ assert_eq!(r.failed[0].0, b);+ assert_eq!(+ env.pending(&host, &token).await,+ Some(BTreeSet::from([b.clone()])),+ "the failed one stays pending"+ );++ let r = sweep(&env.base, &host, &token, &*spaces).await.unwrap();+ assert_eq!((r.deleted, r.failed), (vec![b.clone()], vec![]));+ assert_eq!(env.pending(&host, &token).await, None, "all acked: cleared");+ assert_eq!(+ spaces.local(),+ BTreeSet::from(["space-not-on-the-list".to_owned()])+ );+ assert!(!spaces+ .asked+ .lock()+ .unwrap()+ .iter()+ .any(|s| s == "space-not-on-the-list"));+}++/// The host's cua daemon: deletes the Spaces it has, by relay id, and+/// records whether each call came as this machine's own (no caller).+#[derive(Clone)]+struct FakeDaemon {+ spaces: Arc<FakeSpaces>,+ forwarded: Arc<Mutex<Vec<String>>>,+}++#[tonic::async_trait]+impl HostSpacesService for FakeDaemon {+ async fn get_host_spaces(+ &self,+ _: tonic::Request<GetHostSpacesRequest>,+ ) -> Result<tonic::Response<GetHostSpacesResponse>, tonic::Status> {+ Err(tonic::Status::unimplemented("get_host_spaces"))+ }++ async fn create_host_space(+ &self,+ _: tonic::Request<CreateHostSpaceRequest>,+ ) -> Result<tonic::Response<CreateHostSpaceResponse>, tonic::Status> {+ Err(tonic::Status::unimplemented("create_host_space"))+ }++ async fn cancel_host_space(+ &self,+ _: tonic::Request<CancelHostSpaceRequest>,+ ) -> Result<tonic::Response<CancelHostSpaceResponse>, tonic::Status> {+ Err(tonic::Status::unimplemented("cancel_host_space"))+ }++ async fn set_host_space_power(+ &self,+ _: tonic::Request<SetHostSpacePowerRequest>,+ ) -> Result<tonic::Response<SetHostSpacePowerResponse>, tonic::Status> {+ Err(tonic::Status::unimplemented("set_host_space_power"))+ }++ async fn set_host_warm(+ &self,+ _: tonic::Request<SetHostWarmRequest>,+ ) -> Result<tonic::Response<SetHostWarmResponse>, tonic::Status> {+ Err(tonic::Status::unimplemented("set_host_warm"))+ }++ async fn set_host_space_egress(+ &self,+ _: tonic::Request<SetHostSpaceEgressRequest>,+ ) -> Result<tonic::Response<SetHostSpaceEgressResponse>, tonic::Status> {+ Err(tonic::Status::unimplemented("set_host_space_egress"))+ }++ async fn diagnose_host(+ &self,+ _: tonic::Request<DiagnoseHostRequest>,+ ) -> Result<tonic::Response<DiagnoseHostResponse>, tonic::Status> {+ Err(tonic::Status::unimplemented("diagnose_host"))+ }++ async fn delete_host_space(+ &self,+ req: tonic::Request<DeleteHostSpaceRequest>,+ ) -> Result<tonic::Response<DeleteHostSpaceResponse>, tonic::Status> {+ if let Some(who) = req.metadata().get(CALLER_METADATA) {+ self.forwarded+ .lock()+ .unwrap()+ .push(who.to_str().unwrap_or("").to_owned());+ }+ let space = req.into_inner().space;+ match self.spaces.delete(&space) {+ Ok(true) => Ok(tonic::Response::new(DeleteHostSpaceResponse {+ message: format!("Deleted relay:{space}"),+ })),+ Ok(false) => Err(tonic::Status::not_found(format!(+ "{space} is not a Space this host provides"+ ))),+ Err(e) => Err(tonic::Status::unavailable(e)),+ }+ }++ async fn delete_cloud_space(+ &self,+ req: tonic::Request<DeleteCloudSpaceRequest>,+ ) -> Result<tonic::Response<DeleteCloudSpaceResponse>, tonic::Status> {+ Err(tonic::Status::not_found(format!(+ "relay:{} is not a Space in your cloud that this computer created",+ req.into_inner().space+ )))+ }+}++async fn serve_daemon(socket: &Path, daemon: FakeDaemon) {+ let listener = tokio::net::UnixListener::bind(socket).unwrap();+ let router =+ tonic::service::Routes::new(HostSpacesServiceServer::new(daemon)).into_axum_router();+ tokio::spawn(async move {+ let _ = axum::serve(listener, router).await;+ });+}++/// A host: its policy names the fake daemon.+struct Host {+ dir: tempfile::TempDir,+ spaces: Arc<FakeSpaces>,+ forwarded: Arc<Mutex<Vec<String>>>,+}++impl Host {+ async fn start(local: &[&str]) -> Host {+ let dir = tempfile::tempdir().unwrap();+ let socket = dir.path().join("daemon.sock");+ let spaces = FakeSpaces::with(local);+ let forwarded = Arc::new(Mutex::new(Vec::new()));+ serve_daemon(+ &socket,+ FakeDaemon {+ spaces: spaces.clone(),+ forwarded: forwarded.clone(),+ },+ )+ .await;+ std::fs::write(+ dir.path().join("host.json"),+ json!({+ "owner": "ada-sub",+ "provide_spaces": true,+ "spaces_daemon": {"socket": socket, "cua_home": ""},+ })+ .to_string(),+ )+ .unwrap();+ Host {+ dir,+ spaces,+ forwarded,+ }+ }++ /// The computer comes online: it dials the relay (refused or not) and+ /// sweeps on each dial, as `cua-spacesd join` with a host policy does.+ fn come_online(&self, env: &Env, id: &str, token_file: &Path) -> CancellationToken {+ let stop = CancellationToken::new();+ let dialing = Arc::new(tokio::sync::Notify::new());+ let unused = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let local = unused.local_addr().unwrap();+ drop(unused);+ let mut join = cua_relay::client::JoinConfig::new(+ env.base.replace("http://", "ws://"),+ String::new(),+ id.to_owned(),+ local,+ );+ join.relay_token_file = Some(token_file.to_owned());+ join.max_backoff = Duration::from_millis(200);+ join.dialing = Some(dialing.clone());+ let mut cfg = PendingDeletionsConfig::new(+ join.relay_url.clone(),+ id.to_owned(),+ String::new(),+ dialing,+ );+ cfg.relay_token_file = Some(token_file.to_owned());+ cfg.min_interval = Duration::from_millis(100);+ cfg.retry_interval = Duration::from_millis(100);+ tokio::spawn(cua_relay::client::run(join, stop.clone()));+ tokio::spawn(run(+ cfg,+ Arc::new(DaemonDeleter::new(self.dir.path().join("host.json"))),+ stop.clone(),+ ));+ stop+ }+}++/// Waits (bounded) until `ok` holds.+macro_rules! eventually {+ ($what:expr, $ok:expr) => {{+ let mut done = false;+ for _ in 0..200 {+ if $ok {+ done = true;+ break;+ }+ tokio::time::sleep(Duration::from_millis(50)).await;+ }+ assert!(done, "timed out waiting for: {}", $what);+ }};+}++#[tokio::test]+async fn an_offline_computer_removed_with_delete_deletes_its_spaces_when_it_comes_back() {+ let env = Env::start().await;+ let id = new_id("m");+ let token = env.register(&id, None).await;+ let (a, b) = (new_id("space-"), new_id("space-"));+ env.register(&a, Some(&id)).await;+ env.register(&b, Some(&id)).await;+ let keep = new_id("space-");+ let host = Host::start(&[&a, &b, &keep]).await;+ host.spaces.fail_once.lock().unwrap().insert(b.clone());++ // Offline (never dialed), removed with its Spaces deleted: they leave+ // the account at once.+ env.remove_with_delete(&id).await;+ assert!(env.listed().await.is_empty());+ assert_eq!(+ env.pending(&id, &token).await,+ Some(BTreeSet::from([a.clone(), b.clone()]))+ );++ // It comes back.+ let token_file = host.dir.path().join("relay-token");+ std::fs::write(&token_file, &token).unwrap();+ let _stop = host.come_online(&env, &id, &token_file);+ eventually!(+ "both Spaces deleted and acked",+ env.pending(&id, &token).await.is_none()+ );+ assert_eq!(host.spaces.local(), BTreeSet::from([keep.clone()]));+ assert!(!host.spaces.asked.lock().unwrap().contains(&keep));+ assert!(+ host.forwarded.lock().unwrap().is_empty(),+ "the deletes are this machine's own calls"+ );+}++#[tokio::test]+async fn a_computer_added_again_still_deletes_its_deleted_spaces() {+ let env = Env::start().await;+ let id = new_id("m");+ let old_token = env.register(&id, None).await;+ let kid = new_id("space-");+ env.register(&kid, Some(&id)).await;+ let host = Host::start(&[&kid]).await;+ env.remove_with_delete(&id).await;++ // `cua host setup` adds it again under the same id with a new token+ // before it ever dialed: deleted means deleted.+ let new_token = env.register(&id, None).await;+ assert_ne!(old_token, new_token);+ let (s, _) = env.try_register(&kid, Some(&id)).await;+ assert_eq!(+ s,+ reqwest::StatusCode::GONE,+ "not registered again meanwhile"+ );++ let token_file = host.dir.path().join("relay-token");+ std::fs::write(&token_file, &new_token).unwrap();+ let _stop = host.come_online(&env, &id, &token_file);+ eventually!(+ "the deleted Space is deleted and acked",+ env.pending(&id, &new_token).await == Some(BTreeSet::new())+ );+ assert!(host.spaces.local().is_empty());+ assert!(!env.listed().await.contains(&kid));+ // Acked: the id is free again.+ env.register(&kid, Some(&id)).await;+}diff --git a/libs/cua-spacesd/crates/cua-spacesd/src/main.rs b/libs/cua-spacesd/crates/cua-spacesd/src/main.rsindex 934a77aab9..ce2c4611dd 100644--- a/libs/cua-spacesd/crates/cua-spacesd/src/main.rs+++ b/libs/cua-spacesd/crates/cua-spacesd/src/main.rs@@ -1038,6 +1038,27 @@ async fn serve(args: ServeArgs, join: Option<JoinArgs>) -> Result<Exit, BoxError.pin_jwks_json(json).map_err(|e| format!("CUA_RELAY_JWKS_JSON: {e}"))?;}+ // A host deletes the Spaces its owner deleted while it was removed+ // from the account, each time it dials the relay+ // (cua_spacesd_server::pending_deletions).+ if let Some(policy) = host_policy.clone() {+ let dialing = Arc::new(tokio::sync::Notify::new());+ join_config.dialing = Some(dialing.clone());+ let mut pending = cua_spacesd_server::pending_deletions::PendingDeletionsConfig::new(+ join.relay.clone(),+ machine_id.clone(),+ join.relay_token.clone().unwrap_or_default(),+ dialing,+ );+ pending.relay_token_file = join.relay_token_file.clone();+ tokio::spawn(cua_spacesd_server::pending_deletions::run(+ pending,+ Arc::new(cua_spacesd_server::pending_deletions::DaemonDeleter::new(+ policy,+ )),+ shutdown.clone(),+ ));+ }// Account mode: relay-signed principal assertions authorize// callers (in addition to the local token).ctx.auth().set_external(Arc::new(diff --git a/libs/cua/crates/cua-cli/src/host.rs b/libs/cua/crates/cua-cli/src/host.rsindex d799517f87..2ef04b790f 100644--- a/libs/cua/crates/cua-cli/src/host.rs+++ b/libs/cua/crates/cua-cli/src/host.rs@@ -2156,9 +2156,14 @@ pub async fn remove_machine(String::new()} else if delete_spaces {format!(- " Its {} Space{} and their files will be deleted (the files only if the computer answers).",- children.len(),- if children.len() == 1 { "" } else { "s" }+ " Its {} leave{} your account now, and the computer deletes {}, files included, the next time it comes online.",+ if children.len() == 1 {+ "Space".to_string()+ } else {+ format!("{} Spaces", children.len())+ },+ if children.len() == 1 { "s" } else { "" },+ if children.len() == 1 { "it" } else { "them" })} else {format!(