Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(relay,spacesd): a removed computer deletes its Spaces when it next comes online (CUA-3479) (#6532)

cua-agents[bot]e7e78c8217b1
diff --git a/apps/cua-spaces-web/src/__tests__/spaces-tree.test.tsx b/apps/cua-spaces-web/src/__tests__/spaces-tree.test.tsx
index 9d5f1eb12d..22eb05ded5 100644
--- a/apps/cua-spaces-web/src/__tests__/spaces-tree.test.tsx
+++ b/apps/cua-spaces-web/src/__tests__/spaces-tree.test.tsx
@@ -191,17 +191,15 @@ describe.skipIf(!wasmBuilt)("Spaces as a tree (Labs)", () => {
expect(calls.some(([op]) => op === "spaces.delete")).toBe(false);
expect(calls.find(([op]) => op === "machines.remove")![1]).not.toHaveProperty("deleteSpaces", true);
- // Ticked: each Space is deleted through the computer first, then the computer goes with them.
+ // Ticked: the computer goes with its Spaces in one call; the relay keeps them for the computer to delete when it's back.
calls.length = 0;
dialog = await open();
fireEvent.click(q("[data-tree-also-delete]")!);
- // Ticked, it says what really happens: the Spaces leave the account, their files stay on the offline computer.
- expect(dialog.textContent).toContain("The 2 Spaces are deleted from your account and won't come back. Their files stay on Linux box, which is offline, until you delete them there.");
+ expect(dialog.textContent).toContain("The 2 Spaces leave your account now and won't come back. Linux box deletes them, with their files, the next time it comes online.");
expect(q("[data-tree-remove-confirm]")?.textContent).toBe("Remove and delete");
fireEvent.click(q("[data-tree-remove-confirm]")!);
- await waitFor(() => expect(calls.map(([op]) => op)).toEqual(["spaces.delete", "spaces.delete", "machines.remove"]));
- expect(calls.slice(0, 2).map(([, a]) => a.spaceId).sort()).toEqual(["relay:linux-box/ubuntu-build", "relay:linux-box/windows-legacy"]);
- expect(calls[2]![1]).toMatchObject({ machine: "linux-box", deleteSpaces: true });
+ await waitFor(() => expect(calls.map(([op]) => op)).toEqual(["machines.remove"]));
+ expect(calls[0]![1]).toMatchObject({ machine: "linux-box", deleteSpaces: true });
});
});
diff --git a/apps/cua-spaces-web/src/components/spaces/spaces-tree-page.tsx b/apps/cua-spaces-web/src/components/spaces/spaces-tree-page.tsx
index bb4dbaf09a..833731f729 100644
--- a/apps/cua-spaces-web/src/components/spaces/spaces-tree-page.tsx
+++ b/apps/cua-spaces-web/src/components/spaces/spaces-tree-page.tsx
@@ -49,13 +49,16 @@ export function removeText(
? "It leaves your Spaces list, and your team can't use it any more."
: `It leaves your Spaces list, and your team can't use it any more. The ${plural(n, "Space")} made on it leave${n === 1 ? "s" : ""} the list with it:`;
const back = n === 0 ? "" : n === 1 ? " Add it again and it comes back." : ` Add it again and all ${n} come back.`;
- // Stopgap (CUA-3413): a computer can only be removed while it is offline, so its Spaces can't be deleted on it. Ticked,
- // they leave your account and their files stay on the computer until it deletes them. Say exactly that.
- const them = n === 1 ? "Its files stay" : "Their files stay";
+ // A computer is removed while it is offline (CUA-3479): ticked, its Spaces leave the account now, and the relay keeps
+ // them as pending deletions that the computer carries out, files included, the next time it comes online.
+ const deleted =
+ n === 1
+ ? `The Space leaves your account now and won't come back. ${node.name} deletes it, with its files, the next time it comes online.`
+ : `The ${n} Spaces leave your account now and won't come back. ${node.name} deletes them, with their files, the next time it comes online.`;
return {
title: `Remove ${node.name}?`,
lead,
- keep: alsoDelete ? `${n === 1 ? "The Space is" : `The ${n} Spaces are`} deleted from your account and won't come back. ${them} on ${node.name}, which is offline, until you delete them there.` : `Nothing is deleted from ${node.name}.${back}`,
+ keep: alsoDelete ? deleted : `Nothing is deleted from ${node.name}.${back}`,
box: n === 0 ? null : `Also delete the ${n === 1 ? "Space" : `${n} Spaces`}`,
confirm: alsoDelete ? `Remove and delete` : `Remove ${node.name}`,
};
@@ -293,11 +296,11 @@ export function SpacesTreePage() {
/**
* f9: removing a computer names each Space made on it and any task that
- * stops. They are only detached unless "Also delete" is ticked: then each is
- * deleted through the computer first, and the relay forgets them with it.
+ * stops. They are only detached unless "Also delete" is ticked: then the
+ * relay forgets them at once and the computer deletes them, files included,
+ * the next time it comes online (pending deletions, CUA-3479).
*/
export function RemoveComputerDialog({ node, onClose }: { node: TreeNode; onClose: () => void }) {
- const { deleteSpace } = useSpaces();
const machines = useMachines();
const [alsoDelete, setAlsoDelete] = useState(false);
const [busy, setBusy] = useState(false);
@@ -306,18 +309,6 @@ export function RemoveComputerDialog({ node, onClose }: { node: TreeNode; onClos
const run = async () => {
setBusy(true);
try {
- if (alsoDelete) {
- for (const c of node.children) {
- if (!c.space) continue;
- try {
- await deleteSpace(c.space.id);
- } catch (e) {
- // A Space that couldn't be deleted stops here: the computer stays, so nothing is half-removed.
- toastError(`Couldn't delete ${c.name}, so ${node.name} wasn't removed`)(e);
- return;
- }
- }
- }
await machines.remove(r?.machine ?? node.machine!.id, r?.device ?? null, alsoDelete);
toast(`Removed ${node.name}`, { type: "success" });
onClose();
diff --git a/docs/content/docs/spaces/guides/add-a-computer.mdx b/docs/content/docs/spaces/guides/add-a-computer.mdx
index 738b577095..3662227dd0 100644
--- a/docs/content/docs/spaces/guides/add-a-computer.mdx
+++ b/docs/content/docs/spaces/guides/add-a-computer.mdx
@@ -168,8 +168,10 @@ cua devices revoke dev_0123456789abcdef01234567
A computer that is gone: `cua spaces rm <name>` removes it from your relay directory and revokes its sign-in.
The Spaces it provided stay on your account, hidden, and come back unchanged if the computer is added
-again under the same id. `cua spaces rm <name> --delete-spaces` deletes them too, and their files on
-that computer when it can be reached. (`cua machines rm` still works for one more release and says so.)
+again under the same id. `cua spaces rm <name> --delete-spaces` deletes them instead: they leave your
+account at once, and the computer deletes them, files included, the next time it comes online. If it
+never comes back within 90 days, the request expires; adding the computer again before then doesn't
+bring those Spaces back. (`cua machines rm` still works for one more release and says so.)
In the app, the **Computers** tab holds all of it. A computer that asks to join waits under
**Waiting for you** with **Add computer** and **Not me** (Add computer asks for Touch ID or your login
diff --git a/libs/cua-spacesd/crates/cua-relay/README.md b/libs/cua-spacesd/crates/cua-relay/README.md
index b13bef15b3..04bc4ea817 100644
--- a/libs/cua-spacesd/crates/cua-relay/README.md
+++ b/libs/cua-spacesd/crates/cua-relay/README.md
@@ -211,8 +211,25 @@ Removing a computer (`DELETE /v1/machines/{id}`) keeps the Spaces it provided
connections are refused (403, "its computer was removed from the account")
until the computer registers again with the same id.
`DELETE /v1/machines/{id}?spaces=delete` (owner) forgets them too and cuts
-their connections; deleting their files is the client's job, through the
-computer, before it removes it.
+their connections. A computer can be removed while it is offline, so the
+relay first keeps a pending deletion for it (the Spaces' ids and names, who
+asked, the computer's token hash), and the computer deletes them, files
+included, the next time it dials the relay:
+
+- `GET /v1/machines/{id}/pending-deletions` lists them (`{computer, spaces:
+ [{id, name}], requested_at, expires_at}`; empty `spaces` when none).
+- `POST /v1/machines/{id}/pending-deletions/{space}/ack` marks one deleted
+ (idempotent); the last ack clears the record.
+- Only that computer's machine token is accepted: the one it had when it was
+ removed, or its new one if it was added again with the same id. Another
+ computer's token or an account token gets 403, an unknown token 401.
+- Adding the computer again keeps the record (deleted means deleted), and a
+ pending Space id cannot be registered again (410) until it is acked.
+- A record expires 90 days after the (latest) request, so a computer that
+ never comes back does not pin it forever; erasing the account drops it.
+
+See `src/pending_deletions.rs`; the computer side is
+`cua_spacesd_server::pending_deletions`.
Each account has a default Space and a fallback order, so the apps, CLI,
phone and agents agree where to work: `GET`/`PUT /v1/spaces/settings` and
@@ -557,6 +574,7 @@ Everything the relay keeps lives next to the state file, on one volume:
| Pools | `<state>.pool/<account>/` | `log.jsonl` and `terminal.txt` appended (fsynced) per append; `lease.json`, `keys.json`, `snapshot.bin` replaced whole. A failed append is cut back (no torn line) and answered `500 storage`. |
| Pairing codes | `<state>.pairing.json` | Compact JSON, rewritten (streamed) per mint, spend or revoke. A failed write refuses the mint or spend (`500`); the code stays as it was. |
| Space settings | `<state>.spaces.json` | Compact JSON of each account's default Space and fallback, rewritten (streamed) per change. A failed write refuses the change (`500`). |
+| Pending deletions | `<state>.pending-deletions.json` | Compact JSON of removed computers' Spaces still to delete on them, rewritten (streamed) per removal and ack; records older than 90 days are dropped. A failed write refuses the removal (`500`, nothing removed). |
| Grants | `<state>.grants.json` | Compact JSON of who reached each machine and whose access was revoked, rewritten (streamed) on a change. A failed write is not rolled back (a revoke holds in memory, so old tickets stay refused) and is retried with the next authorization. |
Every replaced file goes through a temp file (`.<name>.<uuid>.tmp`); startup
diff --git a/libs/cua-spacesd/crates/cua-relay/src/account_erase.rs b/libs/cua-spacesd/crates/cua-relay/src/account_erase.rs
index 355bd8f58f..aa2709a6a6 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/account_erase.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/account_erase.rs
@@ -15,7 +15,8 @@
//! accounts' chains are separate and still verify), pairing codes;
//! - **teams** (leave, delete the ones solely owned, scrub invites and the
//! audit log), who-reached-what rows (`grants`), usage and queued
-//! analytics events, the account's default Space settings, the cached
+//! analytics events, the account's default Space settings and its removed
+//! computers' pending Space deletions, the cached
//! plan, cached team-access answers, mesh nodes and pairings, and the
//! account's pool.
//!
@@ -239,6 +240,10 @@ fn run(relay: &Relay, sub: &str, given: &[String]) -> Result<serde_json::Value,
.pool
.erase_account(sub)
.map_err(|e| failed("pool", e))?;
+ let pending_deletions = relay
+ .pending_deletions
+ .erase_account(sub)
+ .map_err(|e| failed("pending_deletions", e))?;
Ok(serde_json::json!({
"machines": machine_ids.len(),
@@ -251,6 +256,7 @@ fn run(relay: &Relay, sub: &str, given: &[String]) -> Result<serde_json::Value,
"plan_cache": plan_cache,
"pool": pool,
"spaces_settings": spaces_settings,
+ "pending_deletions": pending_deletions,
"teams": teams,
"grants": grants,
"posthog_distinct_id": crate::meter::posthog_distinct_id(sub),
diff --git a/libs/cua-spacesd/crates/cua-relay/src/api.rs b/libs/cua-spacesd/crates/cua-relay/src/api.rs
index 8e62eddeae..c04ec7a641 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/api.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/api.rs
@@ -12,7 +12,8 @@
//! listed and refused (403, "its computer was removed from the account")
//! until the computer registers again with the same id.
//! `DELETE ...?spaces=delete` (owner, not a computer token) also removes
-//! them and cuts their connections.
+//! them and cuts their connections; the computer deletes them the next
+//! time it dials the relay ([`crate::pending_deletions`]).
//! - `POST /v1/machines/{id}/stop-sharing` | `start-sharing` (owner or the
//! machine itself): cut every client and refuse new ones, or resume.
//! - `POST /v1/machines/{id}/leave` (anyone it is shared with, never the
@@ -449,6 +450,14 @@ async fn register(
"computer id is connected with a relay token",
);
}
+ // Deleted means deleted: a Space its owner deleted with its computer is
+ // not registered again before that computer has deleted it.
+ if relay.pending_deletions.is_pending_space(&body.id) {
+ return error(
+ StatusCode::GONE,
+ "this Space was deleted; its computer deletes it the next time it comes online",
+ );
+ }
let existing = relay.directory.get(&body.id);
let existed = existing.is_some();
// Registering a new machine needs only the account (hosting never
@@ -975,8 +984,9 @@ pub const DETACHED_MESSAGE: &str = "its computer was removed from the account";
/// (records whose `host` is this id) are kept but detached: not listed, not
/// reachable, back unchanged when the computer registers again with the
/// same id. `?spaces=delete` also removes those records (the computer's
-/// owner only) and cuts their connections; deleting their files is the
-/// client's job, through the computer, before it removes it.
+/// owner only) and cuts their connections, and records them as pending
+/// deletions ([`crate::pending_deletions`]) first: the computer deletes
+/// them, files included, the next time it dials the relay.
async fn delete_one(
State(relay): State<Relay>,
Path(id): Path<String>,
@@ -998,7 +1008,7 @@ async fn delete_one(
if let Err(r) = caller.phone_needs_recent_sign_in(&relay) {
return r;
}
- let (_, role) = match authorize(&caller, &relay, &id) {
+ let (record, role) = match authorize(&caller, &relay, &id) {
Ok(v) => v,
Err(r) => return r,
};
@@ -1011,6 +1021,32 @@ async fn delete_one(
"a computer token cannot delete its Spaces; use your account",
);
}
+ // The intent is durable before any record goes: a failed write refuses
+ // the removal, so no Space is forgotten without its delete pending.
+ if delete_spaces {
+ let spaces = relay
+ .directory
+ .children_of(&id)
+ .into_iter()
+ .map(|c| crate::pending_deletions::PendingSpace {
+ id: c.id,
+ name: c.name,
+ })
+ .collect();
+ if let Err(e) = relay.pending_deletions.record(
+ &id,
+ &record.owner.id,
+ &record.token_sha256,
+ caller.actor().or_else(|| Some(record.owner.id.clone())),
+ spaces,
+ ) {
+ tracing::error!(machine = %id, error = %e, "pending deletions not recorded");
+ return error(
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "could not record the Spaces to delete; nothing was removed",
+ );
+ }
+ }
match relay.directory.remove(&id) {
Ok(removed) => {
relay.drop_machine(&id);
diff --git a/libs/cua-spacesd/crates/cua-relay/src/client.rs b/libs/cua-spacesd/crates/cua-relay/src/client.rs
index 2833b02bd5..76d650eb5d 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/client.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/client.rs
@@ -104,6 +104,10 @@ pub struct JoinConfig {
/// every call). A build that does not must say `false`: the relay then sends
/// it nothing of a phone's.
pub phone_support: bool,
+ /// Notified before every attempt to dial the relay (the first one and
+ /// each reconnect): the machine reads its pending deletions then (see
+ /// [`pending_deletions`]).
+ pub dialing: Option<Arc<tokio::sync::Notify>>,
}
impl JoinConfig {
@@ -126,6 +130,7 @@ impl JoinConfig {
relay_token_file: None,
account: Arc::default(),
phone_support: true,
+ dialing: None,
}
}
}
@@ -210,6 +215,9 @@ pub enum SessionEnd {
pub async fn run(config: JoinConfig, shutdown: CancellationToken) {
let mut attempt = 0u32;
loop {
+ if let Some(dialing) = &config.dialing {
+ dialing.notify_one();
+ }
let started = Instant::now();
let end = session(&config, &shutdown).await;
match &end {
@@ -330,6 +338,112 @@ pub async fn session(config: &JoinConfig, shutdown: &CancellationToken) -> Sessi
end
}
+/// The relay's HTTP base for its REST routes (`ws(s)://` becomes
+/// `http(s)://`; no trailing slash).
+pub fn http_base(relay_url: &str) -> Result<String, String> {
+ let mut url = url::Url::parse(relay_url).map_err(|_| "invalid relay URL".to_owned())?;
+ let scheme = match url.scheme() {
+ "wss" | "https" => "https",
+ "ws" | "http" => "http",
+ _ => return Err("unsupported relay scheme".into()),
+ };
+ url.set_scheme(scheme)
+ .map_err(|_| "cannot set scheme".to_owned())?;
+ Ok(url.to_string().trim_end_matches('/').to_owned())
+}
+
+/// What went wrong asking the relay for this machine's pending deletions.
+#[derive(Debug)]
+pub enum PendingError {
+ /// The relay does not know this token (nothing is pending for a
+ /// removed computer, or the token is wrong): ask again much later.
+ Unknown(String),
+ /// Anything else (network, a relay error, an old relay without the
+ /// route): ask again soon.
+ Failed(String),
+}
+
+impl std::fmt::Display for PendingError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ PendingError::Unknown(m) | PendingError::Failed(m) => f.write_str(m),
+ }
+ }
+}
+
+fn pending_http() -> reqwest::Client {
+ reqwest::Client::builder()
+ .timeout(Duration::from_secs(20))
+ .build()
+ .unwrap_or_default()
+}
+
+async fn pending_error(r: reqwest::Response) -> PendingError {
+ let status = r.status();
+ let body = r.text().await.unwrap_or_default();
+ let message = serde_json::from_str::<serde_json::Value>(&body)
+ .ok()
+ .and_then(|v| v["error"].as_str().map(str::to_owned))
+ .unwrap_or(body);
+ let text = format!("relay answered {status}: {message}");
+ if status == http::StatusCode::UNAUTHORIZED {
+ PendingError::Unknown(text)
+ } else {
+ PendingError::Failed(text)
+ }
+}
+
+/// The Spaces this machine's owner deleted while it was removed from the
+/// account ([`crate::pending_deletions`]), read with its machine token.
+pub async fn pending_deletions(
+ relay_url: &str,
+ machine_id: &str,
+ token: &str,
+) -> Result<Vec<crate::pending_deletions::PendingSpace>, PendingError> {
+ #[derive(serde::Deserialize)]
+ struct Answer {
+ #[serde(default)]
+ spaces: Vec<crate::pending_deletions::PendingSpace>,
+ }
+ let base = http_base(relay_url).map_err(PendingError::Failed)?;
+ let r = pending_http()
+ .get(format!("{base}/v1/machines/{machine_id}/pending-deletions"))
+ .bearer_auth(token)
+ .send()
+ .await
+ .map_err(|e| PendingError::Failed(e.to_string()))?;
+ if !r.status().is_success() {
+ return Err(pending_error(r).await);
+ }
+ r.json::<Answer>()
+ .await
+ .map(|a| a.spaces)
+ .map_err(|e| PendingError::Failed(e.to_string()))
+}
+
+/// Tells the relay this machine deleted `space` (one of its pending
+/// deletions). Idempotent.
+pub async fn ack_pending_deletion(
+ relay_url: &str,
+ machine_id: &str,
+ token: &str,
+ space: &str,
+) -> Result<(), PendingError> {
+ let base = http_base(relay_url).map_err(PendingError::Failed)?;
+ let r = pending_http()
+ .post(format!(
+ "{base}/v1/machines/{machine_id}/pending-deletions/{space}/ack"
+ ))
+ .bearer_auth(token)
+ .send()
+ .await
+ .map_err(|e| PendingError::Failed(e.to_string()))?;
+ if !r.status().is_success() {
+ return Err(pending_error(r).await);
+ }
+ Ok(())
+}
+
/// Records the relay keys and owner from the 101 response. With pinned
/// keys, the relay must present one of them (else it is not the relay this
/// machine was set up with, and the session is refused).
@@ -454,6 +568,23 @@ mod tests {
assert!(backoff(20, cap) >= Duration::from_secs(15));
}
+ #[test]
+ fn http_base_turns_the_relay_url_into_its_rest_base() {
+ assert_eq!(
+ http_base("wss://relay.cua.ai").unwrap(),
+ "https://relay.cua.ai"
+ );
+ assert_eq!(
+ http_base("ws://127.0.0.1:9/").unwrap(),
+ "http://127.0.0.1:9"
+ );
+ assert_eq!(
+ http_base("https://r.example/base/").unwrap(),
+ "https://r.example/base"
+ );
+ assert!(http_base("ftp://x").is_err());
+ }
+
#[test]
fn the_first_reconnect_after_a_session_is_spread_and_later_ones_back_off() {
let cap = Duration::from_secs(30);
diff --git a/libs/cua-spacesd/crates/cua-relay/src/directory.rs b/libs/cua-spacesd/crates/cua-relay/src/directory.rs
index f7e5e4a181..0e8ea4dd4b 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/directory.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/directory.rs
@@ -219,7 +219,7 @@ pub struct Directory {
storage: std::sync::Arc<Storage>,
}
-fn hash_token(token: &str) -> String {
+pub(crate) fn hash_token(token: &str) -> String {
hex::encode(ring::digest::digest(&ring::digest::SHA256, token.as_bytes()).as_ref())
}
diff --git a/libs/cua-spacesd/crates/cua-relay/src/lib.rs b/libs/cua-spacesd/crates/cua-relay/src/lib.rs
index 9733d341a7..087c46c451 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/lib.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/lib.rs
@@ -49,6 +49,7 @@ pub mod mesh;
pub mod mux;
pub mod oidc;
pub mod pairing;
+pub mod pending_deletions;
pub mod phone;
pub mod pool;
pub mod proxies;
diff --git a/libs/cua-spacesd/crates/cua-relay/src/pending_deletions.rs b/libs/cua-spacesd/crates/cua-relay/src/pending_deletions.rs
new file mode 100644
index 0000000000..a084c9a55a
--- /dev/null
+++ b/libs/cua-spacesd/crates/cua-relay/src/pending_deletions.rs
@@ -0,0 +1,463 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Pending deletions: the Spaces of a removed computer that its owner asked
+//! to delete (`DELETE /v1/machines/{id}?spaces=delete`, see [`crate::api`]).
+//!
+//! A computer can be removed while it is offline, so the relay cannot
+//! delete its Spaces on it. The relay forgets their records at once (they
+//! leave every list), and keeps a pending-deletion record per computer:
+//! the Spaces' ids and names, who asked, and the computer's token hash at
+//! that moment. The computer (its cua-spacesd) reads the record whenever it
+//! dials the relay, deletes each listed Space through its cua daemon, and
+//! acks each one; the last ack clears the record.
+//!
+//! - `GET /v1/machines/{id}/pending-deletions`: `{computer, spaces: [{id,
+//! name}], requested_at, expires_at}` (an empty `spaces` when nothing is
+//! pending).
+//! - `POST /v1/machines/{id}/pending-deletions/{space}/ack`: `204`;
+//! idempotent (a Space that is not pending is already done).
+//!
+//! Only that computer's machine token may call them: the token it had when
+//! it was removed (the record keeps its hash), or the one it got when it was
+//! added again with the same id. Another computer's token gets `403`, an
+//! account token `403`, an unknown token `401`.
+//!
+//! Deleted means deleted: adding the computer again keeps the record, and
+//! registering a pending Space id again is refused (`410`).
+//!
+//! A record expires [`TTL_SECS`] (90 days) after the request: a computer
+//! that never comes back must not pin it forever. It persists in
+//! `<state>.pending-deletions.json` (atomic replace, private file mode) and
+//! is erased with the account ([`crate::account_erase`]).
+
+// Handlers short-circuit with ready-made responses.
+#![allow(clippy::result_large_err)]
+
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex};
+
+use axum::extract::{Path, State};
+use axum::http::{header, HeaderMap, StatusCode};
+use axum::response::{IntoResponse, Response};
+use axum::routing::{get, post};
+use axum::{Json, Router};
+use serde::{Deserialize, Serialize};
+
+use crate::api::error;
+use crate::assertion::{now_secs, remove_stale_temps, write_private_with};
+use crate::directory::{hash_token, MACHINE_TOKEN_PREFIX};
+use crate::server::{bearer, Relay};
+use crate::storage::{store, Storage};
+
+/// How long a pending-deletion record lives: 90 days.
+pub const TTL_SECS: u64 = 90 * 86_400;
+
+/// One Space to delete.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PendingSpace {
+ /// Its relay machine id (what the computer's daemon knows it by).
+ pub id: String,
+ /// Its name when it was deleted (for people and logs).
+ #[serde(default)]
+ pub name: String,
+}
+
+/// A removed computer's Spaces to delete.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Tombstone {
+ /// The owning account.
+ pub owner: String,
+ /// SHA-256 (hex) of the computer's machine token when it was removed.
+ pub token_sha256: String,
+ /// Who asked (the device or account that removed it), if known.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub requested_by: Option<String>,
+ /// Unix seconds of the (latest) request.
+ pub requested_at: u64,
+ /// What is left to delete.
+ pub spaces: Vec<PendingSpace>,
+}
+
+impl Tombstone {
+ fn expired(&self, now: u64) -> bool {
+ now >= self.requested_at.saturating_add(TTL_SECS)
+ }
+}
+
+/// Every computer's pending deletions, persisted as JSON (atomic replace)
+/// when a file is configured.
+#[derive(Debug, Default)]
+pub struct PendingDeletions {
+ path: Option<PathBuf>,
+ computers: Mutex<BTreeMap<String, Tombstone>>,
+ storage: Arc<Storage>,
+}
+
+impl PendingDeletions {
+ /// In memory only.
+ pub fn in_memory() -> Self {
+ Self::default()
+ }
+
+ /// Loads (or starts) the store persisted at `path`, reporting writes to
+ /// `storage`. Expired records are dropped (on the next write).
+ pub fn open_with(path: PathBuf, storage: Arc<Storage>) -> std::io::Result<Self> {
+ remove_stale_temps(&path)?;
+ let mut computers: BTreeMap<String, Tombstone> = match std::fs::read(&path) {
+ Ok(raw) => serde_json::from_slice(&raw)
+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?,
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(),
+ Err(e) => return Err(e),
+ };
+ let now = now_secs();
+ computers.retain(|_, t| !t.expired(now));
+ Ok(Self {
+ path: Some(path),
+ computers: Mutex::new(computers),
+ storage,
+ })
+ }
+
+ /// Writes `next` and swaps it in; on a failed write nothing changes.
+ fn commit(
+ &self,
+ computers: &mut BTreeMap<String, Tombstone>,
+ next: BTreeMap<String, Tombstone>,
+ ) -> Result<(), String> {
+ if let Some(path) = &self.path {
+ let written = (|| {
+ if let Some(parent) = path.parent() {
+ std::fs::create_dir_all(parent)?;
+ }
+ write_private_with(path, |out| {
+ serde_json::to_writer(&mut *out, &next).map_err(std::io::Error::other)
+ })
+ })();
+ self.storage.record(store::PENDING_DELETIONS, &written);
+ written.map_err(|e| e.to_string())?;
+ }
+ *computers = next;
+ Ok(())
+ }
+
+ /// Records that `computer` (owned by `owner`, whose token hash was
+ /// `token_sha256`) must delete `spaces`. A record already there gains
+ /// them (and the newest token hash); nothing to delete records nothing.
+ pub fn record(
+ &self,
+ computer: &str,
+ owner: &str,
+ token_sha256: &str,
+ requested_by: Option<String>,
+ spaces: Vec<PendingSpace>,
+ ) -> Result<(), String> {
+ self.record_at(
+ computer,
+ owner,
+ token_sha256,
+ requested_by,
+ spaces,
+ now_secs(),
+ )
+ }
+
+ /// [`Self::record`] at `now`.
+ pub fn record_at(
+ &self,
+ computer: &str,
+ owner: &str,
+ token_sha256: &str,
+ requested_by: Option<String>,
+ spaces: Vec<PendingSpace>,
+ now: u64,
+ ) -> Result<(), String> {
+ if spaces.is_empty() {
+ return Ok(());
+ }
+ let mut computers = self.computers.lock().expect("pending deletions");
+ let mut next = computers.clone();
+ next.retain(|_, t| !t.expired(now));
+ let entry = next
+ .entry(computer.to_owned())
+ .or_insert_with(|| Tombstone {
+ owner: owner.to_owned(),
+ token_sha256: String::new(),
+ requested_by: None,
+ requested_at: now,
+ spaces: Vec::new(),
+ });
+ entry.owner = owner.to_owned();
+ entry.token_sha256 = token_sha256.to_owned();
+ entry.requested_by = requested_by;
+ entry.requested_at = now;
+ for s in spaces {
+ if !entry.spaces.iter().any(|e| e.id == s.id) {
+ entry.spaces.push(s);
+ }
+ }
+ self.commit(&mut computers, next)
+ }
+
+ /// `computer`'s record, unless there is none or it expired.
+ pub fn get(&self, computer: &str) -> Option<Tombstone> {
+ self.get_at(computer, now_secs())
+ }
+
+ /// [`Self::get`] at `now`.
+ pub fn get_at(&self, computer: &str, now: u64) -> Option<Tombstone> {
+ self.computers
+ .lock()
+ .expect("pending deletions")
+ .get(computer)
+ .filter(|t| !t.expired(now))
+ .cloned()
+ }
+
+ /// Whether `space` is waiting to be deleted on some computer (its id
+ /// may not be registered again meanwhile).
+ pub fn is_pending_space(&self, space: &str) -> bool {
+ let now = now_secs();
+ self.computers
+ .lock()
+ .expect("pending deletions")
+ .values()
+ .any(|t| !t.expired(now) && t.spaces.iter().any(|s| s.id == space))
+ }
+
+ /// The computer whose record names `token_sha256`, if any (and not
+ /// expired).
+ fn computer_for_token(&self, token_sha256: &str) -> Option<String> {
+ use subtle::ConstantTimeEq as _;
+ let now = now_secs();
+ self.computers
+ .lock()
+ .expect("pending deletions")
+ .iter()
+ .find(|(_, t)| {
+ !t.expired(now)
+ && !t.token_sha256.is_empty()
+ && bool::from(t.token_sha256.as_bytes().ct_eq(token_sha256.as_bytes()))
+ })
+ .map(|(id, _)| id.clone())
+ }
+
+ /// `computer` deleted `space`: it leaves the record, and the last one
+ /// clears it. Returns whether it was pending.
+ pub fn ack(&self, computer: &str, space: &str) -> Result<bool, String> {
+ self.ack_at(computer, space, now_secs())
+ }
+
+ /// [`Self::ack`] at a given time (tests).
+ pub fn ack_at(&self, computer: &str, space: &str, now: u64) -> Result<bool, String> {
+ let mut computers = self.computers.lock().expect("pending deletions");
+ let mut next = computers.clone();
+ next.retain(|_, t| !t.expired(now));
+ let Some(t) = next.get_mut(computer) else {
+ return Ok(false);
+ };
+ let before = t.spaces.len();
+ t.spaces.retain(|s| s.id != space);
+ let was = t.spaces.len() != before;
+ if t.spaces.is_empty() {
+ next.remove(computer);
+ }
+ if !was && next.len() == computers.len() {
+ return Ok(false);
+ }
+ self.commit(&mut computers, next)?;
+ Ok(was)
+ }
+
+ /// Account deletion: forgets every record of `account`'s computers.
+ /// Returns how many there were. The file is rewritten every call, so a
+ /// retry after a failed write finishes the job.
+ pub fn erase_account(&self, account: &str) -> Result<usize, String> {
+ let mut computers = self.computers.lock().expect("pending deletions");
+ let mut next = computers.clone();
+ let before = next.len();
+ next.retain(|_, t| t.owner != account);
+ let gone = before - next.len();
+ self.commit(&mut computers, next)?;
+ Ok(gone)
+ }
+}
+
+pub(crate) fn routes() -> Router<Relay> {
+ Router::new()
+ .route("/v1/machines/{id}/pending-deletions", get(list))
+ .route("/v1/machines/{id}/pending-deletions/{space}/ack", post(ack))
+}
+
+/// Lets only computer `id`'s own machine token through: the one it had
+/// when removed (kept in the record), or its current one.
+fn authorize(relay: &Relay, headers: &HeaderMap, id: &str) -> Result<(), Response> {
+ let Some(token) = bearer(headers, header::AUTHORIZATION.as_str()) else {
+ return Err(error(StatusCode::UNAUTHORIZED, "missing computer token"));
+ };
+ if !token.starts_with(MACHINE_TOKEN_PREFIX) {
+ return Err(error(
+ StatusCode::FORBIDDEN,
+ "only the computer itself reads its pending deletions",
+ ));
+ }
+ let hashed = hash_token(token);
+ if let Some(t) = relay.pending_deletions.get(id) {
+ use subtle::ConstantTimeEq as _;
+ if bool::from(t.token_sha256.as_bytes().ct_eq(hashed.as_bytes())) {
+ return Ok(());
+ }
+ }
+ let other = match relay.directory.machine_for_token(token) {
+ Some(r) if r.id == id => return Ok(()),
+ Some(_) => true,
+ None => relay
+ .pending_deletions
+ .computer_for_token(&hashed)
+ .is_some(),
+ };
+ if other {
+ return Err(error(
+ StatusCode::FORBIDDEN,
+ "computer token belongs to another computer id",
+ ));
+ }
+ Err(error(StatusCode::UNAUTHORIZED, "invalid relay token"))
+}
+
+/// What `GET .../pending-deletions` answers.
+#[derive(Debug, Serialize)]
+struct View {
+ computer: String,
+ spaces: Vec<PendingSpace>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ requested_at: Option<u64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ expires_at: Option<u64>,
+}
+
+async fn list(State(relay): State<Relay>, Path(id): Path<String>, headers: HeaderMap) -> Response {
+ if let Err(r) = authorize(&relay, &headers, &id) {
+ return r;
+ }
+ let t = relay.pending_deletions.get(&id);
+ Json(View {
+ computer: id,
+ requested_at: t.as_ref().map(|t| t.requested_at),
+ expires_at: t.as_ref().map(|t| t.requested_at.saturating_add(TTL_SECS)),
+ spaces: t.map(|t| t.spaces).unwrap_or_default(),
+ })
+ .into_response()
+}
+
+async fn ack(
+ State(relay): State<Relay>,
+ Path((id, space)): Path<(String, String)>,
+ headers: HeaderMap,
+) -> Response {
+ if let Err(r) = authorize(&relay, &headers, &id) {
+ return r;
+ }
+ let space = space.strip_prefix("relay:").unwrap_or(&space);
+ match relay.pending_deletions.ack(&id, space) {
+ Ok(was) => {
+ if was {
+ tracing::info!(computer = %id, space, "pending Space deletion acknowledged");
+ }
+ StatusCode::NO_CONTENT.into_response()
+ }
+ Err(e) => error(StatusCode::INTERNAL_SERVER_ERROR, e),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn space(id: &str) -> PendingSpace {
+ PendingSpace {
+ id: id.into(),
+ name: format!("{id}-name"),
+ }
+ }
+
+ #[test]
+ fn records_merge_ack_clears_and_expire() {
+ let p = PendingDeletions::in_memory();
+ p.record_at("c1", "ada", "h1", None, vec![], 100).unwrap();
+ assert!(
+ p.get_at("c1", 100).is_none(),
+ "nothing to delete records nothing"
+ );
+
+ p.record_at(
+ "c1",
+ "ada",
+ "h1",
+ Some("dev".into()),
+ vec![space("s1")],
+ 100,
+ )
+ .unwrap();
+ p.record_at("c1", "ada", "h2", None, vec![space("s1"), space("s2")], 200)
+ .unwrap();
+ let t = p.get_at("c1", 200).unwrap();
+ assert_eq!(t.spaces, vec![space("s1"), space("s2")]);
+ assert_eq!((t.token_sha256.as_str(), t.requested_at), ("h2", 200));
+
+ // Expiry counts from the latest request.
+ assert!(p.get_at("c1", 200 + TTL_SECS - 1).is_some());
+ assert!(p.get_at("c1", 200 + TTL_SECS).is_none());
+
+ assert!(p.ack_at("c1", "s1", 200).unwrap());
+ assert!(!p.ack_at("c1", "s1", 200).unwrap(), "idempotent");
+ assert!(!p.ack_at("c1", "nope", 200).unwrap());
+ assert!(!p.ack_at("c9", "s2", 200).unwrap());
+ assert!(p.ack_at("c1", "s2", 200).unwrap());
+ assert!(
+ p.computers.lock().unwrap().is_empty(),
+ "the last ack clears it"
+ );
+ }
+
+ #[test]
+ fn persists_drops_expired_on_load_and_erases_by_account() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("state.pending-deletions.json");
+ let p = PendingDeletions::open_with(path.clone(), Arc::default()).unwrap();
+ let now = now_secs();
+ p.record_at(
+ "old",
+ "ada",
+ "h0",
+ None,
+ vec![space("s0")],
+ now - TTL_SECS - 1,
+ )
+ .unwrap();
+ p.record_at("c1", "ada", "h1", None, vec![space("s1")], now)
+ .unwrap();
+ p.record_at("c2", "bob", "h2", None, vec![space("s2")], now)
+ .unwrap();
+ drop(p);
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt as _;
+ let mode = std::fs::metadata(&path).unwrap().permissions().mode();
+ assert_eq!(mode & 0o077, 0, "private file");
+ }
+
+ let p = PendingDeletions::open_with(path.clone(), Arc::default()).unwrap();
+ assert!(p.get("old").is_none(), "expired on load");
+ assert!(p.get("c1").is_some() && p.get("c2").is_some());
+ assert!(p.is_pending_space("s1") && !p.is_pending_space("s0"));
+ assert_eq!(p.computer_for_token("h2").as_deref(), Some("c2"));
+
+ assert_eq!(p.erase_account("ada").unwrap(), 1);
+ assert_eq!(p.erase_account("ada").unwrap(), 0);
+ drop(p);
+ let p = PendingDeletions::open_with(path, Arc::default()).unwrap();
+ assert!(p.get("c1").is_none() && p.get("c2").is_some());
+ }
+}
diff --git a/libs/cua-spacesd/crates/cua-relay/src/server.rs b/libs/cua-spacesd/crates/cua-relay/src/server.rs
index a199c69d8a..a03c21b24a 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/server.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/server.rs
@@ -411,6 +411,8 @@ pub struct Relay {
pub(crate) pairing: Arc<crate::pairing::PairingStore>,
/// Each account's default Space and fallback order.
pub(crate) spaces: Arc<crate::spaces::SpacesSettings>,
+ /// Removed computers' Spaces still to delete on them.
+ pub(crate) pending_deletions: Arc<crate::pending_deletions::PendingDeletions>,
pub(crate) key: Arc<RelayKey>,
/// Open client streams across every machine (S7): the global memory
/// budget's live counter.
@@ -538,6 +540,13 @@ impl Relay {
)?,
None => crate::spaces::SpacesSettings::in_memory(),
};
+ let pending_deletions = match &config.state_file {
+ Some(path) => crate::pending_deletions::PendingDeletions::open_with(
+ path.with_extension("pending-deletions.json"),
+ storage.clone(),
+ )?,
+ None => crate::pending_deletions::PendingDeletions::in_memory(),
+ };
let key = config
.signing_key
.clone()
@@ -593,6 +602,7 @@ impl Relay {
grants: Arc::new(grants),
pairing: Arc::new(pairing),
spaces: Arc::new(spaces),
+ pending_deletions: Arc::new(pending_deletions),
key,
global_streams: Arc::default(),
mesh,
@@ -650,6 +660,7 @@ impl Relay {
.merge(self.hosted_routes())
.merge(crate::pairing::routes())
.merge(crate::spaces::routes())
+ .merge(crate::pending_deletions::routes())
.merge(crate::tcp_shares::routes())
.merge(if self.share_web_enabled() {
crate::share_web::routes()
diff --git a/libs/cua-spacesd/crates/cua-relay/src/storage.rs b/libs/cua-spacesd/crates/cua-relay/src/storage.rs
index 586475deb2..6c7ba1413e 100644
--- a/libs/cua-spacesd/crates/cua-relay/src/storage.rs
+++ b/libs/cua-spacesd/crates/cua-relay/src/storage.rs
@@ -4,7 +4,7 @@
//! The health of the relay's state volume (CUA-1297).
//!
//! Every state store (devices, audit log, device sessions, machine
-//! directory, mesh registry, pool, usage, teams, pairing codes, grants, Space settings)
+//! directory, mesh registry, pool, usage, teams, pairing codes, grants, Space settings, pending deletions)
//! reports each write here. A failed write
//! is counted, logged with a stable line (`state persist failed`, field
//! `alert = "state_persist_failed"`) and remembered as the store's last
@@ -71,9 +71,23 @@ pub mod store {
pub const GRANTS: &str = "grants";
/// `<state>.spaces.json`: each account's default Space settings.
pub const SPACES: &str = "spaces";
+ /// `<state>.pending-deletions.json`: removed computers' Spaces still to
+ /// delete on them.
+ pub const PENDING_DELETIONS: &str = "pending_deletions";
/// Every store, in report order.
- pub const ALL: [&str; 11] = [
- DEVICES, AUDIT, SESSIONS, DIRECTORY, MESH, POOL, USAGE, TEAMS, PAIRING, GRANTS, SPACES,
+ pub const ALL: [&str; 12] = [
+ DEVICES,
+ AUDIT,
+ SESSIONS,
+ DIRECTORY,
+ MESH,
+ POOL,
+ USAGE,
+ TEAMS,
+ PAIRING,
+ GRANTS,
+ SPACES,
+ PENDING_DELETIONS,
];
}
diff --git a/libs/cua-spacesd/crates/cua-relay/tests/account_erase.rs b/libs/cua-spacesd/crates/cua-relay/tests/account_erase.rs
index 85b10a68b8..8b8674ed8a 100644
--- a/libs/cua-spacesd/crates/cua-relay/tests/account_erase.rs
+++ b/libs/cua-spacesd/crates/cua-relay/tests/account_erase.rs
@@ -429,6 +429,7 @@ async fn an_unknown_account_is_a_200_with_zero_counts() {
"mesh_nodes",
"teams",
"grants",
+ "pending_deletions",
] {
assert_eq!(v[k], 0, "{k}: {v}");
}
@@ -716,6 +717,7 @@ async fn it_erases_one_account_everywhere_and_nobody_elses_and_survives_a_restar
"mesh_nodes",
"teams",
"grants",
+ "pending_deletions",
] {
assert_eq!(again[k], 0, "{k}: {again}");
}
@@ -758,6 +760,62 @@ async fn it_erases_one_account_everywhere_and_nobody_elses_and_survives_a_restar
}
#[cfg(unix)]
+#[tokio::test]
+async fn erasing_an_account_drops_its_removed_computers_pending_deletions() {
+ let env = Env::start().await;
+ let ada = env.enroll(ADA).await;
+ let register = |id: String, host: Option<String>| {
+ let http = env.http.clone();
+ let base = env.base.clone();
+ let ada = ada.clone();
+ async move {
+ let r = http
+ .post(format!("{base}/v1/machines"))
+ .bearer_auth(&ada.token)
+ .header(SESSION, &ada.session)
+ .json(&json!({"id": id, "name": id, "host": host}))
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(r.status(), StatusCode::CREATED);
+ r.json::<Value>().await.unwrap()["machine_token"]
+ .as_str()
+ .unwrap()
+ .to_owned()
+ }
+ };
+ let host = format!("host-{}", &uuid::Uuid::new_v4().simple().to_string()[..16]);
+ let token = register(host.clone(), None).await;
+ register(format!("{host}-kid"), Some(host.clone())).await;
+ let r = env
+ .http
+ .delete(format!("{}/v1/machines/{host}?spaces=delete", env.base))
+ .bearer_auth(&ada.token)
+ .header(SESSION, &ada.session)
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(r.status(), StatusCode::NO_CONTENT);
+ let pending = || {
+ env.http
+ .get(format!("{}/v1/machines/{host}/pending-deletions", env.base))
+ .bearer_auth(&token)
+ .send()
+ };
+ assert_eq!(pending().await.unwrap().status(), StatusCode::OK);
+
+ let (s, v) = env.erase(ADA, &[]).await;
+ assert_eq!(s, StatusCode::OK, "{v}");
+ assert_eq!(v["pending_deletions"], 1, "{v}");
+ assert_eq!(
+ pending().await.unwrap().status(),
+ StatusCode::UNAUTHORIZED,
+ "nothing of hers is left"
+ );
+ let (_, again) = env.erase(ADA, &[]).await;
+ assert_eq!(again["pending_deletions"], 0, "{again}");
+}
+
#[tokio::test]
async fn a_failed_write_answers_500_and_the_same_call_again_finishes() {
let env = Env::start().await;
diff --git a/libs/cua-spacesd/crates/cua-relay/tests/pending_deletions.rs b/libs/cua-spacesd/crates/cua-relay/tests/pending_deletions.rs
new file mode 100644
index 0000000000..8215fb5835
--- /dev/null
+++ b/libs/cua-spacesd/crates/cua-relay/tests/pending_deletions.rs
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Pending deletions (`GET /v1/machines/{id}/pending-deletions`, `POST
+//! .../{space}/ack`): removing a computer with `?spaces=delete` records its
+//! Spaces for it to delete when it is next online. Only that computer's
+//! token reads and acks them; adding it again keeps them, and a deleted
+//! Space's id cannot be registered again until it is acked.
+
+use std::path::Path;
+use std::sync::Arc;
+
+use cua_relay::devices::DevicePolicy;
+use cua_relay::oidc::testing::FakeIssuer;
+use cua_relay::oidc::{OidcConfig, OidcValidator};
+use cua_relay::server::{Relay, RelayConfig};
+use reqwest::{Method, StatusCode};
+use serde_json::{json, Value};
+
+const ISSUER: &str = "https://auth.test/realms/cua";
+
+fn config(issuer: &FakeIssuer, dir: &Path) -> RelayConfig {
+ RelayConfig {
+ oidc: Some(Arc::new(OidcValidator::with_jwks(
+ OidcConfig::new(ISSUER),
+ issuer.jwks(),
+ ))),
+ device_policy: DevicePolicy {
+ grace_secs: 14 * 86_400,
+ ..DevicePolicy::default()
+ },
+ state_file: Some(dir.join("machines.json")),
+ ..RelayConfig::default()
+ }
+}
+
+async fn serve(relay: &Relay) -> String {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let base = format!("http://{}", listener.local_addr().unwrap());
+ let relay = relay.clone();
+ tokio::spawn(async move {
+ let _ = relay.serve(listener, std::future::pending()).await;
+ });
+ base
+}
+
+struct Env {
+ base: String,
+ issuer: FakeIssuer,
+ dir: tempfile::TempDir,
+ ada: String,
+ bob: String,
+}
+
+impl Env {
+ async fn start() -> Env {
+ let issuer = FakeIssuer::new(ISSUER);
+ let dir = tempfile::tempdir().unwrap();
+ let relay = Relay::new(config(&issuer, dir.path()));
+ let base = serve(&relay).await;
+ let ada = issuer.token("ada-sub", Some("[email protected]"), "cua-relay", 3000);
+ let bob = issuer.token("bob-sub", Some("[email protected]"), "cua-relay", 3000);
+ Env {
+ base,
+ issuer,
+ dir,
+ ada,
+ bob,
+ }
+ }
+
+ /// A second relay on the same state files (a restart).
+ async fn restart(&self) -> String {
+ serve(&Relay::new(config(&self.issuer, self.dir.path()))).await
+ }
+
+ async fn call_on(
+ base: &str,
+ method: Method,
+ path: &str,
+ token: Option<&str>,
+ body: Option<Value>,
+ ) -> (StatusCode, Value) {
+ let mut r = reqwest::Client::new().request(method, format!("{base}{path}"));
+ if let Some(t) = token {
+ r = r.bearer_auth(t);
+ }
+ if let Some(b) = body {
+ r = r.json(&b);
+ }
+ let r = r.send().await.unwrap();
+ let status = r.status();
+ let text = r.text().await.unwrap_or_default();
+ (
+ status,
+ serde_json::from_str(&text).unwrap_or(Value::String(text)),
+ )
+ }
+
+ async fn call(
+ &self,
+ method: Method,
+ path: &str,
+ token: Option<&str>,
+ body: Option<Value>,
+ ) -> (StatusCode, Value) {
+ Self::call_on(&self.base, method, path, token, body).await
+ }
+
+ /// Registers a computer (or, with `host`, a Space on it); returns its
+ /// machine token.
+ async fn try_register(&self, token: &str, id: &str, host: Option<&str>) -> (StatusCode, Value) {
+ self.call(
+ Method::POST,
+ "/v1/machines",
+ Some(token),
+ Some(json!({"id": id, "name": format!("{id}-name"), "host": host})),
+ )
+ .await
+ }
+
+ async fn register(&self, token: &str, id: &str, host: Option<&str>) -> String {
+ let (s, v) = self.try_register(token, id, host).await;
+ assert!(s.is_success(), "register {id}: {s} {v}");
+ v["machine_token"].as_str().unwrap().to_owned()
+ }
+
+ fn new_id(&self) -> String {
+ format!("m{}", uuid::Uuid::new_v4().simple())
+ }
+
+ async fn listed(&self, token: &str) -> Vec<String> {
+ let (s, v) = self
+ .call(Method::GET, "/v1/machines", Some(token), None)
+ .await;
+ assert_eq!(s, StatusCode::OK, "{v}");
+ v["machines"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|m| m["id"].as_str().unwrap().to_owned())
+ .collect()
+ }
+
+ async fn remove(&self, computer: &str, delete: bool) {
+ let q = if delete { "?spaces=delete" } else { "" };
+ let (s, v) = self
+ .call(
+ Method::DELETE,
+ &format!("/v1/machines/{computer}{q}"),
+ Some(&self.ada),
+ None,
+ )
+ .await;
+ assert_eq!(s, StatusCode::NO_CONTENT, "{v}");
+ }
+
+ async fn pending(&self, computer: &str, token: Option<&str>) -> (StatusCode, Value) {
+ self.call(
+ Method::GET,
+ &format!("/v1/machines/{computer}/pending-deletions"),
+ token,
+ None,
+ )
+ .await
+ }
+
+ async fn ack(&self, computer: &str, space: &str, token: &str) -> StatusCode {
+ self.call(
+ Method::POST,
+ &format!("/v1/machines/{computer}/pending-deletions/{space}/ack"),
+ Some(token),
+ None,
+ )
+ .await
+ .0
+ }
+}
+
+fn ids(v: &Value) -> Vec<String> {
+ let mut out: Vec<String> = v["spaces"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|s| s["id"].as_str().unwrap().to_owned())
+ .collect();
+ out.sort();
+ out
+}
+
+#[tokio::test]
+async fn removing_with_delete_records_the_spaces_for_the_computer_alone() {
+ let env = Env::start().await;
+ let host = env.new_id();
+ let host_token = env.register(&env.ada, &host, None).await;
+ let kid1 = env.new_id();
+ let kid1_token = env.register(&env.ada, &kid1, Some(&host)).await;
+ let kid2 = env.new_id();
+ env.register(&env.ada, &kid2, Some(&host)).await;
+ let other = env.new_id();
+ let other_token = env.register(&env.ada, &other, None).await;
+ let bobs = env.new_id();
+ let bob_token = env.register(&env.bob, &bobs, None).await;
+
+ // Nothing pending yet: the computer reads an empty list.
+ let (s, v) = env.pending(&host, Some(&host_token)).await;
+ assert_eq!(s, StatusCode::OK, "{v}");
+ assert_eq!(v["spaces"], json!([]));
+ assert!(v.get("expires_at").is_none(), "{v}");
+
+ env.remove(&host, true).await;
+ // The Spaces leave the account at once.
+ assert_eq!(env.listed(&env.ada).await, vec![other.clone()]);
+
+ // The removed computer's token still reads its own record.
+ let (s, v) = env.pending(&host, Some(&host_token)).await;
+ assert_eq!(s, StatusCode::OK, "{v}");
+ let mut want = vec![kid1.clone(), kid2.clone()];
+ want.sort();
+ assert_eq!(ids(&v), want);
+ assert_eq!(v["computer"], host.as_str());
+ let names: Vec<&str> = v["spaces"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|s| s["name"].as_str().unwrap())
+ .collect();
+ assert!(names.contains(&format!("{kid1}-name").as_str()), "{v}");
+ let at = v["requested_at"].as_u64().unwrap();
+ assert_eq!(v["expires_at"].as_u64().unwrap(), at + 90 * 86_400);
+
+ // No one else does.
+ let (s, v) = env.pending(&host, Some(&other_token)).await;
+ assert_eq!(s, StatusCode::FORBIDDEN, "another computer of hers: {v}");
+ let (s, _) = env.pending(&host, Some(&bob_token)).await;
+ assert_eq!(s, StatusCode::FORBIDDEN, "another account's computer");
+ let (s, _) = env.pending(&host, Some(&env.ada)).await;
+ assert_eq!(s, StatusCode::FORBIDDEN, "the owner's account token");
+ let (s, _) = env.pending(&host, None).await;
+ assert_eq!(s, StatusCode::UNAUTHORIZED);
+ let (s, _) = env.pending(&host, Some("cmt_not-a-real-token")).await;
+ assert_eq!(s, StatusCode::UNAUTHORIZED);
+ let (s, _) = env.pending(&host, Some(&kid1_token)).await;
+ assert_eq!(
+ s,
+ StatusCode::UNAUTHORIZED,
+ "a deleted Space's own token is gone"
+ );
+ assert_eq!(
+ env.ack(&host, &kid1, &other_token).await,
+ StatusCode::FORBIDDEN
+ );
+ // ... and the computer's token reads only its own id.
+ let (s, _) = env.pending(&other, Some(&host_token)).await;
+ assert_eq!(s, StatusCode::FORBIDDEN);
+
+ // It survives a restart.
+ let restarted = env.restart().await;
+ let (s, v) = Env::call_on(
+ &restarted,
+ Method::GET,
+ &format!("/v1/machines/{host}/pending-deletions"),
+ Some(&host_token),
+ None,
+ )
+ .await;
+ assert_eq!((s, ids(&v)), (StatusCode::OK, want.clone()));
+
+ // Acks are idempotent; the last one clears the record.
+ assert_eq!(
+ env.ack(&host, &kid1, &host_token).await,
+ StatusCode::NO_CONTENT
+ );
+ assert_eq!(
+ env.ack(&host, &kid1, &host_token).await,
+ StatusCode::NO_CONTENT
+ );
+ let (_, v) = env.pending(&host, Some(&host_token)).await;
+ assert_eq!(ids(&v), vec![kid2.clone()]);
+ assert_eq!(
+ env.ack(&host, &format!("relay:{kid2}"), &host_token).await,
+ StatusCode::NO_CONTENT
+ );
+ // Cleared: the removed computer's token now means nothing here.
+ let (s, _) = env.pending(&host, Some(&host_token)).await;
+ assert_eq!(s, StatusCode::UNAUTHORIZED);
+ let raw =
+ std::fs::read_to_string(env.dir.path().join("machines.pending-deletions.json")).unwrap();
+ assert_eq!(raw.trim(), "{}");
+}
+
+#[tokio::test]
+async fn removing_without_delete_records_nothing() {
+ let env = Env::start().await;
+ let host = env.new_id();
+ let host_token = env.register(&env.ada, &host, None).await;
+ let kid = env.new_id();
+ env.register(&env.ada, &kid, Some(&host)).await;
+ env.remove(&host, false).await;
+ let (s, _) = env.pending(&host, Some(&host_token)).await;
+ assert_eq!(s, StatusCode::UNAUTHORIZED);
+ // A detached Space may come back: its id is not refused.
+ env.register(&env.ada, &host, None).await;
+ assert!(env.listed(&env.ada).await.contains(&kid));
+}
+
+#[tokio::test]
+async fn adding_the_computer_again_keeps_the_deletions_and_refuses_the_ids() {
+ let env = Env::start().await;
+ let host = env.new_id();
+ let old_token = env.register(&env.ada, &host, None).await;
+ let kid = env.new_id();
+ env.register(&env.ada, &kid, Some(&host)).await;
+ env.remove(&host, true).await;
+
+ // `cua host setup` adds it again with the same id and a new token.
+ let new_token = env.register(&env.ada, &host, None).await;
+ assert_ne!(new_token, old_token);
+ for token in [&new_token, &old_token] {
+ let (s, v) = env.pending(&host, Some(token)).await;
+ assert_eq!((s, ids(&v)), (StatusCode::OK, vec![kid.clone()]), "{v}");
+ }
+ assert!(
+ !env.listed(&env.ada).await.contains(&kid),
+ "deleted means deleted"
+ );
+
+ // The deleted Space cannot be registered again meanwhile.
+ let (s, v) = env.try_register(&env.ada, &kid, Some(&host)).await;
+ assert_eq!(s, StatusCode::GONE, "{v}");
+ assert!(v["error"].as_str().unwrap().contains("deleted"), "{v}");
+
+ // Once the computer deleted it, the id is free again.
+ assert_eq!(
+ env.ack(&host, &kid, &new_token).await,
+ StatusCode::NO_CONTENT
+ );
+ let (s, v) = env.pending(&host, Some(&new_token)).await;
+ assert_eq!((s, v["spaces"].clone()), (StatusCode::OK, json!([])));
+ env.register(&env.ada, &kid, Some(&host)).await;
+}
+
+#[tokio::test]
+async fn a_second_removal_adds_to_the_record() {
+ let env = Env::start().await;
+ let host = env.new_id();
+ env.register(&env.ada, &host, None).await;
+ let kid1 = env.new_id();
+ env.register(&env.ada, &kid1, Some(&host)).await;
+ env.remove(&host, true).await;
+ let token = env.register(&env.ada, &host, None).await;
+ let kid2 = env.new_id();
+ env.register(&env.ada, &kid2, Some(&host)).await;
+ env.remove(&host, true).await;
+ let (_, v) = env.pending(&host, Some(&token)).await;
+ let mut want = vec![kid1, kid2];
+ want.sort();
+ assert_eq!(ids(&v), want);
+}
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/host_spaces.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/host_spaces.rs
index 44e945fb8b..7f1ebb4b52 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/host_spaces.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/host_spaces.rs
@@ -165,7 +165,7 @@ mod routed_tests {
}
}
-fn read_policy(path: &Path) -> Result<HostPolicy, Status> {
+pub(crate) fn read_policy(path: &Path) -> Result<HostPolicy, Status> {
let raw = std::fs::read(path).map_err(|e| {
status(
Code::FailedPrecondition,
@@ -484,7 +484,9 @@ fn caller_json_of<T>(req: &Request<T>) -> Result<String, Status> {
#[cfg(unix)]
static STARTING: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
-async fn daemon_channel(daemon: &SpacesDaemon) -> Result<tonic::transport::Channel, Status> {
+pub(crate) async fn daemon_channel(
+ daemon: &SpacesDaemon,
+) -> Result<tonic::transport::Channel, Status> {
#[cfg(unix)]
{
let socket = PathBuf::from(&daemon.socket);
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/lib.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/lib.rs
index 7c228d0111..7946737195 100644
--- a/libs/cua-spacesd/crates/cua-spacesd-server/src/lib.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/lib.rs
@@ -22,6 +22,7 @@ pub mod http;
pub mod identity;
pub mod mesh_host;
pub mod peer;
+pub mod pending_deletions;
pub mod phone_control;
pub mod privilege;
pub mod process;
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/src/pending_deletions.rs b/libs/cua-spacesd/crates/cua-spacesd-server/src/pending_deletions.rs
new file mode 100644
index 0000000000..9d41109ad8
--- /dev/null
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/src/pending_deletions.rs
@@ -0,0 +1,261 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! Deleting the Spaces this computer's owner deleted while it was offline.
+//!
+//! Removing a computer from the account with "Also delete its Spaces"
+//! (`DELETE /v1/machines/{id}?spaces=delete`) works while the computer is
+//! offline: the relay forgets the Spaces at once and keeps a pending
+//! deletion for the computer ([`cua_relay::pending_deletions`]). Every time
+//! this host dials the relay ([`cua_relay::client::JoinConfig::dialing`]),
+//! it reads that list with its machine token, deletes each listed Space
+//! through its cua daemon (`DeleteHostSpace`, else `DeleteCloudSpace`: the
+//! sandbox, its files and, for a cloud Space, its cloud resources) as this
+//! machine's owner, and acks each one.
+//!
+//! - It deletes only Spaces on the list, by their relay id.
+//! - A Space that is not here any more counts as deleted and is acked.
+//! - A delete that fails is not acked: the next sweep tries again.
+//! - A computer added again (`cua host setup` after a removal) keeps its
+//! pending deletions, and the relay refuses to register those Spaces again
+//! until they are acked, so they never come back (deleted means deleted).
+
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+
+use cua_proto::env::v1::host_spaces_service_client::HostSpacesServiceClient;
+use cua_proto::env::v1::{DeleteCloudSpaceRequest, DeleteHostSpaceRequest};
+use cua_relay::client::{ack_pending_deletion, pending_deletions, PendingError};
+use tokio::time::Instant;
+use tokio_util::sync::CancellationToken;
+use tonic::{Code, Request};
+
+use crate::host_spaces::{daemon_channel, read_policy};
+
+/// How long one Space's delete may take (a VM's disk can be large).
+const DELETE_TIMEOUT: Duration = Duration::from_secs(10 * 60);
+
+/// Deletes one Space on this machine.
+#[tonic::async_trait]
+pub trait SpaceDeleter: Send + Sync {
+ /// Deletes `space` (a relay machine id). `Ok(true)`: deleted;
+ /// `Ok(false)`: not on this machine (nothing to delete); `Err`: not
+ /// deleted, try again later.
+ async fn delete(&self, space: &str) -> Result<bool, String>;
+}
+
+/// Deletes through the host's cua daemon named in the host policy.
+pub struct DaemonDeleter {
+ policy_file: PathBuf,
+}
+
+impl DaemonDeleter {
+ /// A deleter reading the host policy at `policy_file` on every call.
+ pub fn new(policy_file: impl Into<PathBuf>) -> Self {
+ Self {
+ policy_file: policy_file.into(),
+ }
+ }
+}
+
+#[tonic::async_trait]
+impl SpaceDeleter for DaemonDeleter {
+ async fn delete(&self, space: &str) -> Result<bool, String> {
+ let policy = read_policy(&self.policy_file).map_err(|s| s.message().to_owned())?;
+ let daemon = policy
+ .spaces_daemon
+ .ok_or("this host's policy names no cua daemon; run `cua host setup` on it again")?;
+ let channel = daemon_channel(&daemon)
+ .await
+ .map_err(|s| s.message().to_owned())?;
+ // No caller metadata: the call is this machine's own (its owner's).
+ let mut client = HostSpacesServiceClient::new(channel);
+ let mut req = Request::new(DeleteHostSpaceRequest {
+ space: space.to_owned(),
+ });
+ req.set_timeout(DELETE_TIMEOUT);
+ match client.delete_host_space(req).await {
+ Ok(_) => return Ok(true),
+ Err(s) if s.code() == Code::NotFound => {}
+ Err(s) => return Err(s.message().to_owned()),
+ }
+ // Not a Space this host provides: maybe one in the owner's cloud
+ // that this computer created.
+ let mut req = Request::new(DeleteCloudSpaceRequest {
+ space: space.to_owned(),
+ });
+ req.set_timeout(DELETE_TIMEOUT);
+ match client.delete_cloud_space(req).await {
+ Ok(_) => Ok(true),
+ Err(s) if s.code() == Code::NotFound => Ok(false),
+ Err(s) => Err(s.message().to_owned()),
+ }
+ }
+}
+
+/// What one sweep did.
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub struct SweepReport {
+ /// Deleted here and acked.
+ pub deleted: Vec<String>,
+ /// Not on this machine (already gone) and acked.
+ pub absent: Vec<String>,
+ /// Not deleted or not acked, with why: the next sweep tries again.
+ pub failed: Vec<(String, String)>,
+}
+
+/// Reads this machine's pending deletions, deletes each listed Space with
+/// `deleter` and acks it. Fails only when the list cannot be read.
+pub async fn sweep(
+ relay_url: &str,
+ machine_id: &str,
+ token: &str,
+ deleter: &dyn SpaceDeleter,
+) -> Result<SweepReport, PendingError> {
+ let mut report = SweepReport::default();
+ for space in pending_deletions(relay_url, machine_id, token).await? {
+ // Only a plain relay id names a Space here; anything else is never
+ // passed on (it could name something that is not on the list).
+ if !cua_relay::valid_machine_id(&space.id) {
+ report
+ .failed
+ .push((space.id.clone(), "not a Space id".into()));
+ continue;
+ }
+ let here = match deleter.delete(&space.id).await {
+ Ok(here) => here,
+ Err(e) => {
+ tracing::warn!(space = %space.id, name = %space.name, error = %e, "pending Space deletion failed; retrying later");
+ report.failed.push((space.id, e));
+ continue;
+ }
+ };
+ if let Err(e) = ack_pending_deletion(relay_url, machine_id, token, &space.id).await {
+ report.failed.push((space.id, format!("ack: {e}")));
+ continue;
+ }
+ tracing::info!(space = %space.id, name = %space.name, here, "deleted a Space its owner deleted while this computer was removed");
+ if here {
+ report.deleted.push(space.id);
+ } else {
+ report.absent.push(space.id);
+ }
+ }
+ Ok(report)
+}
+
+/// How the background sweeper runs.
+pub struct PendingDeletionsConfig {
+ /// The relay (`ws(s)://` or `http(s)://`).
+ pub relay_url: String,
+ /// This machine's id.
+ pub machine_id: String,
+ /// The machine token.
+ pub relay_token: String,
+ /// Re-read before every sweep when readable and non-empty (setup
+ /// rotates it under a running service).
+ pub relay_token_file: Option<PathBuf>,
+ /// Notified on every relay dial ([`cua_relay::client::JoinConfig::dialing`]).
+ pub dialing: Arc<tokio::sync::Notify>,
+ /// Least time between two sweeps.
+ pub min_interval: Duration,
+ /// After a sweep with failures: try again this soon.
+ pub retry_interval: Duration,
+ /// After a clean sweep: look again this often even without a dial.
+ pub idle_interval: Duration,
+ /// After the relay did not know the token (nothing pending for a
+ /// removed computer): ask again this late, unless the token changes.
+ pub unknown_interval: Duration,
+}
+
+impl PendingDeletionsConfig {
+ /// Defaults for everything but the relay, id, token and dial signal.
+ pub fn new(
+ relay_url: String,
+ machine_id: String,
+ relay_token: String,
+ dialing: Arc<tokio::sync::Notify>,
+ ) -> Self {
+ Self {
+ relay_url,
+ machine_id,
+ relay_token,
+ relay_token_file: None,
+ dialing,
+ min_interval: Duration::from_secs(60),
+ retry_interval: Duration::from_secs(5 * 60),
+ idle_interval: Duration::from_secs(6 * 60 * 60),
+ unknown_interval: Duration::from_secs(60 * 60),
+ }
+ }
+
+ fn token(&self) -> String {
+ self.relay_token_file
+ .as_deref()
+ .and_then(|p| std::fs::read_to_string(p).ok())
+ .map(|t| t.trim().to_owned())
+ .filter(|t| !t.is_empty())
+ .unwrap_or_else(|| self.relay_token.clone())
+ }
+}
+
+/// Sweeps at start and on each relay dial (at most once per
+/// `min_interval`), until `shutdown`.
+pub async fn run(
+ config: PendingDeletionsConfig,
+ deleter: Arc<dyn SpaceDeleter>,
+ shutdown: CancellationToken,
+) {
+ let mut due = Instant::now();
+ let mut not_before = Instant::now();
+ // The token the relay last did not know: a new one is asked at once.
+ let mut unknown: Option<String> = None;
+ loop {
+ tokio::select! {
+ _ = shutdown.cancelled() => return,
+ _ = tokio::time::sleep_until(due) => {}
+ _ = config.dialing.notified() => {
+ if unknown.as_deref().is_some_and(|t| t != config.token()) {
+ unknown = None;
+ not_before = Instant::now();
+ }
+ if unknown.is_none() {
+ due = due.min(not_before.max(Instant::now()));
+ }
+ continue;
+ }
+ }
+ let token = config.token();
+ let started = Instant::now();
+ not_before = started + config.min_interval;
+ due = match sweep(&config.relay_url, &config.machine_id, &token, &*deleter).await {
+ Ok(r) => {
+ unknown = None;
+ if !r.deleted.is_empty() || !r.absent.is_empty() || !r.failed.is_empty() {
+ tracing::info!(
+ deleted = r.deleted.len(),
+ absent = r.absent.len(),
+ failed = r.failed.len(),
+ "pending Space deletions swept"
+ );
+ }
+ if r.failed.is_empty() {
+ started + config.idle_interval
+ } else {
+ started + config.retry_interval.max(config.min_interval)
+ }
+ }
+ Err(PendingError::Unknown(e)) => {
+ tracing::debug!(error = %e, "no pending Space deletions for this token");
+ unknown = Some(token);
+ not_before = started + config.unknown_interval;
+ not_before
+ }
+ Err(PendingError::Failed(e)) => {
+ tracing::info!(error = %e, "pending Space deletions not read; retrying later");
+ started + config.retry_interval.max(config.min_interval)
+ }
+ };
+ }
+}
diff --git a/libs/cua-spacesd/crates/cua-spacesd-server/tests/pending_deletions.rs b/libs/cua-spacesd/crates/cua-spacesd-server/tests/pending_deletions.rs
new file mode 100644
index 0000000000..c91c07f50b
--- /dev/null
+++ b/libs/cua-spacesd/crates/cua-spacesd-server/tests/pending_deletions.rs
@@ -0,0 +1,508 @@
+// SPDX-License-Identifier: FSL-1.1-MIT
+// Copyright (c) 2026 Cua AI, Inc.
+
+//! A host deletes the Spaces its owner deleted while it was removed from
+//! the account (`DELETE /v1/machines/{id}?spaces=delete` on the relay), the
+//! next time it dials the relay: the real relay in process (a fake OIDC
+//! issuer) and a fake cua daemon on a Unix socket.
+//!
+//! - Only the listed Spaces are deleted; each is acked; a failed delete is
+//! retried on the next sweep; a Space already gone is acked.
+//! - End to end: an offline computer is removed with delete, comes back,
+//! and its Spaces are deleted and acked; one not on the list stays.
+//! - Added again with the same id (a new token): the deletions still run
+//! first, and the deleted Space ids cannot be registered meanwhile.
+
+#![cfg(unix)]
+
+use std::collections::BTreeSet;
+use std::path::Path;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use cua_proto::env::v1::host_spaces_service_server::{HostSpacesService, HostSpacesServiceServer};
+use cua_proto::env::v1::*;
+use cua_relay::devices::DevicePolicy;
+use cua_relay::oidc::testing::FakeIssuer;
+use cua_relay::oidc::{OidcConfig, OidcValidator};
+use cua_relay::server::{Relay, RelayConfig};
+use cua_spacesd_server::host_spaces::CALLER_METADATA;
+use cua_spacesd_server::pending_deletions::{
+ run, sweep, DaemonDeleter, PendingDeletionsConfig, SpaceDeleter,
+};
+use serde_json::{json, Value};
+use tokio_util::sync::CancellationToken;
+
+const ISSUER: &str = "https://auth.test/realms/cua";
+
+struct Env {
+ base: String,
+ ada: String,
+ _issuer: FakeIssuer,
+ _dir: tempfile::TempDir,
+}
+
+impl Env {
+ async fn start() -> Env {
+ let issuer = FakeIssuer::new(ISSUER);
+ let dir = tempfile::tempdir().unwrap();
+ let relay = Relay::new(RelayConfig {
+ oidc: Some(Arc::new(OidcValidator::with_jwks(
+ OidcConfig::new(ISSUER),
+ issuer.jwks(),
+ ))),
+ device_policy: DevicePolicy {
+ grace_secs: 14 * 86_400,
+ ..DevicePolicy::default()
+ },
+ state_file: Some(dir.path().join("machines.json")),
+ ..RelayConfig::default()
+ });
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let base = format!("http://{}", listener.local_addr().unwrap());
+ tokio::spawn(async move {
+ let _ = relay.serve(listener, std::future::pending()).await;
+ });
+ let ada = issuer.token("ada-sub", Some("[email protected]"), "cua-relay", 3000);
+ Env {
+ base,
+ ada,
+ _issuer: issuer,
+ _dir: dir,
+ }
+ }
+
+ async fn call(
+ &self,
+ method: reqwest::Method,
+ path: &str,
+ token: &str,
+ body: Option<Value>,
+ ) -> (reqwest::StatusCode, Value) {
+ let mut r = reqwest::Client::new()
+ .request(method, format!("{}{path}", self.base))
+ .bearer_auth(token);
+ if let Some(b) = body {
+ r = r.json(&b);
+ }
+ let r = r.send().await.unwrap();
+ let s = r.status();
+ let text = r.text().await.unwrap_or_default();
+ (
+ s,
+ serde_json::from_str(&text).unwrap_or(Value::String(text)),
+ )
+ }
+
+ async fn try_register(&self, id: &str, host: Option<&str>) -> (reqwest::StatusCode, Value) {
+ self.call(
+ reqwest::Method::POST,
+ "/v1/machines",
+ &self.ada,
+ Some(json!({"id": id, "name": id, "host": host})),
+ )
+ .await
+ }
+
+ async fn register(&self, id: &str, host: Option<&str>) -> String {
+ let (s, v) = self.try_register(id, host).await;
+ assert!(s.is_success(), "{s} {v}");
+ v["machine_token"].as_str().unwrap().to_owned()
+ }
+
+ async fn remove_with_delete(&self, computer: &str) {
+ let (s, v) = self
+ .call(
+ reqwest::Method::DELETE,
+ &format!("/v1/machines/{computer}?spaces=delete"),
+ &self.ada,
+ None,
+ )
+ .await;
+ assert_eq!(s, reqwest::StatusCode::NO_CONTENT, "{v}");
+ }
+
+ /// The Space ids still pending for `computer` (`None`: the record is
+ /// gone and the token means nothing to the relay).
+ async fn pending(&self, computer: &str, token: &str) -> Option<BTreeSet<String>> {
+ let (s, v) = self
+ .call(
+ reqwest::Method::GET,
+ &format!("/v1/machines/{computer}/pending-deletions"),
+ token,
+ None,
+ )
+ .await;
+ if s == reqwest::StatusCode::UNAUTHORIZED {
+ return None;
+ }
+ assert_eq!(s, reqwest::StatusCode::OK, "{v}");
+ Some(
+ v["spaces"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|s| s["id"].as_str().unwrap().to_owned())
+ .collect(),
+ )
+ }
+
+ async fn listed(&self) -> Vec<String> {
+ let (_, v) = self
+ .call(reqwest::Method::GET, "/v1/machines", &self.ada, None)
+ .await;
+ v["machines"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|m| m["id"].as_str().unwrap().to_owned())
+ .collect()
+ }
+}
+
+fn new_id(tag: &str) -> String {
+ format!("{tag}{}", uuid::Uuid::new_v4().simple())
+}
+
+/// Local Spaces by relay id; the ones in `fail_once` fail their first
+/// delete.
+#[derive(Default)]
+struct FakeSpaces {
+ local: Mutex<BTreeSet<String>>,
+ fail_once: Mutex<BTreeSet<String>>,
+ asked: Mutex<Vec<String>>,
+}
+
+impl FakeSpaces {
+ fn with(local: &[&str]) -> Arc<Self> {
+ let s = Self::default();
+ s.local
+ .lock()
+ .unwrap()
+ .extend(local.iter().map(|s| s.to_string()));
+ Arc::new(s)
+ }
+
+ fn local(&self) -> BTreeSet<String> {
+ self.local.lock().unwrap().clone()
+ }
+
+ fn delete(&self, space: &str) -> Result<bool, String> {
+ self.asked.lock().unwrap().push(space.to_owned());
+ if self.fail_once.lock().unwrap().remove(space) {
+ return Err("the VM is busy".into());
+ }
+ Ok(self.local.lock().unwrap().remove(space))
+ }
+}
+
+#[tonic::async_trait]
+impl SpaceDeleter for FakeSpaces {
+ async fn delete(&self, space: &str) -> Result<bool, String> {
+ FakeSpaces::delete(self, space)
+ }
+}
+
+#[tokio::test]
+async fn a_sweep_deletes_only_the_listed_spaces_and_retries_a_failure() {
+ let env = Env::start().await;
+ let host = new_id("m");
+ let token = env.register(&host, None).await;
+ let (a, b, gone) = (new_id("space-"), new_id("space-"), new_id("space-"));
+ for kid in [&a, &b, &gone] {
+ env.register(kid, Some(&host)).await;
+ }
+ let spaces = FakeSpaces::with(&[&a, &b, "space-not-on-the-list"]);
+ spaces.fail_once.lock().unwrap().insert(b.clone());
+
+ // Nothing pending yet: nothing is touched.
+ let r = sweep(&env.base, &host, &token, &*spaces).await.unwrap();
+ assert_eq!(r, Default::default());
+
+ env.remove_with_delete(&host).await;
+ let r = sweep(&env.base, &host, &token, &*spaces).await.unwrap();
+ assert_eq!(r.deleted, vec![a.clone()]);
+ assert_eq!(r.absent, vec![gone.clone()], "already gone: acked");
+ assert_eq!(r.failed.len(), 1);
+ assert_eq!(r.failed[0].0, b);
+ assert_eq!(
+ env.pending(&host, &token).await,
+ Some(BTreeSet::from([b.clone()])),
+ "the failed one stays pending"
+ );
+
+ let r = sweep(&env.base, &host, &token, &*spaces).await.unwrap();
+ assert_eq!((r.deleted, r.failed), (vec![b.clone()], vec![]));
+ assert_eq!(env.pending(&host, &token).await, None, "all acked: cleared");
+ assert_eq!(
+ spaces.local(),
+ BTreeSet::from(["space-not-on-the-list".to_owned()])
+ );
+ assert!(!spaces
+ .asked
+ .lock()
+ .unwrap()
+ .iter()
+ .any(|s| s == "space-not-on-the-list"));
+}
+
+/// The host's cua daemon: deletes the Spaces it has, by relay id, and
+/// records whether each call came as this machine's own (no caller).
+#[derive(Clone)]
+struct FakeDaemon {
+ spaces: Arc<FakeSpaces>,
+ forwarded: Arc<Mutex<Vec<String>>>,
+}
+
+#[tonic::async_trait]
+impl HostSpacesService for FakeDaemon {
+ async fn get_host_spaces(
+ &self,
+ _: tonic::Request<GetHostSpacesRequest>,
+ ) -> Result<tonic::Response<GetHostSpacesResponse>, tonic::Status> {
+ Err(tonic::Status::unimplemented("get_host_spaces"))
+ }
+
+ async fn create_host_space(
+ &self,
+ _: tonic::Request<CreateHostSpaceRequest>,
+ ) -> Result<tonic::Response<CreateHostSpaceResponse>, tonic::Status> {
+ Err(tonic::Status::unimplemented("create_host_space"))
+ }
+
+ async fn cancel_host_space(
+ &self,
+ _: tonic::Request<CancelHostSpaceRequest>,
+ ) -> Result<tonic::Response<CancelHostSpaceResponse>, tonic::Status> {
+ Err(tonic::Status::unimplemented("cancel_host_space"))
+ }
+
+ async fn set_host_space_power(
+ &self,
+ _: tonic::Request<SetHostSpacePowerRequest>,
+ ) -> Result<tonic::Response<SetHostSpacePowerResponse>, tonic::Status> {
+ Err(tonic::Status::unimplemented("set_host_space_power"))
+ }
+
+ async fn set_host_warm(
+ &self,
+ _: tonic::Request<SetHostWarmRequest>,
+ ) -> Result<tonic::Response<SetHostWarmResponse>, tonic::Status> {
+ Err(tonic::Status::unimplemented("set_host_warm"))
+ }
+
+ async fn set_host_space_egress(
+ &self,
+ _: tonic::Request<SetHostSpaceEgressRequest>,
+ ) -> Result<tonic::Response<SetHostSpaceEgressResponse>, tonic::Status> {
+ Err(tonic::Status::unimplemented("set_host_space_egress"))
+ }
+
+ async fn diagnose_host(
+ &self,
+ _: tonic::Request<DiagnoseHostRequest>,
+ ) -> Result<tonic::Response<DiagnoseHostResponse>, tonic::Status> {
+ Err(tonic::Status::unimplemented("diagnose_host"))
+ }
+
+ async fn delete_host_space(
+ &self,
+ req: tonic::Request<DeleteHostSpaceRequest>,
+ ) -> Result<tonic::Response<DeleteHostSpaceResponse>, tonic::Status> {
+ if let Some(who) = req.metadata().get(CALLER_METADATA) {
+ self.forwarded
+ .lock()
+ .unwrap()
+ .push(who.to_str().unwrap_or("").to_owned());
+ }
+ let space = req.into_inner().space;
+ match self.spaces.delete(&space) {
+ Ok(true) => Ok(tonic::Response::new(DeleteHostSpaceResponse {
+ message: format!("Deleted relay:{space}"),
+ })),
+ Ok(false) => Err(tonic::Status::not_found(format!(
+ "{space} is not a Space this host provides"
+ ))),
+ Err(e) => Err(tonic::Status::unavailable(e)),
+ }
+ }
+
+ async fn delete_cloud_space(
+ &self,
+ req: tonic::Request<DeleteCloudSpaceRequest>,
+ ) -> Result<tonic::Response<DeleteCloudSpaceResponse>, tonic::Status> {
+ Err(tonic::Status::not_found(format!(
+ "relay:{} is not a Space in your cloud that this computer created",
+ req.into_inner().space
+ )))
+ }
+}
+
+async fn serve_daemon(socket: &Path, daemon: FakeDaemon) {
+ let listener = tokio::net::UnixListener::bind(socket).unwrap();
+ let router =
+ tonic::service::Routes::new(HostSpacesServiceServer::new(daemon)).into_axum_router();
+ tokio::spawn(async move {
+ let _ = axum::serve(listener, router).await;
+ });
+}
+
+/// A host: its policy names the fake daemon.
+struct Host {
+ dir: tempfile::TempDir,
+ spaces: Arc<FakeSpaces>,
+ forwarded: Arc<Mutex<Vec<String>>>,
+}
+
+impl Host {
+ async fn start(local: &[&str]) -> Host {
+ let dir = tempfile::tempdir().unwrap();
+ let socket = dir.path().join("daemon.sock");
+ let spaces = FakeSpaces::with(local);
+ let forwarded = Arc::new(Mutex::new(Vec::new()));
+ serve_daemon(
+ &socket,
+ FakeDaemon {
+ spaces: spaces.clone(),
+ forwarded: forwarded.clone(),
+ },
+ )
+ .await;
+ std::fs::write(
+ dir.path().join("host.json"),
+ json!({
+ "owner": "ada-sub",
+ "provide_spaces": true,
+ "spaces_daemon": {"socket": socket, "cua_home": ""},
+ })
+ .to_string(),
+ )
+ .unwrap();
+ Host {
+ dir,
+ spaces,
+ forwarded,
+ }
+ }
+
+ /// The computer comes online: it dials the relay (refused or not) and
+ /// sweeps on each dial, as `cua-spacesd join` with a host policy does.
+ fn come_online(&self, env: &Env, id: &str, token_file: &Path) -> CancellationToken {
+ let stop = CancellationToken::new();
+ let dialing = Arc::new(tokio::sync::Notify::new());
+ let unused = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let local = unused.local_addr().unwrap();
+ drop(unused);
+ let mut join = cua_relay::client::JoinConfig::new(
+ env.base.replace("http://", "ws://"),
+ String::new(),
+ id.to_owned(),
+ local,
+ );
+ join.relay_token_file = Some(token_file.to_owned());
+ join.max_backoff = Duration::from_millis(200);
+ join.dialing = Some(dialing.clone());
+ let mut cfg = PendingDeletionsConfig::new(
+ join.relay_url.clone(),
+ id.to_owned(),
+ String::new(),
+ dialing,
+ );
+ cfg.relay_token_file = Some(token_file.to_owned());
+ cfg.min_interval = Duration::from_millis(100);
+ cfg.retry_interval = Duration::from_millis(100);
+ tokio::spawn(cua_relay::client::run(join, stop.clone()));
+ tokio::spawn(run(
+ cfg,
+ Arc::new(DaemonDeleter::new(self.dir.path().join("host.json"))),
+ stop.clone(),
+ ));
+ stop
+ }
+}
+
+/// Waits (bounded) until `ok` holds.
+macro_rules! eventually {
+ ($what:expr, $ok:expr) => {{
+ let mut done = false;
+ for _ in 0..200 {
+ if $ok {
+ done = true;
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ assert!(done, "timed out waiting for: {}", $what);
+ }};
+}
+
+#[tokio::test]
+async fn an_offline_computer_removed_with_delete_deletes_its_spaces_when_it_comes_back() {
+ let env = Env::start().await;
+ let id = new_id("m");
+ let token = env.register(&id, None).await;
+ let (a, b) = (new_id("space-"), new_id("space-"));
+ env.register(&a, Some(&id)).await;
+ env.register(&b, Some(&id)).await;
+ let keep = new_id("space-");
+ let host = Host::start(&[&a, &b, &keep]).await;
+ host.spaces.fail_once.lock().unwrap().insert(b.clone());
+
+ // Offline (never dialed), removed with its Spaces deleted: they leave
+ // the account at once.
+ env.remove_with_delete(&id).await;
+ assert!(env.listed().await.is_empty());
+ assert_eq!(
+ env.pending(&id, &token).await,
+ Some(BTreeSet::from([a.clone(), b.clone()]))
+ );
+
+ // It comes back.
+ let token_file = host.dir.path().join("relay-token");
+ std::fs::write(&token_file, &token).unwrap();
+ let _stop = host.come_online(&env, &id, &token_file);
+ eventually!(
+ "both Spaces deleted and acked",
+ env.pending(&id, &token).await.is_none()
+ );
+ assert_eq!(host.spaces.local(), BTreeSet::from([keep.clone()]));
+ assert!(!host.spaces.asked.lock().unwrap().contains(&keep));
+ assert!(
+ host.forwarded.lock().unwrap().is_empty(),
+ "the deletes are this machine's own calls"
+ );
+}
+
+#[tokio::test]
+async fn a_computer_added_again_still_deletes_its_deleted_spaces() {
+ let env = Env::start().await;
+ let id = new_id("m");
+ let old_token = env.register(&id, None).await;
+ let kid = new_id("space-");
+ env.register(&kid, Some(&id)).await;
+ let host = Host::start(&[&kid]).await;
+ env.remove_with_delete(&id).await;
+
+ // `cua host setup` adds it again under the same id with a new token
+ // before it ever dialed: deleted means deleted.
+ let new_token = env.register(&id, None).await;
+ assert_ne!(old_token, new_token);
+ let (s, _) = env.try_register(&kid, Some(&id)).await;
+ assert_eq!(
+ s,
+ reqwest::StatusCode::GONE,
+ "not registered again meanwhile"
+ );
+
+ let token_file = host.dir.path().join("relay-token");
+ std::fs::write(&token_file, &new_token).unwrap();
+ let _stop = host.come_online(&env, &id, &token_file);
+ eventually!(
+ "the deleted Space is deleted and acked",
+ env.pending(&id, &new_token).await == Some(BTreeSet::new())
+ );
+ assert!(host.spaces.local().is_empty());
+ assert!(!env.listed().await.contains(&kid));
+ // Acked: the id is free again.
+ env.register(&kid, Some(&id)).await;
+}
diff --git a/libs/cua-spacesd/crates/cua-spacesd/src/main.rs b/libs/cua-spacesd/crates/cua-spacesd/src/main.rs
index 934a77aab9..ce2c4611dd 100644
--- a/libs/cua-spacesd/crates/cua-spacesd/src/main.rs
+++ b/libs/cua-spacesd/crates/cua-spacesd/src/main.rs
@@ -1038,6 +1038,27 @@ async fn serve(args: ServeArgs, join: Option<JoinArgs>) -> Result<Exit, BoxError
.pin_jwks_json(json)
.map_err(|e| format!("CUA_RELAY_JWKS_JSON: {e}"))?;
}
+ // A host deletes the Spaces its owner deleted while it was removed
+ // from the account, each time it dials the relay
+ // (cua_spacesd_server::pending_deletions).
+ if let Some(policy) = host_policy.clone() {
+ let dialing = Arc::new(tokio::sync::Notify::new());
+ join_config.dialing = Some(dialing.clone());
+ let mut pending = cua_spacesd_server::pending_deletions::PendingDeletionsConfig::new(
+ join.relay.clone(),
+ machine_id.clone(),
+ join.relay_token.clone().unwrap_or_default(),
+ dialing,
+ );
+ pending.relay_token_file = join.relay_token_file.clone();
+ tokio::spawn(cua_spacesd_server::pending_deletions::run(
+ pending,
+ Arc::new(cua_spacesd_server::pending_deletions::DaemonDeleter::new(
+ policy,
+ )),
+ shutdown.clone(),
+ ));
+ }
// Account mode: relay-signed principal assertions authorize
// callers (in addition to the local token).
ctx.auth().set_external(Arc::new(
diff --git a/libs/cua/crates/cua-cli/src/host.rs b/libs/cua/crates/cua-cli/src/host.rs
index d799517f87..2ef04b790f 100644
--- a/libs/cua/crates/cua-cli/src/host.rs
+++ b/libs/cua/crates/cua-cli/src/host.rs
@@ -2156,9 +2156,14 @@ pub async fn remove_machine(
String::new()
} else if delete_spaces {
format!(
- " Its {} Space{} and their files will be deleted (the files only if the computer answers).",
- children.len(),
- if children.len() == 1 { "" } else { "s" }
+ " Its {} leave{} your account now, and the computer deletes {}, files included, the next time it comes online.",
+ if children.len() == 1 {
+ "Space".to_string()
+ } else {
+ format!("{} Spaces", children.len())
+ },
+ if children.len() == 1 { "s" } else { "" },
+ if children.len() == 1 { "it" } else { "them" }
)
} else {
format!(