Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(ownership): team deletion removes team owned and team bot owned entities (#7703)
Will Hutchinson7f7bda5b612b
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex cb9335148a..79718b43bf 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -219,6 +219,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -360,6 +361,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -1229,6 +1231,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/entity_registry/shared_entity_registry","crates/filter_ast","crates/folds/agent_fold","crates/frecency",@@ -1764,6 +1767,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/entity_registry/shared_entity_registry","crates/filter_ast","crates/folds/agent_fold","crates/frecency",@@ -1925,6 +1929,7 @@"crates/macro_queues","crates/macro_redis","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -2309,6 +2314,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/entity_registry/shared_entity_registry","crates/filter_ast","crates/folds/agent_fold","crates/frecency",@@ -2404,6 +2410,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/entity_registry/shared_entity_registry","crates/filter_ast","crates/folds/agent_fold","crates/frecency",@@ -3190,6 +3197,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -4766,6 +4774,7 @@"crates/cowlike","crates/document_storage_service_client","crates/document_sub_type",+ "crates/entity_registry/shared_entity_registry","crates/genai_telemetry","crates/harness_id","crates/harness_token",@@ -4897,6 +4906,7 @@"crates/macro_event_topics","crates/macro_queues","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -5015,6 +5025,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -6283,6 +6294,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -7283,6 +7295,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -7726,6 +7739,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/entity_registry/shared_entity_registry","crates/filter_ast","crates/folds/agent_fold","crates/frecency",@@ -9586,6 +9600,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/entity_registry/shared_entity_registry","crates/filter_ast","crates/folds/agent_fold","crates/frecency",@@ -9947,6 +9962,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -10486,6 +10502,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -10889,6 +10906,7 @@"crates/cowlike","crates/document_storage_service_client","crates/document_sub_type",+ "crates/entity_registry/shared_entity_registry","crates/genai_telemetry","crates/harness_id","crates/harness_token",@@ -11126,6 +11144,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",@@ -11217,6 +11236,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/entity_registry/shared_entity_registry","crates/filter_ast","crates/folds/agent_fold","crates/frecency",@@ -11320,6 +11340,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/entity_registry/shared_entity_registry","crates/filter_ast","crates/folds/agent_fold","crates/frecency",@@ -12177,6 +12198,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/entity_registry/shared_entity_registry","crates/filter_ast","crates/folds/agent_fold","crates/frecency",@@ -12289,6 +12311,7 @@"crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation",+ "crates/entity_registry/shared_entity_registry","crates/filter_ast","crates/folds/agent_fold","crates/frecency",@@ -12601,6 +12624,7 @@"crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_sync_service_jwt","crates/macro_tower_layers","crates/macro_user_id",diff --git a/.sqlx/query-3317b176c3743666b64b1c4189f881064130838b1c285b3e8bc1085a2e95d877.json b/.sqlx/query-3317b176c3743666b64b1c4189f881064130838b1c285b3e8bc1085a2e95d877.jsonnew file mode 100644index 0000000000..939d1def31--- /dev/null+++ b/.sqlx/query-3317b176c3743666b64b1c4189f881064130838b1c285b3e8bc1085a2e95d877.json@@ -0,0 +1,38 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT source_id, source_type AS \"source_type: EntityAccessSourceType\"\n FROM entity_access ORDER BY id",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "source_id",+ "type_info": "Text"+ },+ {+ "ordinal": 1,+ "name": "source_type: EntityAccessSourceType",+ "type_info": {+ "Custom": {+ "name": "entity_access_source_type",+ "kind": {+ "Enum": [+ "channel",+ "team",+ "user",+ "bot"+ ]+ }+ }+ }+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ false,+ false+ ]+ },+ "hash": "3317b176c3743666b64b1c4189f881064130838b1c285b3e8bc1085a2e95d877"+}diff --git a/.sqlx/query-54b1b8d5631c23afdc951c97e5809b08ff47e47b80840df527aa7406a8ff079c.json b/.sqlx/query-54b1b8d5631c23afdc951c97e5809b08ff47e47b80840df527aa7406a8ff079c.jsonnew file mode 100644index 0000000000..b2d76516e8--- /dev/null+++ b/.sqlx/query-54b1b8d5631c23afdc951c97e5809b08ff47e47b80840df527aa7406a8ff079c.json@@ -0,0 +1,27 @@+{+ "db_name": "PostgreSQL",+ "query": "DELETE FROM entity_access WHERE source_type = $1 AND source_id = $2",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ {+ "Custom": {+ "name": "entity_access_source_type",+ "kind": {+ "Enum": [+ "channel",+ "team",+ "user",+ "bot"+ ]+ }+ }+ },+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "54b1b8d5631c23afdc951c97e5809b08ff47e47b80840df527aa7406a8ff079c"+}diff --git a/.sqlx/query-57d2c507ccedb8631d5a8102730934ed0b3b0d311bd5b00dafe75511d05b2882.json b/.sqlx/query-57d2c507ccedb8631d5a8102730934ed0b3b0d311bd5b00dafe75511d05b2882.jsonnew file mode 100644index 0000000000..78d574a7e8--- /dev/null+++ b/.sqlx/query-57d2c507ccedb8631d5a8102730934ed0b3b0d311bd5b00dafe75511d05b2882.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT id\n FROM bots\n WHERE team_id = $1\n ORDER BY id\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "57d2c507ccedb8631d5a8102730934ed0b3b0d311bd5b00dafe75511d05b2882"+}diff --git a/.sqlx/query-75e76748496e7530486ad6b0f51e3fea1b1c203d4906e9dd462f14872b042d66.json b/.sqlx/query-75e76748496e7530486ad6b0f51e3fea1b1c203d4906e9dd462f14872b042d66.jsonnew file mode 100644index 0000000000..82a58b6042--- /dev/null+++ b/.sqlx/query-75e76748496e7530486ad6b0f51e3fea1b1c203d4906e9dd462f14872b042d66.json@@ -0,0 +1,81 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT id, entity_type, owner_type AS \"owner_type: OwnerType\", owner_id,\n created_at, updated_at, deleted_at\n FROM entity\n WHERE owner_type = $1 AND owner_id = $2\n ORDER BY created_at DESC, id DESC\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "entity_type",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "owner_type: OwnerType",+ "type_info": {+ "Custom": {+ "name": "entity_owner_type",+ "kind": {+ "Enum": [+ "user",+ "bot",+ "team"+ ]+ }+ }+ }+ },+ {+ "ordinal": 3,+ "name": "owner_id",+ "type_info": "Text"+ },+ {+ "ordinal": 4,+ "name": "created_at",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 5,+ "name": "updated_at",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 6,+ "name": "deleted_at",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ {+ "Custom": {+ "name": "entity_owner_type",+ "kind": {+ "Enum": [+ "user",+ "bot",+ "team"+ ]+ }+ }+ },+ "Text"+ ]+ },+ "nullable": [+ false,+ false,+ false,+ false,+ false,+ false,+ true+ ]+ },+ "hash": "75e76748496e7530486ad6b0f51e3fea1b1c203d4906e9dd462f14872b042d66"+}diff --git a/Cargo.lock b/Cargo.lockindex b999a5f52f..dae7c7a519 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -379,6 +379,7 @@ dependencies = ["rustls 0.23.45","serde","serde_json",+ "shared_entity_registry","sqlx","thiserror 2.0.18","tokio",@@ -648,6 +649,7 @@ dependencies = ["serde","serde_json","share_permission_db_utils",+ "shared_entity_registry","sqlx","strum 0.27.2","thiserror 2.0.18",@@ -1924,6 +1926,8 @@ dependencies = ["aws-sdk-sqs","axum","base64 0.22.1",+ "bot_id",+ "bots","calendar_events","channels","chrono",@@ -1946,6 +1950,7 @@ dependencies = ["email_db_client","email_validator","entity_access",+ "entity_registry","favorites","foreign_entity","fusionauth",@@ -4035,6 +4040,7 @@ dependencies = ["serde","serde_json","share_permission_db_utils",+ "shared_entity_registry","sqlx","thiserror 2.0.18","tokio",@@ -6343,14 +6349,21 @@ name = "document_storage_service_client"version = "0.1.0"dependencies = ["anyhow",+ "bot_id","model",+ "model_owner","models_bulk_upload","models_permissions","reqwest 0.13.4",+ "rootcause","serde","serde_json",+ "shared_entity_registry","thiserror 2.0.18",+ "tokio","tracing",+ "uuid",+ "wiremock","workspace-hack",]@@ -6480,6 +6493,7 @@ dependencies = ["macro_event_broker","macro_event_topics","macro_service_urls",+ "macro_sha_count_client","macro_sync_service_jwt","macro_tower_layers","macro_user_id",@@ -6507,6 +6521,7 @@ dependencies = ["serde_json","sha2 0.10.9","share_permission_db_utils",+ "shared_entity_registry","sqlx","sqs_client","sync_service_client",@@ -14876,6 +14891,7 @@ dependencies = ["serde","serde_json","share_permission_db_utils",+ "shared_entity_registry","sqlx","sqs_client","system_properties",@@ -17456,6 +17472,8 @@ dependencies = ["model-entity","model_owner","rootcause",+ "serde",+ "serde_json","thiserror 2.0.18","uuid","workspace-hack",@@ -18976,11 +18994,13 @@ dependencies = ["async-stripe","async-trait","axum",+ "bot_id","channels","chrono","contacts","database_env_vars","entity_access",+ "entity_access_db_utils","generic_email_domains","macro_authorization","macro_db_migrator",@@ -18992,6 +19012,7 @@ dependencies = ["model-entity","model-error-response","model_notifications",+ "model_owner","model_user","models_email","models_pagination",@@ -19003,6 +19024,7 @@ dependencies = ["schemars 1.2.1","serde","serde_json",+ "shared_entity_registry","sqlx","sqs_client","thiserror 2.0.18",diff --git a/crates/agent_harness/Cargo.toml b/crates/agent_harness/Cargo.tomlindex 3fccda0a8b..ebee8170cf 100644--- a/crates/agent_harness/Cargo.toml+++ b/crates/agent_harness/Cargo.toml@@ -55,6 +55,7 @@ model-entity = { path = "../model-entity" }model_notifications = { path = "../model_notifications" }model_owner = { path = "../model_owner" }notification = { path = "../notification" }+shared_entity_registry = { path = "../entity_registry/shared_entity_registry" }messages = { path = "../messages", features = ["broker", "ports"] }opentelemetry = { workspace = true, features = ["trace"] }reqwest = { workspace = true }diff --git a/crates/agent_harness/src/domain/service/lifecycle.rs b/crates/agent_harness/src/domain/service/lifecycle.rsindex dcc717c09c..d4839f79be 100644--- a/crates/agent_harness/src/domain/service/lifecycle.rs+++ b/crates/agent_harness/src/domain/service/lifecycle.rs@@ -3,6 +3,8 @@use agent_fold::domain::model::TurnSignal;use agent_session::domain::session::StopReason;+use model_owner::Owner;+use shared_entity_registry::OwnedPurgeOutcome;use super::*;@@ -83,6 +85,26 @@ where.map_err(into_session_error)}+ #[tracing::instrument(+ err,+ skip(self, expected_owner),+ fields(%id, owner.kind = ?expected_owner.owner_type())+ )]+ async fn purge_owned_session(+ &self,+ id: AgentSessionId,+ expected_owner: &Owner,+ ) -> agent_session::domain::error::Result<OwnedPurgeOutcome> {+ let Some(session) = self.inner.sessions.find_session(id).await? else {+ return Ok(OwnedPurgeOutcome::Purged);+ };+ if session.owner_id != *expected_owner {+ return Ok(OwnedPurgeOutcome::OwnedElsewhere);+ }+ self.session_deleted(id).await?;+ Ok(OwnedPurgeOutcome::Purged)+ }+async fn control_event(&self,id: AgentSessionId,diff --git a/crates/agent_harness/src/domain/service/test.rs b/crates/agent_harness/src/domain/service/test.rsindex e3e94866cc..15ed19c79e 100644--- a/crates/agent_harness/src/domain/service/test.rs+++ b/crates/agent_harness/src/domain/service/test.rs@@ -3,6 +3,7 @@//! recording announcer. Only the edges are doubles.mod chat_reply;+mod owned_purge;mod quota;mod user_cleanup;mod warm;diff --git a/crates/agent_harness/src/domain/service/test/owned_purge.rs b/crates/agent_harness/src/domain/service/test/owned_purge.rsnew file mode 100644index 0000000000..9b3abd936f--- /dev/null+++ b/crates/agent_harness/src/domain/service/test/owned_purge.rs@@ -0,0 +1,72 @@+use super::*;+use model_owner::Owner;+use shared_entity_registry::OwnedPurgeOutcome;++const TEAM: Uuid = Uuid::from_u128(0x7EA3);+const OTHER_TEAM: Uuid = Uuid::from_u128(0x7EA4);++#[tokio::test]+async fn purge_owned_session_releases_and_deletes_sessions_their_owner_still_holds() {+ let (service, repo, containers, _, _) = harness();+ for owner in [Owner::Team(TEAM), Owner::Bot(BotId::TEST_A)] {+ let id = disconnected_session_owned_by(&repo, &containers, owner.clone()).await;++ let outcome = service.purge_owned_session(id, &owner).await.unwrap();++ assert_eq!(outcome, OwnedPurgeOutcome::Purged);+ assert!(repo.get(id).await.is_err(), "the row is gone");+ assert!(containers.container(id).is_none(), "the sandbox is gone");+ }+ assert_eq!(containers.torn_down(), 2);+}++#[tokio::test]+async fn purge_owned_session_leaves_a_session_another_owner_holds() {+ let (service, repo, containers, _, _) = harness();+ let id = disconnected_session_owned_by(&repo, &containers, Owner::Team(TEAM)).await;++ for other in [+ Owner::Team(OTHER_TEAM),+ Owner::Bot(BotId::TEST_A),+ Owner::User(sender()),+ ] {+ let outcome = service.purge_owned_session(id, &other).await.unwrap();+ assert_eq!(outcome, OwnedPurgeOutcome::OwnedElsewhere);+ }++ assert!(repo.get(id).await.is_ok());+ assert!(containers.container(id).is_some());+ assert_eq!(containers.torn_down(), 0);+}++#[tokio::test]+async fn purge_owned_session_of_a_missing_session_is_purged_and_releases_nothing_more() {+ let (service, repo, containers, _, _) = harness();+ let id = disconnected_session_owned_by(&repo, &containers, Owner::Team(TEAM)).await;+ let owner = Owner::Team(TEAM);+ service.purge_owned_session(id, &owner).await.unwrap();++ for missing in [id, AgentSessionId::new()] {+ let outcome = service.purge_owned_session(missing, &owner).await.unwrap();+ assert_eq!(outcome, OwnedPurgeOutcome::Purged);+ }++ assert_eq!(containers.torn_down(), 1, "a retry is a no-op");+}++#[tokio::test]+async fn purge_owned_session_keeps_the_row_on_teardown_failure_and_a_retry_finishes() {+ let (service, repo, containers, _, _) = harness();+ let owner = Owner::Team(TEAM);+ let id = disconnected_session_owned_by(&repo, &containers, owner.clone()).await;+ containers.fail_next_teardown();++ assert!(service.purge_owned_session(id, &owner).await.is_err());+ assert!(repo.get(id).await.is_ok());+ assert!(containers.container(id).is_some());++ let outcome = service.purge_owned_session(id, &owner).await.unwrap();+ assert_eq!(outcome, OwnedPurgeOutcome::Purged);+ assert!(repo.get(id).await.is_err());+ assert_eq!(containers.torn_down(), 1);+}diff --git a/crates/agent_harness/src/outbound/cursor/manager/test.rs b/crates/agent_harness/src/outbound/cursor/manager/test.rsindex 351bb7b5c7..2bb6412e86 100644--- a/crates/agent_harness/src/outbound/cursor/manager/test.rs+++ b/crates/agent_harness/src/outbound/cursor/manager/test.rs@@ -68,6 +68,10 @@ impl AgentSessionRepo for StubSessions {unimplemented!("the manager never looks sessions up by egress token")}+ async fn find(&self, _id: AgentSessionId) -> SessionResult<Option<AgentSession>> {+ unimplemented!("the manager never probes for a missing session")+ }+async fn preview(&self,_viewer: &MacroUserIdStr<'static>,diff --git a/crates/agent_harness/src/outbound/routing/test.rs b/crates/agent_harness/src/outbound/routing/test.rsindex d479f76178..0f5b6aecd8 100644--- a/crates/agent_harness/src/outbound/routing/test.rs+++ b/crates/agent_harness/src/outbound/routing/test.rs@@ -127,6 +127,10 @@ impl AgentSessionRepo for FixedBotSessions {unimplemented!("the router never looks sessions up by egress token")}+ async fn find(&self, _id: AgentSessionId) -> SessionResult<Option<AgentSession>> {+ unimplemented!("the router never probes for a missing session")+ }+async fn preview(&self,_viewer: &MacroUserIdStr<'static>,diff --git a/crates/agent_session/Cargo.toml b/crates/agent_session/Cargo.tomlindex 82649df11a..69fa298d24 100644--- a/crates/agent_session/Cargo.toml+++ b/crates/agent_session/Cargo.toml@@ -81,6 +81,7 @@ model-entity = { path = "../model-entity", optional = true }model_owner = { path = "../model_owner" }models_permissions = { path = "../models_permissions" }share_permission_db_utils = { path = "../share_permission_db_utils", optional = true }+shared_entity_registry = { path = "../entity_registry/shared_entity_registry" }clap = { workspace = true, optional = true }macro_authorization = { path = "../macro_authorization", default-features = false, optional = true }macro_user_id = { path = "../macro_user_id" }diff --git a/crates/agent_session/src/domain/ports.rs b/crates/agent_session/src/domain/ports.rsindex bddcd4688a..8861d5bafc 100644--- a/crates/agent_session/src/domain/ports.rs+++ b/crates/agent_session/src/domain/ports.rs@@ -13,6 +13,7 @@ use bots::domain::models::BotId;use macro_user_id::user_id::MacroUserIdStr;use macro_uuid::Uuid;use model_owner::{Owner, OwnerType};+use shared_entity_registry::OwnedPurgeOutcome;use std::num::NonZeroUsize;/// A bidirectional connection to an agent runtime.@@ -383,6 +384,10 @@ pub trait AgentSessionRepo: Send + Sync + 'static {/// Get an agent session by id.fn get(&self, id: AgentSessionId) -> impl Future<Output = Result<AgentSession>> + Send;+ /// The agent session with this id, or `None` when there is none.+ fn find(&self, id: AgentSessionId)+ -> impl Future<Output = Result<Option<AgentSession>>> + Send;+/// The sessions among `ids` that exist, each with what a chip shows and/// whether a materialized grant lets `viewer` see it: their own grant,/// one through a channel they are still in, or one through their teams -@@ -1151,6 +1156,15 @@ pub trait AgentSessionNotificationRecipient: Send + Sync + 'static {/// The session is going away: release its live resources and delete it.fn session_deleted(&self, id: AgentSessionId) -> impl Future<Output = Result<()>> + Send;+ /// Release and delete one session while `expected_owner` still owns it.+ /// Internal owner removal only. A missing session is already purged; one+ /// under another owner is untouched.+ fn purge_owned_session(+ &self,+ id: AgentSessionId,+ expected_owner: &Owner,+ ) -> impl Future<Output = Result<OwnedPurgeOutcome>> + Send;+/// A control operation the live connection has to be told about. Returns/// the action id the caller correlates against the fold stream, and/// whether the action went out or waits in the session's queue.diff --git a/crates/agent_session/src/domain/routines/service/test.rs b/crates/agent_session/src/domain/routines/service/test.rsindex 43a0274e6e..7aa107ff35 100644--- a/crates/agent_session/src/domain/routines/service/test.rs+++ b/crates/agent_session/src/domain/routines/service/test.rs@@ -12,6 +12,7 @@ use agent_runtime_protocol::domain::schema::v0::ToRuntimeMessage;use bots::domain::models::BotId;use macro_user_id::user_id::MacroUserIdStr;use macro_uuid::Uuid;+use shared_entity_registry::OwnedPurgeOutcome;use std::sync::{Arc, Mutex};#[test]@@ -135,6 +136,13 @@ impl AgentSessionNotificationRecipient for Controls {async fn session_deleted(&self, _: AgentSessionId) -> SessionResult<()> {panic!("not a routine capability")}+ async fn purge_owned_session(+ &self,+ _: AgentSessionId,+ _: &Owner,+ ) -> SessionResult<OwnedPurgeOutcome> {+ panic!("not a routine capability")+ }async fn edit_queued_control(&self,_: AgentSessionId,diff --git a/crates/agent_session/src/domain/service.rs b/crates/agent_session/src/domain/service.rsindex 36f05778fc..9c1ea771cb 100644--- a/crates/agent_session/src/domain/service.rs+++ b/crates/agent_session/src/domain/service.rs@@ -161,6 +161,12 @@ pub trait AgentSessionService: Send + Sync + 'static {/// Get a persisted agent session by id.fn get_session(&self, id: AgentSessionId) -> impl Future<Output = Result<AgentSession>> + Send;+ /// A persisted agent session by id, or `None` when there is none.+ fn find_session(+ &self,+ id: AgentSessionId,+ ) -> impl Future<Output = Result<Option<AgentSession>>> + Send;+/// Append a frame observed on the runtime's behalf by something other/// than its session actor - the egress proxy's tool approvals - and push/// it to the session's viewers. Any replica may call this; the frame is@@ -757,6 +763,10 @@ whereself.repo.get(id).await}+ async fn find_session(&self, id: AgentSessionId) -> Result<Option<AgentSession>> {+ self.repo.find(id).await+ }+async fn set_turn_prompter(&self, id: AgentSessionId, prompter: &TurnPrompter) -> Result<()> {self.repo.set_turn_prompter(id, prompter).await}@@ -1663,6 +1673,10 @@ whereself.repo.get(id).await}+ async fn find(&self, id: AgentSessionId) -> Result<Option<AgentSession>> {+ self.repo.find(id).await+ }+async fn preview(&self,viewer: &MacroUserIdStr<'static>,diff --git a/crates/agent_session/src/domain/service/test.rs b/crates/agent_session/src/domain/service/test.rsindex aaf97a6594..e40c6bb733 100644--- a/crates/agent_session/src/domain/service/test.rs+++ b/crates/agent_session/src/domain/service/test.rs@@ -570,6 +570,10 @@ impl AgentSessionRepo for BlockingPromptLogs {self.repo.get(id).await}+ async fn find(&self, id: AgentSessionId) -> Result<Option<AgentSession>> {+ self.repo.find(id).await+ }+async fn preview(&self,viewer: &MacroUserIdStr<'static>,diff --git a/crates/agent_session/src/inbound/axum_router.rs b/crates/agent_session/src/inbound/axum_router.rsindex 212e636b56..5b628f50f4 100644--- a/crates/agent_session/src/inbound/axum_router.rs+++ b/crates/agent_session/src/inbound/axum_router.rs@@ -69,6 +69,8 @@ pub mod sharing;/// Routes associating pull requests with sessions.pub mod pull_requests;+mod owned_purge;+/// Shared state for the agent session router: the agent session service plus/// the authorization state the request extractors authenticate against.pub struct AgentSessionRouterState<T, Access, Auth> {@@ -228,6 +230,10 @@ where"/user/{user_id}",delete(delete_user_sessions_handler::<R, Access, Auth>),)+ .route(+ "/internal/{session_id}",+ delete(owned_purge::purge_owned_session_handler::<R, Access, Auth>),+ ).route("/{session_id}/control",post(control_agent_session_handler::<R, Access, Auth>),diff --git a/crates/agent_session/src/inbound/axum_router/owned_purge.rs b/crates/agent_session/src/inbound/axum_router/owned_purge.rsnew file mode 100644index 0000000000..326977dc12--- /dev/null+++ b/crates/agent_session/src/inbound/axum_router/owned_purge.rs@@ -0,0 +1,49 @@+//! Thin transport adapter for the owner-checked session delete that team+//! deletion calls once per session it removes.++use axum::{+ extract::{Path, Query, State},+ http::StatusCode,+};+use entity_access::domain::ports::EntityAccessService;+use macro_authorization::{InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService};+use macro_uuid::Uuid;+use model_owner::Owner;+use serde::Deserialize;+use shared_entity_registry::OwnedPurgeOutcome;++use super::AgentSessionControlState;+use crate::domain::model::AgentSessionId;+use crate::domain::ports::AgentSessionNotificationRecipient;++#[derive(Debug, Deserialize)]+pub(super) struct OwnerQuery {+ owner: Owner,+}++/// Internal owner removal: never grants delete authority to a user, bot, or+/// harness token. 204 once the session is gone, including when it already+/// was. 409 when another owner holds it, and nothing was deleted. 500 when+/// teardown failed; the purge converges, so the caller retries.+#[tracing::instrument(skip_all, fields(%session_id, owner.kind = ?owner.owner_type()))]+pub(super) async fn purge_owned_session_handler<R, Access, Auth>(+ State(state): State<AgentSessionControlState<R, Access, Auth>>,+ _internal: MacroAuthorizationExtractor<Auth, InternalOnly>,+ Path(session_id): Path<Uuid>,+ Query(OwnerQuery { owner }): Query<OwnerQuery>,+) -> StatusCode+where+ R: AgentSessionNotificationRecipient,+ Access: EntityAccessService,+ Auth: MacroAuthorizationService,+{+ let id = AgentSessionId::new_from_uuid(session_id);+ match state.recipient.purge_owned_session(id, &owner).await {+ Ok(OwnedPurgeOutcome::Purged) => StatusCode::NO_CONTENT,+ Ok(OwnedPurgeOutcome::OwnedElsewhere) => StatusCode::CONFLICT,+ Err(error) => {+ tracing::error!(error = ?error, "unable to purge the owned agent session");+ StatusCode::INTERNAL_SERVER_ERROR+ }+ }+}diff --git a/crates/agent_session/src/inbound/axum_router/test.rs b/crates/agent_session/src/inbound/axum_router/test.rsindex 21e6a7e3c1..261b8fa646 100644--- a/crates/agent_session/src/inbound/axum_router/test.rs+++ b/crates/agent_session/src/inbound/axum_router/test.rs@@ -1309,6 +1309,7 @@ async fn an_external_open_carries_its_instructions() {);}+mod owned_purge;mod read;mod user_cleanup;diff --git a/crates/agent_session/src/inbound/axum_router/test/owned_purge.rs b/crates/agent_session/src/inbound/axum_router/test/owned_purge.rsnew file mode 100644index 0000000000..d07e8eeea5--- /dev/null+++ b/crates/agent_session/src/inbound/axum_router/test/owned_purge.rs@@ -0,0 +1,207 @@+use super::*;+use crate::domain::error::Result as SessionResult;+use crate::domain::ports::{AcceptedControl, QueuedControl};+use shared_entity_registry::OwnedPurgeOutcome;++const INTERNAL_KEY: &str = "test-internal-key";+const SESSION: Uuid = Uuid::from_u128(0x5E55);++/// Answers every purge with `answer`, or with an error the shared mapping+/// would turn into 409 when it is `None`, and records what reached it.+struct OwnedSessions {+ answer: Option<OwnedPurgeOutcome>,+ calls: Mutex<Vec<(AgentSessionId, Owner)>>,+}++impl OwnedSessions {+ fn answering(answer: Option<OwnedPurgeOutcome>) -> Arc<Self> {+ Arc::new(Self {+ answer,+ calls: Mutex::default(),+ })+ }++ fn calls(&self) -> Vec<(AgentSessionId, Owner)> {+ self.calls.lock().unwrap().clone()+ }+}++impl AgentSessionNotificationRecipient for OwnedSessions {+ async fn purge_owned_session(+ &self,+ id: AgentSessionId,+ expected_owner: &Owner,+ ) -> SessionResult<OwnedPurgeOutcome> {+ self.calls+ .lock()+ .unwrap()+ .push((id, expected_owner.clone()));+ self.answer.ok_or(AgentSessionError::Disconnected(id))+ }+ async fn delete_user_sessions(&self, _: MacroUserIdStr<'static>) -> SessionResult<()> {+ unreachable!()+ }+ async fn session_deleted(&self, _: AgentSessionId) -> SessionResult<()> {+ unreachable!()+ }+ async fn control_event(+ &self,+ _: AgentSessionId,+ _: ControlEvent,+ ) -> SessionResult<AcceptedControl> {+ unreachable!()+ }+ async fn queued_controls(&self, _: AgentSessionId) -> SessionResult<Vec<QueuedControl>> {+ unreachable!()+ }+ async fn edit_queued_control(+ &self,+ _: AgentSessionId,+ _: AgentActionId,+ _: String,+ _: Option<MacroUserIdStr<'static>>,+ ) -> SessionResult<()> {+ unreachable!()+ }+ async fn remove_queued_control(+ &self,+ _: AgentSessionId,+ _: AgentActionId,+ _: Option<MacroUserIdStr<'static>>,+ ) -> SessionResult<()> {+ unreachable!()+ }+ async fn steer_queued_control(+ &self,+ _: AgentSessionId,+ _: AgentActionId,+ _: Option<MacroUserIdStr<'static>>,+ ) -> SessionResult<()> {+ unreachable!()+ }+ async fn set_sandbox_size(&self, _: AgentSessionId, _: SandboxSize) -> SessionResult<()> {+ unreachable!()+ }+ async fn session_harness(+ &self,+ _: AgentSessionId,+ ) -> SessionResult<Option<harness_id::HarnessId>> {+ unreachable!()+ }+}++async fn purge(recipient: &Arc<OwnedSessions>, uri: &str, headers: &[(&str, &str)]) -> StatusCode {+ let auth = MacroAuthorizationServiceImpl::new(+ FakeJwtValidator,+ InternalAuthConfig {+ api_key: INTERNAL_KEY.into(),+ default_user_id: None,+ },+ SelfBotAuthorizer,+ NoUserApiKeyAuthorizer,+ )+ .with_harness_authorizer(SelfHarnessAuthorizer);+ let router: Router = agent_session_control_router(AgentSessionControlState::new(+ Arc::clone(recipient),+ Arc::new(entity_access::domain::ports::NoOpEntityAccessService),+ MacroAuthorizationState::new(Arc::new(auth)),+ ));+ let mut request = Request::builder().method("DELETE").uri(uri);+ for (key, value) in headers {+ request = request.header(*key, *value);+ }+ router+ .oneshot(request.body(Body::empty()).unwrap())+ .await+ .unwrap()+ .status()+}++async fn purge_internally(recipient: &Arc<OwnedSessions>, uri: &str) -> StatusCode {+ purge(+ recipient,+ uri,+ &[(macro_authorization::INTERNAL_API_KEY_HEADER, INTERNAL_KEY)],+ )+ .await+}++fn team_purge_uri() -> String {+ format!("/internal/{SESSION}?owner={TEAM_ID}")+}++#[tokio::test]+async fn each_purge_outcome_answers_its_own_status() {+ for (answer, status) in [+ (Some(OwnedPurgeOutcome::Purged), StatusCode::NO_CONTENT),+ (+ Some(OwnedPurgeOutcome::OwnedElsewhere),+ StatusCode::CONFLICT,+ ),+ (None, StatusCode::INTERNAL_SERVER_ERROR),+ ] {+ let recipient = OwnedSessions::answering(answer);++ assert_eq!(+ purge_internally(&recipient, &team_purge_uri()).await,+ status+ );+ assert_eq!(+ recipient.calls(),+ [(AgentSessionId::new_from_uuid(SESSION), Owner::Team(TEAM_ID))]+ );+ }+}++#[tokio::test]+async fn the_owner_query_reaches_the_recipient_as_a_principal() {+ let recipient = OwnedSessions::answering(Some(OwnedPurgeOutcome::Purged));+ let bot = Owner::Bot(BotId::TEST_A);+ let encoded_bot = bot.principal_id().replace('|', "%7C");++ let status = purge_internally(+ &recipient,+ &format!("/internal/{SESSION}?owner={encoded_bot}"),+ )+ .await;++ assert_eq!(status, StatusCode::NO_CONTENT);+ assert_eq!(+ recipient.calls(),+ [(AgentSessionId::new_from_uuid(SESSION), bot)]+ );+}++#[tokio::test]+async fn only_the_internal_key_may_purge() {+ let recipient = OwnedSessions::answering(Some(OwnedPurgeOutcome::Purged));+ let bearer = format!("Bearer {OWNER}");+ for headers in [+ vec![],+ vec![("authorization", bearer.as_str())],+ vec![(BOT_TOKEN_HEADER, BOT_TOKEN), (BOT_SCOPE_HEADER, "user")],+ vec![(HARNESS_TOKEN_HEADER, HARNESS_TOKEN)],+ vec![(macro_authorization::INTERNAL_API_KEY_HEADER, "bad-key")],+ ] {+ let status = purge(&recipient, &team_purge_uri(), &headers).await;+ assert!(status.is_client_error(), "{headers:?} answered {status}");+ }+ assert!(recipient.calls().is_empty());+}++#[tokio::test]+async fn a_malformed_purge_is_refused_before_the_recipient() {+ let recipient = OwnedSessions::answering(Some(OwnedPurgeOutcome::Purged));+ for uri in [+ format!("/internal/not-a-session?owner={TEAM_ID}"),+ format!("/internal/{SESSION}"),+ format!("/internal/{SESSION}?owner=not-an-owner"),+ ] {+ assert_eq!(+ purge_internally(&recipient, &uri).await,+ StatusCode::BAD_REQUEST,+ "{uri}"+ );+ }+ assert!(recipient.calls().is_empty());+}diff --git a/crates/agent_session/src/inbound/axum_router/test/user_cleanup.rs b/crates/agent_session/src/inbound/axum_router/test/user_cleanup.rsindex ef1bab1fc7..7073da1f92 100644--- a/crates/agent_session/src/inbound/axum_router/test/user_cleanup.rs+++ b/crates/agent_session/src/inbound/axum_router/test/user_cleanup.rs@@ -1,6 +1,7 @@use super::*;use crate::domain::error::Result as SessionResult;use crate::domain::ports::{AcceptedControl, QueuedControl};+use shared_entity_registry::OwnedPurgeOutcome;#[derive(Default)]struct Recipient(Mutex<Vec<MacroUserIdStr<'static>>>);@@ -13,6 +14,13 @@ impl AgentSessionNotificationRecipient for Recipient {async fn session_deleted(&self, _: AgentSessionId) -> SessionResult<()> {unreachable!()}+ async fn purge_owned_session(+ &self,+ _: AgentSessionId,+ _: &Owner,+ ) -> SessionResult<OwnedPurgeOutcome> {+ unreachable!()+ }async fn control_event(&self,_: AgentSessionId,diff --git a/crates/agent_session/src/outbound/postgres.rs b/crates/agent_session/src/outbound/postgres.rsindex e3a2b54b3c..4616806b6c 100644--- a/crates/agent_session/src/outbound/postgres.rs+++ b/crates/agent_session/src/outbound/postgres.rs@@ -477,6 +477,10 @@ impl<B: BotFacts + 'static> AgentSessionRepo for PgAgentSessionRepo<B> {}async fn get(&self, id: AgentSessionId) -> Result<AgentSession> {+ Ok(self.find(id).await?.context("agent session not found")?)+ }++ async fn find(&self, id: AgentSessionId) -> Result<Option<AgentSession>> {let row = sqlx::query_as!(AgentSessionRow,r#"@@ -499,10 +503,9 @@ impl<B: BotFacts + 'static> AgentSessionRepo for PgAgentSessionRepo<B> {).fetch_optional(&self.pool).await- .context("failed to get agent session")?- .context("agent session not found")?;+ .context("failed to get agent session")?;- Ok(row.try_into()?)+ Ok(row.map(AgentSession::try_from).transpose()?)}async fn preview(diff --git a/crates/agent_session/src/outbound/postgres/test.rs b/crates/agent_session/src/outbound/postgres/test.rsindex 3b5f307203..3572b7196a 100644--- a/crates/agent_session/src/outbound/postgres/test.rs+++ b/crates/agent_session/src/outbound/postgres/test.rs@@ -795,6 +795,23 @@ async fn get_missing_session_errors(pool: PgPool) {assert!(AgentSessionRepo::get(&repo, missing).await.is_err());}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn find_answers_none_only_for_a_missing_session(pool: PgPool) {+ let repo = test_repo(&pool);+ let bot_id = create_test_bot(&pool).await;+ let stored = create_session(&repo, new_session(bot_id, None, None)).await;++ let found = AgentSessionRepo::find(&repo, stored.id)+ .await+ .expect("find a stored session");+ let missing = AgentSessionRepo::find(&repo, AgentSessionId::new())+ .await+ .expect("find a missing session");++ assert_eq!(found.map(|session| session.id), Some(stored.id));+ assert!(missing.is_none());+}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn delete_removes_session(pool: PgPool) {let repo = test_repo(&pool);diff --git a/crates/agent_session/src/testing.rs b/crates/agent_session/src/testing.rsindex 1624910cf5..ba01162c6f 100644--- a/crates/agent_session/src/testing.rs+++ b/crates/agent_session/src/testing.rs@@ -259,15 +259,19 @@ impl AgentSessionRepo for InMemoryAgentSessionRepo {}async fn get(&self, id: AgentSessionId) -> Result<AgentSession> {+ self.find(id).await?.ok_or_else(|| {+ AgentSessionError::Unknown(anyhow::anyhow!("no agent session {}", id.as_uuid()))+ })+ }++ async fn find(&self, id: AgentSessionId) -> Result<Option<AgentSession>> {self.session_reads.fetch_add(1, Ordering::Relaxed);- self.sessions+ Ok(self+ .sessions.lock().expect("in-memory session store is not poisoned").get(&id)- .cloned()- .ok_or_else(|| {- AgentSessionError::Unknown(anyhow::anyhow!("no agent session {}", id.as_uuid()))- })+ .cloned())}async fn find_all_for_thread(&self, thread_id: Uuid) -> Result<Vec<AgentSession>> {diff --git a/crates/bots/Cargo.toml b/crates/bots/Cargo.tomlindex 3120fe15db..f589540e97 100644--- a/crates/bots/Cargo.toml+++ b/crates/bots/Cargo.toml@@ -29,7 +29,7 @@ inbound = ["ports",]outbound = ["dep:sqlx", "dep:macro_uuid", "dep:entity_registry", "dep:model_owner", "dep:rootcause", "ports"]-ports = ["dep:entity_access", "entity_access/ports", "dep:tokio", "messages/ports"]+ports = ["dep:entity_access", "entity_access/ports", "dep:rootcause", "dep:tokio", "messages/ports"]test-utils = ["dep:mockall", "ports"][dependencies]diff --git a/crates/bots/fixtures/team_roster.sql b/crates/bots/fixtures/team_roster.sqlnew file mode 100644index 0000000000..e555e830a6--- /dev/null+++ b/crates/bots/fixtures/team_roster.sql@@ -0,0 +1,17 @@+INSERT INTO macro_user (id, username, email, stripe_customer_id) VALUES+('91000000-0000-0000-0000-000000000001', 'roster-owner', '[email protected]', 'team-roster-owner'),+('91000000-0000-0000-0000-000000000002', 'roster-other', '[email protected]', 'team-roster-other'),+('91000000-0000-0000-0000-000000000003', 'roster-empty', '[email protected]', 'team-roster-empty');+INSERT INTO "User" (id, email, macro_user_id) VALUES+('macro|[email protected]', '[email protected]', '91000000-0000-0000-0000-000000000001'),+('macro|[email protected]', '[email protected]', '91000000-0000-0000-0000-000000000002'),+('macro|[email protected]', '[email protected]', '91000000-0000-0000-0000-000000000003');+INSERT INTO team (id, name, owner_id) VALUES+('91000000-0000-0000-0000-000000000011', 'Roster team', 'macro|[email protected]'),+('91000000-0000-0000-0000-000000000012', 'Other team', 'macro|[email protected]'),+('91000000-0000-0000-0000-000000000013', 'Team without bots', 'macro|[email protected]');+INSERT INTO bots (id, kind, owner_user_id, team_id, name, handle, created_by) VALUES+('91000000-0000-0000-0000-000000000022', 'owned', NULL, '91000000-0000-0000-0000-000000000011', 'Retired team bot', 'roster-retired', 'macro|[email protected]'),+('91000000-0000-0000-0000-000000000021', 'owned', NULL, '91000000-0000-0000-0000-000000000011', 'Live team bot', 'roster-live', 'macro|[email protected]'),+('91000000-0000-0000-0000-000000000023', 'owned', NULL, '91000000-0000-0000-0000-000000000012', 'Other team bot', 'roster-other', 'macro|[email protected]'),+('91000000-0000-0000-0000-000000000024', 'owned', 'macro|[email protected]', NULL, 'User bot', 'roster-user', 'macro|[email protected]');diff --git a/crates/bots/src/domain/ports.rs b/crates/bots/src/domain/ports.rsindex 1458854474..e9927163bb 100644--- a/crates/bots/src/domain/ports.rs+++ b/crates/bots/src/domain/ports.rs@@ -10,6 +10,7 @@ use super::models::{use bot_token::HashedBotToken;use entity_access::domain::models::{EntityAccessReceipt, MemberParticipantRole};use macro_user_id::user_id::MacroUserIdStr;+use rootcause::Report;use std::{collections::HashMap, future::Future};use uuid::Uuid;@@ -394,6 +395,19 @@ pub trait McpAppCatalog: Send + Sync + 'static {-> impl Future<Output = Result<bool, BotError>> + Send;}+/// Every bot a team has owned, for removing the team.+///+/// `bots.team_id` is the only link from a team bot to its team, and it+/// cascades when the team row is deleted, so ask before deleting the team.+pub trait TeamBotRoster: Send + Sync + 'static {+ /// The bots of `team_id` in id order, soft-deleted bots included. An+ /// unknown team has none.+ fn team_bot_ids(+ &self,+ team_id: Uuid,+ ) -> impl Future<Output = Result<Vec<BotId>, Report>> + Send;+}+/// Bot service error.#[derive(Debug, thiserror::Error)]pub enum BotError {diff --git a/crates/bots/src/outbound/pg_bots_repo.rs b/crates/bots/src/outbound/pg_bots_repo.rsindex 8dbd5e90c9..435695de35 100644--- a/crates/bots/src/outbound/pg_bots_repo.rs+++ b/crates/bots/src/outbound/pg_bots_repo.rs@@ -4,6 +4,7 @@mod tests;mod owner_grants;+mod team_roster;use crate::domain::{models::{diff --git a/crates/bots/src/outbound/pg_bots_repo/team_roster.rs b/crates/bots/src/outbound/pg_bots_repo/team_roster.rsnew file mode 100644index 0000000000..330278ae7e--- /dev/null+++ b/crates/bots/src/outbound/pg_bots_repo/team_roster.rs@@ -0,0 +1,30 @@+//! Team bot roster for team deletion, owned by the bots adapter.++#[cfg(test)]+mod test;++use bot_id::BotId;+use rootcause::{Report, prelude::*};+use uuid::Uuid;++use super::PgBotsRepo;+use crate::domain::ports::TeamBotRoster;++impl TeamBotRoster for PgBotsRepo {+ #[tracing::instrument(skip(self), err)]+ async fn team_bot_ids(&self, team_id: Uuid) -> Result<Vec<BotId>, Report> {+ let ids = sqlx::query_scalar!(+ r#"+ SELECT id+ FROM bots+ WHERE team_id = $1+ ORDER BY id+ "#,+ team_id,+ )+ .fetch_all(&self.pool)+ .await+ .context("unable to list the bots of a team")?;+ Ok(ids.into_iter().map(BotId::new_from_uuid).collect())+ }+}diff --git a/crates/bots/src/outbound/pg_bots_repo/team_roster/test.rs b/crates/bots/src/outbound/pg_bots_repo/team_roster/test.rsnew file mode 100644index 0000000000..e6bbfde0e9--- /dev/null+++ b/crates/bots/src/outbound/pg_bots_repo/team_roster/test.rs@@ -0,0 +1,28 @@+use super::*;+use crate::domain::ports::BotRepo;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::PgPool;++const TEAM: Uuid = Uuid::from_u128(0x91000000_0000_0000_0000_000000000011);+const OTHER_TEAM: Uuid = Uuid::from_u128(0x91000000_0000_0000_0000_000000000012);+const TEAM_WITHOUT_BOTS: Uuid = Uuid::from_u128(0x91000000_0000_0000_0000_000000000013);+const UNKNOWN_TEAM: Uuid = Uuid::from_u128(0x91000000_0000_0000_0000_0000000000ff);++fn bot(n: u128) -> BotId {+ BotId::new_from_uuid(Uuid::from_u128(0x91000000_0000_0000_0000_000000000020 + n))+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("team_roster"))+)]+async fn team_bot_ids_include_soft_deleted_bots_and_exclude_other_owners(pool: PgPool) {+ let repo = PgBotsRepo::new(pool);+ assert!(repo.delete_bot(bot(2)).await.unwrap());+ assert!(repo.get_bot(bot(2)).await.unwrap().is_none());++ assert_eq!(repo.team_bot_ids(TEAM).await.unwrap(), vec![bot(1), bot(2)]);+ assert_eq!(repo.team_bot_ids(OTHER_TEAM).await.unwrap(), vec![bot(3)]);+ assert_eq!(repo.team_bot_ids(TEAM_WITHOUT_BOTS).await.unwrap(), vec![]);+ assert_eq!(repo.team_bot_ids(UNKNOWN_TEAM).await.unwrap(), vec![]);+}diff --git a/crates/chat/Cargo.toml b/crates/chat/Cargo.tomlindex 4049ace7d1..e7ff809b6e 100644--- a/crates/chat/Cargo.toml+++ b/crates/chat/Cargo.toml@@ -60,6 +60,7 @@ models_permissions = { path = "../models_permissions" }rootcause = { workspace = true }serde = { workspace = true }serde_json = { workspace = true }+shared_entity_registry = { path = "../entity_registry/shared_entity_registry" }thiserror = { workspace = true }tokio = { workspace = true }tracing = { workspace = true }diff --git a/crates/chat/src/domain/service/chat.rs b/crates/chat/src/domain/service/chat.rsindex d9528d378c..2760a819dc 100644--- a/crates/chat/src/domain/service/chat.rs+++ b/crates/chat/src/domain/service/chat.rs@@ -1,5 +1,6 @@//! Default [`ChatService`] implementation backed by a [`ChatRepo`].+mod purge_owned;#[cfg(test)]mod test;diff --git a/crates/chat/src/domain/service/chat/purge_owned.rs b/crates/chat/src/domain/service/chat/purge_owned.rsnew file mode 100644index 0000000000..3e37a271ba--- /dev/null+++ b/crates/chat/src/domain/service/chat/purge_owned.rs@@ -0,0 +1,50 @@+//! Owner-checked permanent delete of a chat, for callers removing an owner.++#[cfg(test)]+mod test;++use super::ChatServiceImpl;+use crate::domain::{+ models::ChatErr,+ ports::{ChatRepo, ChatService},+};+use entity_access::domain::models::EntityAccessReceipt;+use entity_access_management::domain::ports::EntityAccessManagementService;+use macro_event_broker::MacroEventBroker;+use model_entity::EntityType;+use model_owner::Owner;+use rootcause::{Report, prelude::*};+use shared_entity_registry::{OwnedPurgeOutcome, PurgeOwnedEntity};++impl<R: ChatRepo, ToolSetContext, Eam: EntityAccessManagementService, B: MacroEventBroker>+ PurgeOwnedEntity for ChatServiceImpl<R, ToolSetContext, Eam, B>+where+ ToolSetContext: Clone + Send + Sync + 'static,+{+ #[tracing::instrument(+ skip(self, expected_owner),+ fields(expected_owner.kind = ?expected_owner.owner_type()),+ err+ )]+ async fn purge_owned(+ &self,+ entity_id: uuid::Uuid,+ expected_owner: &Owner,+ ) -> Result<OwnedPurgeOutcome, Report> {+ let chat_id = entity_id.to_string();+ let chat = match self.repo.get_metadata(&chat_id).await {+ Err(ChatErr::NotFound) => return Ok(OwnedPurgeOutcome::Purged),+ loaded => loaded.context("unable to look up the chat")?,+ };+ if chat.user_id != *expected_owner {+ return Ok(OwnedPurgeOutcome::OwnedElsewhere);+ }+ self.permanently_delete(EntityAccessReceipt::dangerously_assert_internal_user(+ &chat_id,+ EntityType::Chat,+ ))+ .await+ .context("unable to permanently delete the chat")?;+ Ok(OwnedPurgeOutcome::Purged)+ }+}diff --git a/crates/chat/src/domain/service/chat/purge_owned/test.rs b/crates/chat/src/domain/service/chat/purge_owned/test.rsnew file mode 100644index 0000000000..4a6a1abfef--- /dev/null+++ b/crates/chat/src/domain/service/chat/purge_owned/test.rs@@ -0,0 +1,80 @@+use super::super::test::{+ CHAT_ID, PROJECT_ID, RecordingEventBroker, StubChatRepo, TEAM_ID, build_service, owner,+};+use super::*;++fn chat_uuid() -> uuid::Uuid {+ uuid::Uuid::parse_str(CHAT_ID).unwrap()+}++fn team() -> Owner {+ Owner::Team(TEAM_ID)+}++#[tokio::test]+async fn purge_owned_deletes_the_chat_and_publishes_without_an_actor() {+ let repo = StubChatRepo::storing(team(), None);+ let broker = RecordingEventBroker::default();+ let service = build_service(repo.clone(), broker.clone());++ let outcome = service.purge_owned(chat_uuid(), &team()).await.unwrap();++ assert_eq!(outcome, OwnedPurgeOutcome::Purged);+ assert_eq!(repo.permanently_deleted(), [CHAT_ID]);+ let events = broker.events();+ assert_eq!(events.len(), 1);+ assert_eq!(events[0].key, CHAT_ID);+ assert_eq!(events[0].envelope["event_type"], "chat.permanently_deleted");+ let metadata = &events[0].envelope["metadata"];+ assert!(metadata["actor_user_id"].is_null());+ assert_eq!(metadata["project_id"], PROJECT_ID);+}++#[tokio::test]+async fn purge_owned_deletes_a_trashed_chat() {+ let repo = StubChatRepo::storing(team(), Some(chrono::DateTime::<chrono::Utc>::UNIX_EPOCH));+ let service = build_service(repo.clone(), RecordingEventBroker::default());++ let outcome = service.purge_owned(chat_uuid(), &team()).await.unwrap();++ assert_eq!(outcome, OwnedPurgeOutcome::Purged);+ assert_eq!(repo.permanently_deleted(), [CHAT_ID]);+}++#[tokio::test]+async fn purge_owned_refuses_a_chat_owned_by_someone_else() {+ let repo = StubChatRepo::storing(Owner::User(owner()), None);+ let broker = RecordingEventBroker::default();+ let service = build_service(repo.clone(), broker.clone());++ let outcome = service.purge_owned(chat_uuid(), &team()).await.unwrap();++ assert_eq!(outcome, OwnedPurgeOutcome::OwnedElsewhere);+ assert!(repo.permanently_deleted().is_empty());+ assert!(broker.events().is_empty());+}++#[tokio::test]+async fn purge_owned_of_a_missing_chat_is_purged() {+ let repo = StubChatRepo::missing();+ let broker = RecordingEventBroker::default();+ let service = build_service(repo.clone(), broker.clone());++ let outcome = service.purge_owned(chat_uuid(), &team()).await.unwrap();++ assert_eq!(outcome, OwnedPurgeOutcome::Purged);+ assert!(repo.permanently_deleted().is_empty());+ assert!(broker.events().is_empty());+}++#[tokio::test]+async fn purge_owned_returns_a_failed_delete_as_an_error() {+ let broker = RecordingEventBroker::default();+ let service = build_service(+ StubChatRepo::failing_permanent_delete(team()),+ broker.clone(),+ );++ assert!(service.purge_owned(chat_uuid(), &team()).await.is_err());+ assert!(broker.events().is_empty());+}diff --git a/crates/chat/src/domain/service/chat/test.rs b/crates/chat/src/domain/service/chat/test.rsindex f4b3eb9465..b79947b0ed 100644--- a/crates/chat/src/domain/service/chat/test.rs+++ b/crates/chat/src/domain/service/chat/test.rs@@ -13,12 +13,13 @@ use model_owner::Owner;use super::*;use crate::domain::models::{ChatResponse, PatchChatMessageArgs};-const CHAT_ID: &str = "3f6f8b0a-6f9f-4a3f-9c3a-2b1e5d4c7a90";+pub(super) const CHAT_ID: &str = "3f6f8b0a-6f9f-4a3f-9c3a-2b1e5d4c7a90";const NEW_CHAT_ID: &str = "0197f776-6e7b-7c69-a251-780ae754d3e4";-const PROJECT_ID: &str = "c1a2b3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d";+pub(super) const PROJECT_ID: &str = "c1a2b3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d";const OWNER: &str = "macro|[email protected]";const OTHER_USER: &str = "macro|[email protected]";-const TEAM_ID: uuid::Uuid = uuid::Uuid::from_u128(0x7ea3_0000_0000_0000_0000_0000_0000_0001);+pub(super) const TEAM_ID: uuid::Uuid =+ uuid::Uuid::from_u128(0x7ea3_0000_0000_0000_0000_0000_0000_0001);const MESSAGE_ID: &str = "message-id";const TOOL_CALL_ID: &str = "tool-call-id";const TOOL_NAME: &str = "test_tool";@@ -38,8 +39,12 @@ struct MessagePersistence {}#[derive(Clone, Default)]-struct StubChatRepo {+pub(super) struct StubChatRepo {metadata_project_id: Option<String>,+ /// Owner `get_metadata` reports; `None` is [`OWNER`].+ metadata_owner: Option<Owner>,+ metadata_deleted_at: Option<chrono::DateTime<chrono::Utc>>,+ metadata_missing: bool,message_persistence: Arc<Mutex<MessagePersistence>>,team_default: Option<models_permissions::share_permission::TeamLinkShareDefault>,received_owners: Arc<Mutex<Vec<Owner>>>,@@ -48,6 +53,7 @@ struct StubChatRepo {team_share_facts: Option<TeamShareFacts>,team_share_facts_loads: Arc<Mutex<usize>>,received_patch: Arc<Mutex<Option<PatchChatRepoArgs>>>,+ permanently_deleted: Arc<Mutex<Vec<String>>>,fail_create: bool,fail_copy_chat: bool,fail_delete: bool,@@ -65,6 +71,34 @@ impl StubChatRepo {}}+ /// A chat in [`PROJECT_ID`] stored under `owner`, trashed at `deleted_at`.+ pub(super) fn storing(owner: Owner, deleted_at: Option<chrono::DateTime<chrono::Utc>>) -> Self {+ Self {+ metadata_owner: Some(owner),+ metadata_deleted_at: deleted_at,+ ..Self::with_project()+ }+ }++ pub(super) fn missing() -> Self {+ Self {+ metadata_missing: true,+ ..Self::default()+ }+ }++ pub(super) fn failing_permanent_delete(owner: Owner) -> Self {+ Self {+ fail_permanently_delete: true,+ ..Self::storing(owner, None)+ }+ }++ /// Chat ids `permanently_delete` removed, in call order.+ pub(super) fn permanently_deleted(&self) -> Vec<String> {+ self.permanently_deleted.lock().unwrap().clone()+ }+fn with_team_share_facts(facts: TeamShareFacts) -> Self {Self {team_share_facts: Some(facts),@@ -141,17 +175,23 @@ impl ChatRepo for StubChatRepo {}async fn get_metadata(&self, chat_id: &str) -> Result<Chat> {+ if self.metadata_missing {+ return Err(ChatErr::NotFound);+ }Ok(Chat {id: chat_id.to_string(),name: "Source Chat".to_string(),- user_id: Owner::from_principal_str(OWNER).unwrap(),+ user_id: self+ .metadata_owner+ .clone()+ .unwrap_or_else(|| Owner::from_principal_str(OWNER).unwrap()),model: None,project_id: self.metadata_project_id.clone(),created_at: None,updated_at: None,token_count: None,is_persistent: true,- deleted_at: None,+ deleted_at: self.metadata_deleted_at,})}@@ -204,10 +244,14 @@ impl ChatRepo for StubChatRepo {Ok(())}- async fn permanently_delete(&self, _chat_id: &str) -> Result<()> {+ async fn permanently_delete(&self, chat_id: &str) -> Result<()> {if self.fail_permanently_delete {return Err(Self::repo_err());}+ self.permanently_deleted+ .lock()+ .unwrap()+ .push(chat_id.to_string());Ok(())}@@ -284,7 +328,7 @@ impl ChatRepo for StubChatRepo {// -- Stub EntityAccessManagementService --#[derive(Clone)]-struct StubEntityAccessManagement;+pub(super) struct StubEntityAccessManagement;impl EntityAccessManagementService for StubEntityAccessManagement {async fn add_entity_to_project(@@ -318,14 +362,14 @@ impl EntityAccessManagementService for StubEntityAccessManagement {// -- Recording event broker --#[derive(Clone, Debug, PartialEq)]-struct PublishedChatEvent {+pub(super) struct PublishedChatEvent {topic: &'static str,- key: String,- envelope: serde_json::Value,+ pub(super) key: String,+ pub(super) envelope: serde_json::Value,}#[derive(Clone, Default)]-struct RecordingEventBroker {+pub(super) struct RecordingEventBroker {events: Arc<Mutex<Vec<PublishedChatEvent>>>,fail_scheduling: bool,}@@ -338,7 +382,7 @@ impl RecordingEventBroker {}}- fn events(&self) -> Vec<PublishedChatEvent> {+ pub(super) fn events(&self) -> Vec<PublishedChatEvent> {self.events.lock().unwrap().clone()}}@@ -410,7 +454,7 @@ fn tool_message_content() -> ChatMessageContent {])}-fn owner() -> MacroUserIdStr<'static> {+pub(super) fn owner() -> MacroUserIdStr<'static> {MacroUserIdStr::try_from(OWNER.to_string()).expect("valid user id")}@@ -422,7 +466,7 @@ fn view_receipt(chat_id: &str) -> EntityAccessReceipt<ViewAccessLevel> {EntityAccessReceipt::dangerously_assert_authenticated_user(owner(), chat_id, EntityType::Chat)}-fn build_service<B: MacroEventBroker>(+pub(super) fn build_service<B: MacroEventBroker>(repo: StubChatRepo,event_broker: B,) -> ChatServiceImpl<StubChatRepo, (), StubEntityAccessManagement, B> {diff --git a/crates/document_storage_service_client/Cargo.toml b/crates/document_storage_service_client/Cargo.tomlindex 46503fd7c3..8dab65e2ce 100644--- a/crates/document_storage_service_client/Cargo.toml+++ b/crates/document_storage_service_client/Cargo.toml@@ -7,11 +7,20 @@ version = "0.1.0"[dependencies]anyhow = { workspace = true }model = { path = "../model" }+model_owner = { path = "../model_owner" }models_bulk_upload = { path = "../models_bulk_upload" }models_permissions = { path = "../models_permissions" }-reqwest = { workspace = true }+reqwest = { workspace = true, features = ["query"] }+rootcause = { workspace = true }serde = { workspace = true }serde_json = { workspace = true }+shared_entity_registry = { path = "../entity_registry/shared_entity_registry" }thiserror = { workspace = true }tracing = { workspace = true }+uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }++[dev-dependencies]+bot_id = { path = "../bot_id" }+tokio = { workspace = true }+wiremock = { workspace = true }diff --git a/crates/document_storage_service_client/src/delete.rs b/crates/document_storage_service_client/src/delete.rsindex 5ee48f2a43..39c163eac6 100644--- a/crates/document_storage_service_client/src/delete.rs+++ b/crates/document_storage_service_client/src/delete.rs@@ -1,5 +1,31 @@+#[cfg(test)]+mod test;+use super::DocumentStorageServiceClient;use anyhow::Result;+use model_owner::Owner;+use rootcause::{Report, prelude::*};+use shared_entity_registry::RegisteredEntityType;+use uuid::Uuid;++/// Why document storage did not purge an owned entity.+#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]+pub enum PurgeOwnedEntityError {+ /// The entity exists under another owner. Nothing was deleted.+ #[error("the entity has another owner")]+ OwnedElsewhere,+ /// The request did not complete. The cause is the transport error.+ #[error("unable to reach document storage service")]+ Unreachable,+ /// Document storage answered with a status that is not a purge outcome,+ /// such as a 404 from a deployment without the route. The attachment+ /// carries the response body.+ #[error("document storage service answered {status}")]+ UnexpectedResponse {+ /// The HTTP status code.+ status: u16,+ },+}impl DocumentStorageServiceClient {/// Deletes all items for a user@@ -25,4 +51,41 @@ impl DocumentStorageServiceClient {Ok(())}++ /// Permanently delete a document, chat, or project if `owner` still owns it.+ ///+ /// `Ok` means the entity is gone, deleted now or already missing, and the+ /// removal of its stored content, search, and activity is enqueued.+ /// [`PurgeOwnedEntityError::OwnedElsewhere`] means another owner holds it+ /// and nothing was deleted. After any other error the entity may remain,+ /// and a retry repeats the purge.+ #[tracing::instrument(skip(self, owner), fields(owner.kind = ?owner.owner_type()), err)]+ pub async fn purge_owned_entity(+ &self,+ entity_type: RegisteredEntityType,+ entity_id: Uuid,+ owner: &Owner,+ ) -> Result<(), Report<PurgeOwnedEntityError>> {+ let response = self+ .client+ .delete(format!(+ "{}/internal/owned/{entity_type}/{entity_id}",+ self.url+ ))+ .query(&[("owner", owner.principal_id())])+ .send()+ .await+ .context(PurgeOwnedEntityError::Unreachable)?;+ match response.status() {+ reqwest::StatusCode::NO_CONTENT => Ok(()),+ reqwest::StatusCode::CONFLICT => Err(report!(PurgeOwnedEntityError::OwnedElsewhere)),+ status => {+ let body = response.text().await.unwrap_or_default();+ Err(report!(PurgeOwnedEntityError::UnexpectedResponse {+ status: status.as_u16(),+ })+ .attach(body))+ }+ }+ }}diff --git a/crates/document_storage_service_client/src/delete/test.rs b/crates/document_storage_service_client/src/delete/test.rsnew file mode 100644index 0000000000..838cbc1d70--- /dev/null+++ b/crates/document_storage_service_client/src/delete/test.rs@@ -0,0 +1,83 @@+use bot_id::BotId;+use model_owner::Owner;+use shared_entity_registry::RegisteredEntityType;+use uuid::Uuid;+use wiremock::{+ Mock, MockServer, ResponseTemplate,+ matchers::{header, method, path, query_param},+};++use super::PurgeOwnedEntityError;+use crate::DocumentStorageServiceClient;++const INTERNAL_KEY: &str = "internal-key";+const BOT: Uuid = Uuid::from_u128(0xB07);+const DOCUMENT: Uuid = Uuid::from_u128(0xD0C);++async fn dss_answering(response: ResponseTemplate) -> MockServer {+ let server = MockServer::start().await;+ Mock::given(method("DELETE"))+ .and(path(format!("/internal/owned/document/{DOCUMENT}")))+ .and(query_param("owner", format!("bot|{BOT}")))+ .and(header("x-document-storage-service-auth-key", INTERNAL_KEY))+ .respond_with(response)+ .expect(1)+ .mount(&server)+ .await;+ server+}++async fn purge_bot_document(dss_url: String) -> Result<(), PurgeOwnedEntityError> {+ DocumentStorageServiceClient::new(INTERNAL_KEY.into(), dss_url)+ .purge_owned_entity(+ RegisteredEntityType::Document,+ DOCUMENT,+ &Owner::Bot(BotId::new_from_uuid(BOT)),+ )+ .await+ .map_err(|report| *report.current_context())+}++#[tokio::test]+async fn no_content_is_a_purge() {+ let server = dss_answering(ResponseTemplate::new(204)).await;++ assert_eq!(purge_bot_document(server.uri()).await, Ok(()));+}++#[tokio::test]+async fn conflict_is_owned_elsewhere() {+ let server = dss_answering(ResponseTemplate::new(409)).await;++ assert_eq!(+ purge_bot_document(server.uri()).await,+ Err(PurgeOwnedEntityError::OwnedElsewhere)+ );+}++#[tokio::test]+async fn any_other_status_is_an_unexpected_response() {+ for status in [200, 400, 401, 404, 500] {+ let server = dss_answering(ResponseTemplate::new(status)).await;++ assert_eq!(+ purge_bot_document(server.uri()).await,+ Err(PurgeOwnedEntityError::UnexpectedResponse { status }),+ "{status}"+ );+ }+}++#[tokio::test]+async fn a_refused_connection_is_unreachable() {+ let closed_port = std::net::TcpListener::bind("127.0.0.1:0")+ .unwrap()+ .local_addr()+ .unwrap()+ .port();++ assert_eq!(+ purge_bot_document(format!("http://127.0.0.1:{closed_port}")).await,+ Err(PurgeOwnedEntityError::Unreachable)+ );+}diff --git a/crates/documents/Cargo.toml b/crates/documents/Cargo.tomlindex ed2cf89eb6..7d143a4029 100644--- a/crates/documents/Cargo.toml+++ b/crates/documents/Cargo.toml@@ -10,7 +10,14 @@ path = "src/bin/backfill_markdown_content_location.rs"required-features = ["backfill_binary"][features]-purge = ["ports", "outbound", "dep:macro_db_client", "dep:sqs_client", "dep:comms_db_client"]+purge = [+ "ports",+ "outbound",+ "dep:macro_db_client",+ "dep:macro_sha_count_client",+ "dep:sqs_client",+ "dep:comms_db_client",+]ai_tools = ["dep:ai_billing","dep:ai_engine",@@ -85,7 +92,7 @@ outbound = ["dep:sqlx","ports",]-ports = ["dep:tokio", "dep:tokio-retry"]+ports = ["dep:shared_entity_registry", "dep:tokio", "dep:tokio-retry"]schema = ["dep:utoipa", "macro_user_id/schema"]service = ["dep:entity_access_management",@@ -96,6 +103,7 @@ service = [[dependencies]macro_db_client = { path = "../macro_db_client", optional = true }+macro_sha_count_client = { path = "../macro_sha_count_client", optional = true }sqs_client = { path = "../sqs_client", optional = true }activity = { path = "../activity", default-features = false }bot_id = { path = "../bot_id" }@@ -165,6 +173,7 @@ comms_db_client = { path = "../comms_db_client", optional = true }entity_access_db_utils = { path = "../entity_access_db_utils", optional = true }entity_registry = { path = "../entity_registry/entity_registry", optional = true }entity_registry_db_utils = { path = "../entity_registry/entity_registry_db_utils", optional = true }+shared_entity_registry = { path = "../entity_registry/shared_entity_registry", optional = true }model_owner = { path = "../model_owner" }futures = { workspace = true }share_permission_db_utils = { path = "../share_permission_db_utils", optional = true }diff --git a/crates/documents/fixtures/docx_purge.sql b/crates/documents/fixtures/docx_purge.sqlnew file mode 100644index 0000000000..ef18e33d73--- /dev/null+++ b/crates/documents/fixtures/docx_purge.sql@@ -0,0 +1,12 @@+INSERT INTO macro_user (id, username, email, stripe_customer_id) VALUES+ ('93000000-0000-0000-0000-000000000001', 'docx-owner', '[email protected]', 'cus_docx_owner');+INSERT INTO "User" (id, email, macro_user_id) VALUES+ ('macro|[email protected]', '[email protected]', '93000000-0000-0000-0000-000000000001');+INSERT INTO "Document" (id, name, owner, "fileType", "deletedAt") VALUES+ ('93000000-0000-0000-0000-000000000011', 'Trashed docx', 'macro|[email protected]', 'docx', '2026-01-01 00:00:00');+INSERT INTO "DocumentBom" (id, "documentId") VALUES+ (930001, '93000000-0000-0000-0000-000000000011');+INSERT INTO "BomPart" ("documentBomId", sha, path) VALUES+ (930001, 'sha-a', 'word/document.xml'),+ (930001, 'sha-b', 'word/styles.xml'),+ (930001, 'sha-a', 'word/media/copy.xml');diff --git a/crates/documents/src/domain/purge.rs b/crates/documents/src/domain/purge.rsindex d534d26209..579d0fa131 100644--- a/crates/documents/src/domain/purge.rs+++ b/crates/documents/src/domain/purge.rs@@ -1,4 +1,5 @@-//! Internal permanent cleanup for owned description documents and failed creations.+//! Internal permanent cleanup for documents: owned description documents,+//! failed creations, and the documents of an owner being removed.use super::{events::{DocumentMacroEvent, DocumentPurgedMetadata},@@ -6,29 +7,59 @@ use super::{};use macro_event_broker::MacroEventBroker;use model_owner::Owner;+use rootcause::{Report, prelude::*};+use shared_entity_registry::{OwnedPurgeOutcome, PurgeOwnedEntity};use std::future::Future;/// Owning document cleanup; callers must obtain the ID from their own lifecycle state.pub trait DocumentPurgeService: Send + Sync + 'static {- /// Permanently delete document metadata and schedule removal of stored content.+ /// Permanently delete document metadata and schedule removal of stored+ /// content. A document that is already gone is not an error.fn purge(&self,document_id: uuid::Uuid,) -> impl Future<Output = Result<(), DocumentError>> + Send;}+/// What a purge reads from a document while its rows still exist.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct PurgeTarget {+ /// The recorded owner. Stored content is keyed by it.+ pub owner: Owner,+ /// The stored file type, such as `docx`.+ pub file_type: Option<String>,+}++/// What a row delete removed that other stores still hold.+#[derive(Debug, Clone, PartialEq, Eq, Default)]+pub struct PurgedRows {+ /// The SHA of each stored docx part the rows referenced, one entry per+ /// reference. Empty for every other file type.+ pub docx_part_shas: Vec<String>,+}+/// Permanent document persistence cleanup.pub trait DocumentPurgeRepository: Send + Sync + 'static {- /// Remove document metadata and return its owner captured before deletion.+ /// The document, live or trashed. `None` if it does not exist.+ fn find(+ &self,+ document_id: &str,+ ) -> impl Future<Output = Result<Option<PurgeTarget>, DocumentError>> + Send;++ /// Delete the document's rows, its access rows, and its registry row, and+ /// report what those rows referenced elsewhere. A document that is already+ /// gone is not an error.fn purge_rows(&self,document_id: &str,- ) -> impl Future<Output = Result<Owner, DocumentError>> + Send;+ target: &PurgeTarget,+ ) -> impl Future<Output = Result<PurgedRows, DocumentError>> + Send;}/// Existing background content-deletion queue.pub trait DocumentPurgeQueue: Send + Sync + 'static {- /// Schedule content cleanup using the owner retained after metadata deletion.+ /// Schedule content cleanup for a document whose rows still exist. Given+ /// the owner, the worker leaves the rows to this purge.fn enqueue(&self,document_id: String,@@ -36,42 +67,124 @@ pub trait DocumentPurgeQueue: Send + Sync + 'static {) -> impl Future<Output = Result<(), DocumentError>> + Send;}+/// Reference counts of stored docx parts, which documents share by SHA.+pub trait DocxPartReferences: Send + Sync + 'static {+ /// Drop one reference per entry in `shas`. A part left without references+ /// is collected.+ fn release(&self, shas: Vec<String>) -> impl Future<Output = Result<(), DocumentError>> + Send;+}+/// Shared lifecycle orchestration for permanent document cleanup.-pub struct DocumentPurger<R, Q, B> {+pub struct DocumentPurger<R, Q, S, B> {repository: R,queue: Q,+ part_references: S,broker: B,}-impl<R, Q, B> DocumentPurger<R, Q, B> {- /// Compose persistence, cleanup delivery and the shared event broker.- pub fn new(repository: R, queue: Q, broker: B) -> Self {+impl<R, Q, S, B> DocumentPurger<R, Q, S, B> {+ /// Compose persistence, cleanup delivery, docx part references and the+ /// shared event broker.+ pub fn new(repository: R, queue: Q, part_references: S, broker: B) -> Self {Self {repository,queue,+ part_references,broker,}}}-impl<R: DocumentPurgeRepository, Q: DocumentPurgeQueue, B: MacroEventBroker + 'static>- DocumentPurgeService for DocumentPurger<R, Q, B>+impl<R, Q, S, B> DocumentPurger<R, Q, S, B>+where+ R: DocumentPurgeRepository,+ Q: DocumentPurgeQueue,+ S: DocxPartReferences,+ B: MacroEventBroker + 'static,{+ /// Every failure before the row delete leaves the document findable, so+ /// the caller can retry the whole purge. Part references are released+ /// from what the row delete reports, so a retry cannot release them twice.+ async fn purge_found(+ &self,+ document_id: String,+ target: PurgeTarget,+ ) -> Result<(), DocumentError> {+ self.queue+ .enqueue(document_id.clone(), target.owner.clone())+ .await?;+ let event = DocumentMacroEvent::purged(+ document_id.clone(),+ DocumentPurgedMetadata {+ document_id: document_id.clone(),+ },+ );+ self.broker+ .send_event(&event)+ .map_err(|error| DocumentError::Internal(error.into()))?+ .await+ .map_err(|error| DocumentError::Internal(error.into()))?+ .map_err(|error| DocumentError::Internal(error.into()))?;+ let purged = self.repository.purge_rows(&document_id, &target).await?;+ if !purged.docx_part_shas.is_empty()+ && let Err(error) = self.part_references.release(purged.docx_part_shas).await+ {+ tracing::error!(?error, %document_id, "unable to release docx part references");+ }+ Ok(())+ }+}++impl<R, Q, S, B> DocumentPurgeService for DocumentPurger<R, Q, S, B>+where+ R: DocumentPurgeRepository,+ Q: DocumentPurgeQueue,+ S: DocxPartReferences,+ B: MacroEventBroker + 'static,+{+ #[tracing::instrument(skip(self), err)]async fn purge(&self, document_id: uuid::Uuid) -> Result<(), DocumentError> {let document_id = document_id.to_string();- let owner = self.repository.purge_rows(&document_id).await?;- // Both follow-ups must run even if content cleanup delivery is temporarily unavailable.- let cleanup = self.queue.enqueue(document_id.clone(), owner).await;- let event =- DocumentMacroEvent::purged(document_id.clone(), DocumentPurgedMetadata { document_id });- let publication = match self.broker.send_event(&event) {- Ok(delivery) => delivery- .await- .map_err(|error| DocumentError::Internal(error.into()))?- .map_err(|error| DocumentError::Internal(error.into())),- Err(error) => Err(DocumentError::Internal(error.into())),+ match self.repository.find(&document_id).await? {+ Some(target) => self.purge_found(document_id, target).await,+ None => Ok(()),+ }+ }+}++impl<R, Q, S, B> PurgeOwnedEntity for DocumentPurger<R, Q, S, B>+where+ R: DocumentPurgeRepository,+ Q: DocumentPurgeQueue,+ S: DocxPartReferences,+ B: MacroEventBroker + 'static,+{+ #[tracing::instrument(+ skip(self, expected_owner),+ fields(expected_owner.kind = ?expected_owner.owner_type()),+ err+ )]+ async fn purge_owned(+ &self,+ entity_id: uuid::Uuid,+ expected_owner: &Owner,+ ) -> Result<OwnedPurgeOutcome, Report> {+ let document_id = entity_id.to_string();+ let Some(target) = self+ .repository+ .find(&document_id)+ .await+ .context("unable to look up the document")?+ else {+ return Ok(OwnedPurgeOutcome::Purged);};- cleanup.and(publication)+ if target.owner != *expected_owner {+ return Ok(OwnedPurgeOutcome::OwnedElsewhere);+ }+ self.purge_found(document_id, target)+ .await+ .context("unable to purge the document")?;+ Ok(OwnedPurgeOutcome::Purged)}}diff --git a/crates/documents/src/domain/purge/test.rs b/crates/documents/src/domain/purge/test.rsindex 4ea3fa287e..4a940e13df 100644--- a/crates/documents/src/domain/purge/test.rs+++ b/crates/documents/src/domain/purge/test.rs@@ -5,12 +5,18 @@ use std::sync::{use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};use model_owner::Owner;+use shared_entity_registry::{OwnedPurgeOutcome, PurgeOwnedEntity};use tokio::task::JoinHandle;use uuid::Uuid;-use super::{DocumentPurgeQueue, DocumentPurgeRepository, DocumentPurgeService, DocumentPurger};+use super::{+ DocumentPurgeQueue, DocumentPurgeRepository, DocumentPurgeService, DocumentPurger,+ DocxPartReferences, PurgeTarget, PurgedRows,+};use crate::domain::models::DocumentError;+const OWNER: &str = "macro|[email protected]";+#[derive(Clone, Copy, Default)]enum BrokerFailure {#[default]@@ -22,9 +28,11 @@ enum BrokerFailure {struct State {steps: Mutex<Vec<(&'static str, String)>>,- owner: Mutex<Option<Owner>>,+ document: Mutex<Option<PurgeTarget>>,+ part_shas: Vec<String>,fail_rows: AtomicBool,fail_queue: AtomicBool,+ fail_release: AtomicBool,broker_failure: BrokerFailure,}@@ -32,47 +40,86 @@ impl Default for State {fn default() -> Self {Self {steps: Mutex::default(),- owner: Mutex::new(Some(- Owner::from_principal_str("macro|[email protected]").unwrap(),- )),+ document: Mutex::new(Some(PurgeTarget {+ owner: owner(),+ file_type: Some("pdf".to_owned()),+ })),+ part_shas: Vec::new(),fail_rows: AtomicBool::default(),fail_queue: AtomicBool::default(),+ fail_release: AtomicBool::default(),broker_failure: BrokerFailure::default(),}}}impl State {- fn record(&self, step: &'static str, id: &str) {- self.steps.lock().unwrap().push((step, id.to_owned()));+ fn docx(part_shas: &[&str]) -> Self {+ Self {+ document: Mutex::new(Some(PurgeTarget {+ owner: owner(),+ file_type: Some("docx".to_owned()),+ })),+ part_shas: part_shas.iter().map(|sha| (*sha).to_owned()).collect(),+ ..Self::default()+ }+ }++ fn missing() -> Self {+ Self {+ document: Mutex::new(None),+ ..Self::default()+ }+ }++ fn record(&self, step: &'static str, detail: &str) {+ self.steps.lock().unwrap().push((step, detail.to_owned()));}fn steps(&self) -> Vec<(&'static str, String)> {self.steps.lock().unwrap().clone()}++ fn exists(&self) -> bool {+ self.document.lock().unwrap().is_some()+ }+}++fn owner() -> Owner {+ Owner::from_principal_str(OWNER).unwrap()}impl DocumentPurgeRepository for Arc<State> {- async fn purge_rows(&self, document_id: &str) -> Result<Owner, DocumentError> {+ async fn find(&self, document_id: &str) -> Result<Option<PurgeTarget>, DocumentError> {+ self.record("find", document_id);+ Ok(self.document.lock().unwrap().clone())+ }++ async fn purge_rows(+ &self,+ document_id: &str,+ target: &PurgeTarget,+ ) -> Result<PurgedRows, DocumentError> {self.record("rows", document_id);if self.fail_rows.load(Ordering::SeqCst) {return Err(DocumentError::Internal(anyhow::anyhow!("row deletion failed")));}- self.owner- .lock()- .unwrap()- .take()- .ok_or_else(|| DocumentError::Internal(anyhow::anyhow!("document already removed")))+ self.document.lock().unwrap().take();+ let docx_part_shas = if target.file_type.as_deref() == Some("docx") {+ self.part_shas.clone()+ } else {+ Vec::new()+ };+ Ok(PurgedRows { docx_part_shas })}}impl DocumentPurgeQueue for Arc<State> {async fn enqueue(&self, document_id: String, owner: Owner) -> Result<(), DocumentError> {self.record("queue", &document_id);- assert!(self.owner.lock().unwrap().is_none());- assert_eq!(owner.principal_id(), "macro|[email protected]");+ assert_eq!(owner.principal_id(), OWNER);if self.fail_queue.load(Ordering::SeqCst) {return Err(DocumentError::Internal(anyhow::anyhow!("queue unavailable"@@ -82,6 +129,18 @@ impl DocumentPurgeQueue for Arc<State> {}}+impl DocxPartReferences for Arc<State> {+ async fn release(&self, shas: Vec<String>) -> Result<(), DocumentError> {+ self.record("release", &shas.join(","));+ if self.fail_release.load(Ordering::SeqCst) {+ return Err(DocumentError::Internal(anyhow::anyhow!(+ "sha counter unavailable"+ )));+ }+ Ok(())+ }+}+struct Broker(Arc<State>);impl MacroEventBroker for Broker {@@ -111,70 +170,202 @@ impl MacroEventBroker for Broker {}}-fn purger(state: &Arc<State>) -> DocumentPurger<Arc<State>, Arc<State>, Broker> {- DocumentPurger::new(state.clone(), state.clone(), Broker(state.clone()))+fn purger(state: &Arc<State>) -> DocumentPurger<Arc<State>, Arc<State>, Arc<State>, Broker> {+ DocumentPurger::new(+ state.clone(),+ state.clone(),+ state.clone(),+ Broker(state.clone()),+ )}-fn expected_steps(id: Uuid) -> Vec<(&'static str, String)> {- ["rows", "queue", "broker"]- .into_iter()- .map(|step| (step, id.to_string()))- .collect()+fn steps(id: Uuid, names: &[&'static str]) -> Vec<(&'static str, String)> {+ names.iter().map(|name| (*name, id.to_string())).collect()}#[tokio::test]-async fn row_failure_does_not_publish_a_deletion_or_remove_stored_content() {- let state = Arc::new(State {- fail_rows: AtomicBool::new(true),- ..State::default()- });+async fn purge_enqueues_and_publishes_before_deleting_the_rows() {+ let state = Arc::new(State::default());let id = Uuid::now_v7();- assert!(purger(&state).purge(id).await.is_err());- assert_eq!(state.steps(), vec![("rows", id.to_string())]);+ purger(&state).purge(id).await.unwrap();+ assert_eq!(+ state.steps(),+ steps(id, &["find", "queue", "broker", "rows"])+ );+ assert!(!state.exists());}#[tokio::test]-async fn queue_failure_still_publishes_the_committed_deletion() {+async fn queue_failure_stops_before_the_event_and_the_rows() {let state = Arc::new(State {fail_queue: AtomicBool::new(true),..State::default()});let id = Uuid::now_v7();- let service = purger(&state);- assert!(service.purge(id).await.is_err());- assert_eq!(state.steps(), expected_steps(id));- assert!(state.owner.lock().unwrap().is_none());+ assert!(purger(&state).purge(id).await.is_err());+ assert_eq!(state.steps(), steps(id, &["find", "queue"]));+ assert!(state.exists());}#[tokio::test]-async fn broker_start_failure_preserves_the_content_cleanup_attempt() {+async fn broker_start_failure_keeps_the_rows() {let state = Arc::new(State {broker_failure: BrokerFailure::BeforeSpawn,..State::default()});let id = Uuid::now_v7();assert!(purger(&state).purge(id).await.is_err());- assert_eq!(state.steps(), expected_steps(id));+ assert_eq!(state.steps(), steps(id, &["find", "queue", "broker"]));+ assert!(state.exists());}#[tokio::test]-async fn broker_delivery_failure_is_awaited_and_returned() {+async fn broker_delivery_failure_is_awaited_and_keeps_the_rows() {let state = Arc::new(State {broker_failure: BrokerFailure::DuringDelivery,..State::default()});let id = Uuid::now_v7();assert!(purger(&state).purge(id).await.is_err());- assert_eq!(state.steps(), expected_steps(id));+ assert_eq!(state.steps(), steps(id, &["find", "queue", "broker"]));+ assert!(state.exists());}#[tokio::test]-async fn cancelled_broker_delivery_is_returned_after_content_cleanup() {+async fn cancelled_broker_delivery_keeps_the_rows() {let state = Arc::new(State {broker_failure: BrokerFailure::Cancelled,..State::default()});let id = Uuid::now_v7();assert!(purger(&state).purge(id).await.is_err());- assert_eq!(state.steps(), expected_steps(id));+ assert_eq!(state.steps(), steps(id, &["find", "queue", "broker"]));+ assert!(state.exists());+}++#[tokio::test]+async fn row_failure_leaves_the_document_for_a_retry() {+ let state = Arc::new(State {+ fail_rows: AtomicBool::new(true),+ ..State::default()+ });+ let id = Uuid::now_v7();+ let service = purger(&state);+ assert!(service.purge(id).await.is_err());+ assert!(state.exists());++ state.fail_rows.store(false, Ordering::SeqCst);+ service.purge(id).await.unwrap();+ assert_eq!(+ state.steps(),+ steps(+ id,+ &[+ "find", "queue", "broker", "rows", "find", "queue", "broker", "rows"+ ]+ )+ );+ assert!(!state.exists());+}++#[tokio::test]+async fn docx_parts_are_released_after_the_rows_are_gone() {+ let state = Arc::new(State::docx(&["b", "a", "b"]));+ let id = Uuid::now_v7();+ purger(&state).purge(id).await.unwrap();+ let mut expected = steps(id, &["find", "queue", "broker", "rows"]);+ expected.push(("release", "b,a,b".to_owned()));+ assert_eq!(state.steps(), expected);+}++#[tokio::test]+async fn docx_parts_are_released_once_across_a_failed_attempt_and_its_retry() {+ let state = Arc::new(State {+ fail_rows: AtomicBool::new(true),+ ..State::docx(&["a"])+ });+ let id = Uuid::now_v7();+ let service = purger(&state);+ assert!(service.purge(id).await.is_err());+ state.fail_rows.store(false, Ordering::SeqCst);+ service.purge(id).await.unwrap();++ let mut expected = steps(+ id,+ &[+ "find", "queue", "broker", "rows", "find", "queue", "broker", "rows",+ ],+ );+ expected.push(("release", "a".to_owned()));+ assert_eq!(state.steps(), expected);+}++#[tokio::test]+async fn release_failure_after_the_rows_are_gone_still_purges() {+ let state = Arc::new(State {+ fail_release: AtomicBool::new(true),+ ..State::docx(&["a"])+ });+ let id = Uuid::now_v7();+ purger(&state).purge(id).await.unwrap();+ assert_eq!(state.steps().last(), Some(&("release", "a".to_owned())));+ assert!(!state.exists());+}++#[tokio::test]+async fn purge_of_a_missing_document_does_nothing() {+ let state = Arc::new(State::missing());+ let id = Uuid::now_v7();+ purger(&state).purge(id).await.unwrap();+ assert_eq!(state.steps(), steps(id, &["find"]));+}++#[tokio::test]+async fn purge_owned_of_a_missing_document_is_purged() {+ let state = Arc::new(State::missing());+ let id = Uuid::now_v7();+ assert_eq!(+ purger(&state).purge_owned(id, &owner()).await.unwrap(),+ OwnedPurgeOutcome::Purged+ );+ assert_eq!(state.steps(), steps(id, &["find"]));+}++#[tokio::test]+async fn purge_owned_refuses_a_document_owned_by_someone_else() {+ let state = Arc::new(State::default());+ let id = Uuid::now_v7();+ let team = Owner::Team(Uuid::from_u128(1));+ assert_eq!(+ purger(&state).purge_owned(id, &team).await.unwrap(),+ OwnedPurgeOutcome::OwnedElsewhere+ );+ assert_eq!(state.steps(), steps(id, &["find"]));+ assert!(state.exists());+}++#[tokio::test]+async fn purge_owned_of_the_expected_owner_purges() {+ let state = Arc::new(State::default());+ let id = Uuid::now_v7();+ assert_eq!(+ purger(&state).purge_owned(id, &owner()).await.unwrap(),+ OwnedPurgeOutcome::Purged+ );+ assert_eq!(+ state.steps(),+ steps(id, &["find", "queue", "broker", "rows"])+ );+ assert!(!state.exists());+}++#[tokio::test]+async fn purge_owned_returns_a_failed_purge_as_an_error() {+ let state = Arc::new(State {+ fail_queue: AtomicBool::new(true),+ ..State::default()+ });+ let id = Uuid::now_v7();+ assert!(purger(&state).purge_owned(id, &owner()).await.is_err());+ assert!(state.exists());}diff --git a/crates/documents/src/outbound/document_purge.rs b/crates/documents/src/outbound/document_purge.rsindex aa393ca525..edb794ad8d 100644--- a/crates/documents/src/outbound/document_purge.rs+++ b/crates/documents/src/outbound/document_purge.rs@@ -2,7 +2,9 @@use super::super::domain::{models::DocumentError,- purge::{DocumentPurgeQueue, DocumentPurgeRepository},+ purge::{+ DocumentPurgeQueue, DocumentPurgeRepository, DocxPartReferences, PurgeTarget, PurgedRows,+ },};use model_owner::Owner;use std::sync::Arc;@@ -20,11 +22,34 @@ impl LegacyDocumentPurgeRepository {}impl DocumentPurgeRepository for LegacyDocumentPurgeRepository {- async fn purge_rows(&self, document_id: &str) -> Result<Owner, DocumentError> {- let document =- macro_db_client::document::get_deleted_document_info(&self.pool, document_id)+ #[tracing::instrument(skip(self), err)]+ async fn find(&self, document_id: &str) -> Result<Option<PurgeTarget>, DocumentError> {+ match macro_db_client::document::get_deleted_document_info(&self.pool, document_id).await {+ Ok(document) => Ok(Some(PurgeTarget {+ owner: document.owner,+ file_type: document.file_type,+ })),+ Err(sqlx::Error::RowNotFound) => Ok(None),+ Err(error) => Err(DocumentError::Internal(error.into())),+ }+ }++ #[tracing::instrument(skip(self, target), fields(file_type = ?target.file_type), err)]+ async fn purge_rows(+ &self,+ document_id: &str,+ target: &PurgeTarget,+ ) -> Result<PurgedRows, DocumentError> {+ let docx_part_shas = if target.file_type.as_deref() == Some("docx") {+ macro_db_client::document::get_bom_parts(&self.pool, document_id).await- .map_err(|error| DocumentError::Internal(error.into()))?;+ .map_err(DocumentError::Internal)?+ .into_iter()+ .map(|part| part.sha)+ .collect()+ } else {+ Vec::new()+ };macro_db_client::document::delete_document(&self.pool, document_id).await.map_err(DocumentError::Internal)?;@@ -37,7 +62,7 @@ impl DocumentPurgeRepository for LegacyDocumentPurgeRepository {tracing::error!(?error, %document_id, "unable to delete outgoing document mentions");}).ok();- Ok(document.owner)+ Ok(PurgedRows { docx_part_shas })}}@@ -62,5 +87,26 @@ impl DocumentPurgeQueue for SqsDocumentPurgeQueue {}}+/// Docx part references in the shared SHA counter.+pub struct RedisDocxPartReferences {+ sha_counts: Arc<macro_sha_count_client::Redis>,+}++impl RedisDocxPartReferences {+ /// Compose from the application's SHA counter client.+ pub fn new(sha_counts: Arc<macro_sha_count_client::Redis>) -> Self {+ Self { sha_counts }+ }+}++impl DocxPartReferences for RedisDocxPartReferences {+ async fn release(&self, shas: Vec<String>) -> Result<(), DocumentError> {+ self.sha_counts+ .release_shas(shas)+ .await+ .map_err(DocumentError::Internal)+ }+}+#[cfg(test)]mod test;diff --git a/crates/documents/src/outbound/document_purge/test.rs b/crates/documents/src/outbound/document_purge/test.rsindex dc19d740ad..9f1313d7f4 100644--- a/crates/documents/src/outbound/document_purge/test.rs+++ b/crates/documents/src/outbound/document_purge/test.rs@@ -9,9 +9,14 @@ use uuid::Uuid;use super::LegacyDocumentPurgeRepository;use crate::domain::{models::DocumentError,- purge::{DocumentPurgeQueue, DocumentPurgeService, DocumentPurger},+ purge::{+ DocumentPurgeQueue, DocumentPurgeRepository, DocumentPurgeService, DocumentPurger,+ DocxPartReferences,+ },};+const DOCX: &str = "93000000-0000-0000-0000-000000000011";+struct CapturingQueue {pool: PgPool,documents: Mutex<Vec<(String, Owner)>>,@@ -19,32 +24,72 @@ struct CapturingQueue {impl DocumentPurgeQueue for Arc<CapturingQueue> {async fn enqueue(&self, document_id: String, owner: Owner) -> Result<(), DocumentError> {- // The worker can no longer resolve ownership from the database at this point.- let deleted =- macro_db_client::document::get_deleted_document_info(&self.pool, &document_id).await;- assert!(matches!(deleted, Err(sqlx::Error::RowNotFound)));+ let stored = macro_db_client::document::get_deleted_document_info(&self.pool, &document_id)+ .await+ .unwrap();+ assert_eq!(stored.owner, owner);self.documents.lock().unwrap().push((document_id, owner));Ok(())}}-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(- path = "../../../../entity_access/fixtures",- scripts("typed_owner_team")- )-)]-async fn queues_the_stored_owner_after_document_metadata_is_gone(pool: PgPool) {+struct RecordingPartReferences {+ pool: PgPool,+ released: Mutex<Vec<Vec<String>>>,+}++impl DocxPartReferences for Arc<RecordingPartReferences> {+ async fn release(&self, mut shas: Vec<String>) -> Result<(), DocumentError> {+ let deleted = macro_db_client::document::get_deleted_document_info(&self.pool, DOCX).await;+ assert!(matches!(deleted, Err(sqlx::Error::RowNotFound)));+ shas.sort();+ self.released.lock().unwrap().push(shas);+ Ok(())+ }+}++struct Fixture {+ queue: Arc<CapturingQueue>,+ part_references: Arc<RecordingPartReferences>,+ purger: DocumentPurger<+ LegacyDocumentPurgeRepository,+ Arc<CapturingQueue>,+ Arc<RecordingPartReferences>,+ NoopMacroEventBroker,+ >,+}++fn fixture(pool: &PgPool) -> Fixture {let queue = Arc::new(CapturingQueue {pool: pool.clone(),documents: Mutex::default(),});+ let part_references = Arc::new(RecordingPartReferences {+ pool: pool.clone(),+ released: Mutex::default(),+ });let purger = DocumentPurger::new(- LegacyDocumentPurgeRepository::new(pool),+ LegacyDocumentPurgeRepository::new(pool.clone()),queue.clone(),+ part_references.clone(),NoopMacroEventBroker,);+ Fixture {+ queue,+ part_references,+ purger,+ }+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(+ path = "../../../../entity_access/fixtures",+ scripts("typed_owner_team")+ )+)]+async fn queues_the_stored_owner_before_deleting_the_document(pool: PgPool) {+ let fixture = fixture(&pool);let expected = [("90000000-0000-0000-0000-000000000031",@@ -60,10 +105,53 @@ async fn queues_the_stored_owner_after_document_metadata_is_gone(pool: PgPool) {),];for (id, _) in expected {- purger.purge(Uuid::parse_str(id).unwrap()).await.unwrap();+ fixture+ .purger+ .purge(Uuid::parse_str(id).unwrap())+ .await+ .unwrap();}assert_eq!(- *queue.documents.lock().unwrap(),+ *fixture.queue.documents.lock().unwrap(),expected.map(|(id, owner)| (id.to_owned(), Owner::from_principal_str(owner).unwrap())));+ let repository = LegacyDocumentPurgeRepository::new(pool);+ for (id, _) in expected {+ assert_eq!(repository.find(id).await.unwrap(), None);+ }+ assert!(fixture.part_references.released.lock().unwrap().is_empty());+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("docx_purge"))+)]+async fn releases_each_docx_part_reference_once_the_rows_are_gone(pool: PgPool) {+ let fixture = fixture(&pool);+ fixture+ .purger+ .purge(Uuid::parse_str(DOCX).unwrap())+ .await+ .unwrap();+ assert_eq!(+ *fixture.part_references.released.lock().unwrap(),+ vec![vec![+ "sha-a".to_owned(),+ "sha-a".to_owned(),+ "sha-b".to_owned()+ ]]+ );+ assert!(+ macro_db_client::document::get_bom_parts(&pool, DOCX)+ .await+ .unwrap()+ .is_empty()+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn purge_of_a_missing_document_is_ok(pool: PgPool) {+ let fixture = fixture(&pool);+ fixture.purger.purge(Uuid::now_v7()).await.unwrap();+ assert!(fixture.queue.documents.lock().unwrap().is_empty());}diff --git a/crates/entity_access_db_utils/fixtures/source_grants.sql b/crates/entity_access_db_utils/fixtures/source_grants.sqlnew file mode 100644index 0000000000..f10ef19e82--- /dev/null+++ b/crates/entity_access_db_utils/fixtures/source_grants.sql@@ -0,0 +1,7 @@+INSERT INTO entity_access (entity_id, entity_type, source_id, source_type, access_level, granted_from_project_id) VALUES+ ('20000000-0000-0000-0000-000000000004', 'chat', '10000000-0000-0000-0000-000000000001', 'team', 'view', NULL),+ ('20000000-0000-0000-0000-000000000002', 'document', '10000000-0000-0000-0000-000000000001', 'channel', 'view', NULL),+ ('20000000-0000-0000-0000-000000000004', 'chat', 'bot|60000000-0000-0000-0000-000000000001', 'bot', 'owner', NULL),+ ('20000000-0000-0000-0000-000000000002', 'document', 'bot|60000000-0000-0000-0000-000000000001', 'bot', 'view', '20000000-0000-0000-0000-000000000001'),+ ('20000000-0000-0000-0000-000000000002', 'document', 'bot|60000000-0000-0000-0000-000000000002', 'bot', 'edit', NULL),+ ('20000000-0000-0000-0000-000000000002', 'document', 'macro|[email protected]', 'user', 'owner', NULL);diff --git a/crates/entity_access_db_utils/src/lib.rs b/crates/entity_access_db_utils/src/lib.rsindex f7b1470cc5..fd22791907 100644--- a/crates/entity_access_db_utils/src/lib.rs+++ b/crates/entity_access_db_utils/src/lib.rs@@ -6,6 +6,7 @@mod test;pub mod channel_share;+pub mod source_grants;pub mod team_share;pub use channel_share::get_direct_channel_grants;@@ -17,6 +18,7 @@ pub use models_permissions::share_permission::access_level::AccessLevel;use models_permissions::share_permission::channel_share_permission::{UpdateChannelSharePermission, UpdateOperation,};+pub use source_grants::delete_source_grants;use sqlx::{Executor, Postgres, QueryBuilder, Transaction};/// Inserts a row into the entity access tablediff --git a/crates/entity_access_db_utils/src/source_grants.rs b/crates/entity_access_db_utils/src/source_grants.rsnew file mode 100644index 0000000000..8e46b25e7a--- /dev/null+++ b/crates/entity_access_db_utils/src/source_grants.rs@@ -0,0 +1,34 @@+//! Grants keyed by their source, across every entity.++use crate::EntityAccessSourceType;+use sqlx::{Executor, Postgres};++#[cfg(test)]+mod test;++/// Delete every grant whose source is `source_type` and `source_id`, on every+/// entity of every type, project-inherited grants included. Returns how many+/// rows were deleted.+///+/// For removing a principal. Does not authorize and does not commit. Deleting a+/// team source is a team-sharing mutation, so hold+/// [`acquire_guard`](crate::team_share::acquire_guard) in the same transaction+/// first.+#[tracing::instrument(skip(executor), err)]+pub async fn delete_source_grants<'e, E>(+ executor: E,+ source_type: EntityAccessSourceType,+ source_id: &str,+) -> Result<u64, sqlx::Error>+where+ E: Executor<'e, Database = Postgres>,+{+ let result = sqlx::query!(+ "DELETE FROM entity_access WHERE source_type = $1 AND source_id = $2",+ source_type as _,+ source_id,+ )+ .execute(executor)+ .await?;+ Ok(result.rows_affected())+}diff --git a/crates/entity_access_db_utils/src/source_grants/test.rs b/crates/entity_access_db_utils/src/source_grants/test.rsnew file mode 100644index 0000000000..55e2ee8a45--- /dev/null+++ b/crates/entity_access_db_utils/src/source_grants/test.rs@@ -0,0 +1,58 @@+use super::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::PgPool;++const TEAM: &str = "10000000-0000-0000-0000-000000000001";+const BOT: &str = "bot|60000000-0000-0000-0000-000000000001";++async fn remaining_sources(+ pool: &PgPool,+) -> Result<Vec<(String, EntityAccessSourceType)>, sqlx::Error> {+ let rows = sqlx::query!(+ r#"SELECT source_id, source_type AS "source_type: EntityAccessSourceType"+ FROM entity_access ORDER BY id"#+ )+ .fetch_all(pool)+ .await?;+ Ok(rows+ .into_iter()+ .map(|row| (row.source_id, row.source_type))+ .collect())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../fixtures", scripts("team_share", "source_grants"))+)]+async fn deletes_every_grant_of_the_source_and_nothing_else(+ pool: PgPool,+) -> Result<(), sqlx::Error> {+ assert_eq!(+ delete_source_grants(&pool, EntityAccessSourceType::Team, TEAM).await?,+ 3+ );+ assert_eq!(+ delete_source_grants(&pool, EntityAccessSourceType::Bot, BOT).await?,+ 2+ );+ assert_eq!(+ remaining_sources(&pool).await?,+ vec![+ ("other-team".to_owned(), EntityAccessSourceType::Team),+ (TEAM.to_owned(), EntityAccessSourceType::Channel),+ (+ "bot|60000000-0000-0000-0000-000000000002".to_owned(),+ EntityAccessSourceType::Bot+ ),+ (+ "macro|[email protected]".to_owned(),+ EntityAccessSourceType::User+ ),+ ]+ );+ assert_eq!(+ delete_source_grants(&pool, EntityAccessSourceType::Team, TEAM).await?,+ 0+ );+ Ok(())+}diff --git a/crates/entity_registry/entity_registry/src/domain/ports.rs b/crates/entity_registry/entity_registry/src/domain/ports.rsindex 794351ab55..9d9514529a 100644--- a/crates/entity_registry/entity_registry/src/domain/ports.rs+++ b/crates/entity_registry/entity_registry/src/domain/ports.rs@@ -42,6 +42,13 @@ pub trait EntityRegistryRepository: Clone + Send + Sync + 'static {entity_type: Option<RegisteredEntityType>,) -> impl Future<Output = EntityRegistryResult<Vec<EntityRecord>>> + Send;+ /// Live and soft-deleted rows owned by `owner`, newest-created first with+ /// `id` as the tie-break.+ fn list_all_owned_by(+ &self,+ owner: &Owner,+ ) -> impl Future<Output = EntityRegistryResult<Vec<EntityRecord>>> + Send;+/// Live and deleted row counts for one kind.fn count_by_type(&self,@@ -72,6 +79,14 @@ pub trait EntityRegistryService: Clone + Send + Sync + 'static {entity_type: Option<RegisteredEntityType>,) -> impl Future<Output = EntityRegistryResult<Vec<EntityRecord>>> + Send;+ /// Live and soft-deleted rows owned by `owner`, newest-created first with+ /// `id` as the tie-break. For callers that must remove everything an+ /// owner holds, the trash included.+ fn list_all_owned_by(+ &self,+ owner: &Owner,+ ) -> impl Future<Output = EntityRegistryResult<Vec<EntityRecord>>> + Send;+/// Live and deleted row counts for one kind.fn count_by_type(&self,diff --git a/crates/entity_registry/entity_registry/src/domain/service.rs b/crates/entity_registry/entity_registry/src/domain/service.rsindex 7f840553b8..a1e47ab940 100644--- a/crates/entity_registry/entity_registry/src/domain/service.rs+++ b/crates/entity_registry/entity_registry/src/domain/service.rs@@ -50,6 +50,11 @@ whereself.repo.list_owned_by(owner, entity_type).await}+ #[tracing::instrument(skip(self, owner), fields(owner.kind = ?owner.owner_type()), err)]+ async fn list_all_owned_by(&self, owner: &Owner) -> EntityRegistryResult<Vec<EntityRecord>> {+ self.repo.list_all_owned_by(owner).await+ }+#[tracing::instrument(skip(self), err)]async fn count_by_type(&self,diff --git a/crates/entity_registry/entity_registry/src/domain/service/test.rs b/crates/entity_registry/entity_registry/src/domain/service/test.rsindex 2b7451c642..8f289203df 100644--- a/crates/entity_registry/entity_registry/src/domain/service/test.rs+++ b/crates/entity_registry/entity_registry/src/domain/service/test.rs@@ -33,6 +33,7 @@ struct FakeRepo {get: Option<EntityRecord>,get_many: Vec<EntityRecord>,list: Vec<EntityRecord>,+ list_all: Vec<EntityRecord>,count: EntityTypeCount,}@@ -53,6 +54,10 @@ impl EntityRegistryRepository for FakeRepo {Ok(self.list.clone())}+ async fn list_all_owned_by(&self, _owner: &Owner) -> EntityRegistryResult<Vec<EntityRecord>> {+ Ok(self.list_all.clone())+ }+async fn count_by_type(&self,_entity_type: RegisteredEntityType,@@ -104,3 +109,22 @@ async fn get_many_list_and_count_return_repository_values() {count);}++#[tokio::test]+async fn list_all_owned_by_reads_the_trash_inclusive_listing() {+ let live = sample_record(1);+ let trashed = EntityRecord {+ deleted_at: Some(ts()),+ ..sample_record(2)+ };+ let service = EntityRegistryServiceImpl::new(FakeRepo {+ list: vec![live.clone()],+ list_all: vec![trashed.clone(), live.clone()],+ ..FakeRepo::default()+ });++ assert_eq!(+ service.list_all_owned_by(&user_owner()).await.unwrap(),+ vec![trashed, live]+ );+}diff --git a/crates/entity_registry/entity_registry/src/lib.rs b/crates/entity_registry/entity_registry/src/lib.rsindex 059f1a4a41..c699278637 100644--- a/crates/entity_registry/entity_registry/src/lib.rs+++ b/crates/entity_registry/entity_registry/src/lib.rs@@ -39,8 +39,8 @@ pub use domain::owner_grant_policy::OwnerGrantPolicy;pub use domain::ports::{BotFacts, EntityRegistryRepository, EntityRegistryService};pub use domain::service::EntityRegistryServiceImpl;pub use shared_entity_registry::{- EntityRegistryError, EntityRegistryResult, InsertOutcome, NewEntityRecord, Owner,- RegisteredEntityType, UnregisteredEntityType, WriteOutcome,+ EntityRegistryError, EntityRegistryResult, InsertOutcome, NewEntityRecord, OwnedPurgeOutcome,+ Owner, PurgeOwnedEntity, RegisteredEntityType, UnregisteredEntityType, WriteOutcome,};#[cfg(feature = "axum")]diff --git a/crates/entity_registry/entity_registry/src/outbound/pg_entity_registry_repo.rs b/crates/entity_registry/entity_registry/src/outbound/pg_entity_registry_repo.rsindex e780c134be..959a7d5c67 100644--- a/crates/entity_registry/entity_registry/src/outbound/pg_entity_registry_repo.rs+++ b/crates/entity_registry/entity_registry/src/outbound/pg_entity_registry_repo.rs@@ -139,6 +139,30 @@ impl EntityRegistryRepository for PgEntityRegistryRepository {.collect()}+ #[tracing::instrument(skip(self, owner), fields(owner.kind = ?owner.owner_type()), err)]+ async fn list_all_owned_by(&self, owner: &Owner) -> EntityRegistryResult<Vec<EntityRecord>> {+ let owner_type = owner.owner_type();+ let owner_id = owner.principal_id();+ sqlx::query_as!(+ EntityRow,+ r#"+ SELECT id, entity_type, owner_type AS "owner_type: OwnerType", owner_id,+ created_at, updated_at, deleted_at+ FROM entity+ WHERE owner_type = $1 AND owner_id = $2+ ORDER BY created_at DESC, id DESC+ "#,+ owner_type as _,+ owner_id,+ )+ .fetch_all(&self.pool)+ .await+ .context(EntityRegistryError::Infrastructure)?+ .into_iter()+ .map(EntityRow::into_record)+ .collect()+ }+#[tracing::instrument(skip(self), err)]async fn count_by_type(&self,diff --git a/crates/entity_registry/entity_registry/src/outbound/pg_entity_registry_repo/test.rs b/crates/entity_registry/entity_registry/src/outbound/pg_entity_registry_repo/test.rsindex 9af69bdea6..b6124be464 100644--- a/crates/entity_registry/entity_registry/src/outbound/pg_entity_registry_repo/test.rs+++ b/crates/entity_registry/entity_registry/src/outbound/pg_entity_registry_repo/test.rs@@ -229,6 +229,66 @@ async fn list_owned_by_orders_newest_created_first(pool: PgPool) {);}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn list_all_owned_by_returns_live_and_trashed_rows_of_that_owner_newest_first(pool: PgPool) {+ let owner = team_owner();+ let live_chat = Uuid::from_u128(1);+ let trashed_document = Uuid::from_u128(2);+ let other_owners_trashed = Uuid::from_u128(3);+ let old_ts = ts("2024-01-01T00:00:00Z");+ let new_ts = ts("2024-06-01T00:00:00Z");+ insert_committed(+ &pool,+ NewEntityRecord::new(live_chat, RegisteredEntityType::Chat, owner.clone())+ .with_timestamps(old_ts, old_ts),+ )+ .await;+ insert_committed(+ &pool,+ NewEntityRecord::new(+ trashed_document,+ RegisteredEntityType::Document,+ owner.clone(),+ )+ .with_timestamps(new_ts, new_ts),+ )+ .await;+ mark_deleted_committed(&pool, trashed_document, new_ts).await;+ insert_committed(+ &pool,+ NewEntityRecord::new(+ other_owners_trashed,+ RegisteredEntityType::Document,+ bot_owner(),+ ),+ )+ .await;+ mark_deleted_committed(&pool, other_owners_trashed, new_ts).await;++ let repo = PgEntityRegistryRepository::new(pool);+ let rows = repo.list_all_owned_by(&owner).await.unwrap();+ assert_eq!(+ rows.iter()+ .map(|row| (row.id, row.entity_type, row.owner.clone(), row.deleted_at))+ .collect::<Vec<_>>(),+ vec![+ (+ trashed_document,+ RegisteredEntityType::Document,+ owner.clone(),+ Some(new_ts)+ ),+ (live_chat, RegisteredEntityType::Chat, owner.clone(), None),+ ]+ );++ let live = repo.list_owned_by(&owner, None).await.unwrap();+ assert_eq!(+ live.iter().map(|row| row.id).collect::<Vec<_>>(),+ vec![live_chat]+ );+}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn count_by_type_splits_live_and_deleted(pool: PgPool) {let owner = user_owner();diff --git a/crates/entity_registry/shared_entity_registry/Cargo.toml b/crates/entity_registry/shared_entity_registry/Cargo.tomlindex f555ff7b18..48be92374c 100644--- a/crates/entity_registry/shared_entity_registry/Cargo.toml+++ b/crates/entity_registry/shared_entity_registry/Cargo.toml@@ -9,6 +9,10 @@ chrono = { workspace = true }model-entity = { path = "../../model-entity" }model_owner = { path = "../../model_owner" }rootcause = { workspace = true }+serde = { workspace = true }thiserror = { workspace = true }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../../workspace-hack" }++[dev-dependencies]+serde_json = { workspace = true }diff --git a/crates/entity_registry/shared_entity_registry/src/lib.rs b/crates/entity_registry/shared_entity_registry/src/lib.rsindex bc29abd136..2c2f2be669 100644--- a/crates/entity_registry/shared_entity_registry/src/lib.rs+++ b/crates/entity_registry/shared_entity_registry/src/lib.rs@@ -3,7 +3,8 @@//!//! Reads and owner-grant policy live in `entity_registry`. Transactional writes//! go through `entity_registry_db_utils`, which applies that policy when-//! registering owned entities.+//! registering owned entities. [`PurgeOwnedEntity`] is the permanent delete+//! each owning service exposes to callers that remove an owner.//!//! ```//! use model_entity::EntityType;@@ -23,12 +24,17 @@ mod test;use chrono::{DateTime, Utc};use model_entity::EntityType;use rootcause::Report;+use serde::{Deserialize, Serialize};use uuid::Uuid;pub use model_owner::Owner;/// The entity kinds the `entity` table accepts: its CHECK constraint as a type.-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+///+/// Serializes as its [`EntityType`] spelling, and deserializing any other+/// kind fails.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[serde(into = "EntityType", try_from = "EntityType")]pub enum RegisteredEntityType {/// `project`Project,@@ -197,3 +203,27 @@ pub enum EntityRegistryError {/// Typed report returned by every registry operation, read or write.pub type EntityRegistryResult<T> = Result<T, Report<EntityRegistryError>>;++/// What an owner-checked permanent delete did.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum OwnedPurgeOutcome {+ /// The entity is gone: deleted by this call, or already missing.+ Purged,+ /// The entity exists under a different owner. Nothing was deleted.+ OwnedElsewhere,+}++/// Permanent delete of one entity, conditional on its recorded owner.+///+/// The service that owns each [`RegisteredEntityType`] implements it, for+/// callers that remove everything an owner holds. Those callers retry until+/// the owner lists nothing, so a repeated call must converge.+pub trait PurgeOwnedEntity: Send + Sync {+ /// Permanently delete `entity_id`, live or trashed, if `expected_owner`+ /// is its recorded owner. A missing entity is [`OwnedPurgeOutcome::Purged`].+ fn purge_owned(+ &self,+ entity_id: Uuid,+ expected_owner: &Owner,+ ) -> impl Future<Output = Result<OwnedPurgeOutcome, Report>> + Send;+}diff --git a/crates/entity_registry/shared_entity_registry/src/test.rs b/crates/entity_registry/shared_entity_registry/src/test.rsindex ae54cca648..f6e10beb8b 100644--- a/crates/entity_registry/shared_entity_registry/src/test.rs+++ b/crates/entity_registry/shared_entity_registry/src/test.rs@@ -58,6 +58,39 @@ fn try_from_rejects_every_unregistered_entity_type() {}}+#[test]+fn registered_kinds_serialize_as_their_entity_type_spelling() {+ let expected = [+ (RegisteredEntityType::Project, r#""project""#),+ (RegisteredEntityType::Document, r#""document""#),+ (RegisteredEntityType::Chat, r#""chat""#),+ (RegisteredEntityType::AgentSession, r#""agent_session""#),+ (+ RegisteredEntityType::ScheduledAction,+ r#""scheduled_action""#,+ ),+ ];+ for (kind, json) in expected {+ assert_eq!(serde_json::to_string(&kind).unwrap(), json);+ assert_eq!(+ serde_json::from_str::<RegisteredEntityType>(json).unwrap(),+ kind+ );+ }+}++#[test]+fn deserializing_an_unregistered_kind_fails() {+ let error = serde_json::from_str::<RegisteredEntityType>(r#""team""#).unwrap_err();+ assert_eq!(error.to_string(), "team is not a registered entity type");++ let error = serde_json::from_str::<RegisteredEntityType>(r#""Document""#).unwrap_err();+ assert!(+ error.to_string().starts_with("unknown variant `Document`"),+ "{error}"+ );+}+#[test]fn try_new_accepts_chat_and_rejects_initiative() {let owner = Owner::parse(OwnerType::User, "macro|[email protected]").unwrap();diff --git a/crates/macro_sha_count_client/src/lib.rs b/crates/macro_sha_count_client/src/lib.rsindex 3e08cf6615..bc738b75d6 100644--- a/crates/macro_sha_count_client/src/lib.rs+++ b/crates/macro_sha_count_client/src/lib.rs@@ -1,6 +1,10 @@mod decrement_count;mod exists;mod increment_count;+#[cfg(test)]+mod test;++use std::collections::BTreeMap;use model::document::SaveBomPart;@@ -26,6 +30,12 @@ impl Redis {decrement_count::decrement_counts(&self.inner, SHA_DELETE_BUCKET, shas).await}+ /// Release one reference per entry in `shas`, such as the BOM of a deleted+ /// docx. A SHA listed twice loses two references.+ pub async fn release_shas(&self, shas: Vec<String>) -> anyhow::Result<()> {+ self.decrement_counts(&reference_counts(shas)).await+ }+pub async fn find_non_existing_shas(&self,bom_parts: &Vec<SaveBomPart>,@@ -48,3 +58,11 @@ impl Redis {.await}}++fn reference_counts(shas: Vec<String>) -> Vec<(String, i64)> {+ let mut counts = BTreeMap::<String, i64>::new();+ for sha in shas {+ *counts.entry(sha).or_default() += 1;+ }+ counts.into_iter().collect()+}diff --git a/crates/macro_sha_count_client/src/test.rs b/crates/macro_sha_count_client/src/test.rsnew file mode 100644index 0000000000..bfb90068fa--- /dev/null+++ b/crates/macro_sha_count_client/src/test.rs@@ -0,0 +1,15 @@+use super::reference_counts;++#[test]+fn each_listing_of_a_sha_is_one_reference() {+ let shas = ["b", "a", "b", "c", "b"].map(String::from).to_vec();+ assert_eq!(+ reference_counts(shas),+ vec![("a".into(), 1), ("b".into(), 3), ("c".into(), 1)]+ );+}++#[test]+fn no_shas_release_nothing() {+ assert!(reference_counts(Vec::new()).is_empty());+}diff --git a/crates/projects/Cargo.toml b/crates/projects/Cargo.tomlindex 4f5e464d82..1166bcd603 100644--- a/crates/projects/Cargo.toml+++ b/crates/projects/Cargo.toml@@ -42,7 +42,7 @@ outbound = ["ports",]ports = ["dep:tokio"]-service = ["dep:entity_access_management", "ports"]+service = ["dep:entity_access_management", "dep:shared_entity_registry", "ports"][dependencies]activity = { path = "../activity", default-features = false }@@ -78,6 +78,7 @@ models_bulk_upload = { path = "../models_bulk_upload" }models_permissions = { path = "../models_permissions" }rootcause = { workspace = true }s3_key = { path = "../s3_key" }+shared_entity_registry = { path = "../entity_registry/shared_entity_registry", optional = true }# Async runtime (for ports)tokio = { workspace = true, optional = true }diff --git a/crates/projects/src/domain/service.rs b/crates/projects/src/domain/service.rsindex 7c1b7e4eff..442b2450b6 100644--- a/crates/projects/src/domain/service.rs+++ b/crates/projects/src/domain/service.rs@@ -45,6 +45,7 @@ use super::ports::{};use super::upload::{build_destination_map, build_root_folder};+mod purge_owned;#[cfg(test)]mod tests;diff --git a/crates/projects/src/domain/service/purge_owned.rs b/crates/projects/src/domain/service/purge_owned.rsnew file mode 100644index 0000000000..772e4a4ca5--- /dev/null+++ b/crates/projects/src/domain/service/purge_owned.rs@@ -0,0 +1,63 @@+//! Owner-checked permanent delete of a project tree, for callers removing an+//! owner.++#[cfg(test)]+mod test;++use entity_access::domain::models::{EntityAccessReceipt, EntityType, OwnerAccessLevel};+use entity_access_management::domain::ports::EntityAccessManagementService;+use macro_event_broker::MacroEventBroker;+use model_owner::Owner;+use rootcause::{Report, prelude::*};+use shared_entity_registry::{OwnedPurgeOutcome, PurgeOwnedEntity};++use super::ProjectServiceImpl;+use crate::domain::models::ProjectError;+use crate::domain::ports::{+ BulkUploadRequestPort, ProjectRepo, ProjectSearchIndexer, ProjectService, ProjectUploadUrlPort,+ ShaCounterPort,+};++impl<R, U, D, Sha, Eam, Idx, B> PurgeOwnedEntity for ProjectServiceImpl<R, U, D, Sha, Eam, Idx, B>+where+ R: ProjectRepo,+ U: ProjectUploadUrlPort,+ D: BulkUploadRequestPort,+ Sha: ShaCounterPort,+ Eam: EntityAccessManagementService,+ Idx: ProjectSearchIndexer,+ B: MacroEventBroker,+{+ #[tracing::instrument(+ skip(self, expected_owner),+ fields(expected_owner.kind = ?expected_owner.owner_type()),+ err+ )]+ async fn purge_owned(+ &self,+ entity_id: uuid::Uuid,+ expected_owner: &Owner,+ ) -> Result<OwnedPurgeOutcome, Report> {+ let project_id = entity_id.to_string();+ let project = match self.internal_get_basic_project(&project_id).await {+ Err(ProjectError::NotFound(_)) => return Ok(OwnedPurgeOutcome::Purged),+ loaded => loaded.context("unable to look up the project")?,+ };+ if project.user_id != *expected_owner {+ return Ok(OwnedPurgeOutcome::OwnedElsewhere);+ }+ let receipt = EntityAccessReceipt::<OwnerAccessLevel>::dangerously_assert_internal_user(+ &project_id,+ EntityType::Project,+ );+ if project.deleted_at.is_none() {+ self.soft_delete_project(receipt.clone(), project.clone(), String::new())+ .await+ .context("unable to trash the project")?;+ }+ self.permanently_delete_project(receipt, project)+ .await+ .context("unable to permanently delete the project")?;+ Ok(OwnedPurgeOutcome::Purged)+ }+}diff --git a/crates/projects/src/domain/service/purge_owned/test.rs b/crates/projects/src/domain/service/purge_owned/test.rsnew file mode 100644index 0000000000..2b354f1011--- /dev/null+++ b/crates/projects/src/domain/service/purge_owned/test.rs@@ -0,0 +1,167 @@+use std::sync::{Arc, Mutex};++use model::project::BasicProject;+use uuid::Uuid;++use super::super::tests::{+ RecordingBulkUpload, TestEventBroker, assert_project_event, basic_project,+ service_with_event_broker,+};+use super::*;+use crate::domain::models::{PurgedProjectTree, SoftDeleteResult};+use crate::domain::ports::MockProjectRepo;++const TEAM_ID: Uuid = Uuid::from_u128(0x7ea3_0000_0000_0000_0000_0000_0000_0001);++type Steps = Arc<Mutex<Vec<&'static str>>>;++fn team() -> Owner {+ Owner::Team(TEAM_ID)+}++fn team_project(id: Uuid, deleted: bool) -> BasicProject {+ BasicProject {+ user_id: team(),+ ..basic_project(id, None, deleted)+ }+}++/// A repo that serves `project` and expects no other call.+fn serving(project: Option<BasicProject>) -> MockProjectRepo {+ let mut repo = MockProjectRepo::new();+ repo.expect_get_basic_project()+ .return_once(move |_| Box::pin(async move { Ok(project) }));+ repo+}++fn expect_trash(repo: &mut MockProjectRepo, project_id: Uuid, steps: &Steps) {+ let steps = steps.clone();+ repo.expect_soft_delete_project()+ .withf(move |id| id == project_id.to_string())+ .return_once(move |_| {+ steps.lock().unwrap().push("trash");+ Box::pin(async move {+ Ok(SoftDeleteResult {+ project_ids: vec![project_id.to_string()],+ document_ids: Vec::new(),+ chat_ids: Vec::new(),+ })+ })+ });+}++fn expect_purge(repo: &mut MockProjectRepo, project_id: Uuid, steps: &Steps) {+ let steps = steps.clone();+ repo.expect_purge_deleted_project_tree()+ .withf(move |id| id == project_id.to_string())+ .return_once(move |_| {+ steps.lock().unwrap().push("purge");+ Box::pin(async move {+ Ok(PurgedProjectTree {+ project_ids: vec![project_id.to_string()],+ chat_ids: Vec::new(),+ documents: Vec::new(),+ bom_shas: Vec::new(),+ })+ })+ });+}++#[tokio::test]+async fn purge_owned_trashes_a_live_project_then_purges_its_tree() {+ let project_id = Uuid::new_v4();+ let steps = Steps::default();+ let mut repo = serving(Some(team_project(project_id, false)));+ expect_trash(&mut repo, project_id, &steps);+ expect_purge(&mut repo, project_id, &steps);+ let broker = TestEventBroker::default();+ let published = broker.published();+ let service = service_with_event_broker(repo, RecordingBulkUpload::default(), broker);++ let outcome = service.purge_owned(project_id, &team()).await.unwrap();++ assert_eq!(outcome, OwnedPurgeOutcome::Purged);+ assert_eq!(*steps.lock().unwrap(), ["trash", "purge"]);+ let published = published.lock().unwrap();+ assert_eq!(published.len(), 2);+ assert_project_event(&published[0], project_id, "project.deleted");+ assert_project_event(&published[1], project_id, "project.permanently_deleted");+ let metadata = &published[1].payload["metadata"];+ assert_eq!(metadata["owner"], TEAM_ID.to_string());+ assert!(metadata["actor_user_id"].is_null());+}++#[tokio::test]+async fn purge_owned_purges_a_trashed_project_without_trashing_it_again() {+ let project_id = Uuid::new_v4();+ let steps = Steps::default();+ let mut repo = serving(Some(team_project(project_id, true)));+ expect_purge(&mut repo, project_id, &steps);+ let service = service_with_event_broker(+ repo,+ RecordingBulkUpload::default(),+ TestEventBroker::default(),+ );++ let outcome = service.purge_owned(project_id, &team()).await.unwrap();++ assert_eq!(outcome, OwnedPurgeOutcome::Purged);+ assert_eq!(*steps.lock().unwrap(), ["purge"]);+}++#[tokio::test]+async fn purge_owned_refuses_a_project_owned_by_someone_else() {+ let project_id = Uuid::new_v4();+ let broker = TestEventBroker::default();+ let published = broker.published();+ let service = service_with_event_broker(+ serving(Some(basic_project(project_id, None, false))),+ RecordingBulkUpload::default(),+ broker,+ );++ let outcome = service.purge_owned(project_id, &team()).await.unwrap();++ assert_eq!(outcome, OwnedPurgeOutcome::OwnedElsewhere);+ assert!(published.lock().unwrap().is_empty());+}++#[tokio::test]+async fn purge_owned_of_a_missing_project_is_purged() {+ let broker = TestEventBroker::default();+ let published = broker.published();+ let service = service_with_event_broker(serving(None), RecordingBulkUpload::default(), broker);++ let outcome = service.purge_owned(Uuid::new_v4(), &team()).await.unwrap();++ assert_eq!(outcome, OwnedPurgeOutcome::Purged);+ assert!(published.lock().unwrap().is_empty());+}++#[tokio::test]+async fn purge_owned_stops_before_the_purge_when_trashing_fails() {+ let project_id = Uuid::new_v4();+ let mut repo = serving(Some(team_project(project_id, false)));+ repo.expect_soft_delete_project()+ .return_once(|_| Box::pin(async { Err(anyhow::anyhow!("delete failed")) }));+ let broker = TestEventBroker::default();+ let published = broker.published();+ let service = service_with_event_broker(repo, RecordingBulkUpload::default(), broker);++ assert!(service.purge_owned(project_id, &team()).await.is_err());+ assert!(published.lock().unwrap().is_empty());+}++#[tokio::test]+async fn purge_owned_returns_a_failed_purge_as_an_error() {+ let project_id = Uuid::new_v4();+ let mut repo = serving(Some(team_project(project_id, true)));+ repo.expect_purge_deleted_project_tree()+ .return_once(|_| Box::pin(async { Err(anyhow::anyhow!("commit failed")) }));+ let broker = TestEventBroker::default();+ let published = broker.published();+ let service = service_with_event_broker(repo, RecordingBulkUpload::default(), broker);++ assert!(service.purge_owned(project_id, &team()).await.is_err());+ assert!(published.lock().unwrap().is_empty());+}diff --git a/crates/projects/src/domain/service/tests.rs b/crates/projects/src/domain/service/tests.rsindex 703e371931..585f30be4b 100644--- a/crates/projects/src/domain/service/tests.rs+++ b/crates/projects/src/domain/service/tests.rs@@ -37,14 +37,14 @@ use crate::domain::ports::MockProjectRepo;/// A project lifecycle event recorded by [`TestEventBroker`].#[derive(Clone, Debug)]-struct PublishedEvent {+pub(super) struct PublishedEvent {topic: &'static str,key: String,- payload: serde_json::Value,+ pub(super) payload: serde_json::Value,}#[derive(Clone, Default)]-struct TestEventBroker {+pub(super) struct TestEventBroker {published: Arc<Mutex<Vec<PublishedEvent>>>,fail: bool,}@@ -57,7 +57,7 @@ impl TestEventBroker {}}- fn published(&self) -> Arc<Mutex<Vec<PublishedEvent>>> {+ pub(super) fn published(&self) -> Arc<Mutex<Vec<PublishedEvent>>> {Arc::clone(&self.published)}}@@ -81,7 +81,7 @@ impl MacroEventBroker for TestEventBroker {}#[derive(Clone, Copy)]-struct NullPort;+pub(super) struct NullPort;impl ProjectUploadUrlPort for NullPort {async fn put_upload_zip_staging_presigned_url(@@ -277,7 +277,7 @@ fn mutation_service_with_event_broker(}#[derive(Clone, Default)]-struct RecordingBulkUpload {+pub(super) struct RecordingBulkUpload {calls: Arc<Mutex<Vec<String>>>,}@@ -322,7 +322,7 @@ fn service<D: BulkUploadRequestPort>(service_with_event_broker(repo, bulk_upload_service, TestEventBroker::default())}-fn service_with_event_broker<D: BulkUploadRequestPort>(+pub(super) fn service_with_event_broker<D: BulkUploadRequestPort>(repo: MockProjectRepo,bulk_upload_service: D,event_broker: TestEventBroker,@@ -442,7 +442,7 @@ fn receipt(.unwrap()}-fn basic_project(id: Uuid, parent_id: Option<Uuid>, deleted: bool) -> BasicProject {+pub(super) fn basic_project(id: Uuid, parent_id: Option<Uuid>, deleted: bool) -> BasicProject {BasicProject {id: id.to_string(),user_id: owner("macro|[email protected]"),@@ -482,7 +482,7 @@ where.unwrap()}-fn assert_project_event(event: &PublishedEvent, key: Uuid, event_type: &str) {+pub(super) fn assert_project_event(event: &PublishedEvent, key: Uuid, event_type: &str) {assert_eq!(event.topic, "macro.projects");assert_eq!(event.key, key.to_string());assert!(Uuid::parse_str(event.payload["event_id"].as_str().unwrap()).is_ok());diff --git a/crates/teams/Cargo.toml b/crates/teams/Cargo.tomlindex 132b765cfa..c46a3ad5fa 100644--- a/crates/teams/Cargo.toml+++ b/crates/teams/Cargo.toml@@ -39,6 +39,7 @@ inbound = ["axum", "ports"]outbound = ["dep:analytics_client","dep:contacts",+ "dep:entity_access_db_utils","dep:models_email","dep:sqlx","dep:sqs_client",@@ -62,6 +63,7 @@ async-stripe = { workspace = true, default-features = false, features = ["runtime-tokio-hyper-rustls-webpki",] }async-trait = { workspace = true, optional = true }+bot_id = { path = "../bot_id" }chrono = { workspace = true }channels = { path = "../channels", default-features = false, features = ["ports"] }contacts = { path = "../contacts", default-features = false, optional = true }@@ -74,6 +76,7 @@ macro_event_broker = { path = "../macro_event_broker", default-features = falsemacro_event_topics = { path = "../macro_event_topics" }macro_user_id = { path = "../macro_user_id" }macro_uuid = { path = "../macro_uuid" }+model_owner = { path = "../model_owner" }models_permissions = { path = "../models_permissions" }non_empty = { path = "../non_empty" }roles_and_permissions = { path = "../roles_and_permissions", default-features = false, features = [@@ -82,6 +85,7 @@ roles_and_permissions = { path = "../roles_and_permissions", default-features =rootcause = { workspace = true }schemars = { workspace = true, optional = true }serde = { workspace = true }+shared_entity_registry = { path = "../entity_registry/shared_entity_registry" }thiserror = { workspace = true }tracing = { workspace = true }uuid = { workspace = true }@@ -105,6 +109,7 @@ notification = { path = "../notification", optional = true, default-features = f# Outbound dependenciesanalytics_client = { path = "../analytics_client", optional = true }+entity_access_db_utils = { path = "../entity_access_db_utils", optional = true }models_email = { path = "../models_email", optional = true }sqlx = { workspace = true, optional = true }sqs_client = { path = "../sqs_client", default-features = false, features = [diff --git a/crates/teams/fixtures/team_grants.sql b/crates/teams/fixtures/team_grants.sqlnew file mode 100644index 0000000000..f9eb8e86f2--- /dev/null+++ b/crates/teams/fixtures/team_grants.sql@@ -0,0 +1,8 @@+INSERT INTO entity_access (entity_id, entity_type, source_id, source_type, access_level) VALUES+ ('70000000-0000-0000-0000-000000000001', 'document', '11111111-1111-1111-1111-111111111111', 'team', 'edit'),+ ('70000000-0000-0000-0000-000000000002', 'chat', '11111111-1111-1111-1111-111111111111', 'team', 'view'),+ ('70000000-0000-0000-0000-000000000001', 'document', 'bot|60000000-0000-0000-0000-000000000001', 'bot', 'owner'),+ ('70000000-0000-0000-0000-000000000002', 'chat', 'bot|60000000-0000-0000-0000-000000000002', 'bot', 'owner'),+ ('70000000-0000-0000-0000-000000000001', 'document', 'bot|60000000-0000-0000-0000-000000000003', 'bot', 'view'),+ ('70000000-0000-0000-0000-000000000001', 'document', '22222222-2222-2222-2222-222222222222', 'team', 'view'),+ ('70000000-0000-0000-0000-000000000001', 'document', 'macro|[email protected]', 'user', 'owner');diff --git a/crates/teams/src/domain.rs b/crates/teams/src/domain.rsindex a378a8167a..189f1a0b49 100644--- a/crates/teams/src/domain.rs+++ b/crates/teams/src/domain.rs@@ -6,6 +6,7 @@ pub mod events;pub mod join_announcement;pub mod model;pub mod open_seat_release;+pub mod owned_entity_cleanup;pub mod team_analytics;pub mod team_crm_settings_repo;pub mod team_repo;diff --git a/crates/teams/src/domain/events.rs b/crates/teams/src/domain/events.rsindex cfa7d678d3..c92d47e1f4 100644--- a/crates/teams/src/domain/events.rs+++ b/crates/teams/src/domain/events.rs@@ -6,13 +6,16 @@#[cfg(test)]mod test;+use bot_id::BotId;use macro_event_broker::{Event, MacroEvent, TopicEvent};use macro_event_topics::MacroTeamsTopic;use macro_user_id::user_id::MacroUserIdStr;+use model_owner::Owner;use serde::{Deserialize, Serialize};+use shared_entity_registry::RegisteredEntityType;use uuid::Uuid;-use super::model::TeamRole;+use super::{model::TeamRole, owned_entity_cleanup::OwnedEntityRef};/// Metadata for [`TeamTopicEvent::Created`].#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]@@ -55,6 +58,36 @@ pub struct TeamDeletedMetadata {pub actor_user_id: MacroUserIdStr<'static>,/// Users who belonged to the team when it was deleted.pub member_user_ids: Vec<MacroUserIdStr<'static>>,+ /// Every bot the team had, soft-deleted ones included, in id order.+ /// Missing on events published before this field existed.+ #[serde(default)]+ pub bot_ids: Vec<BotId>,+ /// Entities the deletion purged because the team or one of its bots owned+ /// them, in purge order. Entities an earlier, failed attempt purged are+ /// absent. Missing on events published before this field existed.+ #[serde(default)]+ pub owned_entities: Vec<OwnedEntityMetadata>,+}++/// An entity purged by a team deletion, in [`TeamDeletedMetadata`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct OwnedEntityMetadata {+ /// Kind of the purged entity.+ pub entity_type: RegisteredEntityType,+ /// Identifier of the purged entity.+ pub id: Uuid,+ /// The team or bot that owned the entity.+ pub owner: Owner,+}++impl From<&OwnedEntityRef> for OwnedEntityMetadata {+ fn from(entity: &OwnedEntityRef) -> Self {+ Self {+ entity_type: entity.entity_type,+ id: entity.id,+ owner: entity.owner.as_owner(),+ }+ }}/// Metadata for [`TeamTopicEvent::InviteCreated`].diff --git a/crates/teams/src/domain/events/test.rs b/crates/teams/src/domain/events/test.rsindex bdd9c9a0b4..183940ac9e 100644--- a/crates/teams/src/domain/events/test.rs+++ b/crates/teams/src/domain/events/test.rs@@ -8,6 +8,9 @@ use super::*;const TEAM_ID: &str = "3f6f8b0a-6f9f-4a3f-9c3a-2b1e5d4c7a90";const INVITE_ID: &str = "0197f776-6e7b-7c69-a251-780ae754d3e4";const EVENT_ID: &str = "01998a30-1a2b-7c3d-9e4f-5a6b7c8d9e0f";+const BOT_ID: &str = "6d5e2c1b-3a4f-4b8e-9c7d-1e2f3a4b5c6d";+const SESSION_ID: &str = "0199a7c4-52e1-7d3b-8f60-2c9d4e5f6a7b";+const PROJECT_ID: &str = "0199a7c4-52e1-7d3b-8f60-2c9d4e5f6a7c";fn team_id() -> Uuid {Uuid::parse_str(TEAM_ID).expect("valid team id")@@ -17,6 +20,10 @@ fn invite_id() -> Uuid {Uuid::parse_str(INVITE_ID).expect("valid invite id")}+fn bot_id() -> BotId {+ BotId::parse_uuid_str(BOT_ID).expect("valid bot id")+}+fn user_id(value: &str) -> MacroUserIdStr<'static> {MacroUserIdStr::try_from(value.to_string()).expect("valid user id")}@@ -71,13 +78,39 @@ fn topic_events() -> Vec<(TeamTopicEvent, Value)> {user_id("macro|[email protected]"),user_id("macro|[email protected]"),],+ bot_ids: vec![bot_id()],+ owned_entities: vec![+ OwnedEntityMetadata {+ entity_type: RegisteredEntityType::AgentSession,+ id: Uuid::parse_str(SESSION_ID).expect("valid session id"),+ owner: Owner::Bot(bot_id()),+ },+ OwnedEntityMetadata {+ entity_type: RegisteredEntityType::Project,+ id: Uuid::parse_str(PROJECT_ID).expect("valid project id"),+ owner: Owner::Team(team_id()),+ },+ ],}),json!({"event_type": "team.deleted","metadata": {"team_id": TEAM_ID,"actor_user_id": "macro|[email protected]",- "member_user_ids": ["macro|[email protected]", "macro|[email protected]"]+ "member_user_ids": ["macro|[email protected]", "macro|[email protected]"],+ "bot_ids": [BOT_ID],+ "owned_entities": [+ {+ "entity_type": "agent_session",+ "id": SESSION_ID,+ "owner": "bot|6d5e2c1b-3a4f-4b8e-9c7d-1e2f3a4b5c6d"+ },+ {+ "entity_type": "project",+ "id": PROJECT_ID,+ "owner": TEAM_ID+ }+ ]}}),),@@ -306,6 +339,33 @@ fn member_joined_defaults_missing_teammate_ids() {}}+#[test]+fn deleted_defaults_missing_bots_and_owned_entities() {+ let payload = json!({+ "event_id": EVENT_ID,+ "schema_version": 1,+ "event_type": "team.deleted",+ "metadata": {+ "team_id": TEAM_ID,+ "actor_user_id": "macro|[email protected]",+ "member_user_ids": ["macro|[email protected]"]+ }+ });++ let event: Event<TeamTopicEvent> =+ serde_json::from_value(payload).expect("legacy deleted is decodable");+ assert_eq!(+ event.event,+ TeamTopicEvent::Deleted(TeamDeletedMetadata {+ team_id: team_id(),+ actor_user_id: user_id("macro|[email protected]"),+ member_user_ids: vec![user_id("macro|[email protected]")],+ bot_ids: Vec::new(),+ owned_entities: Vec::new(),+ })+ );+}+#[test]fn roles_and_join_methods_serialize_lowercase() {let invite_join = serde_json::to_value(TeamJoinMethod::InviteAccepted {diff --git a/crates/teams/src/domain/model.rs b/crates/teams/src/domain/model.rsindex bf8bba77b8..b47de0a1aa 100644--- a/crates/teams/src/domain/model.rs+++ b/crates/teams/src/domain/model.rs@@ -9,6 +9,8 @@ use macro_user_id::{use models_permissions::share_permission::LinkShare;use roles_and_permissions::domain::model::{ProductTier, RoleId, UserRolesAndPermissionsError};+use crate::domain::owned_entity_cleanup::OwnedEntityCleanupError;+/// Team plans#[derive(Eq,@@ -782,6 +784,9 @@ pub enum DeleteTeamError {/// Remove roles from user error#[error("Remove roles from user error")]RemoveRolesFromUserError(#[from] UserRolesAndPermissionsError),+ /// The team's content could not be purged, so the team was not deleted+ #[error("unable to purge the team's content")]+ OwnedEntityCleanup(#[from] OwnedEntityCleanupError),}/// Errors for detaching a user from every team ahead of account deletiondiff --git a/crates/teams/src/domain/owned_entity_cleanup.rs b/crates/teams/src/domain/owned_entity_cleanup.rsnew file mode 100644index 0000000000..4a8a8e989b--- /dev/null+++ b/crates/teams/src/domain/owned_entity_cleanup.rs@@ -0,0 +1,313 @@+//! Clearing everything a team owns before the team row is deleted.+//!+//! A team owns entities directly ([`Owner::Team`]) and through its bots+//! ([`Owner::Bot`]). `bots.team_id` cascades with the team row, so once the+//! team is deleted nothing can find bot-owned content again. [`clear_team`]+//! runs first, and the [`ClearedTeam`] it returns is the only way to delete+//! the team row.+//!+//! The entity registry is the work queue. An owning service removes an+//! entity's registry row only after it has enqueued that entity's physical+//! cleanup, so a failure at any point leaves exactly the unfinished entities+//! listed for the next attempt.++#[cfg(test)]+mod test;++use std::collections::HashSet;+use std::fmt::{self, Display, Formatter};++use bot_id::BotId;+use model_owner::Owner;+use shared_entity_registry::RegisteredEntityType;+use uuid::Uuid;++/// Discovery passes [`clear_team`] runs before it gives up on a team whose+/// content keeps reappearing. Every pass but the last purges what it found.+pub(crate) const MAX_DISCOVERY_PASSES: u8 = 4;++/// An owner whose entities are deleted with the team. A user cannot be one,+/// because deleting a team never purges a person's content.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum TeamDeletionOwner {+ /// The team itself.+ Team(Uuid),+ /// One of the team's bots, soft-deleted or not.+ Bot(BotId),+}++impl TeamDeletionOwner {+ /// The registry owner this is.+ pub fn as_owner(&self) -> Owner {+ match *self {+ Self::Team(team_id) => Owner::Team(team_id),+ Self::Bot(bot_id) => Owner::Bot(bot_id),+ }+ }+}++impl Display for TeamDeletionOwner {+ fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {+ Display::fmt(&self.as_owner(), formatter)+ }+}++/// One entity a team owns, directly or through one of its bots.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct OwnedEntityRef {+ /// The entity's id.+ pub id: Uuid,+ /// The entity's kind, which decides the owning service that purges it.+ pub entity_type: RegisteredEntityType,+ /// The entity's recorded owner.+ pub owner: TeamDeletionOwner,+}++/// What team deletion needs from outside the teams crate: the team's bots,+/// what each owner holds, and a purge through each entity's owning service.+pub trait OwnedEntityCleanup: Clone + Send + Sync + 'static {+ /// Error returned when a listing or a purge fails.+ type Err: std::error::Error + Send + Sync + 'static;++ /// Every bot whose team is `team_id`, soft-deleted bots included.+ fn team_bots(+ &self,+ team_id: Uuid,+ ) -> impl Future<Output = Result<Vec<BotId>, Self::Err>> + Send;++ /// Every entity recorded as owned by `owner`, trashed entities included.+ fn owned_by(+ &self,+ owner: &TeamDeletionOwner,+ ) -> impl Future<Output = Result<Vec<OwnedEntityRef>, Self::Err>> + Send;++ /// Permanently delete `entity` through its owning service, provided+ /// `entity.owner` still owns it.+ ///+ /// `Ok` means the entity is gone, or was already gone, and its physical+ /// cleanup is enqueued. An entity its owning service records under another+ /// owner is an error, and nothing is deleted.+ fn purge(&self, entity: &OwnedEntityRef) -> impl Future<Output = Result<(), Self::Err>> + Send;+}++/// The cleanup of a team service that was never given one.+///+/// Every call fails, so such a service refuses to delete teams instead of+/// deleting the team row and orphaning its content.+#[derive(Clone, Copy, Debug, Default)]+pub struct UnwiredOwnedEntityCleanup;++/// The error [`UnwiredOwnedEntityCleanup`] returns from every call.+#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]+#[error("team deletion needs an owned-entity cleanup and none is wired")]+pub struct OwnedEntityCleanupUnwired;++impl OwnedEntityCleanup for UnwiredOwnedEntityCleanup {+ type Err = OwnedEntityCleanupUnwired;++ async fn team_bots(&self, _team_id: Uuid) -> Result<Vec<BotId>, Self::Err> {+ Err(OwnedEntityCleanupUnwired)+ }++ async fn owned_by(&self, _owner: &TeamDeletionOwner) -> Result<Vec<OwnedEntityRef>, Self::Err> {+ Err(OwnedEntityCleanupUnwired)+ }++ async fn purge(&self, _entity: &OwnedEntityRef) -> Result<(), Self::Err> {+ Err(OwnedEntityCleanupUnwired)+ }+}++/// Proof that a team, and every bot it has had, owns nothing.+///+/// Only [`clear_team`] constructs one, from a listing that came back empty.+#[derive(Debug)]+pub struct ClearedTeam {+ team_id: Uuid,+ bot_ids: Vec<BotId>,+ purged: Vec<OwnedEntityRef>,+}++impl ClearedTeam {+ /// The team that owns nothing.+ pub fn team_id(&self) -> Uuid {+ self.team_id+ }++ /// Every bot the team has had, soft-deleted ones included, in id order.+ pub fn bot_ids(&self) -> &[BotId] {+ &self.bot_ids+ }++ /// What this attempt purged, in purge order. Entities an earlier, failed+ /// attempt purged are absent, because they were already gone when this+ /// one listed.+ pub fn purged(&self) -> &[OwnedEntityRef] {+ &self.purged+ }++ /// A proof for repository tests that exercise the row delete on its own.+ #[cfg(test)]+ pub(crate) fn assume_cleared(team_id: Uuid, bot_ids: Vec<BotId>) -> Self {+ Self {+ team_id,+ bot_ids,+ purged: Vec::new(),+ }+ }+}++/// Why a team's content could not be cleared.+///+/// Whatever the variant, the team row, its bots, and its subscription are+/// untouched. Entities purged before the failure stay purged, and the next+/// [`clear_team`] resumes with what is left.+#[derive(Debug, thiserror::Error)]+pub enum OwnedEntityCleanupError {+ /// The team's bots could not be listed.+ #[error("unable to list the team's bots")]+ ListBots {+ /// The port's error.+ #[source]+ source: Box<dyn std::error::Error + Send + Sync>,+ },+ /// The entities one owner holds could not be listed.+ #[error("unable to list the entities owned by {owner}")]+ ListEntities {+ /// The team or bot whose entities were being listed.+ owner: TeamDeletionOwner,+ /// The port's error.+ #[source]+ source: Box<dyn std::error::Error + Send + Sync>,+ },+ /// An owning service did not purge an entity.+ #[error("unable to purge {entity_type} {id}")]+ Purge {+ /// The kind of the entity that is still there.+ entity_type: RegisteredEntityType,+ /// The id of the entity that is still there.+ id: Uuid,+ /// The port's error.+ #[source]+ source: Box<dyn std::error::Error + Send + Sync>,+ },+ /// Entities were still owned after the last discovery pass, for example+ /// because members kept creating team content during the deletion.+ #[error("{remaining} entities were still owned after {passes} discovery passes")]+ StillOwned {+ /// How many entities the last pass listed.+ remaining: usize,+ /// How many discovery passes ran.+ passes: u8,+ },+ /// A listing for one owner returned an entity recorded under another.+ #[error("listing the entities owned by {requested} returned {id}, which it does not own")]+ OwnerMismatch {+ /// The owner whose entities were listed.+ requested: TeamDeletionOwner,+ /// The entity recorded under another owner.+ id: Uuid,+ },+}++/// Permanently delete everything `team_id` owns, directly or through any bot+/// it has had, and return the proof that it now owns nothing.+///+/// Each pass lists the team's bots, then what the team and each bot own. An+/// empty listing ends the run. Otherwise the pass purges what it listed, one+/// entity at a time, scheduled actions and agent sessions first and projects+/// last. The first failure ends the run.+#[tracing::instrument(skip(cleanup), err)]+pub async fn clear_team<C: OwnedEntityCleanup>(+ cleanup: &C,+ team_id: Uuid,+) -> Result<ClearedTeam, OwnedEntityCleanupError> {+ let mut purged = Vec::new();+ let mut purged_ids = HashSet::new();+ let mut passes: u8 = 0;+ loop {+ passes += 1;+ let Census { bot_ids, mut owned } = take_census(cleanup, team_id).await?;+ if owned.is_empty() {+ return Ok(ClearedTeam {+ team_id,+ bot_ids,+ purged,+ });+ }+ if passes == MAX_DISCOVERY_PASSES {+ return Err(OwnedEntityCleanupError::StillOwned {+ remaining: owned.len(),+ passes,+ });+ }+ owned.sort_unstable_by_key(|entity| (purge_rank(entity.entity_type), entity.id));+ for entity in owned {+ cleanup+ .purge(&entity)+ .await+ .map_err(|source| OwnedEntityCleanupError::Purge {+ entity_type: entity.entity_type,+ id: entity.id,+ source: Box::new(source),+ })?;+ if purged_ids.insert(entity.id) {+ purged.push(entity);+ }+ }+ }+}++/// Producers go before what they produce, so a running session or a firing+/// scheduled action cannot add content behind the purge. Projects go last.+/// Their tree purge deletes nested items of any owner, so the team's own+/// nested documents and chats take their owner-checked, retry-safe purge first.+fn purge_rank(entity_type: RegisteredEntityType) -> u8 {+ match entity_type {+ RegisteredEntityType::ScheduledAction => 0,+ RegisteredEntityType::AgentSession => 1,+ RegisteredEntityType::Chat => 2,+ RegisteredEntityType::Document => 3,+ RegisteredEntityType::Project => 4,+ }+}++struct Census {+ bot_ids: Vec<BotId>,+ owned: Vec<OwnedEntityRef>,+}++async fn take_census<C: OwnedEntityCleanup>(+ cleanup: &C,+ team_id: Uuid,+) -> Result<Census, OwnedEntityCleanupError> {+ let mut bot_ids =+ cleanup+ .team_bots(team_id)+ .await+ .map_err(|source| OwnedEntityCleanupError::ListBots {+ source: Box::new(source),+ })?;+ bot_ids.sort_unstable_by_key(|bot_id| bot_id.as_uuid());+ bot_ids.dedup();++ let owners = std::iter::once(TeamDeletionOwner::Team(team_id))+ .chain(bot_ids.iter().copied().map(TeamDeletionOwner::Bot));+ let mut owned = Vec::new();+ for owner in owners {+ let listed = cleanup.owned_by(&owner).await.map_err(|source| {+ OwnedEntityCleanupError::ListEntities {+ owner,+ source: Box::new(source),+ }+ })?;+ if let Some(stray) = listed.iter().find(|entity| entity.owner != owner) {+ return Err(OwnedEntityCleanupError::OwnerMismatch {+ requested: owner,+ id: stray.id,+ });+ }+ owned.extend(listed);+ }+ Ok(Census { bot_ids, owned })+}diff --git a/crates/teams/src/domain/owned_entity_cleanup/test.rs b/crates/teams/src/domain/owned_entity_cleanup/test.rsnew file mode 100644index 0000000000..bdac7af3b5--- /dev/null+++ b/crates/teams/src/domain/owned_entity_cleanup/test.rs@@ -0,0 +1,24 @@+use super::*;++#[tokio::test]+async fn unwired_cleanup_refuses_every_call() {+ let team_id = Uuid::from_u128(1);+ let entity = OwnedEntityRef {+ id: Uuid::from_u128(2),+ entity_type: RegisteredEntityType::Document,+ owner: TeamDeletionOwner::Team(team_id),+ };++ assert_eq!(+ UnwiredOwnedEntityCleanup.team_bots(team_id).await,+ Err(OwnedEntityCleanupUnwired)+ );+ assert_eq!(+ UnwiredOwnedEntityCleanup.owned_by(&entity.owner).await,+ Err(OwnedEntityCleanupUnwired)+ );+ assert_eq!(+ UnwiredOwnedEntityCleanup.purge(&entity).await,+ Err(OwnedEntityCleanupUnwired)+ );+}diff --git a/crates/teams/src/domain/team_repo.rs b/crates/teams/src/domain/team_repo.rsindex 7a54cc2396..ff530b412e 100644--- a/crates/teams/src/domain/team_repo.rs+++ b/crates/teams/src/domain/team_repo.rs@@ -15,6 +15,7 @@ use crate::domain::model::{TeamInvite, TeamInviteDetails, TeamMember, TeamMembers, TeamPlan, TeamRole, TeamWithMembers,ToggleAutoJoinDomainError, TryJoinTeamByDomainError,};+use crate::domain::owned_entity_cleanup::ClearedTeam;/// The TeamRepository defines a set of actions to perform on teams datapub trait TeamRepository: Clone + Send + Sync + 'static {@@ -133,10 +134,11 @@ pub trait TeamRepository: Clone + Send + Sync + 'static {paying: bool,) -> impl Future<Output = Result<(), TeamError>> + Send;- /// Deletes a team+ /// Deletes a team that owns nothing, with the access grants it and its+ /// bots were given.fn delete_team(&self,- team_id: &uuid::Uuid,+ cleared: &ClearedTeam,) -> impl Future<Output = Result<(), TeamError>> + Send;/// Gets all members of a team including the owner@@ -394,7 +396,8 @@ pub trait TeamService: Clone + Send + Sync + 'static {team_invite_id: &uuid::Uuid,) -> impl Future<Output = Result<(), RemoveTeamInviteError>> + Send;- /// Cancels the team subscription and deletes the team.+ /// Purges everything the team and its bots own, then cancels the team+ /// subscription and deletes the team.fn delete_team(&self,entity_access_receipt: EntityAccessReceipt<OwnerTeamRole>,diff --git a/crates/teams/src/domain/team_service.rs b/crates/teams/src/domain/team_service.rsindex 865e0301a0..3cbc8f78ed 100644--- a/crates/teams/src/domain/team_service.rs+++ b/crates/teams/src/domain/team_service.rs@@ -36,10 +36,10 @@ use crate::domain::{crm_enqueuer::CrmEnqueuer,customer_repo::CustomerRepository,events::{- TeamAutoJoinDomainToggledMetadata, TeamCreatedMetadata, TeamDeletedMetadata,- TeamInviteCreatedMetadata, TeamInviteRejectedMetadata, TeamInviteRevokedMetadata,- TeamJoinMethod, TeamMacroEvent, TeamMemberJoinedMetadata, TeamMemberRemovedMetadata,- TeamMemberRoleChangedMetadata, TeamUpdatedMetadata,+ OwnedEntityMetadata, TeamAutoJoinDomainToggledMetadata, TeamCreatedMetadata,+ TeamDeletedMetadata, TeamInviteCreatedMetadata, TeamInviteRejectedMetadata,+ TeamInviteRevokedMetadata, TeamJoinMethod, TeamMacroEvent, TeamMemberJoinedMetadata,+ TeamMemberRemovedMetadata, TeamMemberRoleChangedMetadata, TeamUpdatedMetadata,},model::{CreateTeamError, CustomerError, DeleteTeamError, InviteUsersToTeamError, JoinTeamError,@@ -51,6 +51,7 @@ use crate::domain::{TryJoinTeamByDomainError, is_generic_email_domain, team_slug_from_name,},open_seat_release::{NoOpOpenSeatRelease, OpenSeatRelease},+ owned_entity_cleanup::{OwnedEntityCleanup, UnwiredOwnedEntityCleanup, clear_team},team_analytics::{NoOpTeamAnalytics, TeamAnalytics, TeamAnalyticsEvent},team_crm_settings_repo::TeamCrmSettingsRepository,team_repo::{TeamMembersService, TeamRepository, TeamService},@@ -107,6 +108,7 @@ pub struct TeamServiceImpl<CNE = NoOpContactsEnqueuer,EB = NoopMacroEventBroker,OSR = NoOpOpenSeatRelease,+ OEC = UnwiredOwnedEntityCleanup,> whereTR: TeamRepository,CR: CustomerRepository,@@ -119,6 +121,7 @@ pub struct TeamServiceImpl<CNE: ContactsEnqueuer,EB: MacroEventBroker,OSR: OpenSeatRelease,+ OEC: OwnedEntityCleanup,{/// The underlying team repositoryteam_repository: TR,@@ -145,14 +148,17 @@ pub struct TeamServiceImpl<event_broker: EB,/// Outbound port that releases a removed member's open seat.open_seat_release: OSR,+ /// Outbound port that purges everything a team owns before the team is+ /// deleted. The default refuses, so an unwired service cannot delete teams.+ owned_entity_cleanup: OEC,}fn channel_error_to_team_error(error: ChannelMutationErr) -> TeamError {TeamError::StorageLayerError(error.into())}-impl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR> Clone- for TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR>+impl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR, OEC> Clone+ for TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR, OEC>whereTR: TeamRepository,CR: CustomerRepository,@@ -165,6 +171,7 @@ whereCNE: ContactsEnqueuer,EB: MacroEventBroker + Clone,OSR: OpenSeatRelease,+ OEC: OwnedEntityCleanup,{fn clone(&self) -> Self {Self {@@ -179,6 +186,7 @@ wherecontacts_enqueuer: self.contacts_enqueuer.clone(),event_broker: self.event_broker.clone(),open_seat_release: self.open_seat_release.clone(),+ owned_entity_cleanup: self.owned_entity_cleanup.clone(),}}}@@ -196,6 +204,7 @@ impl<TR, CR, CS, URPS, NI, CE, TCRMS>NoOpContactsEnqueuer,NoopMacroEventBroker,NoOpOpenSeatRelease,+ UnwiredOwnedEntityCleanup,>whereTR: TeamRepository,@@ -242,6 +251,7 @@ impl<TR, CR, CS, URPS, NI, CE, TCRMS, TA>NoOpContactsEnqueuer,NoopMacroEventBroker,NoOpOpenSeatRelease,+ UnwiredOwnedEntityCleanup,>whereTR: TeamRepository,@@ -280,12 +290,13 @@ wherecontacts_enqueuer: NoOpContactsEnqueuer,event_broker: NoopMacroEventBroker,open_seat_release: NoOpOpenSeatRelease,+ owned_entity_cleanup: UnwiredOwnedEntityCleanup,}}}-impl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR>- TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR>+impl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR, OEC>+ TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR, OEC>whereTR: TeamRepository,CR: CustomerRepository,@@ -298,12 +309,13 @@ whereCNE: ContactsEnqueuer,EB: MacroEventBroker,OSR: OpenSeatRelease,+ OEC: OwnedEntityCleanup,{/// Replaces the contacts enqueuer while preserving every other service dependency.pub fn with_contacts_enqueuer<CNE2>(self,contacts_enqueuer: CNE2,- ) -> TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE2, EB, OSR>+ ) -> TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE2, EB, OSR, OEC>whereCNE2: ContactsEnqueuer,{@@ -319,6 +331,7 @@ wherecontacts_enqueuer,event_broker: self.event_broker,open_seat_release: self.open_seat_release,+ owned_entity_cleanup: self.owned_entity_cleanup,}}@@ -326,7 +339,7 @@ wherepub fn with_event_broker<EB2>(self,event_broker: EB2,- ) -> TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB2, OSR>+ ) -> TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB2, OSR, OEC>whereEB2: MacroEventBroker,{@@ -342,6 +355,7 @@ wherecontacts_enqueuer: self.contacts_enqueuer,event_broker,open_seat_release: self.open_seat_release,+ owned_entity_cleanup: self.owned_entity_cleanup,}}@@ -349,7 +363,7 @@ wherepub fn with_open_seat_release<OSR2>(self,open_seat_release: OSR2,- ) -> TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR2>+ ) -> TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR2, OEC>whereOSR2: OpenSeatRelease,{@@ -365,6 +379,32 @@ wherecontacts_enqueuer: self.contacts_enqueuer,event_broker: self.event_broker,open_seat_release,+ owned_entity_cleanup: self.owned_entity_cleanup,+ }+ }++ /// Replaces the owned-entity cleanup that team deletion purges through+ /// while preserving every other service dependency.+ pub fn with_owned_entity_cleanup<OEC2>(+ self,+ owned_entity_cleanup: OEC2,+ ) -> TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR, OEC2>+ where+ OEC2: OwnedEntityCleanup,+ {+ TeamServiceImpl {+ team_repository: self.team_repository,+ customer_repository: self.customer_repository,+ channel_service: self.channel_service,+ user_roles_and_permissions_service: self.user_roles_and_permissions_service,+ notification_ingress: self.notification_ingress,+ crm_enqueuer: self.crm_enqueuer,+ team_crm_settings_repository: self.team_crm_settings_repository,+ team_analytics: self.team_analytics,+ contacts_enqueuer: self.contacts_enqueuer,+ event_broker: self.event_broker,+ open_seat_release: self.open_seat_release,+ owned_entity_cleanup,}}@@ -413,10 +453,13 @@ where.ok();}- /// Deletes `team_id` on behalf of `actor_user_id`: cancels the team- /// subscription, removes the team and its memberships, publishes- /// `team.deleted`, and strips the team subscriber role from members who- /// are on no other team.+ /// Deletes `team_id` on behalf of `actor_user_id`. Purges everything the+ /// team and its bots own, cancels the team subscription, deletes the team+ /// with its memberships, bots, and grants, publishes `team.deleted`, and+ /// strips the team subscriber role from members who are on no other team.+ ///+ /// A failed purge stops before billing and the team row, so the team stays+ /// intact and deleting it again resumes where this attempt stopped.async fn delete_team_as(&self,team_id: uuid::Uuid,@@ -428,6 +471,10 @@ where.map(|member| member.user_id.clone().into_owned()).collect();+ let cleared = clear_team(&self.owned_entity_cleanup, team_id)+ .await+ .map_err(DeleteTeamError::OwnedEntityCleanup)?;+let subscription_id = self.team_repository.get_team_subscription_id(&team_id)@@ -441,13 +488,19 @@ where}self.team_repository- .delete_team(&team_id)+ .delete_team(&cleared).await.map_err(DeleteTeamError::TeamError)?;self.publish_team_event(&TeamMacroEvent::deleted(TeamDeletedMetadata {team_id,actor_user_id,member_user_ids,+ bot_ids: cleared.bot_ids().to_vec(),+ owned_entities: cleared+ .purged()+ .iter()+ .map(OwnedEntityMetadata::from)+ .collect(),}));// Remove roles for team members@@ -904,8 +957,8 @@ impl GetTeamSubscriptionError {}}-impl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR> TeamMembersService- for TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR>+impl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR, OEC> TeamMembersService+ for TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR, OEC>whereTR: TeamRepository,CR: CustomerRepository,@@ -918,6 +971,7 @@ whereCNE: ContactsEnqueuer,EB: MacroEventBroker + Clone,OSR: OpenSeatRelease,+ OEC: OwnedEntityCleanup,{#[tracing::instrument(skip(self), err)]async fn list_team_members(@@ -934,8 +988,8 @@ where}}-impl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR> TeamService- for TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR>+impl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR, OEC> TeamService+ for TeamServiceImpl<TR, CR, CS, URPS, NI, CE, TCRMS, TA, CNE, EB, OSR, OEC>whereTR: TeamRepository,CR: CustomerRepository,@@ -948,6 +1002,7 @@ whereCNE: ContactsEnqueuer,EB: MacroEventBroker + Clone,OSR: OpenSeatRelease,+ OEC: OwnedEntityCleanup,{#[tracing::instrument(skip(self), err)]async fn create_team(diff --git a/crates/teams/src/domain/team_service/test.rs b/crates/teams/src/domain/team_service/test.rsindex f50ac4f34a..18aba41cad 100644--- a/crates/teams/src/domain/team_service/test.rs+++ b/crates/teams/src/domain/team_service/test.rs@@ -61,9 +61,14 @@ use crate::domain::{TeamMember, TeamPlan, TeamRole, TeamWithMembers, ToggleAutoJoinDomainError,TryJoinTeamByDomainError,},+ owned_entity_cleanup::ClearedTeam,team_repo::TeamRepository,};+mod delete_team;++use delete_team::{Journal, RecordingOwnedEntityCleanup, Step};+// Historical thresholds are regression fixtures, not membership policy.const FORMER_FREE_MEMBER_LIMIT: i32 = 5;const FORMER_IDEA_MEMBER_LIMIT: i32 = 3;@@ -132,6 +137,7 @@ struct MockTeamRepository {team_ids: Vec<uuid::Uuid>,reject_owner: bool,user_teams: Vec<Team>,+ journal: Journal,}impl MockTeamRepository {@@ -210,6 +216,7 @@ impl MockTeamRepository {team_ids: Vec::new(),reject_owner: false,user_teams: Vec::new(),+ journal: Journal::default(),}}@@ -498,9 +505,16 @@ impl TeamRepository for MockTeamRepository {fn delete_team(&self,- team_id: &uuid::Uuid,+ cleared: &ClearedTeam,) -> impl Future<Output = Result<(), TeamError>> + Send {- self.delete_team_calls.lock().unwrap().push(*team_id);+ self.delete_team_calls+ .lock()+ .unwrap()+ .push(cleared.team_id());+ self.journal.record(Step::DeleteTeam {+ team_id: cleared.team_id(),+ bot_ids: cleared.bot_ids().to_vec(),+ });let fail = self.fail_delete_team;async move {if fail {@@ -841,6 +855,7 @@ struct MockCustomerRepository {fail_increment: bool,fail_decrement: bool,no_active_subscription: bool,+ journal: Journal,}impl Default for MockCustomerRepository {@@ -857,6 +872,7 @@ impl Default for MockCustomerRepository {fail_increment: false,fail_decrement: false,no_active_subscription: false,+ journal: Journal::default(),}}}@@ -968,9 +984,11 @@ impl CustomerRepository for MockCustomerRepository {fn cancel_subscription(&self,- _: &stripe::SubscriptionId,+ subscription_id: &stripe::SubscriptionId,) -> impl Future<Output = Result<(), CustomerError>> + Send {- async { unimplemented!() }+ self.journal+ .record(Step::CancelSubscription(subscription_id.to_string()));+ async { Ok(()) }}}@@ -1333,6 +1351,7 @@ struct PublishedTeamEvent {struct RecordingEventBroker {events: Arc<Mutex<Vec<PublishedTeamEvent>>>,fail_scheduling: bool,+ journal: Journal,}impl RecordingEventBroker {@@ -1363,11 +1382,16 @@ impl MacroEventBroker for RecordingEventBroker {));}- self.events.lock().unwrap().push(PublishedTeamEvent {+ let published = PublishedTeamEvent {topic: event.topic(),key: event.key().to_string(),envelope: serde_json::to_value(event.event())?,- });+ };+ let event_type = published.envelope["event_type"]+ .as_str()+ .unwrap_or_default();+ self.journal.record(Step::Published(event_type.to_owned()));+ self.events.lock().unwrap().push(published);Ok(tokio::spawn(async { Ok(()) }))}@@ -3543,6 +3567,7 @@ fn build_service_with_repo_and_broker(NoOpTeamCrmSettingsRepository,).with_event_broker(event_broker)+ .with_owned_entity_cleanup(RecordingOwnedEntityCleanup::default())}fn build_service_with_team(diff --git a/crates/teams/src/domain/team_service/test/delete_team.rs b/crates/teams/src/domain/team_service/test/delete_team.rsnew file mode 100644index 0000000000..61161bb31e--- /dev/null+++ b/crates/teams/src/domain/team_service/test/delete_team.rs@@ -0,0 +1,615 @@+use std::collections::BTreeMap;++use bot_id::BotId;+use shared_entity_registry::RegisteredEntityType;+use uuid::Uuid;++use super::*;+use crate::domain::owned_entity_cleanup::{+ MAX_DISCOVERY_PASSES, OwnedEntityCleanup, OwnedEntityCleanupError, OwnedEntityCleanupUnwired,+ OwnedEntityRef, TeamDeletionOwner,+};++const TEAM_ID: Uuid = Uuid::from_u128(0x900);+const OWNER: &str = "macro|[email protected]";+const SUBSCRIPTION: &str = "sub_team";++/// One effect of a team deletion, in the order the service caused it.+#[derive(Debug, Clone, PartialEq, Eq)]+pub(super) enum Step {+ Purge(Uuid),+ CancelSubscription(String),+ DeleteTeam { team_id: Uuid, bot_ids: Vec<BotId> },+ Published(String),+}++/// The timeline that the cleanup fake and `test.rs`'s repository, customer+/// repository, and broker mocks append to, so ordering across ports is one+/// assertion.+#[derive(Clone, Default)]+pub(super) struct Journal(Arc<Mutex<Vec<Step>>>);++impl Journal {+ pub(super) fn record(&self, step: Step) {+ self.0.lock().unwrap().push(step);+ }++ fn steps(&self) -> Vec<Step> {+ self.0.lock().unwrap().clone()+ }+}++#[derive(Default)]+struct FakeWorld {+ /// Each team's bots, flagged when soft-deleted. The roster returns both,+ /// as the port requires.+ roster: BTreeMap<Uuid, Vec<(BotId, bool)>>,+ registry: BTreeMap<Uuid, OwnedEntityRef>,+ /// Entities a listing returns for an owner that does not own them.+ misfiled: Vec<(TeamDeletionOwner, OwnedEntityRef)>,+ fail_purge_of: Option<Uuid>,+ /// How many more purges register a fresh entity for the purged one's owner.+ respawns_left: usize,+ spawned: u128,+}++/// An in-memory bot roster and entity registry behind the cleanup port.+#[derive(Clone, Default)]+pub(super) struct RecordingOwnedEntityCleanup {+ journal: Journal,+ world: Arc<Mutex<FakeWorld>>,+}++impl RecordingOwnedEntityCleanup {+ fn new(journal: &Journal) -> Self {+ Self {+ journal: journal.clone(),+ world: Arc::default(),+ }+ }++ fn with_bot(self, team_id: Uuid, bot_id: BotId) -> Self {+ self.with_roster_entry(team_id, bot_id, false)+ }++ fn with_soft_deleted_bot(self, team_id: Uuid, bot_id: BotId) -> Self {+ self.with_roster_entry(team_id, bot_id, true)+ }++ fn with_roster_entry(self, team_id: Uuid, bot_id: BotId, soft_deleted: bool) -> Self {+ self.world+ .lock()+ .unwrap()+ .roster+ .entry(team_id)+ .or_default()+ .push((bot_id, soft_deleted));+ self+ }++ fn owning(+ self,+ owner: TeamDeletionOwner,+ entities: impl IntoIterator<Item = (RegisteredEntityType, Uuid)>,+ ) -> Self {+ let mut world = self.world.lock().unwrap();+ for (entity_type, id) in entities {+ world.registry.insert(+ id,+ OwnedEntityRef {+ id,+ entity_type,+ owner,+ },+ );+ }+ drop(world);+ self+ }++ fn misfiling(self, listed_under: TeamDeletionOwner, entity: OwnedEntityRef) -> Self {+ self.world+ .lock()+ .unwrap()+ .misfiled+ .push((listed_under, entity));+ self+ }++ fn failing_purge_of(self, id: Uuid) -> Self {+ self.world.lock().unwrap().fail_purge_of = Some(id);+ self+ }++ fn respawning(self, times: usize) -> Self {+ self.world.lock().unwrap().respawns_left = times;+ self+ }++ fn heal(&self) {+ self.world.lock().unwrap().fail_purge_of = None;+ }++ fn remaining(&self) -> Vec<Uuid> {+ self.world+ .lock()+ .unwrap()+ .registry+ .keys()+ .copied()+ .collect()+ }+}++impl OwnedEntityCleanup for RecordingOwnedEntityCleanup {+ type Err = std::io::Error;++ async fn team_bots(&self, team_id: Uuid) -> Result<Vec<BotId>, Self::Err> {+ let world = self.world.lock().unwrap();+ Ok(world+ .roster+ .get(&team_id)+ .into_iter()+ .flatten()+ .map(|(bot_id, _soft_deleted)| *bot_id)+ .collect())+ }++ async fn owned_by(&self, owner: &TeamDeletionOwner) -> Result<Vec<OwnedEntityRef>, Self::Err> {+ let world = self.world.lock().unwrap();+ let recorded = world+ .registry+ .values()+ .filter(|entity| entity.owner == *owner);+ let misfiled = world+ .misfiled+ .iter()+ .filter(|(listed_under, _)| listed_under == owner)+ .map(|(_, entity)| entity);+ Ok(recorded.chain(misfiled).copied().collect())+ }++ async fn purge(&self, entity: &OwnedEntityRef) -> Result<(), Self::Err> {+ self.journal.record(Step::Purge(entity.id));+ let mut world = self.world.lock().unwrap();+ if world.fail_purge_of == Some(entity.id) {+ return Err(std::io::Error::other("injected purge failure"));+ }+ world.registry.remove(&entity.id);+ if world.respawns_left > 0 {+ world.respawns_left -= 1;+ let id = Uuid::from_u128(0x1000 + world.spawned);+ world.spawned += 1;+ world.registry.insert(id, OwnedEntityRef { id, ..*entity });+ }+ Ok(())+ }+}++fn scheduled_action(id: u128) -> (RegisteredEntityType, Uuid) {+ (RegisteredEntityType::ScheduledAction, Uuid::from_u128(id))+}++fn session(id: u128) -> (RegisteredEntityType, Uuid) {+ (RegisteredEntityType::AgentSession, Uuid::from_u128(id))+}++fn chat(id: u128) -> (RegisteredEntityType, Uuid) {+ (RegisteredEntityType::Chat, Uuid::from_u128(id))+}++fn document(id: u128) -> (RegisteredEntityType, Uuid) {+ (RegisteredEntityType::Document, Uuid::from_u128(id))+}++fn project(id: u128) -> (RegisteredEntityType, Uuid) {+ (RegisteredEntityType::Project, Uuid::from_u128(id))+}++fn bot(id: u128) -> BotId {+ BotId::new_from_uuid(Uuid::from_u128(id))+}++fn purge(id: u128) -> Step {+ Step::Purge(Uuid::from_u128(id))+}++fn cancel_subscription() -> Step {+ Step::CancelSubscription(SUBSCRIPTION.to_owned())+}++fn delete_team(bot_ids: Vec<BotId>) -> Step {+ Step::DeleteTeam {+ team_id: TEAM_ID,+ bot_ids,+ }+}++fn published_team_deleted() -> Step {+ Step::Published("team.deleted".to_owned())+}++fn owner_receipt() -> EntityAccessReceipt<OwnerTeamRole> {+ let owner = MacroUserIdStr::parse_from_str(OWNER).unwrap();+ test_team_receipt::<OwnerTeamRole>(TEAM_ID, &owner)+}++/// A paying team with one owner, whose repository, customer repository, and+/// broker record onto `journal`.+fn team_ports(+ journal: &Journal,+) -> (+ MockTeamRepository,+ MockCustomerRepository,+ RecordingEventBroker,+) {+ let mut repo = MockTeamRepository::new(Vec::new(), "Team", Arc::new(Mutex::new(Vec::new())));+ repo.team_members = vec![make_team_member(TEAM_ID, OWNER, TeamRole::Owner)];+ repo.team_subscription_id = Some(SUBSCRIPTION.parse().unwrap());+ repo.journal = journal.clone();+ let customer_repo = MockCustomerRepository {+ journal: journal.clone(),+ ..MockCustomerRepository::default()+ };+ let broker = RecordingEventBroker {+ journal: journal.clone(),+ ..RecordingEventBroker::default()+ };+ (repo, customer_repo, broker)+}++fn deleting_service(+ journal: &Journal,+ cleanup: &RecordingOwnedEntityCleanup,+) -> (impl TeamService, RecordingEventBroker) {+ let (repo, customer_repo, broker) = team_ports(journal);+ let service = TeamServiceImpl::new(+ repo,+ customer_repo,+ RecordingChannelService::default(),+ MockUserRolesAndPermissionsService::default(),+ Arc::new(MockNotificationIngress::new(HashSet::new())),+ NoOpCrmEnqueuer,+ NoOpTeamCrmSettingsRepository,+ )+ .with_event_broker(broker.clone())+ .with_owned_entity_cleanup(cleanup.clone());+ (service, broker)+}++fn cleanup_failure(result: Result<(), DeleteTeamError>) -> OwnedEntityCleanupError {+ match result {+ Err(DeleteTeamError::OwnedEntityCleanup(error)) => error,+ other => panic!("expected a cleanup failure, got {other:?}"),+ }+}++fn deleted_metadata(broker: &RecordingEventBroker) -> serde_json::Value {+ let events = broker.events();+ assert_eq!(events.len(), 1, "{events:?}");+ assert_eq!(events[0].envelope["event_type"], "team.deleted");+ events[0].envelope["metadata"].clone()+}++#[tokio::test]+async fn delete_team_purges_what_the_team_owns_directly() {+ let journal = Journal::default();+ let cleanup = RecordingOwnedEntityCleanup::new(&journal).owning(+ TeamDeletionOwner::Team(TEAM_ID),+ [chat(1), document(2), project(3)],+ );+ let (service, _broker) = deleting_service(&journal, &cleanup);++ service.delete_team(owner_receipt()).await.unwrap();++ assert_eq!(+ journal.steps(),+ vec![+ purge(1),+ purge(2),+ purge(3),+ cancel_subscription(),+ delete_team(Vec::new()),+ published_team_deleted(),+ ]+ );+}++#[tokio::test]+async fn delete_team_purges_what_the_teams_bots_own() {+ let journal = Journal::default();+ let cleanup = RecordingOwnedEntityCleanup::new(&journal)+ .with_bot(TEAM_ID, bot(0xb01))+ .owning(+ TeamDeletionOwner::Bot(bot(0xb01)),+ [session(1), document(2)],+ );+ let (service, _broker) = deleting_service(&journal, &cleanup);++ service.delete_team(owner_receipt()).await.unwrap();++ assert_eq!(+ journal.steps(),+ vec![+ purge(1),+ purge(2),+ cancel_subscription(),+ delete_team(vec![bot(0xb01)]),+ published_team_deleted(),+ ]+ );+}++#[tokio::test]+async fn delete_team_purges_what_soft_deleted_team_bots_own() {+ let journal = Journal::default();+ let cleanup = RecordingOwnedEntityCleanup::new(&journal)+ .with_bot(TEAM_ID, bot(0xb01))+ .with_soft_deleted_bot(TEAM_ID, bot(0xb02))+ .owning(+ TeamDeletionOwner::Bot(bot(0xb02)),+ [session(1), document(2)],+ );+ let (service, _broker) = deleting_service(&journal, &cleanup);++ service.delete_team(owner_receipt()).await.unwrap();++ assert_eq!(+ journal.steps(),+ vec![+ purge(1),+ purge(2),+ cancel_subscription(),+ delete_team(vec![bot(0xb01), bot(0xb02)]),+ published_team_deleted(),+ ]+ );+}++#[tokio::test]+async fn delete_team_purges_in_rank_order_before_billing_the_row_and_the_event() {+ let journal = Journal::default();+ let cleanup = RecordingOwnedEntityCleanup::new(&journal)+ .with_bot(TEAM_ID, bot(0xb01))+ .owning(+ TeamDeletionOwner::Team(TEAM_ID),+ [project(1), document(6), document(3), scheduled_action(5)],+ )+ .owning(TeamDeletionOwner::Bot(bot(0xb01)), [chat(2), session(4)]);+ let (service, _broker) = deleting_service(&journal, &cleanup);++ service.delete_team(owner_receipt()).await.unwrap();++ assert_eq!(+ journal.steps(),+ vec![+ purge(5),+ purge(4),+ purge(2),+ purge(3),+ purge(6),+ purge(1),+ cancel_subscription(),+ delete_team(vec![bot(0xb01)]),+ published_team_deleted(),+ ]+ );+}++#[tokio::test]+async fn delete_team_purges_content_created_while_it_purges() {+ let journal = Journal::default();+ let cleanup = RecordingOwnedEntityCleanup::new(&journal)+ .owning(TeamDeletionOwner::Team(TEAM_ID), [session(1)])+ .respawning(1);+ let (service, _broker) = deleting_service(&journal, &cleanup);++ service.delete_team(owner_receipt()).await.unwrap();++ assert_eq!(+ journal.steps(),+ vec![+ purge(1),+ purge(0x1000),+ cancel_subscription(),+ delete_team(Vec::new()),+ published_team_deleted(),+ ]+ );+}++#[tokio::test]+async fn delete_team_keeps_the_team_when_a_purge_fails_and_a_retry_resumes() {+ let journal = Journal::default();+ let cleanup = RecordingOwnedEntityCleanup::new(&journal)+ .owning(+ TeamDeletionOwner::Team(TEAM_ID),+ [+ document(1),+ document(2),+ document(3),+ document(4),+ document(5),+ ],+ )+ .failing_purge_of(Uuid::from_u128(3));+ let (service, broker) = deleting_service(&journal, &cleanup);++ let error = cleanup_failure(service.delete_team(owner_receipt()).await);++ assert!(+ matches!(+ error,+ OwnedEntityCleanupError::Purge {+ entity_type: RegisteredEntityType::Document,+ id,+ ..+ } if id == Uuid::from_u128(3)+ ),+ "{error:?}"+ );+ assert_eq!(journal.steps(), vec![purge(1), purge(2), purge(3)]);+ assert_eq!(+ cleanup.remaining(),+ vec![Uuid::from_u128(3), Uuid::from_u128(4), Uuid::from_u128(5)]+ );++ cleanup.heal();+ service.delete_team(owner_receipt()).await.unwrap();++ assert_eq!(+ journal.steps(),+ vec![+ purge(1),+ purge(2),+ purge(3),+ purge(3),+ purge(4),+ purge(5),+ cancel_subscription(),+ delete_team(Vec::new()),+ published_team_deleted(),+ ]+ );+ assert_eq!(+ deleted_metadata(&broker)["owned_entities"],+ serde_json::json!([+ {+ "entity_type": "document",+ "id": "00000000-0000-0000-0000-000000000003",+ "owner": "00000000-0000-0000-0000-000000000900",+ },+ {+ "entity_type": "document",+ "id": "00000000-0000-0000-0000-000000000004",+ "owner": "00000000-0000-0000-0000-000000000900",+ },+ {+ "entity_type": "document",+ "id": "00000000-0000-0000-0000-000000000005",+ "owner": "00000000-0000-0000-0000-000000000900",+ },+ ])+ );+}++#[tokio::test]+async fn delete_team_gives_up_while_content_keeps_reappearing() {+ let journal = Journal::default();+ let cleanup = RecordingOwnedEntityCleanup::new(&journal)+ .owning(TeamDeletionOwner::Team(TEAM_ID), [chat(1)])+ .respawning(usize::MAX);+ let (service, _broker) = deleting_service(&journal, &cleanup);++ let error = cleanup_failure(service.delete_team(owner_receipt()).await);++ assert!(+ matches!(+ error,+ OwnedEntityCleanupError::StillOwned { remaining: 1, passes }+ if passes == MAX_DISCOVERY_PASSES+ ),+ "{error:?}"+ );+ let steps = journal.steps();+ assert_eq!(+ steps.len(),+ usize::from(MAX_DISCOVERY_PASSES - 1),+ "{steps:?}"+ );+ assert!(+ steps.iter().all(|step| matches!(step, Step::Purge(_))),+ "{steps:?}"+ );+}++#[tokio::test]+async fn delete_team_without_a_cleanup_refuses_before_billing_or_the_row() {+ let journal = Journal::default();+ let (repo, customer_repo, broker) = team_ports(&journal);+ let service = TeamServiceImpl::new(+ repo,+ customer_repo,+ RecordingChannelService::default(),+ MockUserRolesAndPermissionsService::default(),+ Arc::new(MockNotificationIngress::new(HashSet::new())),+ NoOpCrmEnqueuer,+ NoOpTeamCrmSettingsRepository,+ )+ .with_event_broker(broker);++ let error = cleanup_failure(service.delete_team(owner_receipt()).await);++ let OwnedEntityCleanupError::ListBots { source } = &error else {+ panic!("expected the bot listing to fail, got {error:?}");+ };+ assert_eq!(+ source.downcast_ref::<OwnedEntityCleanupUnwired>(),+ Some(&OwnedEntityCleanupUnwired)+ );+ assert_eq!(journal.steps(), Vec::new());+}++#[tokio::test]+async fn delete_team_refuses_a_listing_that_returns_another_owners_entity() {+ let journal = Journal::default();+ let cleanup = RecordingOwnedEntityCleanup::new(&journal)+ .owning(TeamDeletionOwner::Team(TEAM_ID), [document(1)])+ .misfiling(+ TeamDeletionOwner::Team(TEAM_ID),+ OwnedEntityRef {+ id: Uuid::from_u128(2),+ entity_type: RegisteredEntityType::Document,+ owner: TeamDeletionOwner::Team(Uuid::from_u128(0x901)),+ },+ );+ let (service, _broker) = deleting_service(&journal, &cleanup);++ let error = cleanup_failure(service.delete_team(owner_receipt()).await);++ assert!(+ matches!(+ error,+ OwnedEntityCleanupError::OwnerMismatch {+ requested: TeamDeletionOwner::Team(requested),+ id,+ } if requested == TEAM_ID && id == Uuid::from_u128(2)+ ),+ "{error:?}"+ );+ assert_eq!(journal.steps(), Vec::new());+ assert_eq!(cleanup.remaining(), vec![Uuid::from_u128(1)]);+}++#[tokio::test]+async fn team_deleted_event_lists_the_bots_and_each_purged_entity_with_its_owner() {+ let journal = Journal::default();+ let cleanup = RecordingOwnedEntityCleanup::new(&journal)+ .with_bot(TEAM_ID, bot(0xb01))+ .owning(TeamDeletionOwner::Team(TEAM_ID), [project(2)])+ .owning(TeamDeletionOwner::Bot(bot(0xb01)), [document(1)]);+ let (service, broker) = deleting_service(&journal, &cleanup);++ service.delete_team(owner_receipt()).await.unwrap();++ let metadata = deleted_metadata(&broker);+ assert_eq!(+ metadata["bot_ids"],+ serde_json::json!(["00000000-0000-0000-0000-000000000b01"])+ );+ assert_eq!(+ metadata["owned_entities"],+ serde_json::json!([+ {+ "entity_type": "document",+ "id": "00000000-0000-0000-0000-000000000001",+ "owner": "bot|00000000-0000-0000-0000-000000000b01",+ },+ {+ "entity_type": "project",+ "id": "00000000-0000-0000-0000-000000000002",+ "owner": "00000000-0000-0000-0000-000000000900",+ },+ ])+ );+}diff --git a/crates/teams/src/inbound/axum_router.rs b/crates/teams/src/inbound/axum_router.rsindex a294363653..1c8c678b66 100644--- a/crates/teams/src/inbound/axum_router.rs+++ b/crates/teams/src/inbound/axum_router.rs@@ -245,6 +245,12 @@ impl IntoResponse for DeleteTeamError {message: "internal server error".into(),}),),+ DeleteTeamError::OwnedEntityCleanup(_) => (+ StatusCode::INTERNAL_SERVER_ERROR,+ Json(ErrorResponse {+ message: "unable to delete the team's content; the team was not deleted".into(),+ }),+ ),}.into_response()}diff --git a/crates/teams/src/inbound/axum_router/test.rs b/crates/teams/src/inbound/axum_router/test.rsindex 2774ed2a7c..7056293a4d 100644--- a/crates/teams/src/inbound/axum_router/test.rs+++ b/crates/teams/src/inbound/axum_router/test.rs@@ -31,6 +31,7 @@ use model_user::UserContext;use roles_and_permissions::domain::model::PermissionId;use rootcause::Report;use serde_json::{Value, json};+use shared_entity_registry::RegisteredEntityType;use tower::ServiceExt;use crate::domain::{@@ -42,6 +43,7 @@ use crate::domain::{TeamInvite, TeamInviteDetails, TeamMember, TeamWithMembers, ToggleAutoJoinDomainError,TryJoinTeamByDomainError,},+ owned_entity_cleanup::OwnedEntityCleanupError,team_repo::TeamService,};@@ -83,6 +85,22 @@ async fn delete_team_customer_error_response_is_obfuscated() {assert_customer_error_is_obfuscated(DeleteTeamError::CustomerError(customer_error())).await;}+#[tokio::test]+async fn delete_team_cleanup_error_says_the_team_survived_without_details() {+ let error = DeleteTeamError::OwnedEntityCleanup(OwnedEntityCleanupError::Purge {+ entity_type: RegisteredEntityType::Document,+ id: uuid::Uuid::from_u128(1),+ source: Box::new(std::io::Error::other("document storage unreachable")),+ });+ let (status, body_text, _) = response_parts(error).await;++ assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);+ assert_eq!(+ body_text,+ r#"{"message":"unable to delete the team's content; the team was not deleted"}"#+ );+}+#[tokio::test]async fn invite_users_to_team_customer_error_response_is_obfuscated() {assert_customer_error_is_obfuscated(InviteUsersToTeamError::CustomerError(customer_error()))diff --git a/crates/teams/src/outbound/team_repo.rs b/crates/teams/src/outbound/team_repo.rsindex 015eeae53b..11aac40715 100644--- a/crates/teams/src/outbound/team_repo.rs+++ b/crates/teams/src/outbound/team_repo.rs@@ -6,14 +6,17 @@ use crate::domain::{TeamInviteDetails, TeamInviteSnapshot, TeamMember, TeamMembers, TeamPlan, TeamRole,TeamWithMembers, ToggleAutoJoinDomainError, is_generic_email_domain, normalize_team_slug,},+ owned_entity_cleanup::ClearedTeam,team_repo::{TeamMembersService, TeamRepository},};+use entity_access_db_utils::{EntityAccessSourceType, delete_source_grants, team_share};use macro_user_id::{cowlike::CowLike,email::{Email, ReadEmailParts},lowercased::Lowercase,user_id::MacroUserIdStr,};+use model_owner::Owner;use models_permissions::share_permission::LinkShare;use sqlx::{PgPool, Row};use std::str::FromStr;@@ -691,9 +694,26 @@ impl TeamRepository for TeamRepositoryImpl {Ok(())}- #[tracing::instrument(skip(self), err)]- async fn delete_team(&self, team_id: &uuid::Uuid) -> Result<(), TeamError> {+ #[tracing::instrument(skip(self, cleared), fields(team_id = %cleared.team_id()), err)]+ async fn delete_team(&self, cleared: &ClearedTeam) -> Result<(), TeamError> {+ let team_id = cleared.team_id();let mut transaction = self.pool.begin().await?;+ team_share::acquire_guard(&mut transaction).await?;++ delete_source_grants(+ &mut *transaction,+ EntityAccessSourceType::Team,+ &Owner::Team(team_id).principal_id(),+ )+ .await?;+ for bot_id in cleared.bot_ids() {+ delete_source_grants(+ &mut *transaction,+ EntityAccessSourceType::Bot,+ &Owner::Bot(*bot_id).principal_id(),+ )+ .await?;+ }sqlx::query!(r#"diff --git a/crates/teams/src/outbound/team_repo/test.rs b/crates/teams/src/outbound/team_repo/test.rsindex 084be85466..fe1b71092e 100644--- a/crates/teams/src/outbound/team_repo/test.rs+++ b/crates/teams/src/outbound/team_repo/test.rs@@ -6,6 +6,8 @@ use sqlx::{Pool, Postgres, Row};use super::*;use crate::domain::model::SeatPlan;+mod delete_team;+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS",fixtures(path = "../../../fixtures", scripts("teams"))@@ -940,7 +942,9 @@ async fn test_delete_team(pool: Pool<Postgres>) -> anyhow::Result<()> {.execute(&pool).await?;- team_repo.delete_team(&team_id).await?;+ team_repo+ .delete_team(&ClearedTeam::assume_cleared(team_id, Vec::new()))+ .await?;let team = sqlx::query!(r#"diff --git a/crates/teams/src/outbound/team_repo/test/delete_team.rs b/crates/teams/src/outbound/team_repo/test/delete_team.rsnew file mode 100644index 0000000000..924023843b--- /dev/null+++ b/crates/teams/src/outbound/team_repo/test/delete_team.rs@@ -0,0 +1,53 @@+use bot_id::BotId;+use uuid::Uuid;++use super::*;++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("teams", "team_grants"))+)]+async fn delete_team_sweeps_the_grants_of_the_team_and_its_bots_only(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let team_repo = TeamRepositoryImpl::new(pool.clone());+ let team_id = Uuid::parse_str("11111111-1111-1111-1111-111111111111")?;+ let bot_ids = vec![+ BotId::parse_uuid_str("60000000-0000-0000-0000-000000000001")?,+ BotId::parse_uuid_str("60000000-0000-0000-0000-000000000002")?,+ ];++ team_repo+ .delete_team(&ClearedTeam::assume_cleared(team_id, bot_ids))+ .await?;++ let remaining = sqlx::query!(+ r#"SELECT source_id, source_type AS "source_type: EntityAccessSourceType"+ FROM entity_access ORDER BY id"#+ )+ .fetch_all(&pool)+ .await?+ .into_iter()+ .map(|row| (row.source_id, row.source_type))+ .collect::<Vec<_>>();++ assert_eq!(+ remaining,+ vec![+ (+ "bot|60000000-0000-0000-0000-000000000003".to_owned(),+ EntityAccessSourceType::Bot+ ),+ (+ "22222222-2222-2222-2222-222222222222".to_owned(),+ EntityAccessSourceType::Team+ ),+ (+ "macro|[email protected]".to_owned(),+ EntityAccessSourceType::User+ ),+ ]+ );++ Ok(())+}diff --git a/services/authentication_service/Cargo.toml b/services/authentication_service/Cargo.tomlindex d4a46e498b..46594e5497 100644--- a/services/authentication_service/Cargo.toml+++ b/services/authentication_service/Cargo.toml@@ -56,6 +56,8 @@ aws-sdk-sesv2 = { workspace = true }aws-sdk-sqs = { workspace = true }axum = { workspace = true, features = ["macros"] }base64 = { workspace = true }+bot_id = { path = "../../crates/bot_id" }+bots = { path = "../../crates/bots", default-features = false, features = ["outbound"] }channels = { path = "../../crates/channels", default-features = false, features = ["outbound",] }@@ -73,6 +75,9 @@ doppleganger = { workspace = true }email_db_client = { path = "../../crates/email_db_client" }email_validator = { path = "../../crates/email_validator" }entity_access = { path = "../../crates/entity_access" }+entity_registry = { path = "../../crates/entity_registry/entity_registry", features = [+ "postgres",+] }foreign_entity = { path = "../../crates/foreign_entity" }favorites = { path = "../../crates/favorites", features = ["postgres"] }fusionauth = { path = "../../crates/fusionauth" }@@ -142,7 +147,7 @@ rate_limit = { path = "../../crates/rate_limit" }redis = { workspace = true }referral = { path = "../../crates/referral" }remote_env_var = { path = "../../crates/remote_env_var" }-reqwest = { workspace = true }+reqwest = { workspace = true, features = ["query"] }roles_and_permissions = { path = "../../crates/roles_and_permissions" }rootcause = { workspace = true }rsa = { version = "0.9.10", features = ["getrandom"] }diff --git a/services/authentication_service/src/api/context.rs b/services/authentication_service/src/api/context.rsindex 02e8852462..67b51b51f3 100644--- a/services/authentication_service/src/api/context.rs+++ b/services/authentication_service/src/api/context.rs@@ -105,6 +105,10 @@ pub(crate) type TeamsServiceType = teams::domain::team_service::TeamServiceImpl<>,AuthenticationEventBroker,AiBillingServiceType,+ crate::outbound::team_owned_entity_cleanup::TeamOwnedEntityCleanupAdapter<+ bots::outbound::pg_bots_repo::PgBotsRepo,+ entity_registry::EntityRegistryServiceImpl<entity_registry::PgEntityRegistryRepository>,+ >,>;pub(crate) type RateLimiter = RateLimitServiceImpl<RedisRateLimitAdapter<redis::Client>>;diff --git a/services/authentication_service/src/outbound.rs b/services/authentication_service/src/outbound.rsindex 34496650e6..aacc43bd2f 100644--- a/services/authentication_service/src/outbound.rs+++ b/services/authentication_service/src/outbound.rs@@ -1,4 +1,5 @@//! Adapters for authentication service use cases.pub mod subscription_checkout;+pub mod team_owned_entity_cleanup;pub mod user_deletion;diff --git a/services/authentication_service/src/outbound/team_owned_entity_cleanup.rs b/services/authentication_service/src/outbound/team_owned_entity_cleanup.rsnew file mode 100644index 0000000000..03900ca5cf--- /dev/null+++ b/services/authentication_service/src/outbound/team_owned_entity_cleanup.rs@@ -0,0 +1,186 @@+//! Team deletion's view of the bot roster, the entity registry, and the+//! services that own each entity kind.++#[cfg(test)]+mod test;++use std::{sync::Arc, time::Duration};++use bot_id::BotId;+use bots::domain::ports::TeamBotRoster;+use document_storage_service_client::DocumentStorageServiceClient;+use entity_registry::{EntityRecord, EntityRegistryService, Owner, RegisteredEntityType};+use macro_authorization::INTERNAL_API_KEY_HEADER;+use rootcause::{Report, compat::ReportAsError, prelude::*};+use teams::domain::owned_entity_cleanup::{OwnedEntityCleanup, OwnedEntityRef, TeamDeletionOwner};+use uuid::Uuid;++/// [`OwnedEntityCleanup`] over the bot roster and the entity registry.+#[derive(Clone)]+pub struct TeamOwnedEntityCleanupAdapter<B, R> {+ bots: B,+ registry: R,+ documents: Arc<DocumentStorageServiceClient>,+ client: reqwest::Client,+ internal_key: String,+ harness_url: String,+ scheduled_action_url: String,+}++impl<B, R> TeamOwnedEntityCleanupAdapter<B, R> {+ /// Construct a bounded HTTP client. Redirects must not turn a failed+ /// internal DELETE into an unrelated successful response.+ ///+ /// `internal_key` is the fleet-wide internal service key that the agent+ /// harness and scheduled-action services validate.+ pub fn new(+ bots: B,+ registry: R,+ documents: Arc<DocumentStorageServiceClient>,+ internal_key: String,+ harness_url: String,+ scheduled_action_url: String,+ ) -> Result<Self, Report> {+ Ok(Self {+ bots,+ registry,+ documents,+ client: reqwest::Client::builder()+ .timeout(Duration::from_secs(300))+ .redirect(reqwest::redirect::Policy::none())+ .build()?,+ internal_key,+ harness_url,+ scheduled_action_url,+ })+ }++ async fn purge_through_route(+ &self,+ base: &str,+ resource: &str,+ id: Uuid,+ owner: &Owner,+ ) -> Result<(), Report<RoutePurgeError>> {+ let response = self+ .client+ .delete(format!(+ "{}/{resource}/internal/{id}",+ base.trim_end_matches('/')+ ))+ .query(&[("owner", owner.principal_id())])+ .header(INTERNAL_API_KEY_HEADER, &self.internal_key)+ .send()+ .await+ .context(RoutePurgeError::Unreachable)?;+ match route_purge_outcome(response.status()) {+ Ok(()) => Ok(()),+ Err(error) => {+ let body = response.text().await.unwrap_or_default();+ Err(report!(error).attach(body))+ }+ }+ }+}++impl<B, R> OwnedEntityCleanup for TeamOwnedEntityCleanupAdapter<B, R>+where+ B: TeamBotRoster + Clone,+ R: EntityRegistryService,+{+ type Err = ReportAsError;++ #[tracing::instrument(skip(self), err)]+ async fn team_bots(&self, team_id: Uuid) -> Result<Vec<BotId>, Self::Err> {+ self.bots+ .team_bot_ids(team_id)+ .await+ .map_err(ReportAsError::from)+ }++ #[tracing::instrument(skip(self), err)]+ async fn owned_by(&self, owner: &TeamDeletionOwner) -> Result<Vec<OwnedEntityRef>, Self::Err> {+ let records = self+ .registry+ .list_all_owned_by(&owner.as_owner())+ .await+ .map_err(Report::into_dynamic)?;+ records+ .into_iter()+ .map(owned_entity_ref)+ .collect::<Result<_, _>>()+ .map_err(ReportAsError::from)+ }++ #[tracing::instrument(skip(self), err)]+ async fn purge(&self, entity: &OwnedEntityRef) -> Result<(), Self::Err> {+ let owner = entity.owner.as_owner();+ match entity.entity_type {+ RegisteredEntityType::Document+ | RegisteredEntityType::Chat+ | RegisteredEntityType::Project => self+ .documents+ .purge_owned_entity(entity.entity_type, entity.id, &owner)+ .await+ .map_err(Report::into_dynamic)?,+ RegisteredEntityType::AgentSession => self+ .purge_through_route(&self.harness_url, "agent-sessions", entity.id, &owner)+ .await+ .map_err(Report::into_dynamic)?,+ RegisteredEntityType::ScheduledAction => self+ .purge_through_route(+ &self.scheduled_action_url,+ "scheduled-actions",+ entity.id,+ &owner,+ )+ .await+ .map_err(Report::into_dynamic)?,+ }+ Ok(())+ }+}++/// A registry row as an entity team deletion may purge. A listing for a team+/// or a bot that returns a user's entity is a bad read, and purging it would+/// delete a person's content, so it is an error.+fn owned_entity_ref(record: EntityRecord) -> Result<OwnedEntityRef, Report> {+ let owner = match record.owner {+ Owner::Team(team_id) => TeamDeletionOwner::Team(team_id),+ Owner::Bot(bot_id) => TeamDeletionOwner::Bot(bot_id),+ Owner::User(_) => {+ return Err(+ report!("the entity registry listed a user's entity for a team or bot")+ .attach(format!("entity: {}", record.id)),+ );+ }+ };+ Ok(OwnedEntityRef {+ id: record.id,+ entity_type: record.entity_type,+ owner,+ })+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]+enum RoutePurgeError {+ #[error("the entity has another owner")]+ OwnedElsewhere,+ #[error("unable to reach the owning service")]+ Unreachable,+ #[error("the owning service answered {status}")]+ UnexpectedResponse { status: u16 },+}++/// Only 204 is a purge, of the entity or of nothing because it was already+/// gone. A 404 means the route is missing on that deployment, so it fails like+/// any other unexpected answer.+fn route_purge_outcome(status: reqwest::StatusCode) -> Result<(), RoutePurgeError> {+ match status {+ reqwest::StatusCode::NO_CONTENT => Ok(()),+ reqwest::StatusCode::CONFLICT => Err(RoutePurgeError::OwnedElsewhere),+ status => Err(RoutePurgeError::UnexpectedResponse {+ status: status.as_u16(),+ }),+ }+}diff --git a/services/authentication_service/src/outbound/team_owned_entity_cleanup/test.rs b/services/authentication_service/src/outbound/team_owned_entity_cleanup/test.rsnew file mode 100644index 0000000000..bd6d51f668--- /dev/null+++ b/services/authentication_service/src/outbound/team_owned_entity_cleanup/test.rs@@ -0,0 +1,62 @@+use chrono::{DateTime, Utc};+use macro_user_id::user_id::MacroUserIdStr;+use reqwest::StatusCode;++use super::*;++fn record(owner: Owner) -> EntityRecord {+ EntityRecord {+ id: Uuid::from_u128(0xc4a7),+ entity_type: RegisteredEntityType::Chat,+ owner,+ created_at: DateTime::<Utc>::UNIX_EPOCH,+ updated_at: DateTime::<Utc>::UNIX_EPOCH,+ deleted_at: Some(DateTime::<Utc>::UNIX_EPOCH),+ }+}++#[test]+fn only_no_content_counts_as_purged() {+ assert_eq!(route_purge_outcome(StatusCode::NO_CONTENT), Ok(()));+ assert_eq!(+ route_purge_outcome(StatusCode::CONFLICT),+ Err(RoutePurgeError::OwnedElsewhere)+ );+ assert_eq!(+ route_purge_outcome(StatusCode::NOT_FOUND),+ Err(RoutePurgeError::UnexpectedResponse { status: 404 })+ );+ assert_eq!(+ route_purge_outcome(StatusCode::OK),+ Err(RoutePurgeError::UnexpectedResponse { status: 200 })+ );+ assert_eq!(+ route_purge_outcome(StatusCode::INTERNAL_SERVER_ERROR),+ Err(RoutePurgeError::UnexpectedResponse { status: 500 })+ );+}++#[test]+fn registry_rows_of_teams_and_bots_become_purgeable_and_a_users_row_is_refused() {+ let bot = BotId::new_from_uuid(Uuid::from_u128(0xb07));+ let team = Uuid::from_u128(0x7ea);++ assert_eq!(+ owned_entity_ref(record(Owner::Bot(bot))).unwrap(),+ OwnedEntityRef {+ id: Uuid::from_u128(0xc4a7),+ entity_type: RegisteredEntityType::Chat,+ owner: TeamDeletionOwner::Bot(bot),+ }+ );+ assert_eq!(+ owned_entity_ref(record(Owner::Team(team))).unwrap(),+ OwnedEntityRef {+ id: Uuid::from_u128(0xc4a7),+ entity_type: RegisteredEntityType::Chat,+ owner: TeamDeletionOwner::Team(team),+ }+ );+ let user = MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap();+ assert!(owned_entity_ref(record(Owner::User(user))).is_err());+}diff --git a/services/authentication_service/src/server.rs b/services/authentication_service/src/server.rsindex 9c068785d8..2c735960b7 100644--- a/services/authentication_service/src/server.rs+++ b/services/authentication_service/src/server.rs@@ -5,6 +5,7 @@ use analytics_client::{AnalyticsClient, AnalyticsClientConfig, GoogleAnalyticsConfig, MetaConfig, PostHogConfig,};use anyhow::{Context, anyhow};+use bots::outbound::pg_bots_repo::PgBotsRepo;use channels::{domain::{service::ChannelServiceImpl,@@ -22,6 +23,7 @@ use connection_gateway_client::ConnectionGatewayClient;use contacts::{domain::service::SqsContactsIngress, outbound::ingress::SqsContactsQueue};use document_storage_service_client::DocumentStorageServiceClient;use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};+use entity_registry::{EntityRegistryServiceImpl, PgEntityRegistryRepository};use foreign_entity::{domain::service::ForeignEntityServiceImpl,outbound::pg_foreign_entity_repo::PgForeignEntityRepo,@@ -89,6 +91,7 @@ use crate::{microsoft_token_cipher::{EnvelopeMicrosoftTokenCipher, KmsDataKeyProvider, MicrosoftTokenCipher,},+ outbound::team_owned_entity_cleanup::TeamOwnedEntityCleanupAdapter,};use std::{sync::Arc, time::Duration};use tokio_util::task::TaskTracker;@@ -529,6 +532,21 @@ pub async fn run() -> anyhow::Result<()> {.with_enforcement(config.enable_ai_usage_enforcement).with_billing(config.enable_ai_usage_billing),);+ let document_storage_service_client = Arc::new(document_storage_service_client);+ // The harness and scheduled-action services validate the fleet-wide+ // internal key, not this service's own inbound key.+ let internal_key = config.service_internal_auth_key.to_string();+ let harness_url = macro_service_urls::AgentHarnessServiceUrl::new()?.to_string();+ let scheduled_action_url = macro_service_urls::ScheduledActionServiceUrl::new()?.to_string();+ let team_owned_entity_cleanup = TeamOwnedEntityCleanupAdapter::new(+ PgBotsRepo::new(db.clone()),+ EntityRegistryServiceImpl::new(PgEntityRegistryRepository::new(db.clone())),+ document_storage_service_client.clone(),+ internal_key.clone(),+ harness_url.clone(),+ scheduled_action_url.clone(),+ )+ .map_err(|error| anyhow::anyhow!("{error:?}"))?;let teams_service_impl = TeamServiceImpl::new_with_analytics(teams_repo_impl.clone(),customer_repo_impl,@@ -541,11 +559,9 @@ pub async fn run() -> anyhow::Result<()> {).with_contacts_enqueuer(contacts_enqueuer).with_event_broker(macro_event_broker)- .with_open_seat_release((*ai_billing_service).clone());+ .with_open_seat_release((*ai_billing_service).clone())+ .with_owned_entity_cleanup(team_owned_entity_cleanup);let teams_service = Arc::new(teams_service_impl);- let document_storage_service_client = Arc::new(document_storage_service_client);- // The harness and scheduled-action services validate the fleet-wide- // internal key, not this service's own inbound key.let user_deletion = Arc::new(crate::outbound::user_deletion::UserDeletionAdapter::new(db.clone(),@@ -553,9 +569,9 @@ pub async fn run() -> anyhow::Result<()> {teams_service.clone(),onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),stripe_client.clone(),- config.service_internal_auth_key.to_string(),- macro_service_urls::AgentHarnessServiceUrl::new()?.to_string(),- macro_service_urls::ScheduledActionServiceUrl::new()?.to_string(),+ internal_key,+ harness_url,+ scheduled_action_url,).map_err(|error| anyhow::anyhow!("{error:?}"))?,);diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.tomlindex 5e8d30ef49..f2c434ae85 100644--- a/services/document_storage_service/Cargo.toml+++ b/services/document_storage_service/Cargo.toml@@ -282,6 +282,7 @@ zip = { workspace = true }[dev-dependencies]cool_asserts = "2"frecency = { path = "../../crates/frecency", features = ["mock"] }+macro_auth = { path = "../../crates/macro_auth", features = ["testing"] }ordered-float = { workspace = true }[package.metadata.cargo-machete]diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rsindex 823be1eaa8..9f03a83e66 100644--- a/services/document_storage_service/src/api/context.rs+++ b/services/document_storage_service/src/api/context.rs@@ -46,8 +46,12 @@ use connection::{};use connection_gateway_client::client::ConnectionGatewayClient;use documents_hex::domain::ports::{TaskPropertiesPort, task_property_edit_receipt};+use documents_hex::domain::purge::DocumentPurger;use documents_hex::domain::service::DocumentServiceImpl;use documents_hex::inbound::axum_router::DocumentRouterState;+use documents_hex::outbound::document_purge::{+ LegacyDocumentPurgeRepository, RedisDocxPartReferences, SqsDocumentPurgeQueue,+};use documents_hex::outbound::pg_document_repo::PgDocumentRepo;use documents_hex::outbound::s3_upload_url::S3UploadUrlAdapter;use dynamodb_client::DynamodbClient;@@ -348,6 +352,14 @@ pub(crate) type DocumentService = DocumentServiceImpl<sync_service_client::SyncServiceClient,>;+/// Permanent document purge wired into DSS.+pub(crate) type DssDocumentPurger = DocumentPurger<+ LegacyDocumentPurgeRepository,+ SqsDocumentPurgeQueue,+ RedisDocxPartReferences,+ DssEventBroker,+>;+/// Type alias for the authorization service.pub(crate) type AuthorizationService = MacroAuthorizationServiceImpl<MacroAuthJwtValidator>;@@ -485,6 +497,20 @@ pub(crate) type DssChatMutationService = chat::domain::service::ChatServiceImpl<EntityAccessManagementService,>;+/// Chat service for owner-checked permanent deletes. Unlike+/// [`DssChatMutationService`] it publishes chat events, which is how search+/// and activity learn a chat is gone.+pub(crate) type DssChatPurgeService = chat::domain::service::ChatServiceImpl<+ chat::outbound::postgres::PgChatRepo<PgBotsRepo>,+ (),+ EntityAccessManagementService,+ DssEventBroker,+>;++/// State of `DELETE /internal/owned/{entity_type}/{entity_id}`.+pub(crate) type DssOwnedPurgeState =+ super::internal::OwnedPurgeState<DssDocumentPurger, DssChatPurgeService, ProjectService>;+/// Concrete unified entity mutation service wired into GraphQL.pub(crate) type DssEntityMutationService =crate::service::entity_mutation::DssEntityMutationService<@@ -798,6 +824,7 @@ pub(crate) struct ApiContext {pub foreign_entity_state: DssForeignEntityState,pub macro_event_broker: DssEventBroker,pub sqs_client: Arc<sqs_client::SQS>,+ pub document_purger: Arc<DssDocumentPurger>,pub contacts_ingress: Arc<SqsContactsIngress<SqsContactsQueue>>,pub notification_ingress_service: Arc<NotificationIngressType>,pub conn_gateway_client: Arc<ConnectionGatewayClient>,@@ -835,6 +862,7 @@ pub(crate) struct ApiContext {pub cal_webhook_state: DssCalWebhookState,pub entity_access_management_service: EntityAccessManagementService,pub crm_state: DssCrmState,+ pub owned_purge_state: DssOwnedPurgeState,}env_var! {diff --git a/services/document_storage_service/src/api/documents/delete_document.rs b/services/document_storage_service/src/api/documents/delete_document.rsindex b4ccb3bb40..af4ac3abbc 100644--- a/services/document_storage_service/src/api/documents/delete_document.rs+++ b/services/document_storage_service/src/api/documents/delete_document.rs@@ -1,16 +1,12 @@use crate::api::context::ApiContext;use crate::api::context::{AuthorizationService, EntityAccessService};-use crate::api::util::count_occurrences;-use crate::service::document_event_publisher::publish_document_purged_event;use axum::Json;use axum::extract::State;use axum::response::Response;-use axum::{Extension, extract::Path, http::StatusCode, response::IntoResponse};+use axum::{extract::Path, http::StatusCode, response::IntoResponse};+use documents_hex::domain::purge::DocumentPurgeService as _;use entity_access::inbound::axum_extractors::DocumentAccessExtractor;-#[allow(unused_imports)]-use futures::stream::TryStreamExt;use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};-use model::document::DocumentBasic;use model::response::{ErrorResponse, GenericErrorResponse, GenericResponse, GenericSuccessResponse, SuccessResponse,};@@ -19,7 +15,7 @@ use serde::Deserialize;#[derive(Deserialize)]pub struct Params {- pub document_id: String,+ pub document_id: uuid::Uuid,}/// Permanently deletes a document.@@ -43,105 +39,25 @@ pub async fn permanently_delete_document_handler(_access: DocumentAccessExtractor<OwnerAccessLevel, EntityAccessService, AuthorizationService>,State(state): State<ApiContext>,user: MacroAuthorizationExtractor<AuthorizationService, UserOrInternal>,- document_context: Extension<DocumentBasic>,Path(Params { document_id }): Path<Params>,) -> Result<Response, Response> {tracing::info!("permanently_delete_document");- // Decrement sha counts for docx files- if let Some(file_type) = document_context.file_type.as_deref()- && file_type == "docx"- {- let bom_parts = macro_db_client::document::get_bom_parts(&state.db, &document_id)- .await- .map_err(|e| {- tracing::error!(error=?e, "unable to get bom parts");- (- StatusCode::INTERNAL_SERVER_ERROR,- Json(ErrorResponse {- message: "unable to get bom parts".into(),- }),- )- .into_response()- })?;-- // Transform bom parts into Vec<(sha, count)>- let sha_counts = count_occurrences(- bom_parts- .iter()- .map(|bp| bp.sha.clone())- .collect::<Vec<String>>(),- );-- tracing::trace!("decrementing sha ref count");- state- .redis_client- .decrement_counts(&sha_counts)- .await- .map_err(|e| {- tracing::error!(error=?e, "unable to decrement sha ref counts");- (- StatusCode::INTERNAL_SERVER_ERROR,- Json(ErrorResponse {- message: "unable to decrement sha ref counts".into(),- }),- )- .into_response()- })?;- }-- // Delete document info from db- macro_db_client::document::delete_document(&state.db, &document_id)- .await- .map_err(|e| {- tracing::error!(error=?e, "unable to delete document");- (- StatusCode::INTERNAL_SERVER_ERROR,- Json(ErrorResponse {- message: "unable to delete document".into(),- }),- )- .into_response()- })?;-- // Delete entity mentions where this doc is the source- if let Err(e) = comms_db_client::entity_mentions::delete_entity_mentions_by_source(- &state.db,- vec![document_id.clone()],- )- .await- {- tracing::error!(error=?e, "unable to delete entity mentions");- }-- // Queue document for deletion- let owner = document_context.owner.principal_id();state- .sqs_client- .enqueue_document_delete(&owner, &document_id)+ .document_purger+ .purge(document_id).await.map_err(|e| {- tracing::error!(error=?e, "unable to enqueue document delete");+ tracing::error!(error=?e, "unable to permanently delete document");(StatusCode::INTERNAL_SERVER_ERROR,Json(ErrorResponse {- message: "unable to enqueue document delete".into(),+ message: "unable to permanently delete document".into(),}),).into_response()})?;- publish_document_purged_event(&state.macro_event_broker, &document_id).map_err(|e| {- tracing::error!(error=?e, "unable to publish document purged event");- (- StatusCode::INTERNAL_SERVER_ERROR,- Json(ErrorResponse {- message: "unable to publish document purged event".into(),- }),- )- .into_response()- })?;-let response_data = GenericSuccessResponse { success: true };Ok(GenericResponse::builder()diff --git a/services/document_storage_service/src/api/internal.rs b/services/document_storage_service/src/api/internal.rsindex 64cda00881..9e0fbed51f 100644--- a/services/document_storage_service/src/api/internal.rs+++ b/services/document_storage_service/src/api/internal.rs@@ -1,3 +1,7 @@+mod purge_owned_entity;++pub(crate) use purge_owned_entity::OwnedPurgeState;+use super::{context::ApiContext,documents::{export_document, get_document_version},@@ -14,7 +18,8 @@ use super::{user::populate_items,};use crate::api::context::{- AuthorizationService, DocumentService, EntityAccessService, ProjectService,+ AuthorizationService, DocumentService, DssChatPurgeService, DssDocumentPurger,+ EntityAccessService, ProjectService,};use crate::api::items::get_item_ids;use crate::api::threads::get_thread_access_level;@@ -204,6 +209,16 @@ pub fn router(state: ApiContext) -> Router<ApiContext> {).route("/item_ids", get(get_item_ids::get_item_ids_handler)).route("/validate_item_ids", post(validate_item_ids::handler))+ .route(+ purge_owned_entity::ROUTE,+ delete(+ purge_owned_entity::handler::<+ DssDocumentPurger,+ DssChatPurgeService,+ ProjectService,+ >,+ ),+ ).route("/health", get(health_handler))}diff --git a/services/document_storage_service/src/api/internal/purge_owned_entity.rs b/services/document_storage_service/src/api/internal/purge_owned_entity.rsnew file mode 100644index 0000000000..eada601cac--- /dev/null+++ b/services/document_storage_service/src/api/internal/purge_owned_entity.rs@@ -0,0 +1,93 @@+//! The owner-checked permanent delete that team deletion calls once per+//! entity it removes.+//!+//! DSS owns documents, chats, and projects. Agent sessions and scheduled+//! actions belong to the agent harness and scheduled-action services, so this+//! route refuses them instead of deleting rows those services own.++#[cfg(test)]+mod test;++use std::sync::Arc;++use axum::{+ extract::{Path, Query, State},+ http::StatusCode,+};+use entity_registry::{OwnedPurgeOutcome, Owner, PurgeOwnedEntity, RegisteredEntityType};+use macro_authorization::{InternalOnly, MacroAuthorizationExtractor};+use uuid::Uuid;++use crate::api::context::AuthorizationService;++/// Path of the route under `/internal`.+pub(super) const ROUTE: &str = "/owned/{entity_type}/{entity_id}";++/// The owning services the route purges through, one per kind DSS owns.+pub(crate) struct OwnedPurgeState<D, C, P> {+ documents: Arc<D>,+ chats: Arc<C>,+ projects: Arc<P>,+}++impl<D, C, P> OwnedPurgeState<D, C, P> {+ pub(crate) fn new(documents: Arc<D>, chats: Arc<C>, projects: Arc<P>) -> Self {+ Self {+ documents,+ chats,+ projects,+ }+ }+}++impl<D, C, P> Clone for OwnedPurgeState<D, C, P> {+ fn clone(&self) -> Self {+ Self {+ documents: Arc::clone(&self.documents),+ chats: Arc::clone(&self.chats),+ projects: Arc::clone(&self.projects),+ }+ }+}++#[derive(Debug, serde::Deserialize)]+pub(super) struct OwnerQuery {+ owner: Owner,+}++/// `DELETE /internal/owned/{entity_type}/{entity_id}?owner=<principal>`.+///+/// 204 once the entity is gone, including when it already was. 409 when it+/// exists under another owner, and nothing was deleted. 400 for a kind DSS+/// does not own. 500 when the owning service failed. The purge converges, so+/// the caller retries.+#[tracing::instrument(skip(state, _auth, owner), fields(owner.kind = ?owner.owner_type()))]+pub(super) async fn handler<D, C, P>(+ State(state): State<OwnedPurgeState<D, C, P>>,+ _auth: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,+ Path((entity_type, entity_id)): Path<(RegisteredEntityType, Uuid)>,+ Query(OwnerQuery { owner }): Query<OwnerQuery>,+) -> StatusCode+where+ D: PurgeOwnedEntity + 'static,+ C: PurgeOwnedEntity + 'static,+ P: PurgeOwnedEntity + 'static,+{+ let purged = match entity_type {+ RegisteredEntityType::Document => state.documents.purge_owned(entity_id, &owner).await,+ RegisteredEntityType::Chat => state.chats.purge_owned(entity_id, &owner).await,+ RegisteredEntityType::Project => state.projects.purge_owned(entity_id, &owner).await,+ RegisteredEntityType::AgentSession | RegisteredEntityType::ScheduledAction => {+ tracing::warn!("another service owns this kind");+ return StatusCode::BAD_REQUEST;+ }+ };+ match purged {+ Ok(OwnedPurgeOutcome::Purged) => StatusCode::NO_CONTENT,+ Ok(OwnedPurgeOutcome::OwnedElsewhere) => StatusCode::CONFLICT,+ Err(error) => {+ tracing::error!(error = ?error, "unable to purge the owned entity");+ StatusCode::INTERNAL_SERVER_ERROR+ }+ }+}diff --git a/services/document_storage_service/src/api/internal/purge_owned_entity/test.rs b/services/document_storage_service/src/api/internal/purge_owned_entity/test.rsnew file mode 100644index 0000000000..2921366e49--- /dev/null+++ b/services/document_storage_service/src/api/internal/purge_owned_entity/test.rs@@ -0,0 +1,292 @@+use std::{+ sync::{Arc, Mutex},+ time::{SystemTime, UNIX_EPOCH},+};++use axum::{+ Router,+ body::Body,+ extract::FromRef,+ http::{Request, StatusCode, header::AUTHORIZATION},+ routing::delete,+};+use bot_id::BotId;+use entity_registry::{OwnedPurgeOutcome, Owner, PurgeOwnedEntity};+use jsonwebtoken::{Algorithm, EncodingKey, Header};+use macro_auth::middleware::decode_jwt::{JwtValidationArgs, MacroAccessToken};+use macro_authorization::{+ InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,+ MacroAuthorizationState, NoBotAuthorizer, NoUserApiKeyAuthorizer,+};+use rootcause::Report;+use tower::ServiceExt;+use uuid::Uuid;++use super::{OwnedPurgeState, ROUTE, handler};+use crate::api::context::AuthorizationService;++const INTERNAL_KEY: &str = "test-internal-key";+/// The header `document_storage_service_client` sends its key in.+const DSS_CLIENT_KEY_HEADER: &str = "x-document-storage-service-auth-key";+const ENTITY: Uuid = Uuid::from_u128(0xE1);+const TEAM: Uuid = Uuid::from_u128(0x7EA);+const BOT: Uuid = Uuid::from_u128(0xB07);++/// An owning service that records each purge and answers the same way.+struct FakePurger {+ answer: Option<OwnedPurgeOutcome>,+ calls: Mutex<Vec<(Uuid, Owner)>>,+}++impl FakePurger {+ fn answering(answer: OwnedPurgeOutcome) -> Arc<Self> {+ Arc::new(Self {+ answer: Some(answer),+ calls: Mutex::default(),+ })+ }++ fn failing() -> Arc<Self> {+ Arc::new(Self {+ answer: None,+ calls: Mutex::default(),+ })+ }++ fn calls(&self) -> Vec<(Uuid, Owner)> {+ self.calls.lock().unwrap().clone()+ }+}++impl PurgeOwnedEntity for FakePurger {+ async fn purge_owned(+ &self,+ entity_id: Uuid,+ expected_owner: &Owner,+ ) -> Result<OwnedPurgeOutcome, Report> {+ self.calls+ .lock()+ .unwrap()+ .push((entity_id, expected_owner.clone()));+ self.answer+ .ok_or_else(|| rootcause::report!("injected purge failure"))+ }+}++struct Services {+ documents: Arc<FakePurger>,+ chats: Arc<FakePurger>,+ projects: Arc<FakePurger>,+}++impl Services {+ fn all_answering(answer: OwnedPurgeOutcome) -> Self {+ Self {+ documents: FakePurger::answering(answer),+ chats: FakePurger::answering(answer),+ projects: FakePurger::answering(answer),+ }+ }++ fn calls(&self) -> [Vec<(Uuid, Owner)>; 3] {+ [+ self.documents.calls(),+ self.chats.calls(),+ self.projects.calls(),+ ]+ }++ async fn send(&self, request: Request<Body>) -> StatusCode {+ let state = TestState {+ purge: OwnedPurgeState::new(+ self.documents.clone(),+ self.chats.clone(),+ self.projects.clone(),+ ),+ authorization: MacroAuthorizationState::new(Arc::new(+ MacroAuthorizationServiceImpl::new(+ MacroAuthJwtValidator::new(JwtValidationArgs::new_testing()),+ InternalAuthConfig {+ api_key: INTERNAL_KEY.into(),+ default_user_id: None,+ },+ NoBotAuthorizer,+ NoUserApiKeyAuthorizer,+ ),+ )),+ };+ Router::new()+ .route(ROUTE, delete(handler::<FakePurger, FakePurger, FakePurger>))+ .with_state(state)+ .oneshot(request)+ .await+ .unwrap()+ .status()+ }+}++#[derive(Clone, FromRef)]+struct TestState {+ purge: OwnedPurgeState<FakePurger, FakePurger, FakePurger>,+ authorization: MacroAuthorizationState<AuthorizationService>,+}++fn team_owner() -> Owner {+ Owner::Team(TEAM)+}++fn purge(uri: &str) -> Request<Body> {+ Request::delete(uri)+ .header(DSS_CLIENT_KEY_HEADER, INTERNAL_KEY)+ .body(Body::empty())+ .unwrap()+}++fn user_access_token() -> String {+ let expires_in_an_hour = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap()+ .as_secs() as usize+ + 3_600;+ let claims = MacroAccessToken {+ aud: String::new(),+ exp: expires_in_an_hour,+ tid: "test-tenant".into(),+ iss: String::new(),+ email: "[email protected]".into(),+ fusion_user_id: "test-fusion-user".into(),+ macro_user_id: "macro|[email protected]".into(),+ macro_organization_id: None,+ root_macro_id: None,+ };+ let mut header = Header::new(Algorithm::HS256);+ header.kid = Some("test-access-token".into());+ jsonwebtoken::encode(&header, &claims, &EncodingKey::from_secret(b"")).unwrap()+}++#[tokio::test]+async fn each_kind_dss_owns_is_purged_by_its_owning_service() {+ let purged = vec![(ENTITY, team_owner())];+ let cases = [+ ("document", [purged.clone(), vec![], vec![]]),+ ("chat", [vec![], purged.clone(), vec![]]),+ ("project", [vec![], vec![], purged.clone()]),+ ];+ for (kind, expected_calls) in cases {+ let services = Services::all_answering(OwnedPurgeOutcome::Purged);++ let status = services+ .send(purge(&format!("/owned/{kind}/{ENTITY}?owner={TEAM}")))+ .await;++ assert_eq!(status, StatusCode::NO_CONTENT, "{kind}");+ assert_eq!(services.calls(), expected_calls, "{kind}");+ }+}++#[tokio::test]+async fn the_owner_query_reaches_the_owning_service_as_a_principal() {+ let cases = [+ (TEAM.to_string(), Owner::Team(TEAM)),+ (+ format!("bot%7C{BOT}"),+ Owner::Bot(BotId::new_from_uuid(BOT)),+ ),+ ];+ for (query, owner) in cases {+ let services = Services::all_answering(OwnedPurgeOutcome::Purged);++ services+ .send(purge(&format!("/owned/document/{ENTITY}?owner={query}")))+ .await;++ assert_eq!(services.documents.calls(), vec![(ENTITY, owner)], "{query}");+ }+}++#[tokio::test]+async fn an_entity_under_another_owner_is_a_conflict() {+ let services = Services::all_answering(OwnedPurgeOutcome::OwnedElsewhere);++ let status = services+ .send(purge(&format!("/owned/chat/{ENTITY}?owner={TEAM}")))+ .await;++ assert_eq!(status, StatusCode::CONFLICT);+}++#[tokio::test]+async fn a_failed_purge_is_a_server_error() {+ let services = Services {+ projects: FakePurger::failing(),+ ..Services::all_answering(OwnedPurgeOutcome::Purged)+ };++ let status = services+ .send(purge(&format!("/owned/project/{ENTITY}?owner={TEAM}")))+ .await;++ assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR);+}++#[tokio::test]+async fn kinds_other_services_own_are_refused_without_a_purge() {+ for kind in ["agent_session", "scheduled_action"] {+ let services = Services::all_answering(OwnedPurgeOutcome::Purged);++ let status = services+ .send(purge(&format!("/owned/{kind}/{ENTITY}?owner={TEAM}")))+ .await;++ assert_eq!(status, StatusCode::BAD_REQUEST, "{kind}");+ assert_eq!(services.calls(), [vec![], vec![], vec![]], "{kind}");+ }+}++#[tokio::test]+async fn a_malformed_request_is_refused_without_a_purge() {+ for uri in [+ format!("/owned/channel/{ENTITY}?owner={TEAM}"),+ format!("/owned/bogus/{ENTITY}?owner={TEAM}"),+ format!("/owned/document/not-a-uuid?owner={TEAM}"),+ format!("/owned/document/{ENTITY}"),+ format!("/owned/document/{ENTITY}?owner=not-a-principal"),+ ] {+ let services = Services::all_answering(OwnedPurgeOutcome::Purged);++ let status = services.send(purge(&uri)).await;++ assert_eq!(status, StatusCode::BAD_REQUEST, "{uri}");+ assert_eq!(services.calls(), [vec![], vec![], vec![]], "{uri}");+ }+}++#[tokio::test]+async fn a_signed_in_user_is_forbidden_to_purge() {+ let services = Services::all_answering(OwnedPurgeOutcome::Purged);+ let request = Request::delete(format!("/owned/document/{ENTITY}?owner={TEAM}"))+ .header(AUTHORIZATION, format!("Bearer {}", user_access_token()))+ .body(Body::empty())+ .unwrap();++ let status = services.send(request).await;++ assert_eq!(status, StatusCode::FORBIDDEN);+ assert_eq!(services.calls(), [vec![], vec![], vec![]]);+}++#[tokio::test]+async fn a_request_without_the_internal_key_is_refused_without_a_purge() {+ for key in [None, Some("wrong-key")] {+ let services = Services::all_answering(OwnedPurgeOutcome::Purged);+ let mut request = Request::delete(format!("/owned/document/{ENTITY}?owner={TEAM}"));+ if let Some(key) = key {+ request = request.header(DSS_CLIENT_KEY_HEADER, key);+ }++ let status = services.send(request.body(Body::empty()).unwrap()).await;++ assert!(status.is_client_error(), "{key:?}: {status}");+ assert_eq!(services.calls(), [vec![], vec![], vec![]], "{key:?}");+ }+}diff --git a/services/document_storage_service/src/server.rs b/services/document_storage_service/src/server.rsindex b96e7d6efd..3cfa3a66a9 100644--- a/services/document_storage_service/src/server.rs+++ b/services/document_storage_service/src/server.rs@@ -5,7 +5,8 @@ use crate::{api::{self, MACRO_INTERNAL_USER_ID,context::{- ApiContext, AuthorizationService, DocumentStorageServiceAuthKey, TaskPropertiesAdapter,+ ApiContext, AuthorizationService, DocumentStorageServiceAuthKey, DssOwnedPurgeState,+ TaskPropertiesAdapter,},},config::{@@ -1805,6 +1806,30 @@ pub async fn run() -> anyhow::Result<()> {let redis_sha_client = Arc::new(Redis::new(redis_client));+ let document_purger = Arc::new(documents_hex::domain::purge::DocumentPurger::new(+ documents_hex::outbound::document_purge::LegacyDocumentPurgeRepository::new(db.clone()),+ documents_hex::outbound::document_purge::SqsDocumentPurgeQueue::new(sqs_client.clone()),+ documents_hex::outbound::document_purge::RedisDocxPartReferences::new(+ redis_sha_client.clone(),+ ),+ macro_event_broker.clone(),+ ));++ let owned_purge_state = DssOwnedPurgeState::new(+ document_purger.clone(),+ Arc::new(+ chat::domain::service::ChatServiceImpl::new_without_tools(+ chat::outbound::postgres::PgChatRepo::new(+ db.clone(),+ owned_entity_registrar.clone(),+ ),+ entity_access_management_service.clone(),+ )+ .with_event_broker(macro_event_broker.clone()),+ ),+ project_service.clone(),+ );+let graphql_entity_mutation_service =Arc::new(service::entity_mutation::DssEntityMutationService::new(document_service.clone(),@@ -1949,6 +1974,7 @@ pub async fn run() -> anyhow::Result<()> {dynamo_db,macro_event_broker: macro_event_broker.clone(),sqs_client: sqs_client.clone(),+ document_purger,notification_ingress_service: notification_ingress_service.clone(),conn_gateway_client: conn_gateway_client.clone(),sync_service_client: sync_service_client.clone(),@@ -2018,6 +2044,7 @@ pub async fn run() -> anyhow::Result<()> {entity_access_service: entity_access_service.clone(),authorization_state: authorization_state.clone(),},+ owned_purge_state,};#[cfg(feature = "delete_document_worker")]diff --git a/services/scheduled_action/src/domain/ports.rs b/services/scheduled_action/src/domain/ports.rsindex 5ff913f849..9339ee1a8f 100644--- a/services/scheduled_action/src/domain/ports.rs+++ b/services/scheduled_action/src/domain/ports.rs@@ -10,9 +10,10 @@ use chrono::{DateTime, Utc};use entity_access::domain::models::{EditAccessLevel, EntityAccessReceipt, OwnerAccessLevel, ViewAccessLevel,};+use entity_registry::OwnedPurgeOutcome;use macro_user_id::user_id::MacroUserIdStr;use macro_uuid::Uuid;-use model_owner::CreationPrincipal;+use model_owner::{CreationPrincipal, Owner};use rootcause::Report;use tokio::sync::mpsc::{Receiver, Sender};@@ -132,6 +133,15 @@ pub trait ScheduledActionService: Send + Sync + 'static {user_id: MacroUserIdStr<'static>,) -> impl Future<Output = Result<()>> + Send;+ /// Delete one action while `expected_owner` still owns it, including a+ /// disabled or claimed one. Internal owner removal only. A missing action+ /// is already purged; one under another owner is untouched.+ fn purge_owned_action(+ &self,+ id: Uuid,+ expected_owner: &Owner,+ ) -> impl Future<Output = Result<OwnedPurgeOutcome>> + Send;+/// Records `principal.owner()`. A non-user owner is `OwnerNotUserError`/// before validation and before any write. `BotForUser` records the user.fn create_action(diff --git a/services/scheduled_action/src/domain/service.rs b/services/scheduled_action/src/domain/service.rsindex d3c93b8c1e..fbe3afb5d6 100644--- a/services/scheduled_action/src/domain/service.rs+++ b/services/scheduled_action/src/domain/service.rs@@ -9,9 +9,10 @@ use entity_access::domain::{},ports::ScheduledActionGrants,};+use entity_registry::OwnedPurgeOutcome;use macro_user_id::user_id::MacroUserIdStr;use macro_uuid::Uuid;-use model_owner::CreationPrincipal;+use model_owner::{CreationPrincipal, Owner};use tokio::sync::mpsc::Sender;use super::event_runs::ConfigurationRevision;@@ -109,6 +110,16 @@ impl<Rpo: ScheduledActionRepo, Exe, Grants, Targets>}}+ /// Delete `action` and hand its removal to the dispatcher. The permit is+ /// reserved first so a stopped dispatcher fails while the row still exists+ /// for a retry to find.+ async fn delete_and_dispatch(&self, id: &Uuid, action: ScheduledAction) -> Result<()> {+ let permit = self.dispatcher_tx.reserve().await?;+ self.repo.delete_action(id).await?;+ permit.send(DispatchEvent::Delete(action));+ Ok(())+ }+fn check_event_management(&self, trigger: &ActionTrigger) -> Result<()> {if trigger.event_filters().is_some() && !self.event_management_enabled {return Err(ActionPolicyError::EventManagementDisabled.into());@@ -256,15 +267,31 @@ wherelet Some(id) = action.id else {bail!("cannot delete action without id");};- // Reserve before deleting: a stopped dispatcher must not leave us- // reporting failure after losing the row needed to retry its event.- let permit = self.dispatcher_tx.reserve().await?;- self.repo.delete_action(&id).await?;- permit.send(DispatchEvent::Delete(action));+ self.delete_and_dispatch(&id, action).await?;}Ok(())}+ #[tracing::instrument(+ err,+ skip(self, expected_owner),+ fields(%id, owner.kind = ?expected_owner.owner_type())+ )]+ async fn purge_owned_action(+ &self,+ id: Uuid,+ expected_owner: &Owner,+ ) -> Result<OwnedPurgeOutcome> {+ let Some(action) = self.repo.get_action(&id).await? else {+ return Ok(OwnedPurgeOutcome::Purged);+ };+ if action.owner != *expected_owner {+ return Ok(OwnedPurgeOutcome::OwnedElsewhere);+ }+ self.delete_and_dispatch(&id, action).await?;+ Ok(OwnedPurgeOutcome::Purged)+ }+async fn create_action(&self,principal: &CreationPrincipal,diff --git a/services/scheduled_action/src/domain/service/test.rs b/services/scheduled_action/src/domain/service/test.rsindex bac17b527c..4b4c77ff76 100644--- a/services/scheduled_action/src/domain/service/test.rs+++ b/services/scheduled_action/src/domain/service/test.rs@@ -1,4 +1,5 @@mod admission;+mod owned_purge;mod user_cleanup;use super::*;@@ -200,6 +201,28 @@ pub(crate) fn set_stored_owner(service: &TestService, id: Uuid, owner: Owner) {action.owner = owner;}+pub(crate) fn stored_owner(service: &TestService, id: Uuid) -> Option<Owner> {+ let actions = service.repo.actions.lock().unwrap();+ actions+ .iter()+ .find(|action| action.id == Some(id))+ .map(|action| action.owner.clone())+}++/// The same store behind a dispatcher that has stopped.+pub(crate) fn with_stopped_dispatcher(service: &TestService) -> Arc<TestService> {+ let (tx, _) = tokio::sync::mpsc::channel(1);+ Arc::new(+ TestService::new(+ Arc::clone(&service.repo),+ Arc::clone(&service.executor),+ tx,+ Arc::clone(&service.grants),+ )+ .with_event_management_enabled(service.event_management_enabled),+ )+}+pub(crate) fn grant_to(service: &TestService, user_id: &str, id: Uuid) {service.grantsdiff --git a/services/scheduled_action/src/domain/service/test/owned_purge.rs b/services/scheduled_action/src/domain/service/test/owned_purge.rsnew file mode 100644index 0000000000..82e4b5d693--- /dev/null+++ b/services/scheduled_action/src/domain/service/test/owned_purge.rs@@ -0,0 +1,79 @@+use super::user_cleanup::action;+use super::*;+use tokio::sync::mpsc::Receiver;++const TEAM: Uuid = Uuid::from_u128(0x7EA3);+const OTHER_TEAM: Uuid = Uuid::from_u128(0x7EA5);++fn purging_service(+ actions: Vec<ScheduledAction>,+) -> (TestService, Arc<FakeRepo>, Receiver<DispatchEvent>) {+ let repo = Arc::new(FakeRepo::default());+ *repo.actions.lock().unwrap() = actions;+ let (tx, rx) = tokio::sync::mpsc::channel(10);+ let service = TestService::new(+ Arc::clone(&repo),+ Arc::new(FakeExecutor::default()),+ tx,+ grants(&repo),+ );+ (service, repo, rx)+}++#[tokio::test]+async fn purges_a_disabled_claimed_action_once_and_announces_its_delete() {+ let mut held = action(Owner::Team(TEAM), true);+ held.enabled = false;+ held.claimed = Some(Utc::now());+ let unrelated = action(Owner::Bot(bot_id::BotId::TEST_A), false);+ let (service, repo, mut rx) = purging_service(vec![held.clone(), unrelated.clone()]);+ let id = held.id.unwrap();++ for _ in 0..2 {+ let outcome = service+ .purge_owned_action(id, &Owner::Team(TEAM))+ .await+ .unwrap();+ assert_eq!(outcome, OwnedPurgeOutcome::Purged);+ }++ let remaining: Vec<_> = repo.actions.lock().unwrap().iter().map(|a| a.id).collect();+ assert_eq!(remaining, [unrelated.id]);+ assert!(matches!(rx.try_recv().unwrap(), DispatchEvent::Delete(a) if a.id == held.id));+ assert!(rx.try_recv().is_err(), "the retry announces nothing");+}++#[tokio::test]+async fn leaves_an_action_another_owner_holds() {+ let held = action(Owner::Team(TEAM), false);+ let (service, repo, mut rx) = purging_service(vec![held.clone()]);++ for other in [+ Owner::Team(OTHER_TEAM),+ Owner::Bot(bot_id::BotId::TEST_A),+ Owner::User(user()),+ ] {+ let outcome = service+ .purge_owned_action(held.id.unwrap(), &other)+ .await+ .unwrap();+ assert_eq!(outcome, OwnedPurgeOutcome::OwnedElsewhere);+ }++ assert_eq!(repo.actions.lock().unwrap().len(), 1);+ assert!(rx.try_recv().is_err());+}++#[tokio::test]+async fn a_stopped_dispatcher_fails_the_purge_before_deleting_the_row() {+ let held = action(Owner::Team(TEAM), false);+ let (service, repo, rx) = purging_service(vec![held.clone()]);+ drop(rx);++ let purged = service+ .purge_owned_action(held.id.unwrap(), &Owner::Team(TEAM))+ .await;++ assert!(purged.is_err());+ assert_eq!(repo.actions.lock().unwrap().len(), 1);+}diff --git a/services/scheduled_action/src/domain/service/test/user_cleanup.rs b/services/scheduled_action/src/domain/service/test/user_cleanup.rsindex 1e5cc75855..16d4fdd3b3 100644--- a/services/scheduled_action/src/domain/service/test/user_cleanup.rs+++ b/services/scheduled_action/src/domain/service/test/user_cleanup.rs@@ -91,7 +91,7 @@ impl ScheduledActionExecutor for NeverExecutor {}}-fn action(owner: Owner, events: bool) -> ScheduledAction {+pub(super) fn action(owner: Owner, events: bool) -> ScheduledAction {let now = Utc::now();let config = configuration(events);ScheduledAction {diff --git a/services/scheduled_action/src/inbound/axum_router.rs b/services/scheduled_action/src/inbound/axum_router.rsindex 157dea5056..d67d6360eb 100644--- a/services/scheduled_action/src/inbound/axum_router.rs+++ b/services/scheduled_action/src/inbound/axum_router.rs@@ -18,13 +18,15 @@ use chrono_tz::Tz;use entity_access::domain::models::{EditAccessLevel, OwnerAccessLevel, ViewAccessLevel};use entity_access::domain::ports::EntityAccessService;use entity_access::inbound::axum_extractors::ScheduledActionAccessExtractor;-use entity_registry::{CreationPrincipalExtractor, NonUserOwners};+use entity_registry::{CreationPrincipalExtractor, NonUserOwners, OwnedPurgeOutcome};use macro_authorization::{InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService, MacroAuthorizationState,UserOrInternal,};use macro_user_id::user_id::MacroUserIdStr;+use macro_uuid::Uuid;use model::response::EmptyResponse;+use model_owner::Owner;use serde::{Deserialize, Serialize};use utoipa::{IntoParams, ToSchema};@@ -131,6 +133,10 @@ where"/scheduled-actions/user/{user_id}",delete(delete_user_actions::<S, Svc, Auth>),)+ .route(+ "/scheduled-actions/internal/{id}",+ delete(purge_owned_action::<S, Svc, Auth>),+ ).route("/scheduled-actions",get(list_actions::<S, Svc, Auth>).post(create_action::<S, Svc, Auth>),@@ -169,6 +175,36 @@ whereOk(StatusCode::NO_CONTENT)}+#[derive(Debug, Deserialize)]+struct OwnerQuery {+ owner: Owner,+}++/// Internal owner removal: never grants delete authority to a user or bot+/// token. 204 once the action is gone, including when it already was. 409+/// when another owner holds it, and nothing was deleted. 500 when the delete+/// failed; the purge converges, so the caller retries.+#[tracing::instrument(skip_all, fields(%id, owner.kind = ?owner.owner_type()))]+async fn purge_owned_action<S, Svc, Auth>(+ State(state): State<ScheduledActionRouterState<S, Svc, Auth>>,+ _internal: MacroAuthorizationExtractor<Auth, InternalOnly>,+ Path(id): Path<Uuid>,+ Query(OwnerQuery { owner }): Query<OwnerQuery>,+) -> StatusCode+where+ S: ScheduledActionService,+ Auth: MacroAuthorizationService,+{+ match state.service.purge_owned_action(id, &owner).await {+ Ok(OwnedPurgeOutcome::Purged) => StatusCode::NO_CONTENT,+ Ok(OwnedPurgeOutcome::OwnedElsewhere) => StatusCode::CONFLICT,+ Err(error) => {+ tracing::error!(error = ?error, "unable to purge the owned scheduled action");+ StatusCode::INTERNAL_SERVER_ERROR+ }+ }+}+#[utoipa::path(get,path = "/health",diff --git a/services/scheduled_action/src/inbound/axum_router/test.rs b/services/scheduled_action/src/inbound/axum_router/test.rsindex 2066310726..d38a6b3fe7 100644--- a/services/scheduled_action/src/inbound/axum_router/test.rs+++ b/services/scheduled_action/src/inbound/axum_router/test.rs@@ -1,3 +1,5 @@+mod owned_purge;+use super::*;use crate::domain::service::test::{USER, configuration, grant_to, service, set_stored_owner};use axum::{diff --git a/services/scheduled_action/src/inbound/axum_router/test/owned_purge.rs b/services/scheduled_action/src/inbound/axum_router/test/owned_purge.rsnew file mode 100644index 0000000000..9f6642f087--- /dev/null+++ b/services/scheduled_action/src/inbound/axum_router/test/owned_purge.rs@@ -0,0 +1,117 @@+use super::*;+use crate::domain::service::test::{TestService, stored_owner, with_stopped_dispatcher};++const INTERNAL_KEY: &str = "test-internal-key";++/// A routine its owner created, then handed to the team.+async fn team_action(service: &Arc<TestService>) -> Uuid {+ let app = router_with(Arc::clone(service), FakeEntityAccessService::owner_only());+ let (_, created) = request(&app, "POST", "/scheduled-actions", "owner", legacy()).await;+ let id = Uuid::parse_str(created["id"].as_str().unwrap()).unwrap();+ set_stored_owner(service, id, Owner::Team(TEAM_ID));+ id+}++async fn purge(app: &Router, uri: &str, headers: &[(&str, &str)]) -> StatusCode {+ let mut request = Request::builder().method("DELETE").uri(uri);+ for (key, value) in headers {+ request = request.header(*key, *value);+ }+ app.clone()+ .oneshot(request.body(Body::empty()).unwrap())+ .await+ .unwrap()+ .status()+}++async fn purge_internally(app: &Router, uri: &str) -> StatusCode {+ purge(+ app,+ uri,+ &[(macro_authorization::INTERNAL_API_KEY_HEADER, INTERNAL_KEY)],+ )+ .await+}++#[tokio::test]+async fn purges_only_for_the_owner_it_names() {+ let service = service(true);+ let id = team_action(&service).await;+ let app = router_with(Arc::clone(&service), FakeEntityAccessService::owner_only());+ let other_team = Uuid::from_u128(0x7EA5);+ let bot = Owner::Bot(bot_id::BotId::TEST_A)+ .principal_id()+ .replace('|', "%7C");++ for owner in [other_team.to_string(), bot] {+ let uri = format!("/scheduled-actions/internal/{id}?owner={owner}");+ assert_eq!(purge_internally(&app, &uri).await, StatusCode::CONFLICT);+ }+ assert_eq!(stored_owner(&service, id), Some(Owner::Team(TEAM_ID)));++ let uri = format!("/scheduled-actions/internal/{id}?owner={TEAM_ID}");+ assert_eq!(purge_internally(&app, &uri).await, StatusCode::NO_CONTENT);+ assert_eq!(stored_owner(&service, id), None);+ assert_eq!(+ purge_internally(&app, &uri).await,+ StatusCode::NO_CONTENT,+ "a retry finds nothing left"+ );+}++#[tokio::test]+async fn a_failed_purge_is_a_server_error_and_keeps_the_action() {+ let service = service(true);+ let id = team_action(&service).await;+ let app = router_with(+ with_stopped_dispatcher(&service),+ FakeEntityAccessService::owner_only(),+ );++ let uri = format!("/scheduled-actions/internal/{id}?owner={TEAM_ID}");++ assert_eq!(+ purge_internally(&app, &uri).await,+ StatusCode::INTERNAL_SERVER_ERROR+ );+ assert_eq!(stored_owner(&service, id), Some(Owner::Team(TEAM_ID)));+}++#[tokio::test]+async fn only_the_internal_key_may_purge() {+ let service = service(true);+ let id = team_action(&service).await;+ let app = router_with(Arc::clone(&service), FakeEntityAccessService::owner_only());+ let uri = format!("/scheduled-actions/internal/{id}?owner={TEAM_ID}");++ for headers in [+ vec![],+ vec![("authorization", "Bearer owner")],+ vec![(BOT_TOKEN_HEADER, BOT_TOKEN), (BOT_SCOPE_HEADER, "team")],+ vec![(macro_authorization::INTERNAL_API_KEY_HEADER, "bad-key")],+ ] {+ let status = purge(&app, &uri, &headers).await;+ assert!(status.is_client_error(), "{headers:?} answered {status}");+ }+ assert_eq!(stored_owner(&service, id), Some(Owner::Team(TEAM_ID)));+}++#[tokio::test]+async fn a_malformed_purge_is_a_bad_request() {+ let service = service(true);+ let id = team_action(&service).await;+ let app = router_with(Arc::clone(&service), FakeEntityAccessService::owner_only());++ for uri in [+ format!("/scheduled-actions/internal/not-an-id?owner={TEAM_ID}"),+ format!("/scheduled-actions/internal/{id}"),+ format!("/scheduled-actions/internal/{id}?owner=not-an-owner"),+ ] {+ assert_eq!(+ purge_internally(&app, &uri).await,+ StatusCode::BAD_REQUEST,+ "{uri}"+ );+ }+ assert_eq!(stored_owner(&service, id), Some(Owner::Team(TEAM_ID)));+}