Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(auth): keep team cleanup retry-safe during account deletion (follow-up to #7401) (#7405)
Will Hutchinson5ddc97b8828d
diff --git a/crates/teams/src/domain/customer_repo.rs b/crates/teams/src/domain/customer_repo.rsindex 24bd2ad529..b49c22036e 100644--- a/crates/teams/src/domain/customer_repo.rs+++ b/crates/teams/src/domain/customer_repo.rs@@ -53,7 +53,8 @@ pub trait CustomerRepository: Clone + Send + Sync + 'static {to: SeatPlan,) -> impl Future<Output = Result<(), CustomerError>> + Send;- /// Cancels a subscription immediately.+ /// Cancels a subscription immediately. A subscription that is already+ /// cancelled or no longer exists counts as cancelled.fn cancel_subscription(&self,subscription_id: &stripe::SubscriptionId,diff --git a/crates/teams/src/domain/team_service.rs b/crates/teams/src/domain/team_service.rsindex 0668b1e581..865e0301a0 100644--- a/crates/teams/src/domain/team_service.rs+++ b/crates/teams/src/domain/team_service.rs@@ -1307,8 +1307,16 @@ whereself.delete_team_as(team_id, user_id.clone().into_owned()).await?;} else {- self.remove_member(team_id, user_id, user_id.clone().into_owned())- .await?;+ match self+ .remove_member(team_id, user_id, user_id.clone().into_owned())+ .await+ {+ // The goal is that the user is off the team, so a+ // membership that vanished between the lookup and the+ // removal is already done.+ Ok(()) | Err(RemoveUserFromTeamError::UserNotInTeam) => {}+ Err(error) => return Err(error.into()),+ }}}Ok(())diff --git a/crates/teams/src/domain/team_service/test.rs b/crates/teams/src/domain/team_service/test.rsindex 5ec8ad4367..6ca754ab78 100644--- a/crates/teams/src/domain/team_service/test.rs+++ b/crates/teams/src/domain/team_service/test.rs@@ -4061,6 +4061,24 @@ async fn remove_user_from_all_teams_leaves_the_team_the_user_belongs_to() {assert_eq!(metadata["removed_by"], member.as_ref());}+#[tokio::test]+async fn remove_user_from_all_teams_treats_a_vanished_membership_as_left() {+ let team_id = uuid::Uuid::from_u128(812);+ let owner = MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap();+ let member = MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap();+ let mut repo = MockTeamRepository::new(Vec::new(), "Team", Arc::new(Mutex::new(Vec::new())));+ repo.user_teams = vec![account_deletion_team(team_id, &owner)];+ // No `removed_member`: the repository reports the user is no longer on the team.+ let remove_user_calls = repo.remove_user_calls.clone();+ let broker = RecordingEventBroker::default();+ let service = build_service_with_repo_and_broker(repo, broker.clone());++ service.remove_user_from_all_teams(&member).await.unwrap();++ assert_eq!(*remove_user_calls.lock().unwrap(), 1);+ assert!(broker.events().is_empty());+}+#[tokio::test]async fn remove_user_from_all_teams_without_a_team_changes_nothing() {let user = MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap();diff --git a/crates/teams/src/outbound/customer_repo.rs b/crates/teams/src/outbound/customer_repo.rsindex 07bf397ae9..f5de63fb1a 100644--- a/crates/teams/src/outbound/customer_repo.rs+++ b/crates/teams/src/outbound/customer_repo.rs@@ -197,9 +197,24 @@ impl CustomerRepository for CustomerRepositoryImpl {&self,subscription_id: &stripe::SubscriptionId,) -> Result<(), CustomerError> {- let cancel_parmas = stripe::CancelSubscription::default();+ // Cancelling is idempotent: a subscription that is already cancelled,+ // or that went away with its customer, needs nothing further.+ match stripe::Subscription::retrieve(&self.client, subscription_id, &[]).await {+ Ok(subscription) if subscription.status == stripe::SubscriptionStatus::Canceled => {+ return Ok(());+ }+ Ok(_) => {}+ Err(stripe::StripeError::Stripe(error))+ if matches!(error.code, Some(stripe::ErrorCode::ResourceMissing)) =>+ {+ return Ok(());+ }+ Err(error) => return Err(CustomerError::StorageLayerError(error.into())),+ }++ let cancel_params = stripe::CancelSubscription::default();- stripe::Subscription::cancel(&self.client, subscription_id, cancel_parmas)+ stripe::Subscription::cancel(&self.client, subscription_id, cancel_params).await.map_err(|e| CustomerError::StorageLayerError(e.into()))?;diff --git a/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs b/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rsindex 49e72a49de..6d1c1bb7e6 100644--- a/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs+++ b/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs@@ -13,7 +13,6 @@ use model::{authentication::webhooks::FusionAuthUserWebhook, user::UserInfoWithMuse notification::domain::ports::NotificationRepository;use notification::outbound::repository::DbNotificationRepository;use sqlx::{Pool, Postgres};-use stripe::CustomerId;use tracing::Instrument;use crate::api::context::{ApiContext, AuthorizationService};@@ -137,34 +136,6 @@ async fn delete_user(}});- // Handle stripe user deletion- if let Some(stripe_customer_id) = macro_user.stripe_customer_id {- tokio::spawn({- let stripe_customer_id = stripe_customer_id.clone();- let stripe_client = ctx.stripe_client.clone();- async move {- tracing::trace!(stripe_customer_id, "delete_stripe_customer");-- let customer_id: CustomerId = match stripe_customer_id.parse() {- Ok(id) => id,- Err(e) => {- tracing::error!(error=?e, stripe_customer_id, "unable to parse stripe customer id");- return;- }- };-- if let Err(e) = stripe::Customer::delete(&stripe_client, &customer_id).await {- tracing::error!(error=?e, stripe_customer_id, "unable to delete stripe customer");- }-- tracing::trace!(- stripe_customer_id,- "delete_stripe_customer complete"- );- }- }.in_current_span());- }-// Fixed: Create futures and await them all concurrentlylet user_info_futures = user_ids.clone()diff --git a/services/authentication_service/src/main.rs b/services/authentication_service/src/main.rsindex c9c7458b04..365b5f232c 100644--- a/services/authentication_service/src/main.rs+++ b/services/authentication_service/src/main.rs@@ -541,6 +541,7 @@ async fn main() -> anyhow::Result<()> {db.clone(),document_storage_service_client.clone(),teams_service.clone(),+ stripe_client.clone(),config.service_internal_auth_key.to_string(),macro_service_urls::AgentHarnessServiceUrl::new()?.to_string(),macro_service_urls::ScheduledActionServiceUrl::new()?.to_string(),diff --git a/services/authentication_service/src/outbound/user_deletion.rs b/services/authentication_service/src/outbound/user_deletion.rsindex 648725acd5..1a3e968649 100644--- a/services/authentication_service/src/outbound/user_deletion.rs+++ b/services/authentication_service/src/outbound/user_deletion.rs@@ -15,12 +15,13 @@ use uuid::Uuid;use crate::service::user::delete_user::UserDeletionGateway;-/// HTTP, in-process team service, and database adapter composed at+/// HTTP, in-process team service, Stripe, and database adapter composed at/// authentication-service startup.pub struct UserDeletionAdapter<T: TeamService> {db: PgPool,documents: Arc<DocumentStorageServiceClient>,teams: Arc<T>,+ stripe: Arc<stripe::Client>,client: reqwest::Client,internal_key: String,harness_url: String,@@ -38,6 +39,7 @@ impl<T: TeamService> UserDeletionAdapter<T> {db: PgPool,documents: Arc<DocumentStorageServiceClient>,teams: Arc<T>,+ stripe: Arc<stripe::Client>,internal_key: String,harness_url: String,scheduled_action_url: String,@@ -46,6 +48,7 @@ impl<T: TeamService> UserDeletionAdapter<T> {db,documents,teams,+ stripe,client: reqwest::Client::builder().timeout(Duration::from_secs(300)).redirect(reqwest::redirect::Policy::none())@@ -122,6 +125,55 @@ impl<T: TeamService> UserDeletionGateway for UserDeletionAdapter<T> {Ok(())}+ async fn delete_billing_customer(&self, account: &Uuid) -> Result<(), Report> {+ let customer_id =+ match macro_db_client::macro_user::get_macro_user(&self.db, &account.to_string()).await+ {+ Ok(account) => account.stripe_customer_id,+ // A retry after the account row is gone has nothing left to delete.+ Err(error)+ if matches!(+ error.downcast_ref::<sqlx::Error>(),+ Some(sqlx::Error::RowNotFound)+ ) =>+ {+ None+ }+ Err(error) => Err(error.into_rootcause())+ .context("failed to load account for billing cleanup")?,+ };+ let Some(customer_id) = customer_id else {+ return Ok(());+ };+ let customer_id: stripe::CustomerId = match customer_id.parse() {+ Ok(customer_id) => customer_id,+ // Local stacks store placeholder ids: there is nothing in Stripe to+ // delete, and failing here would block the deletion forever.+ Err(error) => {+ tracing::warn!(+ error = ?error,+ "stripe customer id is not a stripe id; skipping customer deletion"+ );+ return Ok(());+ }+ };++ stripe::Customer::delete(&self.stripe, &customer_id)+ .await+ .map(drop)+ .or_else(|error| match error {+ // Already deleted, for example by an earlier attempt.+ stripe::StripeError::Stripe(request)+ if matches!(request.code, Some(stripe::ErrorCode::ResourceMissing)) =>+ {+ Ok(())+ }+ error => Err(Report::new(error).into_dynamic()),+ })+ .context("failed to delete stripe customer")?;+ Ok(())+ }+async fn delete_account(&self, account: &Uuid) -> Result<(), Report> {macro_db_client::macro_user::delete_macro_user(&self.db, account).awaitdiff --git a/services/authentication_service/src/service/user/delete_user.rs b/services/authentication_service/src/service/user/delete_user.rsindex 3dc0452e27..9dbd8da7b6 100644--- a/services/authentication_service/src/service/user/delete_user.rs+++ b/services/authentication_service/src/service/user/delete_user.rs@@ -37,6 +37,15 @@ pub trait UserDeletionGateway: Send + Sync {user: &MacroUserIdStr<'static>,account: &Uuid,) -> impl Future<Output = Result<(), Report>> + Send;+ /// Delete the account's billing customer once every profile is gone and+ /// before the account row that remembers the customer, so a failure here+ /// is retried instead of orphaning the customer. Deleting the customer+ /// also cancels any subscription it still has, so this must run after the+ /// team subscription has been handled through `leave_teams`.+ fn delete_billing_customer(+ &self,+ account: &Uuid,+ ) -> impl Future<Output = Result<(), Report>> + Send;/// Delete the account only after all profiles have been cleaned up.fn delete_account(&self, account: &Uuid) -> impl Future<Output = Result<(), Report>> + Send;}@@ -55,5 +64,6 @@ pub async fn delete_user_data(gateway.delete_items(user).await?;gateway.delete_profile(user, account).await?;}+ gateway.delete_billing_customer(account).await?;gateway.delete_account(account).await}diff --git a/services/authentication_service/src/service/user/delete_user/test.rs b/services/authentication_service/src/service/user/delete_user/test.rsindex 6daef09b51..ed00be3733 100644--- a/services/authentication_service/src/service/user/delete_user/test.rs+++ b/services/authentication_service/src/service/user/delete_user/test.rs@@ -34,6 +34,9 @@ impl UserDeletionGateway for FakeGateway {async fn delete_profile(&self, user: &MacroUserIdStr<'static>, _: &Uuid) -> Result<(), Report> {self.call("profile", user.as_ref())}+ async fn delete_billing_customer(&self, _: &Uuid) -> Result<(), Report> {+ self.call("customer", "")+ }async fn delete_account(&self, _: &Uuid) -> Result<(), Report> {self.call("account", "")}@@ -53,6 +56,7 @@ fn expected_calls() -> Vec<String> {expected.push(format!("{step}:{user}"));}}+ expected.push("customer:".into());expected.push("account:".into());expected}@@ -89,5 +93,5 @@ async fn retry_after_profiles_are_gone_still_deletes_the_account() {delete_user_data(&gateway, &Uuid::now_v7(), &[]).await.unwrap();- assert_eq!(*gateway.calls.lock().unwrap(), ["account:"]);+ assert_eq!(*gateway.calls.lock().unwrap(), ["customer:", "account:"]);}diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env.rs b/tooling/xtask/crates/xtask_local/src/local/local_env.rsindex b51e71c93d..574d71a5a8 100644--- a/tooling/xtask/crates/xtask_local/src/local/local_env.rs+++ b/tooling/xtask/crates/xtask_local/src/local/local_env.rs@@ -523,7 +523,6 @@ impl AgentHarnessEnv {/// container (services, sync, lexical) agrees. `INTERNAL_API_SECRET_KEY` is the/// literal `"local"` to match the FusionAuth webhook's `x-internal-auth-key`.struct ServiceAuthEnv {- dss_auth: String,doc_perm_jwt: String,internal_call: String,url_signing: String,@@ -532,7 +531,6 @@ struct ServiceAuthEnv {impl ServiceAuthEnv {fn for_instance(name: &str) -> Self {ServiceAuthEnv {- dss_auth: identity::instance_secret("dss-auth", name),// Must match sync-service's local DOCUMENT_PERMISSIONS_SECRET// ("local") so locally-minted tokens verify. This is ONLY for local// dev use obv@@ -564,15 +562,21 @@ impl ServiceAuthEnv {"SYNC_SERVICE_AUTH_KEY".into(),identity::INTERNAL_AUTH_KEY.into(),);- // The key the authentication service presents to document storage on- // internal calls (e.g. seeding starter docs at signup). In dev/prod- // Doppler points it at the *same* secret as DSS's own auth key- // (document-storage-service-auth-key-*), so locally the two must be- // one value or DSS 401s every auth-service internal call.- env.insert("SERVICE_INTERNAL_AUTH_KEY".into(), self.dss_auth.clone());+ // The key the authentication service presents on internal calls to+ // document storage (seeding starter docs at signup), the connection+ // gateway, the agent harness, and the scheduled-action service+ // (account deletion). Deployed environments point it at the *same*+ // secret those services validate, which is also DSS's own auth key+ // (document-storage-service-auth-key-*). Locally every service+ // validates the one shared internal key, so this must be that key or+ // every auth-service internal call 401s.+ env.insert(+ "SERVICE_INTERNAL_AUTH_KEY".into(),+ identity::INTERNAL_AUTH_KEY.into(),+ );env.insert("DOCUMENT_STORAGE_SERVICE_AUTH_KEY".into(),- self.dss_auth.clone(),+ identity::INTERNAL_AUTH_KEY.into(),);env.insert("DOCUMENT_PERMISSION_JWT".into(), self.doc_perm_jwt.clone());env.insert("INTERNAL_CALL_SECRET".into(), self.internal_call.clone());diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rsindex 73ffcad669..4f3b189490 100644--- a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs@@ -295,6 +295,19 @@ fn auth_service_internal_key_matches_dss_auth_key() {);}+/// The auth service also presents `SERVICE_INTERNAL_AUTH_KEY` to the connection+/// gateway, the agent harness, and the scheduled-action service (account+/// deletion), which validate `INTERNAL_API_KEY`. Deployed environments point+/// both at one secret; locally they must match too.+#[test]+fn auth_service_internal_key_matches_the_shared_internal_key() {+ let env = local_env();+ assert_eq!(+ env.get("SERVICE_INTERNAL_AUTH_KEY"),+ env.get("INTERNAL_API_KEY"),+ );+}+#[test]fn aws_creds_are_dummy() {let env = local_env();@@ -322,8 +335,8 @@ fn instance_secrets_are_scoped_but_identity_is_fixed() {.to_env();assert_ne!(- a.get("SERVICE_INTERNAL_AUTH_KEY"),- b.get("SERVICE_INTERNAL_AUTH_KEY"),+ a.get("INTERNAL_CALL_SECRET"),+ b.get("INTERNAL_CALL_SECRET"),"per-instance secrets should differ between instances");assert_eq!(