Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(auth): add structured logging to create and delete user webhooks (#7967)
Will Hutchinsoneedefc3ed8a2
diff --git a/services/authentication_service/src/api/webhooks/user/create_user_webhook.rs b/services/authentication_service/src/api/webhooks/user/create_user_webhook.rsindex 2e7cee8919..8c35eb50d9 100644--- a/services/authentication_service/src/api/webhooks/user/create_user_webhook.rs+++ b/services/authentication_service/src/api/webhooks/user/create_user_webhook.rs@@ -38,6 +38,7 @@ use macro_user_id::{use model::authentication::webhooks::{FusionAuthUserWebhook, User as FusionAuthWebhookUser};use model_entity::EntityType;use teams::domain::team_repo::TeamService;+use tracing::Instrument;/// Macro support team members added to every new user's support channel.const MACRO_SUPPORT_EMAILS: [&str; 5] = [@@ -85,18 +86,30 @@ fn identity_provider_name(user: &FusionAuthWebhookUser) -> (Option<String>, Opti}/// FusionAuth create user webhook-#[tracing::instrument(skip(ctx, req, _internal_authorization), fields(email=%req.event.user.email, fusionauth_user_id=%req.event.user.id, username=?req.event.user.username, event_type=%req.event.event_type, ip_address=%req.event.info.ip_address))]+#[tracing::instrument(+ skip(ctx, req, _internal_authorization),+ fields(+ event_id = %req.event.id,+ event_type = %req.event.event_type,+ email = %req.event.user.email,+ fusionauth_user_id = %req.event.user.id,+ username = ?req.event.user.username,+ verified = req.event.user.verified,+ ip_address = %req.event.info.ip_address,+ )+)]pub async fn handler(State(ctx): State<ApiContext>,_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,extract::Json(req): extract::Json<FusionAuthUserWebhook>,) -> Result<Response, Response> {- tracing::info!("create_user_webhook");+ tracing::info!("received FusionAuth user webhook");+ let start_time = std::time::Instant::now();match req.event.event_type.as_str() {"user.create.complete" => {create_user_webhook_complete(&ctx, req).await.map_err(|e| {- tracing::error!(error=?e, "unable to user.create.complete");+ tracing::error!(error=?e, "unable to handle user.create.complete");(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response()})?;}@@ -109,16 +122,21 @@ pub async fn handler(}"user.email.verified" => {verify_user_email_webhook(&ctx, req).await.map_err(|e| {- tracing::error!(error=?e, "unable to user.email.verified");+ tracing::error!(error=?e, "unable to handle user.email.verified");(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response()})?;}_ => {- tracing::error!("unexpected event type");+ tracing::error!("unexpected FusionAuth event type; webhook is not implemented for it");return Ok(StatusCode::NOT_IMPLEMENTED.into_response());}}+ tracing::info!(+ elapsed_ms = start_time.elapsed().as_millis() as u64,+ "handled FusionAuth user webhook"+ );+Ok(StatusCode::OK.into_response())}@@ -129,9 +147,10 @@ enum UserCreateWebhookError {fn user_create_webhook_error_response(error: UserCreateWebhookError) -> Response {match error {+ // Already logged where the policy was evaluated.UserCreateWebhookError::Forbidden => signup_forbidden_response(),UserCreateWebhookError::Onboarding(e) => {- tracing::error!(error=?e, "unable to user.create");+ tracing::error!(error=?e, "unable to handle user.create; FusionAuth user creation will be aborted");(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response()}}@@ -148,14 +167,22 @@ where{signup_policy.authorize_public_email(&req.event.user.email)- .map_err(|_| UserCreateWebhookError::Forbidden)?;+ .map_err(|denial| {+ // The denial variant is redacted and never names configured addresses.+ tracing::warn!(denial=?denial, "rejected user.create: signup policy denied public email");+ UserCreateWebhookError::Forbidden+ })?;onboard_user(req).await.map_err(UserCreateWebhookError::Onboarding)}-#[tracing::instrument(skip(ctx, req), err)]+#[tracing::instrument(+ skip(ctx, req),+ fields(email = %req.event.user.email, fusionauth_user_id = %req.event.user.id),+ err+)]async fn verify_user_email_webhook(ctx: &ApiContext,req: FusionAuthUserWebhook,@@ -171,9 +198,16 @@ async fn verify_user_email_webhook().await?;+ tracing::info!("marked user email as verified");+Ok(())}+#[tracing::instrument(+ skip(ctx, req),+ fields(email = %req.event.user.email, fusionauth_user_id = %req.event.user.id),+ err+)]async fn create_user_webhook_complete(ctx: &ApiContext,req: FusionAuthUserWebhook,@@ -181,18 +215,39 @@ async fn create_user_webhook_complete(let fusionauth_user_id = req.event.user.id;match ctx.auth_client.register_user(&fusionauth_user_id).await {- Ok(_) => Ok(()),+ Ok(_) => {+ tracing::info!("registered user with the FusionAuth application");+ Ok(())+ }Err(e) => match e {- FusionAuthClientError::UserAlreadyRegistered => Ok(()),+ FusionAuthClientError::UserAlreadyRegistered => {+ tracing::info!("user is already registered with the FusionAuth application");+ Ok(())+ }_ => {- tracing::error!(error=?e, "unable to register user in fusionauth");+ tracing::error!(error=?e, "unable to register user with the FusionAuth application");Err(e.into())}},}}-#[tracing::instrument(skip(ctx, req), err)]+/// Onboards a new FusionAuth user.+///+/// `user_id` and `organization_id` are recorded on the span once the profile+/// exists so that every later log line, including those from the spawned+/// follow-up tasks, carries them.+#[tracing::instrument(+ skip(ctx, req),+ fields(+ email = %req.event.user.email,+ fusionauth_user_id = %req.event.user.id,+ ip_address = %req.event.info.ip_address,+ user_id = tracing::field::Empty,+ organization_id = tracing::field::Empty,+ ),+ err+)]async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> anyhow::Result<()> {let ip_address = req.event.info.ip_address;let email = req.event.user.email.to_lowercase();@@ -218,6 +273,11 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> anif let Some(count) = rate_limit&& count >= RATE_LIMIT_CONFIG.create_user_hourly.0{+ tracing::warn!(+ count,+ limit = RATE_LIMIT_CONFIG.create_user_hourly.0,+ "create user hourly rate limit exceeded for ip address"+ );anyhow::bail!("rate limit exceeded")}@@ -229,7 +289,7 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> an).await?{- tracing::info!(user_id=?user_id, organization_id=?organization_id, "user profile already exists for FusionAuth user");+ tracing::info!(user_id=?user_id, organization_id=?organization_id, "user profile already exists for FusionAuth user; skipping onboarding");return Ok(());}@@ -240,11 +300,16 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> an{// The macro_user already exists for that email// We do not allow a user to login through their secondary linked account for SSO so we shouldn't allow for passwordless either- tracing::info!(user_id=?user_id, "user already exists");+ tracing::warn!(existing_user_id=?user_id, "email already belongs to an existing user; rejecting signup through secondary account link");anyhow::bail!("cannot login through secondary account link");}+ tracing::info!(+ has_first_name = first_name.is_some(),+ has_last_name = last_name.is_some(),+ "creating user"+ );let start_time = std::time::Instant::now();let (user_id, organization_id) = create_user(&fusionauth_user_id,@@ -256,7 +321,15 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> an).await?;- tracing::trace!(user_id=?user_id, organization_id=?organization_id, "created user");+ let span = tracing::Span::current();+ span.record("user_id", user_id.as_str());+ if let Some(organization_id) = organization_id {+ span.record("organization_id", organization_id);+ }+ tracing::info!(+ elapsed_ms = start_time.elapsed().as_millis() as u64,+ "created user; dispatching signup follow-ups"+ );// Mark the account as freshly created so the auth callback completing this// flow can attribute the login as a signup (analytics). Best-effort.@@ -297,6 +370,7 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> anlast_name,).await+ .inspect(|_| tracing::info!("set user name from identity provider")).inspect_err(|e| tracing::error!(error=?e, "unable to set user name from identity provider"),);@@ -312,6 +386,7 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> an}),).await+ .inspect(|_| tracing::info!("tracked PostHog sign_up event")).inspect_err(|e| tracing::warn!(error=?e, "failed to track PostHog sign_up event"));let user_data = MetaUserData {@@ -340,10 +415,12 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> an}),).await+ .inspect(|_| tracing::info!("tracked Meta CompleteRegistration event")).inspect_err(|e| {tracing::warn!(error=?e, "failed to track Meta CompleteRegistration");});}+ .in_current_span()});// Tell Loops the account now exists. This upserts the contact into our@@ -368,10 +445,12 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> anSome(&format!("user-registered-{user_id}")),).await+ .inspect(|_| tracing::info!("sent Loops user_registered event")).inspect_err(|e| tracing::error!(error=?e, "failed to send Loops user_registered event"),);}+ .in_current_span()});// add user to all active experiments@@ -379,10 +458,16 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> anlet db = ctx.db.clone();let user_id = user_id.clone();async move {- if let Err(e) = initialize_user_experiments(&db, &user_id).await {- tracing::error!(error=?e, "failed to initialize user experiments");+ match initialize_user_experiments(&db, &user_id).await {+ Ok(experiment_count) => {+ tracing::info!(experiment_count, "initialized user experiments");+ }+ Err(e) => {+ tracing::error!(error=?e, "failed to initialize user experiments");+ }}}+ .in_current_span()});// Automatically add the new user to a team whose auto-join domain@@ -401,14 +486,17 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> an};match teams_service.try_join_team_by_domain(&user_id).await {Ok(Some(member)) => {- tracing::info!(team_id=%member.team_id, %user_id, "auto-joined user to team by email domain");+ tracing::info!(team_id=%member.team_id, "auto-joined user to team by email domain");+ }+ Ok(None) => {+ tracing::info!("no team auto-join domain matched the user's email domain");}- Ok(None) => {}Err(e) => {- tracing::error!(error=?e, %user_id, "failed to auto-join user to team by email domain");+ tracing::error!(error=?e, "failed to auto-join user to team by email domain");}}}+ .in_current_span()});// Seed the starter documents (the "Macro how to guide" and the starter@@ -425,10 +513,10 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> anlet entity_access_service = ctx.entity_access_service.clone();let favorites_service = ctx.favorites_service.clone();let user_id = user_id.clone();- let email = email.clone();let support_channel_name = support_channel_name.clone();async move {- initialize_starter_docs_with_retries(&document_storage_service_client, &user_id).await;+ initialize_starter_docs_with_retries(&document_storage_service_client, &user_id)+ .await;let owner_id = match MacroUserIdStr::try_from(user_id) {Ok(owner_id) => owner_id,@@ -465,31 +553,34 @@ async fn create_user_webhook(ctx: &ApiContext, req: FusionAuthUserWebhook) -> an{Ok(channel) => channel,Err(e) => {- tracing::error!(error=?e, %email, "failed to create Macro support channel");+ tracing::error!(error=?e, "failed to create Macro support channel");return;}};+ tracing::info!(channel_id=%channel.id, "created Macro support channel");let channel_entity = EntityType::Channel.with_entity_str(&channel.id);if let Err(e) = favorites_service.add_favorite_with_established_access(&owner_id, &channel_entity).await{- tracing::error!(error=?e, channel_id=%channel.id, %email, "failed to favorite Macro support channel");+ tracing::error!(error=?e, channel_id=%channel.id, "failed to favorite Macro support channel");}let welcome_gateway =AuthorizedSupportChannelMessages::new(channel_messages, entity_access_service);let _ = post_support_channel_welcome(&welcome_gateway, &channel.id, owner_id).await+ .inspect(|_| {+ tracing::info!(channel_id=%channel.id, "posted Macro support welcome message");+ }).inspect_err(|e| {- tracing::error!(error=?e, channel_id=%channel.id, %email, "failed to post Macro support welcome message");- });+ tracing::error!(error=?e, channel_id=%channel.id, "failed to post Macro support welcome message");+ });}+ .in_current_span()});- tracing::trace!(email, fusionauth_user_id, elapsed=?start_time.elapsed(), "created user");-// Allow to fail silentlylet _ = ctx.macro_cache_client@@ -509,24 +600,28 @@ async fn initialize_starter_docs_with_retries(const RETRY_DELAY_SECS: u64 = 2;for attempt in 1..=MAX_ATTEMPTS {match client.initialize_starter_docs(user_id).await {- Ok(_) => return,+ Ok(_) => {+ tracing::info!(attempt, "initialized starter docs");+ return;+ }Err(e) if attempt < MAX_ATTEMPTS => {tracing::warn!(error=?e, attempt, "failed to initialize starter docs, retrying");tokio::time::sleep(std::time::Duration::from_secs(RETRY_DELAY_SECS)).await;}Err(e) => {- tracing::error!(error=?e, "failed to initialize starter docs after {MAX_ATTEMPTS} attempts");+ tracing::error!(error=?e, attempt, "failed to initialize starter docs after {MAX_ATTEMPTS} attempts");}}}}-/// Initializes the experiments for a provided user-#[allow(dead_code)]+/// Initializes the experiments for a provided user.+///+/// Returns the number of active experiments the user was enrolled in.async fn initialize_user_experiments(db: &sqlx::Pool<sqlx::Postgres>,user_id: &str,-) -> anyhow::Result<()> {+) -> anyhow::Result<usize> {let active_experiments = macro_db_client::experiment::get_active_experiments(db).await.context("failed to get active experiments")?;@@ -548,5 +643,5 @@ async fn initialize_user_experiments(.await.context("failed to bulk create experiment logs")?;- Ok(())+ Ok(active_experiments.len())}diff --git a/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs b/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rsindex a5b786a903..c53c47cf0f 100644--- a/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs+++ b/services/authentication_service/src/api/webhooks/user/delete_user_webhook.rs@@ -9,7 +9,7 @@ use comms_db_client::{};use macro_authorization::{InternalOnly, MacroAuthorizationExtractor};use macro_user_id::user_id::MacroUserIdStr;-use model::{authentication::webhooks::FusionAuthUserWebhook, user::UserInfoWithMacroUserId};+use model::{authentication::webhooks::FusionAuthUserWebhook, user::UserInfo};use notification::domain::ports::NotificationRepository;use notification::outbound::repository::DbNotificationRepository;use sqlx::{Pool, Postgres};@@ -19,13 +19,22 @@ use crate::api::context::{ApiContext, AuthorizationService};use sqs_client::email::LinkManagerMessage;/// Delete user webhook-#[tracing::instrument(skip(ctx, req, _internal_authorization), fields(event_id=req.event.id, email=req.event.user.email,fusionauth_user_id=req.event.user.id))]+#[tracing::instrument(+ skip(ctx, req, _internal_authorization),+ fields(+ event_id = %req.event.id,+ event_type = %req.event.event_type,+ email = %req.event.user.email,+ fusionauth_user_id = %req.event.user.id,+ )+)]pub async fn handler(State(ctx): State<ApiContext>,_internal_authorization: MacroAuthorizationExtractor<AuthorizationService, InternalOnly>,extract::Json(req): extract::Json<FusionAuthUserWebhook>,) -> Result<Response, Response> {- tracing::info!("delete user webhook");+ tracing::info!("received FusionAuth delete user webhook");+ let start_time = std::time::Instant::now();let fusionauth_user_id = req.event.user.id;// if fusionauth_user_id is part of an account_merge_request, return early as we are in@@ -36,7 +45,7 @@ pub async fn handler().await.map_err(|e| {- tracing::error!(error=?e, "unable to get merge request");+ tracing::error!(error=?e, "unable to check for an account merge request");(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response()})?.is_some()@@ -47,23 +56,27 @@ pub async fn handler(return Ok(StatusCode::OK.into_response());}- let macro_user =- match macro_db_client::macro_user::get_macro_user(&ctx.db, &fusionauth_user_id).await {- Ok(user) => user,- // A retry after the final delete has nothing left to clean up.- Err(error)- if matches!(- error.downcast_ref::<sqlx::Error>(),- Some(sqlx::Error::RowNotFound)- ) =>- {- return Ok(StatusCode::OK.into_response());- }- Err(error) => {- tracing::error!(error=?error, "unable to get macro user");- return Err(StatusCode::INTERNAL_SERVER_ERROR.into_response());- }- };+ let macro_user = match macro_db_client::macro_user::get_macro_user(&ctx.db, &fusionauth_user_id)+ .await+ {+ Ok(user) => user,+ // A retry after the final delete has nothing left to clean up.+ Err(error)+ if matches!(+ error.downcast_ref::<sqlx::Error>(),+ Some(sqlx::Error::RowNotFound)+ ) =>+ {+ tracing::info!(+ "macro user not found; nothing left to delete (already deleted or retried after completion)"+ );+ return Ok(StatusCode::OK.into_response());+ }+ Err(error) => {+ tracing::error!(error=?error, "unable to get macro user");+ return Err(StatusCode::INTERNAL_SERVER_ERROR.into_response());+ }+ };let user_ids: Vec<String> =macro_db_client::user::get::get_user_profiles_by_fusionauth_user_id(@@ -72,7 +85,7 @@ pub async fn handler().await.map_err(|e| {- tracing::error!(error=?e, "unable to get user info by email");+ tracing::error!(error=?e, "unable to get user profiles for FusionAuth user");(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response()})?;@@ -81,14 +94,28 @@ pub async fn handler(delete_user(ctx, macro_user, fusionauth_user_id, user_ids).await.map_err(|error| {- tracing::error!(error=?error, "unable to complete user deletion");+ tracing::error!(error=?error, "unable to complete user deletion; FusionAuth will retry");StatusCode::INTERNAL_SERVER_ERROR.into_response()})?;+ tracing::info!(+ elapsed_ms = start_time.elapsed().as_millis() as u64,+ "completed FusionAuth delete user webhook"+ );+Ok(StatusCode::OK.into_response())}-#[tracing::instrument(skip(ctx, user_ids))]+#[tracing::instrument(+ skip(ctx, macro_user, fusionauth_user_id, user_ids),+ fields(+ fusionauth_user_id = %fusionauth_user_id,+ stripe_customer_id = ?macro_user.stripe_customer_id,+ user_ids = ?user_ids,+ profile_count = user_ids.len(),+ ),+ err+)]async fn delete_user(ctx: ApiContext,macro_user: macro_db_client::macro_user::MacroUser,@@ -105,54 +132,75 @@ async fn delete_user(let message = LinkManagerMessage::DeleteUser {fusionauth_user_id: fusionauth_user_id.clone(),};- if let Err(e) = sqs_client- .enqueue_link_manager_notification(message)- .await- {- tracing::error!(error=?e, fusionauth_user_id, "unable to enqueue email link deletion");+ match sqs_client.enqueue_link_manager_notification(message).await {+ Ok(_) => tracing::info!("enqueued email link deletion"),+ Err(e) => {+ tracing::error!(error=?e, "unable to enqueue email link deletion");+ }}}- }.in_current_span());+ .in_current_span()+ });tokio::spawn({let user_ids = user_ids.clone();let sqs_client = ctx.sqs_client.clone();async move {for user_id in user_ids {- tracing::trace!(user_id, "deleting user search data");- if let Err(e) = sqs_client+ match sqs_client.send_message_to_search_event_queue(- sqs_client::search::SearchQueueMessage::RemoveUserProfile(user_id.clone()),+ sqs_client::search::SearchQueueMessage::RemoveUserProfile(+ user_id.clone(),+ ),).await{- tracing::error!(error=?e, user_id, "unable to send delete user message to search event queue");+ Ok(_) => {+ tracing::info!(user_id, "enqueued user profile removal from search");+ }+ Err(e) => {+ tracing::error!(error=?e, user_id, "unable to send delete user message to search event queue");+ }}- tracing::trace!(- user_id,- "sending delete user message to search event queue complete"- );}}+ .in_current_span()});- // Fixed: Create futures and await them all concurrently+ // `user_ids` are `User.id` profile ids (`macro|<email>`), so look them up by+ // id rather than by the email column.let user_info_futures = user_ids.clone().into_iter().map(|user_id| {let db = ctx.db.clone();- async move { macro_db_client::user::get::get_user_info_by_email(&db, &user_id).await }+ async move {+ let result = macro_db_client::user::get::get_user_profile(&db, &user_id).await;+ (user_id, result)+ }}).collect::<Vec<_>>();// Await all futures concurrentlylet user_info_results = futures::future::join_all(user_info_futures).await;- let user_infos: Vec<UserInfoWithMacroUserId> = user_info_results+ // Lookups are best-effort: a profile whose info cannot be loaded still has+ // its durable data deleted below, but skips the cache/channel/notification+ // cleanup, so record which ones were skipped.+ let user_infos: Vec<UserInfo> = user_info_results.into_iter()- .filter_map(|r| r.ok())+ .filter_map(|(user_id, result)| {+ result+ .inspect_err(|e| {+ tracing::error!(error=?e, user_id, "unable to load user info; skipping channel and notification cleanup for this profile");+ })+ .ok()+ }).collect();+ tracing::info!(+ loaded_profile_count = user_infos.len(),+ "loaded user info for profiles"+ );// MacroCache deletiontokio::spawn(@@ -161,11 +209,12 @@ async fn delete_user(let user_ids = user_ids.clone();async move {for user_id in user_ids {- tracing::trace!(user_id, "delete_user_redis_session");- if let Err(e) = redis_client.delete_user(&user_id).await {- tracing::error!(error=?e, user_id, "unable to delete user from redis");+ match redis_client.delete_user(&user_id).await {+ Ok(_) => tracing::info!(user_id, "deleted user session from cache"),+ Err(e) => {+ tracing::error!(error=?e, user_id, "unable to delete user from redis");+ }}- tracing::trace!(user_id, "delete_user_redis_session_complete");}}}@@ -180,16 +229,20 @@ async fn delete_user(async move {for user_info in user_infos {let user_id = user_info.id.clone();- tracing::trace!(user_id, "remove_user_from_org_channels",);// TODO: create delete user endpoint in comms service and handle removing this user and// deleting all of their channels. Keep the messages for now.- if let Some(org_id) = user_info.organization_id- && let Err(err) =- remove_user_from_org_channels(&db, &user_id, org_id as i64).await- {- tracing::error!(error=?err, "unable to remove user from org channels");+ let Some(org_id) = user_info.organization_id else {+ tracing::info!(user_id, "user has no organization; no organization channels to leave");+ continue;+ };+ match remove_user_from_org_channels(&db, &user_id, org_id as i64).await {+ Ok(channel_count) => {+ tracing::info!(user_id, organization_id = org_id, channel_count, "removed user from organization channels");+ }+ Err(err) => {+ tracing::error!(error=?err, user_id, organization_id = org_id, "unable to remove user from org channels");+ }}- tracing::trace!(user_id, "remove_user_from_org_channels complete",);}}}@@ -206,21 +259,22 @@ async fn delete_user(async move {for user_info in user_infos {let user_id = user_info.id.clone();- tracing::trace!(user_id, "delete_user_notifications");match MacroUserIdStr::parse_from_str(&user_id) {Ok(macro_user_id) => {- if let Err(e) = notification_repo+ match notification_repo.delete_all_user_notifications(macro_user_id).await{- tracing::error!(error=?e, user_id, "unable to delete user notifications");+ Ok(_) => tracing::info!(user_id, "deleted user notifications"),+ Err(e) => {+ tracing::error!(error=?e, user_id, "unable to delete user notifications");+ }}}Err(e) => {- tracing::error!(error=?e, user_id, "unable to parse user id");+ tracing::error!(error=?e, user_id, "unable to parse user id for notification cleanup");}}- tracing::trace!(user_id, "delete_user_notifications complete");}}}@@ -232,6 +286,7 @@ async fn delete_user(.iter().map(|id| MacroUserIdStr::try_from(id.clone())).collect::<Result<Vec<_>, _>>()?;+ let start_time = std::time::Instant::now();crate::service::user::delete_user::delete_user_data(ctx.user_deletion.as_ref(),¯o_user.id,@@ -239,16 +294,22 @@ async fn delete_user().await.map_err(|error| anyhow::anyhow!("{error:?}"))?;+ tracing::info!(+ elapsed_ms = start_time.elapsed().as_millis() as u64,+ "deleted user data, profiles, billing customer, and account"+ );Ok(())}-/// Removes a user from all organization channels+/// Removes a user from all organization channels.+///+/// Returns the number of organization channels the user was removed from.async fn remove_user_from_org_channels(db: &Pool<Postgres>,user_id: &str,org_id: i64,-) -> anyhow::Result<()> {+) -> anyhow::Result<usize> {let org_channels = get_org_channels(db, &org_id).await?;for channel in org_channels.iter() {@@ -262,5 +323,5 @@ async fn remove_user_from_org_channels(.await?;}- Ok(())+ Ok(org_channels.len())}diff --git a/services/authentication_service/src/outbound/user_deletion.rs b/services/authentication_service/src/outbound/user_deletion.rsindex fccef4de22..f8f5f472a0 100644--- a/services/authentication_service/src/outbound/user_deletion.rs+++ b/services/authentication_service/src/outbound/user_deletion.rs@@ -67,6 +67,7 @@ impl<T: TeamService, O: OnboardingRepo> UserDeletionAdapter<T, O> {})}+ #[tracing::instrument(skip(self, base, user), fields(user = %user), err)]async fn delete_owned(&self,base: &str,@@ -85,9 +86,11 @@ impl<T: TeamService, O: OnboardingRepo> UserDeletionAdapter<T, O> {.send().await.context_with(|| format!("failed to delete user's {resource}"))?;- if response.status() != reqwest::StatusCode::NO_CONTENT {+ let status = response.status();+ if status != reqwest::StatusCode::NO_CONTENT {+ tracing::error!(%status, "owning service rejected user cleanup request");return Err(rootcause::report!("user cleanup did not complete")- .attach(format!("{resource}: {}", response.status())));+ .attach(format!("{resource}: {status}")));}Ok(())}@@ -104,6 +107,7 @@ impl<T: TeamService, O: OnboardingRepo> UserDeletionGateway for UserDeletionAdap.await}+ #[tracing::instrument(skip(self, user), fields(user = %user), err)]async fn leave_teams(&self, user: &MacroUserIdStr<'static>) -> Result<(), Report> {self.teams.remove_user_from_all_teams(user)@@ -113,6 +117,7 @@ impl<T: TeamService, O: OnboardingRepo> UserDeletionGateway for UserDeletionAdapOk(())}+ #[tracing::instrument(skip(self, user), fields(user = %user), err)]async fn delete_items(&self, user: &MacroUserIdStr<'static>) -> Result<(), Report> {self.documents.delete_all_user_items(user.as_ref())@@ -122,6 +127,7 @@ impl<T: TeamService, O: OnboardingRepo> UserDeletionGateway for UserDeletionAdapOk(())}+ #[tracing::instrument(skip(self, user), fields(user = %user), err)]async fn delete_onboarding(&self, user: &MacroUserIdStr<'static>) -> Result<(), Report> {self.onboarding.delete_row(user)@@ -131,17 +137,21 @@ impl<T: TeamService, O: OnboardingRepo> UserDeletionGateway for UserDeletionAdapOk(())}+ #[tracing::instrument(skip(self, user), fields(user = %user), err)]async fn delete_profile(&self,user: &MacroUserIdStr<'static>,account: &Uuid,) -> Result<(), Report> {- macro_db_client::user::delete_user::delete_user(&self.db, user.as_ref(), account)- .await- .context("failed to delete user profile")?;+ let deleted =+ macro_db_client::user::delete_user::delete_user(&self.db, user.as_ref(), account)+ .await+ .context("failed to delete user profile")?;+ tracing::info!(organization_id = ?deleted.organization_id, "deleted user profile row");Ok(())}+ #[tracing::instrument(skip(self), err)]async fn delete_billing_customer(&self, account: &Uuid) -> Result<(), Report> {let customer_id =match macro_db_client::macro_user::get_macro_user(&self.db, &account.to_string()).await@@ -154,12 +164,14 @@ impl<T: TeamService, O: OnboardingRepo> UserDeletionGateway for UserDeletionAdapSome(sqlx::Error::RowNotFound)) =>{+ tracing::info!("account row already gone; no billing customer to delete");None}Err(error) => Err(error.into_rootcause()).context("failed to load account for billing cleanup")?,};let Some(customer_id) = customer_id else {+ tracing::info!("account has no billing customer; skipping customer deletion");return Ok(());};let customer_id: stripe::CustomerId = match customer_id.parse() {@@ -177,12 +189,15 @@ impl<T: TeamService, O: OnboardingRepo> UserDeletionGateway for UserDeletionAdapstripe::Customer::delete(&self.stripe, &customer_id).await- .map(drop)+ .map(|_| {+ tracing::info!(stripe_customer_id = %customer_id, "deleted stripe customer");+ }).or_else(|error| match error {// Already deleted, for example by an earlier attempt.stripe::StripeError::Stripe(request)if matches!(request.code, Some(stripe::ErrorCode::ResourceMissing)) =>{+ tracing::info!(stripe_customer_id = %customer_id, "stripe customer already deleted");Ok(())}error => Err(Report::new(error).into_dynamic()),@@ -191,6 +206,7 @@ impl<T: TeamService, O: OnboardingRepo> UserDeletionGateway for UserDeletionAdapOk(())}+ #[tracing::instrument(skip(self), err)]async fn delete_account(&self, account: &Uuid) -> Result<(), Report> {macro_db_client::macro_user::delete_macro_user(&self.db, account).awaitdiff --git a/services/authentication_service/src/service/user/create_user.rs b/services/authentication_service/src/service/user/create_user.rsindex ede1b35403..609a23f6ce 100644--- a/services/authentication_service/src/service/user/create_user.rs+++ b/services/authentication_service/src/service/user/create_user.rs@@ -30,7 +30,7 @@ fn local_stripe_customer_id(email: &str) -> String {/// multiple easy-to-follow functions////// Returns the (user_id, Option<organization_id>)-#[tracing::instrument(skip(db, stripe_client))]+#[tracing::instrument(skip(db, stripe_client), err)]pub async fn create_user(fusionauth_user_id: &str,username: &str,@@ -42,12 +42,18 @@ pub async fn create_user(let stripe_customer_id = if is_local_stripe_stub() {// Local mode uses a stub key; don't call Stripe (it would 401 and// abort the transactional user.create webhook, blocking all signups).+ tracing::info!("skipping Stripe customer creation for the local Stripe stub");None} else {// NOTE: stripe adds in ~400ms of latency to this request. We may want to update our// requirement that each customer exists in stripe and create stripe customers as needed.+ let start_time = std::time::Instant::now();let stripe_customer = create_stripe_user(email, stripe_client).await?;- tracing::trace!(stripe_customer_id=?stripe_customer.id.to_string(), "created stripe customer");+ tracing::info!(+ stripe_customer_id = %stripe_customer.id,+ elapsed_ms = start_time.elapsed().as_millis() as u64,+ "created Stripe customer"+ );Some(stripe_customer.id.to_string())};// The legacy account key is required, but a profile without a real Stripe@@ -57,7 +63,6 @@ pub async fn create_user(.unwrap_or_else(|| local_stripe_customer_id(email));let organization_id = match_user_to_organization(db, email).await?;- tracing::trace!(organization_id=?organization_id, "matched user to organization");let roles = match organization_id {Some(organization_id) => {@@ -66,7 +71,7 @@ pub async fn create_user(None => ["self_serve".to_string()].into_iter().collect(),};- tracing::trace!(roles=?roles, "got roles for user");+ tracing::info!(organization_id=?organization_id, roles=?roles, "resolved organization and roles for new user");// Create user in macrodblet user_id = macro_db_client::user::create_user::create_user(@@ -82,12 +87,13 @@ pub async fn create_user().await?;+ tracing::info!(user_id, "created macro user and user profile in macrodb");+if organization_id.is_some() {macro_db_client::organization::delete_organization_invitation(db.clone(), email).await?;+ tracing::info!(organization_id=?organization_id, "deleted pending organization invitation");}- tracing::trace!("created user in macrodb");-Ok((user_id, organization_id))}@@ -95,7 +101,7 @@ pub async fn create_user(/// Note: This does not check if a user with the same email already exists./// We can safely assume that this is not the case when we create a new user as the webhook is only/// run for new users.-#[tracing::instrument(skip(stripe_client))]+#[tracing::instrument(skip(stripe_client), err)]pub async fn create_stripe_user(email: &str,stripe_client: &stripe::Client,diff --git a/services/authentication_service/src/service/user/delete_user.rs b/services/authentication_service/src/service/user/delete_user.rsindex 796c4463f1..8d08796a51 100644--- a/services/authentication_service/src/service/user/delete_user.rs+++ b/services/authentication_service/src/service/user/delete_user.rs@@ -58,6 +58,10 @@ pub trait UserDeletionGateway: Send + Sync {/// Cleanup is retryable: failures preserve the profile/account and never fall/// through to a cascading delete. Callers must await this and report failure.+///+/// Each completed step is logged so a failed or retried deletion can be traced+/// to the exact step it reached.+#[tracing::instrument(skip(gateway, users), fields(profile_count = users.len()), err)]pub async fn delete_user_data(gateway: &impl UserDeletionGateway,account: &Uuid,@@ -65,12 +69,21 @@ pub async fn delete_user_data() -> Result<(), Report> {for user in users {gateway.delete_scheduled_actions(user).await?;+ tracing::info!(%user, "deleted scheduled actions");gateway.delete_agent_sessions(user).await?;+ tracing::info!(%user, "deleted agent sessions");gateway.leave_teams(user).await?;+ tracing::info!(%user, "left and cleaned up teams");gateway.delete_items(user).await?;+ tracing::info!(%user, "deleted documents, chats, and projects");gateway.delete_onboarding(user).await?;+ tracing::info!(%user, "deleted onboarding state");gateway.delete_profile(user, account).await?;+ tracing::info!(%user, "deleted user profile");}gateway.delete_billing_customer(account).await?;- gateway.delete_account(account).await+ tracing::info!("deleted billing customer");+ gateway.delete_account(account).await?;+ tracing::info!("deleted account");+ Ok(())}