Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(calendar_service): notify inbox reauth on backfill grant failure (#6706)
gbirman3c38cc956098
diff --git a/Cargo.lock b/Cargo.lockindex 04426045ec..baf1fbde86 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3259,6 +3259,7 @@ dependencies = ["models_email","redis","reqwest 0.13.4",+ "rootcause","secretsmanager_client","serde","serde_json",diff --git a/crates/calendar_events/src/domain/ports.rs b/crates/calendar_events/src/domain/ports.rsindex 82439be601..a631bc07ee 100644--- a/crates/calendar_events/src/domain/ports.rs+++ b/crates/calendar_events/src/domain/ports.rs@@ -625,6 +625,21 @@ pub trait CalendarRefreshNotifier: Send + Sync + 'static {) -> impl Future<Output = ()> + Send;}+/// Outbound port that announces one connected inbox's Google grant has gone+/// dead and needs reconnection. The notification itself — reconnect-your-inbox+/// email, `email.link_reauth_required` topic event, activity row — is owned by+/// email_service's link-manager consumer; calendar_service backs this port by+/// enqueuing onto the shared link-manager queue that consumer already drains.+/// Best effort at the edge: the announcer logs and swallows a delivery failure,+/// since the failure the notification describes is already durably recorded.+pub trait CalendarReauthNotifier: Send + Sync + 'static {+ /// Announce that `email_link_id`'s grant requires reauthorization.+ fn notify_reauth_required(+ &self,+ email_link_id: Uuid,+ ) -> impl Future<Output = Result<(), Report>> + Send;+}+/// Inbound service port for user-initiated calendar event mutations.pub trait CalendarMutationService: Send + Sync + 'static {/// Create an event on the selected calendar — or the requester's (ordiff --git a/crates/calendar_events/src/domain/service.rs b/crates/calendar_events/src/domain/service.rsindex 2b8ef9b98f..0dc73bb84b 100644--- a/crates/calendar_events/src/domain/service.rs+++ b/crates/calendar_events/src/domain/service.rs@@ -17,9 +17,9 @@ use super::{},ports::{CalendarBackfillRepository, CalendarEventChange, CalendarEventWrite,- CalendarEventWriteOutcome, CalendarOccurrenceService, CalendarRepository,- GoogleCalendarProvider, GoogleCalendarSyncRepository, GoogleEventSyncContext,- GoogleProviderError, GoogleProviderErrorKind, RetiredCalendarEvent,+ CalendarEventWriteOutcome, CalendarOccurrenceService, CalendarReauthNotifier,+ CalendarRepository, GoogleCalendarProvider, GoogleCalendarSyncRepository,+ GoogleEventSyncContext, GoogleProviderError, GoogleProviderErrorKind, RetiredCalendarEvent,},};@@ -542,6 +542,78 @@ where}}+/// Fires the reauth-required notification on exactly the healthy-to-reauth edge+/// a backfill failure consumed, and never on any other failure.+///+/// The edge is decided in the failure transaction, not here:+/// `link_reauth_transitioned` is `true` only when this failure was the one that+/// first flipped the inbox into needs-reauth, which only the `ReauthRequired`+/// disposition does — a `CalendarPermissionRequired` failure never sets it, so+/// a missing calendar scope on an otherwise healthy Gmail grant stays quiet.+/// This service owns only the policy that such a consumed edge is what warrants+/// a notification, forwarding it to the notifier port and leaving the single+/// notification implementation to email_service's link-manager consumer.+///+/// Best effort: a notifier failure is logged and swallowed. The failure the+/// notification describes is already durably recorded, so the caller's delivery+/// must still ack; a lost enqueue means that link never notifies, matching+/// email_service's own calendar branch rather than diverging from it.+pub struct CalendarReauthAnnouncer<N> {+ notifier: N,+}++impl<N> CalendarReauthAnnouncer<N>+where+ N: CalendarReauthNotifier,+{+ /// Construct the announcer over its notifier port.+ pub fn new(notifier: N) -> Self {+ Self { notifier }+ }++ /// Announce after an unclaimed job's terminal failure, iff that failure+ /// newly transitioned the inbox into needs-reauth.+ pub async fn announce_unclaimed(+ &self,+ email_link_id: Uuid,+ outcome: &CalendarBackfillFailureOutcome,+ ) {+ if outcome.link_reauth_transitioned {+ self.fire(email_link_id).await;+ }+ }++ /// Announce after a fenced run failed, iff the failure was a grant+ /// reauthorization failure that newly transitioned the inbox into+ /// needs-reauth. Every other run error — including a `ReauthRequired`+ /// carrying `link_reauth_transitioned: false`, which a+ /// `CalendarPermissionRequired` disposition or a lost race produces — stays+ /// quiet.+ pub async fn announce_run_error(+ &self,+ email_link_id: Uuid,+ error: &GoogleCalendarBackfillRunError,+ ) {+ if let GoogleCalendarBackfillRunError::ReauthRequired {+ link_reauth_transitioned: true,+ ..+ } = error+ {+ self.fire(email_link_id).await;+ }+ }++ async fn fire(&self, email_link_id: Uuid) {+ if let Err(error) = self.notifier.notify_reauth_required(email_link_id).await {+ tracing::warn!(+ ?error,+ %email_link_id,+ "failed to enqueue calendar reauth-required notification"+ );+ }+ }+}+impl<R, G, B> GoogleCalendarBackfillService<R, G, B>whereR: CalendarRepository,diff --git a/crates/calendar_events/src/domain/service/test.rs b/crates/calendar_events/src/domain/service/test.rsindex cf6e125ec8..3dab0bd994 100644--- a/crates/calendar_events/src/domain/service/test.rs+++ b/crates/calendar_events/src/domain/service/test.rs@@ -3,17 +3,17 @@ use crate::domain::ports::{CalendarEventChange, CalendarEventWriteOutcome, Retiruse crate::domain::{models::{AttendeeResponseStatus, CalendarAttendee, CalendarBackfillClaim,- CalendarBackfillFailureDisposition, CalendarBackfillJobKey, CalendarCreationTarget,- CalendarEvent, CalendarEventMutationTarget, CalendarEventSource, CalendarOccurrence,- CalendarSyncStatus, DisconnectedGoogleCalendar, EventReminders, EventStatus, EventTime,- EventTransparency, EventType, EventVisibility, GOOGLE_CALENDAR_FULL_SCOPE,- GOOGLE_CALENDAR_SCOPES, GoogleBackfillRunReport, GoogleCalendarSyncSnapshot,- GoogleEventSource, GoogleEventSyncBatch, GoogleWatchChannel, GoogleWatchConfig,- ProviderCalendar, StoredGoogleCalendar,+ CalendarBackfillFailureDisposition, CalendarBackfillFailureOutcome, CalendarBackfillJobKey,+ CalendarCreationTarget, CalendarEvent, CalendarEventMutationTarget, CalendarEventSource,+ CalendarOccurrence, CalendarSyncStatus, DisconnectedGoogleCalendar, EventReminders,+ EventStatus, EventTime, EventTransparency, EventType, EventVisibility,+ GOOGLE_CALENDAR_FULL_SCOPE, GOOGLE_CALENDAR_SCOPES, GoogleBackfillRunReport,+ GoogleCalendarSyncSnapshot, GoogleEventSource, GoogleEventSyncBatch, GoogleWatchChannel,+ GoogleWatchConfig, ProviderCalendar, StoredGoogleCalendar,},ports::{- CalendarBackfillRepository, CalendarEventWrite, CalendarRepository, GoogleCalendarProvider,- GoogleEventSyncContext, GoogleProviderError,+ CalendarBackfillRepository, CalendarEventWrite, CalendarReauthNotifier, CalendarRepository,+ GoogleCalendarProvider, GoogleEventSyncContext, GoogleProviderError,},};use chrono::{TimeZone, Utc};@@ -1335,3 +1335,132 @@ async fn google_coordinator_keeps_calendar_permission_health_separate_from_gmail&[CalendarBackfillFailureDisposition::CalendarPermissionRequired]);}++#[derive(Clone, Default)]+struct FakeReauthNotifier {+ notified: Arc<Mutex<Vec<Uuid>>>,+ fail: bool,+}++impl FakeReauthNotifier {+ fn failing() -> Self {+ Self {+ notified: Arc::default(),+ fail: true,+ }+ }+}++impl CalendarReauthNotifier for FakeReauthNotifier {+ async fn notify_reauth_required(&self, email_link_id: Uuid) -> Result<(), Report> {+ self.notified.lock().unwrap().push(email_link_id);+ if self.fail {+ return Err(rootcause::report!("reauth notifier unavailable"));+ }+ Ok(())+ }+}++fn reauth_outcome(link_reauth_transitioned: bool) -> CalendarBackfillFailureOutcome {+ CalendarBackfillFailureOutcome {+ job_transitioned: true,+ link_reauth_transitioned,+ }+}++#[tokio::test]+async fn reauth_announcer_fires_when_unclaimed_failure_consumed_the_edge() {+ let notifier = FakeReauthNotifier::default();+ let announcer = CalendarReauthAnnouncer::new(notifier.clone());+ let link = Uuid::now_v7();++ announcer+ .announce_unclaimed(link, &reauth_outcome(true))+ .await;++ assert_eq!(notifier.notified.lock().unwrap().as_slice(), &[link]);+}++#[tokio::test]+async fn reauth_announcer_is_silent_when_unclaimed_failure_did_not_transition() {+ let notifier = FakeReauthNotifier::default();+ let announcer = CalendarReauthAnnouncer::new(notifier.clone());++ announcer+ .announce_unclaimed(Uuid::now_v7(), &reauth_outcome(false))+ .await;++ assert!(notifier.notified.lock().unwrap().is_empty());+}++#[tokio::test]+async fn reauth_announcer_fires_when_run_error_consumed_the_edge() {+ let notifier = FakeReauthNotifier::default();+ let announcer = CalendarReauthAnnouncer::new(notifier.clone());+ let link = Uuid::now_v7();++ announcer+ .announce_run_error(+ link,+ &GoogleCalendarBackfillRunError::ReauthRequired {+ message: "grant revoked".to_string(),+ link_reauth_transitioned: true,+ },+ )+ .await;++ assert_eq!(notifier.notified.lock().unwrap().as_slice(), &[link]);+}++#[tokio::test]+async fn reauth_announcer_is_silent_on_calendar_permission_required() {+ // A `CalendarPermissionRequired` disposition surfaces as a `ReauthRequired`+ // run error carrying `link_reauth_transitioned: false`: the calendar scope+ // is missing but the Gmail grant is healthy, so no inbox edge was consumed.+ let notifier = FakeReauthNotifier::default();+ let announcer = CalendarReauthAnnouncer::new(notifier.clone());++ announcer+ .announce_run_error(+ Uuid::now_v7(),+ &GoogleCalendarBackfillRunError::ReauthRequired {+ message: "calendar scope missing".to_string(),+ link_reauth_transitioned: false,+ },+ )+ .await;++ assert!(notifier.notified.lock().unwrap().is_empty());+}++#[tokio::test]+async fn reauth_announcer_is_silent_on_non_reauth_run_errors() {+ let notifier = FakeReauthNotifier::default();+ let announcer = CalendarReauthAnnouncer::new(notifier.clone());+ let link = Uuid::now_v7();++ for error in [+ GoogleCalendarBackfillRunError::Permanent("permanent".to_string()),+ GoogleCalendarBackfillRunError::Retryable("transient".to_string()),+ GoogleCalendarBackfillRunError::LeaseLost,+ ] {+ announcer.announce_run_error(link, &error).await;+ }++ assert!(notifier.notified.lock().unwrap().is_empty());+}++#[tokio::test]+async fn reauth_announcer_swallows_a_failing_notifier() {+ let notifier = FakeReauthNotifier::failing();+ let announcer = CalendarReauthAnnouncer::new(notifier.clone());+ let link = Uuid::now_v7();++ // A notifier failure must not surface: the DB edge is already consumed and+ // the caller's delivery must still ack.+ announcer+ .announce_unclaimed(link, &reauth_outcome(true))+ .await;++ assert_eq!(notifier.notified.lock().unwrap().as_slice(), &[link]);+}diff --git a/infra/stacks/calendar-service/index.ts b/infra/stacks/calendar-service/index.tsindex f19bd49917..0109ae51d6 100644--- a/infra/stacks/calendar-service/index.ts+++ b/infra/stacks/calendar-service/index.ts@@ -4,6 +4,7 @@ import { Queue, Redis } from '../../packages/resources';import {BASE_DOMAIN,config,+ getLinkManagerQueue,getMacroApiToken,stack,} from '../../packages/shared';@@ -91,6 +92,11 @@ const backfillQueue = new Queue('calendar-service-backfill', {export const calendarBackfillQueueArn = pulumi.interpolate`${backfillQueue.queue.arn}`;export const calendarBackfillQueueName = pulumi.interpolate`${backfillQueue.queue.name}`;+// Reauth-required notifications are enqueued onto email-service's shared+// link-manager queue, whose consumer owns the reconnect-your-inbox+// notification. calendar-service is granted send-only access to it.+const { linkManagerQueueArn } = getLinkManagerQueue();+const secretKeyArns = [authenticationServiceInternalApiKeyArn,jwtSecretKeyArn,@@ -117,6 +123,7 @@ const containerEnvVars = [const calendarService = new CalendarService('calendar-service', {calendarBackfillQueueArn,+ linkManagerQueueArn,vpc: coparse_api_vpc,tags,containerEnvVars,diff --git a/infra/stacks/calendar-service/service.ts b/infra/stacks/calendar-service/service.tsindex 332a7c42a0..f6516351ce 100644--- a/infra/stacks/calendar-service/service.ts+++ b/infra/stacks/calendar-service/service.ts@@ -27,6 +27,7 @@ const REPO_ROOT = '../../..';type CreateCalendarServiceArgs = {calendarBackfillQueueArn: pulumi.Output<string> | string;+ linkManagerQueueArn: pulumi.Output<string> | string;vpc: {vpcId: pulumi.Output<string> | string;privateSubnetIds: pulumi.Output<string[]> | string[];@@ -55,6 +56,7 @@ export class CalendarService extends pulumi.ComponentResource {name: string,{calendarBackfillQueueArn,+ linkManagerQueueArn,vpc,tags,platform,@@ -112,6 +114,11 @@ export class CalendarService extends pulumi.ComponentResource {Resource: [pulumi.interpolate`${calendarBackfillQueueArn}`],Effect: 'Allow',},+ {+ Action: ['sqs:SendMessage'],+ Resource: [pulumi.interpolate`${linkManagerQueueArn}`],+ Effect: 'Allow',+ },],},tags: this.tags,diff --git a/services/calendar_service/Cargo.toml b/services/calendar_service/Cargo.tomlindex 98431e6654..b8319b0b1f 100644--- a/services/calendar_service/Cargo.toml+++ b/services/calendar_service/Cargo.toml@@ -50,6 +50,7 @@ macro_service_urls = { path = "../../crates/macro_service_urls" }models_email = { path = "../../crates/models_email" }redis = { workspace = true, features = ["tokio-rustls-comp"] }reqwest = { workspace = true }+rootcause = { workspace = true }secretsmanager_client = { path = "../../crates/secretsmanager_client" }serde = { workspace = true }serde_json = { workspace = true }diff --git a/services/calendar_service/src/calendar_backfill.rs b/services/calendar_service/src/calendar_backfill.rsindex a79f5a5efc..7ce59e6325 100644--- a/services/calendar_service/src/calendar_backfill.rs+++ b/services/calendar_service/src/calendar_backfill.rs@@ -15,8 +15,8 @@ use calendar_events::domain::{GoogleWatchConfig, OccurrenceRange,},service::{- GoogleCalendarBackfillCoordinator, GoogleCalendarBackfillFailureService,- GoogleCalendarBackfillRunError,+ CalendarReauthAnnouncer, GoogleCalendarBackfillCoordinator,+ GoogleCalendarBackfillFailureService, GoogleCalendarBackfillRunError,},};use calendar_events::outbound::{google::GoogleCalendarClient, pg::PgCalendarRepository};@@ -33,6 +33,7 @@ use crate::backfill_queue::CalendarBackfillMessage;use crate::calendar_backfill_adapters::RedisCalendarRequestGate;use crate::calendar_outbox::republish_calendar_job;use crate::calendar_ratelimit::CalendarRateLimiter;+use crate::calendar_reauth::LinkManagerReauthNotifier;use crate::pubsub_util::cg_refresh_calendar;/// The event broker used by the backfill coordinator.@@ -50,6 +51,9 @@ type GoogleCalendarBackfillService = GoogleCalendarBackfillCoordinator<type GoogleCalendarBackfillFailureHandler =GoogleCalendarBackfillFailureService<PgCalendarRepository>;+/// Concrete reauth-required announcer for this service.+type CalendarReauth = CalendarReauthAnnouncer<LinkManagerReauthNotifier>;+/// Everything a backfill worker needs to process one calendar delivery.#[derive(Clone)]pub struct CalendarBackfillContext {@@ -60,6 +64,7 @@ pub struct CalendarBackfillContext {connection_gateway_client: connection_gateway_client::client::ConnectionGatewayClient,coordinator: Arc<GoogleCalendarBackfillService>,failure: Arc<GoogleCalendarBackfillFailureHandler>,+ reauth: Arc<CalendarReauth>,calendar_sync_enabled: bool,}@@ -74,6 +79,7 @@ impl CalendarBackfillContext {auth_service_client: AuthServiceClient,connection_gateway_client: connection_gateway_client::client::ConnectionGatewayClient,macro_event_broker: CalendarEventBroker,+ reauth_notifier: LinkManagerReauthNotifier,watch: Option<GoogleWatchConfig>,calendar_sync_enabled: bool,) -> Self {@@ -92,6 +98,7 @@ impl CalendarBackfillContext {watch,));let failure = Arc::new(GoogleCalendarBackfillFailureService::new(repository));+ let reauth = Arc::new(CalendarReauthAnnouncer::new(reauth_notifier));Self {db,sqs_worker,@@ -100,6 +107,7 @@ impl CalendarBackfillContext {connection_gateway_client,coordinator,failure,+ reauth,calendar_sync_enabled,}}@@ -318,6 +326,7 @@ async fn run_coordinator(match run_result {Ok(()) => Disposition::Ack,Err(error) => {+ ctx.reauth.announce_run_error(link.id, &error).await;let retryable = matches!(error,GoogleCalendarBackfillRunError::Busy@@ -358,7 +367,10 @@ async fn fail_unclaimed().await{- Ok(_) => Disposition::Ack,+ Ok(outcome) => {+ ctx.reauth.announce_unclaimed(link_id, &outcome).await;+ Disposition::Ack+ }Err(error) => {tracing::error!(error = ?error, %calendar_job_id, "failed to record terminal calendar backfill failure");Disposition::Retry(anyhow::anyhow!(diff --git a/services/calendar_service/src/calendar_reauth.rs b/services/calendar_service/src/calendar_reauth.rsnew file mode 100644index 0000000000..b68eaee1ad--- /dev/null+++ b/services/calendar_service/src/calendar_reauth.rs@@ -0,0 +1,80 @@+//! Link-manager queue adapter that announces a dead Google grant.+//!+//! Backs [`CalendarReauthNotifier`] by enqueuing+//! [`LinkManagerMessage::NotifyReauthRequired`] onto the shared email+//! link-manager queue, whose email_service consumer owns the single+//! reconnect-your-inbox notification. Enqueued directly rather than through+//! `sqs_client`, whose `email` feature would pull `crates/email` back into+//! this service; the wire body is the shared `models_email` serde impl, so it+//! stays byte-compatible with the messages email_service enqueues.++use calendar_events::domain::ports::CalendarReauthNotifier;+use models_email::email::service::pubsub::LinkManagerMessage;+use rootcause::Report;+use uuid::Uuid;++/// [`CalendarReauthNotifier`] backed by the email link-manager SQS queue.+#[derive(Clone)]+pub struct LinkManagerReauthNotifier {+ client: aws_sdk_sqs::Client,+ queue_url: String,+}++impl LinkManagerReauthNotifier {+ /// Construct the notifier over an SQS client and the resolved queue URL.+ pub fn new(client: aws_sdk_sqs::Client, queue_url: impl Into<String>) -> Self {+ Self {+ client,+ queue_url: queue_url.into(),+ }+ }+}++impl CalendarReauthNotifier for LinkManagerReauthNotifier {+ #[tracing::instrument(skip(self))]+ async fn notify_reauth_required(&self, email_link_id: Uuid) -> Result<(), Report> {+ let body = encode_reauth_message(email_link_id).map_err(report)?;+ self.client+ .send_message()+ .queue_url(&self.queue_url)+ .message_body(body)+ .send()+ .await+ .map_err(report)?;+ Ok(())+ }+}++fn encode_reauth_message(email_link_id: Uuid) -> serde_json::Result<String> {+ serde_json::to_string(&LinkManagerMessage::NotifyReauthRequired {+ link_id: email_link_id,+ })+}++fn report(error: impl std::error::Error + Send + Sync + 'static) -> Report {+ rootcause::report!(error).into()+}++#[cfg(test)]+mod test {+ use super::*;++ #[test]+ fn notify_reauth_required_wire_body_round_trips_as_link_manager_message() {+ let link_id = Uuid::now_v7();++ let body = encode_reauth_message(link_id).expect("message serializes");++ assert_eq!(+ body,+ format!(r#"{{"operation":"NotifyReauthRequired","link_id":"{link_id}"}}"#)+ );++ let decoded: LinkManagerMessage =+ serde_json::from_str(&body).expect("email_service deserializes the same bytes");+ assert!(matches!(+ decoded,+ LinkManagerMessage::NotifyReauthRequired { link_id: decoded_id } if decoded_id == link_id+ ));+ }+}diff --git a/services/calendar_service/src/lib.rs b/services/calendar_service/src/lib.rsindex bd2d197f4b..fb5fbfb913 100644--- a/services/calendar_service/src/lib.rs+++ b/services/calendar_service/src/lib.rs@@ -17,6 +17,8 @@ pub mod calendar_backfill_adapters;pub mod calendar_outbox;/// Redis-backed Google Calendar API quota gate.pub mod calendar_ratelimit;+/// Link-manager queue adapter for reauth-required notifications.+pub mod calendar_reauth;/// Connection-gateway calendar refresh notifier.pub mod calendar_refresh;/// Access-token adapter for user-initiated calendar mutations.diff --git a/services/calendar_service/src/main.rs b/services/calendar_service/src/main.rsindex 20435888c2..ca30c1a3a9 100644--- a/services/calendar_service/src/main.rs+++ b/services/calendar_service/src/main.rs@@ -12,6 +12,7 @@ use calendar_service::backfill_queue::CalendarBackfillQueueClient;use calendar_service::calendar_backfill::CalendarBackfillContext;use calendar_service::calendar_backfill_adapters::RedisCalendarRequestGate;use calendar_service::calendar_ratelimit::CalendarRateLimiter;+use calendar_service::calendar_reauth::LinkManagerReauthNotifier;use calendar_service::calendar_refresh::ConnectionGatewayCalendarRefresh;use calendar_service::calendar_tokens::CalendarTokenProviderAdapter;use calendar_service::config::{Config, calendar_watch_config};@@ -109,6 +110,15 @@ async fn main() -> anyhow::Result<()> {let rate_limiter =CalendarRateLimiter::new(redis_inner_client, config.redis_rate_limit_window_secs);+ // Reauth-required notifications ride the shared email link-manager queue,+ // whose email_service consumer owns the single reconnect-your-inbox+ // notification; the queue URL resolves from per-environment defaults.+ let link_manager_queue = macro_queues::LinkManagerQueue::new();+ let reauth_notifier = LinkManagerReauthNotifier::new(+ aws_sdk_sqs::Client::new(&aws_config),+ link_manager_queue.to_string(),+ );+// Sync scheduler + calendar outbox drain.worker_tracker.spawn(calendar_service::calendar_outbox::run(db.clone(),@@ -136,6 +146,7 @@ async fn main() -> anyhow::Result<()> {auth_service_client.clone(),connection_gateway_client.clone(),macro_event_broker.clone(),+ reauth_notifier.clone(),calendar_watch_config(),config.calendar_sync_enabled,);