Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(calendar): log Outlook sync error changes for calendar caches (#9633)
gbirmandd82f4b61cc2
diff --git a/.sqlx/query-1efb2374db84fd648f2f43e00ca122eecab96b69a7bc0128e920a2b467c21da1.json b/.sqlx/query-1efb2374db84fd648f2f43e00ca122eecab96b69a7bc0128e920a2b467c21da1.jsondeleted file mode 100644index c4fec0d6db..0000000000--- a/.sqlx/query-1efb2374db84fd648f2f43e00ca122eecab96b69a7bc0128e920a2b467c21da1.json+++ /dev/null@@ -1,15 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "UPDATE calendars SET last_sync_error=$2,last_sync_error_at=now(),consecutive_sync_failures=consecutive_sync_failures+1 WHERE id=$1",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Uuid",- "Text"- ]- },- "nullable": []- },- "hash": "1efb2374db84fd648f2f43e00ca122eecab96b69a7bc0128e920a2b467c21da1"-}diff --git a/.sqlx/query-599a42d00923771e0fb8a6a36d845dba18b117b256392e6ad5b28c9f5ac143d4.json b/.sqlx/query-599a42d00923771e0fb8a6a36d845dba18b117b256392e6ad5b28c9f5ac143d4.jsonnew file mode 100644index 0000000000..d57f1c9cf5--- /dev/null+++ b/.sqlx/query-599a42d00923771e0fb8a6a36d845dba18b117b256392e6ad5b28c9f5ac143d4.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE calendars SET last_sync_error=$2,last_sync_error_at=now(),consecutive_sync_failures=consecutive_sync_failures+1 WHERE id=$1 RETURNING consecutive_sync_failures",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "consecutive_sync_failures",+ "type_info": "Int4"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Text"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "599a42d00923771e0fb8a6a36d845dba18b117b256392e6ad5b28c9f5ac143d4"+}diff --git a/crates/calendar_events/AGENTS.md b/crates/calendar_events/AGENTS.mdindex 24e8a9a813..8da946cc6b 100644--- a/crates/calendar_events/AGENTS.md+++ b/crates/calendar_events/AGENTS.md@@ -377,6 +377,17 @@ reminder dispatch, team availability, and provider/Postgres persistence.(reminders and automatic declines anchor all-day dates at UTC, teamavailability reports the zone missing). The warning `ignoring an Outlookmailbox time zone Macro cannot map` and the span carry `unmapped_time_zone`.+- A calendar's sync error reaches calendar caches only as an `UpsertCalendar`+ change-log entry, since they read the calendar list once and then follow+ the log. Google and Outlook log one when the shown error+ (`visible_calendar_sync_error_tx`, past+ `CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD` failures) appears or is reworded+ (`failure_changes_sync_error_badge`) and when the round that clears it+ commits, never on a routine round (Outlook polls every two minutes).+ Neither nudges open clients for it, so a tab sees it on its next delta.+ Outlook logged none before November 2026;+ `20261105002046_log_outlook_calendars_for_their_sync_error.sql` logs every+ visible Outlook calendar once so caches that missed a change refetch it.- A sync of a `ready`, already synced account stays `ready`, on Google as onOutlook, including after a re-grant that kept calendar access, whose sync tokenssurvive. Any other state is catching up and marks it `syncing`: a first import,diff --git a/crates/calendar_events/src/outbound/pg.rs b/crates/calendar_events/src/outbound/pg.rsindex 88a80875ab..cf4cedf83c 100644--- a/crates/calendar_events/src/outbound/pg.rs+++ b/crates/calendar_events/src/outbound/pg.rs@@ -2003,9 +2003,9 @@ impl CalendarRepository for PgCalendarRepository {// Clients see a calendar's error only once it crosses the badge// threshold, so only a newly shown or reworded badge is a change.let mut log = ChangeLogBatch::default();- let badged_after =- failures.is_some_and(|failures| failures >= CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD);- if badged_after && badged_before.as_deref() != Some(message) {+ if failures.is_some_and(|failures| {+ failure_changes_sync_error_badge(badged_before.as_deref(), failures, message)+ }) {log.record(key.email_link_id,CalendarChangeOp::UpsertCalendar(calendar_id),@@ -3176,6 +3176,16 @@ async fn visible_calendar_sync_error_tx(}))}+/// Whether a failure stored as `message`, leaving `failures` in a row, newly+/// shows or rewords the badge that `shown_before` held.+fn failure_changes_sync_error_badge(+ shown_before: Option<&str>,+ failures: i32,+ message: &str,+) -> bool {+ failures >= CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD && shown_before != Some(message)+}+fn stored_google_calendar(row: StoredCalendarRow,team_ooo_changed: bool,diff --git a/crates/calendar_events/src/outbound/pg/outlook.rs b/crates/calendar_events/src/outbound/pg/outlook.rsindex ba9bab9f3a..ddf69e0359 100644--- a/crates/calendar_events/src/outbound/pg/outlook.rs+++ b/crates/calendar_events/src/outbound/pg/outlook.rs@@ -203,6 +203,15 @@ impl OutlookCalendarRepository for PgCalendarRepository {}sqlx::query!(r#"UPDATE calendar_outlook_work SET cursor=$2,page_cursor=$2,terminal_cursor=NULL,page_loaded=false,full_scan=$3,scan_id=gen_random_uuid(),visited_pages='{}'WHERE id=$1"#,lease.id,if lease.is_primary{state.terminal_cursor}else{None},!lease.is_primary).execute(&mut *tx).await.map_err(report)?;+ if visible_calendar_sync_error_tx(&mut tx, target.calendar_id)+ .await?+ .is_some()+ {+ log.record(+ lease.binding.link_id,+ CalendarChangeOp::UpsertCalendar(target.calendar_id),+ );+ }sqlx::query!(r#"UPDATE calendars SET synced_at=now(),last_sync_error=NULL,last_sync_error_at=NULL,consecutive_sync_failures=0,materialized_starts_at=$2,materialized_ends_at=$3,materialized_start_date=$4,materialized_end_date=$5 WHERE id=$1"#,target.calendar_id,target.range.starts_at,target.range.ends_at,target.range.start_date,target.range.end_date).execute(&mut *tx).await.map_err(report)?;@@ -424,6 +433,7 @@ impl OutlookCalendarRepository for PgCalendarRepository {_ => "Microsoft calendar synchronization will retry",};let mut tx = self.pool.begin().await.map_err(report)?;+ let mut log = ChangeLogBatch::default();// An old worker must not alter the replacement grant's health.if fence(&mut tx, lease).await.is_err() {return Ok(());@@ -431,7 +441,16 @@ impl OutlookCalendarRepository for PgCalendarRepository {sqlx::query!(r#"UPDATE calendar_outlook_work SET lease_id=NULL,lease_until=NULL,last_error=$2,attempts=attempts+1,next_run_at=now()+make_interval(secs=>LEAST(900,30*power(2,LEAST(attempts,5)))::int) WHERE id=$1"#,lease.id,message).execute(&mut *tx).await.map_err(report)?;if let Some(target) = &lease.target {- sqlx::query!("UPDATE calendars SET last_sync_error=$2,last_sync_error_at=now(),consecutive_sync_failures=consecutive_sync_failures+1 WHERE id=$1",target.calendar_id,message).execute(&mut *tx).await.map_err(report)?;+ let shown_before = visible_calendar_sync_error_tx(&mut tx, target.calendar_id).await?;+ let failures = sqlx::query_scalar!("UPDATE calendars SET last_sync_error=$2,last_sync_error_at=now(),consecutive_sync_failures=consecutive_sync_failures+1 WHERE id=$1 RETURNING consecutive_sync_failures",target.calendar_id,message).fetch_optional(&mut *tx).await.map_err(report)?;+ if failures.is_some_and(|failures| {+ failure_changes_sync_error_badge(shown_before.as_deref(), failures, message)+ }) {+ log.record(+ lease.binding.link_id,+ CalendarChangeOp::UpsertCalendar(target.calendar_id),+ );+ }}sqlx::query!("UPDATE calendar_accounts SET sync_status=$2,last_sync_error=$3 WHERE id=$1",@@ -446,7 +465,7 @@ impl OutlookCalendarRepository for PgCalendarRepository {.execute(&mut *tx).await.map_err(report)?;- tx.commit().await.map_err(report)+ log.commit(tx).await}}diff --git a/crates/calendar_events/src/outbound/pg/test/changes.rs b/crates/calendar_events/src/outbound/pg/test/changes.rsindex e575dac039..2e3881bf14 100644--- a/crates/calendar_events/src/outbound/pg/test/changes.rs+++ b/crates/calendar_events/src/outbound/pg/test/changes.rs@@ -30,7 +30,7 @@ async fn logged(pool: &PgPool, link_id: Uuid) -> Vec<(i64, CalendarChangeOp)> {.collect()}-async fn ops_after(pool: &PgPool, link_id: Uuid, after: i64) -> Vec<CalendarChangeOp> {+pub(super) async fn ops_after(pool: &PgPool, link_id: Uuid, after: i64) -> Vec<CalendarChangeOp> {logged(pool, link_id).await.into_iter()@@ -39,7 +39,7 @@ async fn ops_after(pool: &PgPool, link_id: Uuid, after: i64) -> Vec<CalendarChan.collect()}-async fn counter(pool: &PgPool, link_id: Uuid) -> i64 {+pub(super) async fn counter(pool: &PgPool, link_id: Uuid) -> i64 {sqlx::query_scalar!("SELECT seq FROM calendar_change_counters WHERE link_id = $1",link_id,diff --git a/crates/calendar_events/src/outbound/pg/test/outlook.rs b/crates/calendar_events/src/outbound/pg/test/outlook.rsindex 530c19742a..5c14bf9983 100644--- a/crates/calendar_events/src/outbound/pg/test/outlook.rs+++ b/crates/calendar_events/src/outbound/pg/test/outlook.rs@@ -1,3 +1,4 @@+use super::changes::{counter, ops_after};use super::join_requests::{insert_join_request_notification, recipient_holds};use super::*;use crate::domain::models::{@@ -1459,3 +1460,251 @@ async fn an_outlook_cancellation_or_removal_retires_join_request_notifications(p.unwrap();assert!(!recipient_holds(&pool, on_removed).await);}++/// Makes a released work row due before any other and claims it.+async fn reclaim(+ pool: &PgPool,+ repo: &PgCalendarRepository,+ work_id: Uuid,+) -> OutlookCalendarLease {+ sqlx::query!(+ "UPDATE calendar_outlook_work SET next_run_at=now()-interval '1 day' WHERE id=$1",+ work_id+ )+ .execute(pool)+ .await+ .unwrap();+ let lease = claim(repo).await.unwrap();+ assert_eq!(lease.id, work_id);+ lease+}++/// Fails a primary calendar's sync `times` times with `kind`, reclaiming it+/// after each failure, and returns the next lease.+async fn fail_times(+ pool: &PgPool,+ repo: &PgCalendarRepository,+ mut lease: OutlookCalendarLease,+ kind: CalendarProviderErrorKind,+ times: i32,+) -> OutlookCalendarLease {+ for _ in 0..times {+ repo.fail_outlook_calendar(&lease, kind).await.unwrap();+ lease = reclaim(pool, repo, lease.id).await;+ }+ lease+}++/// Runs one clean round of a primary calendar: its last page, then the finish.+async fn clean_round(repo: &PgCalendarRepository, lease: &OutlookCalendarLease) {+ repo.checkpoint_outlook_calendar(+ lease,+ OutlookCalendarProgress::Page(OutlookCalendarPage {+ members: vec![],+ removed: vec![],+ next: None,+ delta: Some("terminal".into()),+ }),+ )+ .await+ .unwrap();+ let finish = claim(repo).await.unwrap();+ assert_eq!(finish.id, lease.id);+ repo.checkpoint_outlook_calendar(&finish, OutlookCalendarProgress::Finish)+ .await+ .unwrap();+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn an_outlook_sync_error_badge_logs_when_it_appears_and_when_it_changes(pool: PgPool) {+ let (repo, link, _) = mailbox(&pool).await;+ let lease = calendar(&repo).await;+ let calendar_id = lease.target.as_ref().unwrap().calendar_id;+ let before = counter(&pool, link).await;++ let lease = fail_times(+ &pool,+ &repo,+ lease,+ CalendarProviderErrorKind::Permanent,+ CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD + 1,+ )+ .await;+ assert_eq!(+ ops_after(&pool, link, before).await,+ vec![CalendarChangeOp::UpsertCalendar(calendar_id)],+ "only the failure that crosses the threshold shows a badge"+ );++ let lease = fail_times(&pool, &repo, lease, CalendarProviderErrorKind::Transient, 2).await;+ repo.fail_outlook_calendar(&lease, CalendarProviderErrorKind::ReauthRequired)+ .await+ .unwrap();+ assert_eq!(+ ops_after(&pool, link, before).await,+ vec![CalendarChangeOp::UpsertCalendar(calendar_id); 3],+ "each reworded badge logs once; a repeat of the shown one does not"+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_clean_outlook_round_logs_a_cleared_badge_once(pool: PgPool) {+ let (repo, link, _) = mailbox(&pool).await;+ let lease = calendar(&repo).await;+ let calendar_id = lease.target.as_ref().unwrap().calendar_id;+ let lease = fail_times(+ &pool,+ &repo,+ lease,+ CalendarProviderErrorKind::Permanent,+ CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD,+ )+ .await;+ let before = counter(&pool, link).await;++ clean_round(&repo, &lease).await;+ assert_eq!(+ ops_after(&pool, link, before).await,+ vec![CalendarChangeOp::UpsertCalendar(calendar_id)],+ "the round that clears a shown badge logs it"+ );++ let lease = reclaim(&pool, &repo, lease.id).await;+ clean_round(&repo, &lease).await;+ let lease = reclaim(&pool, &repo, lease.id).await;+ let lease = fail_times(+ &pool,+ &repo,+ lease,+ CalendarProviderErrorKind::Permanent,+ CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD - 1,+ )+ .await;+ clean_round(&repo, &lease).await;+ assert_eq!(+ ops_after(&pool, link, before).await,+ vec![CalendarChangeOp::UpsertCalendar(calendar_id)],+ "routine rounds and an error that never showed log nothing"+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn reconnecting_clears_a_reauth_badge_through_the_change_log(pool: PgPool) {+ let (repo, link, _) = mailbox(&pool).await;+ let lease = calendar(&repo).await;+ let calendar_id = lease.target.as_ref().unwrap().calendar_id;+ let before = counter(&pool, link).await;+ let lease = fail_times(+ &pool,+ &repo,+ lease,+ CalendarProviderErrorKind::Permanent,+ CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD - 1,+ )+ .await;+ repo.fail_outlook_calendar(&lease, CalendarProviderErrorKind::ReauthRequired)+ .await+ .unwrap();+ assert!(+ claim(&repo).await.is_none(),+ "reauthorization parks the account"+ );+ assert_eq!(+ ops_after(&pool, link, before).await,+ vec![CalendarChangeOp::UpsertCalendar(calendar_id)],+ "the reauthorization failure shows the badge"+ );++ sqlx::query!(+ "UPDATE email_links SET grant_generation=2,sync_generation=2 WHERE id=$1",+ link+ )+ .execute(&pool)+ .await+ .unwrap();+ sqlx::query!(+ "UPDATE email_link_microsoft_scopes SET grant_generation=2 WHERE link_id=$1",+ link+ )+ .execute(&pool)+ .await+ .unwrap();+ let catalog = claim(&repo).await.unwrap();+ assert!(catalog.target.is_none());+ repo.commit_outlook_calendars(&catalog, vec![discovered_calendar("primary")])+ .await+ .unwrap();+ let lease = claim(&repo).await.unwrap();+ assert_eq!(lease.target.as_ref().unwrap().calendar_id, calendar_id);+ clean_round(&repo, &lease).await;+ assert_eq!(+ ops_after(&pool, link, before).await,+ vec![CalendarChangeOp::UpsertCalendar(calendar_id); 2],+ "the first clean round on the new grant clears the badge"+ );+}++/// The migration that logs every visible Outlook calendar, so caches that+/// missed a change to its sync error badge refetch it.+const LOG_OUTLOOK_CALENDARS: &str = include_str!(+ "../../../../../macro_db_client/migrations/20261105002046_log_outlook_calendars_for_their_sync_error.sql"+);++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn the_badge_backfill_logs_every_visible_outlook_calendar(pool: PgPool) {+ let (repo, link, owner) = mailbox(&pool).await;+ let discovery = claim(&repo).await.unwrap();+ repo.commit_outlook_calendars(+ &discovery,+ vec![+ discovered_calendar("primary"),+ discovered_calendar("shared"),+ discovered_calendar("removed"),+ ],+ )+ .await+ .unwrap();+ let discovery = reclaim(&pool, &repo, discovery.id).await;+ repo.commit_outlook_calendars(+ &discovery,+ vec![+ discovered_calendar("primary"),+ discovered_calendar("shared"),+ ],+ )+ .await+ .unwrap();+ let google_link = insert_link(&pool, &owner).await;+ provider_ids(&repo, google_link).await;+ let visible = sqlx::query_scalar!(+ r#"SELECT c.id FROM calendars c JOIN calendar_accounts a ON a.id=c.account_id+ WHERE a.email_link_id=$1 AND NOT c.is_deleted ORDER BY c.id"#,+ link+ )+ .fetch_all(&pool)+ .await+ .unwrap();+ assert_eq!(visible.len(), 2);+ let before = counter(&pool, link).await;+ let google_before = counter(&pool, google_link).await;++ sqlx::raw_sql(LOG_OUTLOOK_CALENDARS)+ .execute(&pool)+ .await+ .unwrap();++ assert_eq!(+ ops_after(&pool, link, before).await,+ visible+ .into_iter()+ .map(CalendarChangeOp::UpsertCalendar)+ .collect::<Vec<_>>(),+ "each visible Outlook calendar logs once; a removed one does not"+ );+ assert_eq!(counter(&pool, link).await, before + 2);+ assert_eq!(+ ops_after(&pool, google_link, google_before).await,+ Vec::<CalendarChangeOp>::new(),+ "Google calendars already log their badge"+ );+}diff --git a/crates/macro_db_client/migrations/20261105002046_log_outlook_calendars_for_their_sync_error.sql b/crates/macro_db_client/migrations/20261105002046_log_outlook_calendars_for_their_sync_error.sqlnew file mode 100644index 0000000000..ce148b25a0--- /dev/null+++ b/crates/macro_db_client/migrations/20261105002046_log_outlook_calendars_for_their_sync_error.sql@@ -0,0 +1,37 @@+-- Until November 2026 the Outlook sync logged no change when a calendar's+-- sync error badge appeared, changed or cleared, so a client holding a+-- calendar cache can still show an error the server has cleared, or miss one+-- it shows. Which caches missed which change is unknown, so log an upsert for+-- every visible Outlook calendar on its inbox's link, and every cache+-- refetches those calendars on its next sync. Counter locks are taken in link+-- order, as every calendar write takes them. A rerun logs each calendar+-- again, which costs clients one more refetch.+WITH logged AS (+ SELECT account.email_link_id AS link_id,+ calendar.id AS calendar_id,+ row_number() OVER (PARTITION BY account.email_link_id ORDER BY calendar.id) AS position,+ count(*) OVER (PARTITION BY account.email_link_id) AS total+ FROM calendars calendar+ JOIN calendar_accounts account ON account.id = calendar.account_id+ WHERE account.provider = 'outlook'+ AND account.sync_status <> 'disabled'+ AND NOT calendar.is_deleted+),+counters AS (+ INSERT INTO calendar_change_counters (link_id, seq)+ SELECT link_id, count(*)+ FROM logged+ GROUP BY link_id+ ORDER BY link_id+ ON CONFLICT (link_id) DO UPDATE+ SET seq = calendar_change_counters.seq + EXCLUDED.seq,+ updated_at = now()+ RETURNING link_id, seq+)+INSERT INTO calendar_change_log (link_id, seq, kind, calendar_id)+SELECT logged.link_id,+ counters.seq - logged.total + logged.position,+ 3,+ logged.calendar_id+FROM logged+JOIN counters ON counters.link_id = logged.link_id;