Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(calendar): log Outlook sync error changes for calendar caches (#9633)

gbirmandd82f4b61cc2
diff --git a/.sqlx/query-1efb2374db84fd648f2f43e00ca122eecab96b69a7bc0128e920a2b467c21da1.json b/.sqlx/query-1efb2374db84fd648f2f43e00ca122eecab96b69a7bc0128e920a2b467c21da1.json
deleted file mode 100644
index c4fec0d6db..0000000000
--- a/.sqlx/query-1efb2374db84fd648f2f43e00ca122eecab96b69a7bc0128e920a2b467c21da1.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "UPDATE calendars SET last_sync_error=$2,last_sync_error_at=now(),consecutive_sync_failures=consecutive_sync_failures+1 WHERE id=$1",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid",
- "Text"
- ]
- },
- "nullable": []
- },
- "hash": "1efb2374db84fd648f2f43e00ca122eecab96b69a7bc0128e920a2b467c21da1"
-}
diff --git a/.sqlx/query-599a42d00923771e0fb8a6a36d845dba18b117b256392e6ad5b28c9f5ac143d4.json b/.sqlx/query-599a42d00923771e0fb8a6a36d845dba18b117b256392e6ad5b28c9f5ac143d4.json
new file mode 100644
index 0000000000..d57f1c9cf5
--- /dev/null
+++ b/.sqlx/query-599a42d00923771e0fb8a6a36d845dba18b117b256392e6ad5b28c9f5ac143d4.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE calendars SET last_sync_error=$2,last_sync_error_at=now(),consecutive_sync_failures=consecutive_sync_failures+1 WHERE id=$1 RETURNING consecutive_sync_failures",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "consecutive_sync_failures",
+ "type_info": "Int4"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "599a42d00923771e0fb8a6a36d845dba18b117b256392e6ad5b28c9f5ac143d4"
+}
diff --git a/crates/calendar_events/AGENTS.md b/crates/calendar_events/AGENTS.md
index 24e8a9a813..8da946cc6b 100644
--- a/crates/calendar_events/AGENTS.md
+++ b/crates/calendar_events/AGENTS.md
@@ -377,6 +377,17 @@ reminder dispatch, team availability, and provider/Postgres persistence.
(reminders and automatic declines anchor all-day dates at UTC, team
availability reports the zone missing). The warning `ignoring an Outlook
mailbox time zone Macro cannot map` and the span carry `unmapped_time_zone`.
+- A calendar's sync error reaches calendar caches only as an `UpsertCalendar`
+ change-log entry, since they read the calendar list once and then follow
+ the log. Google and Outlook log one when the shown error
+ (`visible_calendar_sync_error_tx`, past
+ `CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD` failures) appears or is reworded
+ (`failure_changes_sync_error_badge`) and when the round that clears it
+ commits, never on a routine round (Outlook polls every two minutes).
+ Neither nudges open clients for it, so a tab sees it on its next delta.
+ Outlook logged none before November 2026;
+ `20261105002046_log_outlook_calendars_for_their_sync_error.sql` logs every
+ visible Outlook calendar once so caches that missed a change refetch it.
- A sync of a `ready`, already synced account stays `ready`, on Google as on
Outlook, including after a re-grant that kept calendar access, whose sync tokens
survive. Any other state is catching up and marks it `syncing`: a first import,
diff --git a/crates/calendar_events/src/outbound/pg.rs b/crates/calendar_events/src/outbound/pg.rs
index 88a80875ab..cf4cedf83c 100644
--- a/crates/calendar_events/src/outbound/pg.rs
+++ b/crates/calendar_events/src/outbound/pg.rs
@@ -2003,9 +2003,9 @@ impl CalendarRepository for PgCalendarRepository {
// Clients see a calendar's error only once it crosses the badge
// threshold, so only a newly shown or reworded badge is a change.
let mut log = ChangeLogBatch::default();
- let badged_after =
- failures.is_some_and(|failures| failures >= CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD);
- if badged_after && badged_before.as_deref() != Some(message) {
+ if failures.is_some_and(|failures| {
+ failure_changes_sync_error_badge(badged_before.as_deref(), failures, message)
+ }) {
log.record(
key.email_link_id,
CalendarChangeOp::UpsertCalendar(calendar_id),
@@ -3176,6 +3176,16 @@ async fn visible_calendar_sync_error_tx(
}))
}
+/// Whether a failure stored as `message`, leaving `failures` in a row, newly
+/// shows or rewords the badge that `shown_before` held.
+fn failure_changes_sync_error_badge(
+ shown_before: Option<&str>,
+ failures: i32,
+ message: &str,
+) -> bool {
+ failures >= CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD && shown_before != Some(message)
+}
+
fn stored_google_calendar(
row: StoredCalendarRow,
team_ooo_changed: bool,
diff --git a/crates/calendar_events/src/outbound/pg/outlook.rs b/crates/calendar_events/src/outbound/pg/outlook.rs
index ba9bab9f3a..ddf69e0359 100644
--- a/crates/calendar_events/src/outbound/pg/outlook.rs
+++ b/crates/calendar_events/src/outbound/pg/outlook.rs
@@ -203,6 +203,15 @@ impl OutlookCalendarRepository for PgCalendarRepository {
}
sqlx::query!(r#"UPDATE calendar_outlook_work SET cursor=$2,page_cursor=$2,terminal_cursor=NULL,page_loaded=false,full_scan=$3,scan_id=gen_random_uuid(),visited_pages='{}'
WHERE id=$1"#,lease.id,if lease.is_primary{state.terminal_cursor}else{None},!lease.is_primary).execute(&mut *tx).await.map_err(report)?;
+ if visible_calendar_sync_error_tx(&mut tx, target.calendar_id)
+ .await?
+ .is_some()
+ {
+ log.record(
+ lease.binding.link_id,
+ CalendarChangeOp::UpsertCalendar(target.calendar_id),
+ );
+ }
sqlx::query!(r#"UPDATE calendars SET synced_at=now(),last_sync_error=NULL,last_sync_error_at=NULL,consecutive_sync_failures=0,
materialized_starts_at=$2,materialized_ends_at=$3,materialized_start_date=$4,materialized_end_date=$5 WHERE id=$1"#,
target.calendar_id,target.range.starts_at,target.range.ends_at,target.range.start_date,target.range.end_date).execute(&mut *tx).await.map_err(report)?;
@@ -424,6 +433,7 @@ impl OutlookCalendarRepository for PgCalendarRepository {
_ => "Microsoft calendar synchronization will retry",
};
let mut tx = self.pool.begin().await.map_err(report)?;
+ let mut log = ChangeLogBatch::default();
// An old worker must not alter the replacement grant's health.
if fence(&mut tx, lease).await.is_err() {
return Ok(());
@@ -431,7 +441,16 @@ impl OutlookCalendarRepository for PgCalendarRepository {
sqlx::query!(r#"UPDATE calendar_outlook_work SET lease_id=NULL,lease_until=NULL,last_error=$2,attempts=attempts+1,
next_run_at=now()+make_interval(secs=>LEAST(900,30*power(2,LEAST(attempts,5)))::int) WHERE id=$1"#,lease.id,message).execute(&mut *tx).await.map_err(report)?;
if let Some(target) = &lease.target {
- sqlx::query!("UPDATE calendars SET last_sync_error=$2,last_sync_error_at=now(),consecutive_sync_failures=consecutive_sync_failures+1 WHERE id=$1",target.calendar_id,message).execute(&mut *tx).await.map_err(report)?;
+ let shown_before = visible_calendar_sync_error_tx(&mut tx, target.calendar_id).await?;
+ let failures = sqlx::query_scalar!("UPDATE calendars SET last_sync_error=$2,last_sync_error_at=now(),consecutive_sync_failures=consecutive_sync_failures+1 WHERE id=$1 RETURNING consecutive_sync_failures",target.calendar_id,message).fetch_optional(&mut *tx).await.map_err(report)?;
+ if failures.is_some_and(|failures| {
+ failure_changes_sync_error_badge(shown_before.as_deref(), failures, message)
+ }) {
+ log.record(
+ lease.binding.link_id,
+ CalendarChangeOp::UpsertCalendar(target.calendar_id),
+ );
+ }
}
sqlx::query!(
"UPDATE calendar_accounts SET sync_status=$2,last_sync_error=$3 WHERE id=$1",
@@ -446,7 +465,7 @@ impl OutlookCalendarRepository for PgCalendarRepository {
.execute(&mut *tx)
.await
.map_err(report)?;
- tx.commit().await.map_err(report)
+ log.commit(tx).await
}
}
diff --git a/crates/calendar_events/src/outbound/pg/test/changes.rs b/crates/calendar_events/src/outbound/pg/test/changes.rs
index e575dac039..2e3881bf14 100644
--- a/crates/calendar_events/src/outbound/pg/test/changes.rs
+++ b/crates/calendar_events/src/outbound/pg/test/changes.rs
@@ -30,7 +30,7 @@ async fn logged(pool: &PgPool, link_id: Uuid) -> Vec<(i64, CalendarChangeOp)> {
.collect()
}
-async fn ops_after(pool: &PgPool, link_id: Uuid, after: i64) -> Vec<CalendarChangeOp> {
+pub(super) async fn ops_after(pool: &PgPool, link_id: Uuid, after: i64) -> Vec<CalendarChangeOp> {
logged(pool, link_id)
.await
.into_iter()
@@ -39,7 +39,7 @@ async fn ops_after(pool: &PgPool, link_id: Uuid, after: i64) -> Vec<CalendarChan
.collect()
}
-async fn counter(pool: &PgPool, link_id: Uuid) -> i64 {
+pub(super) async fn counter(pool: &PgPool, link_id: Uuid) -> i64 {
sqlx::query_scalar!(
"SELECT seq FROM calendar_change_counters WHERE link_id = $1",
link_id,
diff --git a/crates/calendar_events/src/outbound/pg/test/outlook.rs b/crates/calendar_events/src/outbound/pg/test/outlook.rs
index 530c19742a..5c14bf9983 100644
--- a/crates/calendar_events/src/outbound/pg/test/outlook.rs
+++ b/crates/calendar_events/src/outbound/pg/test/outlook.rs
@@ -1,3 +1,4 @@
+use super::changes::{counter, ops_after};
use super::join_requests::{insert_join_request_notification, recipient_holds};
use super::*;
use crate::domain::models::{
@@ -1459,3 +1460,251 @@ async fn an_outlook_cancellation_or_removal_retires_join_request_notifications(p
.unwrap();
assert!(!recipient_holds(&pool, on_removed).await);
}
+
+/// Makes a released work row due before any other and claims it.
+async fn reclaim(
+ pool: &PgPool,
+ repo: &PgCalendarRepository,
+ work_id: Uuid,
+) -> OutlookCalendarLease {
+ sqlx::query!(
+ "UPDATE calendar_outlook_work SET next_run_at=now()-interval '1 day' WHERE id=$1",
+ work_id
+ )
+ .execute(pool)
+ .await
+ .unwrap();
+ let lease = claim(repo).await.unwrap();
+ assert_eq!(lease.id, work_id);
+ lease
+}
+
+/// Fails a primary calendar's sync `times` times with `kind`, reclaiming it
+/// after each failure, and returns the next lease.
+async fn fail_times(
+ pool: &PgPool,
+ repo: &PgCalendarRepository,
+ mut lease: OutlookCalendarLease,
+ kind: CalendarProviderErrorKind,
+ times: i32,
+) -> OutlookCalendarLease {
+ for _ in 0..times {
+ repo.fail_outlook_calendar(&lease, kind).await.unwrap();
+ lease = reclaim(pool, repo, lease.id).await;
+ }
+ lease
+}
+
+/// Runs one clean round of a primary calendar: its last page, then the finish.
+async fn clean_round(repo: &PgCalendarRepository, lease: &OutlookCalendarLease) {
+ repo.checkpoint_outlook_calendar(
+ lease,
+ OutlookCalendarProgress::Page(OutlookCalendarPage {
+ members: vec![],
+ removed: vec![],
+ next: None,
+ delta: Some("terminal".into()),
+ }),
+ )
+ .await
+ .unwrap();
+ let finish = claim(repo).await.unwrap();
+ assert_eq!(finish.id, lease.id);
+ repo.checkpoint_outlook_calendar(&finish, OutlookCalendarProgress::Finish)
+ .await
+ .unwrap();
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn an_outlook_sync_error_badge_logs_when_it_appears_and_when_it_changes(pool: PgPool) {
+ let (repo, link, _) = mailbox(&pool).await;
+ let lease = calendar(&repo).await;
+ let calendar_id = lease.target.as_ref().unwrap().calendar_id;
+ let before = counter(&pool, link).await;
+
+ let lease = fail_times(
+ &pool,
+ &repo,
+ lease,
+ CalendarProviderErrorKind::Permanent,
+ CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD + 1,
+ )
+ .await;
+ assert_eq!(
+ ops_after(&pool, link, before).await,
+ vec![CalendarChangeOp::UpsertCalendar(calendar_id)],
+ "only the failure that crosses the threshold shows a badge"
+ );
+
+ let lease = fail_times(&pool, &repo, lease, CalendarProviderErrorKind::Transient, 2).await;
+ repo.fail_outlook_calendar(&lease, CalendarProviderErrorKind::ReauthRequired)
+ .await
+ .unwrap();
+ assert_eq!(
+ ops_after(&pool, link, before).await,
+ vec![CalendarChangeOp::UpsertCalendar(calendar_id); 3],
+ "each reworded badge logs once; a repeat of the shown one does not"
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_clean_outlook_round_logs_a_cleared_badge_once(pool: PgPool) {
+ let (repo, link, _) = mailbox(&pool).await;
+ let lease = calendar(&repo).await;
+ let calendar_id = lease.target.as_ref().unwrap().calendar_id;
+ let lease = fail_times(
+ &pool,
+ &repo,
+ lease,
+ CalendarProviderErrorKind::Permanent,
+ CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD,
+ )
+ .await;
+ let before = counter(&pool, link).await;
+
+ clean_round(&repo, &lease).await;
+ assert_eq!(
+ ops_after(&pool, link, before).await,
+ vec![CalendarChangeOp::UpsertCalendar(calendar_id)],
+ "the round that clears a shown badge logs it"
+ );
+
+ let lease = reclaim(&pool, &repo, lease.id).await;
+ clean_round(&repo, &lease).await;
+ let lease = reclaim(&pool, &repo, lease.id).await;
+ let lease = fail_times(
+ &pool,
+ &repo,
+ lease,
+ CalendarProviderErrorKind::Permanent,
+ CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD - 1,
+ )
+ .await;
+ clean_round(&repo, &lease).await;
+ assert_eq!(
+ ops_after(&pool, link, before).await,
+ vec![CalendarChangeOp::UpsertCalendar(calendar_id)],
+ "routine rounds and an error that never showed log nothing"
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn reconnecting_clears_a_reauth_badge_through_the_change_log(pool: PgPool) {
+ let (repo, link, _) = mailbox(&pool).await;
+ let lease = calendar(&repo).await;
+ let calendar_id = lease.target.as_ref().unwrap().calendar_id;
+ let before = counter(&pool, link).await;
+ let lease = fail_times(
+ &pool,
+ &repo,
+ lease,
+ CalendarProviderErrorKind::Permanent,
+ CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD - 1,
+ )
+ .await;
+ repo.fail_outlook_calendar(&lease, CalendarProviderErrorKind::ReauthRequired)
+ .await
+ .unwrap();
+ assert!(
+ claim(&repo).await.is_none(),
+ "reauthorization parks the account"
+ );
+ assert_eq!(
+ ops_after(&pool, link, before).await,
+ vec![CalendarChangeOp::UpsertCalendar(calendar_id)],
+ "the reauthorization failure shows the badge"
+ );
+
+ sqlx::query!(
+ "UPDATE email_links SET grant_generation=2,sync_generation=2 WHERE id=$1",
+ link
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ sqlx::query!(
+ "UPDATE email_link_microsoft_scopes SET grant_generation=2 WHERE link_id=$1",
+ link
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ let catalog = claim(&repo).await.unwrap();
+ assert!(catalog.target.is_none());
+ repo.commit_outlook_calendars(&catalog, vec![discovered_calendar("primary")])
+ .await
+ .unwrap();
+ let lease = claim(&repo).await.unwrap();
+ assert_eq!(lease.target.as_ref().unwrap().calendar_id, calendar_id);
+ clean_round(&repo, &lease).await;
+ assert_eq!(
+ ops_after(&pool, link, before).await,
+ vec![CalendarChangeOp::UpsertCalendar(calendar_id); 2],
+ "the first clean round on the new grant clears the badge"
+ );
+}
+
+/// The migration that logs every visible Outlook calendar, so caches that
+/// missed a change to its sync error badge refetch it.
+const LOG_OUTLOOK_CALENDARS: &str = include_str!(
+ "../../../../../macro_db_client/migrations/20261105002046_log_outlook_calendars_for_their_sync_error.sql"
+);
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn the_badge_backfill_logs_every_visible_outlook_calendar(pool: PgPool) {
+ let (repo, link, owner) = mailbox(&pool).await;
+ let discovery = claim(&repo).await.unwrap();
+ repo.commit_outlook_calendars(
+ &discovery,
+ vec![
+ discovered_calendar("primary"),
+ discovered_calendar("shared"),
+ discovered_calendar("removed"),
+ ],
+ )
+ .await
+ .unwrap();
+ let discovery = reclaim(&pool, &repo, discovery.id).await;
+ repo.commit_outlook_calendars(
+ &discovery,
+ vec![
+ discovered_calendar("primary"),
+ discovered_calendar("shared"),
+ ],
+ )
+ .await
+ .unwrap();
+ let google_link = insert_link(&pool, &owner).await;
+ provider_ids(&repo, google_link).await;
+ let visible = sqlx::query_scalar!(
+ r#"SELECT c.id FROM calendars c JOIN calendar_accounts a ON a.id=c.account_id
+ WHERE a.email_link_id=$1 AND NOT c.is_deleted ORDER BY c.id"#,
+ link
+ )
+ .fetch_all(&pool)
+ .await
+ .unwrap();
+ assert_eq!(visible.len(), 2);
+ let before = counter(&pool, link).await;
+ let google_before = counter(&pool, google_link).await;
+
+ sqlx::raw_sql(LOG_OUTLOOK_CALENDARS)
+ .execute(&pool)
+ .await
+ .unwrap();
+
+ assert_eq!(
+ ops_after(&pool, link, before).await,
+ visible
+ .into_iter()
+ .map(CalendarChangeOp::UpsertCalendar)
+ .collect::<Vec<_>>(),
+ "each visible Outlook calendar logs once; a removed one does not"
+ );
+ assert_eq!(counter(&pool, link).await, before + 2);
+ assert_eq!(
+ ops_after(&pool, google_link, google_before).await,
+ Vec::<CalendarChangeOp>::new(),
+ "Google calendars already log their badge"
+ );
+}
diff --git a/crates/macro_db_client/migrations/20261105002046_log_outlook_calendars_for_their_sync_error.sql b/crates/macro_db_client/migrations/20261105002046_log_outlook_calendars_for_their_sync_error.sql
new file mode 100644
index 0000000000..ce148b25a0
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261105002046_log_outlook_calendars_for_their_sync_error.sql
@@ -0,0 +1,37 @@
+-- Until November 2026 the Outlook sync logged no change when a calendar's
+-- sync error badge appeared, changed or cleared, so a client holding a
+-- calendar cache can still show an error the server has cleared, or miss one
+-- it shows. Which caches missed which change is unknown, so log an upsert for
+-- every visible Outlook calendar on its inbox's link, and every cache
+-- refetches those calendars on its next sync. Counter locks are taken in link
+-- order, as every calendar write takes them. A rerun logs each calendar
+-- again, which costs clients one more refetch.
+WITH logged AS (
+ SELECT account.email_link_id AS link_id,
+ calendar.id AS calendar_id,
+ row_number() OVER (PARTITION BY account.email_link_id ORDER BY calendar.id) AS position,
+ count(*) OVER (PARTITION BY account.email_link_id) AS total
+ FROM calendars calendar
+ JOIN calendar_accounts account ON account.id = calendar.account_id
+ WHERE account.provider = 'outlook'
+ AND account.sync_status <> 'disabled'
+ AND NOT calendar.is_deleted
+),
+counters AS (
+ INSERT INTO calendar_change_counters (link_id, seq)
+ SELECT link_id, count(*)
+ FROM logged
+ GROUP BY link_id
+ ORDER BY link_id
+ ON CONFLICT (link_id) DO UPDATE
+ SET seq = calendar_change_counters.seq + EXCLUDED.seq,
+ updated_at = now()
+ RETURNING link_id, seq
+)
+INSERT INTO calendar_change_log (link_id, seq, kind, calendar_id)
+SELECT logged.link_id,
+ counters.seq - logged.total + logged.position,
+ 3,
+ logged.calendar_id
+FROM logged
+JOIN counters ON counters.link_id = logged.link_id;