Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(calendar): isolate per-calendar Google sync failures (#6268)

gbirmanaa1e56dc0eb9
diff --git a/.sqlx/query-0cdd9104762f5146528b4a7abd4a38d839eaa7bbfa37e5f956cd97dfda4d6380.json b/.sqlx/query-579009b8a97c81ee796f1109588b2b1e914ad489c967fa77ab8343ef969527d8.json
similarity index 70%
rename from .sqlx/query-0cdd9104762f5146528b4a7abd4a38d839eaa7bbfa37e5f956cd97dfda4d6380.json
rename to .sqlx/query-579009b8a97c81ee796f1109588b2b1e914ad489c967fa77ab8343ef969527d8.json
index 69603330fa..a471001026 100644
--- a/.sqlx/query-0cdd9104762f5146528b4a7abd4a38d839eaa7bbfa37e5f956cd97dfda4d6380.json
+++ b/.sqlx/query-579009b8a97c81ee796f1109588b2b1e914ad489c967fa77ab8343ef969527d8.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n INSERT INTO calendars (\n id, account_id, provider_calendar_id, name, description,\n time_zone, color, access_role, is_primary, is_selected,\n default_reminders\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)\n ON CONFLICT (account_id, provider_calendar_id) DO UPDATE SET\n name = EXCLUDED.name,\n description = EXCLUDED.description,\n time_zone = EXCLUDED.time_zone,\n color = EXCLUDED.color,\n access_role = EXCLUDED.access_role,\n is_primary = EXCLUDED.is_primary,\n is_selected = EXCLUDED.is_selected,\n is_deleted = false,\n default_reminders = EXCLUDED.default_reminders,\n updated_at = now()\n RETURNING\n id,\n sync_token,\n materialized_starts_at,\n materialized_ends_at,\n materialized_start_date,\n materialized_end_date,\n synced_at,\n watch_expires_at\n ",
+ "query": "\n INSERT INTO calendars (\n id, account_id, provider_calendar_id, name, description,\n time_zone, color, access_role, is_primary, is_selected,\n default_reminders\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)\n ON CONFLICT (account_id, provider_calendar_id) DO UPDATE SET\n name = EXCLUDED.name,\n description = EXCLUDED.description,\n time_zone = EXCLUDED.time_zone,\n color = EXCLUDED.color,\n access_role = EXCLUDED.access_role,\n is_primary = EXCLUDED.is_primary,\n is_selected = EXCLUDED.is_selected,\n is_deleted = false,\n default_reminders = EXCLUDED.default_reminders,\n last_sync_error = CASE\n WHEN calendars.is_deleted THEN NULL\n ELSE calendars.last_sync_error\n END,\n last_sync_error_at = CASE\n WHEN calendars.is_deleted THEN NULL\n ELSE calendars.last_sync_error_at\n END,\n consecutive_sync_failures = CASE\n WHEN calendars.is_deleted THEN 0\n ELSE calendars.consecutive_sync_failures\n END,\n updated_at = now()\n RETURNING\n id,\n sync_token,\n materialized_starts_at,\n materialized_ends_at,\n materialized_start_date,\n materialized_end_date,\n synced_at,\n watch_expires_at\n ",
"describe": {
"columns": [
{
@@ -70,5 +70,5 @@
true
]
},
- "hash": "0cdd9104762f5146528b4a7abd4a38d839eaa7bbfa37e5f956cd97dfda4d6380"
+ "hash": "579009b8a97c81ee796f1109588b2b1e914ad489c967fa77ab8343ef969527d8"
}
diff --git a/.sqlx/query-707e3644cb7cc2292b4b3415aee2802d5ea1c34c434b854b8bccd877e57cdcd4.json b/.sqlx/query-707e3644cb7cc2292b4b3415aee2802d5ea1c34c434b854b8bccd877e57cdcd4.json
new file mode 100644
index 0000000000..842ce6e005
--- /dev/null
+++ b/.sqlx/query-707e3644cb7cc2292b4b3415aee2802d5ea1c34c434b854b8bccd877e57cdcd4.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE calendars\n SET last_sync_error = $3,\n last_sync_error_at = now(),\n consecutive_sync_failures = consecutive_sync_failures + 1,\n updated_at = now()\n WHERE id = $1\n AND account_id = $2\n AND NOT is_deleted\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "707e3644cb7cc2292b4b3415aee2802d5ea1c34c434b854b8bccd877e57cdcd4"
+}
diff --git a/.sqlx/query-96789ed93a8cfa77b5b52ad7ddf3d0dd91ed51285b810f0219a5b16414c6475b.json b/.sqlx/query-96789ed93a8cfa77b5b52ad7ddf3d0dd91ed51285b810f0219a5b16414c6475b.json
deleted file mode 100644
index c7f208bae4..0000000000
--- a/.sqlx/query-96789ed93a8cfa77b5b52ad7ddf3d0dd91ed51285b810f0219a5b16414c6475b.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n UPDATE calendars\n SET sync_token = $3,\n synced_at = now(),\n materialized_starts_at = CASE\n WHEN $4 THEN $5\n ELSE materialized_starts_at\n END,\n materialized_ends_at = CASE\n WHEN $4 THEN $6\n ELSE materialized_ends_at\n END,\n materialized_start_date = CASE\n WHEN $4 THEN $7\n ELSE materialized_start_date\n END,\n materialized_end_date = CASE\n WHEN $4 THEN $8\n ELSE materialized_end_date\n END,\n updated_at = now()\n WHERE id = $1\n AND account_id = $2\n AND NOT is_deleted\n ",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid",
- "Uuid",
- "Text",
- "Bool",
- "Timestamptz",
- "Timestamptz",
- "Date",
- "Date"
- ]
- },
- "nullable": []
- },
- "hash": "96789ed93a8cfa77b5b52ad7ddf3d0dd91ed51285b810f0219a5b16414c6475b"
-}
diff --git a/.sqlx/query-009679f4874d10c31d6588e31dc40bc82c4f6288ed4971117be3aa4dcc1c64b6.json b/.sqlx/query-d7c9243a18937c63aea0ffee3fa8aab0f711fb8d40d9990d5ead5212eb23f4ec.json
similarity index 51%
rename from .sqlx/query-009679f4874d10c31d6588e31dc40bc82c4f6288ed4971117be3aa4dcc1c64b6.json
rename to .sqlx/query-d7c9243a18937c63aea0ffee3fa8aab0f711fb8d40d9990d5ead5212eb23f4ec.json
index c79d249b36..54cb502ac7 100644
--- a/.sqlx/query-009679f4874d10c31d6588e31dc40bc82c4f6288ed4971117be3aa4dcc1c64b6.json
+++ b/.sqlx/query-d7c9243a18937c63aea0ffee3fa8aab0f711fb8d40d9990d5ead5212eb23f4ec.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n SELECT\n calendar.id,\n link.id AS email_link_id,\n link.email_address,\n calendar.name,\n calendar.color,\n calendar.is_primary,\n calendar.access_role,\n calendar.provider_calendar_id,\n calendar.default_reminders\n FROM email_links link\n JOIN calendar_accounts account ON account.email_link_id = link.id\n JOIN calendars calendar ON calendar.account_id = account.id\n WHERE NOT calendar.is_deleted\n AND account.sync_status <> 'disabled'\n AND (\n link.macro_id = $1\n OR EXISTS (\n SELECT 1\n FROM macro_user_links delegation\n WHERE delegation.link_id = link.id\n AND delegation.primary_macro_id = $1\n )\n )\n ORDER BY\n (link.macro_id = $1) DESC,\n link.is_primary DESC,\n link.created_at ASC,\n calendar.is_primary DESC,\n (calendar.access_role IN ('owner', 'writer')) DESC,\n calendar.name ASC\n ",
+ "query": "\n SELECT\n calendar.id,\n link.id AS email_link_id,\n link.email_address,\n calendar.name,\n calendar.color,\n calendar.is_primary,\n calendar.access_role,\n calendar.provider_calendar_id,\n calendar.default_reminders,\n calendar.last_sync_error,\n calendar.consecutive_sync_failures\n FROM email_links link\n JOIN calendar_accounts account ON account.email_link_id = link.id\n JOIN calendars calendar ON calendar.account_id = account.id\n WHERE NOT calendar.is_deleted\n AND account.sync_status <> 'disabled'\n AND (\n link.macro_id = $1\n OR EXISTS (\n SELECT 1\n FROM macro_user_links delegation\n WHERE delegation.link_id = link.id\n AND delegation.primary_macro_id = $1\n )\n )\n ORDER BY\n (link.macro_id = $1) DESC,\n link.is_primary DESC,\n link.created_at ASC,\n calendar.is_primary DESC,\n (calendar.access_role IN ('owner', 'writer')) DESC,\n calendar.name ASC\n ",
"describe": {
"columns": [
{
@@ -47,6 +47,16 @@
"ordinal": 8,
"name": "default_reminders",
"type_info": "Jsonb"
+ },
+ {
+ "ordinal": 9,
+ "name": "last_sync_error",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 10,
+ "name": "consecutive_sync_failures",
+ "type_info": "Int4"
}
],
"parameters": {
@@ -63,8 +73,10 @@
false,
true,
false,
+ false,
+ true,
false
]
},
- "hash": "009679f4874d10c31d6588e31dc40bc82c4f6288ed4971117be3aa4dcc1c64b6"
+ "hash": "d7c9243a18937c63aea0ffee3fa8aab0f711fb8d40d9990d5ead5212eb23f4ec"
}
diff --git a/.sqlx/query-efae87e3ab644d9f07a4043e2df6acc2f791657ef69b764b7bcb59a660a7d19e.json b/.sqlx/query-efae87e3ab644d9f07a4043e2df6acc2f791657ef69b764b7bcb59a660a7d19e.json
new file mode 100644
index 0000000000..78f336b32b
--- /dev/null
+++ b/.sqlx/query-efae87e3ab644d9f07a4043e2df6acc2f791657ef69b764b7bcb59a660a7d19e.json
@@ -0,0 +1,21 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE calendars\n SET sync_token = $3,\n synced_at = now(),\n last_sync_error = NULL,\n last_sync_error_at = NULL,\n consecutive_sync_failures = 0,\n materialized_starts_at = CASE\n WHEN $4 THEN $5\n ELSE materialized_starts_at\n END,\n materialized_ends_at = CASE\n WHEN $4 THEN $6\n ELSE materialized_ends_at\n END,\n materialized_start_date = CASE\n WHEN $4 THEN $7\n ELSE materialized_start_date\n END,\n materialized_end_date = CASE\n WHEN $4 THEN $8\n ELSE materialized_end_date\n END,\n updated_at = now()\n WHERE id = $1\n AND account_id = $2\n AND NOT is_deleted\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Text",
+ "Bool",
+ "Timestamptz",
+ "Timestamptz",
+ "Date",
+ "Date"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "efae87e3ab644d9f07a4043e2df6acc2f791657ef69b764b7bcb59a660a7d19e"
+}
diff --git a/apps/web/src/features/calendar/components/SourceControls.test.tsx b/apps/web/src/features/calendar/components/SourceControls.test.tsx
index 46fe0fdf4b..63329fdc0e 100644
--- a/apps/web/src/features/calendar/components/SourceControls.test.tsx
+++ b/apps/web/src/features/calendar/components/SourceControls.test.tsx
@@ -32,6 +32,7 @@ const SOURCES: CalendarSource[] = [
emailAddress: '[email protected]',
emailLinkId: 'link-test',
isPrimary: true,
+ syncError: 'Precondition check failed.',
},
];
@@ -110,6 +111,21 @@ describe('SourceControls', () => {
expect(indicators).toHaveLength(1);
});
+ it('badges a calendar with a persistent sync failure', () => {
+ const { expandAccount, container } = renderControls();
+ expandAccount('[email protected]');
+ const badge = container.querySelector(
+ '[title="Sync failed: Precondition check failed."]'
+ );
+ expect(badge).toBeTruthy();
+ });
+
+ it('leaves a healthy calendar unbadged', () => {
+ const { expandAccount, container } = renderControls();
+ expandAccount('[email protected]');
+ expect(container.querySelector('[title^="Sync failed:"]')).toBeNull();
+ });
+
it('collapses an account again to hide its calendars', () => {
const { expandAccount, getByRole, queryByText } = renderControls();
expandAccount('[email protected]');
diff --git a/apps/web/src/features/calendar/components/SourceControls.tsx b/apps/web/src/features/calendar/components/SourceControls.tsx
index 2bc268bcbc..7c26d6b0ce 100644
--- a/apps/web/src/features/calendar/components/SourceControls.tsx
+++ b/apps/web/src/features/calendar/components/SourceControls.tsx
@@ -1,5 +1,6 @@
import CaretRightIcon from '@phosphor/caret-right.svg';
import RssIcon from '@phosphor/rss.svg';
+import WarningIcon from '@phosphor/warning.svg';
import { Checkbox } from '@ui';
import { createSignal, For, Show } from 'solid-js';
import type { CalendarSource } from '../types';
@@ -98,6 +99,19 @@ export function SourceControls(props: SourceControlsProps) {
<span class="min-w-0 flex-1 truncate">
{source.name}
</span>
+ <Show when={source.syncError}>
+ {(error) => (
+ <span
+ title={`Sync failed: ${error()}`}
+ class="flex shrink-0 text-alert-ink"
+ >
+ <WarningIcon
+ class="size-3"
+ aria-label={`Sync failed: ${error()}`}
+ />
+ </span>
+ )}
+ </Show>
<Show when={source.isSubscription}>
<span
title="Subscription calendar"
diff --git a/apps/web/src/features/calendar/hooks/use-calendar-sources.ts b/apps/web/src/features/calendar/hooks/use-calendar-sources.ts
index 346d66355c..5276bcf103 100644
--- a/apps/web/src/features/calendar/hooks/use-calendar-sources.ts
+++ b/apps/web/src/features/calendar/hooks/use-calendar-sources.ts
@@ -20,6 +20,7 @@ export function useCalendarSources() {
emailLinkId: calendar.emailLinkId,
isPrimary: calendar.isPrimary,
isSubscription: calendar.isSubscription,
+ syncError: calendar.syncError ?? undefined,
}));
});
const sourceById = createMemo(
diff --git a/apps/web/src/features/calendar/types.ts b/apps/web/src/features/calendar/types.ts
index aab845709c..57ddf48552 100644
--- a/apps/web/src/features/calendar/types.ts
+++ b/apps/web/src/features/calendar/types.ts
@@ -38,6 +38,8 @@ export interface CalendarSource {
isPrimary?: boolean;
/** Whether this is a subscribed system calendar (holidays, birthdays). */
isSubscription?: boolean;
+ /** A persistent sync failure isolated to this calendar, for a settings badge. */
+ syncError?: string;
}
/** Calendar occurrence data, independent from FullCalendar. */
diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts
index dc306ea669..7128cb6063 100644
--- a/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts
+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/index.ts
@@ -360,3 +360,4 @@ export * from './userProvider';
export * from './value';
export * from './visibleCalendar';
export * from './visibleCalendarColor';
+export * from './visibleCalendarSyncError';
diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/visibleCalendar.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/visibleCalendar.ts
index 13f5946bb5..8699158162 100644
--- a/apps/web/src/lib/service-clients/service-email/generated/schemas/visibleCalendar.ts
+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/visibleCalendar.ts
@@ -7,6 +7,7 @@
import type { EventReminderOverride } from './eventReminderOverride';
import type { VisibleCalendarColor } from './visibleCalendarColor';
+import type { VisibleCalendarSyncError } from './visibleCalendarSyncError';
/**
* A calendar visible to a requester, listed for pickers and filters.
@@ -31,4 +32,8 @@ birthdays) the account subscribes to rather than one a person maintains. */
isWritable: boolean;
/** Provider display name. */
name: string;
+ /** A persistent sync failure isolated to this calendar, surfaced so the
+settings row can badge it. `None` while the calendar is syncing
+normally or a failure has not yet crossed the persistence threshold. */
+ syncError?: VisibleCalendarSyncError;
}
diff --git a/apps/web/src/lib/service-clients/service-email/generated/schemas/visibleCalendarSyncError.ts b/apps/web/src/lib/service-clients/service-email/generated/schemas/visibleCalendarSyncError.ts
new file mode 100644
index 0000000000..489f33f1c8
--- /dev/null
+++ b/apps/web/src/lib/service-clients/service-email/generated/schemas/visibleCalendarSyncError.ts
@@ -0,0 +1,13 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * email_service
+ * OpenAPI spec version: 0.1.0
+ */
+
+/**
+ * A persistent sync failure isolated to this calendar, surfaced so the
+settings row can badge it. `None` while the calendar is syncing
+normally or a failure has not yet crossed the persistence threshold.
+ */
+export type VisibleCalendarSyncError = string | null;
diff --git a/apps/web/src/lib/service-clients/service-email/openapi.json b/apps/web/src/lib/service-clients/service-email/openapi.json
index 5166b29ae5..7b0c3e4b31 100644
--- a/apps/web/src/lib/service-clients/service-email/openapi.json
+++ b/apps/web/src/lib/service-clients/service-email/openapi.json
@@ -6609,6 +6609,10 @@
"name": {
"type": "string",
"description": "Provider display name."
+ },
+ "syncError": {
+ "type": ["string", "null"],
+ "description": "A persistent sync failure isolated to this calendar, surfaced so the\nsettings row can badge it. `None` while the calendar is syncing\nnormally or a failure has not yet crossed the persistence threshold."
}
}
}
diff --git a/crates/calendar_events/src/domain/models.rs b/crates/calendar_events/src/domain/models.rs
index 5d5ca6f8ec..3ab7888578 100644
--- a/crates/calendar_events/src/domain/models.rs
+++ b/crates/calendar_events/src/domain/models.rs
@@ -1135,6 +1135,10 @@ pub struct VisibleCalendar {
/// Whether this is one of Google's shared system calendars (holidays,
/// birthdays) the account subscribes to rather than one a person maintains.
pub is_subscription: bool,
+ /// A persistent sync failure isolated to this calendar, surfaced so the
+ /// settings row can badge it. `None` while the calendar is syncing
+ /// normally or a failure has not yet crossed the persistence threshold.
+ pub sync_error: Option<String>,
/// Default reminders applied to events that keep `useDefault`.
pub default_reminders: Vec<EventReminderOverride>,
}
@@ -1199,6 +1203,11 @@ pub struct StoredGoogleCalendar {
/// snapshot; a daily cadence keeps them fresh without that churn.
pub const SYSTEM_CALENDAR_SYNC_INTERVAL: chrono::Duration = chrono::Duration::hours(24);
+/// Consecutive isolated sync failures a calendar must accumulate before its
+/// error surfaces to the user. A one-off transient failure clears on the next
+/// successful poll, so only a persistent failure earns a settings-row badge.
+pub const CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD: i32 = 3;
+
/// Whether a provider calendar is one of Google's shared system calendars
/// (`en.usa#[email protected]` and friends) rather than a
/// calendar a person maintains.
diff --git a/crates/calendar_events/src/domain/mutations/test.rs b/crates/calendar_events/src/domain/mutations/test.rs
index c1f2dbc7a0..94550f93b8 100644
--- a/crates/calendar_events/src/domain/mutations/test.rs
+++ b/crates/calendar_events/src/domain/mutations/test.rs
@@ -275,6 +275,17 @@ impl CalendarRepository for FakeRepo {
unreachable!()
}
+ async fn record_google_calendar_sync_error(
+ &self,
+ _key: CalendarBackfillJobKey,
+ _lease_token: Uuid,
+ _account_id: Uuid,
+ _calendar_id: Uuid,
+ _message: &str,
+ ) -> Result<(), rootcause::Report> {
+ unreachable!()
+ }
+
async fn record_watch_channel(
&self,
_key: CalendarBackfillJobKey,
diff --git a/crates/calendar_events/src/domain/ports.rs b/crates/calendar_events/src/domain/ports.rs
index e2250f44a7..82439be601 100644
--- a/crates/calendar_events/src/domain/ports.rs
+++ b/crates/calendar_events/src/domain/ports.rs
@@ -54,6 +54,11 @@ impl GoogleProviderError {
pub fn kind(&self) -> GoogleProviderErrorKind {
self.kind
}
+
+ /// Return the provider failure detail, without the Display prefix.
+ pub fn message(&self) -> &str {
+ &self.message
+ }
}
/// Stable identifiers and sync policy for one provider calendar fetch.
@@ -481,6 +486,20 @@ pub trait CalendarRepository: Send + Sync + 'static {
events_upserted: usize,
) -> impl Future<Output = Result<Vec<RetiredCalendarEvent>, Report>> + Send;
+ /// Record a provider failure isolated to one calendar under the backfill's
+ /// fencing token: store the message, stamp the time, and bump the
+ /// consecutive-failure counter that gates whether the failure surfaces to
+ /// the user. The calendar's sync state is left untouched so the next poll
+ /// retries it. A successful `commit_google_calendar_sync` clears all three.
+ fn record_google_calendar_sync_error(
+ &self,
+ key: CalendarBackfillJobKey,
+ lease_token: Uuid,
+ account_id: Uuid,
+ calendar_id: Uuid,
+ message: &str,
+ ) -> impl Future<Output = Result<(), Report>> + Send;
+
/// Record a freshly opened push channel for one calendar under the
/// backfill's fencing token.
fn record_watch_channel(
diff --git a/crates/calendar_events/src/domain/service.rs b/crates/calendar_events/src/domain/service.rs
index e4b5bfb612..2b8ef9b98f 100644
--- a/crates/calendar_events/src/domain/service.rs
+++ b/crates/calendar_events/src/domain/service.rs
@@ -641,6 +641,12 @@ where
.await
.map_err(|error| -> Report { rootcause::report!(error).into() })?;
let mut calendar_ids = Vec::with_capacity(calendars.len());
+ // One calendar's provider failure is recorded and skipped rather than
+ // failing the account. The run fails only when no calendar is healthy,
+ // so the coordinator can still classify a wholesale outage.
+ let mut any_calendar_healthy = false;
+ let mut isolated_failures: Vec<(Uuid, String)> = Vec::new();
+ let mut last_isolated_error: Option<GoogleProviderError> = None;
for provider_calendar in calendars {
let provider_calendar_id = provider_calendar.provider_calendar_id.clone();
@@ -660,10 +666,11 @@ where
synced_at > Utc::now() - super::models::SYSTEM_CALENDAR_SYNC_INTERVAL
})
{
+ any_calendar_healthy = true;
continue;
}
let plan = stored_calendar.sync_plan(&range);
- let batch = self
+ let batch = match self
.provider
.sync_events(
access_token,
@@ -682,7 +689,34 @@ where
},
)
.await
- .map_err(|error| -> Report { rootcause::report!(error).into() })?;
+ {
+ Ok(batch) => batch,
+ Err(error) => {
+ // A bad or insufficient grant is account-wide, not
+ // calendar-local: surface it immediately so the coordinator
+ // prompts reauthorization rather than marking the account
+ // ready off the calendars that happened to sync.
+ if error.kind() == GoogleProviderErrorKind::ReauthRequired {
+ return Err(rootcause::report!(error).into());
+ }
+ tracing::warn!(
+ error=?error,
+ calendar_id=%calendar_id,
+ "isolating a failed Google Calendar and continuing the account sync"
+ );
+ isolated_failures.push((calendar_id, error.message().to_owned()));
+ // A retryable failure outranks a permanent one so a total
+ // failure still surfaces as retryable.
+ last_isolated_error = Some(match last_isolated_error {
+ Some(previous) if previous.kind() != GoogleProviderErrorKind::Permanent => {
+ previous
+ }
+ _ => error,
+ });
+ continue;
+ }
+ };
+ any_calendar_healthy = true;
let mut calendar_count = 0;
for upsert in batch.upserts {
if let Err(error) = validate_upsert(&upsert) {
@@ -788,6 +822,35 @@ where
}
}
+ // No calendar is healthy: a wholesale outage the coordinator classifies
+ // for retry. The account-level failure carries it, so no calendar is
+ // badged for it.
+ if !any_calendar_healthy && let Some(error) = last_isolated_error {
+ return Err(rootcause::report!(error).into());
+ }
+
+ // Record each isolated failure for the settings badge, leaving the
+ // calendar's sync state untouched so the next poll retries it.
+ for (calendar_id, message) in isolated_failures {
+ self.repository
+ .record_google_calendar_sync_error(
+ key,
+ lease_token,
+ account_id,
+ calendar_id,
+ &message,
+ )
+ .await
+ .inspect_err(|error| {
+ tracing::warn!(
+ error=?error,
+ calendar_id=%calendar_id,
+ "failed to record isolated Google Calendar sync error"
+ );
+ })
+ .ok();
+ }
+
// A calendar dropped from the provider's list retires its sources, so
// the events it backed announce their own fate here too.
let retired = self
diff --git a/crates/calendar_events/src/domain/service/test.rs b/crates/calendar_events/src/domain/service/test.rs
index ac20d062d5..cf6e125ec8 100644
--- a/crates/calendar_events/src/domain/service/test.rs
+++ b/crates/calendar_events/src/domain/service/test.rs
@@ -27,6 +27,8 @@ struct FakeRepo {
/// Retirements the snapshot commit reports back, standing in for sources
/// the change feed cancelled or a full snapshot no longer observed.
sync_retirements: Vec<RetiredCalendarEvent>,
+ /// Provider calendars whose isolated sync failure was recorded, in order.
+ recorded_sync_errors: Arc<Mutex<Vec<String>>>,
}
impl CalendarRepository for FakeRepo {
@@ -192,6 +194,21 @@ impl CalendarRepository for FakeRepo {
Ok(Vec::new())
}
+ async fn record_google_calendar_sync_error(
+ &self,
+ _key: CalendarBackfillJobKey,
+ _lease_token: Uuid,
+ _account_id: Uuid,
+ _calendar_id: Uuid,
+ message: &str,
+ ) -> Result<(), Report> {
+ self.recorded_sync_errors
+ .lock()
+ .unwrap()
+ .push(message.to_string());
+ Ok(())
+ }
+
async fn record_watch_channel(
&self,
_key: CalendarBackfillJobKey,
@@ -216,6 +233,27 @@ impl CalendarRepository for FakeRepo {
}
}
+fn provider_calendar(id: &str, primary: bool) -> ProviderCalendar {
+ ProviderCalendar {
+ provider_calendar_id: id.to_string(),
+ name: id.to_string(),
+ description: None,
+ time_zone: Some("UTC".to_string()),
+ color: None,
+ access_role: Some("owner".to_string()),
+ is_primary: primary,
+ is_selected: true,
+ default_reminders: Vec::new(),
+ }
+}
+
+fn two_calendars() -> Vec<ProviderCalendar> {
+ vec![
+ provider_calendar("primary", true),
+ provider_calendar("team", false),
+ ]
+}
+
fn valid_upsert() -> CalendarEventUpsert {
let event_id = Uuid::now_v7();
let starts_at = Utc.with_ymd_and_hms(2026, 7, 24, 14, 0, 0).unwrap();
@@ -518,18 +556,7 @@ impl GoogleCalendarProvider for PartialFailureGoogleProvider {
_access_token: &str,
_email_link_id: Uuid,
) -> Result<Vec<ProviderCalendar>, GoogleProviderError> {
- let calendar = |id: &str, primary: bool| ProviderCalendar {
- provider_calendar_id: id.to_string(),
- name: id.to_string(),
- description: None,
- time_zone: Some("UTC".to_string()),
- color: None,
- access_role: Some("owner".to_string()),
- is_primary: primary,
- is_selected: true,
- default_reminders: Vec::new(),
- };
- Ok(vec![calendar("primary", true), calendar("team", false)])
+ Ok(two_calendars())
}
async fn sync_events(
@@ -567,17 +594,196 @@ impl GoogleCalendarProvider for PartialFailureGoogleProvider {
}
}
+/// Fails the poll of every calendar it lists.
+#[derive(Clone)]
+struct TotalFailureGoogleProvider;
+
+impl GoogleCalendarProvider for TotalFailureGoogleProvider {
+ async fn list_calendars(
+ &self,
+ _access_token: &str,
+ _email_link_id: Uuid,
+ ) -> Result<Vec<ProviderCalendar>, GoogleProviderError> {
+ Ok(two_calendars())
+ }
+
+ async fn sync_events(
+ &self,
+ _access_token: &str,
+ _context: GoogleEventSyncContext,
+ ) -> Result<GoogleEventSyncBatch, GoogleProviderError> {
+ Err(GoogleProviderError::new(
+ GoogleProviderErrorKind::Transient,
+ "every calendar's poll failed",
+ ))
+ }
+
+ async fn watch_calendar(
+ &self,
+ _access_token: &str,
+ _email_link_id: Uuid,
+ _provider_calendar_id: &str,
+ _channel_id: Uuid,
+ _config: &GoogleWatchConfig,
+ ) -> Result<GoogleWatchChannel, GoogleProviderError> {
+ unreachable!("watch is disabled in these tests")
+ }
+}
+
+/// Syncs the first calendar with one change, then returns a reauthorization
+/// signal on the second — standing in for a grant that lost calendar scope
+/// mid-run.
+#[derive(Clone)]
+struct ReauthOnColleagueCalendarProvider;
+
+impl GoogleCalendarProvider for ReauthOnColleagueCalendarProvider {
+ async fn list_calendars(
+ &self,
+ _access_token: &str,
+ _email_link_id: Uuid,
+ ) -> Result<Vec<ProviderCalendar>, GoogleProviderError> {
+ Ok(two_calendars())
+ }
+
+ async fn sync_events(
+ &self,
+ _access_token: &str,
+ context: GoogleEventSyncContext,
+ ) -> Result<GoogleEventSyncBatch, GoogleProviderError> {
+ if context.target.provider_calendar_id == "team" {
+ return Err(GoogleProviderError::new(
+ GoogleProviderErrorKind::ReauthRequired,
+ "insufficient permissions",
+ ));
+ }
+ let mut upsert = valid_upsert();
+ let CalendarEventSource::Google(source) = &mut upsert.source;
+ source.calendar_id = Uuid::nil();
+ Ok(GoogleEventSyncBatch {
+ upserts: vec![upsert],
+ observed_provider_event_ids: Some(vec!["provider-event".to_string()]),
+ next_sync_token: "next".to_string(),
+ materialized_range: Some(context.target.range),
+ cancelled_provider_event_ids: Vec::new(),
+ })
+ }
+
+ async fn watch_calendar(
+ &self,
+ _access_token: &str,
+ _email_link_id: Uuid,
+ _provider_calendar_id: &str,
+ _channel_id: Uuid,
+ _config: &GoogleWatchConfig,
+ ) -> Result<GoogleWatchChannel, GoogleProviderError> {
+ unreachable!("watch is disabled in these tests")
+ }
+}
+
+/// Fails both calendars, one permanently and one transiently.
+#[derive(Clone)]
+struct MixedTotalFailureGoogleProvider;
+
+impl GoogleCalendarProvider for MixedTotalFailureGoogleProvider {
+ async fn list_calendars(
+ &self,
+ _access_token: &str,
+ _email_link_id: Uuid,
+ ) -> Result<Vec<ProviderCalendar>, GoogleProviderError> {
+ Ok(two_calendars())
+ }
+
+ async fn sync_events(
+ &self,
+ _access_token: &str,
+ context: GoogleEventSyncContext,
+ ) -> Result<GoogleEventSyncBatch, GoogleProviderError> {
+ // The transient failure comes first, so last-write-wins would surface
+ // the permanent one and fail the assertion.
+ let kind = if context.target.provider_calendar_id == "primary" {
+ GoogleProviderErrorKind::Transient
+ } else {
+ GoogleProviderErrorKind::Permanent
+ };
+ Err(GoogleProviderError::new(kind, "failed"))
+ }
+
+ async fn watch_calendar(
+ &self,
+ _access_token: &str,
+ _email_link_id: Uuid,
+ _provider_calendar_id: &str,
+ _channel_id: Uuid,
+ _config: &GoogleWatchConfig,
+ ) -> Result<GoogleWatchChannel, GoogleProviderError> {
+ unreachable!("watch is disabled in these tests")
+ }
+}
+
#[tokio::test]
-async fn partial_progress_reports_changes_when_a_later_calendar_fails() {
+async fn one_failed_calendar_is_isolated_and_the_account_completes() {
let lifecycle = FakeLifecycle::claimed();
+ let repository = FakeRepo::default();
+ let recorded_sync_errors = repository.recorded_sync_errors.clone();
let coordinator = GoogleCalendarBackfillCoordinator::new(
- FakeRepo::default(),
+ repository,
PartialFailureGoogleProvider,
lifecycle.clone(),
NoopMacroEventBroker,
None,
);
+ let mut report = GoogleBackfillRunReport::default();
+ coordinator
+ .run(
+ CalendarBackfillJobKey {
+ job_id: Uuid::now_v7(),
+ email_link_id: Uuid::now_v7(),
+ },
+ "secret",
+ OccurrenceRange::maintenance_horizon(Utc::now()),
+ &mut report,
+ )
+ .await
+ .expect("a single unreadable calendar must not fail the whole account");
+
+ assert_eq!(
+ report.events_upserted, 1,
+ "the healthy calendar's commit must still land"
+ );
+ assert!(report.changed());
+ let recorded = recorded_sync_errors.lock().unwrap();
+ assert_eq!(
+ recorded.len(),
+ 1,
+ "the failed calendar records its own error"
+ );
+ assert!(recorded[0].contains("the second calendar's poll failed"));
+ assert_eq!(
+ lifecycle.completions.lock().unwrap().len(),
+ 1,
+ "the run completes rather than failing"
+ );
+ assert!(lifecycle.failures.lock().unwrap().is_empty());
+}
+
+/// Every calendar failing is a wholesale outage: the run fails so the
+/// coordinator can classify it (transient here) for retry, rather than
+/// quietly reporting success on an account that synced nothing.
+#[tokio::test]
+async fn every_calendar_failing_fails_the_run_for_retry() {
+ let lifecycle = FakeLifecycle::claimed();
+ let repository = FakeRepo::default();
+ let recorded_sync_errors = repository.recorded_sync_errors.clone();
+ let coordinator = GoogleCalendarBackfillCoordinator::new(
+ repository,
+ TotalFailureGoogleProvider,
+ lifecycle.clone(),
+ NoopMacroEventBroker,
+ None,
+ );
+
let mut report = GoogleBackfillRunReport::default();
let error = coordinator
.run(
@@ -597,11 +803,102 @@ async fn partial_progress_reports_changes_when_a_later_calendar_fails() {
error,
GoogleCalendarBackfillRunError::Retryable(_)
));
+ assert_eq!(report.events_upserted, 0);
+ assert!(
+ recorded_sync_errors.lock().unwrap().is_empty(),
+ "a wholesale outage is recorded on the account, not on every calendar"
+ );
+ assert!(lifecycle.completions.lock().unwrap().is_empty());
+}
+
+/// A reauthorization signal on one calendar is account-wide: it fails the run
+/// immediately (so the user is prompted) rather than being isolated and
+/// swallowed by the other calendars completing.
+#[tokio::test]
+async fn a_reauth_signal_on_one_calendar_fails_the_run_immediately() {
+ let lifecycle = FakeLifecycle::claimed();
+ let repository = FakeRepo::default();
+ let recorded_sync_errors = repository.recorded_sync_errors.clone();
+ let coordinator = GoogleCalendarBackfillCoordinator::new(
+ repository,
+ ReauthOnColleagueCalendarProvider,
+ lifecycle.clone(),
+ NoopMacroEventBroker,
+ None,
+ );
+
+ let mut report = GoogleBackfillRunReport::default();
+ let error = coordinator
+ .run(
+ CalendarBackfillJobKey {
+ job_id: Uuid::now_v7(),
+ email_link_id: Uuid::now_v7(),
+ },
+ "secret",
+ OccurrenceRange::maintenance_horizon(Utc::now()),
+ &mut report,
+ )
+ .await
+ .unwrap_err();
+
+ assert!(matches!(
+ error,
+ GoogleCalendarBackfillRunError::ReauthRequired { .. }
+ ));
assert_eq!(
report.events_upserted, 1,
"the first calendar's durable commit must surface through the failed run"
);
assert!(report.changed());
+ assert_eq!(
+ lifecycle.failures.lock().unwrap().as_slice(),
+ [CalendarBackfillFailureDisposition::CalendarPermissionRequired]
+ );
+ assert!(
+ recorded_sync_errors.lock().unwrap().is_empty(),
+ "a reauth signal propagates before it is recorded as an isolated failure"
+ );
+ assert!(lifecycle.completions.lock().unwrap().is_empty());
+}
+
+/// When every calendar fails with a mix of kinds, the surfaced error prefers a
+/// retryable one so one calendar's permanent error cannot stop the whole inbox
+/// from polling.
+#[tokio::test]
+async fn a_wholesale_failure_prefers_a_retryable_error() {
+ let lifecycle = FakeLifecycle::claimed();
+ let coordinator = GoogleCalendarBackfillCoordinator::new(
+ FakeRepo::default(),
+ MixedTotalFailureGoogleProvider,
+ lifecycle.clone(),
+ NoopMacroEventBroker,
+ None,
+ );
+
+ let mut report = GoogleBackfillRunReport::default();
+ let error = coordinator
+ .run(
+ CalendarBackfillJobKey {
+ job_id: Uuid::now_v7(),
+ email_link_id: Uuid::now_v7(),
+ },
+ "secret",
+ OccurrenceRange::maintenance_horizon(Utc::now()),
+ &mut report,
+ )
+ .await
+ .unwrap_err();
+
+ assert!(
+ matches!(error, GoogleCalendarBackfillRunError::Retryable(_)),
+ "a permanent failure on one calendar must not stop polling when another was transient"
+ );
+ assert_eq!(
+ lifecycle.failures.lock().unwrap().as_slice(),
+ [CalendarBackfillFailureDisposition::Retry]
+ );
}
#[tokio::test]
@@ -854,6 +1151,91 @@ async fn freshly_synced_system_calendars_are_skipped() {
assert_eq!(lifecycle.completions.lock().unwrap().len(), 1);
}
+/// Lists a freshly synced system calendar beside a primary whose poll fails,
+/// so the only calendar attempted this run errors.
+#[derive(Clone)]
+struct FailingPrimaryBesideSystemCalendarProvider;
+
+impl GoogleCalendarProvider for FailingPrimaryBesideSystemCalendarProvider {
+ async fn list_calendars(
+ &self,
+ _access_token: &str,
+ _email_link_id: Uuid,
+ ) -> Result<Vec<ProviderCalendar>, GoogleProviderError> {
+ Ok(vec![
+ provider_calendar("primary", true),
+ provider_calendar("en.usa#[email protected]", false),
+ ])
+ }
+
+ async fn sync_events(
+ &self,
+ _access_token: &str,
+ context: GoogleEventSyncContext,
+ ) -> Result<GoogleEventSyncBatch, GoogleProviderError> {
+ if context.target.provider_calendar_id != "primary" {
+ unreachable!("freshly synced system calendars must not sync")
+ }
+ Err(GoogleProviderError::new(
+ GoogleProviderErrorKind::Transient,
+ "the primary calendar's poll failed",
+ ))
+ }
+
+ async fn watch_calendar(
+ &self,
+ _access_token: &str,
+ _email_link_id: Uuid,
+ _provider_calendar_id: &str,
+ _channel_id: Uuid,
+ _config: &GoogleWatchConfig,
+ ) -> Result<GoogleWatchChannel, GoogleProviderError> {
+ unreachable!("watch is disabled in these tests")
+ }
+}
+
+/// A freshly synced system calendar counts as healthy, so a failing calendar
+/// beside it is isolated rather than read as a wholesale outage.
+#[tokio::test]
+async fn a_fresh_system_calendar_keeps_a_failing_calendar_isolated() {
+ let lifecycle = FakeLifecycle::claimed();
+ let repository = FakeRepo {
+ stored_synced_at: Some(Utc::now()),
+ ..FakeRepo::default()
+ };
+ let recorded_sync_errors = repository.recorded_sync_errors.clone();
+ let coordinator = GoogleCalendarBackfillCoordinator::new(
+ repository,
+ FailingPrimaryBesideSystemCalendarProvider,
+ lifecycle.clone(),
+ NoopMacroEventBroker,
+ None,
+ );
+
+ let mut report = GoogleBackfillRunReport::default();
+ coordinator
+ .run(
+ CalendarBackfillJobKey {
+ job_id: Uuid::now_v7(),
+ email_link_id: Uuid::now_v7(),
+ },
+ "secret",
+ OccurrenceRange::maintenance_horizon(Utc::now()),
+ &mut report,
+ )
+ .await
+ .expect("a fresh system calendar keeps the account healthy");
+
+ assert_eq!(report, GoogleBackfillRunReport::default());
+ assert_eq!(
+ recorded_sync_errors.lock().unwrap().as_slice(),
+ ["the primary calendar's poll failed"]
+ );
+ assert_eq!(lifecycle.completions.lock().unwrap().len(), 1);
+ assert!(lifecycle.failures.lock().unwrap().is_empty());
+}
+
#[derive(Clone)]
struct HangingGoogleProvider;
diff --git a/crates/calendar_events/src/inbound/toolset/test.rs b/crates/calendar_events/src/inbound/toolset/test.rs
index 15b14be918..5586d95e87 100644
--- a/crates/calendar_events/src/inbound/toolset/test.rs
+++ b/crates/calendar_events/src/inbound/toolset/test.rs
@@ -987,6 +987,7 @@ async fn list_calendars_maps_visible_calendars() {
is_primary: true,
is_writable: true,
is_subscription: false,
+ sync_error: None,
default_reminders: Vec::new(),
}]),
..Default::default()
diff --git a/crates/calendar_events/src/outbound/google.rs b/crates/calendar_events/src/outbound/google.rs
index 78ad961f3d..952668bfd5 100644
--- a/crates/calendar_events/src/outbound/google.rs
+++ b/crates/calendar_events/src/outbound/google.rs
@@ -88,7 +88,8 @@ impl<G: GoogleRequestGate> GoogleCalendarClient<G> {
if let Some(token) = &page_token {
request = request.query(&[("pageToken", token)]);
}
- let page: GoogleCalendarListResponse = send_google(request).await?;
+ let page: GoogleCalendarListResponse =
+ send_google(GoogleRequestKind::AccountRead, request).await?;
result.extend(page.items);
page_token = page.next_page_token;
if page_token.is_none() {
@@ -129,7 +130,8 @@ impl<G: GoogleRequestGate> GoogleCalendarClient<G> {
if let Some(token) = &page_token {
request = request.query(&[("pageToken", token)]);
}
- let page: GoogleEventListResponse = send_google(request).await?;
+ let page: GoogleEventListResponse =
+ send_google(GoogleRequestKind::Read, request).await?;
result.extend(page.items);
page_token = page.next_page_token;
if page_token.is_none() {
@@ -176,7 +178,8 @@ impl<G: GoogleRequestGate> GoogleCalendarClient<G> {
if let Some(token) = &page_token {
request = request.query(&[("pageToken", token)]);
}
- let page: GoogleEventListResponse = send_google(request).await?;
+ let page: GoogleEventListResponse =
+ send_google(GoogleRequestKind::Read, request).await?;
result.extend(page.items);
page_token = page.next_page_token;
if page_token.is_none() {
@@ -218,7 +221,11 @@ impl<G: GoogleRequestGate> GoogleCalendarClient<G> {
}
if !status.is_success() {
let body = response.text().await.map_err(provider_transport_error)?;
- return Err(provider_response_error(status, &body));
+ return Err(provider_response_error(
+ GoogleRequestKind::Read,
+ status,
+ &body,
+ ));
}
response
.json()
@@ -252,7 +259,8 @@ impl<G: GoogleRequestGate> GoogleCalendarClient<G> {
if let Some(token) = &page_token {
request = request.query(&[("pageToken", token)]);
}
- let page: GoogleEventListResponse = send_google(request).await?;
+ let page: GoogleEventListResponse =
+ send_google(GoogleRequestKind::Read, request).await?;
result.extend(page.items);
page_token = page.next_page_token;
if page_token.is_none() {
@@ -292,7 +300,24 @@ impl<G: GoogleRequestGate> GoogleCalendarClient<G> {
if let Some(token) = &page_token {
request = request.query(&[("pageToken", token)]);
}
- let page: GoogleEventListResponse = send_google(request).await?;
+ let response = request.send().await.map_err(provider_transport_error)?;
+ let status = response.status();
+ if !status.is_success() {
+ let body = response.text().await.map_err(provider_transport_error)?;
+ let error = provider_response_error(GoogleRequestKind::Read, status, &body);
+ // A missing series master is not a Google quirk to retry.
+ if status == StatusCode::NOT_FOUND
+ && error.kind() == GoogleProviderErrorKind::Transient
+ {
+ return Err(GoogleProviderError::new(
+ GoogleProviderErrorKind::Permanent,
+ error.message(),
+ ));
+ }
+ return Err(error);
+ }
+ let page: GoogleEventListResponse =
+ response.json().await.map_err(provider_transport_error)?;
result.extend(page.items);
page_token = page.next_page_token;
if page_token.is_none() {
@@ -303,14 +328,34 @@ impl<G: GoogleRequestGate> GoogleCalendarClient<G> {
}
}
+/// Whether a request reads calendar state or mutates it. An unrecognized 4xx
+/// means different things for each: our read requests are well-formed, so an
+/// unknown 4xx from a read is a Google-side quirk (like the undocumented 412)
+/// and is retried. A mutation's unknown 4xx is a real client rejection and
+/// stays permanent so it does not retry forever.
+#[derive(Clone, Copy)]
+enum GoogleRequestKind {
+ /// A read scoped to one calendar. Retrying is safe because the backfill
+ /// loop isolates a calendar that keeps failing, so a deterministic 4xx is
+ /// bounded to a badge rather than the whole account.
+ Read,
+ /// A read at account scope (the calendar list). Nothing isolates it, so a
+ /// deterministic unknown 4xx retried here would hold the account at
+ /// `pending` indefinitely; it stays terminal like a mutation. The
+ /// undocumented 412 is still classified retryable regardless of kind.
+ AccountRead,
+ Mutation,
+}
+
async fn send_google<T: DeserializeOwned>(
+ kind: GoogleRequestKind,
request: RequestBuilder,
) -> Result<T, GoogleProviderError> {
let response = request.send().await.map_err(provider_transport_error)?;
let status = response.status();
if !status.is_success() {
let body = response.text().await.map_err(provider_transport_error)?;
- return Err(provider_response_error(status, &body));
+ return Err(provider_response_error(kind, status, &body));
}
response.json().await.map_err(provider_transport_error)
}
@@ -319,7 +364,11 @@ fn provider_transport_error(error: reqwest::Error) -> GoogleProviderError {
GoogleProviderError::new(GoogleProviderErrorKind::Transient, format!("{error:?}"))
}
-fn provider_response_error(status: StatusCode, body: &str) -> GoogleProviderError {
+fn provider_response_error(
+ request_kind: GoogleRequestKind,
+ status: StatusCode,
+ body: &str,
+) -> GoogleProviderError {
let payload = serde_json::from_str::<GoogleErrorResponse>(body).ok();
let reasons: Vec<_> = payload
.as_ref()
@@ -339,6 +388,11 @@ fn provider_response_error(status: StatusCode, body: &str) -> GoogleProviderErro
} else if status == StatusCode::UNAUTHORIZED
|| status == StatusCode::REQUEST_TIMEOUT
|| status == StatusCode::TOO_MANY_REQUESTS
+ // Google returns an undocumented 412 ("Precondition check failed.")
+ // transiently, and we never send an If-Match, so it is never the
+ // documented precondition failure. Reads already fall through to
+ // retryable below, so this arm keeps a mutation's 412 retryable too.
+ || status == StatusCode::PRECONDITION_FAILED
|| status.is_server_error()
|| reasons.contains(&"authError")
|| reasons.contains(&"backendError")
@@ -349,13 +403,35 @@ fn provider_response_error(status: StatusCode, body: &str) -> GoogleProviderErro
{
GoogleProviderErrorKind::Transient
} else {
- GoogleProviderErrorKind::Permanent
+ match request_kind {
+ GoogleRequestKind::Read => GoogleProviderErrorKind::Transient,
+ GoogleRequestKind::AccountRead | GoogleRequestKind::Mutation => {
+ GoogleProviderErrorKind::Permanent
+ }
+ }
};
- let message = payload
- .map(|payload| payload.error.message)
+ let message = provider_error_message(payload.as_ref(), status, &reasons);
+ GoogleProviderError::new(kind, message)
+}
+
+/// Compose the provider error message, keeping Google's `reason` strings
+/// alongside the human-readable `message` so a failure is diagnosable from a
+/// stored `last_error` alone.
+fn provider_error_message(
+ payload: Option<&GoogleErrorResponse>,
+ status: StatusCode,
+ reasons: &[&str],
+) -> String {
+ let base = payload
+ .map(|payload| &payload.error.message)
.filter(|message| !message.is_empty())
+ .cloned()
.unwrap_or_else(|| format!("Google Calendar returned HTTP {status}"));
- GoogleProviderError::new(kind, message)
+ if reasons.is_empty() {
+ base
+ } else {
+ format!("{base} (reasons: {})", reasons.join(", "))
+ }
}
impl<G: GoogleRequestGate> GoogleCalendarProvider for GoogleCalendarClient<G> {
@@ -496,6 +572,7 @@ impl<G: GoogleRequestGate> GoogleCalendarProvider for GoogleCalendarClient<G> {
let calendar = urlencoding::encode(provider_calendar_id);
self.gate.acquire(email_link_id).await?;
let response: GoogleChannelResponse = send_google(
+ GoogleRequestKind::Mutation,
self.client
.post(format!(
"{GOOGLE_CALENDAR_API}/calendars/{calendar}/events/watch"
@@ -788,11 +865,25 @@ impl<G: GoogleRequestGate> GoogleCalendarClient<G> {
/// Recurring series are re-read from the provider so Google stays the
/// recurrence expansion authority, exactly like ingestion. `None` means
/// the series master disappeared between the mutation and the refresh.
+ ///
+ /// The write has already landed by the time this runs, so no failure here
+ /// may surface as retryable — see [`non_retryable_after_write`].
async fn mutation_readback(
&self,
access_token: &str,
target: &GoogleCalendarTarget,
event: GoogleEvent,
+ ) -> Result<Option<CalendarEventUpsert>, GoogleProviderError> {
+ self.readback_mutation(access_token, target, event)
+ .await
+ .map_err(non_retryable_after_write)
+ }
+
+ async fn readback_mutation(
+ &self,
+ access_token: &str,
+ target: &GoogleCalendarTarget,
+ event: GoogleEvent,
) -> Result<Option<CalendarEventUpsert>, GoogleProviderError> {
let event = self
.resolve_pending_conference(access_token, target, event)
@@ -844,15 +935,33 @@ impl<G: GoogleRequestGate> GoogleCalendarClient<G> {
return Ok(());
}
let body = response.text().await.map_err(provider_transport_error)?;
- Err(provider_response_error(status, &body))
+ Err(provider_response_error(
+ GoogleRequestKind::Mutation,
+ status,
+ &body,
+ ))
}
/// Refresh a series after reshaping it, mapping the outcome for callers.
+ ///
+ /// The reshaping write has already landed, so no failure here may surface
+ /// as retryable — see [`non_retryable_after_write`].
async fn series_outcome(
&self,
access_token: &str,
target: &GoogleCalendarTarget,
master_provider_event_id: &str,
+ ) -> Result<GoogleSeriesMutationOutcome, GoogleProviderError> {
+ self.refreshed_series_outcome(access_token, target, master_provider_event_id)
+ .await
+ .map_err(non_retryable_after_write)
+ }
+
+ async fn refreshed_series_outcome(
+ &self,
+ access_token: &str,
+ target: &GoogleCalendarTarget,
+ master_provider_event_id: &str,
) -> Result<GoogleSeriesMutationOutcome, GoogleProviderError> {
match self
.refresh_series(access_token, target, master_provider_event_id, None)
@@ -895,7 +1004,11 @@ impl<G: GoogleRequestGate> GoogleCalendarClient<G> {
}
if !status.is_success() {
let body = response.text().await.map_err(provider_transport_error)?;
- return Err(provider_response_error(status, &body));
+ return Err(provider_response_error(
+ GoogleRequestKind::Mutation,
+ status,
+ &body,
+ ));
}
response
.json()
@@ -925,8 +1038,11 @@ impl<G: GoogleRequestGate> GoogleCalendarMutationProvider for GoogleCalendarClie
.post(format!("{GOOGLE_CALENDAR_API}/calendars/{calendar}/events"))
.bearer_auth(access_token)
.query(&[SEND_UPDATES]);
- let created: GoogleEvent =
- send_google(with_conference_query(request, &body).json(&body)).await?;
+ let created: GoogleEvent = send_google(
+ GoogleRequestKind::Mutation,
+ with_conference_query(request, &body).json(&body),
+ )
+ .await?;
// The insert already happened and carries no idempotency key, so a
// readback miss must not surface as retryable: a client retry would
// POST a duplicate event.
@@ -1073,12 +1189,20 @@ impl<G: GoogleRequestGate> GoogleCalendarMutationProvider for GoogleCalendarClie
.and_then(google_start)
.is_some_and(|candidate| candidate == start)
});
- if let Some(instance) = matched {
- self.delete_event_raw(access_token, target, &instance.id)
- .await?;
+ // Only a landed delete makes the refresh non-retryable; when no
+ // instance matched nothing was written, so a flaky refresh may retry.
+ match matched {
+ Some(instance) => {
+ self.delete_event_raw(access_token, target, &instance.id)
+ .await?;
+ self.series_outcome(access_token, target, master_provider_event_id)
+ .await
+ }
+ None => {
+ self.refreshed_series_outcome(access_token, target, master_provider_event_id)
+ .await
+ }
}
- self.series_outcome(access_token, target, master_provider_event_id)
- .await
}
#[tracing::instrument(
@@ -1167,23 +1291,31 @@ impl<G: GoogleRequestGate> GoogleCalendarMutationProvider for GoogleCalendarClie
.await?
}
};
- if let Some(provider_event_id) = &patch_target {
+ let wrote = if let Some(provider_event_id) = &patch_target {
match self
.patch_actor_response(access_token, target, provider_event_id, actor, response)
.await?
{
- RsvpPatch::Applied => {}
+ RsvpPatch::Applied => true,
RsvpPatch::NotAttendee => return Ok(GoogleRsvpOutcome::NotAttendee),
RsvpPatch::Gone => return Ok(GoogleRsvpOutcome::Gone),
}
- }
+ } else {
+ false
+ };
// Every scope resolves by refreshing the series from Google, which
// owns recurrence expansion and now holds the exceptions just written.
- match self
- .series_outcome(access_token, target, master_provider_event_id)
- .await?
- {
+ // Only a landed patch makes that refresh non-retryable; with no
+ // occurrence to patch nothing was written, so a flaky refresh may retry.
+ let outcome = if wrote {
+ self.series_outcome(access_token, target, master_provider_event_id)
+ .await?
+ } else {
+ self.refreshed_series_outcome(access_token, target, master_provider_event_id)
+ .await?
+ };
+ match outcome {
GoogleSeriesMutationOutcome::Applied(upsert) => Ok(GoogleRsvpOutcome::Applied(upsert)),
GoogleSeriesMutationOutcome::SeriesDeleted | GoogleSeriesMutationOutcome::Gone => {
Ok(GoogleRsvpOutcome::Gone)
@@ -1217,7 +1349,11 @@ impl<G: GoogleRequestGate> GoogleCalendarMutationProvider for GoogleCalendarClie
return Ok(());
}
let body = response.text().await.map_err(provider_transport_error)?;
- Err(provider_response_error(status, &body))
+ Err(provider_response_error(
+ GoogleRequestKind::Mutation,
+ status,
+ &body,
+ ))
}
}
@@ -1386,6 +1522,28 @@ fn truncate_recurrence_lines(lines: &[String], cutoff: &EventStart) -> Vec<Strin
.collect()
}
+/// A read that runs after a mutation has already landed must never surface as
+/// retryable: the caller's retry would re-apply the write. `create_event`
+/// carries no idempotency key, so it would POST a duplicate event, and a
+/// re-sent patch re-notifies every guest. The write is in Google either way.
+/// The next sync converges the projection, so the demotion loses nothing.
+/// A reauthorization signal is kept — retrying would not help it either, and
+/// the caller maps it to a reauth prompt rather than a retry.
+fn non_retryable_after_write(error: GoogleProviderError) -> GoogleProviderError {
+ match error.kind() {
+ GoogleProviderErrorKind::Transient | GoogleProviderErrorKind::SyncTokenExpired => {
+ GoogleProviderError::new(
+ GoogleProviderErrorKind::Permanent,
+ format!(
+ "Google Calendar applied the change but has not returned it yet, refresh to see it: {}",
+ error.message()
+ ),
+ )
+ }
+ GoogleProviderErrorKind::Permanent | GoogleProviderErrorKind::ReauthRequired => error,
+ }
+}
+
fn mutation_normalization_error(error: Report) -> GoogleProviderError {
GoogleProviderError::new(
GoogleProviderErrorKind::Permanent,
diff --git a/crates/calendar_events/src/outbound/google/test.rs b/crates/calendar_events/src/outbound/google/test.rs
index de186cf963..f95a2e8e68 100644
--- a/crates/calendar_events/src/outbound/google/test.rs
+++ b/crates/calendar_events/src/outbound/google/test.rs
@@ -327,6 +327,7 @@ fn malformed_master_is_quarantined_without_deleting_its_provider_identity() {
#[test]
fn quota_forbidden_response_is_retryable() {
let error = provider_response_error(
+ GoogleRequestKind::Read,
StatusCode::FORBIDDEN,
r#"{"error":{"message":"Quota exceeded","errors":[{"reason":"userRateLimitExceeded"}]}}"#,
);
@@ -337,6 +338,7 @@ fn quota_forbidden_response_is_retryable() {
#[test]
fn insufficient_permissions_require_reauthorization() {
let error = provider_response_error(
+ GoogleRequestKind::Read,
StatusCode::FORBIDDEN,
r#"{"error":{"message":"Insufficient Permission","errors":[{"reason":"insufficientPermissions"}]}}"#,
);
@@ -347,6 +349,7 @@ fn insufficient_permissions_require_reauthorization() {
#[test]
fn expired_sync_token_requests_a_full_resync() {
let error = provider_response_error(
+ GoogleRequestKind::Read,
StatusCode::GONE,
r#"{"error":{"message":"Sync token is no longer valid","errors":[{"reason":"fullSyncRequired"}]}}"#,
);
@@ -357,6 +360,7 @@ fn expired_sync_token_requests_a_full_resync() {
#[test]
fn rejected_access_token_is_retryable_with_a_fresh_token() {
let error = provider_response_error(
+ GoogleRequestKind::Read,
StatusCode::UNAUTHORIZED,
r#"{"error":{"message":"Invalid Credentials","errors":[{"reason":"authError"}]}}"#,
);
@@ -365,8 +369,9 @@ fn rejected_access_token_is_retryable_with_a_fresh_token() {
}
#[test]
-fn unrelated_forbidden_response_is_permanent() {
+fn unrelated_forbidden_mutation_is_permanent() {
let error = provider_response_error(
+ GoogleRequestKind::Mutation,
StatusCode::FORBIDDEN,
r#"{"error":{"message":"Forbidden","errors":[{"reason":"forbidden"}]}}"#,
);
@@ -374,6 +379,98 @@ fn unrelated_forbidden_response_is_permanent() {
assert_eq!(error.kind(), GoogleProviderErrorKind::Permanent);
}
+#[test]
+fn undocumented_precondition_failure_is_retryable() {
+ // We never send an If-Match, so a 412 is retryable for mutations too.
+ for kind in [GoogleRequestKind::Read, GoogleRequestKind::Mutation] {
+ let error = provider_response_error(
+ kind,
+ StatusCode::PRECONDITION_FAILED,
+ r#"{"error":{"message":"Precondition check failed."}}"#,
+ );
+
+ assert_eq!(error.kind(), GoogleProviderErrorKind::Transient);
+ }
+}
+
+#[test]
+fn unknown_client_errors_are_retryable_on_reads_but_permanent_on_mutations() {
+ let read = provider_response_error(
+ GoogleRequestKind::Read,
+ StatusCode::CONFLICT,
+ r#"{"error":{"message":"Conflict"}}"#,
+ );
+ assert_eq!(read.kind(), GoogleProviderErrorKind::Transient);
+
+ let mutation = provider_response_error(
+ GoogleRequestKind::Mutation,
+ StatusCode::CONFLICT,
+ r#"{"error":{"message":"Conflict"}}"#,
+ );
+ assert_eq!(mutation.kind(), GoogleProviderErrorKind::Permanent);
+}
+
+/// The calendar list has no per-calendar isolation to bound a deterministic
+/// failure, so an unknown 4xx there stays terminal like a mutation, while the
+/// undocumented 412 is still retried at account scope.
+#[test]
+fn unknown_client_errors_on_the_account_read_stay_permanent() {
+ let forbidden = provider_response_error(
+ GoogleRequestKind::AccountRead,
+ StatusCode::FORBIDDEN,
+ r#"{"error":{"message":"Forbidden","errors":[{"reason":"forbidden"}]}}"#,
+ );
+ assert_eq!(forbidden.kind(), GoogleProviderErrorKind::Permanent);
+
+ let precondition = provider_response_error(
+ GoogleRequestKind::AccountRead,
+ StatusCode::PRECONDITION_FAILED,
+ r#"{"error":{"message":"Precondition check failed."}}"#,
+ );
+ assert_eq!(precondition.kind(), GoogleProviderErrorKind::Transient);
+}
+
+#[test]
+fn provider_error_keeps_google_reason_strings() {
+ let error = provider_response_error(
+ GoogleRequestKind::Read,
+ StatusCode::FORBIDDEN,
+ r#"{"error":{"message":"Forbidden","errors":[{"reason":"variableTermLimitExceeded"}]}}"#,
+ );
+
+ let rendered = error.to_string();
+ assert!(rendered.contains("Forbidden"), "{rendered}");
+ assert!(rendered.contains("variableTermLimitExceeded"), "{rendered}");
+}
+
+#[test]
+fn readback_failures_after_a_write_are_never_retryable() {
+ // A retry of the outer mutation would re-apply the write (a duplicate
+ // POST, re-notified guests), so a retryable readback failure is demoted.
+ for kind in [
+ GoogleProviderErrorKind::Transient,
+ GoogleProviderErrorKind::SyncTokenExpired,
+ ] {
+ let demoted = non_retryable_after_write(GoogleProviderError::new(kind, "Conflict"));
+ assert_eq!(demoted.kind(), GoogleProviderErrorKind::Permanent);
+ assert!(
+ demoted.message().contains("Conflict"),
+ "{}",
+ demoted.message()
+ );
+ }
+
+ // Kinds a retry would not help pass through untouched.
+ for kind in [
+ GoogleProviderErrorKind::Permanent,
+ GoogleProviderErrorKind::ReauthRequired,
+ ] {
+ let kept = non_retryable_after_write(GoogleProviderError::new(kind, "kept"));
+ assert_eq!(kept.kind(), kind);
+ assert_eq!(kept.message(), "kept");
+ }
+}
+
#[test]
fn cancelled_single_events_become_tombstones() {
let cancelled: GoogleEvent = serde_json::from_value(serde_json::json!({
diff --git a/crates/calendar_events/src/outbound/pg.rs b/crates/calendar_events/src/outbound/pg.rs
index 9e71503e38..7743cdd709 100644
--- a/crates/calendar_events/src/outbound/pg.rs
+++ b/crates/calendar_events/src/outbound/pg.rs
@@ -11,10 +11,11 @@ use uuid::Uuid;
use crate::domain::{
models::{
- ActorInboxes, AppliedGoogleGrant, AttendeeResponseStatus, CalendarAttendee,
- CalendarBackfillClaim, CalendarBackfillFailureDisposition, CalendarBackfillFailureOutcome,
- CalendarBackfillJob, CalendarBackfillJobKey, CalendarBackfillKind, CalendarCreationTarget,
- CalendarEvent, CalendarEventMutationTarget, CalendarEventOverride, CalendarEventSource,
+ ActorInboxes, AppliedGoogleGrant, AttendeeResponseStatus,
+ CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD, CalendarAttendee, CalendarBackfillClaim,
+ CalendarBackfillFailureDisposition, CalendarBackfillFailureOutcome, CalendarBackfillJob,
+ CalendarBackfillJobKey, CalendarBackfillKind, CalendarCreationTarget, CalendarEvent,
+ CalendarEventMutationTarget, CalendarEventOverride, CalendarEventSource,
CalendarEventSourceContent, CalendarEventUpsert, CalendarGrantIntent,
CalendarLinkTokenIdentity, CalendarMentionEvent, CalendarMentionPreview,
CalendarMentionRequestItem, CalendarOccurrence, CalendarOccurrenceCursor,
@@ -1384,6 +1385,9 @@ impl CalendarRepository for PgCalendarRepository {
UPDATE calendars
SET sync_token = $3,
synced_at = now(),
+ last_sync_error = NULL,
+ last_sync_error_at = NULL,
+ consecutive_sync_failures = 0,
materialized_starts_at = CASE
WHEN $4 THEN $5
ELSE materialized_starts_at
@@ -1427,6 +1431,38 @@ impl CalendarRepository for PgCalendarRepository {
Ok(retired)
}
+ #[tracing::instrument(skip(self, message), fields(job_id = %key.job_id), err)]
+ async fn record_google_calendar_sync_error(
+ &self,
+ key: CalendarBackfillJobKey,
+ lease_token: Uuid,
+ account_id: Uuid,
+ calendar_id: Uuid,
+ message: &str,
+ ) -> Result<(), Report> {
+ let mut tx = self.pool.begin().await.map_err(report)?;
+ fence_google_mutation_tx(&mut tx, key, lease_token, Some(account_id)).await?;
+ sqlx::query!(
+ r#"
+ UPDATE calendars
+ SET last_sync_error = $3,
+ last_sync_error_at = now(),
+ consecutive_sync_failures = consecutive_sync_failures + 1,
+ updated_at = now()
+ WHERE id = $1
+ AND account_id = $2
+ AND NOT is_deleted
+ "#,
+ calendar_id,
+ account_id,
+ message,
+ )
+ .execute(&mut *tx)
+ .await
+ .map_err(report)?;
+ tx.commit().await.map_err(report)
+ }
+
#[tracing::instrument(skip(self, channel), fields(job_id = %key.job_id), err)]
async fn record_watch_channel(
&self,
@@ -1922,7 +1958,9 @@ impl CalendarRepository for PgCalendarRepository {
calendar.is_primary,
calendar.access_role,
calendar.provider_calendar_id,
- calendar.default_reminders
+ calendar.default_reminders,
+ calendar.last_sync_error,
+ calendar.consecutive_sync_failures
FROM email_links link
JOIN calendar_accounts account ON account.email_link_id = link.id
JOIN calendars calendar ON calendar.account_id = account.id
@@ -1961,6 +1999,9 @@ impl CalendarRepository for PgCalendarRepository {
is_primary: row.is_primary,
is_writable: matches!(row.access_role.as_deref(), Some("owner" | "writer")),
is_subscription: is_system_calendar(&row.provider_calendar_id),
+ sync_error: row.last_sync_error.filter(|_| {
+ row.consecutive_sync_failures >= CALENDAR_SYNC_FAILURE_BADGE_THRESHOLD
+ }),
default_reminders: serde_json::from_value(row.default_reminders)
.inspect_err(|e| {
tracing::error!(error = ?e, calendar_id = %row.id, "malformed calendar default_reminders json");
@@ -2238,6 +2279,18 @@ async fn upsert_calendar_tx(
is_selected = EXCLUDED.is_selected,
is_deleted = false,
default_reminders = EXCLUDED.default_reminders,
+ last_sync_error = CASE
+ WHEN calendars.is_deleted THEN NULL
+ ELSE calendars.last_sync_error
+ END,
+ last_sync_error_at = CASE
+ WHEN calendars.is_deleted THEN NULL
+ ELSE calendars.last_sync_error_at
+ END,
+ consecutive_sync_failures = CASE
+ WHEN calendars.is_deleted THEN 0
+ ELSE calendars.consecutive_sync_failures
+ END,
updated_at = now()
RETURNING
id,
diff --git a/crates/calendar_events/src/outbound/pg/test.rs b/crates/calendar_events/src/outbound/pg/test.rs
index 4c6e0c5f63..bb89051c07 100644
--- a/crates/calendar_events/src/outbound/pg/test.rs
+++ b/crates/calendar_events/src/outbound/pg/test.rs
@@ -2724,6 +2724,129 @@ async fn creation_target_prefers_the_requesters_own_primary_inbox(pool: PgPool)
);
}
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_persistently_failing_calendar_badges_then_clears_on_recovery(pool: PgPool) {
+ let owner_id = "macro|[email protected]";
+ let link_id = insert_link(&pool, owner_id).await;
+ let repo = PgCalendarRepository::new(pool.clone());
+ let enabled = repo
+ .apply_google_grant(
+ link_id,
+ complete_grant(),
+ CalendarGrantIntent::CalendarRequested,
+ )
+ .await
+ .unwrap();
+ let google_job = enabled
+ .jobs
+ .iter()
+ .find(|job| job.kind == CalendarBackfillKind::GoogleCalendar)
+ .unwrap();
+ let account_id = google_job.account_id.unwrap();
+ let key = CalendarBackfillJobKey {
+ job_id: google_job.id,
+ email_link_id: link_id,
+ };
+ let CalendarBackfillClaim::Claimed { lease_token, .. } =
+ repo.claim_google_backfill(key).await.unwrap()
+ else {
+ panic!("Google job should be claimable");
+ };
+ let primary = ProviderCalendar {
+ provider_calendar_id: "primary".to_string(),
+ name: "Primary".to_string(),
+ description: None,
+ time_zone: Some("UTC".to_string()),
+ color: None,
+ access_role: Some("owner".to_string()),
+ is_primary: true,
+ is_selected: true,
+ default_reminders: Vec::new(),
+ };
+ let calendar_id = repo
+ .upsert_google_calendar(key, lease_token, account_id, primary.clone())
+ .await
+ .unwrap()
+ .id;
+
+ let message = "Precondition check failed. (reasons: conditionNotMet)";
+ let sync_error = || async {
+ repo.list_visible_calendars(owner_id).await.unwrap()[0]
+ .sync_error
+ .clone()
+ };
+
+ // A couple of isolated failures stay below the badge threshold.
+ for _ in 0..2 {
+ repo.record_google_calendar_sync_error(key, lease_token, account_id, calendar_id, message)
+ .await
+ .unwrap();
+ }
+ assert_eq!(sync_error().await, None);
+
+ // A third failure crosses the threshold and surfaces to the user.
+ repo.record_google_calendar_sync_error(key, lease_token, account_id, calendar_id, message)
+ .await
+ .unwrap();
+ assert_eq!(sync_error().await.as_deref(), Some(message));
+
+ // A successful sync clears the badge and resets the failure counter.
+ repo.commit_google_calendar_sync(
+ key,
+ lease_token,
+ account_id,
+ GoogleCalendarSyncSnapshot {
+ calendar_id,
+ next_sync_token: "recovered".to_string(),
+ observed_provider_event_ids: Some(Vec::new()),
+ materialized_range: Some(OccurrenceRange::historical_sync(
+ Utc::now().trunc_subsecs(6),
+ )),
+ cancelled_provider_event_ids: Vec::new(),
+ },
+ 0,
+ )
+ .await
+ .unwrap();
+ assert_eq!(sync_error().await, None);
+
+ // A fresh failure after recovery starts the count over, so one blip does
+ // not immediately re-badge.
+ repo.record_google_calendar_sync_error(key, lease_token, account_id, calendar_id, message)
+ .await
+ .unwrap();
+ assert_eq!(sync_error().await, None);
+
+ // A badged calendar that drops off the provider list and later returns
+ // starts clean rather than resurrecting its old badge.
+ for _ in 0..2 {
+ repo.record_google_calendar_sync_error(key, lease_token, account_id, calendar_id, message)
+ .await
+ .unwrap();
+ }
+ assert_eq!(sync_error().await.as_deref(), Some(message));
+ repo.reconcile_google_calendar_list(key, lease_token, account_id, Vec::new())
+ .await
+ .unwrap();
+ assert!(
+ repo.list_visible_calendars(owner_id)
+ .await
+ .unwrap()
+ .is_empty()
+ );
+ let resurrected_id = repo
+ .upsert_google_calendar(key, lease_token, account_id, primary)
+ .await
+ .unwrap()
+ .id;
+ assert_eq!(resurrected_id, calendar_id);
+ assert_eq!(sync_error().await, None);
+ repo.record_google_calendar_sync_error(key, lease_token, account_id, calendar_id, message)
+ .await
+ .unwrap();
+ assert_eq!(sync_error().await, None);
+}
+
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
async fn removing_a_google_source_restores_the_surviving_calendar_copy(pool: PgPool) {
let owner_id = "macro|[email protected]";
diff --git a/crates/macro_db_client/migrations/20260908191815_calendar_per_calendar_sync_error.sql b/crates/macro_db_client/migrations/20260908191815_calendar_per_calendar_sync_error.sql
new file mode 100644
index 0000000000..291a4c1f84
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260908191815_calendar_per_calendar_sync_error.sql
@@ -0,0 +1,6 @@
+-- Per-calendar sync failure isolation: a single unreadable calendar records
+-- its own error and keeps retrying instead of wedging the whole account.
+ALTER TABLE calendars
+ ADD COLUMN IF NOT EXISTS last_sync_error text,
+ ADD COLUMN IF NOT EXISTS last_sync_error_at timestamptz,
+ ADD COLUMN IF NOT EXISTS consecutive_sync_failures integer NOT NULL DEFAULT 0;
diff --git a/docs/AGENT_GUIDE/surfaces.md b/docs/AGENT_GUIDE/surfaces.md
index 0f6105e1bd..5a03b10ccf 100644
--- a/docs/AGENT_GUIDE/surfaces.md
+++ b/docs/AGENT_GUIDE/surfaces.md
@@ -116,7 +116,9 @@ The side panel's `Calendars` section folds each connected account into a collaps
group: a caret plus the account address header with a checkbox that shows or hides all of
that account's calendars at once, and the account's calendars listed beneath it (color dot,
name, per-calendar checkbox). Subscribed system calendars (Google holidays, birthdays)
-carry a small RSS icon.
+carry a small RSS icon. A calendar whose sync has been failing persistently carries a small
+warning icon whose tooltip shows the provider error; the account keeps syncing its other
+calendars and the badge clears on its own once that calendar syncs again.
The `New event` composer (also opened by dragging a range on the grid) has an `Event kind`
pill choosing between `Event` and `Out of office`. Picking `Out of office` hides the guests,
diff --git a/packages/sdk/generated/email/types.gen.ts b/packages/sdk/generated/email/types.gen.ts
index 454454c154..c88c46dcc2 100644
--- a/packages/sdk/generated/email/types.gen.ts
+++ b/packages/sdk/generated/email/types.gen.ts
@@ -1718,6 +1718,12 @@ export type VisibleCalendar = {
* Provider display name.
*/
name: string;
+ /**
+ * A persistent sync failure isolated to this calendar, surfaced so the
+ * settings row can badge it. `None` while the calendar is syncing
+ * normally or a failure has not yet crossed the persistence threshold.
+ */
+ syncError?: string | null;
};
export type ListCalendarsData = {
diff --git a/packages/sdk/specs/email.json b/packages/sdk/specs/email.json
index 5166b29ae5..7b0c3e4b31 100644
--- a/packages/sdk/specs/email.json
+++ b/packages/sdk/specs/email.json
@@ -6609,6 +6609,10 @@
"name": {
"type": "string",
"description": "Provider display name."
+ },
+ "syncError": {
+ "type": ["string", "null"],
+ "description": "A persistent sync failure isolated to this calendar, surfaced so the\nsettings row can badge it. `None` while the calendar is syncing\nnormally or a failure has not yet crossed the persistence threshold."
}
}
}