Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(calendar): dedupe occurrences and reap wedged Google backfills (#6163)
gbirman3ded3cdc3ff1
diff --git a/.sqlx/query-89463db6b747da3ef34bd6025b5559f8b880a8024c0faf3fd5b08eb342b1db67.json b/.sqlx/query-44925c9a6539cc5f2f6ffad704289a3ba71669803ffed78fbfae2b6ab3dd509b.jsonsimilarity index 51%rename from .sqlx/query-89463db6b747da3ef34bd6025b5559f8b880a8024c0faf3fd5b08eb342b1db67.jsonrename to .sqlx/query-44925c9a6539cc5f2f6ffad704289a3ba71669803ffed78fbfae2b6ab3dd509b.jsonindex 6de6a09bbb..8de6f42877 100644--- a/.sqlx/query-89463db6b747da3ef34bd6025b5559f8b880a8024c0faf3fd5b08eb342b1db67.json+++ b/.sqlx/query-44925c9a6539cc5f2f6ffad704289a3ba71669803ffed78fbfae2b6ab3dd509b.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n INSERT INTO calendar_event_occurrences (\n event_id, owner_id, occurrence_key, recurrence_id,\n starts_at, ends_at, start_date, end_date, is_cancelled\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ",+ "query": "\n INSERT INTO calendar_event_occurrences (\n event_id, owner_id, occurrence_key, recurrence_id,\n starts_at, ends_at, start_date, end_date, is_cancelled\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ON CONFLICT (event_id, occurrence_key) DO UPDATE SET\n owner_id = EXCLUDED.owner_id,\n recurrence_id = EXCLUDED.recurrence_id,\n starts_at = EXCLUDED.starts_at,\n ends_at = EXCLUDED.ends_at,\n start_date = EXCLUDED.start_date,\n end_date = EXCLUDED.end_date,\n is_cancelled = EXCLUDED.is_cancelled\n ","describe": {"columns": [],"parameters": {@@ -18,5 +18,5 @@},"nullable": []},- "hash": "89463db6b747da3ef34bd6025b5559f8b880a8024c0faf3fd5b08eb342b1db67"+ "hash": "44925c9a6539cc5f2f6ffad704289a3ba71669803ffed78fbfae2b6ab3dd509b"}diff --git a/.sqlx/query-b887c6da665cc66e008638eba2cab4e89d5054152d7503868b91984454eee088.json b/.sqlx/query-b887c6da665cc66e008638eba2cab4e89d5054152d7503868b91984454eee088.jsonnew file mode 100644index 0000000000..d8d18965ce--- /dev/null+++ b/.sqlx/query-b887c6da665cc66e008638eba2cab4e89d5054152d7503868b91984454eee088.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH wedged AS (\n UPDATE calendar_backfill_jobs job\n SET status = 'pending',\n last_error = NULL,\n started_at = NULL,\n completed_at = NULL,\n lease_token = NULL,\n lease_expires_at = NULL,\n updated_at = now()\n FROM calendar_sync_outbox outbox,\n calendar_accounts account,\n email_link_google_scopes scopes\n WHERE outbox.backfill_job_id = job.id\n AND account.id = job.account_id\n AND account.email_link_id = job.email_link_id\n AND scopes.link_id = job.email_link_id\n AND job.kind = 'google_calendar'\n AND job.grant_version = scopes.grant_version\n AND scopes.granted_scopes @> $2::text[]\n AND account.sync_status <> 'disabled'\n AND (\n -- Dead-lettered: a delivery reached a worker at least\n -- once (started_at is set) and failed back to pending,\n -- and nothing has touched the job since. started_at\n -- separates this from a job merely waiting its turn in a\n -- backed-up queue — the drain publishes that one without\n -- ever claiming it, so republishing would only duplicate\n -- the message still in flight.\n (job.status = 'pending'\n AND job.started_at IS NOT NULL\n AND outbox.published_at IS NOT NULL\n AND job.updated_at <= $1)\n -- Abandoned: a worker claimed the job and died without\n -- releasing or renewing the lease.\n OR (job.status = 'running'\n AND job.lease_expires_at IS NOT NULL\n AND job.lease_expires_at <= $1)\n )\n RETURNING job.id\n ),\n republished AS (\n UPDATE calendar_sync_outbox outbox\n SET published_at = NULL\n FROM wedged\n WHERE outbox.backfill_job_id = wedged.id\n )\n SELECT count(*) AS \"reaped!\" FROM wedged\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "reaped!",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "Timestamptz",+ "TextArray"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "b887c6da665cc66e008638eba2cab4e89d5054152d7503868b91984454eee088"+}diff --git a/crates/calendar_events/src/domain/ports.rs b/crates/calendar_events/src/domain/ports.rsindex e15a2adb44..e6f81537ae 100644--- a/crates/calendar_events/src/domain/ports.rs+++ b/crates/calendar_events/src/domain/ports.rs@@ -702,6 +702,16 @@ pub trait GoogleCalendarSyncRepository: Send + Sync + 'static {&self,due_before: DateTime<Utc>,) -> impl Future<Output = Result<usize, Report>> + Send;++ /// Re-arm current-grant jobs stranded off the queue: `pending` jobs whose+ /// outbox row is already published yet went untouched since `stalled_before`+ /// (a deterministic delivery that dead-lettered), and `running` jobs whose+ /// lease expired before `stalled_before` (a worker that died mid-run).+ /// Republishing the outbox row lets the drain redeliver them.+ fn reap_wedged_google_syncs(+ &self,+ stalled_before: DateTime<Utc>,+ ) -> impl Future<Output = Result<usize, Report>> + Send;}/// Durable lifecycle and lease operations for calendar backfill jobs.diff --git a/crates/calendar_events/src/domain/service.rs b/crates/calendar_events/src/domain/service.rsindex 110f665253..876c9daf6a 100644--- a/crates/calendar_events/src/domain/service.rs+++ b/crates/calendar_events/src/domain/service.rs@@ -259,6 +259,12 @@ pub struct GoogleCalendarSyncScheduler<R> {repository: R,}+/// How long a backfill job may sit off the queue — pending after a delivery+/// dead-lettered, or running behind a dead worker's lease — before the reaper+/// republishes it. Comfortably past the SQS retry budget so an actively+/// retrying delivery is never duplicated.+const WEDGED_SYNC_STALL_THRESHOLD: chrono::Duration = chrono::Duration::minutes(15);+impl<R> GoogleCalendarSyncScheduler<R>whereR: GoogleCalendarSyncRepository,@@ -275,6 +281,15 @@ where.schedule_due_google_syncs(now - chrono::Duration::minutes(5)).await}++ /// Re-arm jobs stranded off the queue by a dead-lettered delivery or a+ /// dead worker's lease, so they recover without a grant change.+ #[tracing::instrument(skip(self), err)]+ pub async fn reap_once(&self, now: chrono::DateTime<Utc>) -> Result<usize, Report> {+ self.repository+ .reap_wedged_google_syncs(now - WEDGED_SYNC_STALL_THRESHOLD)+ .await+ }}/// Applies terminal Google Calendar failures that happen before lease claim.diff --git a/crates/calendar_events/src/outbound/google.rs b/crates/calendar_events/src/outbound/google.rsindex 2a8453a472..0bf0d73de4 100644--- a/crates/calendar_events/src/outbound/google.rs+++ b/crates/calendar_events/src/outbound/google.rs@@ -1896,6 +1896,24 @@ fn map_upsert(.ok().flatten()})+ .fold(+ BTreeMap::<String, CalendarOccurrence>::new(),+ |mut deduped, occurrence| {+ // Google can expand two instances onto one occurrence key — a+ // moved exception whose original start still lands on the series+ // slot, a DST boundary, a pagination overlap. Both would carry+ // the same (event_id, occurrence_key) primary key, so collapse+ // them here and keep the live instance over a cancelled tombstone.+ let replace = deduped+ .get(&occurrence.occurrence_key)+ .is_none_or(|existing| existing.is_cancelled && !occurrence.is_cancelled);+ if replace {+ deduped.insert(occurrence.occurrence_key.clone(), occurrence);+ }+ deduped+ },+ )+ .into_values().collect();Ok(CalendarEventUpsert {diff --git a/crates/calendar_events/src/outbound/google/test.rs b/crates/calendar_events/src/outbound/google/test.rsindex b6d1324f9d..2f3496b7a8 100644--- a/crates/calendar_events/src/outbound/google/test.rs+++ b/crates/calendar_events/src/outbound/google/test.rs@@ -185,6 +185,97 @@ fn malformed_recurring_instance_does_not_overstate_snapshot_coverage() {assert!(upsert.occurrences.is_empty());}+/// Google's `instances` feed can return two entries that resolve to the same+/// occurrence key — a moved exception whose original start still lands on the+/// series slot, a pagination overlap, a DST boundary. Both would collide on+/// the `(event_id, occurrence_key)` primary key and wedge the whole backfill,+/// so the mapper collapses them and keeps the live instance over a cancelled+/// tombstone.+#[test]+fn duplicate_instances_collapse_to_one_live_occurrence() {+ let master: GoogleEvent = serde_json::from_value(serde_json::json!({+ "id": "provider-master",+ "iCalUID": "[email protected]",+ "summary": "Recurring calendar event",+ "start": {"dateTime": "2026-07-24T14:00:00Z", "timeZone": "UTC"},+ "end": {"dateTime": "2026-07-24T15:00:00Z", "timeZone": "UTC"},+ "recurrence": ["RRULE:FREQ=DAILY"],+ "created": "2026-07-20T14:00:00Z",+ "updated": "2026-07-21T14:00:00Z"+ }))+ .unwrap();+ let cancelled_instance: GoogleEvent = serde_json::from_value(serde_json::json!({+ "id": "provider-master_20260724T140000Z",+ "iCalUID": "[email protected]",+ "recurringEventId": "provider-master",+ "originalStartTime": {"dateTime": "2026-07-24T14:00:00Z", "timeZone": "UTC"},+ "start": {"dateTime": "2026-07-24T14:00:00Z", "timeZone": "UTC"},+ "end": {"dateTime": "2026-07-24T15:00:00Z", "timeZone": "UTC"},+ "status": "cancelled"+ }))+ .unwrap();+ let moved_instance: GoogleEvent = serde_json::from_value(serde_json::json!({+ "id": "provider-master_20260724T140000Z",+ "iCalUID": "[email protected]",+ "recurringEventId": "provider-master",+ "originalStartTime": {"dateTime": "2026-07-24T14:00:00Z", "timeZone": "UTC"},+ "start": {"dateTime": "2026-07-24T16:00:00Z", "timeZone": "UTC"},+ "end": {"dateTime": "2026-07-24T17:00:00Z", "timeZone": "UTC"},+ "status": "confirmed"+ }))+ .unwrap();+ let range = OccurrenceRange {+ starts_at: DateTime::parse_from_rfc3339("2026-07-01T00:00:00Z")+ .unwrap()+ .with_timezone(&Utc),+ ends_at: DateTime::parse_from_rfc3339("2026-08-01T00:00:00Z")+ .unwrap()+ .with_timezone(&Utc),+ start_date: NaiveDate::from_ymd_opt(2026, 7, 1).unwrap(),+ end_date: NaiveDate::from_ymd_opt(2026, 8, 1).unwrap(),+ };+ let target = GoogleCalendarTarget {+ owner_id: "macro|[email protected]".to_string(),+ email_link_id: Uuid::now_v7(),+ account_id: Uuid::now_v7(),+ calendar_id: Uuid::now_v7(),+ provider_calendar_id: "primary".to_string(),+ is_read_only: false,+ range,+ };++ // The tombstone is fed first so the live instance must win by replacement,+ // not merely by arriving first.+ let upsert = map_upsert(+ &target,+ master,+ Vec::new(),+ vec![cancelled_instance, moved_instance],+ )+ .unwrap();++ assert_eq!(upsert.occurrences.len(), 1);+ let occurrence = &upsert.occurrences[0];+ assert_eq!(occurrence.occurrence_key, "2026-07-24T14:00:00+00:00");+ assert!(+ !occurrence.is_cancelled,+ "the live instance must survive the collapse"+ );+ assert_eq!(+ occurrence.time,+ EventTime::Timed {+ starts_at: DateTime::parse_from_rfc3339("2026-07-24T16:00:00Z")+ .unwrap()+ .with_timezone(&Utc),+ ends_at: DateTime::parse_from_rfc3339("2026-07-24T17:00:00Z")+ .unwrap()+ .with_timezone(&Utc),+ time_zone: Some("UTC".to_string()),+ },+ "the surviving row keeps the live instance's own time"+ );+}+#[test]fn malformed_master_is_quarantined_without_deleting_its_provider_identity() {let valid: GoogleEvent = serde_json::from_value(serde_json::json!({diff --git a/crates/calendar_events/src/outbound/pg.rs b/crates/calendar_events/src/outbound/pg.rsindex 71a11590b2..8fc767f3e8 100644--- a/crates/calendar_events/src/outbound/pg.rs+++ b/crates/calendar_events/src/outbound/pg.rs@@ -1971,6 +1971,71 @@ impl GoogleCalendarSyncRepository for PgCalendarRepository {.map_err(report)?;Ok(due_count as usize)}++ #[tracing::instrument(skip(self), err)]+ async fn reap_wedged_google_syncs(+ &self,+ stalled_before: DateTime<Utc>,+ ) -> Result<usize, Report> {+ let required_scopes = GOOGLE_CALENDAR_SCOPES.map(str::to_owned);+ let reaped = sqlx::query_scalar!(+ r#"+ WITH wedged AS (+ UPDATE calendar_backfill_jobs job+ SET status = 'pending',+ last_error = NULL,+ started_at = NULL,+ completed_at = NULL,+ lease_token = NULL,+ lease_expires_at = NULL,+ updated_at = now()+ FROM calendar_sync_outbox outbox,+ calendar_accounts account,+ email_link_google_scopes scopes+ WHERE outbox.backfill_job_id = job.id+ AND account.id = job.account_id+ AND account.email_link_id = job.email_link_id+ AND scopes.link_id = job.email_link_id+ AND job.kind = 'google_calendar'+ AND job.grant_version = scopes.grant_version+ AND scopes.granted_scopes @> $2::text[]+ AND account.sync_status <> 'disabled'+ AND (+ -- Dead-lettered: a delivery reached a worker at least+ -- once (started_at is set) and failed back to pending,+ -- and nothing has touched the job since. started_at+ -- separates this from a job merely waiting its turn in a+ -- backed-up queue — the drain publishes that one without+ -- ever claiming it, so republishing would only duplicate+ -- the message still in flight.+ (job.status = 'pending'+ AND job.started_at IS NOT NULL+ AND outbox.published_at IS NOT NULL+ AND job.updated_at <= $1)+ -- Abandoned: a worker claimed the job and died without+ -- releasing or renewing the lease.+ OR (job.status = 'running'+ AND job.lease_expires_at IS NOT NULL+ AND job.lease_expires_at <= $1)+ )+ RETURNING job.id+ ),+ republished AS (+ UPDATE calendar_sync_outbox outbox+ SET published_at = NULL+ FROM wedged+ WHERE outbox.backfill_job_id = wedged.id+ )+ SELECT count(*) AS "reaped!" FROM wedged+ "#,+ stalled_before,+ &required_scopes,+ )+ .fetch_one(&self.pool)+ .await+ .map_err(report)?;+ Ok(reaped as usize)+ }}async fn fence_google_mutation_tx(@@ -2986,6 +3051,14 @@ async fn replace_occurrences(starts_at, ends_at, start_date, end_date, is_cancelled)VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)+ ON CONFLICT (event_id, occurrence_key) DO UPDATE SET+ owner_id = EXCLUDED.owner_id,+ recurrence_id = EXCLUDED.recurrence_id,+ starts_at = EXCLUDED.starts_at,+ ends_at = EXCLUDED.ends_at,+ start_date = EXCLUDED.start_date,+ end_date = EXCLUDED.end_date,+ is_cancelled = EXCLUDED.is_cancelled"#,event_id,owner_id,diff --git a/crates/calendar_events/src/outbound/pg/test.rs b/crates/calendar_events/src/outbound/pg/test.rsindex 6f244c66a8..4e0d0c3024 100644--- a/crates/calendar_events/src/outbound/pg/test.rs+++ b/crates/calendar_events/src/outbound/pg/test.rs@@ -335,6 +335,203 @@ async fn calendar_capability_transition_schedules_once_and_failed_jobs_can_retryassert!(published_at.is_none());}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn reaper_rearms_a_dead_lettered_pending_job(pool: PgPool) {+ let owner_id = "macro|[email protected]";+ let link_id = insert_link(&pool, owner_id).await;+ let repo = PgCalendarRepository::new(pool.clone());+ let enabled = repo+ .apply_google_grant(+ link_id,+ complete_grant(),+ CalendarGrantIntent::CalendarRequested,+ )+ .await+ .unwrap();+ let job_id = enabled.jobs[0].id;++ // The drain published the delivery; the deterministic failure then sat the+ // job back to pending without touching the outbox — exactly the wedge.+ sqlx::query!(+ "UPDATE calendar_sync_outbox SET published_at = now() WHERE backfill_job_id = $1",+ job_id,+ )+ .execute(&pool)+ .await+ .unwrap();++ // A job still inside its SQS retry budget keeps getting touched, so the+ // reaper leaves it alone.+ assert_eq!(+ repo.reap_wedged_google_syncs(Utc::now() - Duration::minutes(15))+ .await+ .unwrap(),+ 0+ );++ // The job reached a worker once (started_at set) and a deterministic+ // failure sat it back to pending; once the delivery dead-letters nothing+ // touches it again.+ sqlx::query!(+ "UPDATE calendar_backfill_jobs SET started_at = now() - interval '25 minutes', updated_at = now() - interval '20 minutes' WHERE id = $1",+ job_id,+ )+ .execute(&pool)+ .await+ .unwrap();++ assert_eq!(+ repo.reap_wedged_google_syncs(Utc::now() - Duration::minutes(15))+ .await+ .unwrap(),+ 1+ );+ let published_at = sqlx::query_scalar!(+ "SELECT published_at FROM calendar_sync_outbox WHERE backfill_job_id = $1",+ job_id,+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert!(+ published_at.is_none(),+ "the reaper republishes the outbox row"+ );++ // Republished rows are no longer published, so a second pass is a no-op.+ assert_eq!(+ repo.reap_wedged_google_syncs(Utc::now() - Duration::minutes(15))+ .await+ .unwrap(),+ 0+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn reaper_leaves_a_never_delivered_queued_job_alone(pool: PgPool) {+ let owner_id = "macro|[email protected]";+ let link_id = insert_link(&pool, owner_id).await;+ let repo = PgCalendarRepository::new(pool.clone());+ let enabled = repo+ .apply_google_grant(+ link_id,+ complete_grant(),+ CalendarGrantIntent::CalendarRequested,+ )+ .await+ .unwrap();+ let job_id = enabled.jobs[0].id;++ // The drain published the delivery, but a backed-up queue means it has not+ // reached a worker yet: started_at is still NULL though the outbox row and+ // the job both look old.+ sqlx::query!(+ "UPDATE calendar_sync_outbox SET published_at = now() - interval '30 minutes' WHERE backfill_job_id = $1",+ job_id,+ )+ .execute(&pool)+ .await+ .unwrap();+ sqlx::query!(+ "UPDATE calendar_backfill_jobs SET updated_at = now() - interval '30 minutes' WHERE id = $1",+ job_id,+ )+ .execute(&pool)+ .await+ .unwrap();++ // Republishing would only duplicate the delivery still in flight, so the+ // reaper leaves an un-claimed job alone.+ assert_eq!(+ repo.reap_wedged_google_syncs(Utc::now() - Duration::minutes(15))+ .await+ .unwrap(),+ 0+ );+ let published_at = sqlx::query_scalar!(+ "SELECT published_at FROM calendar_sync_outbox WHERE backfill_job_id = $1",+ job_id,+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert!(+ published_at.is_some(),+ "the queued delivery stays published, not re-armed"+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn reaper_rearms_a_running_job_behind_a_dead_lease(pool: PgPool) {+ let owner_id = "macro|[email protected]";+ let link_id = insert_link(&pool, owner_id).await;+ let repo = PgCalendarRepository::new(pool.clone());+ let enabled = repo+ .apply_google_grant(+ link_id,+ complete_grant(),+ CalendarGrantIntent::CalendarRequested,+ )+ .await+ .unwrap();+ let job_id = enabled.jobs[0].id;+ let key = CalendarBackfillJobKey {+ job_id,+ email_link_id: link_id,+ };++ let CalendarBackfillClaim::Claimed { .. } = repo.claim_google_backfill(key).await.unwrap()+ else {+ panic!("the freshly granted job should be claimable");+ };+ sqlx::query!(+ "UPDATE calendar_sync_outbox SET published_at = now() WHERE backfill_job_id = $1",+ job_id,+ )+ .execute(&pool)+ .await+ .unwrap();++ // A live lease means a worker is on it, so the reaper leaves it alone.+ assert_eq!(+ repo.reap_wedged_google_syncs(Utc::now() - Duration::minutes(15))+ .await+ .unwrap(),+ 0+ );++ // The worker died: its lease expired long ago and no longer renews.+ sqlx::query!(+ "UPDATE calendar_backfill_jobs SET lease_expires_at = now() - interval '20 minutes' WHERE id = $1",+ job_id,+ )+ .execute(&pool)+ .await+ .unwrap();++ assert_eq!(+ repo.reap_wedged_google_syncs(Utc::now() - Duration::minutes(15))+ .await+ .unwrap(),+ 1+ );+ let row = sqlx::query!(+ r#"+ SELECT job.status, job.lease_token, outbox.published_at+ FROM calendar_backfill_jobs job+ JOIN calendar_sync_outbox outbox ON outbox.backfill_job_id = job.id+ WHERE job.id = $1+ "#,+ job_id,+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(row.status, "pending");+ assert!(row.lease_token.is_none());+ assert!(row.published_at.is_none());+}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn removing_calendar_scope_disables_sources_and_fences_the_running_job(pool: PgPool) {let owner_id = "macro|[email protected]";diff --git a/services/email_service/src/calendar_outbox.rs b/services/email_service/src/calendar_outbox.rsindex d7226f814a..4a84b78882 100644--- a/services/email_service/src/calendar_outbox.rs+++ b/services/email_service/src/calendar_outbox.rs@@ -63,6 +63,13 @@ pub async fn run<R>(tracing::error!(error=?error, "failed to schedule due Google Calendar syncs");}).ok();+ scheduler+ .reap_once(Utc::now())+ .await+ .inspect_err(|error| {+ tracing::error!(error=?error, "failed to reap wedged Google Calendar syncs");+ })+ .ok();drain_calendar(&db, &sqs).await.inspect_err(|error| {