Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): cap concurrent backfill message writes across the fleet (#8912)
teofa8069b7bb97
diff --git a/crates/models_email/src/email/service/pubsub.rs b/crates/models_email/src/email/service/pubsub.rsindex d51b07175b..56f55cf803 100644--- a/crates/models_email/src/email/service/pubsub.rs+++ b/crates/models_email/src/email/service/pubsub.rs@@ -43,6 +43,8 @@ pub enum FailureReason {CalendarBackfillBusy,GmailApiFailed,GmailApiRateLimited,+ /// Every fleet-wide backfill write slot stayed taken; the operation is deferred.+ BackfillWritesBusy,OutdatedHistoryId,AttachmentParsingFailed,DSSUploadFailed,diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.mdindex 2f5c53876a..bcbaecbe60 100644--- a/services/email_service/AGENTS.md+++ b/services/email_service/AGENTS.md@@ -81,6 +81,14 @@ the gateway `/email` prefix.would spend the queue's 20 deliveries.Inbox sync releases the upload claims of a refused attachment fetch andrequeues the upsert, because a claimed attachment is never offered again.+- Backfill message writes hold one of twenty fleet-wide Redis write slots+ (`util/redis/backfill_write_slots.rs`), however many workers and tasks run,+ so a large backfill can't queue enough writers on the primary to slow every+ other query. A write waits up to 10 s for a slot and is then deferred like a+ rate limit; Redis failing to decide also defers it. Run the write through+ `BackfillWriteSlot::hold`, which renews the 120 s lease while the write runs,+ so a stalled write can't lose its slot to another writer. Live inbox+ sync writes take no slot. Keep new bulk backfill writes behind the same slots.- `gmail_ops` workers decode both label write forms: single-message`ModifyMessageLabels` and `ModifyMessageLabelsBatch` (up to 1,000 messages;one `batchModify` from ten messages up, separate `messages.modify` callsdiff --git a/services/email_service/src/pubsub/backfill/backfill_message.rs b/services/email_service/src/pubsub/backfill/backfill_message.rsindex 518d9390cb..1033f98c6c 100644--- a/services/email_service/src/pubsub/backfill/backfill_message.rs+++ b/services/email_service/src/pubsub/backfill/backfill_message.rs@@ -1,6 +1,6 @@use crate::pubsub::backfill::db_error::map_db_error;use crate::pubsub::backfill::email_api_error::map_email_api_error;-use crate::pubsub::backfill::increment_counters;+use crate::pubsub::backfill::{increment_counters, write_slot};use crate::pubsub::context::PubSubContext;use crate::pubsub::util::{CrmContactRecipient, enqueue_populate_crm_contacts};use crate::util::process_pre_insert::process_message_pre_insert;@@ -33,17 +33,19 @@ pub async fn backfill_message(process_message_pre_insert(&mut message).await;- // insert message into database- email_db_client::messages::insert::insert_message(- &ctx.db,- p.thread_db_id,- &mut message,- link.id,- // we update the thread metadata once all messages in the thread have been backfilled- false,- )- .await- .map_err(|e| map_db_error(e, "Failed to insert final message into database"))?;+ // Insert the message while holding one of the fleet's backfill write slots.+ let slot = write_slot::acquire(ctx).await?;+ let inserted = slot+ .hold(email_db_client::messages::insert::insert_message(+ &ctx.db,+ p.thread_db_id,+ &mut message,+ link.id,+ // we update the thread metadata once all messages in the thread have been backfilled+ false,+ ))+ .await;+ inserted.map_err(|e| map_db_error(e, "Failed to insert final message into database"))?;if !message.is_draft && !fetched.calendar_parts.is_empty() {match email_db_client::messages::get::get_message_and_thread_id_by_provider_id(diff --git a/services/email_service/src/pubsub/backfill/deferral.rs b/services/email_service/src/pubsub/backfill/deferral.rsindex 0eae567272..644a194bd8 100644--- a/services/email_service/src/pubsub/backfill/deferral.rs+++ b/services/email_service/src/pubsub/backfill/deferral.rs@@ -27,16 +27,20 @@ const MAX_DEFERRAL: Duration = Duration::from_secs(60);/// SQS's longest message delay, which bounds a provider's longer Retry-After.const MAX_SQS_DELAY: Duration = Duration::from_secs(900);-/// The wait a rate-limited operation was told to observe, when the failure is-/// a rate limit: the request gate's or Gmail's Retry-After, if either gave one.+/// The wait a refused operation was told to observe, when the failure is a+/// refusal: a Gmail rate limit (the request gate's or Gmail's Retry-After, if+/// either gave one) or no free backfill write slot.pub(crate) fn rate_limit(error: &DetailedError) -> Option<Option<Duration>> {- if error.reason != FailureReason::GmailApiRateLimited {- return None;- }- Some(match error.source.downcast_ref::<EmailApiError>() {- Some(EmailApiError::RateLimited { retry_after, .. }) => *retry_after,+ match error.reason {+ FailureReason::GmailApiRateLimited => {+ Some(match error.source.downcast_ref::<EmailApiError>() {+ Some(EmailApiError::RateLimited { retry_after, .. }) => *retry_after,+ _ => None,+ })+ }+ FailureReason::BackfillWritesBusy => Some(None),_ => None,- })+ }}/// How often the received operation has already been deferred.diff --git a/services/email_service/src/pubsub/backfill/deferral/test.rs b/services/email_service/src/pubsub/backfill/deferral/test.rsindex ff001f258b..32be6c7685 100644--- a/services/email_service/src/pubsub/backfill/deferral/test.rs+++ b/services/email_service/src/pubsub/backfill/deferral/test.rs@@ -53,6 +53,16 @@ fn a_gmail_rate_limit_without_retry_after_is_deferred_too() {assert_eq!(rate_limit(&throttled), Some(None));}+#[test]+fn a_write_refused_a_backfill_write_slot_is_deferred_with_backoff() {+ let refused = DetailedError {+ reason: FailureReason::BackfillWritesBusy,+ source: anyhow::anyhow!("no backfill write slot freed up in time"),+ };++ assert_eq!(rate_limit(&refused), Some(None));+}+#[test]fn other_retryable_failures_keep_waiting_out_the_visibility_timeout() {let database = retryable(map_db_error(diff --git a/services/email_service/src/pubsub/backfill/error_handlers.rs b/services/email_service/src/pubsub/backfill/error_handlers.rsindex 8e15bf5cad..1c572b06da 100644--- a/services/email_service/src/pubsub/backfill/error_handlers.rs+++ b/services/email_service/src/pubsub/backfill/error_handlers.rs@@ -155,10 +155,14 @@ pub async fn handle_retryable_error(let error_chain = format!("{:#}", _e.source);tracing::Span::current().record("error", &error_chain);- // Rate limits are expected backpressure, redelivered after the visibility- // timeout; the email API client logs provider throttling at ERROR.- if _e.reason == FailureReason::GmailApiRateLimited {- tracing::debug!("Rate-limited backfill operation will be redelivered");+ // Rate limits and busy write slots are expected backpressure, redelivered+ // after the visibility timeout; the email API client logs provider+ // throttling at ERROR.+ if matches!(+ _e.reason,+ FailureReason::GmailApiRateLimited | FailureReason::BackfillWritesBusy+ ) {+ tracing::debug!("Refused backfill operation will be redelivered");return Ok(());}diff --git a/services/email_service/src/pubsub/backfill/mod.rs b/services/email_service/src/pubsub/backfill/mod.rsindex 459a9edad3..764154760a 100644--- a/services/email_service/src/pubsub/backfill/mod.rs+++ b/services/email_service/src/pubsub/backfill/mod.rs@@ -16,3 +16,4 @@ mod process;mod seed_sent_contact;mod update_metadata;pub mod worker;+mod write_slot;diff --git a/services/email_service/src/pubsub/backfill/write_slot.rs b/services/email_service/src/pubsub/backfill/write_slot.rsnew file mode 100644index 0000000000..eb7d50491d--- /dev/null+++ b/services/email_service/src/pubsub/backfill/write_slot.rs@@ -0,0 +1,27 @@+//! Admission to the fleet's backfill write slots+//! (`util::redis::backfill_write_slots`).++use crate::pubsub::context::PubSubContext;+use crate::util::redis::backfill_write_slots::BackfillWriteSlot;+use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};++/// Takes a backfill write slot for one message write. A slot that stays taken,+/// or Redis failing to decide, refuses the write so the operation is deferred+/// instead of written unmetered.+pub(super) async fn acquire(ctx: &PubSubContext) -> Result<BackfillWriteSlot, ProcessingError> {+ let refused = |source: anyhow::Error| {+ ProcessingError::Retryable(DetailedError {+ reason: FailureReason::BackfillWritesBusy,+ source,+ })+ };+ match ctx.redis_client.acquire_backfill_write_slot().await {+ Ok(Some(slot)) => Ok(slot),+ Ok(None) => Err(refused(anyhow::anyhow!(+ "no backfill write slot freed up in time"+ ))),+ Err(error) => Err(refused(+ anyhow::Error::new(error).context("backfill write slots are unavailable"),+ )),+ }+}diff --git a/services/email_service/src/util/redis.rs b/services/email_service/src/util/redis.rsindex 299e4f37f7..4df98494c5 100644--- a/services/email_service/src/util/redis.rs+++ b/services/email_service/src/util/redis.rs@@ -1,5 +1,6 @@pub mod access_token;pub mod backfill;+pub mod backfill_write_slots;pub mod health_probe;pub mod lock;pub mod rate_limit;diff --git a/services/email_service/src/util/redis/backfill_write_slots.rs b/services/email_service/src/util/redis/backfill_write_slots.rsnew file mode 100644index 0000000000..289cf42a15--- /dev/null+++ b/services/email_service/src/util/redis/backfill_write_slots.rs@@ -0,0 +1,234 @@+//! A fleet-wide cap on concurrent backfill message writes.+//!+//! Every backfill worker of every email-service task writes messages straight+//! to the primary, one transaction per message, and nothing else bounds how+//! many run at once (25 workers per task, 5 to 10 tasks). Message bodies land+//! in one TOAST table, so a large backfill queues its writers on that table's+//! relation-extension lock and slows every other query on the database. A slot+//! is a Redis lease; at most [`CAPACITY`] backfill writes hold one at a time,+//! however many tasks are running.++use crate::util::redis::RedisClient;+use rand::Rng;+use std::sync::LazyLock;+use std::time::Duration;+use uuid::Uuid;++#[cfg(test)]+mod test;++/// Concurrent backfill message writes across the fleet. A healthy write (the+/// contacts upsert plus the message transaction) averages about 100 ms, so+/// twenty slots admit roughly 200 messages a second while the database is+/// healthy, above normal busy hours, and fewer as it slows down. Unbounded,+/// 125 to 250 backfill workers could write at once.+const CAPACITY: u32 = 20;+/// Frees the slot of a crashed worker. A held slot renews it every+/// [`RENEW_EVERY`], so a write that outlasts it keeps its slot.+const LEASE: Duration = Duration::from_secs(120);+const RENEW_EVERY: Duration = Duration::from_secs(30);+/// How long a write waits for a slot before its operation is deferred. With the+/// Gmail gate's wait before it, the handler stays inside the backfill queue's+/// 60-second visibility timeout.+const WAIT: Duration = Duration::from_secs(10);+/// First and longest pause between admission checks. Waiters back off so a+/// saturated fleet doesn't poll Redis in lockstep.+const FIRST_POLL: Duration = Duration::from_millis(25);+const MAX_POLL: Duration = Duration::from_millis(400);+const KEY: &str = "email:backfill:write_slots";++/// Admits the caller when fewer than `capacity` unexpired leases are held.+/// Returns the slots in use counting this one, or 0 when every slot is taken.+static ACQUIRE: LazyLock<redis::Script> = LazyLock::new(|| {+ redis::Script::new(+ r#"+ local clock = redis.call('TIME')+ local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)+ redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)+ local in_flight = redis.call('ZCARD', KEYS[1])+ if in_flight >= tonumber(ARGV[2]) then return 0 end+ redis.call('ZADD', KEYS[1], now + tonumber(ARGV[3]), ARGV[1])+ redis.call('PEXPIRE', KEYS[1], ARGV[3])+ return in_flight + 1+ "#,+ )+});++/// Pushes a held lease forward. Returns 1, or 0 when the lease already expired+/// and was dropped; it is never re-added, so an expired slot can't overfill.+static RENEW: LazyLock<redis::Script> = LazyLock::new(|| {+ redis::Script::new(+ r#"+ local clock = redis.call('TIME')+ local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)+ local renewed = redis.call('ZADD', KEYS[1], 'XX', 'CH', now + tonumber(ARGV[2]), ARGV[1])+ if renewed == 1 then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end+ return renewed+ "#,+ )+});++/// One set of write slots. Production uses [`SLOTS`]; tests use their own keys.+struct SlotSet<'a> {+ key: &'a str,+ capacity: u32,+ lease: Duration,+ wait: Duration,+}++const SLOTS: SlotSet<'static> = SlotSet {+ key: KEY,+ capacity: CAPACITY,+ lease: LEASE,+ wait: WAIT,+};++/// A held backfill write slot.+///+/// Run the write with [`BackfillWriteSlot::hold`], which keeps the lease alive+/// and releases the slot afterwards. A dropped slot is released by a spawned+/// task, and the lease frees it if the process dies first.+pub struct BackfillWriteSlot {+ /// `None` after an explicit release, so `Drop` does nothing.+ inner: Option<SlotLease>,+}++struct SlotLease {+ client: redis::Client,+ key: String,+ id: String,+ lease: Duration,+}++impl SlotLease {+ /// Pushes the lease forward; `false` means it had already expired.+ async fn renew(&self) -> redis::RedisResult<bool> {+ let mut connection = self.client.get_multiplexed_async_connection().await?;+ let renewed: u32 = RENEW+ .key(&self.key)+ .arg(&self.id)+ .arg(self.lease.as_millis().to_string())+ .invoke_async(&mut connection)+ .await?;+ Ok(renewed == 1)+ }++ async fn release(&self) {+ let result: redis::RedisResult<i32> = async {+ let mut connection = self.client.get_multiplexed_async_connection().await?;+ redis::cmd("ZREM")+ .arg(&self.key)+ .arg(&self.id)+ .query_async(&mut connection)+ .await+ }+ .await;+ if let Err(error) = result {+ tracing::warn!(error = %error, "failed to release a backfill write slot; its lease will expire");+ }+ }+}++impl BackfillWriteSlot {+ /// Runs `work` while holding the slot, renewing its lease until the work+ /// finishes, then returns the slot to the fleet.+ pub async fn hold<F: Future>(self, work: F) -> F::Output {+ let mut work = std::pin::pin!(work);+ let mut renewals =+ tokio::time::interval_at(tokio::time::Instant::now() + RENEW_EVERY, RENEW_EVERY);+ let output = loop {+ tokio::select! {+ output = &mut work => break output,+ _ = renewals.tick() => {+ if let Some(lease) = &self.inner {+ match lease.renew().await {+ Ok(true) => {}+ Ok(false) => tracing::warn!("a backfill write outlived its slot's lease"),+ Err(error) => tracing::warn!(error = %error, "failed to renew a backfill write slot"),+ }+ }+ }+ }+ };+ self.release().await;+ output+ }++ /// Returns the slot to the fleet.+ pub async fn release(mut self) {+ if let Some(lease) = self.inner.take() {+ lease.release().await;+ }+ }+}++impl Drop for BackfillWriteSlot {+ fn drop(&mut self) {+ if let Some(lease) = self.inner.take() {+ // Drop cannot wait. Without a runtime, the lease frees the slot.+ if let Ok(runtime) = tokio::runtime::Handle::try_current() {+ runtime.spawn(async move { lease.release().await });+ }+ }+ }+}++impl RedisClient {+ /// Waits for one of the fleet's backfill write slots. Returns `None` when+ /// none frees up in time; the caller defers its operation.+ pub async fn acquire_backfill_write_slot(+ &self,+ ) -> redis::RedisResult<Option<BackfillWriteSlot>> {+ self.acquire_write_slot(&SLOTS).await+ }++ #[tracing::instrument(+ name = "email.backfill.write_slot",+ skip(self, slots),+ fields(admitted, in_flight, waited_ms),+ err+ )]+ async fn acquire_write_slot(+ &self,+ slots: &SlotSet<'_>,+ ) -> redis::RedisResult<Option<BackfillWriteSlot>> {+ let id = Uuid::now_v7().to_string();+ let started = tokio::time::Instant::now();+ let deadline = started + slots.wait;+ let mut poll = FIRST_POLL;+ let mut connection = self.inner.get_multiplexed_async_connection().await?;+ let span = tracing::Span::current();+ loop {+ let in_flight: u32 = ACQUIRE+ .key(slots.key)+ .arg(&id)+ .arg(slots.capacity)+ .arg(slots.lease.as_millis().to_string())+ .invoke_async(&mut connection)+ .await?;+ let now = tokio::time::Instant::now();+ if in_flight > 0 || now >= deadline {+ span.record("admitted", in_flight > 0);+ let waited_ms = u64::try_from(now.duration_since(started).as_millis());+ span.record("waited_ms", waited_ms.unwrap_or(u64::MAX));+ if in_flight == 0 {+ return Ok(None);+ }+ span.record("in_flight", in_flight);+ return Ok(Some(BackfillWriteSlot {+ inner: Some(SlotLease {+ client: self.inner.clone(),+ key: slots.key.to_string(),+ id,+ lease: slots.lease,+ }),+ }));+ }+ let pause = rand::rng()+ .random_range(poll / 2..=poll)+ .min(deadline - now);+ tokio::time::sleep(pause).await;+ poll = (poll * 2).min(MAX_POLL);+ }+ }+}diff --git a/services/email_service/src/util/redis/backfill_write_slots/test.rs b/services/email_service/src/util/redis/backfill_write_slots/test.rsnew file mode 100644index 0000000000..f3e3916daf--- /dev/null+++ b/services/email_service/src/util/redis/backfill_write_slots/test.rs@@ -0,0 +1,179 @@+use super::*;++fn client() -> RedisClient {+ RedisClient::new(redis::Client::open("redis://127.0.0.1:6379/").unwrap(), 60)+}++/// A slot set of its own, so tests never touch the fleet's slots or each other's.+fn unique_key() -> String {+ format!("test:email:backfill:write_slots:{}", Uuid::now_v7())+}++fn slots(key: &str, capacity: u32) -> SlotSet<'_> {+ SlotSet {+ key,+ capacity,+ lease: LEASE,+ wait: Duration::ZERO,+ }+}++async fn cleanup(client: &RedisClient, key: &str) {+ let mut connection = client+ .inner+ .get_multiplexed_async_connection()+ .await+ .unwrap();+ let _: i32 = redis::cmd("DEL")+ .arg(key)+ .query_async(&mut connection)+ .await+ .unwrap();+}++#[tokio::test]+async fn writes_beyond_capacity_are_refused_until_a_slot_is_released() {+ let client = client();+ let key = unique_key();+ let slots = slots(&key, 2);++ let first = client+ .acquire_write_slot(&slots)+ .await+ .unwrap()+ .expect("the first write gets a slot");+ let second = client+ .acquire_write_slot(&slots)+ .await+ .unwrap()+ .expect("the second write gets a slot");+ assert!(+ client.acquire_write_slot(&slots).await.unwrap().is_none(),+ "a third concurrent write must wait"+ );++ first.release().await;+ let third = client+ .acquire_write_slot(&slots)+ .await+ .unwrap()+ .expect("a released slot admits the next write");++ second.release().await;+ third.release().await;+ cleanup(&client, &key).await;+}++#[tokio::test]+async fn a_crashed_writers_expired_lease_frees_its_slot() {+ let client = client();+ let key = unique_key();+ let mut connection = client+ .inner+ .get_multiplexed_async_connection()+ .await+ .unwrap();+ // A lease that expired long ago, as left by a worker that died holding it.+ let _: i32 = redis::cmd("ZADD")+ .arg(&key)+ .arg(1)+ .arg("crashed-writer")+ .query_async(&mut connection)+ .await+ .unwrap();++ let slot = client+ .acquire_write_slot(&slots(&key, 1))+ .await+ .unwrap()+ .expect("an expired lease does not hold the only slot");++ slot.release().await;+ cleanup(&client, &key).await;+}++fn lease_id(slot: &BackfillWriteSlot) -> String {+ slot.inner.as_ref().expect("a held slot").id.clone()+}++#[tokio::test]+async fn renewing_a_held_slot_pushes_its_lease_forward() {+ let client = client();+ let key = unique_key();+ let slot = client+ .acquire_write_slot(&slots(&key, 1))+ .await+ .unwrap()+ .expect("the only slot is free");+ let id = lease_id(&slot);+ let mut connection = client+ .inner+ .get_multiplexed_async_connection()+ .await+ .unwrap();+ // A lease about to expire, as after a long write.+ let _: i32 = redis::cmd("ZADD")+ .arg(&key)+ .arg("XX")+ .arg(1)+ .arg(&id)+ .query_async(&mut connection)+ .await+ .unwrap();++ let renewed = slot.inner.as_ref().unwrap().renew().await.unwrap();++ let expires_at: f64 = redis::cmd("ZSCORE")+ .arg(&key)+ .arg(&id)+ .query_async(&mut connection)+ .await+ .unwrap();+ assert!(renewed);+ assert!(+ expires_at > 1.0e12,+ "the lease now ends at a real time ahead of now"+ );+ slot.release().await;+ cleanup(&client, &key).await;+}++#[tokio::test]+async fn an_expired_lease_is_not_renewed_back_into_a_full_set() {+ let client = client();+ let key = unique_key();+ let slot = client+ .acquire_write_slot(&slots(&key, 1))+ .await+ .unwrap()+ .expect("the only slot is free");+ let mut connection = client+ .inner+ .get_multiplexed_async_connection()+ .await+ .unwrap();+ // The lease expired and another writer took the slot.+ let _: i32 = redis::cmd("ZREM")+ .arg(&key)+ .arg(lease_id(&slot))+ .query_async(&mut connection)+ .await+ .unwrap();+ let other = client+ .acquire_write_slot(&slots(&key, 1))+ .await+ .unwrap()+ .expect("the expired slot is free again");++ let renewed = slot.inner.as_ref().unwrap().renew().await.unwrap();++ let held: i32 = redis::cmd("ZCARD")+ .arg(&key)+ .query_async(&mut connection)+ .await+ .unwrap();+ assert!(!renewed);+ assert_eq!(held, 1, "only the new writer holds the slot");+ other.release().await;+ cleanup(&client, &key).await;+}