Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(email): cap concurrent backfill message writes across the fleet (#8912)

teofa8069b7bb97
diff --git a/crates/models_email/src/email/service/pubsub.rs b/crates/models_email/src/email/service/pubsub.rs
index d51b07175b..56f55cf803 100644
--- a/crates/models_email/src/email/service/pubsub.rs
+++ b/crates/models_email/src/email/service/pubsub.rs
@@ -43,6 +43,8 @@ pub enum FailureReason {
CalendarBackfillBusy,
GmailApiFailed,
GmailApiRateLimited,
+ /// Every fleet-wide backfill write slot stayed taken; the operation is deferred.
+ BackfillWritesBusy,
OutdatedHistoryId,
AttachmentParsingFailed,
DSSUploadFailed,
diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.md
index 2f5c53876a..bcbaecbe60 100644
--- a/services/email_service/AGENTS.md
+++ b/services/email_service/AGENTS.md
@@ -81,6 +81,14 @@ the gateway `/email` prefix.
would spend the queue's 20 deliveries.
Inbox sync releases the upload claims of a refused attachment fetch and
requeues the upsert, because a claimed attachment is never offered again.
+- Backfill message writes hold one of twenty fleet-wide Redis write slots
+ (`util/redis/backfill_write_slots.rs`), however many workers and tasks run,
+ so a large backfill can't queue enough writers on the primary to slow every
+ other query. A write waits up to 10 s for a slot and is then deferred like a
+ rate limit; Redis failing to decide also defers it. Run the write through
+ `BackfillWriteSlot::hold`, which renews the 120 s lease while the write runs,
+ so a stalled write can't lose its slot to another writer. Live inbox
+ sync writes take no slot. Keep new bulk backfill writes behind the same slots.
- `gmail_ops` workers decode both label write forms: single-message
`ModifyMessageLabels` and `ModifyMessageLabelsBatch` (up to 1,000 messages;
one `batchModify` from ten messages up, separate `messages.modify` calls
diff --git a/services/email_service/src/pubsub/backfill/backfill_message.rs b/services/email_service/src/pubsub/backfill/backfill_message.rs
index 518d9390cb..1033f98c6c 100644
--- a/services/email_service/src/pubsub/backfill/backfill_message.rs
+++ b/services/email_service/src/pubsub/backfill/backfill_message.rs
@@ -1,6 +1,6 @@
use crate::pubsub::backfill::db_error::map_db_error;
use crate::pubsub::backfill::email_api_error::map_email_api_error;
-use crate::pubsub::backfill::increment_counters;
+use crate::pubsub::backfill::{increment_counters, write_slot};
use crate::pubsub::context::PubSubContext;
use crate::pubsub::util::{CrmContactRecipient, enqueue_populate_crm_contacts};
use crate::util::process_pre_insert::process_message_pre_insert;
@@ -33,17 +33,19 @@ pub async fn backfill_message(
process_message_pre_insert(&mut message).await;
- // insert message into database
- email_db_client::messages::insert::insert_message(
- &ctx.db,
- p.thread_db_id,
- &mut message,
- link.id,
- // we update the thread metadata once all messages in the thread have been backfilled
- false,
- )
- .await
- .map_err(|e| map_db_error(e, "Failed to insert final message into database"))?;
+ // Insert the message while holding one of the fleet's backfill write slots.
+ let slot = write_slot::acquire(ctx).await?;
+ let inserted = slot
+ .hold(email_db_client::messages::insert::insert_message(
+ &ctx.db,
+ p.thread_db_id,
+ &mut message,
+ link.id,
+ // we update the thread metadata once all messages in the thread have been backfilled
+ false,
+ ))
+ .await;
+ inserted.map_err(|e| map_db_error(e, "Failed to insert final message into database"))?;
if !message.is_draft && !fetched.calendar_parts.is_empty() {
match email_db_client::messages::get::get_message_and_thread_id_by_provider_id(
diff --git a/services/email_service/src/pubsub/backfill/deferral.rs b/services/email_service/src/pubsub/backfill/deferral.rs
index 0eae567272..644a194bd8 100644
--- a/services/email_service/src/pubsub/backfill/deferral.rs
+++ b/services/email_service/src/pubsub/backfill/deferral.rs
@@ -27,16 +27,20 @@ const MAX_DEFERRAL: Duration = Duration::from_secs(60);
/// SQS's longest message delay, which bounds a provider's longer Retry-After.
const MAX_SQS_DELAY: Duration = Duration::from_secs(900);
-/// The wait a rate-limited operation was told to observe, when the failure is
-/// a rate limit: the request gate's or Gmail's Retry-After, if either gave one.
+/// The wait a refused operation was told to observe, when the failure is a
+/// refusal: a Gmail rate limit (the request gate's or Gmail's Retry-After, if
+/// either gave one) or no free backfill write slot.
pub(crate) fn rate_limit(error: &DetailedError) -> Option<Option<Duration>> {
- if error.reason != FailureReason::GmailApiRateLimited {
- return None;
- }
- Some(match error.source.downcast_ref::<EmailApiError>() {
- Some(EmailApiError::RateLimited { retry_after, .. }) => *retry_after,
+ match error.reason {
+ FailureReason::GmailApiRateLimited => {
+ Some(match error.source.downcast_ref::<EmailApiError>() {
+ Some(EmailApiError::RateLimited { retry_after, .. }) => *retry_after,
+ _ => None,
+ })
+ }
+ FailureReason::BackfillWritesBusy => Some(None),
_ => None,
- })
+ }
}
/// How often the received operation has already been deferred.
diff --git a/services/email_service/src/pubsub/backfill/deferral/test.rs b/services/email_service/src/pubsub/backfill/deferral/test.rs
index ff001f258b..32be6c7685 100644
--- a/services/email_service/src/pubsub/backfill/deferral/test.rs
+++ b/services/email_service/src/pubsub/backfill/deferral/test.rs
@@ -53,6 +53,16 @@ fn a_gmail_rate_limit_without_retry_after_is_deferred_too() {
assert_eq!(rate_limit(&throttled), Some(None));
}
+#[test]
+fn a_write_refused_a_backfill_write_slot_is_deferred_with_backoff() {
+ let refused = DetailedError {
+ reason: FailureReason::BackfillWritesBusy,
+ source: anyhow::anyhow!("no backfill write slot freed up in time"),
+ };
+
+ assert_eq!(rate_limit(&refused), Some(None));
+}
+
#[test]
fn other_retryable_failures_keep_waiting_out_the_visibility_timeout() {
let database = retryable(map_db_error(
diff --git a/services/email_service/src/pubsub/backfill/error_handlers.rs b/services/email_service/src/pubsub/backfill/error_handlers.rs
index 8e15bf5cad..1c572b06da 100644
--- a/services/email_service/src/pubsub/backfill/error_handlers.rs
+++ b/services/email_service/src/pubsub/backfill/error_handlers.rs
@@ -155,10 +155,14 @@ pub async fn handle_retryable_error(
let error_chain = format!("{:#}", _e.source);
tracing::Span::current().record("error", &error_chain);
- // Rate limits are expected backpressure, redelivered after the visibility
- // timeout; the email API client logs provider throttling at ERROR.
- if _e.reason == FailureReason::GmailApiRateLimited {
- tracing::debug!("Rate-limited backfill operation will be redelivered");
+ // Rate limits and busy write slots are expected backpressure, redelivered
+ // after the visibility timeout; the email API client logs provider
+ // throttling at ERROR.
+ if matches!(
+ _e.reason,
+ FailureReason::GmailApiRateLimited | FailureReason::BackfillWritesBusy
+ ) {
+ tracing::debug!("Refused backfill operation will be redelivered");
return Ok(());
}
diff --git a/services/email_service/src/pubsub/backfill/mod.rs b/services/email_service/src/pubsub/backfill/mod.rs
index 459a9edad3..764154760a 100644
--- a/services/email_service/src/pubsub/backfill/mod.rs
+++ b/services/email_service/src/pubsub/backfill/mod.rs
@@ -16,3 +16,4 @@ mod process;
mod seed_sent_contact;
mod update_metadata;
pub mod worker;
+mod write_slot;
diff --git a/services/email_service/src/pubsub/backfill/write_slot.rs b/services/email_service/src/pubsub/backfill/write_slot.rs
new file mode 100644
index 0000000000..eb7d50491d
--- /dev/null
+++ b/services/email_service/src/pubsub/backfill/write_slot.rs
@@ -0,0 +1,27 @@
+//! Admission to the fleet's backfill write slots
+//! (`util::redis::backfill_write_slots`).
+
+use crate::pubsub::context::PubSubContext;
+use crate::util::redis::backfill_write_slots::BackfillWriteSlot;
+use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
+
+/// Takes a backfill write slot for one message write. A slot that stays taken,
+/// or Redis failing to decide, refuses the write so the operation is deferred
+/// instead of written unmetered.
+pub(super) async fn acquire(ctx: &PubSubContext) -> Result<BackfillWriteSlot, ProcessingError> {
+ let refused = |source: anyhow::Error| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::BackfillWritesBusy,
+ source,
+ })
+ };
+ match ctx.redis_client.acquire_backfill_write_slot().await {
+ Ok(Some(slot)) => Ok(slot),
+ Ok(None) => Err(refused(anyhow::anyhow!(
+ "no backfill write slot freed up in time"
+ ))),
+ Err(error) => Err(refused(
+ anyhow::Error::new(error).context("backfill write slots are unavailable"),
+ )),
+ }
+}
diff --git a/services/email_service/src/util/redis.rs b/services/email_service/src/util/redis.rs
index 299e4f37f7..4df98494c5 100644
--- a/services/email_service/src/util/redis.rs
+++ b/services/email_service/src/util/redis.rs
@@ -1,5 +1,6 @@
pub mod access_token;
pub mod backfill;
+pub mod backfill_write_slots;
pub mod health_probe;
pub mod lock;
pub mod rate_limit;
diff --git a/services/email_service/src/util/redis/backfill_write_slots.rs b/services/email_service/src/util/redis/backfill_write_slots.rs
new file mode 100644
index 0000000000..289cf42a15
--- /dev/null
+++ b/services/email_service/src/util/redis/backfill_write_slots.rs
@@ -0,0 +1,234 @@
+//! A fleet-wide cap on concurrent backfill message writes.
+//!
+//! Every backfill worker of every email-service task writes messages straight
+//! to the primary, one transaction per message, and nothing else bounds how
+//! many run at once (25 workers per task, 5 to 10 tasks). Message bodies land
+//! in one TOAST table, so a large backfill queues its writers on that table's
+//! relation-extension lock and slows every other query on the database. A slot
+//! is a Redis lease; at most [`CAPACITY`] backfill writes hold one at a time,
+//! however many tasks are running.
+
+use crate::util::redis::RedisClient;
+use rand::Rng;
+use std::sync::LazyLock;
+use std::time::Duration;
+use uuid::Uuid;
+
+#[cfg(test)]
+mod test;
+
+/// Concurrent backfill message writes across the fleet. A healthy write (the
+/// contacts upsert plus the message transaction) averages about 100 ms, so
+/// twenty slots admit roughly 200 messages a second while the database is
+/// healthy, above normal busy hours, and fewer as it slows down. Unbounded,
+/// 125 to 250 backfill workers could write at once.
+const CAPACITY: u32 = 20;
+/// Frees the slot of a crashed worker. A held slot renews it every
+/// [`RENEW_EVERY`], so a write that outlasts it keeps its slot.
+const LEASE: Duration = Duration::from_secs(120);
+const RENEW_EVERY: Duration = Duration::from_secs(30);
+/// How long a write waits for a slot before its operation is deferred. With the
+/// Gmail gate's wait before it, the handler stays inside the backfill queue's
+/// 60-second visibility timeout.
+const WAIT: Duration = Duration::from_secs(10);
+/// First and longest pause between admission checks. Waiters back off so a
+/// saturated fleet doesn't poll Redis in lockstep.
+const FIRST_POLL: Duration = Duration::from_millis(25);
+const MAX_POLL: Duration = Duration::from_millis(400);
+const KEY: &str = "email:backfill:write_slots";
+
+/// Admits the caller when fewer than `capacity` unexpired leases are held.
+/// Returns the slots in use counting this one, or 0 when every slot is taken.
+static ACQUIRE: LazyLock<redis::Script> = LazyLock::new(|| {
+ redis::Script::new(
+ r#"
+ local clock = redis.call('TIME')
+ local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
+ redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
+ local in_flight = redis.call('ZCARD', KEYS[1])
+ if in_flight >= tonumber(ARGV[2]) then return 0 end
+ redis.call('ZADD', KEYS[1], now + tonumber(ARGV[3]), ARGV[1])
+ redis.call('PEXPIRE', KEYS[1], ARGV[3])
+ return in_flight + 1
+ "#,
+ )
+});
+
+/// Pushes a held lease forward. Returns 1, or 0 when the lease already expired
+/// and was dropped; it is never re-added, so an expired slot can't overfill.
+static RENEW: LazyLock<redis::Script> = LazyLock::new(|| {
+ redis::Script::new(
+ r#"
+ local clock = redis.call('TIME')
+ local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)
+ local renewed = redis.call('ZADD', KEYS[1], 'XX', 'CH', now + tonumber(ARGV[2]), ARGV[1])
+ if renewed == 1 then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end
+ return renewed
+ "#,
+ )
+});
+
+/// One set of write slots. Production uses [`SLOTS`]; tests use their own keys.
+struct SlotSet<'a> {
+ key: &'a str,
+ capacity: u32,
+ lease: Duration,
+ wait: Duration,
+}
+
+const SLOTS: SlotSet<'static> = SlotSet {
+ key: KEY,
+ capacity: CAPACITY,
+ lease: LEASE,
+ wait: WAIT,
+};
+
+/// A held backfill write slot.
+///
+/// Run the write with [`BackfillWriteSlot::hold`], which keeps the lease alive
+/// and releases the slot afterwards. A dropped slot is released by a spawned
+/// task, and the lease frees it if the process dies first.
+pub struct BackfillWriteSlot {
+ /// `None` after an explicit release, so `Drop` does nothing.
+ inner: Option<SlotLease>,
+}
+
+struct SlotLease {
+ client: redis::Client,
+ key: String,
+ id: String,
+ lease: Duration,
+}
+
+impl SlotLease {
+ /// Pushes the lease forward; `false` means it had already expired.
+ async fn renew(&self) -> redis::RedisResult<bool> {
+ let mut connection = self.client.get_multiplexed_async_connection().await?;
+ let renewed: u32 = RENEW
+ .key(&self.key)
+ .arg(&self.id)
+ .arg(self.lease.as_millis().to_string())
+ .invoke_async(&mut connection)
+ .await?;
+ Ok(renewed == 1)
+ }
+
+ async fn release(&self) {
+ let result: redis::RedisResult<i32> = async {
+ let mut connection = self.client.get_multiplexed_async_connection().await?;
+ redis::cmd("ZREM")
+ .arg(&self.key)
+ .arg(&self.id)
+ .query_async(&mut connection)
+ .await
+ }
+ .await;
+ if let Err(error) = result {
+ tracing::warn!(error = %error, "failed to release a backfill write slot; its lease will expire");
+ }
+ }
+}
+
+impl BackfillWriteSlot {
+ /// Runs `work` while holding the slot, renewing its lease until the work
+ /// finishes, then returns the slot to the fleet.
+ pub async fn hold<F: Future>(self, work: F) -> F::Output {
+ let mut work = std::pin::pin!(work);
+ let mut renewals =
+ tokio::time::interval_at(tokio::time::Instant::now() + RENEW_EVERY, RENEW_EVERY);
+ let output = loop {
+ tokio::select! {
+ output = &mut work => break output,
+ _ = renewals.tick() => {
+ if let Some(lease) = &self.inner {
+ match lease.renew().await {
+ Ok(true) => {}
+ Ok(false) => tracing::warn!("a backfill write outlived its slot's lease"),
+ Err(error) => tracing::warn!(error = %error, "failed to renew a backfill write slot"),
+ }
+ }
+ }
+ }
+ };
+ self.release().await;
+ output
+ }
+
+ /// Returns the slot to the fleet.
+ pub async fn release(mut self) {
+ if let Some(lease) = self.inner.take() {
+ lease.release().await;
+ }
+ }
+}
+
+impl Drop for BackfillWriteSlot {
+ fn drop(&mut self) {
+ if let Some(lease) = self.inner.take() {
+ // Drop cannot wait. Without a runtime, the lease frees the slot.
+ if let Ok(runtime) = tokio::runtime::Handle::try_current() {
+ runtime.spawn(async move { lease.release().await });
+ }
+ }
+ }
+}
+
+impl RedisClient {
+ /// Waits for one of the fleet's backfill write slots. Returns `None` when
+ /// none frees up in time; the caller defers its operation.
+ pub async fn acquire_backfill_write_slot(
+ &self,
+ ) -> redis::RedisResult<Option<BackfillWriteSlot>> {
+ self.acquire_write_slot(&SLOTS).await
+ }
+
+ #[tracing::instrument(
+ name = "email.backfill.write_slot",
+ skip(self, slots),
+ fields(admitted, in_flight, waited_ms),
+ err
+ )]
+ async fn acquire_write_slot(
+ &self,
+ slots: &SlotSet<'_>,
+ ) -> redis::RedisResult<Option<BackfillWriteSlot>> {
+ let id = Uuid::now_v7().to_string();
+ let started = tokio::time::Instant::now();
+ let deadline = started + slots.wait;
+ let mut poll = FIRST_POLL;
+ let mut connection = self.inner.get_multiplexed_async_connection().await?;
+ let span = tracing::Span::current();
+ loop {
+ let in_flight: u32 = ACQUIRE
+ .key(slots.key)
+ .arg(&id)
+ .arg(slots.capacity)
+ .arg(slots.lease.as_millis().to_string())
+ .invoke_async(&mut connection)
+ .await?;
+ let now = tokio::time::Instant::now();
+ if in_flight > 0 || now >= deadline {
+ span.record("admitted", in_flight > 0);
+ let waited_ms = u64::try_from(now.duration_since(started).as_millis());
+ span.record("waited_ms", waited_ms.unwrap_or(u64::MAX));
+ if in_flight == 0 {
+ return Ok(None);
+ }
+ span.record("in_flight", in_flight);
+ return Ok(Some(BackfillWriteSlot {
+ inner: Some(SlotLease {
+ client: self.inner.clone(),
+ key: slots.key.to_string(),
+ id,
+ lease: slots.lease,
+ }),
+ }));
+ }
+ let pause = rand::rng()
+ .random_range(poll / 2..=poll)
+ .min(deadline - now);
+ tokio::time::sleep(pause).await;
+ poll = (poll * 2).min(MAX_POLL);
+ }
+ }
+}
diff --git a/services/email_service/src/util/redis/backfill_write_slots/test.rs b/services/email_service/src/util/redis/backfill_write_slots/test.rs
new file mode 100644
index 0000000000..f3e3916daf
--- /dev/null
+++ b/services/email_service/src/util/redis/backfill_write_slots/test.rs
@@ -0,0 +1,179 @@
+use super::*;
+
+fn client() -> RedisClient {
+ RedisClient::new(redis::Client::open("redis://127.0.0.1:6379/").unwrap(), 60)
+}
+
+/// A slot set of its own, so tests never touch the fleet's slots or each other's.
+fn unique_key() -> String {
+ format!("test:email:backfill:write_slots:{}", Uuid::now_v7())
+}
+
+fn slots(key: &str, capacity: u32) -> SlotSet<'_> {
+ SlotSet {
+ key,
+ capacity,
+ lease: LEASE,
+ wait: Duration::ZERO,
+ }
+}
+
+async fn cleanup(client: &RedisClient, key: &str) {
+ let mut connection = client
+ .inner
+ .get_multiplexed_async_connection()
+ .await
+ .unwrap();
+ let _: i32 = redis::cmd("DEL")
+ .arg(key)
+ .query_async(&mut connection)
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn writes_beyond_capacity_are_refused_until_a_slot_is_released() {
+ let client = client();
+ let key = unique_key();
+ let slots = slots(&key, 2);
+
+ let first = client
+ .acquire_write_slot(&slots)
+ .await
+ .unwrap()
+ .expect("the first write gets a slot");
+ let second = client
+ .acquire_write_slot(&slots)
+ .await
+ .unwrap()
+ .expect("the second write gets a slot");
+ assert!(
+ client.acquire_write_slot(&slots).await.unwrap().is_none(),
+ "a third concurrent write must wait"
+ );
+
+ first.release().await;
+ let third = client
+ .acquire_write_slot(&slots)
+ .await
+ .unwrap()
+ .expect("a released slot admits the next write");
+
+ second.release().await;
+ third.release().await;
+ cleanup(&client, &key).await;
+}
+
+#[tokio::test]
+async fn a_crashed_writers_expired_lease_frees_its_slot() {
+ let client = client();
+ let key = unique_key();
+ let mut connection = client
+ .inner
+ .get_multiplexed_async_connection()
+ .await
+ .unwrap();
+ // A lease that expired long ago, as left by a worker that died holding it.
+ let _: i32 = redis::cmd("ZADD")
+ .arg(&key)
+ .arg(1)
+ .arg("crashed-writer")
+ .query_async(&mut connection)
+ .await
+ .unwrap();
+
+ let slot = client
+ .acquire_write_slot(&slots(&key, 1))
+ .await
+ .unwrap()
+ .expect("an expired lease does not hold the only slot");
+
+ slot.release().await;
+ cleanup(&client, &key).await;
+}
+
+fn lease_id(slot: &BackfillWriteSlot) -> String {
+ slot.inner.as_ref().expect("a held slot").id.clone()
+}
+
+#[tokio::test]
+async fn renewing_a_held_slot_pushes_its_lease_forward() {
+ let client = client();
+ let key = unique_key();
+ let slot = client
+ .acquire_write_slot(&slots(&key, 1))
+ .await
+ .unwrap()
+ .expect("the only slot is free");
+ let id = lease_id(&slot);
+ let mut connection = client
+ .inner
+ .get_multiplexed_async_connection()
+ .await
+ .unwrap();
+ // A lease about to expire, as after a long write.
+ let _: i32 = redis::cmd("ZADD")
+ .arg(&key)
+ .arg("XX")
+ .arg(1)
+ .arg(&id)
+ .query_async(&mut connection)
+ .await
+ .unwrap();
+
+ let renewed = slot.inner.as_ref().unwrap().renew().await.unwrap();
+
+ let expires_at: f64 = redis::cmd("ZSCORE")
+ .arg(&key)
+ .arg(&id)
+ .query_async(&mut connection)
+ .await
+ .unwrap();
+ assert!(renewed);
+ assert!(
+ expires_at > 1.0e12,
+ "the lease now ends at a real time ahead of now"
+ );
+ slot.release().await;
+ cleanup(&client, &key).await;
+}
+
+#[tokio::test]
+async fn an_expired_lease_is_not_renewed_back_into_a_full_set() {
+ let client = client();
+ let key = unique_key();
+ let slot = client
+ .acquire_write_slot(&slots(&key, 1))
+ .await
+ .unwrap()
+ .expect("the only slot is free");
+ let mut connection = client
+ .inner
+ .get_multiplexed_async_connection()
+ .await
+ .unwrap();
+ // The lease expired and another writer took the slot.
+ let _: i32 = redis::cmd("ZREM")
+ .arg(&key)
+ .arg(lease_id(&slot))
+ .query_async(&mut connection)
+ .await
+ .unwrap();
+ let other = client
+ .acquire_write_slot(&slots(&key, 1))
+ .await
+ .unwrap()
+ .expect("the expired slot is free again");
+
+ let renewed = slot.inner.as_ref().unwrap().renew().await.unwrap();
+
+ let held: i32 = redis::cmd("ZCARD")
+ .arg(&key)
+ .query_async(&mut connection)
+ .await
+ .unwrap();
+ assert!(!renewed);
+ assert_eq!(held, 1, "only the new writer holds the slot");
+ other.release().await;
+ cleanup(&client, &key).await;
+}