Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(email): backfill each thread in one Gmail read and one transaction (#8913)
teoa949192ceb6b
diff --git a/crates/email_api_client/src/domain/ports.rs b/crates/email_api_client/src/domain/ports.rsindex 418503b569..dc1daacf8d 100644--- a/crates/email_api_client/src/domain/ports.rs+++ b/crates/email_api_client/src/domain/ports.rs@@ -6,7 +6,6 @@ pub use mailbox::*;use std::future::Future;use models_email::email::service::label::Label;-use models_email::email::service::message::Message;use models_email::service::contact::{Contact, ContactList};use models_email::service::link::Link;use uuid::Uuid;@@ -84,13 +83,14 @@ pub trait MailboxMessageClient: Send + Sync + 'static {provider_thread_id: &str,) -> impl Future<Output = Result<Vec<String>, EmailApiError>> + Send;- /// Fetches and normalizes every message in a provider thread.+ /// Fetches and normalizes every message in a provider thread, each with+ /// its calendar invitation parts, in one provider read.fn get_thread(&self,access_token: &AccessToken,link_id: Uuid,provider_thread_id: &str,- ) -> impl Future<Output = Result<Vec<Message>, EmailApiError>> + Send;+ ) -> impl Future<Output = Result<Vec<MessageWithCalendarParts>, EmailApiError>> + Send;/// Lists one page of provider threads.fn list_threads(@@ -429,7 +429,7 @@ impl MailboxMessageClient for NoOpMailboxClient {_: &AccessToken,_: Uuid,_: &str,- ) -> Result<Vec<Message>, EmailApiError> {+ ) -> Result<Vec<MessageWithCalendarParts>, EmailApiError> {Err(unavailable())}diff --git a/crates/email_api_client/src/domain/service/messages.rs b/crates/email_api_client/src/domain/service/messages.rsindex 59c3557b74..f75ee04b6e 100644--- a/crates/email_api_client/src/domain/service/messages.rs+++ b/crates/email_api_client/src/domain/service/messages.rs@@ -1,4 +1,3 @@-use models_email::email::service::message::Message;use uuid::Uuid;use super::super::models::{CalendarPart, EmailApiError, MessageWithCalendarParts, ThreadListPage};@@ -79,13 +78,14 @@ where.await}- /// Fetches and normalizes every message in one provider thread.+ /// Fetches and normalizes every message in one provider thread, with any+ /// calendar invitation parts, in a single provider read.#[tracing::instrument(skip(self), fields(retry_count = 0, gate_in_flight = tracing::field::Empty, gate_cap = tracing::field::Empty))]pub async fn get_thread(&self,link_id: Uuid,provider_thread_id: &str,- ) -> Result<Vec<Message>, EmailApiError> {+ ) -> Result<Vec<MessageWithCalendarParts>, EmailApiError> {self.read_with_token(link_id, ApiOperationKind::GetThread, |token| async move {self.repository.get_thread(&token, link_id, provider_thread_id)diff --git a/crates/email_api_client/src/domain/service/messages/test.rs b/crates/email_api_client/src/domain/service/messages/test.rsindex cd6b50654e..62feae72ca 100644--- a/crates/email_api_client/src/domain/service/messages/test.rs+++ b/crates/email_api_client/src/domain/service/messages/test.rs@@ -1,6 +1,5 @@use std::sync::{Arc, Mutex};-use models_email::email::service::message::Message;use uuid::Uuid;use super::super::test_support::{Call, FakeGate, FakeTokenSource, call_log};@@ -96,7 +95,7 @@ impl MailboxMessageClient for MessageClient {access_token: &AccessToken,link_id: Uuid,thread_id: &str,- ) -> Result<Vec<Message>, EmailApiError> {+ ) -> Result<Vec<crate::domain::models::MessageWithCalendarParts>, EmailApiError> {Err(self.record(access_token,MessageCall::Thread(link_id, thread_id.to_string()),diff --git a/crates/email_api_client/src/domain/service/test_support.rs b/crates/email_api_client/src/domain/service/test_support.rsindex ddcfa39c77..34d6be531c 100644--- a/crates/email_api_client/src/domain/service/test_support.rs+++ b/crates/email_api_client/src/domain/service/test_support.rs@@ -1,7 +1,6 @@use std::sync::{Arc, Mutex};use models_email::email::service::label::Label;-use models_email::email::service::message::Message;use models_email::service::contact::{Contact, ContactList};use uuid::Uuid;@@ -217,7 +216,7 @@ impl MailboxMessageClient for FakeRepository {_: &AccessToken,_: Uuid,_: &str,- ) -> Result<Vec<Message>, EmailApiError> {+ ) -> Result<Vec<crate::domain::models::MessageWithCalendarParts>, EmailApiError> {self.unavailable("get_thread")}diff --git a/crates/email_api_client/src/outbound/gmail/messages.rs b/crates/email_api_client/src/outbound/gmail/messages.rsindex 0124cbdf45..4a2749b789 100644--- a/crates/email_api_client/src/outbound/gmail/messages.rs+++ b/crates/email_api_client/src/outbound/gmail/messages.rs@@ -1,10 +1,11 @@//! Gmail message capability implementation.+use std::collections::HashMap;+use base64::{Engine as _,engine::general_purpose::{URL_SAFE, URL_SAFE_NO_PAD},};-use models_email::email::service::message::Message;use models_email::email::service::thread::ThreadSummary;use models_email::gmail::MessagePart;use uuid::Uuid;@@ -135,13 +136,36 @@ impl MailboxMessageClient for GmailApiClientRepository {access_token: &AccessToken,link_id: Uuid,provider_thread_id: &str,- ) -> Result<Vec<Message>, EmailApiError> {+ ) -> Result<Vec<MessageWithCalendarParts>, EmailApiError> {let resource = self.client.get_thread(access_token.expose_secret(), provider_thread_id).await.map_err(map_gmail_error)?;- Ok(map_thread_resource_to_service(resource, link_id)?.messages)+ // Collect calendar parts from the wire messages before conversion+ // consumes them, so a thread read needs no messages.get per message.+ let mut calendar_parts = resource+ .messages+ .iter()+ .map(|message| {+ Ok((+ message.id.clone(),+ collect_calendar_parts(&message.payload)?,+ ))+ })+ .collect::<Result<HashMap<_, _>, EmailApiError>>()?;+ Ok(map_thread_resource_to_service(resource, link_id)?+ .messages+ .into_iter()+ .map(|message| MessageWithCalendarParts {+ calendar_parts: message+ .provider_id+ .as_ref()+ .and_then(|id| calendar_parts.remove(id))+ .unwrap_or_default(),+ message,+ })+ .collect())}async fn list_threads(diff --git a/crates/email_api_client/src/outbound/gmail/test/messages.rs b/crates/email_api_client/src/outbound/gmail/test/messages.rsindex 81cd2454b0..afa592b1ce 100644--- a/crates/email_api_client/src/outbound/gmail/test/messages.rs+++ b/crates/email_api_client/src/outbound/gmail/test/messages.rs@@ -145,7 +145,44 @@ async fn normalizes_full_threads() {.await.unwrap();assert_eq!(messages.len(), 1);- assert_eq!(messages[0].provider_thread_id.as_deref(), Some("thread-1"));+ assert_eq!(+ messages[0].message.provider_thread_id.as_deref(),+ Some("thread-1")+ );+}++#[tokio::test]+async fn get_thread_keeps_each_messages_calendar_parts_from_one_wire_fetch() {+ let (server, repository) = repository().await;+ let thread: serde_json::Value =+ serde_json::from_str(include_str!("fixtures/thread_full.json")).unwrap();+ let mut messages = thread["messages"].as_array().unwrap().clone();+ // The invitation is the later message, so normalization reorders nothing+ // and each message must keep the parts found on its own wire payload.+ let mut invitation = calendar_message_body();+ invitation["internalDate"] = serde_json::json!("1700000001000");+ messages.push(invitation);+ Mock::given(method("GET"))+ .and(path("/users/me/threads/thread-1"))+ .and(query_param("format", "full"))+ .respond_with(+ ResponseTemplate::new(200)+ .set_body_json(serde_json::json!({ "id": "thread-1", "messages": messages })),+ )+ .expect(1)+ .mount(&server)+ .await;++ let fetched = repository+ .get_thread(&AccessToken::new("token"), Uuid::now_v7(), "thread-1")+ .await+ .unwrap();++ assert_eq!(fetched.len(), 2);+ assert_eq!(fetched[0].message.provider_id.as_deref(), Some("message-1"));+ assert!(fetched[0].calendar_parts.is_empty());+ assert_eq!(fetched[1].message.provider_id.as_deref(), Some("calendar"));+ assert_calendar_parts(&fetched[1].calendar_parts);}#[tokio::test]diff --git a/crates/email_db_client/src/messages/insert.rs b/crates/email_db_client/src/messages/insert.rsindex 3d35573b22..af1c67cf39 100644--- a/crates/email_db_client/src/messages/insert.rs+++ b/crates/email_db_client/src/messages/insert.rs@@ -194,6 +194,118 @@ async fn insert_db_message(Ok(existing_id)}+/// A message a backfill write left out because its own data can never be+/// stored; the rest of its thread was still written.+#[derive(Debug)]+pub struct SkippedMessage {+ /// The message's position in the slice passed to the write.+ pub index: usize,+ /// Why it could not be stored.+ pub error: anyhow::Error,+}++/// Writes backfilled messages one savepoint at a time.+///+/// A message whose contacts or rows fail with an error `is_unstorable`+/// accepts is skipped and reported, and the others still commit: one bad+/// message must not cost its whole thread. Any other error fails the write.+pub(crate) struct BackfillMessageWriter<'a, U> {+ pub(crate) pool: &'a PgPool,+ pub(crate) link_id: Uuid,+ pub(crate) is_unstorable: U,+}++impl<U: Fn(&anyhow::Error) -> bool> BackfillMessageWriter<'_, U> {+ /// Upserts each message's contacts, in order. Contacts are shared across+ /// messages, so this runs before the messages' transaction to avoid+ /// deadlocks. A skipped message gets `None`.+ pub(crate) async fn upsert_contacts(+ &self,+ messages: &[message::Message],+ skipped: &mut Vec<SkippedMessage>,+ ) -> anyhow::Result<Vec<Option<UpsertedRecipients>>> {+ let mut recipients = Vec::with_capacity(messages.len());+ for (index, message) in messages.iter().enumerate() {+ match contacts::upsert_message::parse_and_upsert_message_contacts(+ self.pool,+ self.link_id,+ addresses_from_message(message),+ )+ .await+ {+ Ok(upserted) => recipients.push(Some(upserted)),+ Err(error) if (self.is_unstorable)(&error) => {+ skipped.push(SkippedMessage { index, error });+ recipients.push(None);+ }+ Err(error) => return Err(error),+ }+ }+ Ok(recipients)+ }++ /// Inserts each message whose contacts were stored, under its own savepoint.+ pub(crate) async fn insert_messages(+ &self,+ tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,+ thread_db_id: Uuid,+ messages: &mut [message::Message],+ recipients: Vec<Option<UpsertedRecipients>>,+ skipped: &mut Vec<SkippedMessage>,+ ) -> anyhow::Result<()> {+ for (index, (message, recipients)) in messages.iter_mut().zip(recipients).enumerate() {+ let Some(recipients) = recipients else {+ continue;+ };+ let mut savepoint = sqlx::Acquire::begin(&mut *tx).await?;+ match insert_message_with_tx(+ &mut savepoint,+ thread_db_id,+ message,+ self.link_id,+ recipients,+ false,+ )+ .await+ {+ Ok(()) => savepoint.commit().await?,+ Err(error) if (self.is_unstorable)(&error) => {+ savepoint.rollback().await?;+ skipped.push(SkippedMessage { index, error });+ }+ Err(error) => return Err(error),+ }+ }+ Ok(())+ }+}++/// Inserts messages into an existing thread in one transaction, leaving its+/// metadata and `replying_to` links to the caller's later metadata update.+/// Returns the messages it skipped because `is_unstorable` accepted their error.+#[tracing::instrument(skip(pool, messages, is_unstorable), fields(message_count = messages.len()), err)]+pub async fn insert_thread_messages(+ pool: &PgPool,+ thread_db_id: Uuid,+ link_id: Uuid,+ messages: &mut [message::Message],+ is_unstorable: impl Fn(&anyhow::Error) -> bool,+) -> anyhow::Result<Vec<SkippedMessage>> {+ let writer = BackfillMessageWriter {+ pool,+ link_id,+ is_unstorable,+ };+ let mut skipped = Vec::new();+ let recipients = writer.upsert_contacts(messages, &mut skipped).await?;+ let mut tx = pool.begin().await?;+ writer+ .insert_messages(&mut tx, thread_db_id, messages, recipients, &mut skipped)+ .await?;+ tx.commit().await?;+ Ok(skipped)+}+/// Inserts a single message into the database with transaction handling#[tracing::instrument(skip(pool, message), fields(link_id = %message.link_id), err)]pub async fn insert_message(diff --git a/crates/email_db_client/src/threads/insert.rs b/crates/email_db_client/src/threads/insert.rsindex 4030d64236..632d8ebbe2 100644--- a/crates/email_db_client/src/threads/insert.rs+++ b/crates/email_db_client/src/threads/insert.rs@@ -6,7 +6,7 @@ use crate::parse::service_to_db::addresses_from_message;use crate::{contacts, messages, parse};use models_email::email::db::address::UpsertedRecipients;-use models_email::email::service::thread;+use models_email::email::service::{message, thread};use sqlx::types::Uuid;use sqlx::{PgConnection, PgPool};use std::collections::HashMap;@@ -156,15 +156,9 @@ pub async fn insert_thread(Ok(existing_id)}-/// Inserts a thread with no metadata, returning its id, or `None` when the-/// mailbox already has the thread, including when a concurrent insert won.-#[tracing::instrument(skip(pool), err)]-pub async fn insert_blank_thread(- pool: &PgPool,- thread_provider_id: &str,- link_id: Uuid,-) -> anyhow::Result<Option<Uuid>> {- let thread = thread::Thread {+/// A thread row with no metadata yet, for writers that fill it in afterwards.+fn blank_thread(thread_provider_id: &str, link_id: Uuid) -> thread::Thread {+ thread::Thread {db_id: macro_uuid::generate_uuid_v7(),provider_id: Some(thread_provider_id.to_string()),link_id,@@ -176,10 +170,61 @@ pub async fn insert_blank_thread(created_at: Default::default(),updated_at: Default::default(),messages: vec![],- };+ }+}++/// Inserts a thread with no metadata, returning its id, or `None` when the+/// mailbox already has the thread, including when a concurrent insert won.+#[tracing::instrument(skip(pool), err)]+pub async fn insert_blank_thread(+ pool: &PgPool,+ thread_provider_id: &str,+ link_id: Uuid,+) -> anyhow::Result<Option<Uuid>> {+ let thread = blank_thread(thread_provider_id, link_id);let mut tx = pool.begin().await?;let id = insert_thread(&mut tx, &thread, link_id).await?;tx.commit().await?;Ok((id == thread.db_id).then_some(id))}++/// Inserts a backfilled thread and all of its messages in one transaction,+/// returning the thread id and the messages it skipped, or `None` when the+/// mailbox already has the thread, including when a concurrent insert won;+/// nothing is written then.+///+/// A message `is_unstorable` rejects is left out under its own savepoint while+/// the rest commit. Any other failure leaves no thread behind, so a retried+/// backfill writes the whole thread again. Thread metadata and `replying_to`+/// links are left to the backfill's metadata step, as with+/// [`insert_blank_thread`].+#[tracing::instrument(skip(pool, messages, is_unstorable), fields(message_count = messages.len()), err)]+pub async fn insert_backfilled_thread(+ pool: &PgPool,+ thread_provider_id: &str,+ link_id: Uuid,+ messages: &mut [message::Message],+ is_unstorable: impl Fn(&anyhow::Error) -> bool,+) -> anyhow::Result<Option<(Uuid, Vec<messages::insert::SkippedMessage>)>> {+ let writer = messages::insert::BackfillMessageWriter {+ pool,+ link_id,+ is_unstorable,+ };+ let mut skipped = Vec::new();+ let recipients = writer.upsert_contacts(messages, &mut skipped).await?;+ let thread = blank_thread(thread_provider_id, link_id);++ let mut tx = pool.begin().await?;+ let id = insert_thread(&mut tx, &thread, link_id).await?;+ if id != thread.db_id {+ tx.rollback().await?;+ return Ok(None);+ }+ writer+ .insert_messages(&mut tx, id, messages, recipients, &mut skipped)+ .await?;+ tx.commit().await?;+ Ok(Some((id, skipped)))+}diff --git a/crates/email_db_client/src/threads/insert/test.rs b/crates/email_db_client/src/threads/insert/test.rsindex 42c6582ad5..ea7fc45e44 100644--- a/crates/email_db_client/src/threads/insert/test.rs+++ b/crates/email_db_client/src/threads/insert/test.rs@@ -348,3 +348,169 @@ async fn oversized_provider_values_do_not_block_the_thread_insert(Ok(())}++fn backfilled_messages(link_id: Uuid) -> Vec<message::Message> {+ use crate::messages::insert::test::{PARENT_GLOBAL_ID, SyncedMessage};++ let parent = SyncedMessage {+ provider_id: "backfill-parent",+ global_id: PARENT_GLOBAL_ID,+ in_reply_to: None,+ to: &["[email protected]"],+ cc: &[],+ labels: &[],+ body: "parent body",+ };+ let reply = SyncedMessage {+ provider_id: "backfill-reply",+ global_id: "<[email protected]>",+ in_reply_to: Some(PARENT_GLOBAL_ID),+ to: &["[email protected]"],+ cc: &["[email protected]"],+ labels: &[],+ body: "reply body",+ };+ // Backfill writes the thread row itself, so the messages' own thread id is unused.+ let unused_thread_id = Uuid::now_v7();+ vec![+ parent.build(link_id, unused_thread_id),+ reply.build(link_id, unused_thread_id),+ ]+}++/// Postgres "data exception" (SQLSTATE class 22): the value itself can't be stored.+fn data_exception(error: &anyhow::Error) -> bool {+ error.chain().any(|cause| {+ cause+ .downcast_ref::<sqlx::Error>()+ .and_then(|error| error.as_database_error())+ .and_then(|error| error.code())+ .is_some_and(|code| code.starts_with("22"))+ })+}++async fn thread_message_count(pool: &Pool<Postgres>, thread_id: Uuid) -> anyhow::Result<i64> {+ Ok(sqlx::query_scalar!(+ r#"SELECT COUNT(*) AS "count!" FROM email_messages WHERE thread_id = $1"#,+ thread_id+ )+ .fetch_one(pool)+ .await?)+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("links"))+)]+async fn a_backfilled_thread_is_written_with_all_of_its_messages(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;+ let mut messages = backfilled_messages(link_id);++ let (thread_id, skipped) = insert_backfilled_thread(+ &pool,+ "backfill-thread",+ link_id,+ &mut messages,+ data_exception,+ )+ .await?+ .expect("the thread is new");++ assert!(skipped.is_empty());++ assert_eq!(thread_message_count(&pool, thread_id).await?, 2);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("links"))+)]+async fn a_thread_another_copy_already_wrote_gets_no_messages_from_this_one(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;+ let existing_id = insert_blank_thread(&pool, "backfill-thread", link_id)+ .await?+ .expect("the thread is new");+ let mut messages = backfilled_messages(link_id);++ let inserted = insert_backfilled_thread(+ &pool,+ "backfill-thread",+ link_id,+ &mut messages,+ data_exception,+ )+ .await?;++ assert!(+ inserted.is_none(),+ "only the copy that wrote the thread finishes it"+ );+ assert_eq!(thread_message_count(&pool, existing_id).await?, 0);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("links"))+)]+async fn missing_messages_are_written_into_their_existing_thread(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;+ let thread_id = insert_blank_thread(&pool, "backfill-thread", link_id)+ .await?+ .expect("the thread is new");+ let mut messages = backfilled_messages(link_id);++ let skipped = crate::messages::insert::insert_thread_messages(+ &pool,+ thread_id,+ link_id,+ &mut messages,+ data_exception,+ )+ .await?;++ assert!(skipped.is_empty());++ assert_eq!(thread_message_count(&pool, thread_id).await?, 2);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("links"))+)]+async fn a_message_that_cannot_be_stored_is_skipped_and_the_rest_of_its_thread_is_written(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;+ let mut messages = backfilled_messages(link_id);+ // Postgres text can't hold NUL, so this message fails identically every time.+ messages[0].body_text = Some("broken \0 body".to_string());++ let (thread_id, skipped) = insert_backfilled_thread(+ &pool,+ "backfill-thread",+ link_id,+ &mut messages,+ data_exception,+ )+ .await?+ .expect("the thread is new");++ assert_eq!(+ skipped+ .iter()+ .map(|skipped| skipped.index)+ .collect::<Vec<_>>(),+ vec![0]+ );+ assert_eq!(thread_message_count(&pool, thread_id).await?, 1);+ Ok(())+}diff --git a/crates/sqs_worker/src/lib.rs b/crates/sqs_worker/src/lib.rsindex ebc45cdc7f..c9c3527b7e 100644--- a/crates/sqs_worker/src/lib.rs+++ b/crates/sqs_worker/src/lib.rs@@ -57,6 +57,25 @@ impl SQSWorker {pub async fn delete_message(&self, receipt_handle: &str) -> anyhow::Result<()> {delete_message::delete_message(&self.inner, &self.queue_url, receipt_handle).await}++ /// Hides a received message from other consumers for `timeout_seconds`+ /// from now, so work that outlasts the queue's visibility timeout is not+ /// delivered again while it still runs.+ #[tracing::instrument(skip(self, receipt_handle), err)]+ pub async fn change_message_visibility(+ &self,+ receipt_handle: &str,+ timeout_seconds: i32,+ ) -> anyhow::Result<()> {+ self.inner+ .change_message_visibility()+ .queue_url(&self.queue_url)+ .receipt_handle(receipt_handle)+ .visibility_timeout(timeout_seconds)+ .send()+ .await?;+ Ok(())+ }}#[tracing::instrument(skip(sqs_worker, message), fields(message_id=message.message_id, message_receipt_handle=message.receipt_handle))]diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.mdindex bcbaecbe60..04cd301407 100644--- a/services/email_service/AGENTS.md+++ b/services/email_service/AGENTS.md@@ -81,7 +81,23 @@ the gateway `/email` prefix.would spend the queue's 20 deliveries.Inbox sync releases the upload claims of a refused attachment fetch andrequeues the upsert, because a claimed attachment is never offered again.-- Backfill message writes hold one of twenty fleet-wide Redis write slots+- BackfillThread reads a whole thread with one Gmail `threads.get` and writes+ the thread row and all of its messages in one transaction+ (`insert_backfilled_thread`), so a failed write leaves nothing for a retry+ to skip. Each message has its own savepoint: one whose data can never be+ stored (`db_error::is_unstorable`) is skipped and logged, and the rest of the+ thread commits. The CRM fan-out is enqueued after the write slot is taken+ and before the write: after the commit nothing in the handler may request a+ retry, because a retry would find the thread and skip it, and an operation+ deferred for a busy slot must not repeat the fan-out. Invitation and attachment fan-outs only log their+ failures, and the metadata step is enqueued last. BackfillMessage only drains+ operations enqueued before thread writes.+- Backfill keeps a message hidden while its operation runs+ (`process::keep_invisible`, a 20 s heartbeat restoring the queue's 60 s+ visibility timeout). Without it, a slow thread is delivered to a second+ worker, which counts the written thread again.+- Each backfill thread write, and each BackfillMessage write still draining,+ holds one of twenty fleet-wide Redis write slots(`util/redis/backfill_write_slots.rs`), however many workers and tasks run,so a large backfill can't queue enough writers on the primary to slow everyother query. A write waits up to 10 s for a slot and is then deferred like adiff --git a/services/email_service/src/backfill_completion_service.rs b/services/email_service/src/backfill_completion_service.rsindex 97c9f07245..6d1d9da13e 100644--- a/services/email_service/src/backfill_completion_service.rs+++ b/services/email_service/src/backfill_completion_service.rs@@ -433,16 +433,29 @@ async fn handle_thread_completed(job_id: Uuid,p: &BackfillMessagePayload,) -> Result<(), ProcessingError> {- let new_payload = UpdateMetadataPayload {- thread_provider_id: p.thread_provider_id.clone(),- thread_db_id: p.thread_db_id,- };+ enqueue_thread_metadata_update(ctx, link, job_id, &p.thread_provider_id, p.thread_db_id)+ .await?;+ handle_thread_attachment_upload(ctx, link, job_id, p.thread_db_id).await+}+/// Hands a fully written thread to its UpdateThreadMetadata step, which+/// computes the thread's metadata and counts it as completed.+#[tracing::instrument(skip(ctx))]+pub async fn enqueue_thread_metadata_update(+ ctx: &PubSubContext,+ link: &Link,+ job_id: Uuid,+ thread_provider_id: &str,+ thread_db_id: Uuid,+) -> Result<(), ProcessingError> {let ps_message = BackfillPubsubMessage {backfill_operation: BackfillOperation::UpdateThreadMetadata(JobScopedPayload {link_id: link.id,job_id,- payload: new_payload,+ payload: UpdateMetadataPayload {+ thread_provider_id: thread_provider_id.to_string(),+ thread_db_id,+ },}),};@@ -454,11 +467,7 @@ async fn handle_thread_completed(reason: FailureReason::SqsEnqueueFailed,source: e.context("Failed to enqueue metadata message".to_string()),})- })?;-- handle_thread_attachment_upload(ctx, link, job_id, p.thread_db_id).await?;-- Ok(())+ })}/// Creates BackfillAttachmentPayload messages and enqueues them via SQS for the given attachments@@ -543,8 +552,9 @@ async fn send_attachment_backfill_messages(Ok(())}+/// Enqueues the uploads of a backfilled thread's attachments.#[tracing::instrument(skip(ctx))]-async fn handle_thread_attachment_upload(+pub async fn handle_thread_attachment_upload(ctx: &PubSubContext,link: &Link,job_id: Uuid,diff --git a/services/email_service/src/pubsub/backfill/backfill_message.rs b/services/email_service/src/pubsub/backfill/backfill_message.rsindex 1033f98c6c..33c9047898 100644--- a/services/email_service/src/pubsub/backfill/backfill_message.rs+++ b/services/email_service/src/pubsub/backfill/backfill_message.rs@@ -4,13 +4,18 @@ use crate::pubsub::backfill::{increment_counters, write_slot};use crate::pubsub::context::PubSubContext;use crate::pubsub::util::{CrmContactRecipient, enqueue_populate_crm_contacts};use crate::util::process_pre_insert::process_message_pre_insert;+use email_api_client::domain::models::CalendarPart;use models_email::email::service::backfill::{BackfillMessagePayload, JobScopedPayload};use models_email::email::service::link;+use models_email::email::service::message::Message;use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};-/// This step is invoked by BackfillThread once for each message in the thread.-/// Creates a message object in the database. If the message is the last message in-/// the thread to be processed, it sends an UpdateThreadMetadata message for the thread.+/// Backfills one message of a thread whose messages were enqueued one by one.+///+/// BackfillThread now writes a thread's messages itself; this step only drains+/// BackfillMessage operations enqueued before that. Creates a message object in+/// the database. If the message is the last message in the thread to be+/// processed, it sends an UpdateThreadMetadata message for the thread.#[tracing::instrument(skip(ctx))]pub async fn backfill_message(ctx: &PubSubContext,@@ -47,11 +52,27 @@ pub async fn backfill_message(.await;inserted.map_err(|e| map_db_error(e, "Failed to insert final message into database"))?;- if !message.is_draft && !fetched.calendar_parts.is_empty() {+ save_invitations(ctx, link, &message, &fetched.calendar_parts).await;+ enqueue_crm_contacts(ctx, link, &message).await?;++ // Handle all success-related operations+ increment_counters::incr_completed_messages(ctx, link, scope.job_id, p).await+}++/// Saves the calendar invitations a written backfill message carries. Best+/// effort: a failure is logged, as invitation extraction always has been.+pub(super) async fn save_invitations(+ ctx: &PubSubContext,+ link: &link::Link,+ message: &Message,+ calendar_parts: &[CalendarPart],+) {+ let provider_id = message.provider_id.as_deref().unwrap_or_default();+ if !message.is_draft && !calendar_parts.is_empty() {match email_db_client::messages::get::get_message_and_thread_id_by_provider_id(&ctx.db,link.id,- &p.message_provider_id,+ provider_id,).await{@@ -59,16 +80,24 @@ pub async fn backfill_message(crate::pubsub::invitation_extraction::save_discovered(ctx,link.id,- &p.message_provider_id,+ provider_id,message_id,- &fetched.calendar_parts,+ calendar_parts,).await;}Err(error) => tracing::warn!(error = ?error, "invitation extraction skipped"),}}+}+/// Enqueues the CRM contact updates a backfill message implies. Needs no+/// database row, so it can run before the message is written.+pub(super) async fn enqueue_crm_contacts(+ ctx: &PubSubContext,+ link: &link::Link,+ message: &Message,+) -> Result<(), ProcessingError> {// Fan out a PopulateCrmContact job per address involved in the// message — every non-draft message contributes, in both// directions. Sent: enumerate to/cc/bcc (recipients the team@@ -115,6 +144,5 @@ pub async fn backfill_message(}}- // Handle all success-related operations- increment_counters::incr_completed_messages(ctx, link, scope.job_id, p).await+ Ok(())}diff --git a/services/email_service/src/pubsub/backfill/backfill_thread.rs b/services/email_service/src/pubsub/backfill/backfill_thread.rsindex e051b36e09..a34186b815 100644--- a/services/email_service/src/pubsub/backfill/backfill_thread.rs+++ b/services/email_service/src/pubsub/backfill/backfill_thread.rs@@ -1,19 +1,26 @@-use crate::pubsub::backfill::db_error::map_db_error;+use crate::backfill_completion_service::{+ enqueue_thread_metadata_update, handle_thread_attachment_upload,+};+use crate::pubsub::backfill::backfill_message::{enqueue_crm_contacts, save_invitations};+use crate::pubsub::backfill::db_error::{is_unstorable, map_db_error};use crate::pubsub::backfill::email_api_error::map_email_api_error;use crate::pubsub::backfill::increment_counters::incr_completed_threads;+use crate::pubsub::backfill::write_slot;use crate::pubsub::context::PubSubContext;-use models_email::email::service::backfill::{- BackfillMessagePayload, BackfillOperation, BackfillPubsubMessage, BackfillThreadPayload,- JobScopedPayload,-};+use crate::util::process_pre_insert::process_message_pre_insert;+use email_api_client::domain::models::{CalendarPart, MessageWithCalendarParts};+use email_db_client::messages::insert::SkippedMessage;+use models_email::email::service::backfill::{BackfillThreadPayload, JobScopedPayload};use models_email::email::service::link;-use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};+use models_email::email::service::message::Message;+use models_email::email::service::pubsub::ProcessingError;use std::collections::HashSet;/// This step is invoked by ListThreads for each thread being backfilled.-/// Creates the thread object in the database, fetches the message ids for the thread-/// from the gmail api, and sends a BackfillMessage message for each message_id.-#[tracing::instrument(skip(ctx))]+/// Reads the whole thread from the provider at once and writes the thread with+/// all of its messages in one transaction, holding one of the fleet's backfill+/// write slots, then hands the thread to its attachment and metadata steps.+#[tracing::instrument(skip(ctx), fields(message_count))]pub async fn backfill_thread(ctx: &PubSubContext,scope: &JobScopedPayload<BackfillThreadPayload>,@@ -46,46 +53,50 @@ pub async fn backfill_thread(return Ok(());}- // fetch all message_ids of the thread- let message_ids = ctx- .email_api- .get_message_ids_for_thread(link.id, &thread_provider_id)- .await- .map_err(|error| map_email_api_error(error, "Failed to get provider thread message IDs"))?;+ let fetched = fetch_thread(ctx, link, &thread_provider_id).await?;+ let (mut messages, calendar_parts) = prepare_messages(fetched).await;// The priority pass and the normal sweep both enqueue a thread, and both// copies can pass the check above while they wait for Gmail. Only the copy- // that inserts the thread backfills its messages; counting them twice- // would complete the thread early and strand the duplicates.- let Some(thread_db_id) = email_db_client::threads::insert::insert_blank_thread(- &ctx.db,- &thread_provider_id,- link.id,- )- .await- .map_err(|e| {- map_db_error(- e,- format!("Failed to insert thread shell for {}", thread_provider_id),- )- })?- else {- incr_completed_threads(ctx, link, scope.job_id).await?;- return Ok(());+ // that inserts the thread finishes it; the other just counts it as done.+ let slot = write_slot::acquire(ctx).await?;+ let inserted = slot+ .hold(async {+ enqueue_threads_crm_contacts(ctx, link, &messages).await?;+ email_db_client::threads::insert::insert_backfilled_thread(+ &ctx.db,+ &thread_provider_id,+ link.id,+ &mut messages,+ is_unstorable,+ )+ .await+ .map_err(|e| map_db_error(e, format!("Failed to insert thread {}", thread_provider_id)))+ })+ .await?;+ let Some((thread_db_id, skipped)) = inserted else {+ return incr_completed_threads(ctx, link, scope.job_id).await;};- // create backfill pubsub message for each email message- enqueue_backfill_messages(ctx, scope, link, thread_db_id, message_ids).await+ finish_written_thread(+ ctx,+ scope,+ link,+ thread_db_id,+ &messages,+ &calendar_parts,+ &skipped,+ )+ .await}-/// Recovery-job path for a thread that is already in the database: fetch the-/// provider's message ids and backfill only the messages we are missing, so-/// replies that arrived during a sync gap are recovered.+/// Recovery-job path for a thread that is already in the database: read the+/// provider's thread and write only the messages we are missing, so replies+/// that arrived during a sync gap are recovered.////// Job accounting matches the fresh-thread path: with nothing missing the-/// thread completes immediately (like the skip path); otherwise its redis-/// progress counter is initialized to the missing count and the enqueued-/// messages complete it.+/// thread completes immediately (like the skip path); otherwise its metadata+/// step completes it.async fn refresh_existing_thread(ctx: &PubSubContext,scope: &JobScopedPayload<BackfillThreadPayload>,@@ -93,18 +104,17 @@ async fn refresh_existing_thread(thread_db_id: uuid::Uuid,) -> Result<(), ProcessingError> {let p = &scope.payload;+ let fetched = fetch_thread(ctx, link, &p.thread_provider_id).await?;- let message_ids = ctx- .email_api- .get_message_ids_for_thread(link.id, &p.thread_provider_id)- .await- .map_err(|error| map_email_api_error(error, "Failed to get provider thread message IDs"))?;-+ let provider_ids: Vec<String> = fetched+ .iter()+ .filter_map(|fetched| fetched.message.provider_id.clone())+ .collect();let existing_message_ids =email_db_client::messages::get::filter_existing_provider_message_ids(&ctx.db,link.id,- &message_ids,+ &provider_ids,).await.map_err(|e| {@@ -117,71 +127,139 @@ async fn refresh_existing_thread()})?;- let missing_message_ids: Vec<String> = message_ids+ let missing: Vec<MessageWithCalendarParts> = fetched.into_iter()- .filter(|message_id| !existing_message_ids.contains(message_id))+ .filter(|fetched| {+ fetched+ .message+ .provider_id+ .as_ref()+ .is_some_and(|id| !existing_message_ids.contains(id))+ }).collect();-- if missing_message_ids.is_empty() {+ if missing.is_empty() {return incr_completed_threads(ctx, link, scope.job_id).await;}- enqueue_backfill_messages(ctx, scope, link, thread_db_id, missing_message_ids).await+ let (mut messages, calendar_parts) = prepare_messages(missing).await;+ let slot = write_slot::acquire(ctx).await?;+ let skipped = slot+ .hold(async {+ enqueue_threads_crm_contacts(ctx, link, &messages).await?;+ email_db_client::messages::insert::insert_thread_messages(+ &ctx.db,+ thread_db_id,+ link.id,+ &mut messages,+ is_unstorable,+ )+ .await+ .map_err(|e| {+ map_db_error(+ e,+ format!(+ "Failed to insert missing messages of thread {}",+ p.thread_provider_id+ ),+ )+ })+ })+ .await?;++ finish_written_thread(+ ctx,+ scope,+ link,+ thread_db_id,+ &messages,+ &calendar_parts,+ &skipped,+ )+ .await}-/// Initializes the thread's redis progress counter and enqueues one-/// BackfillMessage per message id.-async fn enqueue_backfill_messages(+/// Reads every message of the provider thread, with its calendar parts.+async fn fetch_thread(ctx: &PubSubContext,- scope: &JobScopedPayload<BackfillThreadPayload>,link: &link::Link,- thread_db_id: uuid::Uuid,- message_ids: Vec<String>,-) -> Result<(), ProcessingError> {- let p = &scope.payload;-- ctx.redis_client- .init_backfill_thread_progress(- scope.job_id,- &p.thread_provider_id,- message_ids.len() as i32,- )+ thread_provider_id: &str,+) -> Result<Vec<MessageWithCalendarParts>, ProcessingError> {+ let fetched = ctx+ .email_api+ .get_thread(link.id, thread_provider_id).await- .map_err(|e| {- ProcessingError::Retryable(DetailedError {- reason: FailureReason::RedisQueryFailed,- source: e.context(format!(- "Failed to create entry in redis for thread {}",- &p.thread_provider_id- )),- })- })?;-- for message_id in message_ids {- let new_payload = BackfillMessagePayload {- thread_provider_id: p.thread_provider_id.clone(),- thread_db_id,- message_provider_id: message_id.clone(),- };-- let ps_message = BackfillPubsubMessage {- backfill_operation: BackfillOperation::BackfillMessage(JobScopedPayload {- link_id: link.id,- job_id: scope.job_id,- payload: new_payload,- }),- };+ .map_err(|error| map_email_api_error(error, "Failed to get provider thread"))?;+ tracing::Span::current().record("message_count", fetched.len());+ Ok(fetched)+}- ctx.sqs_client- .enqueue_email_backfill_message(ps_message)- .await- .map_err(|e| {- ProcessingError::NonRetryable(DetailedError {- reason: FailureReason::SqsEnqueueFailed,- source: e.context(format!("Failed to enqueue message {}", message_id)),- })- })?;+/// Cleans each message for insertion, keeping its calendar parts at the same index.+async fn prepare_messages(+ fetched: Vec<MessageWithCalendarParts>,+) -> (Vec<Message>, Vec<Vec<CalendarPart>>) {+ let mut messages = Vec::with_capacity(fetched.len());+ let mut calendar_parts = Vec::with_capacity(fetched.len());+ for MessageWithCalendarParts {+ mut message,+ calendar_parts: parts,+ } in fetched+ {+ process_message_pre_insert(&mut message).await;+ messages.push(message);+ calendar_parts.push(parts);}+ (messages, calendar_parts)+}+/// Enqueues the CRM contact updates of every message in the thread. This runs+/// just before the thread is written, while a failure can still be retried:+/// after the commit a retry would skip the thread. It runs once a write slot is+/// held, so an operation deferred for a busy slot doesn't repeat it each time.+/// The rare duplicate from a retried write is harmless.+async fn enqueue_threads_crm_contacts(+ ctx: &PubSubContext,+ link: &link::Link,+ messages: &[Message],+) -> Result<(), ProcessingError> {+ for message in messages {+ enqueue_crm_contacts(ctx, link, message).await?;+ }Ok(())}++/// Finishes a thread whose messages are committed.+///+/// A retried BackfillThread finds the thread written and skips it, so nothing+/// here may ask for a retry. Messages the write skipped are logged. The+/// invitation and attachment fan-outs only log their failures; the metadata+/// step goes last and is the only one that can fail the operation, as a+/// non-retryable error that counts the thread once.+async fn finish_written_thread(+ ctx: &PubSubContext,+ scope: &JobScopedPayload<BackfillThreadPayload>,+ link: &link::Link,+ thread_db_id: uuid::Uuid,+ messages: &[Message],+ calendar_parts: &[Vec<CalendarPart>],+ skipped: &[SkippedMessage],+) -> Result<(), ProcessingError> {+ let thread_provider_id = &scope.payload.thread_provider_id;+ for SkippedMessage { index, error } in skipped {+ tracing::warn!(+ error = ?error,+ %thread_provider_id,+ message_id = messages[*index].provider_id.as_deref().unwrap_or_default(),+ "Skipped a backfilled message that cannot be stored"+ );+ }+ for (index, (message, parts)) in messages.iter().zip(calendar_parts).enumerate() {+ if !skipped.iter().any(|skipped| skipped.index == index) {+ save_invitations(ctx, link, message, parts).await;+ }+ }+ if let Err(error) = handle_thread_attachment_upload(ctx, link, scope.job_id, thread_db_id).await+ {+ tracing::warn!(error = ?error, %thread_provider_id, "Skipped a backfilled thread's attachment uploads");+ }+ enqueue_thread_metadata_update(ctx, link, scope.job_id, thread_provider_id, thread_db_id).await+}diff --git a/services/email_service/src/pubsub/backfill/db_error.rs b/services/email_service/src/pubsub/backfill/db_error.rsindex bf145953a7..14e7b582bb 100644--- a/services/email_service/src/pubsub/backfill/db_error.rs+++ b/services/email_service/src/pubsub/backfill/db_error.rs@@ -43,6 +43,12 @@ where}}+/// Whether a write failed on its own data, failing identically on every+/// retry. Backfill skips such a message instead of failing its whole thread.+pub(crate) fn is_unstorable(error: &anyhow::Error) -> bool {+ deterministic_sqlstate(error).is_some()+}+/// Returns the SQLSTATE of the first database error in the chain that no/// amount of retrying will get past.fn deterministic_sqlstate(error: &anyhow::Error) -> Option<String> {diff --git a/services/email_service/src/pubsub/backfill/process.rs b/services/email_service/src/pubsub/backfill/process.rsindex 36c0684573..f45508f91a 100644--- a/services/email_service/src/pubsub/backfill/process.rs+++ b/services/email_service/src/pubsub/backfill/process.rs@@ -11,6 +11,7 @@ use models_email::email::service::backfill::{use models_email::email::service::link;use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};use sqs_worker::cleanup_message;+use std::time::Duration;use uuid::Uuid;// Process a single message from the backfill queue@@ -30,7 +31,8 @@ pub async fn process_message(}};- let processing_result = inner_process_message(&ctx, &data).await;+ let processing_result =+ keep_invisible(message, &ctx.sqs_worker, inner_process_message(&ctx, &data)).await;match processing_result {// Processing success. Clean up the message@@ -55,6 +57,47 @@ pub async fn process_message(}}+/// How often a running operation pushes its message's visibility out again.+const VISIBILITY_HEARTBEAT: Duration = Duration::from_secs(20);+/// The backfill queue's visibility timeout, restored by every heartbeat.+const VISIBILITY_TIMEOUT_SECONDS: i32 = 60;++/// Runs `work` while keeping its message hidden from other workers.+///+/// A whole thread, or any operation slowed by a struggling database, can+/// outlast the queue's visibility timeout. Delivered again while the first+/// copy still runs, the second copy would find the thread written and count+/// it, and the first would count it again through its metadata step,+/// completing the job early. A failed heartbeat only logs: the message then+/// reappears as it would have without one.+async fn keep_invisible<F: Future>(+ message: &aws_sdk_sqs::types::Message,+ sqs_worker: &sqs_worker::SQSWorker,+ work: F,+) -> F::Output {+ let Some(receipt_handle) = message.receipt_handle() else {+ return work.await;+ };+ let mut work = std::pin::pin!(work);+ let mut heartbeats = tokio::time::interval_at(+ tokio::time::Instant::now() + VISIBILITY_HEARTBEAT,+ VISIBILITY_HEARTBEAT,+ );+ loop {+ tokio::select! {+ output = &mut work => return output,+ _ = heartbeats.tick() => {+ if let Err(error) = sqs_worker+ .change_message_visibility(receipt_handle, VISIBILITY_TIMEOUT_SECONDS)+ .await+ {+ tracing::warn!(error = ?error, "failed to keep a running backfill operation hidden");+ }+ }+ }+ }+}+#[tracing::instrument(skip(ctx))]async fn inner_process_message(ctx: &PubSubContext,diff --git a/services/email_service/src/pubsub/backfill/write_slot.rs b/services/email_service/src/pubsub/backfill/write_slot.rsindex eb7d50491d..81437bdbd7 100644--- a/services/email_service/src/pubsub/backfill/write_slot.rs+++ b/services/email_service/src/pubsub/backfill/write_slot.rs@@ -5,7 +5,7 @@ use crate::pubsub::context::PubSubContext;use crate::util::redis::backfill_write_slots::BackfillWriteSlot;use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};-/// Takes a backfill write slot for one message write. A slot that stays taken,+/// Takes a backfill write slot for one write. A slot that stays taken,/// or Redis failing to decide, refuses the write so the operation is deferred/// instead of written unmetered.pub(super) async fn acquire(ctx: &PubSubContext) -> Result<BackfillWriteSlot, ProcessingError> {diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rsindex 506cbe8585..a403d2584a 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs@@ -602,7 +602,7 @@ async fn fetch_and_insert_thread(provider_thread_id: &str,) -> result::Result<(), ProcessingError> {let messages = match ctx.email_api.get_thread(link_id, provider_thread_id).await {- Ok(messages) => messages,+ Ok(fetched) => fetched.into_iter().map(|fetched| fetched.message).collect(),Err(error) => {return Err(handle_operation_error(ctx,diff --git a/services/email_service/src/util/redis/backfill.rs b/services/email_service/src/util/redis/backfill.rsindex 608c2d5685..71c113d627 100644--- a/services/email_service/src/util/redis/backfill.rs+++ b/services/email_service/src/util/redis/backfill.rs@@ -133,34 +133,6 @@ impl RedisClient {Ok(())}- /// Initialize backfill thread progress in Redis- pub async fn init_backfill_thread_progress(- &self,- job_id: Uuid,- thread_id: &str,- total_messages: i32,- ) -> anyhow::Result<()> {- let key = Self::thread_status_key(job_id, thread_id);-- let mut redis_connection = self- .inner- .get_multiplexed_async_connection()- .await- .context("unable to connect to redis")?;-- redis_connection- .hset_multiple::<&str, &str, i32, ()>(- &key,- &[- ("total_messages", total_messages),- ("completed_messages", 0),- ],- )- .await?;-- Ok(())- }-/// Increment completed messages count and return true if thread is complete, deleting redis entry for thread if sopub async fn incr_completed_messages(&self,diff --git a/services/email_service/src/util/redis/backfill_write_slots.rs b/services/email_service/src/util/redis/backfill_write_slots.rsindex 289cf42a15..a46f9e8879 100644--- a/services/email_service/src/util/redis/backfill_write_slots.rs+++ b/services/email_service/src/util/redis/backfill_write_slots.rs@@ -1,12 +1,11 @@-//! A fleet-wide cap on concurrent backfill message writes.+//! A fleet-wide cap on concurrent backfill writes.//!-//! Every backfill worker of every email-service task writes messages straight-//! to the primary, one transaction per message, and nothing else bounds how-//! many run at once (25 workers per task, 5 to 10 tasks). Message bodies land-//! in one TOAST table, so a large backfill queues its writers on that table's-//! relation-extension lock and slows every other query on the database. A slot-//! is a Redis lease; at most [`CAPACITY`] backfill writes hold one at a time,-//! however many tasks are running.+//! Every backfill worker of every email-service task writes threads straight+//! to the primary, and nothing else bounds how many run at once (25 workers+//! per task, 5 to 10 tasks). Message bodies land in one TOAST table, so a large+//! backfill queues its writers on that table's relation-extension lock and+//! slows every other query on the database. A slot is a Redis lease; at most+//! [`CAPACITY`] backfill writes hold one at a time, however many tasks run.use crate::util::redis::RedisClient;use rand::Rng;@@ -17,11 +16,12 @@ use uuid::Uuid;#[cfg(test)]mod test;-/// Concurrent backfill message writes across the fleet. A healthy write (the-/// contacts upsert plus the message transaction) averages about 100 ms, so-/// twenty slots admit roughly 200 messages a second while the database is-/// healthy, above normal busy hours, and fewer as it slows down. Unbounded,-/// 125 to 250 backfill workers could write at once.+/// Concurrent backfill writes across the fleet. A healthy single-message write+/// (the contacts upsert plus its transaction) averages about 100 ms, so twenty+/// slots admit at least roughly 200 messages a second while the database is+/// healthy, above normal busy hours, more when a thread's messages share one+/// transaction, and fewer as the database slows down. Unbounded, 125 to 250+/// backfill workers could write at once.const CAPACITY: u32 = 20;/// Frees the slot of a crashed worker. A held slot renews it every/// [`RENEW_EVERY`], so a write that outlasts it keeps its slot.