Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

perf(email): backfill each thread in one Gmail read and one transaction (#8913)

teoa949192ceb6b
diff --git a/crates/email_api_client/src/domain/ports.rs b/crates/email_api_client/src/domain/ports.rs
index 418503b569..dc1daacf8d 100644
--- a/crates/email_api_client/src/domain/ports.rs
+++ b/crates/email_api_client/src/domain/ports.rs
@@ -6,7 +6,6 @@ pub use mailbox::*;
use std::future::Future;
use models_email::email::service::label::Label;
-use models_email::email::service::message::Message;
use models_email::service::contact::{Contact, ContactList};
use models_email::service::link::Link;
use uuid::Uuid;
@@ -84,13 +83,14 @@ pub trait MailboxMessageClient: Send + Sync + 'static {
provider_thread_id: &str,
) -> impl Future<Output = Result<Vec<String>, EmailApiError>> + Send;
- /// Fetches and normalizes every message in a provider thread.
+ /// Fetches and normalizes every message in a provider thread, each with
+ /// its calendar invitation parts, in one provider read.
fn get_thread(
&self,
access_token: &AccessToken,
link_id: Uuid,
provider_thread_id: &str,
- ) -> impl Future<Output = Result<Vec<Message>, EmailApiError>> + Send;
+ ) -> impl Future<Output = Result<Vec<MessageWithCalendarParts>, EmailApiError>> + Send;
/// Lists one page of provider threads.
fn list_threads(
@@ -429,7 +429,7 @@ impl MailboxMessageClient for NoOpMailboxClient {
_: &AccessToken,
_: Uuid,
_: &str,
- ) -> Result<Vec<Message>, EmailApiError> {
+ ) -> Result<Vec<MessageWithCalendarParts>, EmailApiError> {
Err(unavailable())
}
diff --git a/crates/email_api_client/src/domain/service/messages.rs b/crates/email_api_client/src/domain/service/messages.rs
index 59c3557b74..f75ee04b6e 100644
--- a/crates/email_api_client/src/domain/service/messages.rs
+++ b/crates/email_api_client/src/domain/service/messages.rs
@@ -1,4 +1,3 @@
-use models_email::email::service::message::Message;
use uuid::Uuid;
use super::super::models::{CalendarPart, EmailApiError, MessageWithCalendarParts, ThreadListPage};
@@ -79,13 +78,14 @@ where
.await
}
- /// Fetches and normalizes every message in one provider thread.
+ /// Fetches and normalizes every message in one provider thread, with any
+ /// calendar invitation parts, in a single provider read.
#[tracing::instrument(skip(self), fields(retry_count = 0, gate_in_flight = tracing::field::Empty, gate_cap = tracing::field::Empty))]
pub async fn get_thread(
&self,
link_id: Uuid,
provider_thread_id: &str,
- ) -> Result<Vec<Message>, EmailApiError> {
+ ) -> Result<Vec<MessageWithCalendarParts>, EmailApiError> {
self.read_with_token(link_id, ApiOperationKind::GetThread, |token| async move {
self.repository
.get_thread(&token, link_id, provider_thread_id)
diff --git a/crates/email_api_client/src/domain/service/messages/test.rs b/crates/email_api_client/src/domain/service/messages/test.rs
index cd6b50654e..62feae72ca 100644
--- a/crates/email_api_client/src/domain/service/messages/test.rs
+++ b/crates/email_api_client/src/domain/service/messages/test.rs
@@ -1,6 +1,5 @@
use std::sync::{Arc, Mutex};
-use models_email::email::service::message::Message;
use uuid::Uuid;
use super::super::test_support::{Call, FakeGate, FakeTokenSource, call_log};
@@ -96,7 +95,7 @@ impl MailboxMessageClient for MessageClient {
access_token: &AccessToken,
link_id: Uuid,
thread_id: &str,
- ) -> Result<Vec<Message>, EmailApiError> {
+ ) -> Result<Vec<crate::domain::models::MessageWithCalendarParts>, EmailApiError> {
Err(self.record(
access_token,
MessageCall::Thread(link_id, thread_id.to_string()),
diff --git a/crates/email_api_client/src/domain/service/test_support.rs b/crates/email_api_client/src/domain/service/test_support.rs
index ddcfa39c77..34d6be531c 100644
--- a/crates/email_api_client/src/domain/service/test_support.rs
+++ b/crates/email_api_client/src/domain/service/test_support.rs
@@ -1,7 +1,6 @@
use std::sync::{Arc, Mutex};
use models_email::email::service::label::Label;
-use models_email::email::service::message::Message;
use models_email::service::contact::{Contact, ContactList};
use uuid::Uuid;
@@ -217,7 +216,7 @@ impl MailboxMessageClient for FakeRepository {
_: &AccessToken,
_: Uuid,
_: &str,
- ) -> Result<Vec<Message>, EmailApiError> {
+ ) -> Result<Vec<crate::domain::models::MessageWithCalendarParts>, EmailApiError> {
self.unavailable("get_thread")
}
diff --git a/crates/email_api_client/src/outbound/gmail/messages.rs b/crates/email_api_client/src/outbound/gmail/messages.rs
index 0124cbdf45..4a2749b789 100644
--- a/crates/email_api_client/src/outbound/gmail/messages.rs
+++ b/crates/email_api_client/src/outbound/gmail/messages.rs
@@ -1,10 +1,11 @@
//! Gmail message capability implementation.
+use std::collections::HashMap;
+
use base64::{
Engine as _,
engine::general_purpose::{URL_SAFE, URL_SAFE_NO_PAD},
};
-use models_email::email::service::message::Message;
use models_email::email::service::thread::ThreadSummary;
use models_email::gmail::MessagePart;
use uuid::Uuid;
@@ -135,13 +136,36 @@ impl MailboxMessageClient for GmailApiClientRepository {
access_token: &AccessToken,
link_id: Uuid,
provider_thread_id: &str,
- ) -> Result<Vec<Message>, EmailApiError> {
+ ) -> Result<Vec<MessageWithCalendarParts>, EmailApiError> {
let resource = self
.client
.get_thread(access_token.expose_secret(), provider_thread_id)
.await
.map_err(map_gmail_error)?;
- Ok(map_thread_resource_to_service(resource, link_id)?.messages)
+ // Collect calendar parts from the wire messages before conversion
+ // consumes them, so a thread read needs no messages.get per message.
+ let mut calendar_parts = resource
+ .messages
+ .iter()
+ .map(|message| {
+ Ok((
+ message.id.clone(),
+ collect_calendar_parts(&message.payload)?,
+ ))
+ })
+ .collect::<Result<HashMap<_, _>, EmailApiError>>()?;
+ Ok(map_thread_resource_to_service(resource, link_id)?
+ .messages
+ .into_iter()
+ .map(|message| MessageWithCalendarParts {
+ calendar_parts: message
+ .provider_id
+ .as_ref()
+ .and_then(|id| calendar_parts.remove(id))
+ .unwrap_or_default(),
+ message,
+ })
+ .collect())
}
async fn list_threads(
diff --git a/crates/email_api_client/src/outbound/gmail/test/messages.rs b/crates/email_api_client/src/outbound/gmail/test/messages.rs
index 81cd2454b0..afa592b1ce 100644
--- a/crates/email_api_client/src/outbound/gmail/test/messages.rs
+++ b/crates/email_api_client/src/outbound/gmail/test/messages.rs
@@ -145,7 +145,44 @@ async fn normalizes_full_threads() {
.await
.unwrap();
assert_eq!(messages.len(), 1);
- assert_eq!(messages[0].provider_thread_id.as_deref(), Some("thread-1"));
+ assert_eq!(
+ messages[0].message.provider_thread_id.as_deref(),
+ Some("thread-1")
+ );
+}
+
+#[tokio::test]
+async fn get_thread_keeps_each_messages_calendar_parts_from_one_wire_fetch() {
+ let (server, repository) = repository().await;
+ let thread: serde_json::Value =
+ serde_json::from_str(include_str!("fixtures/thread_full.json")).unwrap();
+ let mut messages = thread["messages"].as_array().unwrap().clone();
+ // The invitation is the later message, so normalization reorders nothing
+ // and each message must keep the parts found on its own wire payload.
+ let mut invitation = calendar_message_body();
+ invitation["internalDate"] = serde_json::json!("1700000001000");
+ messages.push(invitation);
+ Mock::given(method("GET"))
+ .and(path("/users/me/threads/thread-1"))
+ .and(query_param("format", "full"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .set_body_json(serde_json::json!({ "id": "thread-1", "messages": messages })),
+ )
+ .expect(1)
+ .mount(&server)
+ .await;
+
+ let fetched = repository
+ .get_thread(&AccessToken::new("token"), Uuid::now_v7(), "thread-1")
+ .await
+ .unwrap();
+
+ assert_eq!(fetched.len(), 2);
+ assert_eq!(fetched[0].message.provider_id.as_deref(), Some("message-1"));
+ assert!(fetched[0].calendar_parts.is_empty());
+ assert_eq!(fetched[1].message.provider_id.as_deref(), Some("calendar"));
+ assert_calendar_parts(&fetched[1].calendar_parts);
}
#[tokio::test]
diff --git a/crates/email_db_client/src/messages/insert.rs b/crates/email_db_client/src/messages/insert.rs
index 3d35573b22..af1c67cf39 100644
--- a/crates/email_db_client/src/messages/insert.rs
+++ b/crates/email_db_client/src/messages/insert.rs
@@ -194,6 +194,118 @@ async fn insert_db_message(
Ok(existing_id)
}
+/// A message a backfill write left out because its own data can never be
+/// stored; the rest of its thread was still written.
+#[derive(Debug)]
+pub struct SkippedMessage {
+ /// The message's position in the slice passed to the write.
+ pub index: usize,
+ /// Why it could not be stored.
+ pub error: anyhow::Error,
+}
+
+/// Writes backfilled messages one savepoint at a time.
+///
+/// A message whose contacts or rows fail with an error `is_unstorable`
+/// accepts is skipped and reported, and the others still commit: one bad
+/// message must not cost its whole thread. Any other error fails the write.
+pub(crate) struct BackfillMessageWriter<'a, U> {
+ pub(crate) pool: &'a PgPool,
+ pub(crate) link_id: Uuid,
+ pub(crate) is_unstorable: U,
+}
+
+impl<U: Fn(&anyhow::Error) -> bool> BackfillMessageWriter<'_, U> {
+ /// Upserts each message's contacts, in order. Contacts are shared across
+ /// messages, so this runs before the messages' transaction to avoid
+ /// deadlocks. A skipped message gets `None`.
+ pub(crate) async fn upsert_contacts(
+ &self,
+ messages: &[message::Message],
+ skipped: &mut Vec<SkippedMessage>,
+ ) -> anyhow::Result<Vec<Option<UpsertedRecipients>>> {
+ let mut recipients = Vec::with_capacity(messages.len());
+ for (index, message) in messages.iter().enumerate() {
+ match contacts::upsert_message::parse_and_upsert_message_contacts(
+ self.pool,
+ self.link_id,
+ addresses_from_message(message),
+ )
+ .await
+ {
+ Ok(upserted) => recipients.push(Some(upserted)),
+ Err(error) if (self.is_unstorable)(&error) => {
+ skipped.push(SkippedMessage { index, error });
+ recipients.push(None);
+ }
+ Err(error) => return Err(error),
+ }
+ }
+ Ok(recipients)
+ }
+
+ /// Inserts each message whose contacts were stored, under its own savepoint.
+ pub(crate) async fn insert_messages(
+ &self,
+ tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+ thread_db_id: Uuid,
+ messages: &mut [message::Message],
+ recipients: Vec<Option<UpsertedRecipients>>,
+ skipped: &mut Vec<SkippedMessage>,
+ ) -> anyhow::Result<()> {
+ for (index, (message, recipients)) in messages.iter_mut().zip(recipients).enumerate() {
+ let Some(recipients) = recipients else {
+ continue;
+ };
+ let mut savepoint = sqlx::Acquire::begin(&mut *tx).await?;
+ match insert_message_with_tx(
+ &mut savepoint,
+ thread_db_id,
+ message,
+ self.link_id,
+ recipients,
+ false,
+ )
+ .await
+ {
+ Ok(()) => savepoint.commit().await?,
+ Err(error) if (self.is_unstorable)(&error) => {
+ savepoint.rollback().await?;
+ skipped.push(SkippedMessage { index, error });
+ }
+ Err(error) => return Err(error),
+ }
+ }
+ Ok(())
+ }
+}
+
+/// Inserts messages into an existing thread in one transaction, leaving its
+/// metadata and `replying_to` links to the caller's later metadata update.
+/// Returns the messages it skipped because `is_unstorable` accepted their error.
+#[tracing::instrument(skip(pool, messages, is_unstorable), fields(message_count = messages.len()), err)]
+pub async fn insert_thread_messages(
+ pool: &PgPool,
+ thread_db_id: Uuid,
+ link_id: Uuid,
+ messages: &mut [message::Message],
+ is_unstorable: impl Fn(&anyhow::Error) -> bool,
+) -> anyhow::Result<Vec<SkippedMessage>> {
+ let writer = BackfillMessageWriter {
+ pool,
+ link_id,
+ is_unstorable,
+ };
+ let mut skipped = Vec::new();
+ let recipients = writer.upsert_contacts(messages, &mut skipped).await?;
+ let mut tx = pool.begin().await?;
+ writer
+ .insert_messages(&mut tx, thread_db_id, messages, recipients, &mut skipped)
+ .await?;
+ tx.commit().await?;
+ Ok(skipped)
+}
+
/// Inserts a single message into the database with transaction handling
#[tracing::instrument(skip(pool, message), fields(link_id = %message.link_id), err)]
pub async fn insert_message(
diff --git a/crates/email_db_client/src/threads/insert.rs b/crates/email_db_client/src/threads/insert.rs
index 4030d64236..632d8ebbe2 100644
--- a/crates/email_db_client/src/threads/insert.rs
+++ b/crates/email_db_client/src/threads/insert.rs
@@ -6,7 +6,7 @@ use crate::parse::service_to_db::addresses_from_message;
use crate::{contacts, messages, parse};
use models_email::email::db::address::UpsertedRecipients;
-use models_email::email::service::thread;
+use models_email::email::service::{message, thread};
use sqlx::types::Uuid;
use sqlx::{PgConnection, PgPool};
use std::collections::HashMap;
@@ -156,15 +156,9 @@ pub async fn insert_thread(
Ok(existing_id)
}
-/// Inserts a thread with no metadata, returning its id, or `None` when the
-/// mailbox already has the thread, including when a concurrent insert won.
-#[tracing::instrument(skip(pool), err)]
-pub async fn insert_blank_thread(
- pool: &PgPool,
- thread_provider_id: &str,
- link_id: Uuid,
-) -> anyhow::Result<Option<Uuid>> {
- let thread = thread::Thread {
+/// A thread row with no metadata yet, for writers that fill it in afterwards.
+fn blank_thread(thread_provider_id: &str, link_id: Uuid) -> thread::Thread {
+ thread::Thread {
db_id: macro_uuid::generate_uuid_v7(),
provider_id: Some(thread_provider_id.to_string()),
link_id,
@@ -176,10 +170,61 @@ pub async fn insert_blank_thread(
created_at: Default::default(),
updated_at: Default::default(),
messages: vec![],
- };
+ }
+}
+
+/// Inserts a thread with no metadata, returning its id, or `None` when the
+/// mailbox already has the thread, including when a concurrent insert won.
+#[tracing::instrument(skip(pool), err)]
+pub async fn insert_blank_thread(
+ pool: &PgPool,
+ thread_provider_id: &str,
+ link_id: Uuid,
+) -> anyhow::Result<Option<Uuid>> {
+ let thread = blank_thread(thread_provider_id, link_id);
let mut tx = pool.begin().await?;
let id = insert_thread(&mut tx, &thread, link_id).await?;
tx.commit().await?;
Ok((id == thread.db_id).then_some(id))
}
+
+/// Inserts a backfilled thread and all of its messages in one transaction,
+/// returning the thread id and the messages it skipped, or `None` when the
+/// mailbox already has the thread, including when a concurrent insert won;
+/// nothing is written then.
+///
+/// A message `is_unstorable` rejects is left out under its own savepoint while
+/// the rest commit. Any other failure leaves no thread behind, so a retried
+/// backfill writes the whole thread again. Thread metadata and `replying_to`
+/// links are left to the backfill's metadata step, as with
+/// [`insert_blank_thread`].
+#[tracing::instrument(skip(pool, messages, is_unstorable), fields(message_count = messages.len()), err)]
+pub async fn insert_backfilled_thread(
+ pool: &PgPool,
+ thread_provider_id: &str,
+ link_id: Uuid,
+ messages: &mut [message::Message],
+ is_unstorable: impl Fn(&anyhow::Error) -> bool,
+) -> anyhow::Result<Option<(Uuid, Vec<messages::insert::SkippedMessage>)>> {
+ let writer = messages::insert::BackfillMessageWriter {
+ pool,
+ link_id,
+ is_unstorable,
+ };
+ let mut skipped = Vec::new();
+ let recipients = writer.upsert_contacts(messages, &mut skipped).await?;
+ let thread = blank_thread(thread_provider_id, link_id);
+
+ let mut tx = pool.begin().await?;
+ let id = insert_thread(&mut tx, &thread, link_id).await?;
+ if id != thread.db_id {
+ tx.rollback().await?;
+ return Ok(None);
+ }
+ writer
+ .insert_messages(&mut tx, id, messages, recipients, &mut skipped)
+ .await?;
+ tx.commit().await?;
+ Ok(Some((id, skipped)))
+}
diff --git a/crates/email_db_client/src/threads/insert/test.rs b/crates/email_db_client/src/threads/insert/test.rs
index 42c6582ad5..ea7fc45e44 100644
--- a/crates/email_db_client/src/threads/insert/test.rs
+++ b/crates/email_db_client/src/threads/insert/test.rs
@@ -348,3 +348,169 @@ async fn oversized_provider_values_do_not_block_the_thread_insert(
Ok(())
}
+
+fn backfilled_messages(link_id: Uuid) -> Vec<message::Message> {
+ use crate::messages::insert::test::{PARENT_GLOBAL_ID, SyncedMessage};
+
+ let parent = SyncedMessage {
+ provider_id: "backfill-parent",
+ global_id: PARENT_GLOBAL_ID,
+ in_reply_to: None,
+ to: &["[email protected]"],
+ cc: &[],
+ labels: &[],
+ body: "parent body",
+ };
+ let reply = SyncedMessage {
+ provider_id: "backfill-reply",
+ global_id: "<[email protected]>",
+ in_reply_to: Some(PARENT_GLOBAL_ID),
+ to: &["[email protected]"],
+ cc: &["[email protected]"],
+ labels: &[],
+ body: "reply body",
+ };
+ // Backfill writes the thread row itself, so the messages' own thread id is unused.
+ let unused_thread_id = Uuid::now_v7();
+ vec![
+ parent.build(link_id, unused_thread_id),
+ reply.build(link_id, unused_thread_id),
+ ]
+}
+
+/// Postgres "data exception" (SQLSTATE class 22): the value itself can't be stored.
+fn data_exception(error: &anyhow::Error) -> bool {
+ error.chain().any(|cause| {
+ cause
+ .downcast_ref::<sqlx::Error>()
+ .and_then(|error| error.as_database_error())
+ .and_then(|error| error.code())
+ .is_some_and(|code| code.starts_with("22"))
+ })
+}
+
+async fn thread_message_count(pool: &Pool<Postgres>, thread_id: Uuid) -> anyhow::Result<i64> {
+ Ok(sqlx::query_scalar!(
+ r#"SELECT COUNT(*) AS "count!" FROM email_messages WHERE thread_id = $1"#,
+ thread_id
+ )
+ .fetch_one(pool)
+ .await?)
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("links"))
+)]
+async fn a_backfilled_thread_is_written_with_all_of_its_messages(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;
+ let mut messages = backfilled_messages(link_id);
+
+ let (thread_id, skipped) = insert_backfilled_thread(
+ &pool,
+ "backfill-thread",
+ link_id,
+ &mut messages,
+ data_exception,
+ )
+ .await?
+ .expect("the thread is new");
+
+ assert!(skipped.is_empty());
+
+ assert_eq!(thread_message_count(&pool, thread_id).await?, 2);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("links"))
+)]
+async fn a_thread_another_copy_already_wrote_gets_no_messages_from_this_one(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;
+ let existing_id = insert_blank_thread(&pool, "backfill-thread", link_id)
+ .await?
+ .expect("the thread is new");
+ let mut messages = backfilled_messages(link_id);
+
+ let inserted = insert_backfilled_thread(
+ &pool,
+ "backfill-thread",
+ link_id,
+ &mut messages,
+ data_exception,
+ )
+ .await?;
+
+ assert!(
+ inserted.is_none(),
+ "only the copy that wrote the thread finishes it"
+ );
+ assert_eq!(thread_message_count(&pool, existing_id).await?, 0);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("links"))
+)]
+async fn missing_messages_are_written_into_their_existing_thread(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;
+ let thread_id = insert_blank_thread(&pool, "backfill-thread", link_id)
+ .await?
+ .expect("the thread is new");
+ let mut messages = backfilled_messages(link_id);
+
+ let skipped = crate::messages::insert::insert_thread_messages(
+ &pool,
+ thread_id,
+ link_id,
+ &mut messages,
+ data_exception,
+ )
+ .await?;
+
+ assert!(skipped.is_empty());
+
+ assert_eq!(thread_message_count(&pool, thread_id).await?, 2);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("links"))
+)]
+async fn a_message_that_cannot_be_stored_is_skipped_and_the_rest_of_its_thread_is_written(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;
+ let mut messages = backfilled_messages(link_id);
+ // Postgres text can't hold NUL, so this message fails identically every time.
+ messages[0].body_text = Some("broken \0 body".to_string());
+
+ let (thread_id, skipped) = insert_backfilled_thread(
+ &pool,
+ "backfill-thread",
+ link_id,
+ &mut messages,
+ data_exception,
+ )
+ .await?
+ .expect("the thread is new");
+
+ assert_eq!(
+ skipped
+ .iter()
+ .map(|skipped| skipped.index)
+ .collect::<Vec<_>>(),
+ vec![0]
+ );
+ assert_eq!(thread_message_count(&pool, thread_id).await?, 1);
+ Ok(())
+}
diff --git a/crates/sqs_worker/src/lib.rs b/crates/sqs_worker/src/lib.rs
index ebc45cdc7f..c9c3527b7e 100644
--- a/crates/sqs_worker/src/lib.rs
+++ b/crates/sqs_worker/src/lib.rs
@@ -57,6 +57,25 @@ impl SQSWorker {
pub async fn delete_message(&self, receipt_handle: &str) -> anyhow::Result<()> {
delete_message::delete_message(&self.inner, &self.queue_url, receipt_handle).await
}
+
+ /// Hides a received message from other consumers for `timeout_seconds`
+ /// from now, so work that outlasts the queue's visibility timeout is not
+ /// delivered again while it still runs.
+ #[tracing::instrument(skip(self, receipt_handle), err)]
+ pub async fn change_message_visibility(
+ &self,
+ receipt_handle: &str,
+ timeout_seconds: i32,
+ ) -> anyhow::Result<()> {
+ self.inner
+ .change_message_visibility()
+ .queue_url(&self.queue_url)
+ .receipt_handle(receipt_handle)
+ .visibility_timeout(timeout_seconds)
+ .send()
+ .await?;
+ Ok(())
+ }
}
#[tracing::instrument(skip(sqs_worker, message), fields(message_id=message.message_id, message_receipt_handle=message.receipt_handle))]
diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.md
index bcbaecbe60..04cd301407 100644
--- a/services/email_service/AGENTS.md
+++ b/services/email_service/AGENTS.md
@@ -81,7 +81,23 @@ the gateway `/email` prefix.
would spend the queue's 20 deliveries.
Inbox sync releases the upload claims of a refused attachment fetch and
requeues the upsert, because a claimed attachment is never offered again.
-- Backfill message writes hold one of twenty fleet-wide Redis write slots
+- BackfillThread reads a whole thread with one Gmail `threads.get` and writes
+ the thread row and all of its messages in one transaction
+ (`insert_backfilled_thread`), so a failed write leaves nothing for a retry
+ to skip. Each message has its own savepoint: one whose data can never be
+ stored (`db_error::is_unstorable`) is skipped and logged, and the rest of the
+ thread commits. The CRM fan-out is enqueued after the write slot is taken
+ and before the write: after the commit nothing in the handler may request a
+ retry, because a retry would find the thread and skip it, and an operation
+ deferred for a busy slot must not repeat the fan-out. Invitation and attachment fan-outs only log their
+ failures, and the metadata step is enqueued last. BackfillMessage only drains
+ operations enqueued before thread writes.
+- Backfill keeps a message hidden while its operation runs
+ (`process::keep_invisible`, a 20 s heartbeat restoring the queue's 60 s
+ visibility timeout). Without it, a slow thread is delivered to a second
+ worker, which counts the written thread again.
+- Each backfill thread write, and each BackfillMessage write still draining,
+ holds one of twenty fleet-wide Redis write slots
(`util/redis/backfill_write_slots.rs`), however many workers and tasks run,
so a large backfill can't queue enough writers on the primary to slow every
other query. A write waits up to 10 s for a slot and is then deferred like a
diff --git a/services/email_service/src/backfill_completion_service.rs b/services/email_service/src/backfill_completion_service.rs
index 97c9f07245..6d1d9da13e 100644
--- a/services/email_service/src/backfill_completion_service.rs
+++ b/services/email_service/src/backfill_completion_service.rs
@@ -433,16 +433,29 @@ async fn handle_thread_completed(
job_id: Uuid,
p: &BackfillMessagePayload,
) -> Result<(), ProcessingError> {
- let new_payload = UpdateMetadataPayload {
- thread_provider_id: p.thread_provider_id.clone(),
- thread_db_id: p.thread_db_id,
- };
+ enqueue_thread_metadata_update(ctx, link, job_id, &p.thread_provider_id, p.thread_db_id)
+ .await?;
+ handle_thread_attachment_upload(ctx, link, job_id, p.thread_db_id).await
+}
+/// Hands a fully written thread to its UpdateThreadMetadata step, which
+/// computes the thread's metadata and counts it as completed.
+#[tracing::instrument(skip(ctx))]
+pub async fn enqueue_thread_metadata_update(
+ ctx: &PubSubContext,
+ link: &Link,
+ job_id: Uuid,
+ thread_provider_id: &str,
+ thread_db_id: Uuid,
+) -> Result<(), ProcessingError> {
let ps_message = BackfillPubsubMessage {
backfill_operation: BackfillOperation::UpdateThreadMetadata(JobScopedPayload {
link_id: link.id,
job_id,
- payload: new_payload,
+ payload: UpdateMetadataPayload {
+ thread_provider_id: thread_provider_id.to_string(),
+ thread_db_id,
+ },
}),
};
@@ -454,11 +467,7 @@ async fn handle_thread_completed(
reason: FailureReason::SqsEnqueueFailed,
source: e.context("Failed to enqueue metadata message".to_string()),
})
- })?;
-
- handle_thread_attachment_upload(ctx, link, job_id, p.thread_db_id).await?;
-
- Ok(())
+ })
}
/// Creates BackfillAttachmentPayload messages and enqueues them via SQS for the given attachments
@@ -543,8 +552,9 @@ async fn send_attachment_backfill_messages(
Ok(())
}
+/// Enqueues the uploads of a backfilled thread's attachments.
#[tracing::instrument(skip(ctx))]
-async fn handle_thread_attachment_upload(
+pub async fn handle_thread_attachment_upload(
ctx: &PubSubContext,
link: &Link,
job_id: Uuid,
diff --git a/services/email_service/src/pubsub/backfill/backfill_message.rs b/services/email_service/src/pubsub/backfill/backfill_message.rs
index 1033f98c6c..33c9047898 100644
--- a/services/email_service/src/pubsub/backfill/backfill_message.rs
+++ b/services/email_service/src/pubsub/backfill/backfill_message.rs
@@ -4,13 +4,18 @@ use crate::pubsub::backfill::{increment_counters, write_slot};
use crate::pubsub::context::PubSubContext;
use crate::pubsub::util::{CrmContactRecipient, enqueue_populate_crm_contacts};
use crate::util::process_pre_insert::process_message_pre_insert;
+use email_api_client::domain::models::CalendarPart;
use models_email::email::service::backfill::{BackfillMessagePayload, JobScopedPayload};
use models_email::email::service::link;
+use models_email::email::service::message::Message;
use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
-/// This step is invoked by BackfillThread once for each message in the thread.
-/// Creates a message object in the database. If the message is the last message in
-/// the thread to be processed, it sends an UpdateThreadMetadata message for the thread.
+/// Backfills one message of a thread whose messages were enqueued one by one.
+///
+/// BackfillThread now writes a thread's messages itself; this step only drains
+/// BackfillMessage operations enqueued before that. Creates a message object in
+/// the database. If the message is the last message in the thread to be
+/// processed, it sends an UpdateThreadMetadata message for the thread.
#[tracing::instrument(skip(ctx))]
pub async fn backfill_message(
ctx: &PubSubContext,
@@ -47,11 +52,27 @@ pub async fn backfill_message(
.await;
inserted.map_err(|e| map_db_error(e, "Failed to insert final message into database"))?;
- if !message.is_draft && !fetched.calendar_parts.is_empty() {
+ save_invitations(ctx, link, &message, &fetched.calendar_parts).await;
+ enqueue_crm_contacts(ctx, link, &message).await?;
+
+ // Handle all success-related operations
+ increment_counters::incr_completed_messages(ctx, link, scope.job_id, p).await
+}
+
+/// Saves the calendar invitations a written backfill message carries. Best
+/// effort: a failure is logged, as invitation extraction always has been.
+pub(super) async fn save_invitations(
+ ctx: &PubSubContext,
+ link: &link::Link,
+ message: &Message,
+ calendar_parts: &[CalendarPart],
+) {
+ let provider_id = message.provider_id.as_deref().unwrap_or_default();
+ if !message.is_draft && !calendar_parts.is_empty() {
match email_db_client::messages::get::get_message_and_thread_id_by_provider_id(
&ctx.db,
link.id,
- &p.message_provider_id,
+ provider_id,
)
.await
{
@@ -59,16 +80,24 @@ pub async fn backfill_message(
crate::pubsub::invitation_extraction::save_discovered(
ctx,
link.id,
- &p.message_provider_id,
+ provider_id,
message_id,
- &fetched.calendar_parts,
+ calendar_parts,
)
.await;
}
Err(error) => tracing::warn!(error = ?error, "invitation extraction skipped"),
}
}
+}
+/// Enqueues the CRM contact updates a backfill message implies. Needs no
+/// database row, so it can run before the message is written.
+pub(super) async fn enqueue_crm_contacts(
+ ctx: &PubSubContext,
+ link: &link::Link,
+ message: &Message,
+) -> Result<(), ProcessingError> {
// Fan out a PopulateCrmContact job per address involved in the
// message — every non-draft message contributes, in both
// directions. Sent: enumerate to/cc/bcc (recipients the team
@@ -115,6 +144,5 @@ pub async fn backfill_message(
}
}
- // Handle all success-related operations
- increment_counters::incr_completed_messages(ctx, link, scope.job_id, p).await
+ Ok(())
}
diff --git a/services/email_service/src/pubsub/backfill/backfill_thread.rs b/services/email_service/src/pubsub/backfill/backfill_thread.rs
index e051b36e09..a34186b815 100644
--- a/services/email_service/src/pubsub/backfill/backfill_thread.rs
+++ b/services/email_service/src/pubsub/backfill/backfill_thread.rs
@@ -1,19 +1,26 @@
-use crate::pubsub::backfill::db_error::map_db_error;
+use crate::backfill_completion_service::{
+ enqueue_thread_metadata_update, handle_thread_attachment_upload,
+};
+use crate::pubsub::backfill::backfill_message::{enqueue_crm_contacts, save_invitations};
+use crate::pubsub::backfill::db_error::{is_unstorable, map_db_error};
use crate::pubsub::backfill::email_api_error::map_email_api_error;
use crate::pubsub::backfill::increment_counters::incr_completed_threads;
+use crate::pubsub::backfill::write_slot;
use crate::pubsub::context::PubSubContext;
-use models_email::email::service::backfill::{
- BackfillMessagePayload, BackfillOperation, BackfillPubsubMessage, BackfillThreadPayload,
- JobScopedPayload,
-};
+use crate::util::process_pre_insert::process_message_pre_insert;
+use email_api_client::domain::models::{CalendarPart, MessageWithCalendarParts};
+use email_db_client::messages::insert::SkippedMessage;
+use models_email::email::service::backfill::{BackfillThreadPayload, JobScopedPayload};
use models_email::email::service::link;
-use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
+use models_email::email::service::message::Message;
+use models_email::email::service::pubsub::ProcessingError;
use std::collections::HashSet;
/// This step is invoked by ListThreads for each thread being backfilled.
-/// Creates the thread object in the database, fetches the message ids for the thread
-/// from the gmail api, and sends a BackfillMessage message for each message_id.
-#[tracing::instrument(skip(ctx))]
+/// Reads the whole thread from the provider at once and writes the thread with
+/// all of its messages in one transaction, holding one of the fleet's backfill
+/// write slots, then hands the thread to its attachment and metadata steps.
+#[tracing::instrument(skip(ctx), fields(message_count))]
pub async fn backfill_thread(
ctx: &PubSubContext,
scope: &JobScopedPayload<BackfillThreadPayload>,
@@ -46,46 +53,50 @@ pub async fn backfill_thread(
return Ok(());
}
- // fetch all message_ids of the thread
- let message_ids = ctx
- .email_api
- .get_message_ids_for_thread(link.id, &thread_provider_id)
- .await
- .map_err(|error| map_email_api_error(error, "Failed to get provider thread message IDs"))?;
+ let fetched = fetch_thread(ctx, link, &thread_provider_id).await?;
+ let (mut messages, calendar_parts) = prepare_messages(fetched).await;
// The priority pass and the normal sweep both enqueue a thread, and both
// copies can pass the check above while they wait for Gmail. Only the copy
- // that inserts the thread backfills its messages; counting them twice
- // would complete the thread early and strand the duplicates.
- let Some(thread_db_id) = email_db_client::threads::insert::insert_blank_thread(
- &ctx.db,
- &thread_provider_id,
- link.id,
- )
- .await
- .map_err(|e| {
- map_db_error(
- e,
- format!("Failed to insert thread shell for {}", thread_provider_id),
- )
- })?
- else {
- incr_completed_threads(ctx, link, scope.job_id).await?;
- return Ok(());
+ // that inserts the thread finishes it; the other just counts it as done.
+ let slot = write_slot::acquire(ctx).await?;
+ let inserted = slot
+ .hold(async {
+ enqueue_threads_crm_contacts(ctx, link, &messages).await?;
+ email_db_client::threads::insert::insert_backfilled_thread(
+ &ctx.db,
+ &thread_provider_id,
+ link.id,
+ &mut messages,
+ is_unstorable,
+ )
+ .await
+ .map_err(|e| map_db_error(e, format!("Failed to insert thread {}", thread_provider_id)))
+ })
+ .await?;
+ let Some((thread_db_id, skipped)) = inserted else {
+ return incr_completed_threads(ctx, link, scope.job_id).await;
};
- // create backfill pubsub message for each email message
- enqueue_backfill_messages(ctx, scope, link, thread_db_id, message_ids).await
+ finish_written_thread(
+ ctx,
+ scope,
+ link,
+ thread_db_id,
+ &messages,
+ &calendar_parts,
+ &skipped,
+ )
+ .await
}
-/// Recovery-job path for a thread that is already in the database: fetch the
-/// provider's message ids and backfill only the messages we are missing, so
-/// replies that arrived during a sync gap are recovered.
+/// Recovery-job path for a thread that is already in the database: read the
+/// provider's thread and write only the messages we are missing, so replies
+/// that arrived during a sync gap are recovered.
///
/// Job accounting matches the fresh-thread path: with nothing missing the
-/// thread completes immediately (like the skip path); otherwise its redis
-/// progress counter is initialized to the missing count and the enqueued
-/// messages complete it.
+/// thread completes immediately (like the skip path); otherwise its metadata
+/// step completes it.
async fn refresh_existing_thread(
ctx: &PubSubContext,
scope: &JobScopedPayload<BackfillThreadPayload>,
@@ -93,18 +104,17 @@ async fn refresh_existing_thread(
thread_db_id: uuid::Uuid,
) -> Result<(), ProcessingError> {
let p = &scope.payload;
+ let fetched = fetch_thread(ctx, link, &p.thread_provider_id).await?;
- let message_ids = ctx
- .email_api
- .get_message_ids_for_thread(link.id, &p.thread_provider_id)
- .await
- .map_err(|error| map_email_api_error(error, "Failed to get provider thread message IDs"))?;
-
+ let provider_ids: Vec<String> = fetched
+ .iter()
+ .filter_map(|fetched| fetched.message.provider_id.clone())
+ .collect();
let existing_message_ids =
email_db_client::messages::get::filter_existing_provider_message_ids(
&ctx.db,
link.id,
- &message_ids,
+ &provider_ids,
)
.await
.map_err(|e| {
@@ -117,71 +127,139 @@ async fn refresh_existing_thread(
)
})?;
- let missing_message_ids: Vec<String> = message_ids
+ let missing: Vec<MessageWithCalendarParts> = fetched
.into_iter()
- .filter(|message_id| !existing_message_ids.contains(message_id))
+ .filter(|fetched| {
+ fetched
+ .message
+ .provider_id
+ .as_ref()
+ .is_some_and(|id| !existing_message_ids.contains(id))
+ })
.collect();
-
- if missing_message_ids.is_empty() {
+ if missing.is_empty() {
return incr_completed_threads(ctx, link, scope.job_id).await;
}
- enqueue_backfill_messages(ctx, scope, link, thread_db_id, missing_message_ids).await
+ let (mut messages, calendar_parts) = prepare_messages(missing).await;
+ let slot = write_slot::acquire(ctx).await?;
+ let skipped = slot
+ .hold(async {
+ enqueue_threads_crm_contacts(ctx, link, &messages).await?;
+ email_db_client::messages::insert::insert_thread_messages(
+ &ctx.db,
+ thread_db_id,
+ link.id,
+ &mut messages,
+ is_unstorable,
+ )
+ .await
+ .map_err(|e| {
+ map_db_error(
+ e,
+ format!(
+ "Failed to insert missing messages of thread {}",
+ p.thread_provider_id
+ ),
+ )
+ })
+ })
+ .await?;
+
+ finish_written_thread(
+ ctx,
+ scope,
+ link,
+ thread_db_id,
+ &messages,
+ &calendar_parts,
+ &skipped,
+ )
+ .await
}
-/// Initializes the thread's redis progress counter and enqueues one
-/// BackfillMessage per message id.
-async fn enqueue_backfill_messages(
+/// Reads every message of the provider thread, with its calendar parts.
+async fn fetch_thread(
ctx: &PubSubContext,
- scope: &JobScopedPayload<BackfillThreadPayload>,
link: &link::Link,
- thread_db_id: uuid::Uuid,
- message_ids: Vec<String>,
-) -> Result<(), ProcessingError> {
- let p = &scope.payload;
-
- ctx.redis_client
- .init_backfill_thread_progress(
- scope.job_id,
- &p.thread_provider_id,
- message_ids.len() as i32,
- )
+ thread_provider_id: &str,
+) -> Result<Vec<MessageWithCalendarParts>, ProcessingError> {
+ let fetched = ctx
+ .email_api
+ .get_thread(link.id, thread_provider_id)
.await
- .map_err(|e| {
- ProcessingError::Retryable(DetailedError {
- reason: FailureReason::RedisQueryFailed,
- source: e.context(format!(
- "Failed to create entry in redis for thread {}",
- &p.thread_provider_id
- )),
- })
- })?;
-
- for message_id in message_ids {
- let new_payload = BackfillMessagePayload {
- thread_provider_id: p.thread_provider_id.clone(),
- thread_db_id,
- message_provider_id: message_id.clone(),
- };
-
- let ps_message = BackfillPubsubMessage {
- backfill_operation: BackfillOperation::BackfillMessage(JobScopedPayload {
- link_id: link.id,
- job_id: scope.job_id,
- payload: new_payload,
- }),
- };
+ .map_err(|error| map_email_api_error(error, "Failed to get provider thread"))?;
+ tracing::Span::current().record("message_count", fetched.len());
+ Ok(fetched)
+}
- ctx.sqs_client
- .enqueue_email_backfill_message(ps_message)
- .await
- .map_err(|e| {
- ProcessingError::NonRetryable(DetailedError {
- reason: FailureReason::SqsEnqueueFailed,
- source: e.context(format!("Failed to enqueue message {}", message_id)),
- })
- })?;
+/// Cleans each message for insertion, keeping its calendar parts at the same index.
+async fn prepare_messages(
+ fetched: Vec<MessageWithCalendarParts>,
+) -> (Vec<Message>, Vec<Vec<CalendarPart>>) {
+ let mut messages = Vec::with_capacity(fetched.len());
+ let mut calendar_parts = Vec::with_capacity(fetched.len());
+ for MessageWithCalendarParts {
+ mut message,
+ calendar_parts: parts,
+ } in fetched
+ {
+ process_message_pre_insert(&mut message).await;
+ messages.push(message);
+ calendar_parts.push(parts);
}
+ (messages, calendar_parts)
+}
+/// Enqueues the CRM contact updates of every message in the thread. This runs
+/// just before the thread is written, while a failure can still be retried:
+/// after the commit a retry would skip the thread. It runs once a write slot is
+/// held, so an operation deferred for a busy slot doesn't repeat it each time.
+/// The rare duplicate from a retried write is harmless.
+async fn enqueue_threads_crm_contacts(
+ ctx: &PubSubContext,
+ link: &link::Link,
+ messages: &[Message],
+) -> Result<(), ProcessingError> {
+ for message in messages {
+ enqueue_crm_contacts(ctx, link, message).await?;
+ }
Ok(())
}
+
+/// Finishes a thread whose messages are committed.
+///
+/// A retried BackfillThread finds the thread written and skips it, so nothing
+/// here may ask for a retry. Messages the write skipped are logged. The
+/// invitation and attachment fan-outs only log their failures; the metadata
+/// step goes last and is the only one that can fail the operation, as a
+/// non-retryable error that counts the thread once.
+async fn finish_written_thread(
+ ctx: &PubSubContext,
+ scope: &JobScopedPayload<BackfillThreadPayload>,
+ link: &link::Link,
+ thread_db_id: uuid::Uuid,
+ messages: &[Message],
+ calendar_parts: &[Vec<CalendarPart>],
+ skipped: &[SkippedMessage],
+) -> Result<(), ProcessingError> {
+ let thread_provider_id = &scope.payload.thread_provider_id;
+ for SkippedMessage { index, error } in skipped {
+ tracing::warn!(
+ error = ?error,
+ %thread_provider_id,
+ message_id = messages[*index].provider_id.as_deref().unwrap_or_default(),
+ "Skipped a backfilled message that cannot be stored"
+ );
+ }
+ for (index, (message, parts)) in messages.iter().zip(calendar_parts).enumerate() {
+ if !skipped.iter().any(|skipped| skipped.index == index) {
+ save_invitations(ctx, link, message, parts).await;
+ }
+ }
+ if let Err(error) = handle_thread_attachment_upload(ctx, link, scope.job_id, thread_db_id).await
+ {
+ tracing::warn!(error = ?error, %thread_provider_id, "Skipped a backfilled thread's attachment uploads");
+ }
+ enqueue_thread_metadata_update(ctx, link, scope.job_id, thread_provider_id, thread_db_id).await
+}
diff --git a/services/email_service/src/pubsub/backfill/db_error.rs b/services/email_service/src/pubsub/backfill/db_error.rs
index bf145953a7..14e7b582bb 100644
--- a/services/email_service/src/pubsub/backfill/db_error.rs
+++ b/services/email_service/src/pubsub/backfill/db_error.rs
@@ -43,6 +43,12 @@ where
}
}
+/// Whether a write failed on its own data, failing identically on every
+/// retry. Backfill skips such a message instead of failing its whole thread.
+pub(crate) fn is_unstorable(error: &anyhow::Error) -> bool {
+ deterministic_sqlstate(error).is_some()
+}
+
/// Returns the SQLSTATE of the first database error in the chain that no
/// amount of retrying will get past.
fn deterministic_sqlstate(error: &anyhow::Error) -> Option<String> {
diff --git a/services/email_service/src/pubsub/backfill/process.rs b/services/email_service/src/pubsub/backfill/process.rs
index 36c0684573..f45508f91a 100644
--- a/services/email_service/src/pubsub/backfill/process.rs
+++ b/services/email_service/src/pubsub/backfill/process.rs
@@ -11,6 +11,7 @@ use models_email::email::service::backfill::{
use models_email::email::service::link;
use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
use sqs_worker::cleanup_message;
+use std::time::Duration;
use uuid::Uuid;
// Process a single message from the backfill queue
@@ -30,7 +31,8 @@ pub async fn process_message(
}
};
- let processing_result = inner_process_message(&ctx, &data).await;
+ let processing_result =
+ keep_invisible(message, &ctx.sqs_worker, inner_process_message(&ctx, &data)).await;
match processing_result {
// Processing success. Clean up the message
@@ -55,6 +57,47 @@ pub async fn process_message(
}
}
+/// How often a running operation pushes its message's visibility out again.
+const VISIBILITY_HEARTBEAT: Duration = Duration::from_secs(20);
+/// The backfill queue's visibility timeout, restored by every heartbeat.
+const VISIBILITY_TIMEOUT_SECONDS: i32 = 60;
+
+/// Runs `work` while keeping its message hidden from other workers.
+///
+/// A whole thread, or any operation slowed by a struggling database, can
+/// outlast the queue's visibility timeout. Delivered again while the first
+/// copy still runs, the second copy would find the thread written and count
+/// it, and the first would count it again through its metadata step,
+/// completing the job early. A failed heartbeat only logs: the message then
+/// reappears as it would have without one.
+async fn keep_invisible<F: Future>(
+ message: &aws_sdk_sqs::types::Message,
+ sqs_worker: &sqs_worker::SQSWorker,
+ work: F,
+) -> F::Output {
+ let Some(receipt_handle) = message.receipt_handle() else {
+ return work.await;
+ };
+ let mut work = std::pin::pin!(work);
+ let mut heartbeats = tokio::time::interval_at(
+ tokio::time::Instant::now() + VISIBILITY_HEARTBEAT,
+ VISIBILITY_HEARTBEAT,
+ );
+ loop {
+ tokio::select! {
+ output = &mut work => return output,
+ _ = heartbeats.tick() => {
+ if let Err(error) = sqs_worker
+ .change_message_visibility(receipt_handle, VISIBILITY_TIMEOUT_SECONDS)
+ .await
+ {
+ tracing::warn!(error = ?error, "failed to keep a running backfill operation hidden");
+ }
+ }
+ }
+ }
+}
+
#[tracing::instrument(skip(ctx))]
async fn inner_process_message(
ctx: &PubSubContext,
diff --git a/services/email_service/src/pubsub/backfill/write_slot.rs b/services/email_service/src/pubsub/backfill/write_slot.rs
index eb7d50491d..81437bdbd7 100644
--- a/services/email_service/src/pubsub/backfill/write_slot.rs
+++ b/services/email_service/src/pubsub/backfill/write_slot.rs
@@ -5,7 +5,7 @@ use crate::pubsub::context::PubSubContext;
use crate::util::redis::backfill_write_slots::BackfillWriteSlot;
use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
-/// Takes a backfill write slot for one message write. A slot that stays taken,
+/// Takes a backfill write slot for one write. A slot that stays taken,
/// or Redis failing to decide, refuses the write so the operation is deferred
/// instead of written unmetered.
pub(super) async fn acquire(ctx: &PubSubContext) -> Result<BackfillWriteSlot, ProcessingError> {
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
index 506cbe8585..a403d2584a 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
@@ -602,7 +602,7 @@ async fn fetch_and_insert_thread(
provider_thread_id: &str,
) -> result::Result<(), ProcessingError> {
let messages = match ctx.email_api.get_thread(link_id, provider_thread_id).await {
- Ok(messages) => messages,
+ Ok(fetched) => fetched.into_iter().map(|fetched| fetched.message).collect(),
Err(error) => {
return Err(handle_operation_error(
ctx,
diff --git a/services/email_service/src/util/redis/backfill.rs b/services/email_service/src/util/redis/backfill.rs
index 608c2d5685..71c113d627 100644
--- a/services/email_service/src/util/redis/backfill.rs
+++ b/services/email_service/src/util/redis/backfill.rs
@@ -133,34 +133,6 @@ impl RedisClient {
Ok(())
}
- /// Initialize backfill thread progress in Redis
- pub async fn init_backfill_thread_progress(
- &self,
- job_id: Uuid,
- thread_id: &str,
- total_messages: i32,
- ) -> anyhow::Result<()> {
- let key = Self::thread_status_key(job_id, thread_id);
-
- let mut redis_connection = self
- .inner
- .get_multiplexed_async_connection()
- .await
- .context("unable to connect to redis")?;
-
- redis_connection
- .hset_multiple::<&str, &str, i32, ()>(
- &key,
- &[
- ("total_messages", total_messages),
- ("completed_messages", 0),
- ],
- )
- .await?;
-
- Ok(())
- }
-
/// Increment completed messages count and return true if thread is complete, deleting redis entry for thread if so
pub async fn incr_completed_messages(
&self,
diff --git a/services/email_service/src/util/redis/backfill_write_slots.rs b/services/email_service/src/util/redis/backfill_write_slots.rs
index 289cf42a15..a46f9e8879 100644
--- a/services/email_service/src/util/redis/backfill_write_slots.rs
+++ b/services/email_service/src/util/redis/backfill_write_slots.rs
@@ -1,12 +1,11 @@
-//! A fleet-wide cap on concurrent backfill message writes.
+//! A fleet-wide cap on concurrent backfill writes.
//!
-//! Every backfill worker of every email-service task writes messages straight
-//! to the primary, one transaction per message, and nothing else bounds how
-//! many run at once (25 workers per task, 5 to 10 tasks). Message bodies land
-//! in one TOAST table, so a large backfill queues its writers on that table's
-//! relation-extension lock and slows every other query on the database. A slot
-//! is a Redis lease; at most [`CAPACITY`] backfill writes hold one at a time,
-//! however many tasks are running.
+//! Every backfill worker of every email-service task writes threads straight
+//! to the primary, and nothing else bounds how many run at once (25 workers
+//! per task, 5 to 10 tasks). Message bodies land in one TOAST table, so a large
+//! backfill queues its writers on that table's relation-extension lock and
+//! slows every other query on the database. A slot is a Redis lease; at most
+//! [`CAPACITY`] backfill writes hold one at a time, however many tasks run.
use crate::util::redis::RedisClient;
use rand::Rng;
@@ -17,11 +16,12 @@ use uuid::Uuid;
#[cfg(test)]
mod test;
-/// Concurrent backfill message writes across the fleet. A healthy write (the
-/// contacts upsert plus the message transaction) averages about 100 ms, so
-/// twenty slots admit roughly 200 messages a second while the database is
-/// healthy, above normal busy hours, and fewer as it slows down. Unbounded,
-/// 125 to 250 backfill workers could write at once.
+/// Concurrent backfill writes across the fleet. A healthy single-message write
+/// (the contacts upsert plus its transaction) averages about 100 ms, so twenty
+/// slots admit at least roughly 200 messages a second while the database is
+/// healthy, above normal busy hours, more when a thread's messages share one
+/// transaction, and fewer as the database slows down. Unbounded, 125 to 250
+/// backfill workers could write at once.
const CAPACITY: u32 = 20;
/// Frees the slot of a crashed worker. A held slot renews it every
/// [`RENEW_EVERY`], so a write that outlasts it keeps its slot.