Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(email): pace mailbox purges and stop duplicate concurrent deletes (#8410)

teoa3021c6634db
diff --git a/Cargo.lock b/Cargo.lock
index 799e4a3e9d..ea4348b189 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7323,6 +7323,7 @@ dependencies = [
"sqlx",
"tokio",
"tracing",
+ "tracing-subscriber",
"workspace-hack",
]
diff --git a/crates/email/src/outbound/email_pg_repo/test/send_attempt/deferred_deletion.rs b/crates/email/src/outbound/email_pg_repo/test/send_attempt/deferred_deletion.rs
index 4d81b9fe17..7cc1b03000 100644
--- a/crates/email/src/outbound/email_pg_repo/test/send_attempt/deferred_deletion.rs
+++ b/crates/email/src/outbound/email_pg_repo/test/send_attempt/deferred_deletion.rs
@@ -288,7 +288,7 @@ async fn explicit_inbox_purge_removes_an_accepted_send_and_its_retained_deletion
let summary = email_db_client::links::delete::purge_link(
&pool,
link(),
- email_db_client::links::delete::PurgeBatchSizes::default(),
+ email_db_client::links::delete::PurgePacing::default(),
)
.await?;
// The fixture has four messages in this inbox and one in a second inbox.
diff --git a/crates/email_db_client/Cargo.toml b/crates/email_db_client/Cargo.toml
index 9adfdae7db..c420298302 100644
--- a/crates/email_db_client/Cargo.toml
+++ b/crates/email_db_client/Cargo.toml
@@ -24,3 +24,4 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
macro_db_migrator = { path = "../macro_db_migrator" }
+tracing-subscriber = { workspace = true }
diff --git a/crates/email_db_client/src/links/delete.rs b/crates/email_db_client/src/links/delete.rs
index 58286c3840..0ba5d36a4b 100644
--- a/crates/email_db_client/src/links/delete.rs
+++ b/crates/email_db_client/src/links/delete.rs
@@ -1,14 +1,17 @@
use sqlx::types::Uuid;
use sqlx::{PgPool, Postgres, Transaction};
use std::num::NonZeroU32;
+use std::time::Duration;
#[cfg(test)]
mod test;
-/// How many rows each [`purge_link`] transaction removes. Every batch commits on its own,
-/// so no single transaction holds locks, WAL or the xmin horizon for a whole mailbox.
+/// How fast [`purge_link`] removes a link's data. Every batch commits on its own, so no
+/// single transaction holds locks, WAL or the xmin horizon for a whole mailbox, and the
+/// pause between batches caps how much write I/O one purge generates, so several purges
+/// running at once can't saturate the database.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct PurgeBatchSizes {
+pub struct PurgePacing {
/// Messages per transaction, deleted together with their labels, recipients,
/// attachments and scheduled sends, and with the threads the batch finishes. Message
/// bodies are most of a mailbox's bytes (TOAST), so this is the knob that bounds a
@@ -18,14 +21,17 @@ pub struct PurgeBatchSizes {
pub threads: NonZeroU32,
/// Contacts per transaction.
pub contacts: NonZeroU32,
+ /// Wait after a full batch before the next one starts.
+ pub pause: Duration,
}
-impl Default for PurgeBatchSizes {
+impl Default for PurgePacing {
fn default() -> Self {
Self {
- messages: NonZeroU32::new(1_000).expect("non-zero"),
+ messages: NonZeroU32::new(500).expect("non-zero"),
threads: NonZeroU32::new(2_000).expect("non-zero"),
contacts: NonZeroU32::new(5_000).expect("non-zero"),
+ pause: Duration::from_secs(2),
}
}
}
@@ -69,8 +75,8 @@ pub async fn mark_link_deleting(pool: &PgPool, link_id: Uuid) -> anyhow::Result<
/// Deletes a link and all of its email data incrementally: messages (deepest children
/// first) with the threads they belong to, then any remaining threads, then contacts,
-/// each in bounded short transactions, and finally the `email_links` row, whose cascade
-/// only has small per-link tables left to clear.
+/// each in bounded short transactions paced by `pacing`, and finally the `email_links`
+/// row, whose cascade only has small per-link tables left to clear.
///
/// Marks the link via [`mark_link_deleting`] first, so it can be called on its own. Safe to
/// rerun after a crash (it resumes from whatever is left) and to run concurrently for the
@@ -79,46 +85,124 @@ pub async fn mark_link_deleting(pool: &PgPool, link_id: Uuid) -> anyhow::Result<
pub async fn purge_link(
pool: &PgPool,
link_id: Uuid,
- batch_sizes: PurgeBatchSizes,
+ pacing: PurgePacing,
) -> anyhow::Result<PurgeSummary> {
mark_link_deleting(pool, link_id).await?;
let mut summary = PurgeSummary::default();
+ (summary.messages, summary.threads) = purge_messages(pool, link_id, pacing).await?;
+ summary.threads += purge_threads(pool, link_id, pacing).await?;
+ summary.contacts = purge_contacts(pool, link_id, pacing).await?;
+ summary.link_deleted = delete_marked_link(pool, link_id).await?;
+
+ tracing::info!(?summary, "Purged email link");
+ Ok(summary)
+}
+/// Deletes the link's messages with the threads each batch finishes, returning how many
+/// of each it deleted.
+#[tracing::instrument(
+ name = "email.purge_link.messages",
+ skip_all,
+ fields(
+ batches = tracing::field::Empty,
+ paused_ms = tracing::field::Empty,
+ messages = tracing::field::Empty,
+ threads = tracing::field::Empty,
+ ),
+ err
+)]
+async fn purge_messages(
+ pool: &PgPool,
+ link_id: Uuid,
+ pacing: PurgePacing,
+) -> anyhow::Result<(u64, u64)> {
+ let (mut batches, mut messages, mut threads) = (0, 0, 0);
let mut thread_cursor = Uuid::nil();
loop {
- let batch = purge_message_batch(pool, link_id, thread_cursor, batch_sizes.messages).await?;
- summary.messages += batch.messages;
- summary.threads += batch.threads;
+ let batch = purge_message_batch(pool, link_id, thread_cursor, pacing.messages).await?;
+ batches += 1;
+ messages += batch.messages;
+ threads += batch.threads;
let Some(next_cursor) = batch.next_cursor else {
break;
};
thread_cursor = next_cursor;
+ tokio::time::sleep(pacing.pause).await;
}
+ let span = record_phase(batches, pacing);
+ span.record("messages", messages);
+ span.record("threads", threads);
+ Ok((messages, threads))
+}
+
+/// Deletes the threads left once every message is gone.
+#[tracing::instrument(
+ name = "email.purge_link.threads",
+ skip_all,
+ fields(
+ batches = tracing::field::Empty,
+ paused_ms = tracing::field::Empty,
+ deleted = tracing::field::Empty,
+ ),
+ err
+)]
+async fn purge_threads(pool: &PgPool, link_id: Uuid, pacing: PurgePacing) -> anyhow::Result<u64> {
+ let (mut batches, mut deleted) = (0, 0);
loop {
- let batch = purge_thread_batch(pool, link_id, batch_sizes.threads).await?;
- summary.threads += batch.deleted;
+ let batch = purge_thread_batch(pool, link_id, pacing.threads).await?;
+ batches += 1;
+ deleted += batch.deleted;
if !batch.full {
break;
}
+ tokio::time::sleep(pacing.pause).await;
}
+ record_phase(batches, pacing).record("deleted", deleted);
+ Ok(deleted)
+}
+
+/// Deletes the link's contacts.
+#[tracing::instrument(
+ name = "email.purge_link.contacts",
+ skip_all,
+ fields(
+ batches = tracing::field::Empty,
+ paused_ms = tracing::field::Empty,
+ deleted = tracing::field::Empty,
+ ),
+ err
+)]
+async fn purge_contacts(pool: &PgPool, link_id: Uuid, pacing: PurgePacing) -> anyhow::Result<u64> {
+ let (mut batches, mut deleted) = (0, 0);
let mut contact_cursor = String::new();
loop {
- let batch =
- purge_contact_batch(pool, link_id, &contact_cursor, batch_sizes.contacts).await?;
- summary.contacts += batch.deleted;
+ let batch = purge_contact_batch(pool, link_id, &contact_cursor, pacing.contacts).await?;
+ batches += 1;
+ deleted += batch.deleted;
let Some(next_cursor) = batch.next_cursor else {
break;
};
contact_cursor = next_cursor;
+ tokio::time::sleep(pacing.pause).await;
}
- summary.link_deleted = delete_marked_link(pool, link_id).await?;
+ record_phase(batches, pacing).record("deleted", deleted);
+ Ok(deleted)
+}
- tracing::info!(?summary, "Purged email link");
- Ok(summary)
+/// Records a finished phase's batch count and pause time on its span. Every batch but the
+/// last came back full and was followed by a pause.
+fn record_phase(batches: u32, pacing: PurgePacing) -> tracing::Span {
+ let span = tracing::Span::current();
+ span.record("batches", batches);
+ span.record(
+ "paused_ms",
+ (pacing.pause * batches.saturating_sub(1)).as_millis(),
+ );
+ span
}
/// Opens a purge batch transaction holding the link's purge lock until commit, so a
diff --git a/crates/email_db_client/src/links/delete/test.rs b/crates/email_db_client/src/links/delete/test.rs
index 7e3ddaefd5..0459465a20 100644
--- a/crates/email_db_client/src/links/delete/test.rs
+++ b/crates/email_db_client/src/links/delete/test.rs
@@ -6,6 +6,11 @@ use macro_user_id::email::EmailStr;
use macro_user_id::user_id::MacroUserIdStr;
use models_email::email::service::link::{Link, UserProvider};
use sqlx::{Pool, Postgres};
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use tracing::field::{Field, Visit};
+use tracing_subscriber::layer::{Context, SubscriberExt as _};
+use tracing_subscriber::registry::LookupSpan;
/// The fixture link that tests purge.
const PURGED: Uuid = Uuid::from_u128(0xa001);
@@ -122,16 +127,17 @@ async fn deletion_state(pool: &Pool<Postgres>, link_id: Uuid) -> anyhow::Result<
.await?)
}
-fn batch_sizes(messages: u32, threads: u32, contacts: u32) -> PurgeBatchSizes {
- PurgeBatchSizes {
+fn batch_sizes(messages: u32, threads: u32, contacts: u32) -> PurgePacing {
+ PurgePacing {
messages: NonZeroU32::new(messages).expect("non-zero"),
threads: NonZeroU32::new(threads).expect("non-zero"),
contacts: NonZeroU32::new(contacts).expect("non-zero"),
+ pause: Duration::ZERO,
}
}
/// Small enough that every phase of the fixture takes several batches, with a partial last one.
-fn tiny_batches() -> PurgeBatchSizes {
+fn tiny_batches() -> PurgePacing {
batch_sizes(4, 3, 3)
}
@@ -177,6 +183,129 @@ async fn purge_removes_all_link_data_in_batches(pool: Pool<Postgres>) -> anyhow:
Ok(())
}
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("purge_link"))
+)]
+async fn purge_pauses_between_batches(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ // 30 messages in batches of 20 take one full batch and one partial one; threads and
+ // contacts each fit in a single partial batch. So the purge pauses exactly once.
+ let pause = Duration::from_secs(1);
+ let started = std::time::Instant::now();
+
+ let summary = purge_link(
+ &pool,
+ PURGED,
+ PurgePacing {
+ pause,
+ ..batch_sizes(20, 100, 100)
+ },
+ )
+ .await?;
+
+ assert!(started.elapsed() >= pause);
+ assert!(summary.link_deleted);
+ assert_eq!(link_rows(&pool, PURGED).await?, LinkRows::EMPTY);
+ assert_eq!(link_rows(&pool, KEPT).await?, LinkRows::FULL);
+
+ Ok(())
+}
+
+/// The integer fields recorded on each span while installed, by span and field name.
+#[derive(Clone, Default)]
+struct RecordedFields(Arc<Mutex<HashMap<&'static str, HashMap<&'static str, u128>>>>);
+
+impl RecordedFields {
+ fn of(&self, span: &str) -> HashMap<&'static str, u128> {
+ let spans = self.0.lock().expect("recorded fields lock poisoned");
+ spans.get(span).cloned().unwrap_or_default()
+ }
+}
+
+impl<S> tracing_subscriber::Layer<S> for RecordedFields
+where
+ S: tracing::Subscriber + for<'a> LookupSpan<'a>,
+{
+ fn on_record(
+ &self,
+ id: &tracing::span::Id,
+ values: &tracing::span::Record<'_>,
+ ctx: Context<'_, S>,
+ ) {
+ let Some(span) = ctx.span(id) else {
+ return;
+ };
+ let mut spans = self.0.lock().expect("recorded fields lock poisoned");
+ values.record(&mut IntegerFields(spans.entry(span.name()).or_default()));
+ }
+}
+
+struct IntegerFields<'a>(&'a mut HashMap<&'static str, u128>);
+
+impl Visit for IntegerFields<'_> {
+ fn record_u64(&mut self, field: &Field, value: u64) {
+ self.0.insert(field.name(), value.into());
+ }
+
+ fn record_u128(&mut self, field: &Field, value: u128) {
+ self.0.insert(field.name(), value);
+ }
+
+ fn record_debug(&mut self, _field: &Field, _value: &dyn std::fmt::Debug) {}
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("purge_link"))
+)]
+async fn purge_phases_record_their_batches_and_pauses(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let recorded = RecordedFields::default();
+ let _guard =
+ tracing::subscriber::set_default(tracing_subscriber::registry().with(recorded.clone()));
+ tracing::callsite::rebuild_interest_cache();
+ let pause = Duration::from_millis(5);
+
+ let summary = purge_link(
+ &pool,
+ PURGED,
+ PurgePacing {
+ pause,
+ ..tiny_batches()
+ },
+ )
+ .await?;
+
+ // 30 messages in batches of 4 take 7 full batches and a partial one. A full batch keeps
+ // its last thread even when it emptied it, so they finish 8 of the 11 threads, and the
+ // thread phase takes the other 3 in one full batch and an empty one. 4 contacts in
+ // batches of 3 take a full batch and a partial one. Every full batch is followed by a
+ // pause.
+ let messages = recorded.of("email.purge_link.messages");
+ assert_eq!(messages["batches"], 8);
+ assert_eq!(messages["paused_ms"], 7 * pause.as_millis());
+ assert_eq!(messages["messages"], 30);
+ assert_eq!(messages["threads"], 8);
+ let threads = recorded.of("email.purge_link.threads");
+ assert_eq!(threads["batches"], 2);
+ assert_eq!(threads["paused_ms"], pause.as_millis());
+ assert_eq!(threads["deleted"], 3);
+ let contacts = recorded.of("email.purge_link.contacts");
+ assert_eq!(contacts["batches"], 2);
+ assert_eq!(contacts["paused_ms"], pause.as_millis());
+ assert_eq!(contacts["deleted"], 4);
+ assert_eq!(
+ summary,
+ PurgeSummary {
+ messages: 30,
+ threads: 11,
+ contacts: 4,
+ link_deleted: true,
+ }
+ );
+
+ Ok(())
+}
+
#[sqlx::test(
migrator = "MACRO_DB_MIGRATIONS",
fixtures(path = "../../../fixtures", scripts("purge_link"))
@@ -299,7 +428,7 @@ async fn sync_does_not_resume_on_a_link_being_deleted(pool: Pool<Postgres>) -> a
assert_eq!(state.deleting_at, Some(deleting_at));
assert!(
- purge_link(&pool, link.id, PurgeBatchSizes::default())
+ purge_link(&pool, link.id, PurgePacing::default())
.await?
.link_deleted
);
@@ -320,7 +449,7 @@ async fn purge_of_a_missing_link_is_a_no_op(pool: Pool<Postgres>) -> anyhow::Res
mark_link_deleting(&pool, missing).await?;
assert_eq!(
- purge_link(&pool, missing, PurgeBatchSizes::default()).await?,
+ purge_link(&pool, missing, PurgePacing::default()).await?,
PurgeSummary::default()
);
diff --git a/crates/sqs_worker/src/change_message_visibility.rs b/crates/sqs_worker/src/change_message_visibility.rs
new file mode 100644
index 0000000000..8430f61c08
--- /dev/null
+++ b/crates/sqs_worker/src/change_message_visibility.rs
@@ -0,0 +1,16 @@
+pub async fn change_message_visibility(
+ inner: &aws_sdk_sqs::Client,
+ queue_url: &str,
+ receipt_handle: &str,
+ visibility_timeout_seconds: i32,
+) -> anyhow::Result<()> {
+ inner
+ .change_message_visibility()
+ .queue_url(queue_url)
+ .receipt_handle(receipt_handle)
+ .visibility_timeout(visibility_timeout_seconds)
+ .send()
+ .await?;
+
+ Ok(())
+}
diff --git a/crates/sqs_worker/src/lib.rs b/crates/sqs_worker/src/lib.rs
index c9c3527b7e..b15a4d026e 100644
--- a/crates/sqs_worker/src/lib.rs
+++ b/crates/sqs_worker/src/lib.rs
@@ -1,3 +1,4 @@
+mod change_message_visibility;
mod delete_message;
mod receive_messages;
@@ -67,14 +68,36 @@ impl SQSWorker {
receipt_handle: &str,
timeout_seconds: i32,
) -> anyhow::Result<()> {
- self.inner
- .change_message_visibility()
- .queue_url(&self.queue_url)
- .receipt_handle(receipt_handle)
- .visibility_timeout(timeout_seconds)
- .send()
- .await?;
- Ok(())
+ change_message_visibility::change_message_visibility(
+ &self.inner,
+ &self.queue_url,
+ receipt_handle,
+ timeout_seconds,
+ )
+ .await
+ }
+
+ /// Hides a received message from other consumers for `seconds` from now.
+ #[tracing::instrument(
+ skip(self, message),
+ fields(message_id = message.message_id),
+ err(level = tracing::Level::WARN)
+ )]
+ pub async fn extend_visibility(
+ &self,
+ message: &aws_sdk_sqs::types::Message,
+ seconds: i32,
+ ) -> anyhow::Result<()> {
+ let Some(receipt_handle) = message.receipt_handle.as_ref() else {
+ anyhow::bail!("no receipt handle found for message");
+ };
+ change_message_visibility::change_message_visibility(
+ &self.inner,
+ &self.queue_url,
+ receipt_handle,
+ seconds,
+ )
+ .await
}
}
diff --git a/deny.toml b/deny.toml
index 43980027b9..0d7f85ede0 100644
--- a/deny.toml
+++ b/deny.toml
@@ -234,6 +234,7 @@ deny = [
"document_storage_service", # GraphQL telemetry span and log tests only (dev-dependency)
"documents", # Dev logging only
"email", # Dev logging only
+ "email_db_client", # Purge phase span tests only
"genai_telemetry", # GenAI span tests
"graphql_email", # Unavailable-thread warning field and span tests only
"graphql_entity_mutation", # Batch failure span field tests only
diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.md
index 53171ac9f0..068eb2b90b 100644
--- a/services/email_service/AGENTS.md
+++ b/services/email_service/AGENTS.md
@@ -11,7 +11,8 @@ the gateway `/email` prefix.
- `src/main.rs` / `src/lib.rs`: entrypoint and `run()`.
- `src/api.rs` + `src/api/context.rs`: HTTP surface and `ApiContext`.
- `src/pubsub/`: SQS pubsub workers, one module per concern — `inbox_sync`,
- `link_manager` (Gmail re-subscribe), `scheduled` (scheduled sends),
+ `link_manager` (Gmail re-subscribe, health checks, inbox deletion),
+ `scheduled` (scheduled sends),
`sfs_uploader`/`sfs_deleter` (attachment storage), `gmail_ops`, `backfill`,
`crm_cleanup`, `worker_lifecycle`.
- `src/backfill_outbox.rs`, `src/backfill_init_service.rs`,
@@ -153,6 +154,10 @@ the gateway `/email` prefix.
- Header addresses that Gmail accepts can fail `MacroUserIdStr::try_from_email`.
Skip the bad address with a WARN that omits it (the parse error echoes it)
instead of failing the step; see `pubsub::util::build_contact_connections`.
+- The `link_manager` worker receives one message at a time and renews its
+ visibility while processing. Don't raise its receive batch: messages queued
+ behind a long mailbox purge would be redelivered and purged twice.
+ `PurgePacing` (`email_db_client::links::delete`) sets the purge's write rate.
## Related
diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
index f73e0d865a..2d8ae35d49 100644
--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
@@ -140,10 +140,13 @@ async fn main() -> anyhow::Result<()> {
),
});
+ // One message per receive: messages are processed one at a time and only the one in
+ // progress renews its visibility, so the rest of a batch would time out behind a long
+ // mailbox purge and be processed again by another worker.
let link_manager_worker = sqs_worker::SQSWorker::new(
aws_sdk_sqs::Client::new(&gmail_queue_aws_config),
link_manager_queue.to_string(),
- config.queue_max_messages,
+ 1,
config.queue_wait_time_seconds,
);
diff --git a/services/email_service/src/pubsub/link_manager/process.rs b/services/email_service/src/pubsub/link_manager/process.rs
index 6495cef0b4..1991728b8b 100644
--- a/services/email_service/src/pubsub/link_manager/process.rs
+++ b/services/email_service/src/pubsub/link_manager/process.rs
@@ -9,7 +9,7 @@ use email::domain::events::{
use email_api_client::domain::models::AccessToken;
#[cfg(test)]
use email_api_client::domain::models::EmailApiError;
-use email_db_client::links::delete::PurgeBatchSizes;
+use email_db_client::links::delete::PurgePacing;
use model_entity::EntityType;
use model_notifications::InboxReauthRequiredMetadata;
use models_email::api::refresh::RefreshEmailEvent;
@@ -202,7 +202,7 @@ async fn handle_delete(
// finally, delete the link and all of its email data in bounded batches; a redelivered
// message resumes the purge where an interrupted one stopped
- email_db_client::links::delete::purge_link(&ctx.db, link.id, PurgeBatchSizes::default())
+ email_db_client::links::delete::purge_link(&ctx.db, link.id, PurgePacing::default())
.await
.context("Failed to purge link in background task")?;
diff --git a/services/email_service/src/pubsub/link_manager/worker.rs b/services/email_service/src/pubsub/link_manager/worker.rs
index 6b634b233a..a6e80640fb 100644
--- a/services/email_service/src/pubsub/link_manager/worker.rs
+++ b/services/email_service/src/pubsub/link_manager/worker.rs
@@ -10,8 +10,22 @@ use contacts::{domain::service::SqsContactsIngress, outbound::ingress::SqsContac
use futures::StreamExt;
use sqlx::PgPool;
use sqs_client::SQS;
+use std::convert::Infallible;
+use std::future::Future;
use std::sync::Arc;
+use std::time::Duration;
+use tokio::time::{Instant, MissedTickBehavior, interval_at};
use tokio_util::sync::CancellationToken;
+use tracing::Instrument;
+
+#[cfg(test)]
+mod test;
+
+/// How long each renewal hides a message being processed from other workers.
+const VISIBILITY_EXTENSION_SECONDS: i32 = 300;
+/// How often a message being processed renews its visibility. Several renewals fit in one
+/// extension, so a failed renewal doesn't expose the message.
+const VISIBILITY_RENEWAL_INTERVAL: Duration = Duration::from_secs(60);
/// method that ingests sqs messages and calls the process function for each
#[allow(clippy::too_many_arguments)]
@@ -144,12 +158,23 @@ pub async fn run_worker_with_cancellation(
let result = futures::stream::iter(messages.iter())
.then(|message| {
let ctx = ctx.clone();
+ let worker = &worker;
+ let span = tracing::info_span!(
+ "email.link_manager.message",
+ message_id = message.message_id.as_deref(),
+ );
async move {
- let result = process::process_message(
- ctx,
- message,
+ let result = while_renewing_visibility(
+ || {
+ worker.extend_visibility(
+ message,
+ VISIBILITY_EXTENSION_SECONDS,
+ )
+ },
+ process::process_message(ctx, message),
)
- .await;
+ .instrument(span)
+ .await;
match result {
Ok(_) => Ok(()),
@@ -213,3 +238,34 @@ pub async fn run_worker_with_cancellation(
}
}
}
+
+/// Runs `work` while calling `renew` every [`VISIBILITY_RENEWAL_INTERVAL`], so a message
+/// that outlives the queue's visibility timeout (a large mailbox purge) isn't redelivered
+/// and processed a second time while the first run is still going.
+async fn while_renewing_visibility<T, R, RenewFuture>(renew: R, work: impl Future<Output = T>) -> T
+where
+ R: FnMut() -> RenewFuture,
+ RenewFuture: Future<Output = anyhow::Result<()>>,
+{
+ tokio::select! {
+ output = work => output,
+ never = renew_visibility(renew) => match never {},
+ }
+}
+
+async fn renew_visibility<R, RenewFuture>(mut renew: R) -> Infallible
+where
+ R: FnMut() -> RenewFuture,
+ RenewFuture: Future<Output = anyhow::Result<()>>,
+{
+ let mut ticks = interval_at(
+ Instant::now() + VISIBILITY_RENEWAL_INTERVAL,
+ VISIBILITY_RENEWAL_INTERVAL,
+ );
+ ticks.set_missed_tick_behavior(MissedTickBehavior::Delay);
+ loop {
+ ticks.tick().await;
+ // A failed renewal is retried on the next tick; the work keeps running either way.
+ renew().await.ok();
+ }
+}
diff --git a/services/email_service/src/pubsub/link_manager/worker/test.rs b/services/email_service/src/pubsub/link_manager/worker/test.rs
new file mode 100644
index 0000000000..c1e40e6ffa
--- /dev/null
+++ b/services/email_service/src/pubsub/link_manager/worker/test.rs
@@ -0,0 +1,111 @@
+use super::*;
+use std::sync::atomic::{AtomicU32, Ordering};
+
+#[tokio::test(start_paused = true)]
+async fn renews_visibility_while_work_outlasts_the_timeout() {
+ let renewals = AtomicU32::new(0);
+
+ let output = while_renewing_visibility(
+ || async {
+ renewals.fetch_add(1, Ordering::SeqCst);
+ Ok(())
+ },
+ async {
+ tokio::time::sleep(Duration::from_secs(150)).await;
+ "purged"
+ },
+ )
+ .await;
+
+ assert_eq!(output, "purged");
+ assert_eq!(renewals.load(Ordering::SeqCst), 2);
+}
+
+#[tokio::test(start_paused = true)]
+async fn failed_renewals_do_not_stop_the_work_or_later_renewals() {
+ let renewals = AtomicU32::new(0);
+
+ let output = while_renewing_visibility(
+ || async {
+ if renewals.fetch_add(1, Ordering::SeqCst) == 0 {
+ anyhow::bail!("throttled");
+ }
+ Ok(())
+ },
+ async {
+ tokio::time::sleep(Duration::from_secs(150)).await;
+ "purged"
+ },
+ )
+ .await;
+
+ assert_eq!(output, "purged");
+ assert_eq!(renewals.load(Ordering::SeqCst), 2);
+}
+
+#[tokio::test(start_paused = true)]
+async fn quick_work_does_not_renew() {
+ let renewals = AtomicU32::new(0);
+
+ while_renewing_visibility(
+ || async {
+ renewals.fetch_add(1, Ordering::SeqCst);
+ Ok(())
+ },
+ async {},
+ )
+ .await;
+
+ assert_eq!(renewals.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test(start_paused = true)]
+async fn failed_work_stops_renewing_so_the_message_is_redelivered() {
+ let renewals = AtomicU32::new(0);
+
+ let output = while_renewing_visibility(
+ || async {
+ renewals.fetch_add(1, Ordering::SeqCst);
+ Ok(())
+ },
+ async {
+ tokio::time::sleep(Duration::from_secs(90)).await;
+ Err::<(), _>(anyhow::anyhow!("purge batch failed"))
+ },
+ )
+ .await;
+ tokio::time::advance(Duration::from_secs(600)).await;
+
+ assert!(output.is_err());
+ assert_eq!(renewals.load(Ordering::SeqCst), 1);
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_crashed_worker_stops_renewing() {
+ let renewals = Arc::new(AtomicU32::new(0));
+
+ let worker = tokio::spawn({
+ let renewals = renewals.clone();
+ async move {
+ while_renewing_visibility(
+ move || {
+ let renewals = renewals.clone();
+ async move {
+ renewals.fetch_add(1, Ordering::SeqCst);
+ Ok(())
+ }
+ },
+ async {
+ tokio::time::sleep(Duration::from_secs(150)).await;
+ panic!("worker crashed mid-purge");
+ },
+ )
+ .await
+ }
+ });
+
+ assert!(worker.await.expect_err("the worker panicked").is_panic());
+ tokio::time::advance(Duration::from_secs(600)).await;
+
+ assert_eq!(renewals.load(Ordering::SeqCst), 2);
+}