Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): pace mailbox purges and stop duplicate concurrent deletes (#8410)
teoa3021c6634db
diff --git a/Cargo.lock b/Cargo.lockindex 799e4a3e9d..ea4348b189 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -7323,6 +7323,7 @@ dependencies = ["sqlx","tokio","tracing",+ "tracing-subscriber","workspace-hack",]diff --git a/crates/email/src/outbound/email_pg_repo/test/send_attempt/deferred_deletion.rs b/crates/email/src/outbound/email_pg_repo/test/send_attempt/deferred_deletion.rsindex 4d81b9fe17..7cc1b03000 100644--- a/crates/email/src/outbound/email_pg_repo/test/send_attempt/deferred_deletion.rs+++ b/crates/email/src/outbound/email_pg_repo/test/send_attempt/deferred_deletion.rs@@ -288,7 +288,7 @@ async fn explicit_inbox_purge_removes_an_accepted_send_and_its_retained_deletionlet summary = email_db_client::links::delete::purge_link(&pool,link(),- email_db_client::links::delete::PurgeBatchSizes::default(),+ email_db_client::links::delete::PurgePacing::default(),).await?;// The fixture has four messages in this inbox and one in a second inbox.diff --git a/crates/email_db_client/Cargo.toml b/crates/email_db_client/Cargo.tomlindex 9adfdae7db..c420298302 100644--- a/crates/email_db_client/Cargo.toml+++ b/crates/email_db_client/Cargo.toml@@ -24,3 +24,4 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]macro_db_migrator = { path = "../macro_db_migrator" }+tracing-subscriber = { workspace = true }diff --git a/crates/email_db_client/src/links/delete.rs b/crates/email_db_client/src/links/delete.rsindex 58286c3840..0ba5d36a4b 100644--- a/crates/email_db_client/src/links/delete.rs+++ b/crates/email_db_client/src/links/delete.rs@@ -1,14 +1,17 @@use sqlx::types::Uuid;use sqlx::{PgPool, Postgres, Transaction};use std::num::NonZeroU32;+use std::time::Duration;#[cfg(test)]mod test;-/// How many rows each [`purge_link`] transaction removes. Every batch commits on its own,-/// so no single transaction holds locks, WAL or the xmin horizon for a whole mailbox.+/// How fast [`purge_link`] removes a link's data. Every batch commits on its own, so no+/// single transaction holds locks, WAL or the xmin horizon for a whole mailbox, and the+/// pause between batches caps how much write I/O one purge generates, so several purges+/// running at once can't saturate the database.#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub struct PurgeBatchSizes {+pub struct PurgePacing {/// Messages per transaction, deleted together with their labels, recipients,/// attachments and scheduled sends, and with the threads the batch finishes. Message/// bodies are most of a mailbox's bytes (TOAST), so this is the knob that bounds a@@ -18,14 +21,17 @@ pub struct PurgeBatchSizes {pub threads: NonZeroU32,/// Contacts per transaction.pub contacts: NonZeroU32,+ /// Wait after a full batch before the next one starts.+ pub pause: Duration,}-impl Default for PurgeBatchSizes {+impl Default for PurgePacing {fn default() -> Self {Self {- messages: NonZeroU32::new(1_000).expect("non-zero"),+ messages: NonZeroU32::new(500).expect("non-zero"),threads: NonZeroU32::new(2_000).expect("non-zero"),contacts: NonZeroU32::new(5_000).expect("non-zero"),+ pause: Duration::from_secs(2),}}}@@ -69,8 +75,8 @@ pub async fn mark_link_deleting(pool: &PgPool, link_id: Uuid) -> anyhow::Result</// Deletes a link and all of its email data incrementally: messages (deepest children/// first) with the threads they belong to, then any remaining threads, then contacts,-/// each in bounded short transactions, and finally the `email_links` row, whose cascade-/// only has small per-link tables left to clear.+/// each in bounded short transactions paced by `pacing`, and finally the `email_links`+/// row, whose cascade only has small per-link tables left to clear.////// Marks the link via [`mark_link_deleting`] first, so it can be called on its own. Safe to/// rerun after a crash (it resumes from whatever is left) and to run concurrently for the@@ -79,46 +85,124 @@ pub async fn mark_link_deleting(pool: &PgPool, link_id: Uuid) -> anyhow::Result<pub async fn purge_link(pool: &PgPool,link_id: Uuid,- batch_sizes: PurgeBatchSizes,+ pacing: PurgePacing,) -> anyhow::Result<PurgeSummary> {mark_link_deleting(pool, link_id).await?;let mut summary = PurgeSummary::default();+ (summary.messages, summary.threads) = purge_messages(pool, link_id, pacing).await?;+ summary.threads += purge_threads(pool, link_id, pacing).await?;+ summary.contacts = purge_contacts(pool, link_id, pacing).await?;+ summary.link_deleted = delete_marked_link(pool, link_id).await?;++ tracing::info!(?summary, "Purged email link");+ Ok(summary)+}+/// Deletes the link's messages with the threads each batch finishes, returning how many+/// of each it deleted.+#[tracing::instrument(+ name = "email.purge_link.messages",+ skip_all,+ fields(+ batches = tracing::field::Empty,+ paused_ms = tracing::field::Empty,+ messages = tracing::field::Empty,+ threads = tracing::field::Empty,+ ),+ err+)]+async fn purge_messages(+ pool: &PgPool,+ link_id: Uuid,+ pacing: PurgePacing,+) -> anyhow::Result<(u64, u64)> {+ let (mut batches, mut messages, mut threads) = (0, 0, 0);let mut thread_cursor = Uuid::nil();loop {- let batch = purge_message_batch(pool, link_id, thread_cursor, batch_sizes.messages).await?;- summary.messages += batch.messages;- summary.threads += batch.threads;+ let batch = purge_message_batch(pool, link_id, thread_cursor, pacing.messages).await?;+ batches += 1;+ messages += batch.messages;+ threads += batch.threads;let Some(next_cursor) = batch.next_cursor else {break;};thread_cursor = next_cursor;+ tokio::time::sleep(pacing.pause).await;}+ let span = record_phase(batches, pacing);+ span.record("messages", messages);+ span.record("threads", threads);+ Ok((messages, threads))+}++/// Deletes the threads left once every message is gone.+#[tracing::instrument(+ name = "email.purge_link.threads",+ skip_all,+ fields(+ batches = tracing::field::Empty,+ paused_ms = tracing::field::Empty,+ deleted = tracing::field::Empty,+ ),+ err+)]+async fn purge_threads(pool: &PgPool, link_id: Uuid, pacing: PurgePacing) -> anyhow::Result<u64> {+ let (mut batches, mut deleted) = (0, 0);loop {- let batch = purge_thread_batch(pool, link_id, batch_sizes.threads).await?;- summary.threads += batch.deleted;+ let batch = purge_thread_batch(pool, link_id, pacing.threads).await?;+ batches += 1;+ deleted += batch.deleted;if !batch.full {break;}+ tokio::time::sleep(pacing.pause).await;}+ record_phase(batches, pacing).record("deleted", deleted);+ Ok(deleted)+}++/// Deletes the link's contacts.+#[tracing::instrument(+ name = "email.purge_link.contacts",+ skip_all,+ fields(+ batches = tracing::field::Empty,+ paused_ms = tracing::field::Empty,+ deleted = tracing::field::Empty,+ ),+ err+)]+async fn purge_contacts(pool: &PgPool, link_id: Uuid, pacing: PurgePacing) -> anyhow::Result<u64> {+ let (mut batches, mut deleted) = (0, 0);let mut contact_cursor = String::new();loop {- let batch =- purge_contact_batch(pool, link_id, &contact_cursor, batch_sizes.contacts).await?;- summary.contacts += batch.deleted;+ let batch = purge_contact_batch(pool, link_id, &contact_cursor, pacing.contacts).await?;+ batches += 1;+ deleted += batch.deleted;let Some(next_cursor) = batch.next_cursor else {break;};contact_cursor = next_cursor;+ tokio::time::sleep(pacing.pause).await;}- summary.link_deleted = delete_marked_link(pool, link_id).await?;+ record_phase(batches, pacing).record("deleted", deleted);+ Ok(deleted)+}- tracing::info!(?summary, "Purged email link");- Ok(summary)+/// Records a finished phase's batch count and pause time on its span. Every batch but the+/// last came back full and was followed by a pause.+fn record_phase(batches: u32, pacing: PurgePacing) -> tracing::Span {+ let span = tracing::Span::current();+ span.record("batches", batches);+ span.record(+ "paused_ms",+ (pacing.pause * batches.saturating_sub(1)).as_millis(),+ );+ span}/// Opens a purge batch transaction holding the link's purge lock until commit, so adiff --git a/crates/email_db_client/src/links/delete/test.rs b/crates/email_db_client/src/links/delete/test.rsindex 7e3ddaefd5..0459465a20 100644--- a/crates/email_db_client/src/links/delete/test.rs+++ b/crates/email_db_client/src/links/delete/test.rs@@ -6,6 +6,11 @@ use macro_user_id::email::EmailStr;use macro_user_id::user_id::MacroUserIdStr;use models_email::email::service::link::{Link, UserProvider};use sqlx::{Pool, Postgres};+use std::collections::HashMap;+use std::sync::{Arc, Mutex};+use tracing::field::{Field, Visit};+use tracing_subscriber::layer::{Context, SubscriberExt as _};+use tracing_subscriber::registry::LookupSpan;/// The fixture link that tests purge.const PURGED: Uuid = Uuid::from_u128(0xa001);@@ -122,16 +127,17 @@ async fn deletion_state(pool: &Pool<Postgres>, link_id: Uuid) -> anyhow::Result<.await?)}-fn batch_sizes(messages: u32, threads: u32, contacts: u32) -> PurgeBatchSizes {- PurgeBatchSizes {+fn batch_sizes(messages: u32, threads: u32, contacts: u32) -> PurgePacing {+ PurgePacing {messages: NonZeroU32::new(messages).expect("non-zero"),threads: NonZeroU32::new(threads).expect("non-zero"),contacts: NonZeroU32::new(contacts).expect("non-zero"),+ pause: Duration::ZERO,}}/// Small enough that every phase of the fixture takes several batches, with a partial last one.-fn tiny_batches() -> PurgeBatchSizes {+fn tiny_batches() -> PurgePacing {batch_sizes(4, 3, 3)}@@ -177,6 +183,129 @@ async fn purge_removes_all_link_data_in_batches(pool: Pool<Postgres>) -> anyhow:Ok(())}+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("purge_link"))+)]+async fn purge_pauses_between_batches(pool: Pool<Postgres>) -> anyhow::Result<()> {+ // 30 messages in batches of 20 take one full batch and one partial one; threads and+ // contacts each fit in a single partial batch. So the purge pauses exactly once.+ let pause = Duration::from_secs(1);+ let started = std::time::Instant::now();++ let summary = purge_link(+ &pool,+ PURGED,+ PurgePacing {+ pause,+ ..batch_sizes(20, 100, 100)+ },+ )+ .await?;++ assert!(started.elapsed() >= pause);+ assert!(summary.link_deleted);+ assert_eq!(link_rows(&pool, PURGED).await?, LinkRows::EMPTY);+ assert_eq!(link_rows(&pool, KEPT).await?, LinkRows::FULL);++ Ok(())+}++/// The integer fields recorded on each span while installed, by span and field name.+#[derive(Clone, Default)]+struct RecordedFields(Arc<Mutex<HashMap<&'static str, HashMap<&'static str, u128>>>>);++impl RecordedFields {+ fn of(&self, span: &str) -> HashMap<&'static str, u128> {+ let spans = self.0.lock().expect("recorded fields lock poisoned");+ spans.get(span).cloned().unwrap_or_default()+ }+}++impl<S> tracing_subscriber::Layer<S> for RecordedFields+where+ S: tracing::Subscriber + for<'a> LookupSpan<'a>,+{+ fn on_record(+ &self,+ id: &tracing::span::Id,+ values: &tracing::span::Record<'_>,+ ctx: Context<'_, S>,+ ) {+ let Some(span) = ctx.span(id) else {+ return;+ };+ let mut spans = self.0.lock().expect("recorded fields lock poisoned");+ values.record(&mut IntegerFields(spans.entry(span.name()).or_default()));+ }+}++struct IntegerFields<'a>(&'a mut HashMap<&'static str, u128>);++impl Visit for IntegerFields<'_> {+ fn record_u64(&mut self, field: &Field, value: u64) {+ self.0.insert(field.name(), value.into());+ }++ fn record_u128(&mut self, field: &Field, value: u128) {+ self.0.insert(field.name(), value);+ }++ fn record_debug(&mut self, _field: &Field, _value: &dyn std::fmt::Debug) {}+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("purge_link"))+)]+async fn purge_phases_record_their_batches_and_pauses(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let recorded = RecordedFields::default();+ let _guard =+ tracing::subscriber::set_default(tracing_subscriber::registry().with(recorded.clone()));+ tracing::callsite::rebuild_interest_cache();+ let pause = Duration::from_millis(5);++ let summary = purge_link(+ &pool,+ PURGED,+ PurgePacing {+ pause,+ ..tiny_batches()+ },+ )+ .await?;++ // 30 messages in batches of 4 take 7 full batches and a partial one. A full batch keeps+ // its last thread even when it emptied it, so they finish 8 of the 11 threads, and the+ // thread phase takes the other 3 in one full batch and an empty one. 4 contacts in+ // batches of 3 take a full batch and a partial one. Every full batch is followed by a+ // pause.+ let messages = recorded.of("email.purge_link.messages");+ assert_eq!(messages["batches"], 8);+ assert_eq!(messages["paused_ms"], 7 * pause.as_millis());+ assert_eq!(messages["messages"], 30);+ assert_eq!(messages["threads"], 8);+ let threads = recorded.of("email.purge_link.threads");+ assert_eq!(threads["batches"], 2);+ assert_eq!(threads["paused_ms"], pause.as_millis());+ assert_eq!(threads["deleted"], 3);+ let contacts = recorded.of("email.purge_link.contacts");+ assert_eq!(contacts["batches"], 2);+ assert_eq!(contacts["paused_ms"], pause.as_millis());+ assert_eq!(contacts["deleted"], 4);+ assert_eq!(+ summary,+ PurgeSummary {+ messages: 30,+ threads: 11,+ contacts: 4,+ link_deleted: true,+ }+ );++ Ok(())+}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS",fixtures(path = "../../../fixtures", scripts("purge_link"))@@ -299,7 +428,7 @@ async fn sync_does_not_resume_on_a_link_being_deleted(pool: Pool<Postgres>) -> aassert_eq!(state.deleting_at, Some(deleting_at));assert!(- purge_link(&pool, link.id, PurgeBatchSizes::default())+ purge_link(&pool, link.id, PurgePacing::default()).await?.link_deleted);@@ -320,7 +449,7 @@ async fn purge_of_a_missing_link_is_a_no_op(pool: Pool<Postgres>) -> anyhow::Resmark_link_deleting(&pool, missing).await?;assert_eq!(- purge_link(&pool, missing, PurgeBatchSizes::default()).await?,+ purge_link(&pool, missing, PurgePacing::default()).await?,PurgeSummary::default());diff --git a/crates/sqs_worker/src/change_message_visibility.rs b/crates/sqs_worker/src/change_message_visibility.rsnew file mode 100644index 0000000000..8430f61c08--- /dev/null+++ b/crates/sqs_worker/src/change_message_visibility.rs@@ -0,0 +1,16 @@+pub async fn change_message_visibility(+ inner: &aws_sdk_sqs::Client,+ queue_url: &str,+ receipt_handle: &str,+ visibility_timeout_seconds: i32,+) -> anyhow::Result<()> {+ inner+ .change_message_visibility()+ .queue_url(queue_url)+ .receipt_handle(receipt_handle)+ .visibility_timeout(visibility_timeout_seconds)+ .send()+ .await?;++ Ok(())+}diff --git a/crates/sqs_worker/src/lib.rs b/crates/sqs_worker/src/lib.rsindex c9c3527b7e..b15a4d026e 100644--- a/crates/sqs_worker/src/lib.rs+++ b/crates/sqs_worker/src/lib.rs@@ -1,3 +1,4 @@+mod change_message_visibility;mod delete_message;mod receive_messages;@@ -67,14 +68,36 @@ impl SQSWorker {receipt_handle: &str,timeout_seconds: i32,) -> anyhow::Result<()> {- self.inner- .change_message_visibility()- .queue_url(&self.queue_url)- .receipt_handle(receipt_handle)- .visibility_timeout(timeout_seconds)- .send()- .await?;- Ok(())+ change_message_visibility::change_message_visibility(+ &self.inner,+ &self.queue_url,+ receipt_handle,+ timeout_seconds,+ )+ .await+ }++ /// Hides a received message from other consumers for `seconds` from now.+ #[tracing::instrument(+ skip(self, message),+ fields(message_id = message.message_id),+ err(level = tracing::Level::WARN)+ )]+ pub async fn extend_visibility(+ &self,+ message: &aws_sdk_sqs::types::Message,+ seconds: i32,+ ) -> anyhow::Result<()> {+ let Some(receipt_handle) = message.receipt_handle.as_ref() else {+ anyhow::bail!("no receipt handle found for message");+ };+ change_message_visibility::change_message_visibility(+ &self.inner,+ &self.queue_url,+ receipt_handle,+ seconds,+ )+ .await}}diff --git a/deny.toml b/deny.tomlindex 43980027b9..0d7f85ede0 100644--- a/deny.toml+++ b/deny.toml@@ -234,6 +234,7 @@ deny = ["document_storage_service", # GraphQL telemetry span and log tests only (dev-dependency)"documents", # Dev logging only"email", # Dev logging only+ "email_db_client", # Purge phase span tests only"genai_telemetry", # GenAI span tests"graphql_email", # Unavailable-thread warning field and span tests only"graphql_entity_mutation", # Batch failure span field tests onlydiff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.mdindex 53171ac9f0..068eb2b90b 100644--- a/services/email_service/AGENTS.md+++ b/services/email_service/AGENTS.md@@ -11,7 +11,8 @@ the gateway `/email` prefix.- `src/main.rs` / `src/lib.rs`: entrypoint and `run()`.- `src/api.rs` + `src/api/context.rs`: HTTP surface and `ApiContext`.- `src/pubsub/`: SQS pubsub workers, one module per concern — `inbox_sync`,- `link_manager` (Gmail re-subscribe), `scheduled` (scheduled sends),+ `link_manager` (Gmail re-subscribe, health checks, inbox deletion),+ `scheduled` (scheduled sends),`sfs_uploader`/`sfs_deleter` (attachment storage), `gmail_ops`, `backfill`,`crm_cleanup`, `worker_lifecycle`.- `src/backfill_outbox.rs`, `src/backfill_init_service.rs`,@@ -153,6 +154,10 @@ the gateway `/email` prefix.- Header addresses that Gmail accepts can fail `MacroUserIdStr::try_from_email`.Skip the bad address with a WARN that omits it (the parse error echoes it)instead of failing the step; see `pubsub::util::build_contact_connections`.+- The `link_manager` worker receives one message at a time and renews its+ visibility while processing. Don't raise its receive batch: messages queued+ behind a long mailbox purge would be redelivered and purged twice.+ `PurgePacing` (`email_db_client::links::delete`) sets the purge's write rate.## Relateddiff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rsindex f73e0d865a..2d8ae35d49 100644--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs@@ -140,10 +140,13 @@ async fn main() -> anyhow::Result<()> {),});+ // One message per receive: messages are processed one at a time and only the one in+ // progress renews its visibility, so the rest of a batch would time out behind a long+ // mailbox purge and be processed again by another worker.let link_manager_worker = sqs_worker::SQSWorker::new(aws_sdk_sqs::Client::new(&gmail_queue_aws_config),link_manager_queue.to_string(),- config.queue_max_messages,+ 1,config.queue_wait_time_seconds,);diff --git a/services/email_service/src/pubsub/link_manager/process.rs b/services/email_service/src/pubsub/link_manager/process.rsindex 6495cef0b4..1991728b8b 100644--- a/services/email_service/src/pubsub/link_manager/process.rs+++ b/services/email_service/src/pubsub/link_manager/process.rs@@ -9,7 +9,7 @@ use email::domain::events::{use email_api_client::domain::models::AccessToken;#[cfg(test)]use email_api_client::domain::models::EmailApiError;-use email_db_client::links::delete::PurgeBatchSizes;+use email_db_client::links::delete::PurgePacing;use model_entity::EntityType;use model_notifications::InboxReauthRequiredMetadata;use models_email::api::refresh::RefreshEmailEvent;@@ -202,7 +202,7 @@ async fn handle_delete(// finally, delete the link and all of its email data in bounded batches; a redelivered// message resumes the purge where an interrupted one stopped- email_db_client::links::delete::purge_link(&ctx.db, link.id, PurgeBatchSizes::default())+ email_db_client::links::delete::purge_link(&ctx.db, link.id, PurgePacing::default()).await.context("Failed to purge link in background task")?;diff --git a/services/email_service/src/pubsub/link_manager/worker.rs b/services/email_service/src/pubsub/link_manager/worker.rsindex 6b634b233a..a6e80640fb 100644--- a/services/email_service/src/pubsub/link_manager/worker.rs+++ b/services/email_service/src/pubsub/link_manager/worker.rs@@ -10,8 +10,22 @@ use contacts::{domain::service::SqsContactsIngress, outbound::ingress::SqsContacuse futures::StreamExt;use sqlx::PgPool;use sqs_client::SQS;+use std::convert::Infallible;+use std::future::Future;use std::sync::Arc;+use std::time::Duration;+use tokio::time::{Instant, MissedTickBehavior, interval_at};use tokio_util::sync::CancellationToken;+use tracing::Instrument;++#[cfg(test)]+mod test;++/// How long each renewal hides a message being processed from other workers.+const VISIBILITY_EXTENSION_SECONDS: i32 = 300;+/// How often a message being processed renews its visibility. Several renewals fit in one+/// extension, so a failed renewal doesn't expose the message.+const VISIBILITY_RENEWAL_INTERVAL: Duration = Duration::from_secs(60);/// method that ingests sqs messages and calls the process function for each#[allow(clippy::too_many_arguments)]@@ -144,12 +158,23 @@ pub async fn run_worker_with_cancellation(let result = futures::stream::iter(messages.iter()).then(|message| {let ctx = ctx.clone();+ let worker = &worker;+ let span = tracing::info_span!(+ "email.link_manager.message",+ message_id = message.message_id.as_deref(),+ );async move {- let result = process::process_message(- ctx,- message,+ let result = while_renewing_visibility(+ || {+ worker.extend_visibility(+ message,+ VISIBILITY_EXTENSION_SECONDS,+ )+ },+ process::process_message(ctx, message),)- .await;+ .instrument(span)+ .await;match result {Ok(_) => Ok(()),@@ -213,3 +238,34 @@ pub async fn run_worker_with_cancellation(}}}++/// Runs `work` while calling `renew` every [`VISIBILITY_RENEWAL_INTERVAL`], so a message+/// that outlives the queue's visibility timeout (a large mailbox purge) isn't redelivered+/// and processed a second time while the first run is still going.+async fn while_renewing_visibility<T, R, RenewFuture>(renew: R, work: impl Future<Output = T>) -> T+where+ R: FnMut() -> RenewFuture,+ RenewFuture: Future<Output = anyhow::Result<()>>,+{+ tokio::select! {+ output = work => output,+ never = renew_visibility(renew) => match never {},+ }+}++async fn renew_visibility<R, RenewFuture>(mut renew: R) -> Infallible+where+ R: FnMut() -> RenewFuture,+ RenewFuture: Future<Output = anyhow::Result<()>>,+{+ let mut ticks = interval_at(+ Instant::now() + VISIBILITY_RENEWAL_INTERVAL,+ VISIBILITY_RENEWAL_INTERVAL,+ );+ ticks.set_missed_tick_behavior(MissedTickBehavior::Delay);+ loop {+ ticks.tick().await;+ // A failed renewal is retried on the next tick; the work keeps running either way.+ renew().await.ok();+ }+}diff --git a/services/email_service/src/pubsub/link_manager/worker/test.rs b/services/email_service/src/pubsub/link_manager/worker/test.rsnew file mode 100644index 0000000000..c1e40e6ffa--- /dev/null+++ b/services/email_service/src/pubsub/link_manager/worker/test.rs@@ -0,0 +1,111 @@+use super::*;+use std::sync::atomic::{AtomicU32, Ordering};++#[tokio::test(start_paused = true)]+async fn renews_visibility_while_work_outlasts_the_timeout() {+ let renewals = AtomicU32::new(0);++ let output = while_renewing_visibility(+ || async {+ renewals.fetch_add(1, Ordering::SeqCst);+ Ok(())+ },+ async {+ tokio::time::sleep(Duration::from_secs(150)).await;+ "purged"+ },+ )+ .await;++ assert_eq!(output, "purged");+ assert_eq!(renewals.load(Ordering::SeqCst), 2);+}++#[tokio::test(start_paused = true)]+async fn failed_renewals_do_not_stop_the_work_or_later_renewals() {+ let renewals = AtomicU32::new(0);++ let output = while_renewing_visibility(+ || async {+ if renewals.fetch_add(1, Ordering::SeqCst) == 0 {+ anyhow::bail!("throttled");+ }+ Ok(())+ },+ async {+ tokio::time::sleep(Duration::from_secs(150)).await;+ "purged"+ },+ )+ .await;++ assert_eq!(output, "purged");+ assert_eq!(renewals.load(Ordering::SeqCst), 2);+}++#[tokio::test(start_paused = true)]+async fn quick_work_does_not_renew() {+ let renewals = AtomicU32::new(0);++ while_renewing_visibility(+ || async {+ renewals.fetch_add(1, Ordering::SeqCst);+ Ok(())+ },+ async {},+ )+ .await;++ assert_eq!(renewals.load(Ordering::SeqCst), 0);+}++#[tokio::test(start_paused = true)]+async fn failed_work_stops_renewing_so_the_message_is_redelivered() {+ let renewals = AtomicU32::new(0);++ let output = while_renewing_visibility(+ || async {+ renewals.fetch_add(1, Ordering::SeqCst);+ Ok(())+ },+ async {+ tokio::time::sleep(Duration::from_secs(90)).await;+ Err::<(), _>(anyhow::anyhow!("purge batch failed"))+ },+ )+ .await;+ tokio::time::advance(Duration::from_secs(600)).await;++ assert!(output.is_err());+ assert_eq!(renewals.load(Ordering::SeqCst), 1);+}++#[tokio::test(start_paused = true)]+async fn a_crashed_worker_stops_renewing() {+ let renewals = Arc::new(AtomicU32::new(0));++ let worker = tokio::spawn({+ let renewals = renewals.clone();+ async move {+ while_renewing_visibility(+ move || {+ let renewals = renewals.clone();+ async move {+ renewals.fetch_add(1, Ordering::SeqCst);+ Ok(())+ }+ },+ async {+ tokio::time::sleep(Duration::from_secs(150)).await;+ panic!("worker crashed mid-purge");+ },+ )+ .await+ }+ });++ assert!(worker.await.expect_err("the worker panicked").is_panic());+ tokio::time::advance(Duration::from_secs(600)).await;++ assert_eq!(renewals.load(Ordering::SeqCst), 2);+}