Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): do not strand a send whose enqueue fails (#6580)
gbirmanea1f4047d773
diff --git a/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.json b/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.jsonnew file mode 100644index 0000000000..92cdd72f4b--- /dev/null+++ b/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "\n DELETE FROM email_scheduled_messages\n WHERE link_id = $1 AND message_id = $2 AND sent = false\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b"+}diff --git a/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.json b/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.jsondeleted file mode 100644index 5c0001485c..0000000000--- a/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.json+++ /dev/null@@ -1,26 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n SELECT\n esm.link_id, esm.message_id\n FROM email_scheduled_messages esm\n JOIN email_messages em ON em.id = esm.message_id\n WHERE\n esm.send_time < now()\n AND esm.sent = FALSE\n AND em.is_draft = TRUE\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "link_id",- "type_info": "Uuid"- },- {- "ordinal": 1,- "name": "message_id",- "type_info": "Uuid"- }- ],- "parameters": {- "Left": []- },- "nullable": [- false,- false- ]- },- "hash": "2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01"-}diff --git a/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.json b/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.jsonnew file mode 100644index 0000000000..127c7e74ba--- /dev/null+++ b/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT processing FROM email_scheduled_messages\n WHERE link_id = $1 AND message_id = $2\n FOR UPDATE\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "processing",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351"+}diff --git a/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.json b/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.jsonnew file mode 100644index 0000000000..7e470f50d3--- /dev/null+++ b/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE email_messages\n SET is_draft = true, is_sent = false, updated_at = NOW()\n WHERE id = $1 AND link_id = $2 AND is_sent = false\n RETURNING thread_id\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "thread_id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe"+}diff --git a/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.json b/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.jsonnew file mode 100644index 0000000000..b641ef3d21--- /dev/null+++ b/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.json@@ -0,0 +1,34 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT\n esm.link_id,\n esm.message_id,\n NOT em.is_draft AS \"stranded!\"\n FROM email_scheduled_messages esm\n JOIN email_messages em ON em.id = esm.message_id\n WHERE\n esm.send_time < now()\n AND esm.sent = FALSE\n AND em.is_sent = FALSE\n AND (\n em.is_draft = TRUE\n OR (\n esm.processing = FALSE\n AND esm.send_time < now() - make_interval(secs => $1)\n )\n )\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "link_id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "message_id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 2,+ "name": "stranded!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Float8"+ ]+ },+ "nullable": [+ false,+ false,+ null+ ]+ },+ "hash": "c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538"+}diff --git a/crates/email/src/domain/ports.rs b/crates/email/src/domain/ports.rsindex cbdf85c9b3..fb95ce91dd 100644--- a/crates/email/src/domain/ports.rs+++ b/crates/email/src/domain/ports.rs@@ -269,6 +269,16 @@ pub trait EmailRepo: Send + Sync + 'static {is_draft: bool,) -> impl Future<Output = Result<(), Self::Err>> + Send;+ /// Undo an `insert_message(.., is_draft = false)` whose send never made it+ /// onto a queue: put the message back to an unsent draft, drop its pending+ /// scheduled row, and recompute the thread's denormalized metadata.+ /// Leaves a message the scheduled worker already delivered untouched.+ fn revert_sent_message_to_draft(+ &self,+ message_id: Uuid,+ link_id: Uuid,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;+/// Fetch a label by its database ID and link ID.fn get_label_by_id(&self,diff --git a/crates/email/src/domain/service/send.rs b/crates/email/src/domain/service/send.rsindex 5d19639aa3..b154d4cca9 100644--- a/crates/email/src/domain/service/send.rs+++ b/crates/email/src/domain/service/send.rs@@ -38,10 +38,37 @@ where// FE displays "Undo" button for delay_secs. Give extra time for round trip of cancel requestlet sqs_delay = delay_secs as i32 + 2;- self.enqueuer+ // The insert above already committed the message as non-draft with a+ // pending scheduled row. The queue message is the only thing that will+ // ever deliver it — nothing re-derives the send from the row — so an+ // enqueue failure leaves a message that is no longer a draft, was never+ // sent, and has no worker coming for it. Put it back to an unsent draft+ // so the body survives and the caller can retry, and fail loudly.+ if let Err(e) = self+ .enqueuer.enqueue_scheduled_message(link.id, created.db_id, Some(sqs_delay)).await- .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)))?;+ {+ let err = anyhow::Error::from(e);+ tracing::error!(error=?err, "failed to enqueue scheduled send, reverting the message to a draft");+ if let Err(revert_err) = self+ .email_repo+ .revert_sent_message_to_draft(created.db_id, link.id)+ .await+ {+ // Both writes failed, so the row is still a non-draft unsent+ // send with a pending scheduled row. That is what the+ // email_scheduled_handler sweep re-enqueues once the row is+ // past its grace period.+ tracing::error!(+ error=?anyhow::Error::from(revert_err),+ link_id=%link.id,+ message_id=%created.db_id,+ "failed to revert the message after a failed enqueue; leaving it for the scheduled sweep",+ );+ }+ return Err(EmailErr::EnqueueErr(err));+ }// The undo-window send is now committed and queued; the matching// message_sent / message_send_cancelled event resolves it later,diff --git a/crates/email/src/outbound/email_pg_repo.rs b/crates/email/src/outbound/email_pg_repo.rsindex 9e94094da8..07cf9dfcd5 100644--- a/crates/email/src/outbound/email_pg_repo.rs+++ b/crates/email/src/outbound/email_pg_repo.rs@@ -284,6 +284,14 @@ impl EmailRepo for EmailPgRepo {draft::insert_message(&self.pool, input, contacts, link_id, new_thread, is_draft).await}+ async fn revert_sent_message_to_draft(+ &self,+ message_id: Uuid,+ link_id: Uuid,+ ) -> Result<(), Self::Err> {+ draft::revert_sent_message_to_draft(&self.pool, message_id, link_id).await+ }+async fn get_label_by_id(&self,label_id: Uuid,diff --git a/crates/email/src/outbound/email_pg_repo/draft.rs b/crates/email/src/outbound/email_pg_repo/draft.rsindex b68c2c84f0..e6b4305fea 100644--- a/crates/email/src/outbound/email_pg_repo/draft.rs+++ b/crates/email/src/outbound/email_pg_repo/draft.rs@@ -60,6 +60,87 @@ pub(crate) async fn insert_message(Ok(())}+/// Revert a message that `insert_message(.., is_draft = false)` optimistically+/// marked as outgoing back to an unsent draft, and drop its pending scheduled+/// row.+///+/// The whole revert hinges on the send not having happened: a message the+/// scheduled worker already delivered (`is_sent`), or is delivering right now+/// (`processing` on its scheduled row), is left exactly as it is, message row+/// and scheduled row both. `get_message_to_send` does not re-check `is_draft`,+/// so a worker that has claimed the row sends the mail whatever this does —+/// reverting underneath it would deliver the email *and* leave a live draft to+/// send again.+#[tracing::instrument(skip(pool), err)]+pub(crate) async fn revert_sent_message_to_draft(+ pool: &PgPool,+ message_id: Uuid,+ link_id: Uuid,+) -> Result<(), sqlx::Error> {+ let mut tx = pool.begin().await?;++ // Lock the scheduled row up front rather than testing `processing` inside+ // the UPDATE below. A worker's claim is a single committed statement, so an+ // unlocked read only proves nothing had claimed the row at that instant: a+ // claim landing before the DELETE would have its row deleted out from under+ // it and still send. Holding the lock makes a concurrent claim wait — it+ // then finds no row and skips — and makes a claim that got here first+ // visible to the check.+ let claimed = sqlx::query_scalar!(+ r#"+ SELECT processing FROM email_scheduled_messages+ WHERE link_id = $1 AND message_id = $2+ FOR UPDATE+ "#,+ link_id,+ message_id,+ )+ .fetch_optional(&mut *tx)+ .await?;++ if claimed == Some(true) {+ tx.commit().await?;+ return Ok(());+ }++ let thread_db_id = sqlx::query_scalar!(+ r#"+ UPDATE email_messages+ SET is_draft = true, is_sent = false, updated_at = NOW()+ WHERE id = $1 AND link_id = $2 AND is_sent = false+ RETURNING thread_id+ "#,+ message_id,+ link_id,+ )+ .fetch_optional(&mut *tx)+ .await?;++ let Some(thread_db_id) = thread_db_id else {+ tx.commit().await?;+ return Ok(());+ };++ sqlx::query!(+ r#"+ DELETE FROM email_scheduled_messages+ WHERE link_id = $1 AND message_id = $2 AND sent = false+ "#,+ link_id,+ message_id,+ )+ .execute(&mut *tx)+ .await?;++ // Drafts count toward inbox_visible and the latest-message timestamps, so+ // the metadata the insert computed for an outgoing message no longer+ // describes the thread. Same recompute delete_draft_message does.+ thread::update_thread_metadata(&mut tx, thread_db_id, link_id).await?;++ tx.commit().await?;+ Ok(())+}+/// Upsert a draft message row.pub(crate) async fn upsert_draft(tx: &mut sqlx::PgConnection,diff --git a/crates/email/src/outbound/email_pg_repo/test.rs b/crates/email/src/outbound/email_pg_repo/test.rsindex cc4284ad1f..f6c77e3f34 100644--- a/crates/email/src/outbound/email_pg_repo/test.rs+++ b/crates/email/src/outbound/email_pg_repo/test.rs@@ -9,6 +9,7 @@ mod message;mod preview;mod project;mod project_scope_dynamic_query;+mod send;mod settings;mod signal_flag;mod thread;diff --git a/crates/email/src/outbound/email_pg_repo/test/send.rs b/crates/email/src/outbound/email_pg_repo/test/send.rsnew file mode 100644index 0000000000..72fd386295--- /dev/null+++ b/crates/email/src/outbound/email_pg_repo/test/send.rs@@ -0,0 +1,485 @@+//! Durability of the undo-window send path.+//!+//! `send_message_impl` commits the message as a non-draft with a pending+//! scheduled row and only then enqueues the delivery, so these tests run the+//! domain service against the real repo with an enqueuer that fails, and pin+//! down what the database is left holding.++use super::*;+use crate::domain::models::{ContactInfo, CreateDraftInput};+use crate::domain::ports::EmailMessageEnqueuer;+use crate::domain::service::EmailServiceImpl;+use crm::domain::service::NoOpCrmService;+use frecency::domain::services::FrecencyQueryServiceImpl;+use frecency::outbound::postgres::FrecencyPgStorage;+use macro_event_broker::NoopMacroEventBroker;+use macro_user_id::email::EmailStr;++const LINK_ID: &str = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";+const SENT_MESSAGE_ID: &str = "ee000001-0000-0000-0000-000000000001";+const DRAFT_MESSAGE_ID: &str = "ee000002-0000-0000-0000-000000000002";+/// The draft that is the only message in its thread, so thread metadata swings+/// entirely on whether it counts as a draft.+const LONE_DRAFT_MESSAGE_ID: &str = "ee000004-0000-0000-0000-000000000004";+const LONE_DRAFT_THREAD_ID: &str = "33333333-3333-3333-3333-333333333333";++/// Stands in for SQS being unreachable — the shape of the outage that stranded+/// sends in production.+#[derive(Clone)]+struct FailingEnqueuer;++impl EmailMessageEnqueuer for FailingEnqueuer {+ type Err = anyhow::Error;++ async fn enqueue_scheduled_message(+ &self,+ _link_id: Uuid,+ _message_id: Uuid,+ _delay_seconds: Option<i32>,+ ) -> Result<(), Self::Err> {+ Err(anyhow::anyhow!("queue unavailable"))+ }++ async fn enqueue_gmail_ops_modify_labels_batch(+ &self,+ _link_id: Uuid,+ _messages: Vec<(Uuid, String)>,+ _labels_to_add: Vec<String>,+ _labels_to_remove: Vec<String>,+ ) -> Result<(), Self::Err> {+ Err(anyhow::anyhow!("queue unavailable"))+ }++ async fn enqueue_gmail_ops_block_sender(+ &self,+ _link_id: Uuid,+ _email_address: String,+ ) -> Result<(), Self::Err> {+ Err(anyhow::anyhow!("queue unavailable"))+ }++ async fn enqueue_gmail_ops_unblock_sender(+ &self,+ _link_id: Uuid,+ _email_address: String,+ ) -> Result<(), Self::Err> {+ Err(anyhow::anyhow!("queue unavailable"))+ }+}++/// The send path touches neither frecency nor CRM nor the event broker, so+/// those collaborators are no-ops; the repo is the real thing.+type TestService<E> = EmailServiceImpl<+ EmailPgRepo,+ FrecencyQueryServiceImpl<FrecencyPgStorage>,+ E,+ NoOpCrmService,+ (),+ NoopMacroEventBroker,+>;++fn service<E: EmailMessageEnqueuer>(pool: Pool<Postgres>, enqueuer: E) -> TestService<E> {+ EmailServiceImpl {+ email_repo: EmailPgRepo::new(pool.clone()),+ frecency_service: FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(pool)),+ enqueuer,+ crm_service: NoOpCrmService,+ entity_access_management_service: (),+ macro_event_broker: NoopMacroEventBroker,+ sent_undo_delay_secs: 10,+ }+}++fn link() -> anyhow::Result<Link> {+ Ok(Link {+ id: Uuid::parse_str(LINK_ID)?,+ macro_id: MacroUserIdStr::try_from_email("[email protected]")?,+ fusionauth_user_id: "fa-user-1".to_owned(),+ email_address: EmailStr::try_from("[email protected]".to_owned())?,+ provider: UserProvider::Gmail,+ is_sync_active: true,+ is_primary: true,+ created_at: Utc::now(),+ updated_at: Utc::now(),+ })+}++fn send_input(db_id: Option<Uuid>) -> CreateDraftInput {+ CreateDraftInput {+ db_id,+ provider_id: None,+ replying_to_id: None,+ provider_thread_id: None,+ thread_db_id: None,+ subject: "Quarterly update".to_owned(),+ to: vec![ContactInfo {+ email: "[email protected]".to_owned(),+ name: None,+ photo_url: None,+ }],+ cc: vec![],+ bcc: vec![],+ body_text: Some("Numbers attached.".to_owned()),+ body_html: None,+ body_macro: None,+ headers_json: None,+ send_time: None,+ include_signature: Some(false),+ actor: None,+ }+}++/// `(is_draft, is_sent)` for a message row.+async fn message_flags(pool: &Pool<Postgres>, message_id: Uuid) -> anyhow::Result<(bool, bool)> {+ let row = sqlx::query!(+ "SELECT is_draft, is_sent FROM email_messages WHERE id = $1",+ message_id,+ )+ .fetch_one(pool)+ .await?;+ Ok((row.is_draft, row.is_sent))+}++async fn scheduled_row_count(pool: &Pool<Postgres>, message_id: Uuid) -> anyhow::Result<i64> {+ let count = sqlx::query_scalar!(+ r#"SELECT COUNT(*) AS "count!" FROM email_scheduled_messages WHERE message_id = $1"#,+ message_id,+ )+ .fetch_one(pool)+ .await?;+ Ok(count)+}++struct ThreadInboxState {+ inbox_visible: bool,+ latest_inbound_message_ts: Option<chrono::DateTime<Utc>>,+}++async fn thread_inbox_state(+ pool: &Pool<Postgres>,+ thread_id: Uuid,+) -> anyhow::Result<ThreadInboxState> {+ let row = sqlx::query!(+ "SELECT inbox_visible, latest_inbound_message_ts FROM email_threads WHERE id = $1",+ thread_id,+ )+ .fetch_one(pool)+ .await?;++ Ok(ThreadInboxState {+ inbox_visible: row.inbox_visible,+ latest_inbound_message_ts: row.latest_inbound_message_ts,+ })+}++// ── send_message_impl ─────────────────────────────────────────────++/// The regression: a failed enqueue used to leave the message flipped to+/// non-draft with a scheduled row and nothing in the queue to deliver it, so+/// the draft vanished from the UI and no mail ever went out. The sweep could+/// not rescue it either, because it only looked at drafts.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("email_draft"))+)]+async fn failed_enqueue_leaves_the_message_as_an_unsent_draft(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link = link()?;+ let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;+ let service = service(pool.clone(), FailingEnqueuer);++ let err = service+ .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))+ .await+ .expect_err("enqueue failed, so the send must fail");++ assert!(+ matches!(err, EmailErr::EnqueueErr(_)),+ "expected EnqueueErr, got {err:?}"+ );++ let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;+ assert!(is_draft, "the draft must survive a failed send");+ assert!(!is_sent);+ assert_eq!(+ scheduled_row_count(&pool, draft_id).await?,+ 0,+ "no scheduled row should be left pointing at a send that will never happen"+ );++ Ok(())+}++/// The agent/AI path sends without an existing draft (`db_id: None`), so the+/// revert has to leave the freshly composed body behind as a draft rather than+/// losing it.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("email_draft"))+)]+async fn failed_enqueue_keeps_a_brand_new_send_as_a_draft(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link = link()?;+ let service = service(pool.clone(), FailingEnqueuer);++ let err = service+ .send_message_impl(&link, &[link.clone()], send_input(None))+ .await+ .expect_err("enqueue failed, so the send must fail");+ assert!(+ matches!(err, EmailErr::EnqueueErr(_)),+ "expected EnqueueErr, got {err:?}"+ );++ let new_message_id: Uuid = sqlx::query_scalar!(+ r#"+ SELECT id FROM email_messages+ WHERE link_id = $1 AND subject = 'Quarterly update'+ "#,+ link.id,+ )+ .fetch_one(&pool)+ .await?;++ let (is_draft, is_sent) = message_flags(&pool, new_message_id).await?;+ assert!(is_draft, "the composed body must survive as a draft");+ assert!(!is_sent);+ assert_eq!(scheduled_row_count(&pool, new_message_id).await?, 0);++ Ok(())+}++/// The happy path still commits the message as outgoing with a pending+/// scheduled row — the revert must not fire when the enqueue succeeds.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("email_draft"))+)]+async fn successful_enqueue_commits_the_send(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let link = link()?;+ let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;+ let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);++ let created = service+ .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))+ .await?;+ assert_eq!(created.db_id, draft_id);++ let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;+ assert!(!is_draft, "an enqueued send is no longer a draft");+ assert!(+ !is_sent,+ "the worker marks it sent once Gmail accepts it, not here"+ );+ assert_eq!(scheduled_row_count(&pool, draft_id).await?, 1);++ Ok(())+}++// ── revert_sent_message_to_draft ──────────────────────────────────++/// Drafts count toward a thread's denormalized `inbox_visible` and+/// latest-message timestamps, so the metadata the insert computed for an+/// outgoing message has to be recomputed on the way back.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("email_draft"))+)]+async fn revert_recomputes_thread_metadata(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let link = link()?;+ let draft_id = Uuid::parse_str(LONE_DRAFT_MESSAGE_ID)?;+ let thread_id = Uuid::parse_str(LONE_DRAFT_THREAD_ID)?;+ let repo = EmailPgRepo::new(pool.clone());+ let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);++ service+ .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))+ .await?;++ let sent = thread_inbox_state(&pool, thread_id).await?;+ assert!(+ !sent.inbox_visible && sent.latest_inbound_message_ts.is_none(),+ "the thread's only message is outgoing, so nothing keeps it in the inbox"+ );++ repo.revert_sent_message_to_draft(draft_id, link.id).await?;++ let reverted = thread_inbox_state(&pool, thread_id).await?;+ assert!(+ reverted.inbox_visible,+ "a thread holding a draft again belongs back in the inbox"+ );+ assert!(reverted.latest_inbound_message_ts.is_some());++ Ok(())+}++/// Losing a queue message must never be able to un-send mail that Gmail has+/// already accepted, so the revert is a no-op once the worker has marked the+/// message sent.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("email_draft"))+)]+async fn revert_leaves_an_already_delivered_message_alone(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link_id = Uuid::parse_str(LINK_ID)?;+ let sent_id = Uuid::parse_str(SENT_MESSAGE_ID)?;+ let repo = EmailPgRepo::new(pool.clone());++ sqlx::query!(+ r#"+ INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, created_at, updated_at)+ VALUES ($1, $2, NOW(), false, NOW(), NOW())+ "#,+ link_id,+ sent_id,+ )+ .execute(&pool)+ .await?;++ repo.revert_sent_message_to_draft(sent_id, link_id).await?;++ let (is_draft, is_sent) = message_flags(&pool, sent_id).await?;+ assert!(!is_draft, "a delivered message must not become a draft");+ assert!(is_sent);+ assert_eq!(+ scheduled_row_count(&pool, sent_id).await?,+ 1,+ "the revert bails before touching the scheduled row"+ );++ Ok(())+}++/// A worker that has claimed the scheduled row will deliver the mail whether or+/// not this revert runs (`get_message_to_send` does not re-check `is_draft`), so+/// reverting underneath it would send the email and leave a live draft behind.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("email_draft"))+)]+async fn revert_leaves_a_message_a_worker_is_delivering_alone(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link = link()?;+ let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;+ let repo = EmailPgRepo::new(pool.clone());+ let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);++ service+ .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))+ .await?;++ sqlx::query!(+ r#"+ UPDATE email_scheduled_messages SET processing = true+ WHERE link_id = $1 AND message_id = $2+ "#,+ link.id,+ draft_id,+ )+ .execute(&pool)+ .await?;++ repo.revert_sent_message_to_draft(draft_id, link.id).await?;++ let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;+ assert!(+ !is_draft,+ "a send in flight must not become a draft that can be sent again"+ );+ assert!(!is_sent);+ assert_eq!(+ scheduled_row_count(&pool, draft_id).await?,+ 1,+ "the worker still needs its row to mark the send as sent"+ );++ Ok(())+}++/// The guard has to be a lock, not a read. A worker's claim is one committed+/// statement, so a revert that merely *read* `processing` could have a claim+/// land between that read and its DELETE, delete the row the worker just+/// claimed, and leave the worker to send from the data it already holds.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("email_draft"))+)]+async fn revert_waits_for_a_claim_racing_it(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let link = link()?;+ let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;+ let repo = EmailPgRepo::new(pool.clone());+ let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);++ service+ .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))+ .await?;++ // A worker claims the row exactly the way the scheduled consumer does, but+ // holds the transaction open so the claim is in flight, not settled.+ let mut claim = pool.begin().await?;+ sqlx::query!(+ r#"+ UPDATE email_scheduled_messages SET processing = true, updated_at = NOW()+ WHERE link_id = $1 AND message_id = $2+ "#,+ link.id,+ draft_id,+ )+ .execute(&mut *claim)+ .await?;++ let mut revert = tokio::spawn({+ let repo = repo.clone();+ let link_id = link.id;+ async move { repo.revert_sent_message_to_draft(draft_id, link_id).await }+ });++ assert!(+ tokio::time::timeout(std::time::Duration::from_millis(500), &mut revert)+ .await+ .is_err(),+ "the revert must wait on the in-flight claim"+ );++ // What the claim lands on is the point. Testing `processing` with an+ // unlocked read lets the revert flip the message to a draft first and only+ // then wait — on the DELETE's row lock — so once the claim commits it+ // deletes the row that claim just took and leaves a live draft behind.+ claim.commit().await?;+ revert.await??;++ let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;+ assert!(+ !is_draft,+ "once the claim lands the revert must see it and leave the send alone"+ );+ assert!(!is_sent);+ assert_eq!(+ scheduled_row_count(&pool, draft_id).await?,+ 1,+ "the claimed row must survive for the worker to mark the send as sent"+ );++ Ok(())+}++/// A message that was never inserted (or belongs to another inbox) must not+/// make the revert fail — it runs on a path that is already handling an error.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../../fixtures", scripts("email_draft"))+)]+async fn revert_is_a_no_op_for_an_unknown_message(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let repo = EmailPgRepo::new(pool);++ repo.revert_sent_message_to_draft(Uuid::from_u128(999), Uuid::parse_str(LINK_ID)?)+ .await?;++ Ok(())+}diff --git a/services/email_scheduled_handler/src/handler.rs b/services/email_scheduled_handler/src/handler.rsindex 6124137641..a6c8900504 100644--- a/services/email_scheduled_handler/src/handler.rs+++ b/services/email_scheduled_handler/src/handler.rs@@ -24,12 +24,25 @@ pub async fn handler(tracing::info!(notifications = ?notifications, "Sending scheduled email pubsub messages");}- for notif in notifications.into_iter() {- let message_id = notif.message_id;- let link_id = notif.link_id;+ for pending in notifications.into_iter() {+ let message_id = pending.message_id;+ let link_id = pending.link_id;++ // A stranded row means the send path committed the message but failed+ // to enqueue it and failed to revert it, so this sweep is the only+ // reason the mail goes out at all. Warn so the underlying failure is+ // visible instead of being quietly papered over.+ if pending.stranded {+ tracing::warn!(+ link_id = link_id.to_string(),+ message_id = message_id.to_string(),+ "Re-enqueueing a send whose original enqueue never landed",+ );+ }+if let Err(e) = ctx.sqs_client- .enqueue_email_scheduled_message(notif, None)+ .enqueue_email_scheduled_message(pending.into(), None).await{tracing::error!(@@ -44,27 +57,73 @@ pub async fn handler(Ok(())}+/// How long past its `send_time` a non-draft scheduled row must sit before the+/// sweep treats it as stranded rather than in flight.+///+/// The undo-window send path (`email`'s `send_message_impl`) inserts its+/// message as a non-draft and puts the queue message on a short delay, so such+/// a row is normally delivered seconds after its `send_time`. The grace period+/// has to clear that delay plus the scheduled queue's whole redelivery cycle+/// (five receives at a 30s visibility timeout) so a rescue can never race a+/// delivery that is still coming and double-send the message.+const STRANDED_SEND_GRACE_SECS: f64 = 300.0;++/// An overdue scheduled send, plus how it reached this sweep.+#[derive(Debug)]+pub struct PendingScheduledSend {+ pub link_id: sqlx::types::Uuid,+ pub message_id: sqlx::types::Uuid,+ /// `true` for an undo-window send this sweep is rescuing, `false` for an+ /// ordinary send-later draft whose delivery this sweep always owned.+ pub stranded: bool,+}++impl From<PendingScheduledSend> for ScheduledPubsubMessage {+ fn from(pending: PendingScheduledSend) -> Self {+ ScheduledPubsubMessage {+ link_id: pending.link_id,+ message_id: pending.message_id,+ }+ }+}+/// Fetches all scheduled messages that are ready to be sent.-/// Only returns messages where:-/// - send_time has passed (< now())-/// - sent = false-/// - the associated email message is still a draft (is_draft = true)+///+/// Returns rows whose `send_time` has passed and that neither the scheduled+/// table (`esm.sent`) nor the message itself (`em.is_sent`) records as sent,+/// in two shapes:+/// - send-later drafts (`em.is_draft = TRUE`), for which this sweep is the+/// only thing that ever enqueues a delivery;+/// - undo-window sends (`em.is_draft = FALSE`), which enqueue their own+/// delivery at send time and so are only picked up once they are+/// [`STRANDED_SEND_GRACE_SECS`] overdue and unclaimed (`processing = FALSE`)+/// — the signature of a send whose enqueue never landed.#[tracing::instrument(skip(pool), err)]pub async fn fetch_pending_scheduled_messages(pool: &PgPool,-) -> anyhow::Result<Vec<ScheduledPubsubMessage>> {+) -> anyhow::Result<Vec<PendingScheduledSend>> {let messages = sqlx::query_as!(- ScheduledPubsubMessage,+ PendingScheduledSend,r#"SELECT- esm.link_id, esm.message_id+ esm.link_id,+ esm.message_id,+ NOT em.is_draft AS "stranded!"FROM email_scheduled_messages esmJOIN email_messages em ON em.id = esm.message_idWHEREesm.send_time < now()AND esm.sent = FALSE- AND em.is_draft = TRUE+ AND em.is_sent = FALSE+ AND (+ em.is_draft = TRUE+ OR (+ esm.processing = FALSE+ AND esm.send_time < now() - make_interval(secs => $1)+ )+ )"#,+ STRANDED_SEND_GRACE_SECS,).fetch_all(pool).await?;diff --git a/services/email_scheduled_handler/src/test.rs b/services/email_scheduled_handler/src/test.rsindex e6aba1e190..2a54a3da45 100644--- a/services/email_scheduled_handler/src/test.rs+++ b/services/email_scheduled_handler/src/test.rs@@ -13,15 +13,17 @@ async fn fetch_pending_returns_only_eligible_messages(pool: Pool<Postgres>) -> Rlet results = fetch_pending_scheduled_messages(&pool).await?;- // Should return exactly 2 messages:+ // Should return exactly 3 messages:// - Message 1: Draft with past send_time, not sent// - Message 5: Draft with past send_time, not sent (different link)- assert_eq!(results.len(), 2);+ // - Message 6: Non-draft undo-window send stranded well past its send_time+ assert_eq!(results.len(), 3);let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f501")?));assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f505")?));+ assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f506")?));Ok(())}@@ -60,13 +62,86 @@ async fn fetch_pending_excludes_already_sent(pool: Pool<Postgres>) -> Result<()>migrator = "MACRO_DB_MIGRATIONS",fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages")))]-async fn fetch_pending_excludes_non_drafts(pool: Pool<Postgres>) -> Result<()> {+async fn fetch_pending_excludes_delivered_messages(pool: Pool<Postgres>) -> Result<()> {let results = fetch_pending_scheduled_messages(&pool).await?;let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();- // Message 4 has is_draft = false, should not be included+ // Messages 4 and 9 both have is_sent = true, so the send already happened+ // and the unsent scheduled row is stale. Re-enqueueing either would send+ // the same mail twice.assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f504")?));+ assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f509")?));++ Ok(())+}++/// The bug this sweep now covers: `send_message_impl` commits the message as a+/// non-draft with a scheduled row and only then enqueues the delivery, so a+/// failed enqueue leaves a message that is not a draft, was never sent, and has+/// no worker coming for it. The old `em.is_draft = TRUE` predicate could not+/// see those rows at all.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))+)]+async fn fetch_pending_includes_stranded_non_draft_sends(pool: Pool<Postgres>) -> Result<()> {+ let results = fetch_pending_scheduled_messages(&pool).await?;++ let rescued = results+ .iter()+ .find(|r| r.message_id == Uuid::parse_str("00000000-0000-0000-0000-00000000f506").unwrap());++ let rescued = rescued.expect("the stranded send should be picked up");+ assert!(+ rescued.stranded,+ "a rescue must be flagged so the underlying send failure is visible in the logs"+ );++ // Ordinary send-later drafts are this sweep's normal work, not rescues.+ let draft = results+ .iter()+ .find(|r| r.message_id == Uuid::parse_str("00000000-0000-0000-0000-00000000f501").unwrap())+ .expect("the send-later draft should be picked up");+ assert!(!draft.stranded);++ Ok(())+}++/// A non-draft send that is only seconds overdue still has its own queue+/// message in flight (or in SQS redelivery), so rescuing it would double-send.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))+)]+async fn fetch_pending_excludes_non_draft_sends_inside_grace_period(+ pool: Pool<Postgres>,+) -> Result<()> {+ let results = fetch_pending_scheduled_messages(&pool).await?;++ let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();++ // Message 7 is 30 seconds overdue, well inside the grace period+ assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f507")?));++ Ok(())+}++/// A worker already claimed the row, so it is being delivered rather than+/// stranded.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))+)]+async fn fetch_pending_excludes_non_draft_sends_being_processed(+ pool: Pool<Postgres>,+) -> Result<()> {+ let results = fetch_pending_scheduled_messages(&pool).await?;++ let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();++ // Message 8 is well overdue but has processing = true+ assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f508")?));Ok(())}diff --git a/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql b/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sqlindex 35a69c4d52..c07602aa38 100644--- a/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql+++ b/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql@@ -110,7 +110,8 @@ VALUES ('00000000-0000-0000-0000-000000000f01',NOW(), NOW());--------------------------------------------------------------- Message 4: NOT a draft (is_draft = false) with past send_time, not sent (SHOULD NOT BE RETURNED)+-- Message 4: NOT a draft, already delivered (is_sent = true), with a stale+-- unsent scheduled row and a past send_time (SHOULD NOT BE RETURNED)------------------------------------------------------------INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,@@ -151,4 +152,99 @@ VALUES ('00000000-0000-0000-0000-000000000f02','00000000-0000-0000-0000-00000000f505','2020-01-01 14:00:00 +00:00', -- Past datefalse,- NOW(), NOW());\ No newline at end of file+ NOW(), NOW());++------------------------------------------------------------+-- Message 6: undo-window send stranded by a failed enqueue — not a draft,+-- not sent, unclaimed, and well past the grace period (SHOULD BE RETURNED)+------------------------------------------------------------++INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,+ has_attachments, is_read, is_starred, is_draft, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-00000000f506',+ '00000000-0000-0000-0000-00000000f201',+ '00000000-0000-0000-0000-000000000f01',+ NULL,+ FALSE,+ '00000000-0000-0000-0000-0000000cf001',+ '2025-01-10 15:00:00 +00:00',+ false, false, false, false, NOW(), NOW()); -- is_draft = false, is_sent = false++INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-000000000f01',+ '00000000-0000-0000-0000-00000000f506',+ NOW() - INTERVAL '10 minutes', -- well past the stranded grace period+ false,+ false,+ NOW(), NOW());++------------------------------------------------------------+-- Message 7: undo-window send inside the grace period — its own queue message+-- is still in flight or being retried (SHOULD NOT BE RETURNED)+------------------------------------------------------------++INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,+ has_attachments, is_read, is_starred, is_draft, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-00000000f507',+ '00000000-0000-0000-0000-00000000f201',+ '00000000-0000-0000-0000-000000000f01',+ NULL,+ FALSE,+ '00000000-0000-0000-0000-0000000cf001',+ '2025-01-10 16:00:00 +00:00',+ false, false, false, false, NOW(), NOW());++INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-000000000f01',+ '00000000-0000-0000-0000-00000000f507',+ NOW() - INTERVAL '30 seconds', -- overdue, but inside the grace period+ false,+ false,+ NOW(), NOW());++------------------------------------------------------------+-- Message 8: overdue undo-window send a worker already claimed+-- (SHOULD NOT BE RETURNED)+------------------------------------------------------------++INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,+ has_attachments, is_read, is_starred, is_draft, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-00000000f508',+ '00000000-0000-0000-0000-00000000f201',+ '00000000-0000-0000-0000-000000000f01',+ NULL,+ FALSE,+ '00000000-0000-0000-0000-0000000cf001',+ '2025-01-10 17:00:00 +00:00',+ false, false, false, false, NOW(), NOW());++INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-000000000f01',+ '00000000-0000-0000-0000-00000000f508',+ NOW() - INTERVAL '10 minutes',+ false,+ true, -- claimed by a worker+ NOW(), NOW());++------------------------------------------------------------+-- Message 9: draft whose message row says it was delivered, with a stale+-- unsent scheduled row (SHOULD NOT BE RETURNED)+------------------------------------------------------------++INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,+ has_attachments, is_read, is_starred, is_draft, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-00000000f509',+ '00000000-0000-0000-0000-00000000f201',+ '00000000-0000-0000-0000-000000000f01',+ 'provider-pending-509',+ TRUE,+ '00000000-0000-0000-0000-0000000cf001',+ '2025-01-10 18:00:00 +00:00',+ false, false, false, true, NOW(), NOW()); -- is_draft = true, is_sent = true++INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, created_at, updated_at)+VALUES ('00000000-0000-0000-0000-000000000f01',+ '00000000-0000-0000-0000-00000000f509',+ '2020-01-01 18:00:00 +00:00',+ false,+ NOW(), NOW());