Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(email): do not strand a send whose enqueue fails (#6580)

gbirmanea1f4047d773
diff --git a/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.json b/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.json
new file mode 100644
index 0000000000..92cdd72f4b
--- /dev/null
+++ b/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n DELETE FROM email_scheduled_messages\n WHERE link_id = $1 AND message_id = $2 AND sent = false\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b"
+}
diff --git a/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.json b/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.json
deleted file mode 100644
index 5c0001485c..0000000000
--- a/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT\n esm.link_id, esm.message_id\n FROM email_scheduled_messages esm\n JOIN email_messages em ON em.id = esm.message_id\n WHERE\n esm.send_time < now()\n AND esm.sent = FALSE\n AND em.is_draft = TRUE\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "link_id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 1,
- "name": "message_id",
- "type_info": "Uuid"
- }
- ],
- "parameters": {
- "Left": []
- },
- "nullable": [
- false,
- false
- ]
- },
- "hash": "2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01"
-}
diff --git a/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.json b/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.json
new file mode 100644
index 0000000000..127c7e74ba
--- /dev/null
+++ b/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT processing FROM email_scheduled_messages\n WHERE link_id = $1 AND message_id = $2\n FOR UPDATE\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "processing",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351"
+}
diff --git a/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.json b/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.json
new file mode 100644
index 0000000000..7e470f50d3
--- /dev/null
+++ b/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE email_messages\n SET is_draft = true, is_sent = false, updated_at = NOW()\n WHERE id = $1 AND link_id = $2 AND is_sent = false\n RETURNING thread_id\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "thread_id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe"
+}
diff --git a/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.json b/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.json
new file mode 100644
index 0000000000..b641ef3d21
--- /dev/null
+++ b/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.json
@@ -0,0 +1,34 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n esm.link_id,\n esm.message_id,\n NOT em.is_draft AS \"stranded!\"\n FROM email_scheduled_messages esm\n JOIN email_messages em ON em.id = esm.message_id\n WHERE\n esm.send_time < now()\n AND esm.sent = FALSE\n AND em.is_sent = FALSE\n AND (\n em.is_draft = TRUE\n OR (\n esm.processing = FALSE\n AND esm.send_time < now() - make_interval(secs => $1)\n )\n )\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "link_id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "message_id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 2,
+ "name": "stranded!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Float8"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ null
+ ]
+ },
+ "hash": "c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538"
+}
diff --git a/crates/email/src/domain/ports.rs b/crates/email/src/domain/ports.rs
index cbdf85c9b3..fb95ce91dd 100644
--- a/crates/email/src/domain/ports.rs
+++ b/crates/email/src/domain/ports.rs
@@ -269,6 +269,16 @@ pub trait EmailRepo: Send + Sync + 'static {
is_draft: bool,
) -> impl Future<Output = Result<(), Self::Err>> + Send;
+ /// Undo an `insert_message(.., is_draft = false)` whose send never made it
+ /// onto a queue: put the message back to an unsent draft, drop its pending
+ /// scheduled row, and recompute the thread's denormalized metadata.
+ /// Leaves a message the scheduled worker already delivered untouched.
+ fn revert_sent_message_to_draft(
+ &self,
+ message_id: Uuid,
+ link_id: Uuid,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+
/// Fetch a label by its database ID and link ID.
fn get_label_by_id(
&self,
diff --git a/crates/email/src/domain/service/send.rs b/crates/email/src/domain/service/send.rs
index 5d19639aa3..b154d4cca9 100644
--- a/crates/email/src/domain/service/send.rs
+++ b/crates/email/src/domain/service/send.rs
@@ -38,10 +38,37 @@ where
// FE displays "Undo" button for delay_secs. Give extra time for round trip of cancel request
let sqs_delay = delay_secs as i32 + 2;
- self.enqueuer
+ // The insert above already committed the message as non-draft with a
+ // pending scheduled row. The queue message is the only thing that will
+ // ever deliver it — nothing re-derives the send from the row — so an
+ // enqueue failure leaves a message that is no longer a draft, was never
+ // sent, and has no worker coming for it. Put it back to an unsent draft
+ // so the body survives and the caller can retry, and fail loudly.
+ if let Err(e) = self
+ .enqueuer
.enqueue_scheduled_message(link.id, created.db_id, Some(sqs_delay))
.await
- .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)))?;
+ {
+ let err = anyhow::Error::from(e);
+ tracing::error!(error=?err, "failed to enqueue scheduled send, reverting the message to a draft");
+ if let Err(revert_err) = self
+ .email_repo
+ .revert_sent_message_to_draft(created.db_id, link.id)
+ .await
+ {
+ // Both writes failed, so the row is still a non-draft unsent
+ // send with a pending scheduled row. That is what the
+ // email_scheduled_handler sweep re-enqueues once the row is
+ // past its grace period.
+ tracing::error!(
+ error=?anyhow::Error::from(revert_err),
+ link_id=%link.id,
+ message_id=%created.db_id,
+ "failed to revert the message after a failed enqueue; leaving it for the scheduled sweep",
+ );
+ }
+ return Err(EmailErr::EnqueueErr(err));
+ }
// The undo-window send is now committed and queued; the matching
// message_sent / message_send_cancelled event resolves it later,
diff --git a/crates/email/src/outbound/email_pg_repo.rs b/crates/email/src/outbound/email_pg_repo.rs
index 9e94094da8..07cf9dfcd5 100644
--- a/crates/email/src/outbound/email_pg_repo.rs
+++ b/crates/email/src/outbound/email_pg_repo.rs
@@ -284,6 +284,14 @@ impl EmailRepo for EmailPgRepo {
draft::insert_message(&self.pool, input, contacts, link_id, new_thread, is_draft).await
}
+ async fn revert_sent_message_to_draft(
+ &self,
+ message_id: Uuid,
+ link_id: Uuid,
+ ) -> Result<(), Self::Err> {
+ draft::revert_sent_message_to_draft(&self.pool, message_id, link_id).await
+ }
+
async fn get_label_by_id(
&self,
label_id: Uuid,
diff --git a/crates/email/src/outbound/email_pg_repo/draft.rs b/crates/email/src/outbound/email_pg_repo/draft.rs
index b68c2c84f0..e6b4305fea 100644
--- a/crates/email/src/outbound/email_pg_repo/draft.rs
+++ b/crates/email/src/outbound/email_pg_repo/draft.rs
@@ -60,6 +60,87 @@ pub(crate) async fn insert_message(
Ok(())
}
+/// Revert a message that `insert_message(.., is_draft = false)` optimistically
+/// marked as outgoing back to an unsent draft, and drop its pending scheduled
+/// row.
+///
+/// The whole revert hinges on the send not having happened: a message the
+/// scheduled worker already delivered (`is_sent`), or is delivering right now
+/// (`processing` on its scheduled row), is left exactly as it is, message row
+/// and scheduled row both. `get_message_to_send` does not re-check `is_draft`,
+/// so a worker that has claimed the row sends the mail whatever this does —
+/// reverting underneath it would deliver the email *and* leave a live draft to
+/// send again.
+#[tracing::instrument(skip(pool), err)]
+pub(crate) async fn revert_sent_message_to_draft(
+ pool: &PgPool,
+ message_id: Uuid,
+ link_id: Uuid,
+) -> Result<(), sqlx::Error> {
+ let mut tx = pool.begin().await?;
+
+ // Lock the scheduled row up front rather than testing `processing` inside
+ // the UPDATE below. A worker's claim is a single committed statement, so an
+ // unlocked read only proves nothing had claimed the row at that instant: a
+ // claim landing before the DELETE would have its row deleted out from under
+ // it and still send. Holding the lock makes a concurrent claim wait — it
+ // then finds no row and skips — and makes a claim that got here first
+ // visible to the check.
+ let claimed = sqlx::query_scalar!(
+ r#"
+ SELECT processing FROM email_scheduled_messages
+ WHERE link_id = $1 AND message_id = $2
+ FOR UPDATE
+ "#,
+ link_id,
+ message_id,
+ )
+ .fetch_optional(&mut *tx)
+ .await?;
+
+ if claimed == Some(true) {
+ tx.commit().await?;
+ return Ok(());
+ }
+
+ let thread_db_id = sqlx::query_scalar!(
+ r#"
+ UPDATE email_messages
+ SET is_draft = true, is_sent = false, updated_at = NOW()
+ WHERE id = $1 AND link_id = $2 AND is_sent = false
+ RETURNING thread_id
+ "#,
+ message_id,
+ link_id,
+ )
+ .fetch_optional(&mut *tx)
+ .await?;
+
+ let Some(thread_db_id) = thread_db_id else {
+ tx.commit().await?;
+ return Ok(());
+ };
+
+ sqlx::query!(
+ r#"
+ DELETE FROM email_scheduled_messages
+ WHERE link_id = $1 AND message_id = $2 AND sent = false
+ "#,
+ link_id,
+ message_id,
+ )
+ .execute(&mut *tx)
+ .await?;
+
+ // Drafts count toward inbox_visible and the latest-message timestamps, so
+ // the metadata the insert computed for an outgoing message no longer
+ // describes the thread. Same recompute delete_draft_message does.
+ thread::update_thread_metadata(&mut tx, thread_db_id, link_id).await?;
+
+ tx.commit().await?;
+ Ok(())
+}
+
/// Upsert a draft message row.
pub(crate) async fn upsert_draft(
tx: &mut sqlx::PgConnection,
diff --git a/crates/email/src/outbound/email_pg_repo/test.rs b/crates/email/src/outbound/email_pg_repo/test.rs
index cc4284ad1f..f6c77e3f34 100644
--- a/crates/email/src/outbound/email_pg_repo/test.rs
+++ b/crates/email/src/outbound/email_pg_repo/test.rs
@@ -9,6 +9,7 @@ mod message;
mod preview;
mod project;
mod project_scope_dynamic_query;
+mod send;
mod settings;
mod signal_flag;
mod thread;
diff --git a/crates/email/src/outbound/email_pg_repo/test/send.rs b/crates/email/src/outbound/email_pg_repo/test/send.rs
new file mode 100644
index 0000000000..72fd386295
--- /dev/null
+++ b/crates/email/src/outbound/email_pg_repo/test/send.rs
@@ -0,0 +1,485 @@
+//! Durability of the undo-window send path.
+//!
+//! `send_message_impl` commits the message as a non-draft with a pending
+//! scheduled row and only then enqueues the delivery, so these tests run the
+//! domain service against the real repo with an enqueuer that fails, and pin
+//! down what the database is left holding.
+
+use super::*;
+use crate::domain::models::{ContactInfo, CreateDraftInput};
+use crate::domain::ports::EmailMessageEnqueuer;
+use crate::domain::service::EmailServiceImpl;
+use crm::domain::service::NoOpCrmService;
+use frecency::domain::services::FrecencyQueryServiceImpl;
+use frecency::outbound::postgres::FrecencyPgStorage;
+use macro_event_broker::NoopMacroEventBroker;
+use macro_user_id::email::EmailStr;
+
+const LINK_ID: &str = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
+const SENT_MESSAGE_ID: &str = "ee000001-0000-0000-0000-000000000001";
+const DRAFT_MESSAGE_ID: &str = "ee000002-0000-0000-0000-000000000002";
+/// The draft that is the only message in its thread, so thread metadata swings
+/// entirely on whether it counts as a draft.
+const LONE_DRAFT_MESSAGE_ID: &str = "ee000004-0000-0000-0000-000000000004";
+const LONE_DRAFT_THREAD_ID: &str = "33333333-3333-3333-3333-333333333333";
+
+/// Stands in for SQS being unreachable — the shape of the outage that stranded
+/// sends in production.
+#[derive(Clone)]
+struct FailingEnqueuer;
+
+impl EmailMessageEnqueuer for FailingEnqueuer {
+ type Err = anyhow::Error;
+
+ async fn enqueue_scheduled_message(
+ &self,
+ _link_id: Uuid,
+ _message_id: Uuid,
+ _delay_seconds: Option<i32>,
+ ) -> Result<(), Self::Err> {
+ Err(anyhow::anyhow!("queue unavailable"))
+ }
+
+ async fn enqueue_gmail_ops_modify_labels_batch(
+ &self,
+ _link_id: Uuid,
+ _messages: Vec<(Uuid, String)>,
+ _labels_to_add: Vec<String>,
+ _labels_to_remove: Vec<String>,
+ ) -> Result<(), Self::Err> {
+ Err(anyhow::anyhow!("queue unavailable"))
+ }
+
+ async fn enqueue_gmail_ops_block_sender(
+ &self,
+ _link_id: Uuid,
+ _email_address: String,
+ ) -> Result<(), Self::Err> {
+ Err(anyhow::anyhow!("queue unavailable"))
+ }
+
+ async fn enqueue_gmail_ops_unblock_sender(
+ &self,
+ _link_id: Uuid,
+ _email_address: String,
+ ) -> Result<(), Self::Err> {
+ Err(anyhow::anyhow!("queue unavailable"))
+ }
+}
+
+/// The send path touches neither frecency nor CRM nor the event broker, so
+/// those collaborators are no-ops; the repo is the real thing.
+type TestService<E> = EmailServiceImpl<
+ EmailPgRepo,
+ FrecencyQueryServiceImpl<FrecencyPgStorage>,
+ E,
+ NoOpCrmService,
+ (),
+ NoopMacroEventBroker,
+>;
+
+fn service<E: EmailMessageEnqueuer>(pool: Pool<Postgres>, enqueuer: E) -> TestService<E> {
+ EmailServiceImpl {
+ email_repo: EmailPgRepo::new(pool.clone()),
+ frecency_service: FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(pool)),
+ enqueuer,
+ crm_service: NoOpCrmService,
+ entity_access_management_service: (),
+ macro_event_broker: NoopMacroEventBroker,
+ sent_undo_delay_secs: 10,
+ }
+}
+
+fn link() -> anyhow::Result<Link> {
+ Ok(Link {
+ id: Uuid::parse_str(LINK_ID)?,
+ macro_id: MacroUserIdStr::try_from_email("[email protected]")?,
+ fusionauth_user_id: "fa-user-1".to_owned(),
+ email_address: EmailStr::try_from("[email protected]".to_owned())?,
+ provider: UserProvider::Gmail,
+ is_sync_active: true,
+ is_primary: true,
+ created_at: Utc::now(),
+ updated_at: Utc::now(),
+ })
+}
+
+fn send_input(db_id: Option<Uuid>) -> CreateDraftInput {
+ CreateDraftInput {
+ db_id,
+ provider_id: None,
+ replying_to_id: None,
+ provider_thread_id: None,
+ thread_db_id: None,
+ subject: "Quarterly update".to_owned(),
+ to: vec![ContactInfo {
+ email: "[email protected]".to_owned(),
+ name: None,
+ photo_url: None,
+ }],
+ cc: vec![],
+ bcc: vec![],
+ body_text: Some("Numbers attached.".to_owned()),
+ body_html: None,
+ body_macro: None,
+ headers_json: None,
+ send_time: None,
+ include_signature: Some(false),
+ actor: None,
+ }
+}
+
+/// `(is_draft, is_sent)` for a message row.
+async fn message_flags(pool: &Pool<Postgres>, message_id: Uuid) -> anyhow::Result<(bool, bool)> {
+ let row = sqlx::query!(
+ "SELECT is_draft, is_sent FROM email_messages WHERE id = $1",
+ message_id,
+ )
+ .fetch_one(pool)
+ .await?;
+ Ok((row.is_draft, row.is_sent))
+}
+
+async fn scheduled_row_count(pool: &Pool<Postgres>, message_id: Uuid) -> anyhow::Result<i64> {
+ let count = sqlx::query_scalar!(
+ r#"SELECT COUNT(*) AS "count!" FROM email_scheduled_messages WHERE message_id = $1"#,
+ message_id,
+ )
+ .fetch_one(pool)
+ .await?;
+ Ok(count)
+}
+
+struct ThreadInboxState {
+ inbox_visible: bool,
+ latest_inbound_message_ts: Option<chrono::DateTime<Utc>>,
+}
+
+async fn thread_inbox_state(
+ pool: &Pool<Postgres>,
+ thread_id: Uuid,
+) -> anyhow::Result<ThreadInboxState> {
+ let row = sqlx::query!(
+ "SELECT inbox_visible, latest_inbound_message_ts FROM email_threads WHERE id = $1",
+ thread_id,
+ )
+ .fetch_one(pool)
+ .await?;
+
+ Ok(ThreadInboxState {
+ inbox_visible: row.inbox_visible,
+ latest_inbound_message_ts: row.latest_inbound_message_ts,
+ })
+}
+
+// ── send_message_impl ─────────────────────────────────────────────
+
+/// The regression: a failed enqueue used to leave the message flipped to
+/// non-draft with a scheduled row and nothing in the queue to deliver it, so
+/// the draft vanished from the UI and no mail ever went out. The sweep could
+/// not rescue it either, because it only looked at drafts.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../fixtures", scripts("email_draft"))
+)]
+async fn failed_enqueue_leaves_the_message_as_an_unsent_draft(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link = link()?;
+ let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;
+ let service = service(pool.clone(), FailingEnqueuer);
+
+ let err = service
+ .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))
+ .await
+ .expect_err("enqueue failed, so the send must fail");
+
+ assert!(
+ matches!(err, EmailErr::EnqueueErr(_)),
+ "expected EnqueueErr, got {err:?}"
+ );
+
+ let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;
+ assert!(is_draft, "the draft must survive a failed send");
+ assert!(!is_sent);
+ assert_eq!(
+ scheduled_row_count(&pool, draft_id).await?,
+ 0,
+ "no scheduled row should be left pointing at a send that will never happen"
+ );
+
+ Ok(())
+}
+
+/// The agent/AI path sends without an existing draft (`db_id: None`), so the
+/// revert has to leave the freshly composed body behind as a draft rather than
+/// losing it.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../fixtures", scripts("email_draft"))
+)]
+async fn failed_enqueue_keeps_a_brand_new_send_as_a_draft(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link = link()?;
+ let service = service(pool.clone(), FailingEnqueuer);
+
+ let err = service
+ .send_message_impl(&link, &[link.clone()], send_input(None))
+ .await
+ .expect_err("enqueue failed, so the send must fail");
+ assert!(
+ matches!(err, EmailErr::EnqueueErr(_)),
+ "expected EnqueueErr, got {err:?}"
+ );
+
+ let new_message_id: Uuid = sqlx::query_scalar!(
+ r#"
+ SELECT id FROM email_messages
+ WHERE link_id = $1 AND subject = 'Quarterly update'
+ "#,
+ link.id,
+ )
+ .fetch_one(&pool)
+ .await?;
+
+ let (is_draft, is_sent) = message_flags(&pool, new_message_id).await?;
+ assert!(is_draft, "the composed body must survive as a draft");
+ assert!(!is_sent);
+ assert_eq!(scheduled_row_count(&pool, new_message_id).await?, 0);
+
+ Ok(())
+}
+
+/// The happy path still commits the message as outgoing with a pending
+/// scheduled row — the revert must not fire when the enqueue succeeds.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../fixtures", scripts("email_draft"))
+)]
+async fn successful_enqueue_commits_the_send(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let link = link()?;
+ let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;
+ let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);
+
+ let created = service
+ .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))
+ .await?;
+ assert_eq!(created.db_id, draft_id);
+
+ let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;
+ assert!(!is_draft, "an enqueued send is no longer a draft");
+ assert!(
+ !is_sent,
+ "the worker marks it sent once Gmail accepts it, not here"
+ );
+ assert_eq!(scheduled_row_count(&pool, draft_id).await?, 1);
+
+ Ok(())
+}
+
+// ── revert_sent_message_to_draft ──────────────────────────────────
+
+/// Drafts count toward a thread's denormalized `inbox_visible` and
+/// latest-message timestamps, so the metadata the insert computed for an
+/// outgoing message has to be recomputed on the way back.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../fixtures", scripts("email_draft"))
+)]
+async fn revert_recomputes_thread_metadata(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let link = link()?;
+ let draft_id = Uuid::parse_str(LONE_DRAFT_MESSAGE_ID)?;
+ let thread_id = Uuid::parse_str(LONE_DRAFT_THREAD_ID)?;
+ let repo = EmailPgRepo::new(pool.clone());
+ let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);
+
+ service
+ .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))
+ .await?;
+
+ let sent = thread_inbox_state(&pool, thread_id).await?;
+ assert!(
+ !sent.inbox_visible && sent.latest_inbound_message_ts.is_none(),
+ "the thread's only message is outgoing, so nothing keeps it in the inbox"
+ );
+
+ repo.revert_sent_message_to_draft(draft_id, link.id).await?;
+
+ let reverted = thread_inbox_state(&pool, thread_id).await?;
+ assert!(
+ reverted.inbox_visible,
+ "a thread holding a draft again belongs back in the inbox"
+ );
+ assert!(reverted.latest_inbound_message_ts.is_some());
+
+ Ok(())
+}
+
+/// Losing a queue message must never be able to un-send mail that Gmail has
+/// already accepted, so the revert is a no-op once the worker has marked the
+/// message sent.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../fixtures", scripts("email_draft"))
+)]
+async fn revert_leaves_an_already_delivered_message_alone(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link_id = Uuid::parse_str(LINK_ID)?;
+ let sent_id = Uuid::parse_str(SENT_MESSAGE_ID)?;
+ let repo = EmailPgRepo::new(pool.clone());
+
+ sqlx::query!(
+ r#"
+ INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, created_at, updated_at)
+ VALUES ($1, $2, NOW(), false, NOW(), NOW())
+ "#,
+ link_id,
+ sent_id,
+ )
+ .execute(&pool)
+ .await?;
+
+ repo.revert_sent_message_to_draft(sent_id, link_id).await?;
+
+ let (is_draft, is_sent) = message_flags(&pool, sent_id).await?;
+ assert!(!is_draft, "a delivered message must not become a draft");
+ assert!(is_sent);
+ assert_eq!(
+ scheduled_row_count(&pool, sent_id).await?,
+ 1,
+ "the revert bails before touching the scheduled row"
+ );
+
+ Ok(())
+}
+
+/// A worker that has claimed the scheduled row will deliver the mail whether or
+/// not this revert runs (`get_message_to_send` does not re-check `is_draft`), so
+/// reverting underneath it would send the email and leave a live draft behind.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../fixtures", scripts("email_draft"))
+)]
+async fn revert_leaves_a_message_a_worker_is_delivering_alone(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let link = link()?;
+ let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;
+ let repo = EmailPgRepo::new(pool.clone());
+ let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);
+
+ service
+ .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))
+ .await?;
+
+ sqlx::query!(
+ r#"
+ UPDATE email_scheduled_messages SET processing = true
+ WHERE link_id = $1 AND message_id = $2
+ "#,
+ link.id,
+ draft_id,
+ )
+ .execute(&pool)
+ .await?;
+
+ repo.revert_sent_message_to_draft(draft_id, link.id).await?;
+
+ let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;
+ assert!(
+ !is_draft,
+ "a send in flight must not become a draft that can be sent again"
+ );
+ assert!(!is_sent);
+ assert_eq!(
+ scheduled_row_count(&pool, draft_id).await?,
+ 1,
+ "the worker still needs its row to mark the send as sent"
+ );
+
+ Ok(())
+}
+
+/// The guard has to be a lock, not a read. A worker's claim is one committed
+/// statement, so a revert that merely *read* `processing` could have a claim
+/// land between that read and its DELETE, delete the row the worker just
+/// claimed, and leave the worker to send from the data it already holds.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../fixtures", scripts("email_draft"))
+)]
+async fn revert_waits_for_a_claim_racing_it(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let link = link()?;
+ let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;
+ let repo = EmailPgRepo::new(pool.clone());
+ let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);
+
+ service
+ .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))
+ .await?;
+
+ // A worker claims the row exactly the way the scheduled consumer does, but
+ // holds the transaction open so the claim is in flight, not settled.
+ let mut claim = pool.begin().await?;
+ sqlx::query!(
+ r#"
+ UPDATE email_scheduled_messages SET processing = true, updated_at = NOW()
+ WHERE link_id = $1 AND message_id = $2
+ "#,
+ link.id,
+ draft_id,
+ )
+ .execute(&mut *claim)
+ .await?;
+
+ let mut revert = tokio::spawn({
+ let repo = repo.clone();
+ let link_id = link.id;
+ async move { repo.revert_sent_message_to_draft(draft_id, link_id).await }
+ });
+
+ assert!(
+ tokio::time::timeout(std::time::Duration::from_millis(500), &mut revert)
+ .await
+ .is_err(),
+ "the revert must wait on the in-flight claim"
+ );
+
+ // What the claim lands on is the point. Testing `processing` with an
+ // unlocked read lets the revert flip the message to a draft first and only
+ // then wait — on the DELETE's row lock — so once the claim commits it
+ // deletes the row that claim just took and leaves a live draft behind.
+ claim.commit().await?;
+ revert.await??;
+
+ let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;
+ assert!(
+ !is_draft,
+ "once the claim lands the revert must see it and leave the send alone"
+ );
+ assert!(!is_sent);
+ assert_eq!(
+ scheduled_row_count(&pool, draft_id).await?,
+ 1,
+ "the claimed row must survive for the worker to mark the send as sent"
+ );
+
+ Ok(())
+}
+
+/// A message that was never inserted (or belongs to another inbox) must not
+/// make the revert fail — it runs on a path that is already handling an error.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../../fixtures", scripts("email_draft"))
+)]
+async fn revert_is_a_no_op_for_an_unknown_message(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let repo = EmailPgRepo::new(pool);
+
+ repo.revert_sent_message_to_draft(Uuid::from_u128(999), Uuid::parse_str(LINK_ID)?)
+ .await?;
+
+ Ok(())
+}
diff --git a/services/email_scheduled_handler/src/handler.rs b/services/email_scheduled_handler/src/handler.rs
index 6124137641..a6c8900504 100644
--- a/services/email_scheduled_handler/src/handler.rs
+++ b/services/email_scheduled_handler/src/handler.rs
@@ -24,12 +24,25 @@ pub async fn handler(
tracing::info!(notifications = ?notifications, "Sending scheduled email pubsub messages");
}
- for notif in notifications.into_iter() {
- let message_id = notif.message_id;
- let link_id = notif.link_id;
+ for pending in notifications.into_iter() {
+ let message_id = pending.message_id;
+ let link_id = pending.link_id;
+
+ // A stranded row means the send path committed the message but failed
+ // to enqueue it and failed to revert it, so this sweep is the only
+ // reason the mail goes out at all. Warn so the underlying failure is
+ // visible instead of being quietly papered over.
+ if pending.stranded {
+ tracing::warn!(
+ link_id = link_id.to_string(),
+ message_id = message_id.to_string(),
+ "Re-enqueueing a send whose original enqueue never landed",
+ );
+ }
+
if let Err(e) = ctx
.sqs_client
- .enqueue_email_scheduled_message(notif, None)
+ .enqueue_email_scheduled_message(pending.into(), None)
.await
{
tracing::error!(
@@ -44,27 +57,73 @@ pub async fn handler(
Ok(())
}
+/// How long past its `send_time` a non-draft scheduled row must sit before the
+/// sweep treats it as stranded rather than in flight.
+///
+/// The undo-window send path (`email`'s `send_message_impl`) inserts its
+/// message as a non-draft and puts the queue message on a short delay, so such
+/// a row is normally delivered seconds after its `send_time`. The grace period
+/// has to clear that delay plus the scheduled queue's whole redelivery cycle
+/// (five receives at a 30s visibility timeout) so a rescue can never race a
+/// delivery that is still coming and double-send the message.
+const STRANDED_SEND_GRACE_SECS: f64 = 300.0;
+
+/// An overdue scheduled send, plus how it reached this sweep.
+#[derive(Debug)]
+pub struct PendingScheduledSend {
+ pub link_id: sqlx::types::Uuid,
+ pub message_id: sqlx::types::Uuid,
+ /// `true` for an undo-window send this sweep is rescuing, `false` for an
+ /// ordinary send-later draft whose delivery this sweep always owned.
+ pub stranded: bool,
+}
+
+impl From<PendingScheduledSend> for ScheduledPubsubMessage {
+ fn from(pending: PendingScheduledSend) -> Self {
+ ScheduledPubsubMessage {
+ link_id: pending.link_id,
+ message_id: pending.message_id,
+ }
+ }
+}
+
/// Fetches all scheduled messages that are ready to be sent.
-/// Only returns messages where:
-/// - send_time has passed (< now())
-/// - sent = false
-/// - the associated email message is still a draft (is_draft = true)
+///
+/// Returns rows whose `send_time` has passed and that neither the scheduled
+/// table (`esm.sent`) nor the message itself (`em.is_sent`) records as sent,
+/// in two shapes:
+/// - send-later drafts (`em.is_draft = TRUE`), for which this sweep is the
+/// only thing that ever enqueues a delivery;
+/// - undo-window sends (`em.is_draft = FALSE`), which enqueue their own
+/// delivery at send time and so are only picked up once they are
+/// [`STRANDED_SEND_GRACE_SECS`] overdue and unclaimed (`processing = FALSE`)
+/// — the signature of a send whose enqueue never landed.
#[tracing::instrument(skip(pool), err)]
pub async fn fetch_pending_scheduled_messages(
pool: &PgPool,
-) -> anyhow::Result<Vec<ScheduledPubsubMessage>> {
+) -> anyhow::Result<Vec<PendingScheduledSend>> {
let messages = sqlx::query_as!(
- ScheduledPubsubMessage,
+ PendingScheduledSend,
r#"
SELECT
- esm.link_id, esm.message_id
+ esm.link_id,
+ esm.message_id,
+ NOT em.is_draft AS "stranded!"
FROM email_scheduled_messages esm
JOIN email_messages em ON em.id = esm.message_id
WHERE
esm.send_time < now()
AND esm.sent = FALSE
- AND em.is_draft = TRUE
+ AND em.is_sent = FALSE
+ AND (
+ em.is_draft = TRUE
+ OR (
+ esm.processing = FALSE
+ AND esm.send_time < now() - make_interval(secs => $1)
+ )
+ )
"#,
+ STRANDED_SEND_GRACE_SECS,
)
.fetch_all(pool)
.await?;
diff --git a/services/email_scheduled_handler/src/test.rs b/services/email_scheduled_handler/src/test.rs
index e6aba1e190..2a54a3da45 100644
--- a/services/email_scheduled_handler/src/test.rs
+++ b/services/email_scheduled_handler/src/test.rs
@@ -13,15 +13,17 @@ async fn fetch_pending_returns_only_eligible_messages(pool: Pool<Postgres>) -> R
let results = fetch_pending_scheduled_messages(&pool).await?;
- // Should return exactly 2 messages:
+ // Should return exactly 3 messages:
// - Message 1: Draft with past send_time, not sent
// - Message 5: Draft with past send_time, not sent (different link)
- assert_eq!(results.len(), 2);
+ // - Message 6: Non-draft undo-window send stranded well past its send_time
+ assert_eq!(results.len(), 3);
let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();
assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f501")?));
assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f505")?));
+ assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f506")?));
Ok(())
}
@@ -60,13 +62,86 @@ async fn fetch_pending_excludes_already_sent(pool: Pool<Postgres>) -> Result<()>
migrator = "MACRO_DB_MIGRATIONS",
fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))
)]
-async fn fetch_pending_excludes_non_drafts(pool: Pool<Postgres>) -> Result<()> {
+async fn fetch_pending_excludes_delivered_messages(pool: Pool<Postgres>) -> Result<()> {
let results = fetch_pending_scheduled_messages(&pool).await?;
let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();
- // Message 4 has is_draft = false, should not be included
+ // Messages 4 and 9 both have is_sent = true, so the send already happened
+ // and the unsent scheduled row is stale. Re-enqueueing either would send
+ // the same mail twice.
assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f504")?));
+ assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f509")?));
+
+ Ok(())
+}
+
+/// The bug this sweep now covers: `send_message_impl` commits the message as a
+/// non-draft with a scheduled row and only then enqueues the delivery, so a
+/// failed enqueue leaves a message that is not a draft, was never sent, and has
+/// no worker coming for it. The old `em.is_draft = TRUE` predicate could not
+/// see those rows at all.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))
+)]
+async fn fetch_pending_includes_stranded_non_draft_sends(pool: Pool<Postgres>) -> Result<()> {
+ let results = fetch_pending_scheduled_messages(&pool).await?;
+
+ let rescued = results
+ .iter()
+ .find(|r| r.message_id == Uuid::parse_str("00000000-0000-0000-0000-00000000f506").unwrap());
+
+ let rescued = rescued.expect("the stranded send should be picked up");
+ assert!(
+ rescued.stranded,
+ "a rescue must be flagged so the underlying send failure is visible in the logs"
+ );
+
+ // Ordinary send-later drafts are this sweep's normal work, not rescues.
+ let draft = results
+ .iter()
+ .find(|r| r.message_id == Uuid::parse_str("00000000-0000-0000-0000-00000000f501").unwrap())
+ .expect("the send-later draft should be picked up");
+ assert!(!draft.stranded);
+
+ Ok(())
+}
+
+/// A non-draft send that is only seconds overdue still has its own queue
+/// message in flight (or in SQS redelivery), so rescuing it would double-send.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))
+)]
+async fn fetch_pending_excludes_non_draft_sends_inside_grace_period(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let results = fetch_pending_scheduled_messages(&pool).await?;
+
+ let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();
+
+ // Message 7 is 30 seconds overdue, well inside the grace period
+ assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f507")?));
+
+ Ok(())
+}
+
+/// A worker already claimed the row, so it is being delivered rather than
+/// stranded.
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))
+)]
+async fn fetch_pending_excludes_non_draft_sends_being_processed(
+ pool: Pool<Postgres>,
+) -> Result<()> {
+ let results = fetch_pending_scheduled_messages(&pool).await?;
+
+ let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();
+
+ // Message 8 is well overdue but has processing = true
+ assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f508")?));
Ok(())
}
diff --git a/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql b/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql
index 35a69c4d52..c07602aa38 100644
--- a/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql
+++ b/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql
@@ -110,7 +110,8 @@ VALUES ('00000000-0000-0000-0000-000000000f01',
NOW(), NOW());
------------------------------------------------------------
--- Message 4: NOT a draft (is_draft = false) with past send_time, not sent (SHOULD NOT BE RETURNED)
+-- Message 4: NOT a draft, already delivered (is_sent = true), with a stale
+-- unsent scheduled row and a past send_time (SHOULD NOT BE RETURNED)
------------------------------------------------------------
INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,
@@ -151,4 +152,99 @@ VALUES ('00000000-0000-0000-0000-000000000f02',
'00000000-0000-0000-0000-00000000f505',
'2020-01-01 14:00:00 +00:00', -- Past date
false,
- NOW(), NOW());
\ No newline at end of file
+ NOW(), NOW());
+
+------------------------------------------------------------
+-- Message 6: undo-window send stranded by a failed enqueue — not a draft,
+-- not sent, unclaimed, and well past the grace period (SHOULD BE RETURNED)
+------------------------------------------------------------
+
+INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,
+ has_attachments, is_read, is_starred, is_draft, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-00000000f506',
+ '00000000-0000-0000-0000-00000000f201',
+ '00000000-0000-0000-0000-000000000f01',
+ NULL,
+ FALSE,
+ '00000000-0000-0000-0000-0000000cf001',
+ '2025-01-10 15:00:00 +00:00',
+ false, false, false, false, NOW(), NOW()); -- is_draft = false, is_sent = false
+
+INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-000000000f01',
+ '00000000-0000-0000-0000-00000000f506',
+ NOW() - INTERVAL '10 minutes', -- well past the stranded grace period
+ false,
+ false,
+ NOW(), NOW());
+
+------------------------------------------------------------
+-- Message 7: undo-window send inside the grace period — its own queue message
+-- is still in flight or being retried (SHOULD NOT BE RETURNED)
+------------------------------------------------------------
+
+INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,
+ has_attachments, is_read, is_starred, is_draft, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-00000000f507',
+ '00000000-0000-0000-0000-00000000f201',
+ '00000000-0000-0000-0000-000000000f01',
+ NULL,
+ FALSE,
+ '00000000-0000-0000-0000-0000000cf001',
+ '2025-01-10 16:00:00 +00:00',
+ false, false, false, false, NOW(), NOW());
+
+INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-000000000f01',
+ '00000000-0000-0000-0000-00000000f507',
+ NOW() - INTERVAL '30 seconds', -- overdue, but inside the grace period
+ false,
+ false,
+ NOW(), NOW());
+
+------------------------------------------------------------
+-- Message 8: overdue undo-window send a worker already claimed
+-- (SHOULD NOT BE RETURNED)
+------------------------------------------------------------
+
+INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,
+ has_attachments, is_read, is_starred, is_draft, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-00000000f508',
+ '00000000-0000-0000-0000-00000000f201',
+ '00000000-0000-0000-0000-000000000f01',
+ NULL,
+ FALSE,
+ '00000000-0000-0000-0000-0000000cf001',
+ '2025-01-10 17:00:00 +00:00',
+ false, false, false, false, NOW(), NOW());
+
+INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-000000000f01',
+ '00000000-0000-0000-0000-00000000f508',
+ NOW() - INTERVAL '10 minutes',
+ false,
+ true, -- claimed by a worker
+ NOW(), NOW());
+
+------------------------------------------------------------
+-- Message 9: draft whose message row says it was delivered, with a stale
+-- unsent scheduled row (SHOULD NOT BE RETURNED)
+------------------------------------------------------------
+
+INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,
+ has_attachments, is_read, is_starred, is_draft, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-00000000f509',
+ '00000000-0000-0000-0000-00000000f201',
+ '00000000-0000-0000-0000-000000000f01',
+ 'provider-pending-509',
+ TRUE,
+ '00000000-0000-0000-0000-0000000cf001',
+ '2025-01-10 18:00:00 +00:00',
+ false, false, false, true, NOW(), NOW()); -- is_draft = true, is_sent = true
+
+INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, created_at, updated_at)
+VALUES ('00000000-0000-0000-0000-000000000f01',
+ '00000000-0000-0000-0000-00000000f509',
+ '2020-01-01 18:00:00 +00:00',
+ false,
+ NOW(), NOW());