Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
revert(email): stop replaying historical sends (#6580) (#6617)
gbirmanbbc115899444
diff --git a/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.json b/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.jsondeleted file mode 100644index 92cdd72f4b..0000000000--- a/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.json+++ /dev/null@@ -1,15 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n DELETE FROM email_scheduled_messages\n WHERE link_id = $1 AND message_id = $2 AND sent = false\n ",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Uuid",- "Uuid"- ]- },- "nullable": []- },- "hash": "06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b"-}diff --git a/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.json b/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.jsonnew file mode 100644index 0000000000..5c0001485c--- /dev/null+++ b/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.json@@ -0,0 +1,26 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT\n esm.link_id, esm.message_id\n FROM email_scheduled_messages esm\n JOIN email_messages em ON em.id = esm.message_id\n WHERE\n esm.send_time < now()\n AND esm.sent = FALSE\n AND em.is_draft = TRUE\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "link_id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "message_id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ false,+ false+ ]+ },+ "hash": "2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01"+}diff --git a/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.json b/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.jsondeleted file mode 100644index 127c7e74ba..0000000000--- a/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.json+++ /dev/null@@ -1,23 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n SELECT processing FROM email_scheduled_messages\n WHERE link_id = $1 AND message_id = $2\n FOR UPDATE\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "processing",- "type_info": "Bool"- }- ],- "parameters": {- "Left": [- "Uuid",- "Uuid"- ]- },- "nullable": [- false- ]- },- "hash": "585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351"-}diff --git a/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.json b/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.jsondeleted file mode 100644index 7e470f50d3..0000000000--- a/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.json+++ /dev/null@@ -1,23 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n UPDATE email_messages\n SET is_draft = true, is_sent = false, updated_at = NOW()\n WHERE id = $1 AND link_id = $2 AND is_sent = false\n RETURNING thread_id\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "thread_id",- "type_info": "Uuid"- }- ],- "parameters": {- "Left": [- "Uuid",- "Uuid"- ]- },- "nullable": [- false- ]- },- "hash": "be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe"-}diff --git a/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.json b/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.jsondeleted file mode 100644index b641ef3d21..0000000000--- a/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.json+++ /dev/null@@ -1,34 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n SELECT\n esm.link_id,\n esm.message_id,\n NOT em.is_draft AS \"stranded!\"\n FROM email_scheduled_messages esm\n JOIN email_messages em ON em.id = esm.message_id\n WHERE\n esm.send_time < now()\n AND esm.sent = FALSE\n AND em.is_sent = FALSE\n AND (\n em.is_draft = TRUE\n OR (\n esm.processing = FALSE\n AND esm.send_time < now() - make_interval(secs => $1)\n )\n )\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "link_id",- "type_info": "Uuid"- },- {- "ordinal": 1,- "name": "message_id",- "type_info": "Uuid"- },- {- "ordinal": 2,- "name": "stranded!",- "type_info": "Bool"- }- ],- "parameters": {- "Left": [- "Float8"- ]- },- "nullable": [- false,- false,- null- ]- },- "hash": "c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538"-}diff --git a/crates/email/src/domain/ports.rs b/crates/email/src/domain/ports.rsindex fb95ce91dd..cbdf85c9b3 100644--- a/crates/email/src/domain/ports.rs+++ b/crates/email/src/domain/ports.rs@@ -269,16 +269,6 @@ pub trait EmailRepo: Send + Sync + 'static {is_draft: bool,) -> impl Future<Output = Result<(), Self::Err>> + Send;- /// Undo an `insert_message(.., is_draft = false)` whose send never made it- /// onto a queue: put the message back to an unsent draft, drop its pending- /// scheduled row, and recompute the thread's denormalized metadata.- /// Leaves a message the scheduled worker already delivered untouched.- fn revert_sent_message_to_draft(- &self,- message_id: Uuid,- link_id: Uuid,- ) -> impl Future<Output = Result<(), Self::Err>> + Send;-/// Fetch a label by its database ID and link ID.fn get_label_by_id(&self,diff --git a/crates/email/src/domain/service/send.rs b/crates/email/src/domain/service/send.rsindex b154d4cca9..5d19639aa3 100644--- a/crates/email/src/domain/service/send.rs+++ b/crates/email/src/domain/service/send.rs@@ -38,37 +38,10 @@ where// FE displays "Undo" button for delay_secs. Give extra time for round trip of cancel requestlet sqs_delay = delay_secs as i32 + 2;- // The insert above already committed the message as non-draft with a- // pending scheduled row. The queue message is the only thing that will- // ever deliver it — nothing re-derives the send from the row — so an- // enqueue failure leaves a message that is no longer a draft, was never- // sent, and has no worker coming for it. Put it back to an unsent draft- // so the body survives and the caller can retry, and fail loudly.- if let Err(e) = self- .enqueuer+ self.enqueuer.enqueue_scheduled_message(link.id, created.db_id, Some(sqs_delay)).await- {- let err = anyhow::Error::from(e);- tracing::error!(error=?err, "failed to enqueue scheduled send, reverting the message to a draft");- if let Err(revert_err) = self- .email_repo- .revert_sent_message_to_draft(created.db_id, link.id)- .await- {- // Both writes failed, so the row is still a non-draft unsent- // send with a pending scheduled row. That is what the- // email_scheduled_handler sweep re-enqueues once the row is- // past its grace period.- tracing::error!(- error=?anyhow::Error::from(revert_err),- link_id=%link.id,- message_id=%created.db_id,- "failed to revert the message after a failed enqueue; leaving it for the scheduled sweep",- );- }- return Err(EmailErr::EnqueueErr(err));- }+ .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)))?;// The undo-window send is now committed and queued; the matching// message_sent / message_send_cancelled event resolves it later,diff --git a/crates/email/src/outbound/email_pg_repo.rs b/crates/email/src/outbound/email_pg_repo.rsindex 07cf9dfcd5..9e94094da8 100644--- a/crates/email/src/outbound/email_pg_repo.rs+++ b/crates/email/src/outbound/email_pg_repo.rs@@ -284,14 +284,6 @@ impl EmailRepo for EmailPgRepo {draft::insert_message(&self.pool, input, contacts, link_id, new_thread, is_draft).await}- async fn revert_sent_message_to_draft(- &self,- message_id: Uuid,- link_id: Uuid,- ) -> Result<(), Self::Err> {- draft::revert_sent_message_to_draft(&self.pool, message_id, link_id).await- }-async fn get_label_by_id(&self,label_id: Uuid,diff --git a/crates/email/src/outbound/email_pg_repo/draft.rs b/crates/email/src/outbound/email_pg_repo/draft.rsindex e6b4305fea..b68c2c84f0 100644--- a/crates/email/src/outbound/email_pg_repo/draft.rs+++ b/crates/email/src/outbound/email_pg_repo/draft.rs@@ -60,87 +60,6 @@ pub(crate) async fn insert_message(Ok(())}-/// Revert a message that `insert_message(.., is_draft = false)` optimistically-/// marked as outgoing back to an unsent draft, and drop its pending scheduled-/// row.-///-/// The whole revert hinges on the send not having happened: a message the-/// scheduled worker already delivered (`is_sent`), or is delivering right now-/// (`processing` on its scheduled row), is left exactly as it is, message row-/// and scheduled row both. `get_message_to_send` does not re-check `is_draft`,-/// so a worker that has claimed the row sends the mail whatever this does —-/// reverting underneath it would deliver the email *and* leave a live draft to-/// send again.-#[tracing::instrument(skip(pool), err)]-pub(crate) async fn revert_sent_message_to_draft(- pool: &PgPool,- message_id: Uuid,- link_id: Uuid,-) -> Result<(), sqlx::Error> {- let mut tx = pool.begin().await?;-- // Lock the scheduled row up front rather than testing `processing` inside- // the UPDATE below. A worker's claim is a single committed statement, so an- // unlocked read only proves nothing had claimed the row at that instant: a- // claim landing before the DELETE would have its row deleted out from under- // it and still send. Holding the lock makes a concurrent claim wait — it- // then finds no row and skips — and makes a claim that got here first- // visible to the check.- let claimed = sqlx::query_scalar!(- r#"- SELECT processing FROM email_scheduled_messages- WHERE link_id = $1 AND message_id = $2- FOR UPDATE- "#,- link_id,- message_id,- )- .fetch_optional(&mut *tx)- .await?;-- if claimed == Some(true) {- tx.commit().await?;- return Ok(());- }-- let thread_db_id = sqlx::query_scalar!(- r#"- UPDATE email_messages- SET is_draft = true, is_sent = false, updated_at = NOW()- WHERE id = $1 AND link_id = $2 AND is_sent = false- RETURNING thread_id- "#,- message_id,- link_id,- )- .fetch_optional(&mut *tx)- .await?;-- let Some(thread_db_id) = thread_db_id else {- tx.commit().await?;- return Ok(());- };-- sqlx::query!(- r#"- DELETE FROM email_scheduled_messages- WHERE link_id = $1 AND message_id = $2 AND sent = false- "#,- link_id,- message_id,- )- .execute(&mut *tx)- .await?;-- // Drafts count toward inbox_visible and the latest-message timestamps, so- // the metadata the insert computed for an outgoing message no longer- // describes the thread. Same recompute delete_draft_message does.- thread::update_thread_metadata(&mut tx, thread_db_id, link_id).await?;-- tx.commit().await?;- Ok(())-}-/// Upsert a draft message row.pub(crate) async fn upsert_draft(tx: &mut sqlx::PgConnection,diff --git a/crates/email/src/outbound/email_pg_repo/test.rs b/crates/email/src/outbound/email_pg_repo/test.rsindex f6c77e3f34..cc4284ad1f 100644--- a/crates/email/src/outbound/email_pg_repo/test.rs+++ b/crates/email/src/outbound/email_pg_repo/test.rs@@ -9,7 +9,6 @@ mod message;mod preview;mod project;mod project_scope_dynamic_query;-mod send;mod settings;mod signal_flag;mod thread;diff --git a/crates/email/src/outbound/email_pg_repo/test/send.rs b/crates/email/src/outbound/email_pg_repo/test/send.rsdeleted file mode 100644index 72fd386295..0000000000--- a/crates/email/src/outbound/email_pg_repo/test/send.rs+++ /dev/null@@ -1,485 +0,0 @@-//! Durability of the undo-window send path.-//!-//! `send_message_impl` commits the message as a non-draft with a pending-//! scheduled row and only then enqueues the delivery, so these tests run the-//! domain service against the real repo with an enqueuer that fails, and pin-//! down what the database is left holding.--use super::*;-use crate::domain::models::{ContactInfo, CreateDraftInput};-use crate::domain::ports::EmailMessageEnqueuer;-use crate::domain::service::EmailServiceImpl;-use crm::domain::service::NoOpCrmService;-use frecency::domain::services::FrecencyQueryServiceImpl;-use frecency::outbound::postgres::FrecencyPgStorage;-use macro_event_broker::NoopMacroEventBroker;-use macro_user_id::email::EmailStr;--const LINK_ID: &str = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";-const SENT_MESSAGE_ID: &str = "ee000001-0000-0000-0000-000000000001";-const DRAFT_MESSAGE_ID: &str = "ee000002-0000-0000-0000-000000000002";-/// The draft that is the only message in its thread, so thread metadata swings-/// entirely on whether it counts as a draft.-const LONE_DRAFT_MESSAGE_ID: &str = "ee000004-0000-0000-0000-000000000004";-const LONE_DRAFT_THREAD_ID: &str = "33333333-3333-3333-3333-333333333333";--/// Stands in for SQS being unreachable — the shape of the outage that stranded-/// sends in production.-#[derive(Clone)]-struct FailingEnqueuer;--impl EmailMessageEnqueuer for FailingEnqueuer {- type Err = anyhow::Error;-- async fn enqueue_scheduled_message(- &self,- _link_id: Uuid,- _message_id: Uuid,- _delay_seconds: Option<i32>,- ) -> Result<(), Self::Err> {- Err(anyhow::anyhow!("queue unavailable"))- }-- async fn enqueue_gmail_ops_modify_labels_batch(- &self,- _link_id: Uuid,- _messages: Vec<(Uuid, String)>,- _labels_to_add: Vec<String>,- _labels_to_remove: Vec<String>,- ) -> Result<(), Self::Err> {- Err(anyhow::anyhow!("queue unavailable"))- }-- async fn enqueue_gmail_ops_block_sender(- &self,- _link_id: Uuid,- _email_address: String,- ) -> Result<(), Self::Err> {- Err(anyhow::anyhow!("queue unavailable"))- }-- async fn enqueue_gmail_ops_unblock_sender(- &self,- _link_id: Uuid,- _email_address: String,- ) -> Result<(), Self::Err> {- Err(anyhow::anyhow!("queue unavailable"))- }-}--/// The send path touches neither frecency nor CRM nor the event broker, so-/// those collaborators are no-ops; the repo is the real thing.-type TestService<E> = EmailServiceImpl<- EmailPgRepo,- FrecencyQueryServiceImpl<FrecencyPgStorage>,- E,- NoOpCrmService,- (),- NoopMacroEventBroker,->;--fn service<E: EmailMessageEnqueuer>(pool: Pool<Postgres>, enqueuer: E) -> TestService<E> {- EmailServiceImpl {- email_repo: EmailPgRepo::new(pool.clone()),- frecency_service: FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(pool)),- enqueuer,- crm_service: NoOpCrmService,- entity_access_management_service: (),- macro_event_broker: NoopMacroEventBroker,- sent_undo_delay_secs: 10,- }-}--fn link() -> anyhow::Result<Link> {- Ok(Link {- id: Uuid::parse_str(LINK_ID)?,- macro_id: MacroUserIdStr::try_from_email("[email protected]")?,- fusionauth_user_id: "fa-user-1".to_owned(),- email_address: EmailStr::try_from("[email protected]".to_owned())?,- provider: UserProvider::Gmail,- is_sync_active: true,- is_primary: true,- created_at: Utc::now(),- updated_at: Utc::now(),- })-}--fn send_input(db_id: Option<Uuid>) -> CreateDraftInput {- CreateDraftInput {- db_id,- provider_id: None,- replying_to_id: None,- provider_thread_id: None,- thread_db_id: None,- subject: "Quarterly update".to_owned(),- to: vec![ContactInfo {- email: "[email protected]".to_owned(),- name: None,- photo_url: None,- }],- cc: vec![],- bcc: vec![],- body_text: Some("Numbers attached.".to_owned()),- body_html: None,- body_macro: None,- headers_json: None,- send_time: None,- include_signature: Some(false),- actor: None,- }-}--/// `(is_draft, is_sent)` for a message row.-async fn message_flags(pool: &Pool<Postgres>, message_id: Uuid) -> anyhow::Result<(bool, bool)> {- let row = sqlx::query!(- "SELECT is_draft, is_sent FROM email_messages WHERE id = $1",- message_id,- )- .fetch_one(pool)- .await?;- Ok((row.is_draft, row.is_sent))-}--async fn scheduled_row_count(pool: &Pool<Postgres>, message_id: Uuid) -> anyhow::Result<i64> {- let count = sqlx::query_scalar!(- r#"SELECT COUNT(*) AS "count!" FROM email_scheduled_messages WHERE message_id = $1"#,- message_id,- )- .fetch_one(pool)- .await?;- Ok(count)-}--struct ThreadInboxState {- inbox_visible: bool,- latest_inbound_message_ts: Option<chrono::DateTime<Utc>>,-}--async fn thread_inbox_state(- pool: &Pool<Postgres>,- thread_id: Uuid,-) -> anyhow::Result<ThreadInboxState> {- let row = sqlx::query!(- "SELECT inbox_visible, latest_inbound_message_ts FROM email_threads WHERE id = $1",- thread_id,- )- .fetch_one(pool)- .await?;-- Ok(ThreadInboxState {- inbox_visible: row.inbox_visible,- latest_inbound_message_ts: row.latest_inbound_message_ts,- })-}--// ── send_message_impl ─────────────────────────────────────────────--/// The regression: a failed enqueue used to leave the message flipped to-/// non-draft with a scheduled row and nothing in the queue to deliver it, so-/// the draft vanished from the UI and no mail ever went out. The sweep could-/// not rescue it either, because it only looked at drafts.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "../../../../fixtures", scripts("email_draft"))-)]-async fn failed_enqueue_leaves_the_message_as_an_unsent_draft(- pool: Pool<Postgres>,-) -> anyhow::Result<()> {- let link = link()?;- let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;- let service = service(pool.clone(), FailingEnqueuer);-- let err = service- .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))- .await- .expect_err("enqueue failed, so the send must fail");-- assert!(- matches!(err, EmailErr::EnqueueErr(_)),- "expected EnqueueErr, got {err:?}"- );-- let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;- assert!(is_draft, "the draft must survive a failed send");- assert!(!is_sent);- assert_eq!(- scheduled_row_count(&pool, draft_id).await?,- 0,- "no scheduled row should be left pointing at a send that will never happen"- );-- Ok(())-}--/// The agent/AI path sends without an existing draft (`db_id: None`), so the-/// revert has to leave the freshly composed body behind as a draft rather than-/// losing it.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "../../../../fixtures", scripts("email_draft"))-)]-async fn failed_enqueue_keeps_a_brand_new_send_as_a_draft(- pool: Pool<Postgres>,-) -> anyhow::Result<()> {- let link = link()?;- let service = service(pool.clone(), FailingEnqueuer);-- let err = service- .send_message_impl(&link, &[link.clone()], send_input(None))- .await- .expect_err("enqueue failed, so the send must fail");- assert!(- matches!(err, EmailErr::EnqueueErr(_)),- "expected EnqueueErr, got {err:?}"- );-- let new_message_id: Uuid = sqlx::query_scalar!(- r#"- SELECT id FROM email_messages- WHERE link_id = $1 AND subject = 'Quarterly update'- "#,- link.id,- )- .fetch_one(&pool)- .await?;-- let (is_draft, is_sent) = message_flags(&pool, new_message_id).await?;- assert!(is_draft, "the composed body must survive as a draft");- assert!(!is_sent);- assert_eq!(scheduled_row_count(&pool, new_message_id).await?, 0);-- Ok(())-}--/// The happy path still commits the message as outgoing with a pending-/// scheduled row — the revert must not fire when the enqueue succeeds.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "../../../../fixtures", scripts("email_draft"))-)]-async fn successful_enqueue_commits_the_send(pool: Pool<Postgres>) -> anyhow::Result<()> {- let link = link()?;- let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;- let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);-- let created = service- .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))- .await?;- assert_eq!(created.db_id, draft_id);-- let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;- assert!(!is_draft, "an enqueued send is no longer a draft");- assert!(- !is_sent,- "the worker marks it sent once Gmail accepts it, not here"- );- assert_eq!(scheduled_row_count(&pool, draft_id).await?, 1);-- Ok(())-}--// ── revert_sent_message_to_draft ──────────────────────────────────--/// Drafts count toward a thread's denormalized `inbox_visible` and-/// latest-message timestamps, so the metadata the insert computed for an-/// outgoing message has to be recomputed on the way back.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "../../../../fixtures", scripts("email_draft"))-)]-async fn revert_recomputes_thread_metadata(pool: Pool<Postgres>) -> anyhow::Result<()> {- let link = link()?;- let draft_id = Uuid::parse_str(LONE_DRAFT_MESSAGE_ID)?;- let thread_id = Uuid::parse_str(LONE_DRAFT_THREAD_ID)?;- let repo = EmailPgRepo::new(pool.clone());- let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);-- service- .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))- .await?;-- let sent = thread_inbox_state(&pool, thread_id).await?;- assert!(- !sent.inbox_visible && sent.latest_inbound_message_ts.is_none(),- "the thread's only message is outgoing, so nothing keeps it in the inbox"- );-- repo.revert_sent_message_to_draft(draft_id, link.id).await?;-- let reverted = thread_inbox_state(&pool, thread_id).await?;- assert!(- reverted.inbox_visible,- "a thread holding a draft again belongs back in the inbox"- );- assert!(reverted.latest_inbound_message_ts.is_some());-- Ok(())-}--/// Losing a queue message must never be able to un-send mail that Gmail has-/// already accepted, so the revert is a no-op once the worker has marked the-/// message sent.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "../../../../fixtures", scripts("email_draft"))-)]-async fn revert_leaves_an_already_delivered_message_alone(- pool: Pool<Postgres>,-) -> anyhow::Result<()> {- let link_id = Uuid::parse_str(LINK_ID)?;- let sent_id = Uuid::parse_str(SENT_MESSAGE_ID)?;- let repo = EmailPgRepo::new(pool.clone());-- sqlx::query!(- r#"- INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, created_at, updated_at)- VALUES ($1, $2, NOW(), false, NOW(), NOW())- "#,- link_id,- sent_id,- )- .execute(&pool)- .await?;-- repo.revert_sent_message_to_draft(sent_id, link_id).await?;-- let (is_draft, is_sent) = message_flags(&pool, sent_id).await?;- assert!(!is_draft, "a delivered message must not become a draft");- assert!(is_sent);- assert_eq!(- scheduled_row_count(&pool, sent_id).await?,- 1,- "the revert bails before touching the scheduled row"- );-- Ok(())-}--/// A worker that has claimed the scheduled row will deliver the mail whether or-/// not this revert runs (`get_message_to_send` does not re-check `is_draft`), so-/// reverting underneath it would send the email and leave a live draft behind.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "../../../../fixtures", scripts("email_draft"))-)]-async fn revert_leaves_a_message_a_worker_is_delivering_alone(- pool: Pool<Postgres>,-) -> anyhow::Result<()> {- let link = link()?;- let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;- let repo = EmailPgRepo::new(pool.clone());- let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);-- service- .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))- .await?;-- sqlx::query!(- r#"- UPDATE email_scheduled_messages SET processing = true- WHERE link_id = $1 AND message_id = $2- "#,- link.id,- draft_id,- )- .execute(&pool)- .await?;-- repo.revert_sent_message_to_draft(draft_id, link.id).await?;-- let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;- assert!(- !is_draft,- "a send in flight must not become a draft that can be sent again"- );- assert!(!is_sent);- assert_eq!(- scheduled_row_count(&pool, draft_id).await?,- 1,- "the worker still needs its row to mark the send as sent"- );-- Ok(())-}--/// The guard has to be a lock, not a read. A worker's claim is one committed-/// statement, so a revert that merely *read* `processing` could have a claim-/// land between that read and its DELETE, delete the row the worker just-/// claimed, and leave the worker to send from the data it already holds.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "../../../../fixtures", scripts("email_draft"))-)]-async fn revert_waits_for_a_claim_racing_it(pool: Pool<Postgres>) -> anyhow::Result<()> {- let link = link()?;- let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;- let repo = EmailPgRepo::new(pool.clone());- let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);-- service- .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))- .await?;-- // A worker claims the row exactly the way the scheduled consumer does, but- // holds the transaction open so the claim is in flight, not settled.- let mut claim = pool.begin().await?;- sqlx::query!(- r#"- UPDATE email_scheduled_messages SET processing = true, updated_at = NOW()- WHERE link_id = $1 AND message_id = $2- "#,- link.id,- draft_id,- )- .execute(&mut *claim)- .await?;-- let mut revert = tokio::spawn({- let repo = repo.clone();- let link_id = link.id;- async move { repo.revert_sent_message_to_draft(draft_id, link_id).await }- });-- assert!(- tokio::time::timeout(std::time::Duration::from_millis(500), &mut revert)- .await- .is_err(),- "the revert must wait on the in-flight claim"- );-- // What the claim lands on is the point. Testing `processing` with an- // unlocked read lets the revert flip the message to a draft first and only- // then wait — on the DELETE's row lock — so once the claim commits it- // deletes the row that claim just took and leaves a live draft behind.- claim.commit().await?;- revert.await??;-- let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;- assert!(- !is_draft,- "once the claim lands the revert must see it and leave the send alone"- );- assert!(!is_sent);- assert_eq!(- scheduled_row_count(&pool, draft_id).await?,- 1,- "the claimed row must survive for the worker to mark the send as sent"- );-- Ok(())-}--/// A message that was never inserted (or belongs to another inbox) must not-/// make the revert fail — it runs on a path that is already handling an error.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "../../../../fixtures", scripts("email_draft"))-)]-async fn revert_is_a_no_op_for_an_unknown_message(pool: Pool<Postgres>) -> anyhow::Result<()> {- let repo = EmailPgRepo::new(pool);-- repo.revert_sent_message_to_draft(Uuid::from_u128(999), Uuid::parse_str(LINK_ID)?)- .await?;-- Ok(())-}diff --git a/services/email_scheduled_handler/src/handler.rs b/services/email_scheduled_handler/src/handler.rsindex a6c8900504..6124137641 100644--- a/services/email_scheduled_handler/src/handler.rs+++ b/services/email_scheduled_handler/src/handler.rs@@ -24,25 +24,12 @@ pub async fn handler(tracing::info!(notifications = ?notifications, "Sending scheduled email pubsub messages");}- for pending in notifications.into_iter() {- let message_id = pending.message_id;- let link_id = pending.link_id;-- // A stranded row means the send path committed the message but failed- // to enqueue it and failed to revert it, so this sweep is the only- // reason the mail goes out at all. Warn so the underlying failure is- // visible instead of being quietly papered over.- if pending.stranded {- tracing::warn!(- link_id = link_id.to_string(),- message_id = message_id.to_string(),- "Re-enqueueing a send whose original enqueue never landed",- );- }-+ for notif in notifications.into_iter() {+ let message_id = notif.message_id;+ let link_id = notif.link_id;if let Err(e) = ctx.sqs_client- .enqueue_email_scheduled_message(pending.into(), None)+ .enqueue_email_scheduled_message(notif, None).await{tracing::error!(@@ -57,73 +44,27 @@ pub async fn handler(Ok(())}-/// How long past its `send_time` a non-draft scheduled row must sit before the-/// sweep treats it as stranded rather than in flight.-///-/// The undo-window send path (`email`'s `send_message_impl`) inserts its-/// message as a non-draft and puts the queue message on a short delay, so such-/// a row is normally delivered seconds after its `send_time`. The grace period-/// has to clear that delay plus the scheduled queue's whole redelivery cycle-/// (five receives at a 30s visibility timeout) so a rescue can never race a-/// delivery that is still coming and double-send the message.-const STRANDED_SEND_GRACE_SECS: f64 = 300.0;--/// An overdue scheduled send, plus how it reached this sweep.-#[derive(Debug)]-pub struct PendingScheduledSend {- pub link_id: sqlx::types::Uuid,- pub message_id: sqlx::types::Uuid,- /// `true` for an undo-window send this sweep is rescuing, `false` for an- /// ordinary send-later draft whose delivery this sweep always owned.- pub stranded: bool,-}--impl From<PendingScheduledSend> for ScheduledPubsubMessage {- fn from(pending: PendingScheduledSend) -> Self {- ScheduledPubsubMessage {- link_id: pending.link_id,- message_id: pending.message_id,- }- }-}-/// Fetches all scheduled messages that are ready to be sent.-///-/// Returns rows whose `send_time` has passed and that neither the scheduled-/// table (`esm.sent`) nor the message itself (`em.is_sent`) records as sent,-/// in two shapes:-/// - send-later drafts (`em.is_draft = TRUE`), for which this sweep is the-/// only thing that ever enqueues a delivery;-/// - undo-window sends (`em.is_draft = FALSE`), which enqueue their own-/// delivery at send time and so are only picked up once they are-/// [`STRANDED_SEND_GRACE_SECS`] overdue and unclaimed (`processing = FALSE`)-/// — the signature of a send whose enqueue never landed.+/// Only returns messages where:+/// - send_time has passed (< now())+/// - sent = false+/// - the associated email message is still a draft (is_draft = true)#[tracing::instrument(skip(pool), err)]pub async fn fetch_pending_scheduled_messages(pool: &PgPool,-) -> anyhow::Result<Vec<PendingScheduledSend>> {+) -> anyhow::Result<Vec<ScheduledPubsubMessage>> {let messages = sqlx::query_as!(- PendingScheduledSend,+ ScheduledPubsubMessage,r#"SELECT- esm.link_id,- esm.message_id,- NOT em.is_draft AS "stranded!"+ esm.link_id, esm.message_idFROM email_scheduled_messages esmJOIN email_messages em ON em.id = esm.message_idWHEREesm.send_time < now()AND esm.sent = FALSE- AND em.is_sent = FALSE- AND (- em.is_draft = TRUE- OR (- esm.processing = FALSE- AND esm.send_time < now() - make_interval(secs => $1)- )- )+ AND em.is_draft = TRUE"#,- STRANDED_SEND_GRACE_SECS,).fetch_all(pool).await?;diff --git a/services/email_scheduled_handler/src/test.rs b/services/email_scheduled_handler/src/test.rsindex 2a54a3da45..e6aba1e190 100644--- a/services/email_scheduled_handler/src/test.rs+++ b/services/email_scheduled_handler/src/test.rs@@ -13,17 +13,15 @@ async fn fetch_pending_returns_only_eligible_messages(pool: Pool<Postgres>) -> Rlet results = fetch_pending_scheduled_messages(&pool).await?;- // Should return exactly 3 messages:+ // Should return exactly 2 messages:// - Message 1: Draft with past send_time, not sent// - Message 5: Draft with past send_time, not sent (different link)- // - Message 6: Non-draft undo-window send stranded well past its send_time- assert_eq!(results.len(), 3);+ assert_eq!(results.len(), 2);let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f501")?));assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f505")?));- assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f506")?));Ok(())}@@ -62,86 +60,13 @@ async fn fetch_pending_excludes_already_sent(pool: Pool<Postgres>) -> Result<()>migrator = "MACRO_DB_MIGRATIONS",fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages")))]-async fn fetch_pending_excludes_delivered_messages(pool: Pool<Postgres>) -> Result<()> {+async fn fetch_pending_excludes_non_drafts(pool: Pool<Postgres>) -> Result<()> {let results = fetch_pending_scheduled_messages(&pool).await?;let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();- // Messages 4 and 9 both have is_sent = true, so the send already happened- // and the unsent scheduled row is stale. Re-enqueueing either would send- // the same mail twice.+ // Message 4 has is_draft = false, should not be includedassert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f504")?));- assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f509")?));-- Ok(())-}--/// The bug this sweep now covers: `send_message_impl` commits the message as a-/// non-draft with a scheduled row and only then enqueues the delivery, so a-/// failed enqueue leaves a message that is not a draft, was never sent, and has-/// no worker coming for it. The old `em.is_draft = TRUE` predicate could not-/// see those rows at all.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))-)]-async fn fetch_pending_includes_stranded_non_draft_sends(pool: Pool<Postgres>) -> Result<()> {- let results = fetch_pending_scheduled_messages(&pool).await?;-- let rescued = results- .iter()- .find(|r| r.message_id == Uuid::parse_str("00000000-0000-0000-0000-00000000f506").unwrap());-- let rescued = rescued.expect("the stranded send should be picked up");- assert!(- rescued.stranded,- "a rescue must be flagged so the underlying send failure is visible in the logs"- );-- // Ordinary send-later drafts are this sweep's normal work, not rescues.- let draft = results- .iter()- .find(|r| r.message_id == Uuid::parse_str("00000000-0000-0000-0000-00000000f501").unwrap())- .expect("the send-later draft should be picked up");- assert!(!draft.stranded);-- Ok(())-}--/// A non-draft send that is only seconds overdue still has its own queue-/// message in flight (or in SQS redelivery), so rescuing it would double-send.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))-)]-async fn fetch_pending_excludes_non_draft_sends_inside_grace_period(- pool: Pool<Postgres>,-) -> Result<()> {- let results = fetch_pending_scheduled_messages(&pool).await?;-- let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();-- // Message 7 is 30 seconds overdue, well inside the grace period- assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f507")?));-- Ok(())-}--/// A worker already claimed the row, so it is being delivered rather than-/// stranded.-#[sqlx::test(- migrator = "MACRO_DB_MIGRATIONS",- fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))-)]-async fn fetch_pending_excludes_non_draft_sends_being_processed(- pool: Pool<Postgres>,-) -> Result<()> {- let results = fetch_pending_scheduled_messages(&pool).await?;-- let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();-- // Message 8 is well overdue but has processing = true- assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f508")?));Ok(())}diff --git a/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql b/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sqlindex c07602aa38..35a69c4d52 100644--- a/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql+++ b/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql@@ -110,8 +110,7 @@ VALUES ('00000000-0000-0000-0000-000000000f01',NOW(), NOW());--------------------------------------------------------------- Message 4: NOT a draft, already delivered (is_sent = true), with a stale--- unsent scheduled row and a past send_time (SHOULD NOT BE RETURNED)+-- Message 4: NOT a draft (is_draft = false) with past send_time, not sent (SHOULD NOT BE RETURNED)------------------------------------------------------------INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,@@ -152,99 +151,4 @@ VALUES ('00000000-0000-0000-0000-000000000f02','00000000-0000-0000-0000-00000000f505','2020-01-01 14:00:00 +00:00', -- Past datefalse,- NOW(), NOW());----------------------------------------------------------------- Message 6: undo-window send stranded by a failed enqueue — not a draft,--- not sent, unclaimed, and well past the grace period (SHOULD BE RETURNED)---------------------------------------------------------------INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,- has_attachments, is_read, is_starred, is_draft, created_at, updated_at)-VALUES ('00000000-0000-0000-0000-00000000f506',- '00000000-0000-0000-0000-00000000f201',- '00000000-0000-0000-0000-000000000f01',- NULL,- FALSE,- '00000000-0000-0000-0000-0000000cf001',- '2025-01-10 15:00:00 +00:00',- false, false, false, false, NOW(), NOW()); -- is_draft = false, is_sent = false--INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)-VALUES ('00000000-0000-0000-0000-000000000f01',- '00000000-0000-0000-0000-00000000f506',- NOW() - INTERVAL '10 minutes', -- well past the stranded grace period- false,- false,- NOW(), NOW());----------------------------------------------------------------- Message 7: undo-window send inside the grace period — its own queue message--- is still in flight or being retried (SHOULD NOT BE RETURNED)---------------------------------------------------------------INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,- has_attachments, is_read, is_starred, is_draft, created_at, updated_at)-VALUES ('00000000-0000-0000-0000-00000000f507',- '00000000-0000-0000-0000-00000000f201',- '00000000-0000-0000-0000-000000000f01',- NULL,- FALSE,- '00000000-0000-0000-0000-0000000cf001',- '2025-01-10 16:00:00 +00:00',- false, false, false, false, NOW(), NOW());--INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)-VALUES ('00000000-0000-0000-0000-000000000f01',- '00000000-0000-0000-0000-00000000f507',- NOW() - INTERVAL '30 seconds', -- overdue, but inside the grace period- false,- false,- NOW(), NOW());----------------------------------------------------------------- Message 8: overdue undo-window send a worker already claimed--- (SHOULD NOT BE RETURNED)---------------------------------------------------------------INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,- has_attachments, is_read, is_starred, is_draft, created_at, updated_at)-VALUES ('00000000-0000-0000-0000-00000000f508',- '00000000-0000-0000-0000-00000000f201',- '00000000-0000-0000-0000-000000000f01',- NULL,- FALSE,- '00000000-0000-0000-0000-0000000cf001',- '2025-01-10 17:00:00 +00:00',- false, false, false, false, NOW(), NOW());--INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)-VALUES ('00000000-0000-0000-0000-000000000f01',- '00000000-0000-0000-0000-00000000f508',- NOW() - INTERVAL '10 minutes',- false,- true, -- claimed by a worker- NOW(), NOW());----------------------------------------------------------------- Message 9: draft whose message row says it was delivered, with a stale--- unsent scheduled row (SHOULD NOT BE RETURNED)---------------------------------------------------------------INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,- has_attachments, is_read, is_starred, is_draft, created_at, updated_at)-VALUES ('00000000-0000-0000-0000-00000000f509',- '00000000-0000-0000-0000-00000000f201',- '00000000-0000-0000-0000-000000000f01',- 'provider-pending-509',- TRUE,- '00000000-0000-0000-0000-0000000cf001',- '2025-01-10 18:00:00 +00:00',- false, false, false, true, NOW(), NOW()); -- is_draft = true, is_sent = true--INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, created_at, updated_at)-VALUES ('00000000-0000-0000-0000-000000000f01',- '00000000-0000-0000-0000-00000000f509',- '2020-01-01 18:00:00 +00:00',- false,- NOW(), NOW());+ NOW(), NOW());\ No newline at end of file