Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

revert(email): stop replaying historical sends (#6580) (#6617)

gbirmanbbc115899444
diff --git a/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.json b/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.json
deleted file mode 100644
index 92cdd72f4b..0000000000
--- a/.sqlx/query-06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n DELETE FROM email_scheduled_messages\n WHERE link_id = $1 AND message_id = $2 AND sent = false\n ",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid",
- "Uuid"
- ]
- },
- "nullable": []
- },
- "hash": "06883296c03b75a3130924d987eb0e15e34380c20568181d32b849f4a5a22d2b"
-}
diff --git a/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.json b/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.json
new file mode 100644
index 0000000000..5c0001485c
--- /dev/null
+++ b/.sqlx/query-2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01.json
@@ -0,0 +1,26 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n esm.link_id, esm.message_id\n FROM email_scheduled_messages esm\n JOIN email_messages em ON em.id = esm.message_id\n WHERE\n esm.send_time < now()\n AND esm.sent = FALSE\n AND em.is_draft = TRUE\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "link_id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "message_id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "2386c156456dbaea33ec5071907677174bdfc3048719f1be818c11b6dd7ada01"
+}
diff --git a/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.json b/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.json
deleted file mode 100644
index 127c7e74ba..0000000000
--- a/.sqlx/query-585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT processing FROM email_scheduled_messages\n WHERE link_id = $1 AND message_id = $2\n FOR UPDATE\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "processing",
- "type_info": "Bool"
- }
- ],
- "parameters": {
- "Left": [
- "Uuid",
- "Uuid"
- ]
- },
- "nullable": [
- false
- ]
- },
- "hash": "585c7d795c18a43ddb4a3f21007730169d11c72d236658452ab0d3b326d47351"
-}
diff --git a/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.json b/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.json
deleted file mode 100644
index 7e470f50d3..0000000000
--- a/.sqlx/query-be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n UPDATE email_messages\n SET is_draft = true, is_sent = false, updated_at = NOW()\n WHERE id = $1 AND link_id = $2 AND is_sent = false\n RETURNING thread_id\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "thread_id",
- "type_info": "Uuid"
- }
- ],
- "parameters": {
- "Left": [
- "Uuid",
- "Uuid"
- ]
- },
- "nullable": [
- false
- ]
- },
- "hash": "be019f365e38f63fd21442137aa0e9c9a034fc115120f37c18145a43ca1598fe"
-}
diff --git a/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.json b/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.json
deleted file mode 100644
index b641ef3d21..0000000000
--- a/.sqlx/query-c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT\n esm.link_id,\n esm.message_id,\n NOT em.is_draft AS \"stranded!\"\n FROM email_scheduled_messages esm\n JOIN email_messages em ON em.id = esm.message_id\n WHERE\n esm.send_time < now()\n AND esm.sent = FALSE\n AND em.is_sent = FALSE\n AND (\n em.is_draft = TRUE\n OR (\n esm.processing = FALSE\n AND esm.send_time < now() - make_interval(secs => $1)\n )\n )\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "link_id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 1,
- "name": "message_id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 2,
- "name": "stranded!",
- "type_info": "Bool"
- }
- ],
- "parameters": {
- "Left": [
- "Float8"
- ]
- },
- "nullable": [
- false,
- false,
- null
- ]
- },
- "hash": "c54f9aedb2f87d706edc410248f8879e136513efae3979d59eb795ca3459d538"
-}
diff --git a/crates/email/src/domain/ports.rs b/crates/email/src/domain/ports.rs
index fb95ce91dd..cbdf85c9b3 100644
--- a/crates/email/src/domain/ports.rs
+++ b/crates/email/src/domain/ports.rs
@@ -269,16 +269,6 @@ pub trait EmailRepo: Send + Sync + 'static {
is_draft: bool,
) -> impl Future<Output = Result<(), Self::Err>> + Send;
- /// Undo an `insert_message(.., is_draft = false)` whose send never made it
- /// onto a queue: put the message back to an unsent draft, drop its pending
- /// scheduled row, and recompute the thread's denormalized metadata.
- /// Leaves a message the scheduled worker already delivered untouched.
- fn revert_sent_message_to_draft(
- &self,
- message_id: Uuid,
- link_id: Uuid,
- ) -> impl Future<Output = Result<(), Self::Err>> + Send;
-
/// Fetch a label by its database ID and link ID.
fn get_label_by_id(
&self,
diff --git a/crates/email/src/domain/service/send.rs b/crates/email/src/domain/service/send.rs
index b154d4cca9..5d19639aa3 100644
--- a/crates/email/src/domain/service/send.rs
+++ b/crates/email/src/domain/service/send.rs
@@ -38,37 +38,10 @@ where
// FE displays "Undo" button for delay_secs. Give extra time for round trip of cancel request
let sqs_delay = delay_secs as i32 + 2;
- // The insert above already committed the message as non-draft with a
- // pending scheduled row. The queue message is the only thing that will
- // ever deliver it — nothing re-derives the send from the row — so an
- // enqueue failure leaves a message that is no longer a draft, was never
- // sent, and has no worker coming for it. Put it back to an unsent draft
- // so the body survives and the caller can retry, and fail loudly.
- if let Err(e) = self
- .enqueuer
+ self.enqueuer
.enqueue_scheduled_message(link.id, created.db_id, Some(sqs_delay))
.await
- {
- let err = anyhow::Error::from(e);
- tracing::error!(error=?err, "failed to enqueue scheduled send, reverting the message to a draft");
- if let Err(revert_err) = self
- .email_repo
- .revert_sent_message_to_draft(created.db_id, link.id)
- .await
- {
- // Both writes failed, so the row is still a non-draft unsent
- // send with a pending scheduled row. That is what the
- // email_scheduled_handler sweep re-enqueues once the row is
- // past its grace period.
- tracing::error!(
- error=?anyhow::Error::from(revert_err),
- link_id=%link.id,
- message_id=%created.db_id,
- "failed to revert the message after a failed enqueue; leaving it for the scheduled sweep",
- );
- }
- return Err(EmailErr::EnqueueErr(err));
- }
+ .map_err(|e| EmailErr::RepoErr(anyhow::Error::from(e)))?;
// The undo-window send is now committed and queued; the matching
// message_sent / message_send_cancelled event resolves it later,
diff --git a/crates/email/src/outbound/email_pg_repo.rs b/crates/email/src/outbound/email_pg_repo.rs
index 07cf9dfcd5..9e94094da8 100644
--- a/crates/email/src/outbound/email_pg_repo.rs
+++ b/crates/email/src/outbound/email_pg_repo.rs
@@ -284,14 +284,6 @@ impl EmailRepo for EmailPgRepo {
draft::insert_message(&self.pool, input, contacts, link_id, new_thread, is_draft).await
}
- async fn revert_sent_message_to_draft(
- &self,
- message_id: Uuid,
- link_id: Uuid,
- ) -> Result<(), Self::Err> {
- draft::revert_sent_message_to_draft(&self.pool, message_id, link_id).await
- }
-
async fn get_label_by_id(
&self,
label_id: Uuid,
diff --git a/crates/email/src/outbound/email_pg_repo/draft.rs b/crates/email/src/outbound/email_pg_repo/draft.rs
index e6b4305fea..b68c2c84f0 100644
--- a/crates/email/src/outbound/email_pg_repo/draft.rs
+++ b/crates/email/src/outbound/email_pg_repo/draft.rs
@@ -60,87 +60,6 @@ pub(crate) async fn insert_message(
Ok(())
}
-/// Revert a message that `insert_message(.., is_draft = false)` optimistically
-/// marked as outgoing back to an unsent draft, and drop its pending scheduled
-/// row.
-///
-/// The whole revert hinges on the send not having happened: a message the
-/// scheduled worker already delivered (`is_sent`), or is delivering right now
-/// (`processing` on its scheduled row), is left exactly as it is, message row
-/// and scheduled row both. `get_message_to_send` does not re-check `is_draft`,
-/// so a worker that has claimed the row sends the mail whatever this does —
-/// reverting underneath it would deliver the email *and* leave a live draft to
-/// send again.
-#[tracing::instrument(skip(pool), err)]
-pub(crate) async fn revert_sent_message_to_draft(
- pool: &PgPool,
- message_id: Uuid,
- link_id: Uuid,
-) -> Result<(), sqlx::Error> {
- let mut tx = pool.begin().await?;
-
- // Lock the scheduled row up front rather than testing `processing` inside
- // the UPDATE below. A worker's claim is a single committed statement, so an
- // unlocked read only proves nothing had claimed the row at that instant: a
- // claim landing before the DELETE would have its row deleted out from under
- // it and still send. Holding the lock makes a concurrent claim wait — it
- // then finds no row and skips — and makes a claim that got here first
- // visible to the check.
- let claimed = sqlx::query_scalar!(
- r#"
- SELECT processing FROM email_scheduled_messages
- WHERE link_id = $1 AND message_id = $2
- FOR UPDATE
- "#,
- link_id,
- message_id,
- )
- .fetch_optional(&mut *tx)
- .await?;
-
- if claimed == Some(true) {
- tx.commit().await?;
- return Ok(());
- }
-
- let thread_db_id = sqlx::query_scalar!(
- r#"
- UPDATE email_messages
- SET is_draft = true, is_sent = false, updated_at = NOW()
- WHERE id = $1 AND link_id = $2 AND is_sent = false
- RETURNING thread_id
- "#,
- message_id,
- link_id,
- )
- .fetch_optional(&mut *tx)
- .await?;
-
- let Some(thread_db_id) = thread_db_id else {
- tx.commit().await?;
- return Ok(());
- };
-
- sqlx::query!(
- r#"
- DELETE FROM email_scheduled_messages
- WHERE link_id = $1 AND message_id = $2 AND sent = false
- "#,
- link_id,
- message_id,
- )
- .execute(&mut *tx)
- .await?;
-
- // Drafts count toward inbox_visible and the latest-message timestamps, so
- // the metadata the insert computed for an outgoing message no longer
- // describes the thread. Same recompute delete_draft_message does.
- thread::update_thread_metadata(&mut tx, thread_db_id, link_id).await?;
-
- tx.commit().await?;
- Ok(())
-}
-
/// Upsert a draft message row.
pub(crate) async fn upsert_draft(
tx: &mut sqlx::PgConnection,
diff --git a/crates/email/src/outbound/email_pg_repo/test.rs b/crates/email/src/outbound/email_pg_repo/test.rs
index f6c77e3f34..cc4284ad1f 100644
--- a/crates/email/src/outbound/email_pg_repo/test.rs
+++ b/crates/email/src/outbound/email_pg_repo/test.rs
@@ -9,7 +9,6 @@ mod message;
mod preview;
mod project;
mod project_scope_dynamic_query;
-mod send;
mod settings;
mod signal_flag;
mod thread;
diff --git a/crates/email/src/outbound/email_pg_repo/test/send.rs b/crates/email/src/outbound/email_pg_repo/test/send.rs
deleted file mode 100644
index 72fd386295..0000000000
--- a/crates/email/src/outbound/email_pg_repo/test/send.rs
+++ /dev/null
@@ -1,485 +0,0 @@
-//! Durability of the undo-window send path.
-//!
-//! `send_message_impl` commits the message as a non-draft with a pending
-//! scheduled row and only then enqueues the delivery, so these tests run the
-//! domain service against the real repo with an enqueuer that fails, and pin
-//! down what the database is left holding.
-
-use super::*;
-use crate::domain::models::{ContactInfo, CreateDraftInput};
-use crate::domain::ports::EmailMessageEnqueuer;
-use crate::domain::service::EmailServiceImpl;
-use crm::domain::service::NoOpCrmService;
-use frecency::domain::services::FrecencyQueryServiceImpl;
-use frecency::outbound::postgres::FrecencyPgStorage;
-use macro_event_broker::NoopMacroEventBroker;
-use macro_user_id::email::EmailStr;
-
-const LINK_ID: &str = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
-const SENT_MESSAGE_ID: &str = "ee000001-0000-0000-0000-000000000001";
-const DRAFT_MESSAGE_ID: &str = "ee000002-0000-0000-0000-000000000002";
-/// The draft that is the only message in its thread, so thread metadata swings
-/// entirely on whether it counts as a draft.
-const LONE_DRAFT_MESSAGE_ID: &str = "ee000004-0000-0000-0000-000000000004";
-const LONE_DRAFT_THREAD_ID: &str = "33333333-3333-3333-3333-333333333333";
-
-/// Stands in for SQS being unreachable — the shape of the outage that stranded
-/// sends in production.
-#[derive(Clone)]
-struct FailingEnqueuer;
-
-impl EmailMessageEnqueuer for FailingEnqueuer {
- type Err = anyhow::Error;
-
- async fn enqueue_scheduled_message(
- &self,
- _link_id: Uuid,
- _message_id: Uuid,
- _delay_seconds: Option<i32>,
- ) -> Result<(), Self::Err> {
- Err(anyhow::anyhow!("queue unavailable"))
- }
-
- async fn enqueue_gmail_ops_modify_labels_batch(
- &self,
- _link_id: Uuid,
- _messages: Vec<(Uuid, String)>,
- _labels_to_add: Vec<String>,
- _labels_to_remove: Vec<String>,
- ) -> Result<(), Self::Err> {
- Err(anyhow::anyhow!("queue unavailable"))
- }
-
- async fn enqueue_gmail_ops_block_sender(
- &self,
- _link_id: Uuid,
- _email_address: String,
- ) -> Result<(), Self::Err> {
- Err(anyhow::anyhow!("queue unavailable"))
- }
-
- async fn enqueue_gmail_ops_unblock_sender(
- &self,
- _link_id: Uuid,
- _email_address: String,
- ) -> Result<(), Self::Err> {
- Err(anyhow::anyhow!("queue unavailable"))
- }
-}
-
-/// The send path touches neither frecency nor CRM nor the event broker, so
-/// those collaborators are no-ops; the repo is the real thing.
-type TestService<E> = EmailServiceImpl<
- EmailPgRepo,
- FrecencyQueryServiceImpl<FrecencyPgStorage>,
- E,
- NoOpCrmService,
- (),
- NoopMacroEventBroker,
->;
-
-fn service<E: EmailMessageEnqueuer>(pool: Pool<Postgres>, enqueuer: E) -> TestService<E> {
- EmailServiceImpl {
- email_repo: EmailPgRepo::new(pool.clone()),
- frecency_service: FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(pool)),
- enqueuer,
- crm_service: NoOpCrmService,
- entity_access_management_service: (),
- macro_event_broker: NoopMacroEventBroker,
- sent_undo_delay_secs: 10,
- }
-}
-
-fn link() -> anyhow::Result<Link> {
- Ok(Link {
- id: Uuid::parse_str(LINK_ID)?,
- macro_id: MacroUserIdStr::try_from_email("[email protected]")?,
- fusionauth_user_id: "fa-user-1".to_owned(),
- email_address: EmailStr::try_from("[email protected]".to_owned())?,
- provider: UserProvider::Gmail,
- is_sync_active: true,
- is_primary: true,
- created_at: Utc::now(),
- updated_at: Utc::now(),
- })
-}
-
-fn send_input(db_id: Option<Uuid>) -> CreateDraftInput {
- CreateDraftInput {
- db_id,
- provider_id: None,
- replying_to_id: None,
- provider_thread_id: None,
- thread_db_id: None,
- subject: "Quarterly update".to_owned(),
- to: vec![ContactInfo {
- email: "[email protected]".to_owned(),
- name: None,
- photo_url: None,
- }],
- cc: vec![],
- bcc: vec![],
- body_text: Some("Numbers attached.".to_owned()),
- body_html: None,
- body_macro: None,
- headers_json: None,
- send_time: None,
- include_signature: Some(false),
- actor: None,
- }
-}
-
-/// `(is_draft, is_sent)` for a message row.
-async fn message_flags(pool: &Pool<Postgres>, message_id: Uuid) -> anyhow::Result<(bool, bool)> {
- let row = sqlx::query!(
- "SELECT is_draft, is_sent FROM email_messages WHERE id = $1",
- message_id,
- )
- .fetch_one(pool)
- .await?;
- Ok((row.is_draft, row.is_sent))
-}
-
-async fn scheduled_row_count(pool: &Pool<Postgres>, message_id: Uuid) -> anyhow::Result<i64> {
- let count = sqlx::query_scalar!(
- r#"SELECT COUNT(*) AS "count!" FROM email_scheduled_messages WHERE message_id = $1"#,
- message_id,
- )
- .fetch_one(pool)
- .await?;
- Ok(count)
-}
-
-struct ThreadInboxState {
- inbox_visible: bool,
- latest_inbound_message_ts: Option<chrono::DateTime<Utc>>,
-}
-
-async fn thread_inbox_state(
- pool: &Pool<Postgres>,
- thread_id: Uuid,
-) -> anyhow::Result<ThreadInboxState> {
- let row = sqlx::query!(
- "SELECT inbox_visible, latest_inbound_message_ts FROM email_threads WHERE id = $1",
- thread_id,
- )
- .fetch_one(pool)
- .await?;
-
- Ok(ThreadInboxState {
- inbox_visible: row.inbox_visible,
- latest_inbound_message_ts: row.latest_inbound_message_ts,
- })
-}
-
-// ── send_message_impl ─────────────────────────────────────────────
-
-/// The regression: a failed enqueue used to leave the message flipped to
-/// non-draft with a scheduled row and nothing in the queue to deliver it, so
-/// the draft vanished from the UI and no mail ever went out. The sweep could
-/// not rescue it either, because it only looked at drafts.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../../../fixtures", scripts("email_draft"))
-)]
-async fn failed_enqueue_leaves_the_message_as_an_unsent_draft(
- pool: Pool<Postgres>,
-) -> anyhow::Result<()> {
- let link = link()?;
- let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;
- let service = service(pool.clone(), FailingEnqueuer);
-
- let err = service
- .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))
- .await
- .expect_err("enqueue failed, so the send must fail");
-
- assert!(
- matches!(err, EmailErr::EnqueueErr(_)),
- "expected EnqueueErr, got {err:?}"
- );
-
- let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;
- assert!(is_draft, "the draft must survive a failed send");
- assert!(!is_sent);
- assert_eq!(
- scheduled_row_count(&pool, draft_id).await?,
- 0,
- "no scheduled row should be left pointing at a send that will never happen"
- );
-
- Ok(())
-}
-
-/// The agent/AI path sends without an existing draft (`db_id: None`), so the
-/// revert has to leave the freshly composed body behind as a draft rather than
-/// losing it.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../../../fixtures", scripts("email_draft"))
-)]
-async fn failed_enqueue_keeps_a_brand_new_send_as_a_draft(
- pool: Pool<Postgres>,
-) -> anyhow::Result<()> {
- let link = link()?;
- let service = service(pool.clone(), FailingEnqueuer);
-
- let err = service
- .send_message_impl(&link, &[link.clone()], send_input(None))
- .await
- .expect_err("enqueue failed, so the send must fail");
- assert!(
- matches!(err, EmailErr::EnqueueErr(_)),
- "expected EnqueueErr, got {err:?}"
- );
-
- let new_message_id: Uuid = sqlx::query_scalar!(
- r#"
- SELECT id FROM email_messages
- WHERE link_id = $1 AND subject = 'Quarterly update'
- "#,
- link.id,
- )
- .fetch_one(&pool)
- .await?;
-
- let (is_draft, is_sent) = message_flags(&pool, new_message_id).await?;
- assert!(is_draft, "the composed body must survive as a draft");
- assert!(!is_sent);
- assert_eq!(scheduled_row_count(&pool, new_message_id).await?, 0);
-
- Ok(())
-}
-
-/// The happy path still commits the message as outgoing with a pending
-/// scheduled row — the revert must not fire when the enqueue succeeds.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../../../fixtures", scripts("email_draft"))
-)]
-async fn successful_enqueue_commits_the_send(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let link = link()?;
- let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;
- let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);
-
- let created = service
- .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))
- .await?;
- assert_eq!(created.db_id, draft_id);
-
- let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;
- assert!(!is_draft, "an enqueued send is no longer a draft");
- assert!(
- !is_sent,
- "the worker marks it sent once Gmail accepts it, not here"
- );
- assert_eq!(scheduled_row_count(&pool, draft_id).await?, 1);
-
- Ok(())
-}
-
-// ── revert_sent_message_to_draft ──────────────────────────────────
-
-/// Drafts count toward a thread's denormalized `inbox_visible` and
-/// latest-message timestamps, so the metadata the insert computed for an
-/// outgoing message has to be recomputed on the way back.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../../../fixtures", scripts("email_draft"))
-)]
-async fn revert_recomputes_thread_metadata(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let link = link()?;
- let draft_id = Uuid::parse_str(LONE_DRAFT_MESSAGE_ID)?;
- let thread_id = Uuid::parse_str(LONE_DRAFT_THREAD_ID)?;
- let repo = EmailPgRepo::new(pool.clone());
- let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);
-
- service
- .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))
- .await?;
-
- let sent = thread_inbox_state(&pool, thread_id).await?;
- assert!(
- !sent.inbox_visible && sent.latest_inbound_message_ts.is_none(),
- "the thread's only message is outgoing, so nothing keeps it in the inbox"
- );
-
- repo.revert_sent_message_to_draft(draft_id, link.id).await?;
-
- let reverted = thread_inbox_state(&pool, thread_id).await?;
- assert!(
- reverted.inbox_visible,
- "a thread holding a draft again belongs back in the inbox"
- );
- assert!(reverted.latest_inbound_message_ts.is_some());
-
- Ok(())
-}
-
-/// Losing a queue message must never be able to un-send mail that Gmail has
-/// already accepted, so the revert is a no-op once the worker has marked the
-/// message sent.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../../../fixtures", scripts("email_draft"))
-)]
-async fn revert_leaves_an_already_delivered_message_alone(
- pool: Pool<Postgres>,
-) -> anyhow::Result<()> {
- let link_id = Uuid::parse_str(LINK_ID)?;
- let sent_id = Uuid::parse_str(SENT_MESSAGE_ID)?;
- let repo = EmailPgRepo::new(pool.clone());
-
- sqlx::query!(
- r#"
- INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, created_at, updated_at)
- VALUES ($1, $2, NOW(), false, NOW(), NOW())
- "#,
- link_id,
- sent_id,
- )
- .execute(&pool)
- .await?;
-
- repo.revert_sent_message_to_draft(sent_id, link_id).await?;
-
- let (is_draft, is_sent) = message_flags(&pool, sent_id).await?;
- assert!(!is_draft, "a delivered message must not become a draft");
- assert!(is_sent);
- assert_eq!(
- scheduled_row_count(&pool, sent_id).await?,
- 1,
- "the revert bails before touching the scheduled row"
- );
-
- Ok(())
-}
-
-/// A worker that has claimed the scheduled row will deliver the mail whether or
-/// not this revert runs (`get_message_to_send` does not re-check `is_draft`), so
-/// reverting underneath it would send the email and leave a live draft behind.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../../../fixtures", scripts("email_draft"))
-)]
-async fn revert_leaves_a_message_a_worker_is_delivering_alone(
- pool: Pool<Postgres>,
-) -> anyhow::Result<()> {
- let link = link()?;
- let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;
- let repo = EmailPgRepo::new(pool.clone());
- let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);
-
- service
- .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))
- .await?;
-
- sqlx::query!(
- r#"
- UPDATE email_scheduled_messages SET processing = true
- WHERE link_id = $1 AND message_id = $2
- "#,
- link.id,
- draft_id,
- )
- .execute(&pool)
- .await?;
-
- repo.revert_sent_message_to_draft(draft_id, link.id).await?;
-
- let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;
- assert!(
- !is_draft,
- "a send in flight must not become a draft that can be sent again"
- );
- assert!(!is_sent);
- assert_eq!(
- scheduled_row_count(&pool, draft_id).await?,
- 1,
- "the worker still needs its row to mark the send as sent"
- );
-
- Ok(())
-}
-
-/// The guard has to be a lock, not a read. A worker's claim is one committed
-/// statement, so a revert that merely *read* `processing` could have a claim
-/// land between that read and its DELETE, delete the row the worker just
-/// claimed, and leave the worker to send from the data it already holds.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../../../fixtures", scripts("email_draft"))
-)]
-async fn revert_waits_for_a_claim_racing_it(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let link = link()?;
- let draft_id = Uuid::parse_str(DRAFT_MESSAGE_ID)?;
- let repo = EmailPgRepo::new(pool.clone());
- let service = service(pool.clone(), crate::domain::ports::NoOpEnqueuer);
-
- service
- .send_message_impl(&link, &[link.clone()], send_input(Some(draft_id)))
- .await?;
-
- // A worker claims the row exactly the way the scheduled consumer does, but
- // holds the transaction open so the claim is in flight, not settled.
- let mut claim = pool.begin().await?;
- sqlx::query!(
- r#"
- UPDATE email_scheduled_messages SET processing = true, updated_at = NOW()
- WHERE link_id = $1 AND message_id = $2
- "#,
- link.id,
- draft_id,
- )
- .execute(&mut *claim)
- .await?;
-
- let mut revert = tokio::spawn({
- let repo = repo.clone();
- let link_id = link.id;
- async move { repo.revert_sent_message_to_draft(draft_id, link_id).await }
- });
-
- assert!(
- tokio::time::timeout(std::time::Duration::from_millis(500), &mut revert)
- .await
- .is_err(),
- "the revert must wait on the in-flight claim"
- );
-
- // What the claim lands on is the point. Testing `processing` with an
- // unlocked read lets the revert flip the message to a draft first and only
- // then wait — on the DELETE's row lock — so once the claim commits it
- // deletes the row that claim just took and leaves a live draft behind.
- claim.commit().await?;
- revert.await??;
-
- let (is_draft, is_sent) = message_flags(&pool, draft_id).await?;
- assert!(
- !is_draft,
- "once the claim lands the revert must see it and leave the send alone"
- );
- assert!(!is_sent);
- assert_eq!(
- scheduled_row_count(&pool, draft_id).await?,
- 1,
- "the claimed row must survive for the worker to mark the send as sent"
- );
-
- Ok(())
-}
-
-/// A message that was never inserted (or belongs to another inbox) must not
-/// make the revert fail — it runs on a path that is already handling an error.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "../../../../fixtures", scripts("email_draft"))
-)]
-async fn revert_is_a_no_op_for_an_unknown_message(pool: Pool<Postgres>) -> anyhow::Result<()> {
- let repo = EmailPgRepo::new(pool);
-
- repo.revert_sent_message_to_draft(Uuid::from_u128(999), Uuid::parse_str(LINK_ID)?)
- .await?;
-
- Ok(())
-}
diff --git a/services/email_scheduled_handler/src/handler.rs b/services/email_scheduled_handler/src/handler.rs
index a6c8900504..6124137641 100644
--- a/services/email_scheduled_handler/src/handler.rs
+++ b/services/email_scheduled_handler/src/handler.rs
@@ -24,25 +24,12 @@ pub async fn handler(
tracing::info!(notifications = ?notifications, "Sending scheduled email pubsub messages");
}
- for pending in notifications.into_iter() {
- let message_id = pending.message_id;
- let link_id = pending.link_id;
-
- // A stranded row means the send path committed the message but failed
- // to enqueue it and failed to revert it, so this sweep is the only
- // reason the mail goes out at all. Warn so the underlying failure is
- // visible instead of being quietly papered over.
- if pending.stranded {
- tracing::warn!(
- link_id = link_id.to_string(),
- message_id = message_id.to_string(),
- "Re-enqueueing a send whose original enqueue never landed",
- );
- }
-
+ for notif in notifications.into_iter() {
+ let message_id = notif.message_id;
+ let link_id = notif.link_id;
if let Err(e) = ctx
.sqs_client
- .enqueue_email_scheduled_message(pending.into(), None)
+ .enqueue_email_scheduled_message(notif, None)
.await
{
tracing::error!(
@@ -57,73 +44,27 @@ pub async fn handler(
Ok(())
}
-/// How long past its `send_time` a non-draft scheduled row must sit before the
-/// sweep treats it as stranded rather than in flight.
-///
-/// The undo-window send path (`email`'s `send_message_impl`) inserts its
-/// message as a non-draft and puts the queue message on a short delay, so such
-/// a row is normally delivered seconds after its `send_time`. The grace period
-/// has to clear that delay plus the scheduled queue's whole redelivery cycle
-/// (five receives at a 30s visibility timeout) so a rescue can never race a
-/// delivery that is still coming and double-send the message.
-const STRANDED_SEND_GRACE_SECS: f64 = 300.0;
-
-/// An overdue scheduled send, plus how it reached this sweep.
-#[derive(Debug)]
-pub struct PendingScheduledSend {
- pub link_id: sqlx::types::Uuid,
- pub message_id: sqlx::types::Uuid,
- /// `true` for an undo-window send this sweep is rescuing, `false` for an
- /// ordinary send-later draft whose delivery this sweep always owned.
- pub stranded: bool,
-}
-
-impl From<PendingScheduledSend> for ScheduledPubsubMessage {
- fn from(pending: PendingScheduledSend) -> Self {
- ScheduledPubsubMessage {
- link_id: pending.link_id,
- message_id: pending.message_id,
- }
- }
-}
-
/// Fetches all scheduled messages that are ready to be sent.
-///
-/// Returns rows whose `send_time` has passed and that neither the scheduled
-/// table (`esm.sent`) nor the message itself (`em.is_sent`) records as sent,
-/// in two shapes:
-/// - send-later drafts (`em.is_draft = TRUE`), for which this sweep is the
-/// only thing that ever enqueues a delivery;
-/// - undo-window sends (`em.is_draft = FALSE`), which enqueue their own
-/// delivery at send time and so are only picked up once they are
-/// [`STRANDED_SEND_GRACE_SECS`] overdue and unclaimed (`processing = FALSE`)
-/// — the signature of a send whose enqueue never landed.
+/// Only returns messages where:
+/// - send_time has passed (< now())
+/// - sent = false
+/// - the associated email message is still a draft (is_draft = true)
#[tracing::instrument(skip(pool), err)]
pub async fn fetch_pending_scheduled_messages(
pool: &PgPool,
-) -> anyhow::Result<Vec<PendingScheduledSend>> {
+) -> anyhow::Result<Vec<ScheduledPubsubMessage>> {
let messages = sqlx::query_as!(
- PendingScheduledSend,
+ ScheduledPubsubMessage,
r#"
SELECT
- esm.link_id,
- esm.message_id,
- NOT em.is_draft AS "stranded!"
+ esm.link_id, esm.message_id
FROM email_scheduled_messages esm
JOIN email_messages em ON em.id = esm.message_id
WHERE
esm.send_time < now()
AND esm.sent = FALSE
- AND em.is_sent = FALSE
- AND (
- em.is_draft = TRUE
- OR (
- esm.processing = FALSE
- AND esm.send_time < now() - make_interval(secs => $1)
- )
- )
+ AND em.is_draft = TRUE
"#,
- STRANDED_SEND_GRACE_SECS,
)
.fetch_all(pool)
.await?;
diff --git a/services/email_scheduled_handler/src/test.rs b/services/email_scheduled_handler/src/test.rs
index 2a54a3da45..e6aba1e190 100644
--- a/services/email_scheduled_handler/src/test.rs
+++ b/services/email_scheduled_handler/src/test.rs
@@ -13,17 +13,15 @@ async fn fetch_pending_returns_only_eligible_messages(pool: Pool<Postgres>) -> R
let results = fetch_pending_scheduled_messages(&pool).await?;
- // Should return exactly 3 messages:
+ // Should return exactly 2 messages:
// - Message 1: Draft with past send_time, not sent
// - Message 5: Draft with past send_time, not sent (different link)
- // - Message 6: Non-draft undo-window send stranded well past its send_time
- assert_eq!(results.len(), 3);
+ assert_eq!(results.len(), 2);
let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();
assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f501")?));
assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f505")?));
- assert!(message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f506")?));
Ok(())
}
@@ -62,86 +60,13 @@ async fn fetch_pending_excludes_already_sent(pool: Pool<Postgres>) -> Result<()>
migrator = "MACRO_DB_MIGRATIONS",
fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))
)]
-async fn fetch_pending_excludes_delivered_messages(pool: Pool<Postgres>) -> Result<()> {
+async fn fetch_pending_excludes_non_drafts(pool: Pool<Postgres>) -> Result<()> {
let results = fetch_pending_scheduled_messages(&pool).await?;
let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();
- // Messages 4 and 9 both have is_sent = true, so the send already happened
- // and the unsent scheduled row is stale. Re-enqueueing either would send
- // the same mail twice.
+ // Message 4 has is_draft = false, should not be included
assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f504")?));
- assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f509")?));
-
- Ok(())
-}
-
-/// The bug this sweep now covers: `send_message_impl` commits the message as a
-/// non-draft with a scheduled row and only then enqueues the delivery, so a
-/// failed enqueue leaves a message that is not a draft, was never sent, and has
-/// no worker coming for it. The old `em.is_draft = TRUE` predicate could not
-/// see those rows at all.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))
-)]
-async fn fetch_pending_includes_stranded_non_draft_sends(pool: Pool<Postgres>) -> Result<()> {
- let results = fetch_pending_scheduled_messages(&pool).await?;
-
- let rescued = results
- .iter()
- .find(|r| r.message_id == Uuid::parse_str("00000000-0000-0000-0000-00000000f506").unwrap());
-
- let rescued = rescued.expect("the stranded send should be picked up");
- assert!(
- rescued.stranded,
- "a rescue must be flagged so the underlying send failure is visible in the logs"
- );
-
- // Ordinary send-later drafts are this sweep's normal work, not rescues.
- let draft = results
- .iter()
- .find(|r| r.message_id == Uuid::parse_str("00000000-0000-0000-0000-00000000f501").unwrap())
- .expect("the send-later draft should be picked up");
- assert!(!draft.stranded);
-
- Ok(())
-}
-
-/// A non-draft send that is only seconds overdue still has its own queue
-/// message in flight (or in SQS redelivery), so rescuing it would double-send.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))
-)]
-async fn fetch_pending_excludes_non_draft_sends_inside_grace_period(
- pool: Pool<Postgres>,
-) -> Result<()> {
- let results = fetch_pending_scheduled_messages(&pool).await?;
-
- let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();
-
- // Message 7 is 30 seconds overdue, well inside the grace period
- assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f507")?));
-
- Ok(())
-}
-
-/// A worker already claimed the row, so it is being delivered rather than
-/// stranded.
-#[sqlx::test(
- migrator = "MACRO_DB_MIGRATIONS",
- fixtures(path = "test/fixtures", scripts("fetch_pending_scheduled_messages"))
-)]
-async fn fetch_pending_excludes_non_draft_sends_being_processed(
- pool: Pool<Postgres>,
-) -> Result<()> {
- let results = fetch_pending_scheduled_messages(&pool).await?;
-
- let message_ids: Vec<Uuid> = results.iter().map(|r| r.message_id).collect();
-
- // Message 8 is well overdue but has processing = true
- assert!(!message_ids.contains(&Uuid::parse_str("00000000-0000-0000-0000-00000000f508")?));
Ok(())
}
diff --git a/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql b/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql
index c07602aa38..35a69c4d52 100644
--- a/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql
+++ b/services/email_scheduled_handler/src/test/fixtures/fetch_pending_scheduled_messages.sql
@@ -110,8 +110,7 @@ VALUES ('00000000-0000-0000-0000-000000000f01',
NOW(), NOW());
------------------------------------------------------------
--- Message 4: NOT a draft, already delivered (is_sent = true), with a stale
--- unsent scheduled row and a past send_time (SHOULD NOT BE RETURNED)
+-- Message 4: NOT a draft (is_draft = false) with past send_time, not sent (SHOULD NOT BE RETURNED)
------------------------------------------------------------
INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,
@@ -152,99 +151,4 @@ VALUES ('00000000-0000-0000-0000-000000000f02',
'00000000-0000-0000-0000-00000000f505',
'2020-01-01 14:00:00 +00:00', -- Past date
false,
- NOW(), NOW());
-
-------------------------------------------------------------
--- Message 6: undo-window send stranded by a failed enqueue — not a draft,
--- not sent, unclaimed, and well past the grace period (SHOULD BE RETURNED)
-------------------------------------------------------------
-
-INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,
- has_attachments, is_read, is_starred, is_draft, created_at, updated_at)
-VALUES ('00000000-0000-0000-0000-00000000f506',
- '00000000-0000-0000-0000-00000000f201',
- '00000000-0000-0000-0000-000000000f01',
- NULL,
- FALSE,
- '00000000-0000-0000-0000-0000000cf001',
- '2025-01-10 15:00:00 +00:00',
- false, false, false, false, NOW(), NOW()); -- is_draft = false, is_sent = false
-
-INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)
-VALUES ('00000000-0000-0000-0000-000000000f01',
- '00000000-0000-0000-0000-00000000f506',
- NOW() - INTERVAL '10 minutes', -- well past the stranded grace period
- false,
- false,
- NOW(), NOW());
-
-------------------------------------------------------------
--- Message 7: undo-window send inside the grace period — its own queue message
--- is still in flight or being retried (SHOULD NOT BE RETURNED)
-------------------------------------------------------------
-
-INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,
- has_attachments, is_read, is_starred, is_draft, created_at, updated_at)
-VALUES ('00000000-0000-0000-0000-00000000f507',
- '00000000-0000-0000-0000-00000000f201',
- '00000000-0000-0000-0000-000000000f01',
- NULL,
- FALSE,
- '00000000-0000-0000-0000-0000000cf001',
- '2025-01-10 16:00:00 +00:00',
- false, false, false, false, NOW(), NOW());
-
-INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)
-VALUES ('00000000-0000-0000-0000-000000000f01',
- '00000000-0000-0000-0000-00000000f507',
- NOW() - INTERVAL '30 seconds', -- overdue, but inside the grace period
- false,
- false,
- NOW(), NOW());
-
-------------------------------------------------------------
--- Message 8: overdue undo-window send a worker already claimed
--- (SHOULD NOT BE RETURNED)
-------------------------------------------------------------
-
-INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,
- has_attachments, is_read, is_starred, is_draft, created_at, updated_at)
-VALUES ('00000000-0000-0000-0000-00000000f508',
- '00000000-0000-0000-0000-00000000f201',
- '00000000-0000-0000-0000-000000000f01',
- NULL,
- FALSE,
- '00000000-0000-0000-0000-0000000cf001',
- '2025-01-10 17:00:00 +00:00',
- false, false, false, false, NOW(), NOW());
-
-INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, processing, created_at, updated_at)
-VALUES ('00000000-0000-0000-0000-000000000f01',
- '00000000-0000-0000-0000-00000000f508',
- NOW() - INTERVAL '10 minutes',
- false,
- true, -- claimed by a worker
- NOW(), NOW());
-
-------------------------------------------------------------
--- Message 9: draft whose message row says it was delivered, with a stale
--- unsent scheduled row (SHOULD NOT BE RETURNED)
-------------------------------------------------------------
-
-INSERT INTO email_messages (id, thread_id, link_id, provider_id, is_sent, from_contact_id, internal_date_ts,
- has_attachments, is_read, is_starred, is_draft, created_at, updated_at)
-VALUES ('00000000-0000-0000-0000-00000000f509',
- '00000000-0000-0000-0000-00000000f201',
- '00000000-0000-0000-0000-000000000f01',
- 'provider-pending-509',
- TRUE,
- '00000000-0000-0000-0000-0000000cf001',
- '2025-01-10 18:00:00 +00:00',
- false, false, false, true, NOW(), NOW()); -- is_draft = true, is_sent = true
-
-INSERT INTO email_scheduled_messages (link_id, message_id, send_time, sent, created_at, updated_at)
-VALUES ('00000000-0000-0000-0000-000000000f01',
- '00000000-0000-0000-0000-00000000f509',
- '2020-01-01 18:00:00 +00:00',
- false,
- NOW(), NOW());
+ NOW(), NOW());
\ No newline at end of file