Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): retry message inserts that race a thread delete (#7889)
teo6cddb8f04c85
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rsindex e061b0b328..c20a86ece3 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs@@ -38,6 +38,7 @@ use models_email::service::pubsub::{DetailedError, FailureReason, ProcessingErrouse models_email::service::thread::Thread;use notification::domain::models::{SendNotificationRequest, SendNotificationRequestBuilder};use notification::domain::service::NotificationIngress;+use sqlx::PgPool;use std::collections::HashSet;use std::result;use std::sync::Arc;@@ -214,14 +215,9 @@ pub async fn upsert_message(// if the message's thread doesn't exist in the database, we need to fetch and insert the whole thread.// if it does exist in the database, we just need to insert the already fetched message.if let Some(thread_db_id) = thread_provider_to_db_map.get(&provider_thread_id) {- process_and_insert_message(ctx, link.id, *thread_db_id, &mut message)- .await- .map_err(|e| {- ProcessingError::NonRetryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e.context("Failed to process and insert message".to_string()),- })- })?;+ // Retryable: a concurrent delete can drop the thread after the lookup+ // above, and the redelivery then takes the whole-thread branch below.+ process_and_insert_message(&ctx.db, link.id, *thread_db_id, &mut message).await?;} else {// Propagated verbatim: fetch_and_insert_thread already routed provider// errors through handle_operation_error (retry queue vs redelivery),@@ -638,31 +634,23 @@ fn thread_from_normalized_messages(}/// Process and insert message-#[tracing::instrument(skip(ctx))]+#[tracing::instrument(skip(db))]async fn process_and_insert_message(- ctx: &PubSubContext,+ db: &PgPool,link_id: Uuid,thread_db_id: Uuid,message: &mut Message,-) -> anyhow::Result<()> {+) -> result::Result<(), ProcessingError> {process_message_pre_insert(message).await;- email_db_client::messages::insert::insert_message(- &ctx.db,- thread_db_id,- message,- link_id,- true,- )- .await- .map_err(|e| {- ProcessingError::Retryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e.context("Failed to insert messages".to_string()),+ email_db_client::messages::insert::insert_message(db, thread_db_id, message, link_id, true)+ .await+ .map_err(|e| {+ ProcessingError::Retryable(DetailedError {+ reason: FailureReason::DatabaseQueryFailed,+ source: e.context("Failed to insert message".to_string()),+ })})- })?;-- Ok(())}/// Notify downstream services about new message in a user's inboxdiff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rsindex 7370120a4b..0c8b6c7554 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs@@ -1,4 +1,5 @@use super::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;fn id(s: &str) -> MacroUserIdStr<'static> {MacroUserIdStr::try_from(s.to_string()).unwrap()@@ -316,3 +317,84 @@ fn signal_filter_requires_importance_and_unshared() {other => panic!("expected thread AND signal predicates, got {other:?}"),}}++fn received_message(link_id: Uuid, thread_db_id: Uuid) -> Message {+ let now = chrono::Utc::now();+ Message {+ db_id: Uuid::now_v7(),+ provider_id: Some("provider-message".to_string()),+ thread_db_id,+ provider_thread_id: Some("provider-thread".to_string()),+ replying_to_id: None,+ global_id: Some("<[email protected]>".to_string()),+ link_id,+ subject: Some("hello".to_string()),+ snippet: None,+ provider_history_id: None,+ internal_date_ts: Some(now),+ sent_at: Some(now),+ size_estimate: None,+ is_read: false,+ is_starred: false,+ is_sent: false,+ is_draft: false,+ scheduled_send_time: None,+ has_attachments: false,+ from: None,+ to: vec![],+ cc: vec![],+ bcc: vec![],+ labels: vec![],+ body_text: Some("body".to_string()),+ body_html_sanitized: None,+ body_macro: None,+ attachments: vec![],+ attachments_draft: vec![],+ attachments_forwarded: vec![],+ headers_json: None,+ created_at: now,+ updated_at: now,+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn inserting_into_a_concurrently_deleted_thread_is_retryable(pool: PgPool) {+ let link_id = Uuid::now_v7();+ let thread_db_id = Uuid::now_v7();+ sqlx::query(+ "INSERT INTO email_links (id, macro_id, fusionauth_user_id, email_address, provider)+ VALUES ($1, 'macro|[email protected]', 'fa-user', '[email protected]', 'GMAIL')",+ )+ .bind(link_id)+ .execute(&pool)+ .await+ .unwrap();+ sqlx::query(+ "INSERT INTO email_threads (id, provider_id, link_id, inbox_visible, is_read)+ VALUES ($1, 'provider-thread', $2, TRUE, FALSE)",+ )+ .bind(thread_db_id)+ .bind(link_id)+ .execute(&pool)+ .await+ .unwrap();+ // The thread lookup found this row; a delete of its last message then+ // removes it before the insert runs.+ sqlx::query("DELETE FROM email_threads WHERE id = $1")+ .bind(thread_db_id)+ .execute(&pool)+ .await+ .unwrap();++ let mut message = received_message(link_id, thread_db_id);+ let result = process_and_insert_message(&pool, link_id, thread_db_id, &mut message).await;++ match result {+ Err(ProcessingError::Retryable(e)) => assert!(+ format!("{:#}", e.source).contains("email_messages_thread_id_fkey"),+ "unexpected insert error: {:#}",+ e.source+ ),+ other => panic!("expected a retryable insert failure, got {other:?}"),+ }+}