Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(email): retry message inserts that race a thread delete (#7889)

teo6cddb8f04c85
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
index e061b0b328..c20a86ece3 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
@@ -38,6 +38,7 @@ use models_email::service::pubsub::{DetailedError, FailureReason, ProcessingErro
use models_email::service::thread::Thread;
use notification::domain::models::{SendNotificationRequest, SendNotificationRequestBuilder};
use notification::domain::service::NotificationIngress;
+use sqlx::PgPool;
use std::collections::HashSet;
use std::result;
use std::sync::Arc;
@@ -214,14 +215,9 @@ pub async fn upsert_message(
// if the message's thread doesn't exist in the database, we need to fetch and insert the whole thread.
// if it does exist in the database, we just need to insert the already fetched message.
if let Some(thread_db_id) = thread_provider_to_db_map.get(&provider_thread_id) {
- process_and_insert_message(ctx, link.id, *thread_db_id, &mut message)
- .await
- .map_err(|e| {
- ProcessingError::NonRetryable(DetailedError {
- reason: FailureReason::DatabaseQueryFailed,
- source: e.context("Failed to process and insert message".to_string()),
- })
- })?;
+ // Retryable: a concurrent delete can drop the thread after the lookup
+ // above, and the redelivery then takes the whole-thread branch below.
+ process_and_insert_message(&ctx.db, link.id, *thread_db_id, &mut message).await?;
} else {
// Propagated verbatim: fetch_and_insert_thread already routed provider
// errors through handle_operation_error (retry queue vs redelivery),
@@ -638,31 +634,23 @@ fn thread_from_normalized_messages(
}
/// Process and insert message
-#[tracing::instrument(skip(ctx))]
+#[tracing::instrument(skip(db))]
async fn process_and_insert_message(
- ctx: &PubSubContext,
+ db: &PgPool,
link_id: Uuid,
thread_db_id: Uuid,
message: &mut Message,
-) -> anyhow::Result<()> {
+) -> result::Result<(), ProcessingError> {
process_message_pre_insert(message).await;
- email_db_client::messages::insert::insert_message(
- &ctx.db,
- thread_db_id,
- message,
- link_id,
- true,
- )
- .await
- .map_err(|e| {
- ProcessingError::Retryable(DetailedError {
- reason: FailureReason::DatabaseQueryFailed,
- source: e.context("Failed to insert messages".to_string()),
+ email_db_client::messages::insert::insert_message(db, thread_db_id, message, link_id, true)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to insert message".to_string()),
+ })
})
- })?;
-
- Ok(())
}
/// Notify downstream services about new message in a user's inbox
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs
index 7370120a4b..0c8b6c7554 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs
@@ -1,4 +1,5 @@
use super::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
fn id(s: &str) -> MacroUserIdStr<'static> {
MacroUserIdStr::try_from(s.to_string()).unwrap()
@@ -316,3 +317,84 @@ fn signal_filter_requires_importance_and_unshared() {
other => panic!("expected thread AND signal predicates, got {other:?}"),
}
}
+
+fn received_message(link_id: Uuid, thread_db_id: Uuid) -> Message {
+ let now = chrono::Utc::now();
+ Message {
+ db_id: Uuid::now_v7(),
+ provider_id: Some("provider-message".to_string()),
+ thread_db_id,
+ provider_thread_id: Some("provider-thread".to_string()),
+ replying_to_id: None,
+ global_id: Some("<[email protected]>".to_string()),
+ link_id,
+ subject: Some("hello".to_string()),
+ snippet: None,
+ provider_history_id: None,
+ internal_date_ts: Some(now),
+ sent_at: Some(now),
+ size_estimate: None,
+ is_read: false,
+ is_starred: false,
+ is_sent: false,
+ is_draft: false,
+ scheduled_send_time: None,
+ has_attachments: false,
+ from: None,
+ to: vec![],
+ cc: vec![],
+ bcc: vec![],
+ labels: vec![],
+ body_text: Some("body".to_string()),
+ body_html_sanitized: None,
+ body_macro: None,
+ attachments: vec![],
+ attachments_draft: vec![],
+ attachments_forwarded: vec![],
+ headers_json: None,
+ created_at: now,
+ updated_at: now,
+ }
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn inserting_into_a_concurrently_deleted_thread_is_retryable(pool: PgPool) {
+ let link_id = Uuid::now_v7();
+ let thread_db_id = Uuid::now_v7();
+ sqlx::query(
+ "INSERT INTO email_links (id, macro_id, fusionauth_user_id, email_address, provider)
+ VALUES ($1, 'macro|[email protected]', 'fa-user', '[email protected]', 'GMAIL')",
+ )
+ .bind(link_id)
+ .execute(&pool)
+ .await
+ .unwrap();
+ sqlx::query(
+ "INSERT INTO email_threads (id, provider_id, link_id, inbox_visible, is_read)
+ VALUES ($1, 'provider-thread', $2, TRUE, FALSE)",
+ )
+ .bind(thread_db_id)
+ .bind(link_id)
+ .execute(&pool)
+ .await
+ .unwrap();
+ // The thread lookup found this row; a delete of its last message then
+ // removes it before the insert runs.
+ sqlx::query("DELETE FROM email_threads WHERE id = $1")
+ .bind(thread_db_id)
+ .execute(&pool)
+ .await
+ .unwrap();
+
+ let mut message = received_message(link_id, thread_db_id);
+ let result = process_and_insert_message(&pool, link_id, thread_db_id, &mut message).await;
+
+ match result {
+ Err(ProcessingError::Retryable(e)) => assert!(
+ format!("{:#}", e.source).contains("email_messages_thread_id_fkey"),
+ "unexpected insert error: {:#}",
+ e.source
+ ),
+ other => panic!("expected a retryable insert failure, got {other:?}"),
+ }
+}