Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(email): stop oversized provider values wedging backfill jobs (#5886)
gbirman8aafa827e372
diff --git a/crates/email_db_client/src/messages/insert.rs b/crates/email_db_client/src/messages/insert.rsindex ce1b5c88a2..2c9898033e 100644--- a/crates/email_db_client/src/messages/insert.rs+++ b/crates/email_db_client/src/messages/insert.rs@@ -1,3 +1,6 @@+#[cfg(test)]+mod test;+use crate::attachments::provider;use crate::messages::replying_to_id;use crate::parse::service_to_db::addresses_from_message;diff --git a/crates/email_db_client/src/messages/insert/test.rs b/crates/email_db_client/src/messages/insert/test.rsnew file mode 100644index 0000000000..5933b5f61d--- /dev/null+++ b/crates/email_db_client/src/messages/insert/test.rs@@ -0,0 +1,157 @@+use super::*;+use chrono::Utc;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use models_email::email::service::address::ContactInfo;+use models_email::email::service::attachment::Attachment;+use sqlx::{Pool, Postgres};++const LINK_ID: &str = "00000000-0000-0000-0000-000000000001";+const THREAD_ID: &str = "10000000-0000-0000-0000-000000000001";++/// The Apple Mail one-click-unsubscribe address that wedged mwatts42's+/// backfill: 505 characters, well past `email_contacts.email_address`.+fn unsubscribe_address() -> String {+ let address = format!(+ "v3_{}@unsubscribe-06.emailinboundprocessing.com",+ "t".repeat(460)+ );+ assert_eq!(address.chars().count(), 505);+ address+}++fn message_with_oversized_fields(link_id: Uuid, thread_db_id: Uuid) -> message::Message {+ let now = Utc::now();+ message::Message {+ db_id: Uuid::now_v7(),+ provider_id: Some("oversized-recipient-message".to_string()),+ thread_db_id,+ provider_thread_id: Some("prov1".to_string()),+ replying_to_id: None,+ global_id: Some("<[email protected]>".to_string()),+ link_id,+ subject: Some("unsubscribe".to_string()),+ snippet: None,+ provider_history_id: None,+ internal_date_ts: Some(now),+ sent_at: Some(now),+ size_estimate: Some(1024),+ is_read: true,+ is_starred: false,+ is_sent: true,+ is_draft: false,+ scheduled_send_time: None,+ has_attachments: true,+ from: Some(ContactInfo {+ email: "[email protected]".to_string(),+ name: Some("f".repeat(400)),+ photo_url: None,+ }),+ to: vec![+ ContactInfo {+ email: unsubscribe_address(),+ name: None,+ photo_url: None,+ },+ ContactInfo {+ email: "[email protected]".to_string(),+ name: Some("n".repeat(400)),+ photo_url: None,+ },+ ],+ cc: vec![],+ bcc: vec![],+ labels: vec![],+ body_text: Some("body".to_string()),+ body_html_sanitized: Some("<p>body</p>".to_string()),+ body_macro: None,+ attachments: vec![Attachment {+ db_id: Uuid::now_v7(),+ provider_id: Some("att1".to_string()),+ data_url: None,+ filename: Some("report.PDF".to_string()),+ mime_type: Some("m".repeat(400)),+ size_bytes: Some(10),+ sfs_id: None,+ content_id: Some("c".repeat(400)),+ }],+ attachments_draft: vec![],+ attachments_forwarded: vec![],+ headers_json: None,+ created_at: now,+ updated_at: now,+ }+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("links", "threads"))+)]+async fn oversized_provider_values_do_not_block_the_message_insert(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let link_id = Uuid::parse_str(LINK_ID)?;+ let thread_db_id = Uuid::parse_str(THREAD_ID)?;+ let mut message = message_with_oversized_fields(link_id, thread_db_id);+ let message_db_id = message.db_id;++ insert_message(&pool, thread_db_id, &mut message, link_id, false).await?;++ let inserted = sqlx::query!(+ r#"SELECT from_name FROM email_messages WHERE id = $1"#,+ message_db_id+ )+ .fetch_one(&pool)+ .await?;+ assert_eq!(+ inserted.from_name.as_deref().map(|n| n.chars().count()),+ Some(255)+ );++ // The unstorable address is dropped; every other recipient still lands.+ let recipients = sqlx::query!(+ r#"+ SELECT c.email_address, r.name+ FROM email_message_recipients r+ JOIN email_contacts c ON c.id = r.contact_id+ WHERE r.message_id = $1+ "#,+ message_db_id+ )+ .fetch_all(&pool)+ .await?;+ assert_eq!(recipients.len(), 1);+ assert_eq!(recipients[0].email_address, "[email protected]");+ assert_eq!(+ recipients[0].name.as_deref().map(|n| n.chars().count()),+ Some(255)+ );++ let contacts = sqlx::query_scalar!(+ r#"SELECT email_address FROM email_contacts WHERE link_id = $1 ORDER BY email_address"#,+ link_id+ )+ .fetch_all(&pool)+ .await?;+ assert_eq!(contacts, vec!["[email protected]", "[email protected]"]);++ let attachment = sqlx::query!(+ r#"+ SELECT mime_type, content_id+ FROM email_attachments+ WHERE message_id = $1+ "#,+ message_db_id+ )+ .fetch_one(&pool)+ .await?;+ assert_eq!(+ attachment.mime_type.as_deref().map(|m| m.chars().count()),+ Some(255)+ );+ assert_eq!(+ attachment.content_id.as_deref().map(|c| c.chars().count()),+ Some(255)+ );++ Ok(())+}diff --git a/crates/email_db_client/src/parse/column_limits.rs b/crates/email_db_client/src/parse/column_limits.rsnew file mode 100644index 0000000000..bf1d9d1703--- /dev/null+++ b/crates/email_db_client/src/parse/column_limits.rs@@ -0,0 +1,69 @@+//! Character limits of the `varchar(n)` columns that provider-sourced strings+//! land in.+//!+//! Gmail hands us headers with no length contract of its own, so a single+//! oversized value (a 500-char one-click-unsubscribe `To:` address, a runaway+//! `From:` display name) used to abort the whole message insert with SQLSTATE+//! 22001. Clamping here — at the service-to-db mapping boundary every insert+//! path funnels through — keeps the message itself ingestible.+//!+//! Postgres counts `varchar(n)` in characters, not bytes, so every limit below+//! is a character count and truncation is done on char boundaries.++/// `email_contacts.email_address`+pub(crate) const EMAIL_ADDRESS: usize = 320;++/// `email_contacts.name`, `email_message_recipients.name`,+/// `email_messages.from_name`+pub(crate) const CONTACT_NAME: usize = 255;++/// `email_attachments.filename`+pub(crate) const ATTACHMENT_FILENAME: usize = 512;++/// `email_attachments.mime_type`+pub(crate) const ATTACHMENT_MIME_TYPE: usize = 255;++/// `email_attachments.content_id`+pub(crate) const ATTACHMENT_CONTENT_ID: usize = 255;++/// True when `value` fits the column without truncation.+pub(crate) fn fits(value: &str, limit: usize) -> bool {+ value.chars().count() <= limit+}++/// Clamps `value` to `limit` characters, returning it untouched when it+/// already fits.+pub(crate) fn clamp(value: String, limit: usize) -> String {+ match value.char_indices().nth(limit) {+ Some((byte_offset, _)) => {+ let mut clamped = value;+ clamped.truncate(byte_offset);+ clamped+ }+ None => value,+ }+}++/// Clamps an optional column value, logging once per truncation so oversized+/// provider data stays visible.+pub(crate) fn clamp_opt(+ value: Option<String>,+ limit: usize,+ column: &'static str,+) -> Option<String> {+ value.map(|value| {+ if fits(&value, limit) {+ return value;+ }+ tracing::warn!(+ column,+ limit,+ length = value.chars().count(),+ "truncating oversized provider value to fit its column"+ );+ clamp(value, limit)+ })+}++#[cfg(test)]+mod test;diff --git a/crates/email_db_client/src/parse/column_limits/test.rs b/crates/email_db_client/src/parse/column_limits/test.rsnew file mode 100644index 0000000000..b243db6086--- /dev/null+++ b/crates/email_db_client/src/parse/column_limits/test.rs@@ -0,0 +1,50 @@+use super::*;++#[test]+fn clamp_leaves_values_that_already_fit() {+ assert_eq!(clamp("abc".to_string(), 5), "abc");+ assert_eq!(clamp("abcde".to_string(), 5), "abcde");+}++#[test]+fn clamp_cuts_to_the_character_limit() {+ assert_eq!(clamp("abcdef".to_string(), 5), "abcde");+ assert_eq!(clamp(String::new(), 0), "");+}++#[test]+fn clamp_counts_characters_not_bytes() {+ // Postgres varchar(n) counts characters, so a 5-char multi-byte string+ // fits a varchar(5) and must survive untouched.+ let five_chars = "ñññññ".to_string();+ assert_eq!(five_chars.len(), 10);+ assert_eq!(clamp(five_chars.clone(), 5), five_chars);+ assert_eq!(clamp(five_chars, 3), "ñññ");+}++#[test]+fn clamp_never_splits_a_multi_byte_character() {+ let emoji = "👩🚀🚀🚀".to_string();+ let clamped = clamp(emoji, 2);+ assert!(clamped.is_char_boundary(clamped.len()));+ assert_eq!(clamped.chars().count(), 2);+}++#[test]+fn fits_compares_characters() {+ assert!(fits("ñññññ", 5));+ assert!(!fits("ñññññ", 4));+}++#[test]+fn clamp_opt_passes_through_none() {+ assert_eq!(clamp_opt(None, 5, "col"), None);+}++#[test]+fn clamp_opt_truncates_some() {+ assert_eq!(+ clamp_opt(Some("abcdef".to_string()), 5, "col"),+ Some("abcde".to_string())+ );+}diff --git a/crates/email_db_client/src/parse/mod.rs b/crates/email_db_client/src/parse/mod.rsindex 2223bbf4e7..bbcde4a5dc 100644--- a/crates/email_db_client/src/parse/mod.rs+++ b/crates/email_db_client/src/parse/mod.rs@@ -1,2 +1,3 @@+pub(crate) mod column_limits;pub mod db_to_service;pub mod service_to_db;diff --git a/crates/email_db_client/src/parse/service_to_db.rs b/crates/email_db_client/src/parse/service_to_db.rsindex 545fc0c2ea..320b4062a1 100644--- a/crates/email_db_client/src/parse/service_to_db.rs+++ b/crates/email_db_client/src/parse/service_to_db.rs@@ -1,3 +1,4 @@+use crate::parse::column_limits;use chrono::Utc;use models_email::email::db::{address, attachment};use models_email::email::service::{message, thread};@@ -7,23 +8,55 @@ use models_email::service::message::HasContactInfo;use sqlx::types::Uuid;use std::mem;-/// Maps a ContactInfo to an EmailAddress-fn map_contact_to_db(contact: &service::address::ContactInfo) -> address::EmailAddress {- address::EmailAddress {+/// Maps a ContactInfo to an EmailAddress, dropping it when the address cannot+/// be stored.+///+/// RFC 5321 caps an address at 254 characters, so anything past the+/// `email_contacts.email_address` column width is tracking-token garbage (the+/// one-click-unsubscribe addresses some senders put in `To:` run to hundreds+/// of characters). Dropping the address is strictly better than failing the+/// whole message insert on it.+fn map_contact_to_db(contact: &service::address::ContactInfo) -> Option<address::EmailAddress> {+ if !column_limits::fits(&contact.email, column_limits::EMAIL_ADDRESS) {+ tracing::warn!(+ length = contact.email.chars().count(),+ limit = column_limits::EMAIL_ADDRESS,+ "dropping unstorable oversized email address"+ );+ return None;+ }++ Some(address::EmailAddress {id: macro_uuid::generate_uuid_v7(), // this value doesn't actually matter as we will be setting it later againemail_address: contact.email.clone(),- name: contact.name.clone(),+ name: column_limits::clamp_opt(+ contact.name.clone(),+ column_limits::CONTACT_NAME,+ "email_contacts.name",+ ),created_at: Utc::now(),- }+ })}/// logic for parsing service-layer structs into db-layer structspub fn addresses_from_message<T: HasContactInfo>(message: &T) -> address::ParsedAddresses {address::ParsedAddresses {- from: message.get_from().map(map_contact_to_db),- to: message.get_to().iter().map(map_contact_to_db).collect(),- cc: message.get_cc().iter().map(map_contact_to_db).collect(),- bcc: message.get_bcc().iter().map(map_contact_to_db).collect(),+ from: message.get_from().and_then(map_contact_to_db),+ to: message+ .get_to()+ .iter()+ .filter_map(map_contact_to_db)+ .collect(),+ cc: message+ .get_cc()+ .iter()+ .filter_map(map_contact_to_db)+ .collect(),+ bcc: message+ .get_bcc()+ .iter()+ .filter_map(map_contact_to_db)+ .collect(),}}@@ -65,7 +98,11 @@ pub fn map_service_message_to_db(snippet: service_msg.snippet.clone(),size_estimate: service_msg.size_estimate,subject: service_msg.subject.clone(),- from_name: service_msg.from.as_ref().and_then(|f| f.name.clone()),+ from_name: column_limits::clamp_opt(+ service_msg.from.as_ref().and_then(|f| f.name.clone()),+ column_limits::CONTACT_NAME,+ "email_messages.from_name",+ ),from_contact_id,sent_at: service_msg.sent_at,has_attachments: service_msg.has_attachments,@@ -115,10 +152,22 @@ pub fn map_service_attachments_to_db(id: service_attachment.db_id,message_id: message_db_id,provider_attachment_id: service_attachment.provider_id.clone(),- filename,- mime_type: service_attachment.mime_type.clone(),+ filename: column_limits::clamp_opt(+ filename,+ column_limits::ATTACHMENT_FILENAME,+ "email_attachments.filename",+ ),+ mime_type: column_limits::clamp_opt(+ service_attachment.mime_type.clone(),+ column_limits::ATTACHMENT_MIME_TYPE,+ "email_attachments.mime_type",+ ),size_bytes: service_attachment.size_bytes,- content_id: service_attachment.content_id.clone(),+ content_id: column_limits::clamp_opt(+ service_attachment.content_id.clone(),+ column_limits::ATTACHMENT_CONTENT_ID,+ "email_attachments.content_id",+ ),sfs_id: service_attachment.sfs_id,created_at: Utc::now(),}@@ -130,7 +179,11 @@ pub fn map_new_contact_to_db(service_contact: &Contact) -> db::contact::Contactdb::contact::Contact {id: service_contact.id,link_id: service_contact.link_id,- name: service_contact.name.clone(),+ name: column_limits::clamp_opt(+ service_contact.name.clone(),+ column_limits::CONTACT_NAME,+ "email_contacts.name",+ ),email_address: service_contact.email_address.clone(),original_photo_url: service_contact.original_photo_url.clone(),sfs_photo_url: service_contact.sfs_photo_url.clone(),@@ -138,3 +191,6 @@ pub fn map_new_contact_to_db(service_contact: &Contact) -> db::contact::Contactupdated_at: Utc::now(),}}++#[cfg(test)]+mod test;diff --git a/crates/email_db_client/src/parse/service_to_db/test.rs b/crates/email_db_client/src/parse/service_to_db/test.rsnew file mode 100644index 0000000000..831b5e3142--- /dev/null+++ b/crates/email_db_client/src/parse/service_to_db/test.rs@@ -0,0 +1,171 @@+use super::*;+use models_email::email::service::address::ContactInfo;++/// The shape that wedged mwatts42's backfill: a one-click-unsubscribe `To:`+/// address far past what `email_contacts.email_address` can hold.+fn oversized_address() -> String {+ format!(+ "v3_{}@unsubscribe-06.emailinboundprocessing.com",+ "a".repeat(470)+ )+}++fn contact(email: &str, name: Option<&str>) -> ContactInfo {+ ContactInfo {+ email: email.to_string(),+ name: name.map(str::to_string),+ photo_url: None,+ }+}++fn message_with(from: Option<ContactInfo>, to: Vec<ContactInfo>) -> message::Message {+ let now = Utc::now();+ message::Message {+ db_id: Uuid::now_v7(),+ provider_id: Some("m1".to_string()),+ thread_db_id: Uuid::now_v7(),+ provider_thread_id: Some("t1".to_string()),+ replying_to_id: None,+ global_id: None,+ link_id: Uuid::now_v7(),+ subject: None,+ snippet: None,+ provider_history_id: None,+ internal_date_ts: Some(now),+ sent_at: Some(now),+ size_estimate: None,+ is_read: false,+ is_starred: false,+ is_sent: true,+ is_draft: false,+ scheduled_send_time: None,+ has_attachments: false,+ from,+ to,+ cc: vec![],+ bcc: vec![],+ labels: vec![],+ body_text: None,+ body_html_sanitized: None,+ body_macro: None,+ attachments: vec![],+ attachments_draft: vec![],+ attachments_forwarded: vec![],+ headers_json: None,+ created_at: now,+ updated_at: now,+ }+}++#[test]+fn oversized_recipient_addresses_are_dropped_and_the_rest_kept() {+ let message = message_with(+ Some(contact("[email protected]", None)),+ vec![+ contact(&oversized_address(), None),+ contact("[email protected]", None),+ ],+ );++ let addresses = addresses_from_message(&message);++ assert_eq!(addresses.to.len(), 1);+ assert_eq!(addresses.to[0].email_address, "[email protected]");+ assert!(addresses.from.is_some());+}++#[test]+fn an_address_exactly_at_the_column_width_is_kept() {+ let at_limit = format!(+ "{}@macro.com",+ "a".repeat(column_limits::EMAIL_ADDRESS - 10)+ );+ assert_eq!(at_limit.chars().count(), column_limits::EMAIL_ADDRESS);++ let message = message_with(None, vec![contact(&at_limit, None)]);++ assert_eq!(addresses_from_message(&message).to.len(), 1);+}++#[test]+fn an_oversized_from_address_is_dropped_without_dropping_the_message() {+ let message = message_with(Some(contact(&oversized_address(), Some("Sender"))), vec![]);++ let addresses = addresses_from_message(&message);++ assert!(addresses.from.is_none());+}++#[test]+fn oversized_contact_names_are_truncated() {+ let long_name = "n".repeat(column_limits::CONTACT_NAME + 100);+ let message = message_with(None, vec![contact("[email protected]", Some(&long_name))]);++ let addresses = addresses_from_message(&message);++ let name = addresses.to[0].name.as_ref().expect("name should survive");+ assert_eq!(name.chars().count(), column_limits::CONTACT_NAME);+}++#[test]+fn oversized_from_names_are_truncated_for_the_message_row() {+ let long_name = "n".repeat(column_limits::CONTACT_NAME + 100);+ let mut message = message_with(Some(contact("[email protected]", Some(&long_name))), vec![]);++ let db_message = map_service_message_to_db(&mut message, Uuid::now_v7(), None);++ let from_name = db_message.from_name.expect("from_name should survive");+ assert_eq!(from_name.chars().count(), column_limits::CONTACT_NAME);+}++#[test]+fn oversized_attachment_metadata_is_truncated() {+ let mut attachments = vec![service::attachment::Attachment {+ db_id: Uuid::now_v7(),+ provider_id: Some("a1".to_string()),+ data_url: None,+ filename: Some(format!(+ "{}.PDF",+ "f".repeat(column_limits::ATTACHMENT_FILENAME)+ )),+ mime_type: Some("x".repeat(column_limits::ATTACHMENT_MIME_TYPE + 1)),+ size_bytes: Some(1),+ sfs_id: None,+ content_id: Some("c".repeat(column_limits::ATTACHMENT_CONTENT_ID + 1)),+ }];++ let mapped = map_service_attachments_to_db(&mut attachments, Uuid::now_v7());++ let attachment = &mapped[0];+ assert_eq!(+ attachment.filename.as_ref().unwrap().chars().count(),+ column_limits::ATTACHMENT_FILENAME+ );+ assert_eq!(+ attachment.mime_type.as_ref().unwrap().chars().count(),+ column_limits::ATTACHMENT_MIME_TYPE+ );+ assert_eq!(+ attachment.content_id.as_ref().unwrap().chars().count(),+ column_limits::ATTACHMENT_CONTENT_ID+ );+}++#[test]+fn oversized_synced_contact_names_are_truncated() {+ let contact = Contact {+ id: Uuid::now_v7(),+ link_id: Uuid::now_v7(),+ name: Some("n".repeat(column_limits::CONTACT_NAME + 1)),+ email_address: Some("[email protected]".to_string()),+ original_photo_url: None,+ sfs_photo_url: None,+ };++ let mapped = map_new_contact_to_db(&contact);++ assert_eq!(+ mapped.name.as_ref().unwrap().chars().count(),+ column_limits::CONTACT_NAME+ );+}diff --git a/crates/email_db_client/src/threads/insert/test.rs b/crates/email_db_client/src/threads/insert/test.rsindex 8b19321b9f..7932bf3b43 100644--- a/crates/email_db_client/src/threads/insert/test.rs+++ b/crates/email_db_client/src/threads/insert/test.rs@@ -147,3 +147,102 @@ async fn blank_insert_creates_new_thread(pool: Pool<Postgres>) -> anyhow::ResultOk(())}++/// The live inbox-sync path funnels through here, so it needs the same+/// protection from unstorable provider values as the backfill path.+#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("links"))+)]+async fn oversized_provider_values_do_not_block_the_thread_insert(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ use models_email::email::service::address::ContactInfo;+ use models_email::email::service::message;++ let link_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?;+ let thread_db_id = Uuid::now_v7();+ let message_db_id = Uuid::now_v7();+ let now = Utc::now();+ let unsubscribe_address = format!(+ "v3_{}@unsubscribe-06.emailinboundprocessing.com",+ "t".repeat(460)+ );++ let thread = thread::Thread {+ db_id: thread_db_id,+ provider_id: Some("oversized-thread".to_string()),+ link_id,+ inbox_visible: true,+ is_read: false,+ latest_inbound_message_ts: Some(now),+ latest_outbound_message_ts: None,+ latest_non_spam_message_ts: Some(now),+ created_at: now,+ updated_at: now,+ messages: vec![message::Message {+ db_id: message_db_id,+ provider_id: Some("oversized-thread-message".to_string()),+ thread_db_id,+ provider_thread_id: Some("oversized-thread".to_string()),+ replying_to_id: None,+ global_id: None,+ link_id,+ subject: Some("unsubscribe".to_string()),+ snippet: None,+ provider_history_id: None,+ internal_date_ts: Some(now),+ sent_at: Some(now),+ size_estimate: Some(1),+ is_read: false,+ is_starred: false,+ is_sent: true,+ is_draft: false,+ scheduled_send_time: None,+ has_attachments: false,+ from: Some(ContactInfo {+ email: "[email protected]".to_string(),+ name: Some("f".repeat(400)),+ photo_url: None,+ }),+ to: vec![ContactInfo {+ email: unsubscribe_address,+ name: None,+ photo_url: None,+ }],+ cc: vec![],+ bcc: vec![],+ labels: vec![],+ body_text: Some("body".to_string()),+ body_html_sanitized: None,+ body_macro: None,+ attachments: vec![],+ attachments_draft: vec![],+ attachments_forwarded: vec![],+ headers_json: None,+ created_at: now,+ updated_at: now,+ }],+ };++ let inserted_thread_id = insert_thread_and_messages(&pool, thread, link_id).await?;++ assert_eq!(inserted_thread_id, thread_db_id);+ let from_name = sqlx::query_scalar!(+ r#"SELECT from_name FROM email_messages WHERE id = $1"#,+ message_db_id+ )+ .fetch_one(&pool)+ .await?;+ assert_eq!(from_name.as_deref().map(|n| n.chars().count()), Some(255));++ let recipient_count = sqlx::query_scalar!(+ r#"SELECT COUNT(*) FROM email_message_recipients WHERE message_id = $1"#,+ message_db_id+ )+ .fetch_one(&pool)+ .await?;+ assert_eq!(recipient_count, Some(0));++ Ok(())+}diff --git a/services/email_service/src/pubsub/backfill/backfill_message.rs b/services/email_service/src/pubsub/backfill/backfill_message.rsindex 4f27bd60ed..69e3138c44 100644--- a/services/email_service/src/pubsub/backfill/backfill_message.rs+++ b/services/email_service/src/pubsub/backfill/backfill_message.rs@@ -1,3 +1,4 @@+use crate::pubsub::backfill::db_error::map_db_error;use crate::pubsub::backfill::email_api_error::map_email_api_error;use crate::pubsub::backfill::increment_counters;use crate::pubsub::context::PubSubContext;@@ -42,12 +43,7 @@ pub async fn backfill_message(false,).await- .map_err(|e| {- ProcessingError::Retryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e.context("Failed to insert final message into database"),- })- })?;+ .map_err(|e| map_db_error(e, "Failed to insert final message into database"))?;// Fan out a PopulateCrmContact job per address involved in the// message — every non-draft message contributes, in bothdiff --git a/services/email_service/src/pubsub/backfill/backfill_thread.rs b/services/email_service/src/pubsub/backfill/backfill_thread.rsindex bfd1e29c89..6be59c5549 100644--- a/services/email_service/src/pubsub/backfill/backfill_thread.rs+++ b/services/email_service/src/pubsub/backfill/backfill_thread.rs@@ -1,3 +1,4 @@+use crate::pubsub::backfill::db_error::map_db_error;use crate::pubsub::backfill::email_api_error::map_email_api_error;use crate::pubsub::backfill::increment_counters::incr_completed_threads;use crate::pubsub::context::PubSubContext;@@ -28,13 +29,10 @@ pub async fn backfill_thread().await.map_err(|e| {- ProcessingError::Retryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e.context(format!(- "DB check for existing thread {} failed",- thread_provider_id- )),- })+ map_db_error(+ e,+ format!("DB check for existing thread {} failed", thread_provider_id),+ )})?;// If the thread already exists: recovery jobs refresh it (backfilling any@@ -63,13 +61,10 @@ pub async fn backfill_thread().await.map_err(|e| {- ProcessingError::Retryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e.context(format!(- "Failed to insert thread shell for {}",- thread_provider_id- )),- })+ map_db_error(+ e,+ format!("Failed to insert thread shell for {}", thread_provider_id),+ )})?;// create backfill pubsub message for each email message@@ -106,13 +101,13 @@ async fn refresh_existing_thread().await.map_err(|e| {- ProcessingError::Retryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e.context(format!(+ map_db_error(+ e,+ format!("DB check for existing messages of thread {} failed",p.thread_provider_id- )),- })+ ),+ )})?;let missing_message_ids: Vec<String> = message_idsdiff --git a/services/email_service/src/pubsub/backfill/db_error.rs b/services/email_service/src/pubsub/backfill/db_error.rsnew file mode 100644index 0000000000..bf145953a7--- /dev/null+++ b/services/email_service/src/pubsub/backfill/db_error.rs@@ -0,0 +1,61 @@+use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};++#[cfg(test)]+mod test;++/// SQLSTATE class 22, "Data Exception": the value itself cannot be represented+/// in its column. `22001` (string too long) is the one that wedged backfills —+/// a Gmail header longer than the `varchar(n)` it lands in — but every member+/// of the class shares the property that matters here: the same payload fails+/// identically on every attempt.+const DATA_EXCEPTION_CLASS: &str = "22";++/// `not_null_violation` — the row is missing a column the schema requires.+const NOT_NULL_VIOLATION: &str = "23502";++/// `check_violation` — the row fails a check constraint.+const CHECK_VIOLATION: &str = "23514";++/// Maps a database failure onto a retry policy.+///+/// The default is `Retryable`: transient connection drops, deadlocks, and+/// serialization failures all clear on redelivery. Errors that are a property+/// of the payload rather than of the moment are `NonRetryable` instead, so the+/// backfill error handlers count the thread as failed and let the job finish.+/// Left retryable, they burn 20 receives and land in the DLQ while the job's+/// completion counter sits one short forever.+///+/// `unique_violation` and `foreign_key_violation` are deliberately absent:+/// concurrent inserts of the same row make both genuinely transient.+pub(crate) fn map_db_error<C>(error: anyhow::Error, context: C) -> ProcessingError+where+ C: std::fmt::Display + Send + Sync + 'static,+{+ match deterministic_sqlstate(&error) {+ Some(sqlstate) => ProcessingError::NonRetryable(DetailedError {+ reason: FailureReason::InvalidData,+ source: error.context(format!("{context} (deterministic SQLSTATE {sqlstate})")),+ }),+ None => ProcessingError::Retryable(DetailedError {+ reason: FailureReason::DatabaseQueryFailed,+ source: error.context(context),+ }),+ }+}++/// Returns the SQLSTATE of the first database error in the chain that no+/// amount of retrying will get past.+fn deterministic_sqlstate(error: &anyhow::Error) -> Option<String> {+ error.chain().find_map(|cause| {+ let code = cause+ .downcast_ref::<sqlx::Error>()?+ .as_database_error()?+ .code()?;+ is_deterministic(&code).then(|| code.into_owned())+ })+}++fn is_deterministic(sqlstate: &str) -> bool {+ sqlstate.starts_with(DATA_EXCEPTION_CLASS)+ || matches!(sqlstate, NOT_NULL_VIOLATION | CHECK_VIOLATION)+}diff --git a/services/email_service/src/pubsub/backfill/db_error/test.rs b/services/email_service/src/pubsub/backfill/db_error/test.rsnew file mode 100644index 0000000000..11ddda9fe5--- /dev/null+++ b/services/email_service/src/pubsub/backfill/db_error/test.rs@@ -0,0 +1,145 @@+use super::*;+use sqlx::error::{DatabaseError, ErrorKind};+use std::borrow::Cow;+use std::error::Error as StdError;+use std::fmt;++#[derive(Debug)]+struct FakeDbError {+ code: &'static str,+ message: &'static str,+}++impl fmt::Display for FakeDbError {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str(self.message)+ }+}++impl StdError for FakeDbError {}++impl DatabaseError for FakeDbError {+ fn message(&self) -> &str {+ self.message+ }++ fn code(&self) -> Option<Cow<'_, str>> {+ Some(Cow::Borrowed(self.code))+ }++ fn as_error(&self) -> &(dyn StdError + Send + Sync + 'static) {+ self+ }++ fn as_error_mut(&mut self) -> &mut (dyn StdError + Send + Sync + 'static) {+ self+ }++ fn into_error(self: Box<Self>) -> Box<dyn StdError + Send + Sync + 'static> {+ self+ }++ fn kind(&self) -> ErrorKind {+ ErrorKind::Other+ }+}++fn db_error(code: &'static str, message: &'static str) -> anyhow::Error {+ anyhow::Error::new(sqlx::Error::Database(Box::new(FakeDbError {+ code,+ message,+ })))+}++fn assert_non_retryable(error: anyhow::Error) -> DetailedError {+ match map_db_error(error, "insert failed") {+ ProcessingError::NonRetryable(detail) => detail,+ other => panic!("expected a non-retryable classification, got {other:?}"),+ }+}++fn assert_retryable(error: anyhow::Error) -> DetailedError {+ match map_db_error(error, "insert failed") {+ ProcessingError::Retryable(detail) => detail,+ other => panic!("expected a retryable classification, got {other:?}"),+ }+}++#[test]+fn string_data_right_truncation_is_terminal() {+ let detail = assert_non_retryable(db_error(+ "22001",+ "value too long for type character varying(320)",+ ));++ assert_eq!(detail.reason, FailureReason::InvalidData);+ let chain = format!("{:#}", detail.source);+ assert!(chain.contains("insert failed"));+ assert!(chain.contains("22001"));+}++#[test]+fn the_whole_data_exception_class_is_terminal() {+ for code in ["22003", "22007", "22021", "22P02"] {+ let detail = assert_non_retryable(db_error(code, "data exception"));+ assert_eq!(detail.reason, FailureReason::InvalidData);+ }+}++#[test]+fn not_null_and_check_violations_are_terminal() {+ for code in [NOT_NULL_VIOLATION, CHECK_VIOLATION] {+ assert_eq!(+ assert_non_retryable(db_error(code, "constraint violation")).reason,+ FailureReason::InvalidData+ );+ }+}++#[test]+fn conflicts_that_concurrency_can_cause_stay_retryable() {+ // A racing insert of the same row resolves on redelivery, so these must+ // not be treated as a property of the payload.+ for code in ["23505", "23503"] {+ assert_eq!(+ assert_retryable(db_error(code, "conflict")).reason,+ FailureReason::DatabaseQueryFailed+ );+ }+}++#[test]+fn transient_failures_stay_retryable() {+ for code in ["40001", "40P01", "53300", "57014", "08006"] {+ assert_eq!(+ assert_retryable(db_error(code, "transient")).reason,+ FailureReason::DatabaseQueryFailed+ );+ }+}++#[test]+fn non_database_errors_stay_retryable() {+ assert_eq!(+ assert_retryable(anyhow::anyhow!("pool timed out")).reason,+ FailureReason::DatabaseQueryFailed+ );+ assert_eq!(+ assert_retryable(anyhow::Error::new(sqlx::Error::PoolClosed)).reason,+ FailureReason::DatabaseQueryFailed+ );+}++#[test]+fn the_sqlstate_is_found_through_added_context() {+ // Production errors reach the mapper already wrapped by the db client's+ // own `.context(...)` calls; the classification must still see through.+ let wrapped = db_error("22001", "value too long")+ .context("Rollback also failed: none")+ .context("Failed to insert final message into database");++ assert_eq!(+ assert_non_retryable(wrapped).reason,+ FailureReason::InvalidData+ );+}diff --git a/services/email_service/src/pubsub/backfill/error_handlers.rs b/services/email_service/src/pubsub/backfill/error_handlers.rsindex 50ad419454..24467849cd 100644--- a/services/email_service/src/pubsub/backfill/error_handlers.rs+++ b/services/email_service/src/pubsub/backfill/error_handlers.rs@@ -184,74 +184,74 @@ pub async fn handle_retryable_error(match &data.backfill_operation {BackfillOperation::Init(_) => {- tracing::debug!("Retryable error in Init")+ tracing::warn!("Retryable error in Init")}BackfillOperation::ListThreads(_) => {- tracing::debug!("Retryable error listing threads")+ tracing::warn!("Retryable error listing threads")}BackfillOperation::BackfillThread(scope) => {- tracing::debug!(+ tracing::warn!(thread_id = %scope.payload.thread_provider_id,"Retryable error backfilling thread");}BackfillOperation::BackfillMessage(scope) => {- tracing::debug!(+ tracing::warn!(thread_id = %scope.payload.thread_provider_id,message_id = %scope.payload.message_provider_id,"Retryable error backfilling message");}BackfillOperation::UpdateThreadMetadata(scope) => {- tracing::debug!(+ tracing::warn!(thread_id = %scope.payload.thread_provider_id,"Retryable error updating thread metadata");}BackfillOperation::BackfillAttachment(scope) => {- tracing::debug!(+ tracing::warn!(attachment_db_id = %scope.payload.metadata.attachment_metadata.attachment_db_id,"Retryable error backfilling attachment")}BackfillOperation::FinalizeBackfill(scope) => {- tracing::debug!(+ tracing::warn!(job_id = %scope.job_id,"Retryable error finalizing completed backfill")}BackfillOperation::CalendarGoogleBackfill(scope) => {- tracing::debug!(+ tracing::warn!(calendar_job_id = %scope.payload.calendar_job_id,"Retryable error backfilling Google Calendar")}BackfillOperation::SeedSentContact(scope) => {- tracing::debug!(+ tracing::warn!(message_id = %scope.payload.message_provider_id,"Retryable error seeding contact from sent message")}BackfillOperation::PopulateCrmContact(scope) => {- tracing::debug!(+ tracing::warn!(contact_email = %scope.payload.contact_email,"Retryable error populating CRM contact")}BackfillOperation::DepopulateCrmContact(scope) => {- tracing::debug!(+ tracing::warn!(contact_email = %scope.payload.contact_email,"Retryable error depopulating CRM contact")}BackfillOperation::PopulateCrmForUser(payload) => {- tracing::debug!(+ tracing::warn!(macro_id = %payload.macro_id,"Retryable error populating CRM for user")}BackfillOperation::DepopulateCrmForUser(payload) => {- tracing::debug!(+ tracing::warn!(macro_id = %payload.macro_id,"Retryable error depopulating CRM for user")diff --git a/services/email_service/src/pubsub/backfill/mod.rs b/services/email_service/src/pubsub/backfill/mod.rsindex 16d0607322..7345db8aa2 100644--- a/services/email_service/src/pubsub/backfill/mod.rs+++ b/services/email_service/src/pubsub/backfill/mod.rs@@ -2,6 +2,7 @@ mod backfill_attachment;mod backfill_message;mod backfill_thread;mod calendar_google_backfill;+pub(crate) mod db_error;mod depopulate_crm_contact;mod depopulate_crm_for_user;pub(crate) mod email_api_error;diff --git a/services/email_service/src/pubsub/backfill/update_metadata.rs b/services/email_service/src/pubsub/backfill/update_metadata.rsindex 47ec8bfe83..4f393d8dce 100644--- a/services/email_service/src/pubsub/backfill/update_metadata.rs+++ b/services/email_service/src/pubsub/backfill/update_metadata.rs@@ -1,3 +1,4 @@+use crate::pubsub::backfill::db_error::map_db_error;use crate::pubsub::backfill::increment_counters::incr_completed_threads;use crate::pubsub::context::PubSubContext;use crate::pubsub::util::publish_email_event;@@ -30,12 +31,7 @@ pub async fn update_thread_metadata(// get to this point.email_db_client::threads::update::update_thread_metadata(&mut tx, p.thread_db_id, link.id).await- .map_err(|e| {- ProcessingError::Retryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e.context("Failed to update thread metadata"),- })- })?;+ .map_err(|e| map_db_error(e, "Failed to update thread metadata"))?;// update the replying_to_id of the messages in the thread. this can only be done once all// messages in the thread have been inserted, and we know this is the case by the time we@@ -46,12 +42,7 @@ pub async fn update_thread_metadata(link.id,).await- .map_err(|e| {- ProcessingError::Retryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e.context("Failed to update message replying to id"),- })- })?;+ .map_err(|e| map_db_error(e, "Failed to update message replying to id"))?;Ok(())}