Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(search_processing_service): email kafka (#5340)
Will Hutchinson8394d21d6938
diff --git a/Cargo.lock b/Cargo.lockindex 96e0baf1b1..927d3ab3b3 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -12975,6 +12975,7 @@ dependencies = ["document_sub_type","documents","doppler-config",+ "email","email_db_client","ensure_exists","futures",diff --git a/crates/email/src/domain/events.rs b/crates/email/src/domain/events.rsindex 6397d0209e..bb491bd366 100644--- a/crates/email/src/domain/events.rs+++ b/crates/email/src/domain/events.rs@@ -8,9 +8,10 @@//! account lifecycle events are ordered against the inbox's content events,//! and per-thread order is preserved (a thread belongs to exactly one link).//!-//! Payloads carry RFC-822 *header* level data only (subject, from, to/cc-//! addresses) — never message bodies, snippets, bcc addresses, or attachment-//! names. Consumers needing content must read back through the API.+//! When payloads carry message data, it is limited to RFC-822 *header* level+//! data (subject, from, to/cc addresses) — never message bodies, snippets, bcc+//! addresses, or attachment names. Consumers needing content, including+//! consumers of reindex notifications, must read back through the API.//!//! Each variant documents its owning emission site: a logical change is//! published from the point closest to the committed Macro-DB write.@@ -84,6 +85,14 @@ pub enum SendCancelReason {ThreadTrashed,}+/// Why a set of threads was asked to be reindexed for search.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum ThreadsReindexReason {+ /// Contact display names changed; sender/recipient names in the index are stale.+ ContactsChanged,+}+/// Metadata for [`EmailTopicEvent::LinkConnected`].#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct LinkConnectedMetadata {@@ -160,6 +169,25 @@ pub struct MessageReceivedMetadata {pub received_at: Option<DateTime<Utc>>,}+/// Metadata for [`EmailTopicEvent::MessageDraftSynced`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct MessageDraftSyncedMetadata {+ /// The link (inbox) containing the draft.+ pub link_id: Uuid,+ /// The Macro user who owns the link.+ pub owner: MacroUserIdStr<'static>,+ /// Macro's message row id.+ pub message_id: Uuid,+ /// Provider (Gmail) message id.+ pub provider_message_id: String,+ /// Macro's thread row id.+ pub thread_id: Uuid,+ /// Provider (Gmail) thread id.+ pub provider_thread_id: String,+ /// Whether the draft is currently spam or trash.+ pub is_spam_or_trash: bool,+}+/// Metadata for [`EmailTopicEvent::MessageSent`].#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct MessageSentMetadata {@@ -315,6 +343,24 @@ pub struct ThreadStarredMetadata {pub origin: EmailEventOrigin,}+/// Metadata for [`EmailTopicEvent::ThreadSpamChanged`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct ThreadSpamChangedMetadata {+ /// The link (inbox) containing the thread.+ pub link_id: Uuid,+ /// The Macro user who owns the link.+ pub owner: MacroUserIdStr<'static>,+ /// The authenticated user who changed the state; `None` for+ /// provider-initiated changes.+ pub actor: Option<MacroUserIdStr<'static>>,+ /// Macro's thread row id.+ pub thread_id: Uuid,+ /// New state: `true` = marked spam, `false` = unmarked.+ pub spam: bool,+ /// Where the change originated.+ pub origin: EmailEventOrigin,+}+/// Metadata for [`EmailTopicEvent::ThreadProjectChanged`].#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct ThreadProjectChangedMetadata {@@ -335,10 +381,10 @@ pub struct ThreadProjectChangedMetadata {/// Metadata for [`EmailTopicEvent::ThreadLabelsUpdated`].////// Carries **user label** changes only; system-label changes (INBOX, TRASH,-/// UNREAD, STARRED) are published as the semantic+/// UNREAD, STARRED, SPAM) are published as the semantic/// [`EmailTopicEvent::ThreadArchived`] / [`EmailTopicEvent::ThreadTrashed`] /-/// [`EmailTopicEvent::ThreadRead`] / [`EmailTopicEvent::ThreadStarred`]-/// events instead.+/// [`EmailTopicEvent::ThreadRead`] / [`EmailTopicEvent::ThreadStarred`] /+/// [`EmailTopicEvent::ThreadSpamChanged`] events instead.#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct ThreadLabelsUpdatedMetadata {/// The link (inbox) containing the thread.@@ -358,6 +404,30 @@ pub struct ThreadLabelsUpdatedMetadata {pub origin: EmailEventOrigin,}+/// Metadata for [`EmailTopicEvent::ThreadBackfilled`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct ThreadBackfilledMetadata {+ /// The link (inbox) containing the thread.+ pub link_id: Uuid,+ /// The Macro user who owns the link.+ pub owner: MacroUserIdStr<'static>,+ /// Macro's thread row id.+ pub thread_id: Uuid,+}++/// Metadata for [`EmailTopicEvent::ThreadsReindexRequested`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct ThreadsReindexRequestedMetadata {+ /// The link (inbox) containing the threads.+ pub link_id: Uuid,+ /// The Macro user who owns the link.+ pub owner: MacroUserIdStr<'static>,+ /// Thread row ids to reindex. Producers must chunk to at most 50 ids.+ pub thread_ids: Vec<Uuid>,+ /// Why the reindex was requested.+ pub reason: ThreadsReindexReason,+}+/// Events that can be published to [`MacroEmailTopic`].#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]#[serde(tag = "event_type", content = "metadata")]@@ -381,6 +451,10 @@ pub enum EmailTopicEvent {/// siblings are persisted silently (they were not just received).#[serde(rename = "email.message_received")]MessageReceived(MessageReceivedMetadata),+ /// A draft was persisted or updated while syncing from the provider.+ /// Emitted on every synced draft update because draft bodies are mutable.+ #[serde(rename = "email.message_draft_synced")]+ MessageDraftSynced(MessageDraftSyncedMetadata),/// A message was accepted by the provider. Emitted from the scheduled/// send worker after the provider accepts (`origin = user_action`), or/// from the inbox-sync worker when a message sent from another client is@@ -421,6 +495,11 @@ pub enum EmailTopicEvent {/// inbox-sync STARRED-label diff (`origin = provider_sync`).#[serde(rename = "email.thread_starred")]ThreadStarred(ThreadStarredMetadata),+ /// A thread was marked or unmarked spam. Emitted from the inbox-sync+ /// SPAM-label diff (`origin = provider_sync`) or the domain label path+ /// (`origin = user_action`).+ #[serde(rename = "email.thread_spam_changed")]+ ThreadSpamChanged(ThreadSpamChangedMetadata),/// A thread was assigned to or removed from a project. Emitted from the/// email domain service's thread-project update.#[serde(rename = "email.thread_project_changed")]@@ -431,6 +510,15 @@ pub enum EmailTopicEvent {/// semantic thread events instead.#[serde(rename = "email.thread_labels_updated")]ThreadLabelsUpdated(ThreadLabelsUpdatedMetadata),+ /// All messages of a backfilled thread are persisted and thread metadata+ /// is final. Emitted from the backfill worker's update-metadata step.+ #[serde(rename = "email.thread_backfilled")]+ ThreadBackfilled(ThreadBackfilledMetadata),+ /// A bounded batch of threads should be re-derived by read-back consumers+ /// (for example, the search index). Emitted from contact sync when contact+ /// names change. Producers must include at most 50 thread ids.+ #[serde(rename = "email.threads_reindex_requested")]+ ThreadsReindexRequested(ThreadsReindexRequestedMetadata),}impl TopicEvent for EmailTopicEvent {@@ -478,6 +566,14 @@ impl EmailMacroEvent {)}+ /// Build a message-draft-synced event keyed by the link id.+ pub fn message_draft_synced(metadata: MessageDraftSyncedMetadata) -> Self {+ Self::new(+ metadata.link_id.to_string(),+ EmailTopicEvent::MessageDraftSynced(metadata),+ )+ }+/// Build a message-sent event keyed by the link id.pub fn message_sent(metadata: MessageSentMetadata) -> Self {Self::new(@@ -542,6 +638,14 @@ impl EmailMacroEvent {)}+ /// Build a thread-spam-changed event keyed by the link id.+ pub fn thread_spam_changed(metadata: ThreadSpamChangedMetadata) -> Self {+ Self::new(+ metadata.link_id.to_string(),+ EmailTopicEvent::ThreadSpamChanged(metadata),+ )+ }+/// Build a thread-project-changed event keyed by the link id.pub fn thread_project_changed(metadata: ThreadProjectChangedMetadata) -> Self {Self::new(@@ -558,10 +662,27 @@ impl EmailMacroEvent {)}+ /// Build a thread-backfilled event keyed by the link id.+ pub fn thread_backfilled(metadata: ThreadBackfilledMetadata) -> Self {+ Self::new(+ metadata.link_id.to_string(),+ EmailTopicEvent::ThreadBackfilled(metadata),+ )+ }++ /// Build a threads-reindex-requested event keyed by the link id.+ pub fn threads_reindex_requested(metadata: ThreadsReindexRequestedMetadata) -> Self {+ Self::new(+ metadata.link_id.to_string(),+ EmailTopicEvent::ThreadsReindexRequested(metadata),+ )+ }+/// Map a thread-scoped label change onto the semantic event for that- /// label: INBOX/TRASH/UNREAD/STARRED become the dedicated thread-state- /// events, SPAM/IMPORTANT/SENT/DRAFT publish nothing (`None`), and any- /// other (user) label becomes a `thread_labels_updated` diff.+ /// label: INBOX/TRASH/UNREAD/STARRED/SPAM become dedicated thread-state+ /// events, IMPORTANT/SENT/DRAFT and category labels publish nothing+ /// (`None`), and any other (user) label becomes a+ /// `thread_labels_updated` diff.pub fn thread_label_change(link_id: Uuid,owner: MacroUserIdStr<'static>,@@ -606,10 +727,15 @@ impl EmailMacroEvent {archived: !added,origin,}),- system_labels::SPAM- | system_labels::IMPORTANT- | system_labels::SENT- | system_labels::DRAFT => return None,+ system_labels::SPAM => Self::thread_spam_changed(ThreadSpamChangedMetadata {+ link_id,+ owner,+ actor,+ thread_id,+ spam: added,+ origin,+ }),+ system_labels::IMPORTANT | system_labels::SENT | system_labels::DRAFT => return None,// Gmail category tabs (CATEGORY_PERSONAL/SOCIAL/...) are// provider system labels, not user labels.l if l.starts_with("CATEGORY_") => return None,diff --git a/crates/email/src/domain/events/test.rs b/crates/email/src/domain/events/test.rsindex 0ba97ce0ec..aed9ae312f 100644--- a/crates/email/src/domain/events/test.rs+++ b/crates/email/src/domain/events/test.rs@@ -14,6 +14,41 @@ fn timestamp() -> DateTime<Utc> {.with_timezone(&Utc)}+fn new_contract_events(link_id: Uuid) -> Vec<EmailMacroEvent> {+ let owner = user_id("macro|[email protected]");++ vec![+ EmailMacroEvent::message_draft_synced(MessageDraftSyncedMetadata {+ link_id,+ owner: owner.clone(),+ message_id: Uuid::new_v4(),+ provider_message_id: "gmail-draft-1".to_string(),+ thread_id: Uuid::new_v4(),+ provider_thread_id: "gmail-thread-1".to_string(),+ is_spam_or_trash: false,+ }),+ EmailMacroEvent::thread_backfilled(ThreadBackfilledMetadata {+ link_id,+ owner: owner.clone(),+ thread_id: Uuid::new_v4(),+ }),+ EmailMacroEvent::threads_reindex_requested(ThreadsReindexRequestedMetadata {+ link_id,+ owner: owner.clone(),+ thread_ids: vec![Uuid::new_v4(), Uuid::new_v4()],+ reason: ThreadsReindexReason::ContactsChanged,+ }),+ EmailMacroEvent::thread_spam_changed(ThreadSpamChangedMetadata {+ link_id,+ owner,+ actor: None,+ thread_id: Uuid::new_v4(),+ spam: true,+ origin: EmailEventOrigin::ProviderSync,+ }),+ ]+}+#[test]fn message_received_wire_shape() {let event = Event::with_event_id(@@ -63,6 +98,43 @@ fn message_received_wire_shape() {);}+#[test]+fn message_draft_synced_wire_shape_preserves_spam_or_trash_state() {+ for is_spam_or_trash in [false, true] {+ let event = Event::with_event_id(+ Uuid::nil(),+ EmailTopicEvent::MessageDraftSynced(MessageDraftSyncedMetadata {+ link_id: Uuid::nil(),+ owner: user_id("macro|[email protected]"),+ message_id: Uuid::nil(),+ provider_message_id: "gmail-draft-1".to_string(),+ thread_id: Uuid::nil(),+ provider_thread_id: "gmail-thread-1".to_string(),+ is_spam_or_trash,+ }),+ );++ let value = serde_json::to_value(&event).expect("serializable");+ assert_eq!(+ value,+ json!({+ "event_id": "00000000-0000-0000-0000-000000000000",+ "schema_version": 1,+ "event_type": "email.message_draft_synced",+ "metadata": {+ "link_id": "00000000-0000-0000-0000-000000000000",+ "owner": "macro|[email protected]",+ "message_id": "00000000-0000-0000-0000-000000000000",+ "provider_message_id": "gmail-draft-1",+ "thread_id": "00000000-0000-0000-0000-000000000000",+ "provider_thread_id": "gmail-thread-1",+ "is_spam_or_trash": is_spam_or_trash,+ },+ })+ );+ }+}+#[test]fn thread_archived_wire_shape_with_origin() {let event = Event::with_event_id(@@ -114,6 +186,19 @@ fn decode_round_trips() {assert_eq!(decoded.event(), original.event());}+#[test]+fn new_contract_events_decode_round_trip() {+ let link_id = Uuid::new_v4();++ for original in new_contract_events(link_id) {+ let payload = serde_json::to_vec(original.event()).expect("serializable");+ let decoded = EmailMacroEvent::decode(original.key(), &payload).expect("decodable payload");++ assert_eq!(decoded.key(), link_id.to_string());+ assert_eq!(decoded.event(), original.event());+ }+}+#[test]fn events_are_keyed_by_link_id() {let link_id = Uuid::new_v4();@@ -126,6 +211,23 @@ fn events_are_keyed_by_link_id() {assert_eq!(event.key(), link_id.to_string());}+#[test]+fn new_contract_event_constructors_are_keyed_by_link_id() {+ let link_id = Uuid::new_v4();++ for event in new_contract_events(link_id) {+ assert_eq!(event.key(), link_id.to_string());+ }+}++#[test]+fn threads_reindex_reason_uses_snake_case() {+ let value = serde_json::to_value(ThreadsReindexReason::ContactsChanged)+ .expect("reason is serializable");++ assert_eq!(value, json!("contacts_changed"));+}+#[test]fn event_type_strings_follow_dot_convention() {let link_id = Uuid::nil();@@ -181,6 +283,18 @@ fn event_type_strings_follow_dot_convention() {}),"email.message_received",),+ (+ EmailMacroEvent::message_draft_synced(MessageDraftSyncedMetadata {+ link_id,+ owner: owner.clone(),+ message_id: Uuid::nil(),+ provider_message_id: "m".to_string(),+ thread_id: Uuid::nil(),+ provider_thread_id: "t".to_string(),+ is_spam_or_trash: false,+ }),+ "email.message_draft_synced",+ ),(EmailMacroEvent::message_sent(MessageSentMetadata {link_id,@@ -275,6 +389,17 @@ fn event_type_strings_follow_dot_convention() {}),"email.thread_starred",),+ (+ EmailMacroEvent::thread_spam_changed(ThreadSpamChangedMetadata {+ link_id,+ owner: owner.clone(),+ actor: actor.clone(),+ thread_id: Uuid::nil(),+ spam: true,+ origin: EmailEventOrigin::UserAction,+ }),+ "email.thread_spam_changed",+ ),(EmailMacroEvent::thread_project_changed(ThreadProjectChangedMetadata {link_id,@@ -302,6 +427,23 @@ fn event_type_strings_follow_dot_convention() {}),"email.thread_labels_updated",),+ (+ EmailMacroEvent::thread_backfilled(ThreadBackfilledMetadata {+ link_id,+ owner: owner.clone(),+ thread_id: Uuid::nil(),+ }),+ "email.thread_backfilled",+ ),+ (+ EmailMacroEvent::threads_reindex_requested(ThreadsReindexRequestedMetadata {+ link_id,+ owner,+ thread_ids: vec![Uuid::nil()],+ reason: ThreadsReindexReason::ContactsChanged,+ }),+ "email.threads_reindex_requested",+ ),];for (event, expected_type) in cases {@@ -349,6 +491,8 @@ fn thread_label_change_maps_system_labels_to_semantic_events() {("TRASH", false, "email.thread_trashed", "trashed", false),("INBOX", true, "email.thread_archived", "archived", false),("INBOX", false, "email.thread_archived", "archived", true),+ ("SPAM", true, "email.thread_spam_changed", "spam", true),+ ("SPAM", false, "email.thread_spam_changed", "spam", false),];for (label, added, expected_type, state_field, expected_state) in cases {@@ -365,7 +509,7 @@ fn thread_label_change_maps_system_labels_to_semantic_events() {#[test]fn thread_label_change_skips_unpublished_system_labels() {- for label in ["SPAM", "IMPORTANT", "SENT", "DRAFT"] {+ for label in ["IMPORTANT", "SENT", "DRAFT"] {assert!(label_change(system_label(label), true).is_none(),"{label} add should not publish"diff --git a/crates/opensearch_client/src/delete/email.rs b/crates/opensearch_client/src/delete/email.rsindex 0f97c57728..d697a31c88 100644--- a/crates/opensearch_client/src/delete/email.rs+++ b/crates/opensearch_client/src/delete/email.rs@@ -2,6 +2,13 @@ use models_opensearch::SearchIndex;use crate::{Result, error::OpensearchClientError};+#[cfg(test)]+mod test;++fn resolve_destination(index_override: Option<&str>) -> &str {+ index_override.unwrap_or(SearchIndex::Emails.as_ref())+}+/// Deletes all email messages with the specified link_id#[tracing::instrument(skip(client))]pub async fn delete_email_by_link_id(client: &opensearch::OpenSearch, link_id: &str) -> Result<()> {@@ -58,6 +65,7 @@ pub async fn delete_email_by_link_id(client: &opensearch::OpenSearch, link_id: &pub async fn delete_email_message_by_id(client: &opensearch::OpenSearch,message_id: &str,+ index_override: Option<&str>,) -> Result<()> {let query = serde_json::json!({"query": {@@ -67,10 +75,9 @@ pub async fn delete_email_message_by_id(},});+ let index = resolve_destination(index_override);let response = client- .delete_by_query(opensearch::DeleteByQueryParts::Index(&[- SearchIndex::Emails.as_ref(),- ]))+ .delete_by_query(opensearch::DeleteByQueryParts::Index(&[index])).body(query).refresh(true) // Ensure the index reflects changes immediately.send()diff --git a/crates/opensearch_client/src/delete/email/test.rs b/crates/opensearch_client/src/delete/email/test.rsnew file mode 100644index 0000000000..09ef960726--- /dev/null+++ b/crates/opensearch_client/src/delete/email/test.rs@@ -0,0 +1,17 @@+use super::*;++#[test]+fn resolves_default_email_destination() {+ assert_eq!(+ resolve_destination(None),+ models_opensearch::SearchIndex::Emails.as_ref()+ );+}++#[test]+fn resolves_overridden_email_destination() {+ assert_eq!(+ resolve_destination(Some("emails_backfill")),+ "emails_backfill"+ );+}diff --git a/crates/opensearch_client/src/email.rs b/crates/opensearch_client/src/email.rsindex d913d188d7..8f17ee22de 100644--- a/crates/opensearch_client/src/email.rs+++ b/crates/opensearch_client/src/email.rs@@ -32,8 +32,12 @@ impl OpensearchClient {}/// Deletes a particular email message- pub async fn delete_email_message_by_id(&self, message_id: &str) -> Result<()> {- delete::email::delete_email_message_by_id(&self.inner, message_id).await+ pub async fn delete_email_message_by_id(+ &self,+ message_id: &str,+ index_override: Option<&str>,+ ) -> Result<()> {+ delete::email::delete_email_message_by_id(&self.inner, message_id, index_override).await}/// Deletes all email messages with the specified link_iddiff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rsindex 08fab0ca4b..db835ed05e 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs@@ -225,33 +225,57 @@ fn patches_from_chat_event(event: &ChatTopicEvent) -> Vec<SoupRealtimePatch> {}fn patches_from_email_event(event: &EmailTopicEvent) -> Vec<SoupRealtimePatch> {- if let EmailTopicEvent::ThreadTrashed(metadata) = event- && metadata.trashed- {- return vec![delete(EntityType::EmailThread, metadata.thread_id)];- }+ let updated_thread = |thread_id| vec![update(EntityType::EmailThread, thread_id)];+ let deleted_thread = |thread_id| vec![delete(EntityType::EmailThread, thread_id)];- let thread_id = match event {- EmailTopicEvent::MessageReceived(metadata) if !metadata.is_spam_or_trash => {- metadata.thread_id- }- EmailTopicEvent::MessageSent(metadata) => metadata.thread_id,- EmailTopicEvent::MessageDeleted(metadata) => metadata.thread_id,- EmailTopicEvent::MessageSendQueued(metadata) => metadata.thread_id,- EmailTopicEvent::MessageSendCancelled(metadata) => metadata.thread_id,- EmailTopicEvent::ThreadArchived(metadata) => metadata.thread_id,- EmailTopicEvent::ThreadTrashed(metadata) if !metadata.trashed => metadata.thread_id,- EmailTopicEvent::ThreadRead(metadata) => metadata.thread_id,- EmailTopicEvent::ThreadStarred(metadata) => metadata.thread_id,- EmailTopicEvent::ThreadProjectChanged(metadata) => metadata.thread_id,- EmailTopicEvent::ThreadLabelsUpdated(metadata) => metadata.thread_id,+ match event {+ EmailTopicEvent::MessageReceived(metadata) => {+ if metadata.is_spam_or_trash {+ Vec::new()+ } else {+ updated_thread(metadata.thread_id)+ }+ }+ EmailTopicEvent::MessageDraftSynced(metadata) => {+ if metadata.is_spam_or_trash {+ Vec::new()+ } else {+ updated_thread(metadata.thread_id)+ }+ }+ EmailTopicEvent::MessageSent(metadata) => updated_thread(metadata.thread_id),+ EmailTopicEvent::MessageDeleted(metadata) => updated_thread(metadata.thread_id),+ EmailTopicEvent::MessageSendQueued(metadata) => updated_thread(metadata.thread_id),+ EmailTopicEvent::MessageSendCancelled(metadata) => updated_thread(metadata.thread_id),+ EmailTopicEvent::ThreadArchived(metadata) => updated_thread(metadata.thread_id),+ EmailTopicEvent::ThreadTrashed(metadata) => {+ if metadata.trashed {+ deleted_thread(metadata.thread_id)+ } else {+ updated_thread(metadata.thread_id)+ }+ }+ EmailTopicEvent::ThreadRead(metadata) => updated_thread(metadata.thread_id),+ EmailTopicEvent::ThreadStarred(metadata) => updated_thread(metadata.thread_id),+ EmailTopicEvent::ThreadSpamChanged(metadata) => {+ if metadata.spam {+ deleted_thread(metadata.thread_id)+ } else {+ updated_thread(metadata.thread_id)+ }+ }+ EmailTopicEvent::ThreadProjectChanged(metadata) => updated_thread(metadata.thread_id),+ EmailTopicEvent::ThreadLabelsUpdated(metadata) => updated_thread(metadata.thread_id),+ EmailTopicEvent::ThreadBackfilled(metadata) => updated_thread(metadata.thread_id),+ EmailTopicEvent::ThreadsReindexRequested(metadata) => metadata+ .thread_ids+ .iter()+ .map(|thread_id| update(EntityType::EmailThread, thread_id))+ .collect(),EmailTopicEvent::LinkConnected(_)| EmailTopicEvent::LinkDisconnected(_)- | EmailTopicEvent::LinkReauthRequired(_)- | EmailTopicEvent::MessageReceived(_)- | EmailTopicEvent::ThreadTrashed(_) => return Vec::new(),- };- vec![update(EntityType::EmailThread, thread_id)]+ | EmailTopicEvent::LinkReauthRequired(_) => Vec::new(),+ }}fn channel_and_thread_entities(diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rsindex c44cf68b5e..1443fe44fd 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs@@ -15,7 +15,9 @@ use documents::domain::events::{DocumentUpdatedMetadata, InteractionReason,};use email::domain::events::{- EmailEventOrigin, EmailTopicEvent, ThreadReadMetadata, ThreadTrashedMetadata,+ EmailEventOrigin, EmailTopicEvent, MessageDraftSyncedMetadata, ThreadBackfilledMetadata,+ ThreadReadMetadata, ThreadSpamChangedMetadata, ThreadTrashedMetadata, ThreadsReindexReason,+ ThreadsReindexRequestedMetadata,};use macro_event_broker::{Event, EventBrokerError, MacroEventCollection as _, MessageParts};use macro_user_id::user_id::MacroUserIdStr;@@ -378,6 +380,89 @@ fn email_state_events_map_to_updated_or_deleted_patches() {));}+#[test]+fn new_email_events_map_to_realtime_thread_patches() {+ let link_id = Uuid::now_v7();+ let thread_id = Uuid::now_v7();+ let second_thread_id = Uuid::now_v7();++ let visible_draft = EmailTopicEvent::MessageDraftSynced(MessageDraftSyncedMetadata {+ link_id,+ owner: user(),+ message_id: Uuid::now_v7(),+ provider_message_id: "message-id".to_string(),+ thread_id,+ provider_thread_id: "thread-id".to_string(),+ is_spam_or_trash: false,+ });+ let hidden_draft = EmailTopicEvent::MessageDraftSynced(MessageDraftSyncedMetadata {+ link_id,+ owner: user(),+ message_id: Uuid::now_v7(),+ provider_message_id: "hidden-message-id".to_string(),+ thread_id,+ provider_thread_id: "thread-id".to_string(),+ is_spam_or_trash: true,+ });+ let marked_spam = EmailTopicEvent::ThreadSpamChanged(ThreadSpamChangedMetadata {+ link_id,+ owner: user(),+ actor: Some(user()),+ thread_id,+ spam: true,+ origin: EmailEventOrigin::UserAction,+ });+ let restored_from_spam = EmailTopicEvent::ThreadSpamChanged(ThreadSpamChangedMetadata {+ link_id,+ owner: user(),+ actor: Some(user()),+ thread_id,+ spam: false,+ origin: EmailEventOrigin::UserAction,+ });+ let backfilled = EmailTopicEvent::ThreadBackfilled(ThreadBackfilledMetadata {+ link_id,+ owner: user(),+ thread_id,+ });+ let reindex_requested =+ EmailTopicEvent::ThreadsReindexRequested(ThreadsReindexRequestedMetadata {+ link_id,+ owner: user(),+ thread_ids: vec![thread_id, second_thread_id],+ reason: ThreadsReindexReason::ContactsChanged,+ });++ assert!(matches!(+ patches_from_email_event(&visible_draft)[0].patch,+ Patch::Updated(_)+ ));+ assert!(patches_from_email_event(&hidden_draft).is_empty());+ assert!(matches!(+ patches_from_email_event(&marked_spam)[0].patch,+ Patch::Deleted(_)+ ));+ assert!(matches!(+ patches_from_email_event(&restored_from_spam)[0].patch,+ Patch::Updated(_)+ ));+ assert!(matches!(+ patches_from_email_event(&backfilled)[0].patch,+ Patch::Updated(_)+ ));++ let reindex_patches = patches_from_email_event(&reindex_requested);+ assert_eq!(reindex_patches.len(), 2);+ assert_eq!(+ patch_entity(&reindex_patches[0]).entity_id,+ thread_id.to_string()+ );+ assert_eq!(+ patch_entity(&reindex_patches[1]).entity_id,+ second_thread_id.to_string()+ );+}+#[test]fn attachment_events_use_only_metadata_available_on_the_existing_event() {let channel_id = Uuid::now_v7();diff --git a/crates/sqs_client/src/search/email.rs b/crates/sqs_client/src/search/email.rsindex 2982d928f1..b7afa62bec 100644--- a/crates/sqs_client/src/search/email.rs+++ b/crates/sqs_client/src/search/email.rs@@ -1,22 +1,3 @@-#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]-pub struct EmailMessage {- /// The message id- pub message_id: String,- /// The macro user id of the user who the message is for- pub macro_user_id: String,-}--#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]-pub struct EmailThreadMessage {- /// The thread id- pub thread_id: String,- /// The macro user id of the user who the message is for- pub macro_user_id: String,- /// Optional override for the target OpenSearch index (e.g. "emails_v2" for migration backfills)- #[serde(skip_serializing_if = "Option::is_none")]- pub index_override: Option<String>,-}-#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]pub struct EmailThreadBatchMessage {/// The thread ids to process@@ -27,11 +8,3 @@ pub struct EmailThreadBatchMessage {#[serde(skip_serializing_if = "Option::is_none")]pub index_override: Option<String>,}--#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]-pub struct EmailLinkMessage {- /// The link id- pub link_id: String,- /// The macro user id associated with the link- pub macro_user_id: String,-}diff --git a/crates/sqs_client/src/search/mod.rs b/crates/sqs_client/src/search/mod.rsindex a3bd10ffbe..30850b9a72 100644--- a/crates/sqs_client/src/search/mod.rs+++ b/crates/sqs_client/src/search/mod.rs@@ -5,7 +5,7 @@ use crate::{channel::ChannelMessageUpdate,chat::ChatMessage,document::SearchExtractorMessage,- email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},+ email::EmailThreadBatchMessage,project::UpsertProject,},};@@ -74,11 +74,7 @@ pub enum SearchQueueMessage {// Chat/// SQS backfill work-queue contract for reconciling a chat message.ChatMessage(ChatMessage),- ExtractEmailMessage(EmailMessage),- RemoveEmailMessage(EmailMessage),- ExtractEmailThreadMessage(EmailThreadMessage),- RemoveEmailLink(EmailLinkMessage),+ // Email backfillsExtractEmailThreadBatch(EmailThreadBatchMessage),// ChannelChannelMessageUpdate(ChannelMessageUpdate),@@ -99,13 +95,9 @@ impl PrimaryId for SearchQueueMessage {SearchQueueMessage::ExtractSync(message) => message.document_id.clone(),// The message id keeps entries unique within an SQS batch.SearchQueueMessage::ChatMessage(message) => message.message_id.clone(),- SearchQueueMessage::ExtractEmailMessage(message)- | SearchQueueMessage::RemoveEmailMessage(message) => message.message_id.clone(),- SearchQueueMessage::ExtractEmailThreadMessage(message) => message.thread_id.clone(),SearchQueueMessage::ExtractEmailThreadBatch(message) => {message.thread_ids.first().cloned().unwrap_or_default()}- SearchQueueMessage::RemoveEmailLink(message) => message.link_id.clone(),SearchQueueMessage::ChannelMessageUpdate(message) => message.message_id.clone(),SearchQueueMessage::CallRecord(message) => message.call_id.clone(),SearchQueueMessage::RemoveCallRecord(message) => format!(@@ -128,12 +120,8 @@ impl SearchQueueMessage {SearchQueueMessage::ExtractSync(_) => Operation::ExtractSync,// ChatSearchQueueMessage::ChatMessage(_) => Operation::ExtractText,- SearchQueueMessage::ExtractEmailMessage(_) => Operation::ExtractText,- SearchQueueMessage::RemoveEmailMessage(_) => Operation::Remove,- SearchQueueMessage::ExtractEmailThreadMessage(_) => Operation::ExtractText,+ // Email backfillsSearchQueueMessage::ExtractEmailThreadBatch(_) => Operation::ExtractText,- SearchQueueMessage::RemoveEmailLink(_) => Operation::Remove,// ChannelsSearchQueueMessage::ChannelMessageUpdate(_) => Operation::ExtractText,// Callsdiff --git a/services/email_service/Cargo.toml b/services/email_service/Cargo.tomlindex a2cda81b6e..8b3f78c2b5 100644--- a/services/email_service/Cargo.toml+++ b/services/email_service/Cargo.toml@@ -158,7 +158,6 @@ sqlx-core = { workspace = true }sqs_client = { path = "../../crates/sqs_client", default-features = false, features = ["email","gmail",- "search","sfs_delete","sfs_uploader"] }diff --git a/services/email_service/src/backfill_init_service.rs b/services/email_service/src/backfill_init_service.rsindex d2bb1be3a3..426ae22cef 100644--- a/services/email_service/src/backfill_init_service.rs+++ b/services/email_service/src/backfill_init_service.rs@@ -91,6 +91,7 @@ async fn initialize_backfill(&ctx.db,&ctx.gmail_client,&ctx.sqs_client,+ &ctx.macro_event_broker,access_token,).awaitdiff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rsindex 773a7f0e2d..9a1c2d1e49 100644--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs@@ -75,7 +75,6 @@ async fn main() -> anyhow::Result<()> {let gmail_inbox_sync_retry_queue = macro_queues::GmailInboxSyncRetryQueue::new();let gmail_ops_queue = macro_queues::GmailOpsQueue::new();let gmail_ops_retry_queue = macro_queues::GmailOpsRetryQueue::new();- let search_event_queue = macro_queues::SearchEventQueue::new();let backfill_queue = macro_queues::EmailBackfillQueue::new();let crm_cleanup_queue = macro_queues::EmailCrmCleanupQueue::new();let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();@@ -90,7 +89,6 @@ async fn main() -> anyhow::Result<()> {.gmail_inbox_sync_retry_queue(&gmail_inbox_sync_retry_queue).gmail_ops_queue(&gmail_ops_queue).gmail_ops_retry_queue(&gmail_ops_retry_queue)- .search_event_queue(&search_event_queue).email_backfill_queue(&backfill_queue).email_crm_cleanup_queue(&crm_cleanup_queue).email_scheduled_queue(&email_scheduled_queue)diff --git a/services/email_service/src/main.rs b/services/email_service/src/main.rsindex 98a3486f5a..93cf3c2a19 100644--- a/services/email_service/src/main.rs+++ b/services/email_service/src/main.rs@@ -66,7 +66,6 @@ async fn main() -> anyhow::Result<()> {let gmail_inbox_sync_queue = macro_queues::GmailInboxSyncQueue::new();let gmail_inbox_sync_retry_queue = macro_queues::GmailInboxSyncRetryQueue::new();let gmail_ops_queue = macro_queues::GmailOpsQueue::new();- let search_event_queue = macro_queues::SearchEventQueue::new();let backfill_queue = macro_queues::EmailBackfillQueue::new();let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();let sfs_uploader_queue = macro_queues::SfsUploaderQueue::new();@@ -75,7 +74,6 @@ async fn main() -> anyhow::Result<()> {.gmail_inbox_sync_queue(&gmail_inbox_sync_queue).gmail_inbox_sync_retry_queue(&gmail_inbox_sync_retry_queue).gmail_ops_queue(&gmail_ops_queue)- .search_event_queue(&search_event_queue).email_backfill_queue(&backfill_queue).email_scheduled_queue(&email_scheduled_queue).sfs_uploader_queue(&sfs_uploader_queue)diff --git a/services/email_service/src/pubsub/backfill/update_metadata.rs b/services/email_service/src/pubsub/backfill/update_metadata.rsindex 7a165abcee..47ec8bfe83 100644--- a/services/email_service/src/pubsub/backfill/update_metadata.rs+++ b/services/email_service/src/pubsub/backfill/update_metadata.rs@@ -1,10 +1,10 @@use crate::pubsub::backfill::increment_counters::incr_completed_threads;use crate::pubsub::context::PubSubContext;+use crate::pubsub::util::publish_email_event;+use email::domain::events::{EmailMacroEvent, ThreadBackfilledMetadata};use models_email::email::service::backfill::{JobScopedPayload, UpdateMetadataPayload};use models_email::email::service::link;use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};-use sqs_client::search::SearchQueueMessage;-use sqs_client::search::email::EmailThreadMessage;/// This step is invoked by BackfillMessage once all messages in a thread have been backfilled./// Updates the thread metadata in the database, the replying_to_id values of its messages, and@@ -80,22 +80,16 @@ pub async fn update_thread_metadata(incr_completed_threads(ctx, link, scope.job_id).await?;- // notify search-service about the new thread- let search_message = SearchQueueMessage::ExtractEmailThreadMessage(EmailThreadMessage {- thread_id: p.thread_db_id.clone().to_string(),- macro_user_id: link.macro_id.to_string(),- index_override: None,- });-- ctx.sqs_client- .send_message_to_search_event_queue(search_message)- .await- .map_err(|e| {- ProcessingError::NonRetryable(DetailedError {- reason: FailureReason::SqsEnqueueFailed,- source: e.context("Failed to send message to search extractor queue"),- })- })?;+ // Best-effort by design: publication failures are logged and dropped so a committed+ // backfill is not retried after its completed-thread counters have advanced.+ publish_email_event(+ &ctx.macro_event_broker,+ &EmailMacroEvent::thread_backfilled(ThreadBackfilledMetadata {+ link_id: link.id,+ owner: link.macro_id.clone(),+ thread_id: p.thread_db_id,+ }),+ );Ok(())}diff --git a/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rsindex 4f0b07bc05..ce1c0d9063 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs@@ -8,8 +8,6 @@ use models_email::api::refresh::RefreshEmailEvent;use models_email::email::service::link;use models_email::gmail::inbox_sync::DeleteMessagePayload;use models_email::service::pubsub::{DetailedError, FailureReason, ProcessingError};-use sqs_client::search::SearchQueueMessage;-use sqs_client::search::email::EmailMessage;use std::result;use uuid::Uuid;@@ -134,19 +132,5 @@ pub async fn delete_message().await;- // send message to search text extractor queue- let _ = ctx- .sqs_client- .bulk_send_message_to_search_event_queue(vec![SearchQueueMessage::RemoveEmailMessage(- EmailMessage {- message_id: message.db_id.to_string(),- macro_user_id: link.macro_id.to_string(),- },- )])- .await- .inspect_err(- |e| tracing::error!(error = ?e, "failed to send message to search extractor queue"),- );-Ok(())}diff --git a/services/email_service/src/pubsub/inbox_sync/operations/mod.rs b/services/email_service/src/pubsub/inbox_sync/operations/mod.rsindex 694cca1d37..c50c3c3df8 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/mod.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/mod.rs@@ -1,5 +1,4 @@pub(in crate::pubsub) mod delete_message;pub(in crate::pubsub) mod gmail_message;-pub(in crate::pubsub) mod shared;pub(in crate::pubsub) mod update_labels;pub(in crate::pubsub) mod upsert_message;diff --git a/services/email_service/src/pubsub/inbox_sync/operations/shared.rs b/services/email_service/src/pubsub/inbox_sync/operations/shared.rsdeleted file mode 100644index 491d8faba4..0000000000--- a/services/email_service/src/pubsub/inbox_sync/operations/shared.rs+++ /dev/null@@ -1,39 +0,0 @@-use crate::pubsub::context::PubSubContext;-use models_email::service::link;-use models_email::service::pubsub::ProcessingError;-use sqs_client::search::SearchQueueMessage;-use sqs_client::search::email::EmailMessage;-use std::result;-use uuid::Uuid;--/// Notify search about a message change. If the message is spam or trash, it will be removed from-/// search. Otherwise, it will be upserted.-#[tracing::instrument(skip(ctx, link, message_db_id))]-pub async fn notify_search(- ctx: &PubSubContext,- link: &link::Link,- message_db_id: Uuid,- is_spam_or_trash: bool,-) -> result::Result<(), ProcessingError> {- let message = if is_spam_or_trash {- SearchQueueMessage::RemoveEmailMessage(EmailMessage {- message_id: message_db_id.to_string(),- macro_user_id: link.macro_id.to_string(),- })- } else {- SearchQueueMessage::ExtractEmailMessage(EmailMessage {- message_id: message_db_id.to_string(),- macro_user_id: link.macro_id.to_string(),- })- };-- ctx.sqs_client- .send_message_to_search_event_queue(message)- .await- .inspect_err(- |e| tracing::error!(error = ?e, "failed to send message to search extractor queue"),- )- .ok();-- Ok(())-}diff --git a/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs b/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rsindex 94eedaa6a2..8d6d07dc5d 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs@@ -1,5 +1,4 @@use crate::pubsub::context::PubSubContext;-use crate::pubsub::inbox_sync::operations::shared::notify_search;use crate::pubsub::inbox_sync::process;use crate::pubsub::inbox_sync::process::check_gmail_rate_limit_inbox_sync;use crate::pubsub::util::{@@ -7,11 +6,13 @@ use crate::pubsub::util::{};use email::domain::events::{EmailEventOrigin, EmailMacroEvent, LabelRef, ThreadArchivedMetadata,- ThreadLabelsUpdatedMetadata, ThreadReadMetadata, ThreadStarredMetadata, ThreadTrashedMetadata,+ ThreadLabelsUpdatedMetadata, ThreadReadMetadata, ThreadSpamChangedMetadata,+ ThreadStarredMetadata, ThreadTrashedMetadata,};use email_db_client::labels::delete::delete_db_message_labels;use email_db_client::labels::insert;use email_db_client::threads::update::update_thread_metadata;+use macro_user_id::user_id::MacroUserIdStr;use models_email::api::refresh::RefreshEmailEvent;use models_email::email::service::link;use models_email::gmail::inbox_sync::{@@ -24,6 +25,9 @@ use sqlx::PgPool;use std::result;use uuid::Uuid;+#[cfg(test)]+mod test;+/// Compares a db message's labels to the gmail message's labels, adding/removing as necessary#[tracing::instrument(skip(ctx))]pub async fn update_labels(@@ -159,9 +163,6 @@ pub async fn update_labels(}if has_label_changes {- // Publish semantic macro.email events before the fallible search- // notify: a notify failure retries the whole op, and on retry the- // label diff is empty so the events would be lost.publish_label_diff_events(ctx,link,@@ -172,13 +173,6 @@ pub async fn update_labels().await;- let is_spam_or_trash = gmail_message_labels.iter().any(|label| {- label == service::label::system_labels::SPAM- || label == service::label::system_labels::TRASH- });-- notify_search(ctx, link, db_message.db_id, is_spam_or_trash).await?;-// tell FE to refresh user's inboxcg_refresh_email(&ctx.connection_gateway_client,@@ -192,8 +186,8 @@ pub async fn update_labels(}/// System labels that never appear in a `thread_labels_updated` event:-/// INBOX/TRASH/UNREAD/STARRED map to dedicated thread-state events, and-/// SPAM/IMPORTANT/SENT/DRAFT changes are not published at all.+/// INBOX/TRASH/UNREAD/STARRED/SPAM map to dedicated thread-state events, and+/// IMPORTANT/SENT/DRAFT changes are not published.const NON_USER_LABELS: [&str; 8] = [service::label::system_labels::INBOX,service::label::system_labels::SPAM,@@ -205,6 +199,36 @@ const NON_USER_LABELS: [&str; 8] = [service::label::system_labels::DRAFT,];+fn build_provider_spam_changed_metadata(+ link_id: Uuid,+ owner: &MacroUserIdStr<'static>,+ thread_id: Uuid,+ labels_to_add: &[String],+ labels_to_delete: &[String],+) -> Option<ThreadSpamChangedMetadata> {+ let spam_label = service::label::system_labels::SPAM;+ let spam = if labels_to_add.iter().any(|label| label == spam_label) {+ true+ } else if labels_to_delete.iter().any(|label| label == spam_label) {+ false+ } else {+ return None;+ };++ Some(ThreadSpamChangedMetadata {+ link_id,+ owner: owner.clone(),+ actor: None,+ thread_id,+ spam,+ origin: EmailEventOrigin::ProviderSync,+ })+}++fn is_user_label(label: &str) -> bool {+ !NON_USER_LABELS.contains(&label) && !label.starts_with("CATEGORY_")+}+/// Publish semantic macro.email events for a provider-side label diff, all/// with `origin = provider_sync`. Changes made through Macro update the DB/// optimistically, so by the time they echo back through Gmail history sync@@ -272,6 +296,16 @@ async fn publish_label_diff_events(}));}+ if let Some(metadata) = build_provider_spam_changed_metadata(+ link.id,+ &link.macro_id,+ thread_db_id,+ labels_to_add,+ labels_to_delete,+ ) {+ events.push(EmailMacroEvent::thread_spam_changed(metadata));+ }+if added(service::label::system_labels::UNREAD)|| removed(service::label::system_labels::UNREAD){@@ -301,13 +335,9 @@ async fn publish_label_diff_events(}));}- // Gmail category tabs (CATEGORY_PERSONAL/SOCIAL/...) are provider system- // labels, not user labels.- let is_user_label = |l: &str| !NON_USER_LABELS.contains(&l) && !l.starts_with("CATEGORY_");-let added_user: Vec<LabelRef> = labels_to_add.iter()- .filter(|l| is_user_label(l.as_str()))+ .filter(|label| is_user_label(label.as_str())).map(|l| LabelRef {label_id: None,provider_label_id: l.clone(),@@ -316,14 +346,14 @@ async fn publish_label_diff_events(.collect();let removed_user: Vec<LabelRef> = db_message_labels.iter()- .filter(|l| {- labels_to_delete.contains(&l.provider_label_id)- && is_user_label(l.provider_label_id.as_str())+ .filter(|label| {+ labels_to_delete.contains(&label.provider_label_id)+ && is_user_label(label.provider_label_id.as_str())})- .map(|l| LabelRef {- label_id: Some(l.id),- provider_label_id: l.provider_label_id.clone(),- name: Some(l.name.clone()),+ .map(|label| LabelRef {+ label_id: Some(label.id),+ provider_label_id: label.provider_label_id.clone(),+ name: Some(label.name.clone()),}).collect();diff --git a/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rsnew file mode 100644index 0000000000..24cba89e10--- /dev/null+++ b/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs@@ -0,0 +1,72 @@+use super::*;++fn owner() -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).unwrap()+}++#[test]+fn builds_provider_origin_spam_addition() {+ let link_id = Uuid::from_u128(1);+ let thread_id = Uuid::from_u128(2);+ let owner = owner();+ let labels_to_add = vec![service::label::system_labels::SPAM.to_string()];++ let metadata =+ build_provider_spam_changed_metadata(link_id, &owner, thread_id, &labels_to_add, &[]);++ assert_eq!(+ metadata,+ Some(ThreadSpamChangedMetadata {+ link_id,+ owner,+ actor: None,+ thread_id,+ spam: true,+ origin: EmailEventOrigin::ProviderSync,+ })+ );+}++#[test]+fn builds_provider_origin_spam_removal() {+ let link_id = Uuid::from_u128(1);+ let thread_id = Uuid::from_u128(2);+ let owner = owner();+ let labels_to_delete = vec![service::label::system_labels::SPAM.to_string()];++ let metadata =+ build_provider_spam_changed_metadata(link_id, &owner, thread_id, &[], &labels_to_delete);++ assert_eq!(+ metadata,+ Some(ThreadSpamChangedMetadata {+ link_id,+ owner,+ actor: None,+ thread_id,+ spam: false,+ origin: EmailEventOrigin::ProviderSync,+ })+ );+}++#[test]+fn ignores_diffs_without_spam_changes() {+ let labels_to_add = vec![service::label::system_labels::STARRED.to_string()];++ assert_eq!(+ build_provider_spam_changed_metadata(+ Uuid::from_u128(1),+ &owner(),+ Uuid::from_u128(2),+ &labels_to_add,+ &[],+ ),+ None+ );+}++#[test]+fn spam_remains_excluded_from_user_label_events() {+ assert!(!is_user_label(service::label::system_labels::SPAM));+}diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rsindex 4e26a21a10..ccf9bc298e 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs@@ -1,6 +1,5 @@use crate::convert::{map_message_resource_to_service, map_thread_resources_to_service};use crate::pubsub::context::PubSubContext;-use crate::pubsub::inbox_sync::operations::shared::notify_search;use crate::pubsub::inbox_sync::process;use crate::pubsub::inbox_sync::process::check_gmail_rate_limit_inbox_sync;use crate::pubsub::util::{@@ -12,7 +11,8 @@ use crate::util::upload_attachment::{UploadAttachmentContext, upload_attachment}use contacts::domain::models::messages::ContactConnection;use contacts::domain::ports::ContactsIngress;use email::domain::events::{- EmailEventOrigin, EmailMacroEvent, MessageReceivedMetadata, MessageSentMetadata,+ EmailEventOrigin, EmailMacroEvent, MessageDraftSyncedMetadata, MessageReceivedMetadata,+ MessageSentMetadata,};use email::domain::models::{PreviewCursorQuery, PreviewView, PreviewViewStandardLabel};use email::domain::ports::EmailRepo;@@ -44,6 +44,37 @@ use uuid::Uuid;#[cfg(test)]mod test;+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum MessageSyncEventKind {+ DraftSynced,+ Received,+ Sent,+}++fn select_message_sync_event(+ existing_message_was_draft: Option<bool>,+ is_draft: bool,+ is_sent: bool,+) -> Option<MessageSyncEventKind> {+ if is_draft {+ return Some(MessageSyncEventKind::DraftSynced);+ }++ if existing_message_was_draft == Some(true) && is_sent {+ return Some(MessageSyncEventKind::Sent);+ }++ if existing_message_was_draft.is_some() {+ return None;+ }++ if is_sent {+ Some(MessageSyncEventKind::Sent)+ } else {+ Some(MessageSyncEventKind::Received)+ }+}+// upsert a message into the db. could be a new message or an existing one that had changes#[tracing::instrument(skip(ctx))]pub async fn upsert_message(@@ -174,20 +205,24 @@ pub async fn upsert_message(})})?;- // before upserting, figure out if the message is new so we can send a notification for it if so- let message_already_exists = email_db_client::messages::get::message_exists_by_provider_id(- &ctx.db,- &payload.provider_message_id,- link.id,- )- .await- .map_err(|e| {- ProcessingError::NonRetryable(DetailedError {- reason: FailureReason::DatabaseQueryFailed,- source: e- .context("Failed to check whether provider_message_id already exists".to_string()),- })- })?;+ // Snapshot the previous draft state before the upsert. Provider drafts are mutable,+ // and this state distinguishes a draft-to-sent transition from an unchanged message.+ let existing_message =+ email_db_client::messages::get_simple_messages::get_simple_message_by_provider_and_link(+ &ctx.db,+ &payload.provider_message_id,+ &link.id,+ )+ .await+ .map_err(|e| {+ ProcessingError::NonRetryable(DetailedError {+ reason: FailureReason::DatabaseQueryFailed,+ source: e+ .context("Failed to fetch existing message before provider upsert".to_string()),+ })+ })?;+ let existing_message_was_draft = existing_message.as_ref().map(|message| message.is_draft);+ let message_already_exists = existing_message.is_some();let is_new_thread = !thread_provider_to_db_map.contains_key(&provider_thread_id);@@ -233,17 +268,44 @@ pub async fn upsert_message(})})?;- // Publish to the macro.email topic immediately after the committed- // insert, BEFORE the fallible side-effect steps below: if any of them- // fails, the SQS retry finds the message already existing and would- // never emit. Drafts are skipped — a draft synced from the provider is- // not a received or sent message.- if !message_already_exists && !is_draft {- let event = if is_sent {- // Sent from another client and first observed via sync; sends- // performed through Macro are published by the scheduled-send- // worker and already exist here.- EmailMacroEvent::message_sent(MessageSentMetadata {+ // Publish to the macro.email topic immediately after the committed insert.+ // Drafts publish after every sync because their bodies are mutable. Existing+ // immutable messages remain suppressed except when a previously synced provider+ // draft becomes sent.+ if let Some(event_kind) =+ select_message_sync_event(existing_message_was_draft, is_draft, is_sent)+ {+ let event = match event_kind {+ MessageSyncEventKind::DraftSynced => {+ EmailMacroEvent::message_draft_synced(MessageDraftSyncedMetadata {+ link_id: link.id,+ owner: link.macro_id.clone(),+ message_id: message_db_id,+ provider_message_id: payload.provider_message_id.clone(),+ thread_id: thread_db_id,+ provider_thread_id: provider_thread_id.clone(),+ is_spam_or_trash,+ })+ }+ MessageSyncEventKind::Received => {+ EmailMacroEvent::message_received(MessageReceivedMetadata {+ link_id: link.id,+ owner: link.macro_id.clone(),+ message_id: message_db_id,+ provider_message_id: payload.provider_message_id.clone(),+ thread_id: thread_db_id,+ provider_thread_id: provider_thread_id.clone(),+ is_new_thread,+ subject: event_subject,+ from_email: event_from.as_ref().map(|contact| contact.email.clone()),+ from_name: event_from.as_ref().and_then(|contact| contact.name.clone()),+ to_emails: event_to_emails,+ attachment_count: message_attachment_count as u32,+ is_spam_or_trash,+ received_at: event_received_at,+ })+ }+ MessageSyncEventKind::Sent => EmailMacroEvent::message_sent(MessageSentMetadata {link_id: link.id,owner: link.macro_id.clone(),actor: None,@@ -256,24 +318,7 @@ pub async fn upsert_message(cc_emails: event_cc_emails,origin: EmailEventOrigin::ProviderSync,sent_at: event_sent_at.unwrap_or_else(chrono::Utc::now),- })- } else {- EmailMacroEvent::message_received(MessageReceivedMetadata {- link_id: link.id,- owner: link.macro_id.clone(),- message_id: message_db_id,- provider_message_id: payload.provider_message_id.clone(),- thread_id: thread_db_id,- provider_thread_id: provider_thread_id.clone(),- is_new_thread,- subject: event_subject,- from_email: event_from.as_ref().map(|c| c.email.clone()),- from_name: event_from.as_ref().and_then(|c| c.name.clone()),- to_emails: event_to_emails,- attachment_count: message_attachment_count as u32,- is_spam_or_trash,- received_at: event_received_at,- })+ }),};publish_email_event(&ctx.macro_event_broker, &event);}@@ -329,8 +374,6 @@ pub async fn upsert_message(enqueue_populate_crm_contacts(ctx, link.id, &self_email, crm_recipients, is_sent).await?;}- notify_search(ctx, link, message_db_id, is_spam_or_trash).await?;-// trigger FE inbox refreshcg_refresh_email(&ctx.connection_gateway_client,diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rsindex cc0f92f975..126d7dac23 100644--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs@@ -49,3 +49,49 @@ fn skips_unparseable_primaries_keeping_valid_ones() {HashSet::from([owner, id("macro|[email protected]")]));}++#[test]+fn selects_draft_sync_for_new_draft() {+ assert_eq!(+ select_message_sync_event(None, true, false),+ Some(MessageSyncEventKind::DraftSynced)+ );+}++#[test]+fn selects_draft_sync_for_draft_edit() {+ assert_eq!(+ select_message_sync_event(Some(true), true, false),+ Some(MessageSyncEventKind::DraftSynced)+ );+}++#[test]+fn selects_sent_for_provider_draft_to_sent_transition() {+ assert_eq!(+ select_message_sync_event(Some(true), false, true),+ Some(MessageSyncEventKind::Sent)+ );+}++#[test]+fn selects_sent_for_new_sent_message() {+ assert_eq!(+ select_message_sync_event(None, false, true),+ Some(MessageSyncEventKind::Sent)+ );+}++#[test]+fn selects_received_for_new_received_message() {+ assert_eq!(+ select_message_sync_event(None, false, false),+ Some(MessageSyncEventKind::Received)+ );+}++#[test]+fn suppresses_existing_immutable_non_drafts() {+ assert_eq!(select_message_sync_event(Some(false), false, false), None);+ assert_eq!(select_message_sync_event(Some(false), false, true), None);+}diff --git a/services/email_service/src/pubsub/link_manager/process.rs b/services/email_service/src/pubsub/link_manager/process.rsindex f1e9b2e5f5..906443ec5f 100644--- a/services/email_service/src/pubsub/link_manager/process.rs+++ b/services/email_service/src/pubsub/link_manager/process.rs@@ -18,8 +18,6 @@ use models_email::service::cache::TokenCacheKey;use models_email::service::link::{Link, UserProvider};use notification::domain::models::SendNotificationRequestBuilder;use notification::domain::service::NotificationIngress;-use sqs_client::search::SearchQueueMessage;-use sqs_client::search::email::EmailLinkMessage;use sqs_worker::cleanup_message;use std::time::Duration;use tokio_retry::RetryIf;@@ -204,6 +202,7 @@ async fn handle_refresh(&ctx.db,&ctx.gmail_client,&ctx.sqs_client,+ &ctx.macro_event_broker,gmail_access_token,).await@@ -369,18 +368,6 @@ async fn handle_delete(}).ok();- // inform search of deletion so it can wipe the email records from OS- ctx.sqs_client- .send_message_to_search_event_queue(SearchQueueMessage::RemoveEmailLink(EmailLinkMessage {- link_id: link.id.to_string(),- macro_user_id: link.macro_id.to_string(),- }))- .await- .inspect_err(|e| {- tracing::error!(error=?e, "failed to send message to search extractor queue");- })- .ok();-// Tear down CRM rows this link contributed to the user's team before// the big cascading link delete fires. Best-effort: a failure here// would only leave orphan `crm_contacts`/`crm_companies` rows behinddiff --git a/services/email_service/src/util/sync_contacts.rs b/services/email_service/src/util/sync_contacts.rsindex e9913436d1..594d2d8360 100644--- a/services/email_service/src/util/sync_contacts.rs+++ b/services/email_service/src/util/sync_contacts.rs@@ -1,24 +1,33 @@use crate::convert::map_person_to_contact;+use crate::pubsub::util::publish_email_event;use anyhow::{Context, anyhow};+use email::domain::events::{+ EmailMacroEvent, ThreadsReindexReason, ThreadsReindexRequestedMetadata,+};use futures::{StreamExt, stream};use gmail_client::GmailClient;+use macro_event_broker::MacroEventBroker;+use macro_user_id::user_id::MacroUserIdStr;use models_email::service::contact::Contact;use models_email::service::link::Link;use models_email::service::pubsub::SFSUploaderMessage;use models_email::service::sync_token::SyncTokens;use sqlx::PgPool;use sqs_client::SQS;-use sqs_client::search::SearchQueueMessage;-use sqs_client::search::email::EmailThreadBatchMessage;use std::collections::HashSet;use std::time::Instant;+use uuid::Uuid;++#[cfg(test)]+mod test;/// Syncs user's contacts with gmail-pub async fn sync_contacts(+pub async fn sync_contacts<B: MacroEventBroker>(link: &Link,db: &PgPool,gmail_client: &GmailClient,sqs_client: &SQS,+ macro_event_broker: &B,gmail_access_token: &str,) -> anyhow::Result<()> {// 1. Get existing sync tokens from our DB@@ -37,7 +46,7 @@ pub async fn sync_contacts(// 3. If we received any new/updated contacts, process and store them.if !new_contacts.is_empty() {- process_and_store_contacts(db, sqs_client, link, new_contacts).await?;+ process_and_store_contacts(db, sqs_client, macro_event_broker, link, new_contacts).await?;}// 4. Store the new sync tokens in our DB@@ -129,9 +138,10 @@ async fn fetch_new_contacts_from_google(}/// Handles processing (SFS uploads) and database storage for a list of contacts.-async fn process_and_store_contacts(+async fn process_and_store_contacts<B: MacroEventBroker>(db: &PgPool,sqs_client: &SQS,+ macro_event_broker: &B,link: &Link,contacts: Vec<Contact>,) -> anyhow::Result<()> {@@ -177,7 +187,8 @@ async fn process_and_store_contacts();if !changed_contact_ids.is_empty() {- reindex_threads_for_changed_contacts(db, sqs_client, link, &changed_contact_ids).await;+ reindex_threads_for_changed_contacts(db, macro_event_broker, link, &changed_contact_ids)+ .await;}if cfg!(not(feature = "sfs_map")) {@@ -215,12 +226,12 @@ async fn process_and_store_contacts(const REINDEX_BATCH_SIZE: usize = 50;-#[tracing::instrument(skip(db, sqs_client, link, changed_contact_ids))]-async fn reindex_threads_for_changed_contacts(+#[tracing::instrument(skip(db, macro_event_broker, link, changed_contact_ids))]+async fn reindex_threads_for_changed_contacts<B: MacroEventBroker>(db: &PgPool,- sqs_client: &SQS,+ macro_event_broker: &B,link: &Link,- changed_contact_ids: &[uuid::Uuid],+ changed_contact_ids: &[Uuid],) {let thread_ids = match email_db_client::threads::get::get_thread_ids_by_contact_ids(db,@@ -239,8 +250,6 @@ async fn reindex_threads_for_changed_contacts(return;}- let macro_user_id = link.macro_id.to_string();-tracing::info!(num_threads = thread_ids.len(),num_changed_contacts = changed_contact_ids.len(),@@ -248,21 +257,24 @@ async fn reindex_threads_for_changed_contacts("Re-indexing threads for contacts with name changes");- let messages: Vec<SearchQueueMessage> = thread_ids- .chunks(REINDEX_BATCH_SIZE)- .map(|chunk| {- SearchQueueMessage::ExtractEmailThreadBatch(EmailThreadBatchMessage {- thread_ids: chunk.iter().map(|id| id.to_string()).collect(),- macro_user_id: macro_user_id.clone(),- index_override: None,- })- })- .collect();+ publish_thread_reindex_batches(macro_event_broker, link.id, &link.macro_id, &thread_ids);+}- if let Err(e) = sqs_client- .bulk_send_message_to_search_event_queue(messages)- .await- {- tracing::error!(error=?e, link_id=%link.id, "Failed to enqueue search re-index messages for contact name changes");+fn publish_thread_reindex_batches<B: MacroEventBroker>(+ macro_event_broker: &B,+ link_id: Uuid,+ owner: &MacroUserIdStr<'static>,+ thread_ids: &[Uuid],+) {+ for thread_ids in thread_ids.chunks(REINDEX_BATCH_SIZE) {+ publish_email_event(+ macro_event_broker,+ &EmailMacroEvent::threads_reindex_requested(ThreadsReindexRequestedMetadata {+ link_id,+ owner: owner.clone(),+ thread_ids: thread_ids.to_vec(),+ reason: ThreadsReindexReason::ContactsChanged,+ }),+ );}}diff --git a/services/email_service/src/util/sync_contacts/test.rs b/services/email_service/src/util/sync_contacts/test.rsnew file mode 100644index 0000000000..ef1f55ca08--- /dev/null+++ b/services/email_service/src/util/sync_contacts/test.rs@@ -0,0 +1,133 @@+use std::sync::Mutex;++use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};+use serde_json::{Value, json};++use super::*;++#[derive(Clone, Debug)]+struct PublishedEvent {+ topic: String,+ key: String,+ envelope: Value,+}++#[derive(Default)]+struct RecordingEventBroker {+ published: Mutex<Vec<PublishedEvent>>,+}++impl MacroEventBroker for RecordingEventBroker {+ fn send_event<E: MacroEvent + ?Sized>(+ &self,+ event: &E,+ ) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {+ self.published+ .lock()+ .expect("published events lock should not be poisoned")+ .push(PublishedEvent {+ topic: event.topic().to_string(),+ key: event.key().to_string(),+ envelope: serde_json::to_value(event.event())?,+ });++ Ok(tokio::spawn(async { Ok(()) }))+ }+}++fn owner() -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid owner")+}++fn thread_ids(count: usize) -> Vec<Uuid> {+ (1..=count)+ .map(|value| Uuid::from_u128(value as u128))+ .collect()+}++fn assert_published_batches(thread_count: usize, expected_batch_sizes: &[usize]) {+ let broker = RecordingEventBroker::default();+ let link_id = Uuid::from_u128(u128::MAX);+ let owner = owner();+ let thread_ids = thread_ids(thread_count);++ publish_thread_reindex_batches(&broker, link_id, &owner, &thread_ids);++ let published = broker+ .published+ .lock()+ .expect("published events lock should not be poisoned");+ let actual_batch_sizes: Vec<usize> = published+ .iter()+ .map(|event| {+ event.envelope["metadata"]["thread_ids"]+ .as_array()+ .expect("thread_ids should be an array")+ .len()+ })+ .collect();+ assert_eq!(actual_batch_sizes, expected_batch_sizes);++ let published_thread_ids: Vec<Uuid> = published+ .iter()+ .flat_map(|event| {+ assert_eq!(event.topic, "macro.email");+ assert_eq!(event.key, link_id.to_string());+ assert_eq!(event.envelope["schema_version"], json!(1));+ assert_eq!(+ event.envelope["event_type"],+ json!("email.threads_reindex_requested")+ );+ assert_eq!(+ event.envelope["metadata"]["link_id"],+ json!(link_id.to_string())+ );+ assert_eq!(+ event.envelope["metadata"]["owner"],+ json!(owner.to_string())+ );+ assert_eq!(+ event.envelope["metadata"]["reason"],+ json!("contacts_changed")+ );++ let batch = event.envelope["metadata"]["thread_ids"]+ .as_array()+ .expect("thread_ids should be an array");+ assert!(!batch.is_empty());+ assert!(batch.len() <= REINDEX_BATCH_SIZE);++ batch.iter().map(|thread_id| {+ Uuid::parse_str(thread_id.as_str().expect("thread id should be a string"))+ .expect("thread id should be a UUID")+ })+ })+ .collect();++ assert_eq!(published_thread_ids, thread_ids);+}++#[tokio::test]+async fn publishes_no_batches_for_zero_threads() {+ assert_published_batches(0, &[]);+}++#[tokio::test]+async fn publishes_one_batch_for_one_thread() {+ assert_published_batches(1, &[1]);+}++#[tokio::test]+async fn publishes_one_full_batch_for_fifty_threads() {+ assert_published_batches(50, &[50]);+}++#[tokio::test]+async fn publishes_a_second_batch_for_fifty_one_threads() {+ assert_published_batches(51, &[50, 1]);+}++#[tokio::test]+async fn publishes_ordered_bounded_batches_for_larger_inputs() {+ assert_published_batches(137, &[50, 50, 37]);+}diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.tomlindex 841d2e9b5b..ca3cf93794 100644--- a/services/search_processing_service/Cargo.toml+++ b/services/search_processing_service/Cargo.toml@@ -12,6 +12,7 @@ processing = ["dep:channels","dep:chat","dep:documents",+ "dep:email","dep:kafka_util","dep:macro_event_broker","dep:macro_event_topics",@@ -51,6 +52,7 @@ channels = { path = "../../crates/channels", default-features = false, optionalchat = { path = "../../crates/chat", default-features = false, optional = true }chrono = { workspace = true }comms_db_client = { path = "../../crates/comms_db_client" }+email = { path = "../../crates/email", default-features = false, optional = true }email_db_client = { path = "../../crates/email_db_client" }futures = { workspace = true }http-body-util = { workspace = true }diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rsindex 289d4e4439..59cc633a27 100644--- a/services/search_processing_service/src/inbound/kafka_consumer.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs@@ -7,8 +7,8 @@//! offsets are already committed.//!//! Per-entity event mapping and processing live in the [`call`], [`channel`],-//! [`chat`], [`document`], [`project`], and [`property`] submodules; this module-//! owns the poll loop, worker, retry policy, and commit semantics.+//! [`chat`], [`document`], [`email`], [`project`], and [`property`] submodules;+//! this module owns the poll loop, worker, retry policy, and commit semantics.#![allow(clippy::enum_variant_names)]@@ -17,6 +17,7 @@ mod channel;mod chat;mod context;mod document;+mod email;mod project;mod property;#[cfg(test)]@@ -26,6 +27,7 @@ use std::{future::Future, time::Duration};use ::call::domain::events::CallMacroEvent;use ::chat::domain::events::ChatMacroEvent;+use ::email::domain::events::EmailMacroEvent;use channels::domain::broker_events::ChannelMacroEvent;use documents::domain::events::DocumentMacroEvent;use kafka_util::{GroupName, KafkaEventConsumer};@@ -42,7 +44,7 @@ use tokio_retry::{Retry, strategy::ExponentialBackoff};use self::{call::process_call_event, channel::process_channel_event, chat::process_chat_event,- document::process_document_event, project::process_project_event,+ document::process_document_event, email::process_email_event, project::process_project_event,property::process_property_event,};@@ -66,6 +68,7 @@ macro_event_broker::declare_topics!(ChannelMacroEvent,ChatMacroEvent,DocumentMacroEvent,+ EmailMacroEvent,ProjectMacroEvent,PropertyMacroEvent,);@@ -190,6 +193,9 @@ async fn process_event(DeclaredMacroEvent::DocumentMacroEvent(event) => {process_document_event(context, event, partition, offset).await}+ DeclaredMacroEvent::EmailMacroEvent(event) => {+ process_email_event(db, opensearch_client, event, partition, offset).await+ }DeclaredMacroEvent::ProjectMacroEvent(event) => {process_project_event(db, opensearch_client, event, partition, offset).await}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/email.rs b/services/search_processing_service/src/inbound/kafka_consumer/email.rsnew file mode 100644index 0000000000..b812cead3f--- /dev/null+++ b/services/search_processing_service/src/inbound/kafka_consumer/email.rs@@ -0,0 +1,286 @@+//! Maps email lifecycle events to search-index actions and processes them.++use ::email::domain::events::{EmailMacroEvent, EmailTopicEvent};+use macro_event_broker::MacroEvent as _;+use opensearch_client::OpensearchClient;+use sqlx::PgPool;+use uuid::Uuid;++use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};+use crate::process::email::{remove, upsert};++#[derive(Debug, Clone, PartialEq, Eq)]+pub(super) enum EmailIndexAction {+ UpsertMessage {+ message_id: Uuid,+ owner: String,+ },+ RemoveMessage {+ message_id: Uuid,+ },+ ReindexThread {+ thread_id: Uuid,+ owner: String,+ },+ ReindexThreads {+ thread_ids: Vec<Uuid>,+ owner: String,+ },+ RemoveLink {+ link_id: Uuid,+ },+ Ignore,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub(super) struct EmailEventDescription {+ pub(super) action: EmailIndexAction,+ pub(super) link_id: Uuid,+ pub(super) event_type: &'static str,+}++pub(super) fn describe_email_event(event: &EmailTopicEvent) -> EmailEventDescription {+ match event {+ EmailTopicEvent::LinkConnected(metadata) => EmailEventDescription {+ action: EmailIndexAction::Ignore,+ link_id: metadata.link_id,+ event_type: "email.link_connected",+ },+ EmailTopicEvent::LinkDisconnected(metadata) => EmailEventDescription {+ action: EmailIndexAction::RemoveLink {+ link_id: metadata.link_id,+ },+ link_id: metadata.link_id,+ event_type: "email.link_disconnected",+ },+ EmailTopicEvent::LinkReauthRequired(metadata) => EmailEventDescription {+ action: EmailIndexAction::Ignore,+ link_id: metadata.link_id,+ event_type: "email.link_reauth_required",+ },+ EmailTopicEvent::MessageReceived(metadata) => EmailEventDescription {+ action: if metadata.is_spam_or_trash {+ EmailIndexAction::RemoveMessage {+ message_id: metadata.message_id,+ }+ } else {+ EmailIndexAction::UpsertMessage {+ message_id: metadata.message_id,+ owner: metadata.owner.to_string(),+ }+ },+ link_id: metadata.link_id,+ event_type: "email.message_received",+ },+ EmailTopicEvent::MessageDraftSynced(metadata) => EmailEventDescription {+ action: if metadata.is_spam_or_trash {+ EmailIndexAction::RemoveMessage {+ message_id: metadata.message_id,+ }+ } else {+ EmailIndexAction::UpsertMessage {+ message_id: metadata.message_id,+ owner: metadata.owner.to_string(),+ }+ },+ link_id: metadata.link_id,+ event_type: "email.message_draft_synced",+ },+ EmailTopicEvent::MessageSent(metadata) => EmailEventDescription {+ action: EmailIndexAction::UpsertMessage {+ message_id: metadata.message_id,+ owner: metadata.owner.to_string(),+ },+ link_id: metadata.link_id,+ event_type: "email.message_sent",+ },+ EmailTopicEvent::MessageDeleted(metadata) => EmailEventDescription {+ action: EmailIndexAction::RemoveMessage {+ message_id: metadata.message_id,+ },+ link_id: metadata.link_id,+ event_type: "email.message_deleted",+ },+ EmailTopicEvent::MessageSendQueued(metadata) => EmailEventDescription {+ action: EmailIndexAction::Ignore,+ link_id: metadata.link_id,+ event_type: "email.message_send_queued",+ },+ EmailTopicEvent::MessageSendCancelled(metadata) => EmailEventDescription {+ action: EmailIndexAction::Ignore,+ link_id: metadata.link_id,+ event_type: "email.message_send_cancelled",+ },+ EmailTopicEvent::ThreadArchived(metadata) => EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: metadata.thread_id,+ owner: metadata.owner.to_string(),+ },+ link_id: metadata.link_id,+ event_type: "email.thread_archived",+ },+ EmailTopicEvent::ThreadTrashed(metadata) => EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: metadata.thread_id,+ owner: metadata.owner.to_string(),+ },+ link_id: metadata.link_id,+ event_type: "email.thread_trashed",+ },+ EmailTopicEvent::ThreadRead(metadata) => EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: metadata.thread_id,+ owner: metadata.owner.to_string(),+ },+ link_id: metadata.link_id,+ event_type: "email.thread_read",+ },+ EmailTopicEvent::ThreadStarred(metadata) => EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: metadata.thread_id,+ owner: metadata.owner.to_string(),+ },+ link_id: metadata.link_id,+ event_type: "email.thread_starred",+ },+ EmailTopicEvent::ThreadSpamChanged(metadata) => EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: metadata.thread_id,+ owner: metadata.owner.to_string(),+ },+ link_id: metadata.link_id,+ event_type: "email.thread_spam_changed",+ },+ EmailTopicEvent::ThreadProjectChanged(metadata) => EmailEventDescription {+ action: EmailIndexAction::Ignore,+ link_id: metadata.link_id,+ event_type: "email.thread_project_changed",+ },+ EmailTopicEvent::ThreadLabelsUpdated(metadata) => EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: metadata.thread_id,+ owner: metadata.owner.to_string(),+ },+ link_id: metadata.link_id,+ event_type: "email.thread_labels_updated",+ },+ EmailTopicEvent::ThreadBackfilled(metadata) => EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: metadata.thread_id,+ owner: metadata.owner.to_string(),+ },+ link_id: metadata.link_id,+ event_type: "email.thread_backfilled",+ },+ EmailTopicEvent::ThreadsReindexRequested(metadata) => EmailEventDescription {+ action: EmailIndexAction::ReindexThreads {+ thread_ids: metadata.thread_ids.clone(),+ owner: metadata.owner.to_string(),+ },+ link_id: metadata.link_id,+ event_type: "email.threads_reindex_requested",+ },+ }+}++async fn process_email_index_action(+ db: &PgPool,+ opensearch_client: &OpensearchClient,+ action: EmailIndexAction,+) -> anyhow::Result<()> {+ match action {+ EmailIndexAction::UpsertMessage { message_id, owner } => {+ upsert::process_upsert_message(opensearch_client, db, message_id, &owner).await+ }+ EmailIndexAction::RemoveMessage { message_id } => {+ remove::process_remove_message(opensearch_client, message_id, None).await+ }+ EmailIndexAction::ReindexThread { thread_id, owner } => {+ upsert::process_upsert_thread_message(opensearch_client, db, thread_id, &owner, None)+ .await+ }+ EmailIndexAction::ReindexThreads { thread_ids, owner } => {+ upsert::process_upsert_thread_batch_message(+ opensearch_client,+ db,+ &thread_ids,+ &owner,+ None,+ )+ .await+ }+ EmailIndexAction::RemoveLink { link_id } => {+ remove::process_remove_messages_by_link_id(opensearch_client, link_id).await+ }+ EmailIndexAction::Ignore => Ok(()),+ }+}++pub(super) async fn process_email_event(+ db: &PgPool,+ opensearch_client: &OpensearchClient,+ event: &EmailMacroEvent,+ partition: i32,+ offset: i64,+) -> EventOutcome {+ let description = describe_email_event(&event.event().event);+ if description.action == EmailIndexAction::Ignore {+ tracing::trace!(+ link_id = %description.link_id,+ event_type = description.event_type,+ partition,+ offset,+ "ignoring email event without a search-index action"+ );+ return EventOutcome::Ignored;+ }++ let result = retry_processing(|attempt| {+ let action = description.action.clone();+ async move {+ tracing::trace!(+ link_id = %description.link_id,+ event_type = description.event_type,+ partition,+ offset,+ attempt,+ "processing email search-index event"+ );+ process_email_index_action(db, opensearch_client, action)+ .await+ .inspect_err(|error| {+ if attempt < MAX_PROCESSING_ATTEMPTS {+ let retry_delay =+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));+ tracing::warn!(+ error = ?error,+ link_id = %description.link_id,+ event_type = description.event_type,+ partition,+ offset,+ attempt,+ delay_secs = retry_delay.as_secs(),+ "email search-index processing failed, retrying"+ );+ }+ })+ }+ })+ .await;++ match result {+ Ok(()) => EventOutcome::Indexed,+ Err(error) => {+ tracing::error!(+ error = ?error,+ link_id = %description.link_id,+ event_type = description.event_type,+ partition,+ offset,+ attempts = MAX_PROCESSING_ATTEMPTS,+ "dropping email event after processing retries were exhausted"+ );+ EventOutcome::Dropped+ }+ }+}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rsindex cf8daa25e0..2809077124 100644--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs@@ -11,6 +11,15 @@ use ::chat::domain::events::{ChatMessageRole, ChatMessageSentMetadata, ChatPermanentlyDeletedMetadata, ChatRestoredMetadata,ChatTopicEvent, ChatUpdatedMetadata,};+use ::email::domain::events::{+ EmailEventOrigin, EmailTopicEvent, LinkConnectedMetadata, LinkDisconnectReason,+ LinkDisconnectedMetadata, LinkReauthRequiredMetadata, MessageDeletedMetadata,+ MessageDraftSyncedMetadata, MessageReceivedMetadata, MessageSendCancelledMetadata,+ MessageSendQueuedMetadata, MessageSentMetadata, SendCancelReason, ThreadArchivedMetadata,+ ThreadBackfilledMetadata, ThreadLabelsUpdatedMetadata, ThreadProjectChangedMetadata,+ ThreadReadMetadata, ThreadSpamChangedMetadata, ThreadStarredMetadata, ThreadTrashedMetadata,+ ThreadsReindexReason, ThreadsReindexRequestedMetadata,+};use channels::domain::{broker_events::{ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,@@ -30,8 +39,8 @@ use documents::domain::events::{};use macro_event_broker::{Event, EventBrokerError, MacroEvent as _, MessageParts};use macro_event_topics::{- MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroDocumentsTopic, MacroProjectsTopic,- MacroPropertiesTopic, Topic as _,+ MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroDocumentsTopic, MacroEmailTopic,+ MacroProjectsTopic, MacroPropertiesTopic, Topic as _,};use macro_user_id::user_id::MacroUserIdStr;use model::document::FileType;@@ -57,6 +66,7 @@ use super::{DocumentEventDescription, DocumentIndexAction, describe_document_event,stored_extractor_message, sync_extractor_message,},+ email::{EmailEventDescription, EmailIndexAction, describe_email_event},project::{ProjectEventDescription, ProjectIndexAction, collect_project_ids, describe_project_event,},@@ -80,6 +90,9 @@ const PROPERTY_ENTITY_ID: &str = "property-entity-id";const PROPERTY_DEFINITION_ID: Uuid = Uuid::from_u128(4);const PROPERTY_OPTION_ID: Uuid = Uuid::from_u128(5);const ENTITY_PROPERTY_ID: Uuid = Uuid::from_u128(6);+const EMAIL_LINK_ID: Uuid = Uuid::from_u128(7);+const EMAIL_THREAD_ID: Uuid = Uuid::from_u128(8);+const SECOND_EMAIL_THREAD_ID: Uuid = Uuid::from_u128(9);struct TestMessage {topic: &'static str,@@ -178,6 +191,334 @@ fn encoded_message<E: serde::Serialize>(}}+fn received_email_event(is_spam_or_trash: bool) -> EmailTopicEvent {+ EmailTopicEvent::MessageReceived(MessageReceivedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ message_id: MESSAGE_ID,+ provider_message_id: "provider-message-id".to_string(),+ thread_id: EMAIL_THREAD_ID,+ provider_thread_id: "provider-thread-id".to_string(),+ is_new_thread: true,+ subject: Some("Subject".to_string()),+ from_email: Some("[email protected]".to_string()),+ from_name: Some("Sender".to_string()),+ to_emails: vec!["[email protected]".to_string()],+ attachment_count: 1,+ is_spam_or_trash,+ received_at: Some(Utc::now()),+ })+}++fn draft_email_event(is_spam_or_trash: bool) -> EmailTopicEvent {+ EmailTopicEvent::MessageDraftSynced(MessageDraftSyncedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ message_id: MESSAGE_ID,+ provider_message_id: "provider-message-id".to_string(),+ thread_id: EMAIL_THREAD_ID,+ provider_thread_id: "provider-thread-id".to_string(),+ is_spam_or_trash,+ })+}++fn email_event_cases() -> Vec<(EmailTopicEvent, EmailEventDescription)> {+ let owner = "macro|[email protected]".to_string();++ vec![+ (+ EmailTopicEvent::LinkConnected(LinkConnectedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ email_address: "[email protected]".to_string(),+ provider: "GMAIL".to_string(),+ is_primary: true,+ connected_at: Utc::now(),+ }),+ EmailEventDescription {+ action: EmailIndexAction::Ignore,+ link_id: EMAIL_LINK_ID,+ event_type: "email.link_connected",+ },+ ),+ (+ EmailTopicEvent::LinkDisconnected(LinkDisconnectedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ email_address: "[email protected]".to_string(),+ reason: LinkDisconnectReason::ManuallyDisabled,+ }),+ EmailEventDescription {+ action: EmailIndexAction::RemoveLink {+ link_id: EMAIL_LINK_ID,+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.link_disconnected",+ },+ ),+ (+ EmailTopicEvent::LinkReauthRequired(LinkReauthRequiredMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ email_address: "[email protected]".to_string(),+ observed_at: Utc::now(),+ }),+ EmailEventDescription {+ action: EmailIndexAction::Ignore,+ link_id: EMAIL_LINK_ID,+ event_type: "email.link_reauth_required",+ },+ ),+ (+ received_email_event(false),+ EmailEventDescription {+ action: EmailIndexAction::UpsertMessage {+ message_id: MESSAGE_ID,+ owner: owner.clone(),+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.message_received",+ },+ ),+ (+ draft_email_event(true),+ EmailEventDescription {+ action: EmailIndexAction::RemoveMessage {+ message_id: MESSAGE_ID,+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.message_draft_synced",+ },+ ),+ (+ EmailTopicEvent::MessageSent(MessageSentMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ actor: Some(user_id()),+ message_id: MESSAGE_ID,+ provider_message_id: "provider-message-id".to_string(),+ thread_id: EMAIL_THREAD_ID,+ provider_thread_id: "provider-thread-id".to_string(),+ subject: Some("Subject".to_string()),+ to_emails: vec!["[email protected]".to_string()],+ cc_emails: vec![],+ origin: EmailEventOrigin::UserAction,+ sent_at: Utc::now(),+ }),+ EmailEventDescription {+ action: EmailIndexAction::UpsertMessage {+ message_id: MESSAGE_ID,+ owner: owner.clone(),+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.message_sent",+ },+ ),+ (+ EmailTopicEvent::MessageDeleted(MessageDeletedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ message_id: MESSAGE_ID,+ provider_message_id: "provider-message-id".to_string(),+ thread_id: EMAIL_THREAD_ID,+ }),+ EmailEventDescription {+ action: EmailIndexAction::RemoveMessage {+ message_id: MESSAGE_ID,+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.message_deleted",+ },+ ),+ (+ EmailTopicEvent::MessageSendQueued(MessageSendQueuedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ actor: Some(user_id()),+ message_id: MESSAGE_ID,+ thread_id: EMAIL_THREAD_ID,+ scheduled_send_at: Utc::now(),+ is_scheduled: false,+ }),+ EmailEventDescription {+ action: EmailIndexAction::Ignore,+ link_id: EMAIL_LINK_ID,+ event_type: "email.message_send_queued",+ },+ ),+ (+ EmailTopicEvent::MessageSendCancelled(MessageSendCancelledMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ actor: Some(user_id()),+ message_id: MESSAGE_ID,+ thread_id: EMAIL_THREAD_ID,+ reason: SendCancelReason::Undo,+ }),+ EmailEventDescription {+ action: EmailIndexAction::Ignore,+ link_id: EMAIL_LINK_ID,+ event_type: "email.message_send_cancelled",+ },+ ),+ (+ EmailTopicEvent::ThreadArchived(ThreadArchivedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ actor: Some(user_id()),+ thread_id: EMAIL_THREAD_ID,+ archived: true,+ origin: EmailEventOrigin::UserAction,+ }),+ EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: EMAIL_THREAD_ID,+ owner: owner.clone(),+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.thread_archived",+ },+ ),+ (+ EmailTopicEvent::ThreadTrashed(ThreadTrashedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ actor: None,+ thread_id: EMAIL_THREAD_ID,+ trashed: true,+ origin: EmailEventOrigin::ProviderSync,+ }),+ EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: EMAIL_THREAD_ID,+ owner: owner.clone(),+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.thread_trashed",+ },+ ),+ (+ EmailTopicEvent::ThreadRead(ThreadReadMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ actor: None,+ thread_id: EMAIL_THREAD_ID,+ is_read: true,+ origin: EmailEventOrigin::ProviderSync,+ }),+ EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: EMAIL_THREAD_ID,+ owner: owner.clone(),+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.thread_read",+ },+ ),+ (+ EmailTopicEvent::ThreadStarred(ThreadStarredMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ actor: None,+ thread_id: EMAIL_THREAD_ID,+ starred: true,+ origin: EmailEventOrigin::ProviderSync,+ }),+ EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: EMAIL_THREAD_ID,+ owner: owner.clone(),+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.thread_starred",+ },+ ),+ (+ EmailTopicEvent::ThreadSpamChanged(ThreadSpamChangedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ actor: None,+ thread_id: EMAIL_THREAD_ID,+ spam: true,+ origin: EmailEventOrigin::ProviderSync,+ }),+ EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: EMAIL_THREAD_ID,+ owner: owner.clone(),+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.thread_spam_changed",+ },+ ),+ (+ EmailTopicEvent::ThreadProjectChanged(ThreadProjectChangedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ actor: user_id(),+ thread_id: EMAIL_THREAD_ID,+ previous_project_id: Some("old-project".to_string()),+ project_id: Some("new-project".to_string()),+ }),+ EmailEventDescription {+ action: EmailIndexAction::Ignore,+ link_id: EMAIL_LINK_ID,+ event_type: "email.thread_project_changed",+ },+ ),+ (+ EmailTopicEvent::ThreadLabelsUpdated(ThreadLabelsUpdatedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ actor: None,+ thread_id: EMAIL_THREAD_ID,+ added: vec![],+ removed: vec![],+ origin: EmailEventOrigin::ProviderSync,+ }),+ EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: EMAIL_THREAD_ID,+ owner: owner.clone(),+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.thread_labels_updated",+ },+ ),+ (+ EmailTopicEvent::ThreadBackfilled(ThreadBackfilledMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ thread_id: EMAIL_THREAD_ID,+ }),+ EmailEventDescription {+ action: EmailIndexAction::ReindexThread {+ thread_id: EMAIL_THREAD_ID,+ owner: owner.clone(),+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.thread_backfilled",+ },+ ),+ (+ EmailTopicEvent::ThreadsReindexRequested(ThreadsReindexRequestedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ thread_ids: vec![EMAIL_THREAD_ID, SECOND_EMAIL_THREAD_ID],+ reason: ThreadsReindexReason::ContactsChanged,+ }),+ EmailEventDescription {+ action: EmailIndexAction::ReindexThreads {+ thread_ids: vec![EMAIL_THREAD_ID, SECOND_EMAIL_THREAD_ID],+ owner,+ },+ link_id: EMAIL_LINK_ID,+ event_type: "email.threads_reindex_requested",+ },+ ),+ ]+}+fn chat_event_cases() -> Vec<(ChatTopicEvent, ChatEventDescription<'static>)> {let owner = user_id();@@ -891,6 +1232,7 @@ fn subscribes_to_declared_search_processing_topics_with_durable_group() {assert!(topics.contains(&MacroChannelsTopic::TOPIC_STR));assert!(topics.contains(&MacroChatsTopic::TOPIC_STR));assert!(topics.contains(&MacroDocumentsTopic::TOPIC_STR));+ assert!(topics.contains(&MacroEmailTopic::TOPIC_STR));assert!(topics.contains(&MacroProjectsTopic::TOPIC_STR));assert!(topics.contains(&MacroPropertiesTopic::TOPIC_STR));}@@ -950,6 +1292,35 @@ fn maps_all_call_lifecycle_events_to_index_actions() {);}+#[test]+fn maps_all_email_lifecycle_events_to_index_actions() {+ let cases = email_event_cases();+ assert_eq!(cases.len(), 18);++ for (event, expected) in cases {+ let serialized = serde_json::to_value(&event).expect("serializable email event");+ assert_eq!(serialized["event_type"], expected.event_type);+ assert_eq!(describe_email_event(&event), expected);+ }+}++#[test]+fn email_message_sync_actions_remove_spam_or_trash_and_upsert_other_messages() {+ assert_eq!(+ describe_email_event(&received_email_event(true)).action,+ EmailIndexAction::RemoveMessage {+ message_id: MESSAGE_ID,+ }+ );+ assert_eq!(+ describe_email_event(&draft_email_event(false)).action,+ EmailIndexAction::UpsertMessage {+ message_id: MESSAGE_ID,+ owner: "macro|[email protected]".to_string(),+ }+ );+}+#[test]fn maps_all_chat_lifecycle_events_to_index_actions() {let cases = chat_event_cases();@@ -1078,6 +1449,23 @@ fn project_id_collection_is_stable_and_drops_missing_or_empty_ids() {);}+#[test]+fn email_envelope_decodes_round_trip() {+ let event = draft_email_event(false);+ let message = encoded_message(+ MacroEmailTopic::TOPIC_STR,+ EMAIL_LINK_ID,+ Event::new(event.clone()),+ );++ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable email event");+ let DeclaredMacroEvent::EmailMacroEvent(decoded_event) = decoded else {+ panic!("expected email event");+ };+ assert_eq!(decoded_event.key(), EMAIL_LINK_ID.to_string());+ assert_eq!(decoded_event.event().event, event);+}+#[test]fn chat_envelope_decodes_round_trip() {let event = ChatTopicEvent::MessageSent(ChatMessageSentMetadata {@@ -1243,6 +1631,61 @@ fn exact_macro_documents_envelopes_decode_into_document_events() {}}+#[tokio::test]+async fn malformed_missing_key_and_unsupported_email_messages_are_commit_safe() {+ let malformed = TestMessage {+ topic: MacroEmailTopic::TOPIC_STR,+ key: Some(EMAIL_LINK_ID.to_string()),+ payload: Some(b"not json".to_vec()),+ };+ let malformed = attach_event_coordinates(DeclaredMacroEvent::decode(&malformed), 7, 60);++ let missing_key = TestMessage {+ topic: MacroEmailTopic::TOPIC_STR,+ key: None,+ payload: encoded_message(+ MacroEmailTopic::TOPIC_STR,+ EMAIL_LINK_ID,+ Event::new(draft_email_event(false)),+ )+ .payload,+ };+ let missing_key = attach_event_coordinates(DeclaredMacroEvent::decode(&missing_key), 7, 61);++ let unsupported = encoded_message(+ MacroEmailTopic::TOPIC_STR,+ EMAIL_LINK_ID,+ Event::with_schema_version(draft_email_event(false), 2),+ );+ let unsupported = attach_event_coordinates(DeclaredMacroEvent::decode(&unsupported), 7, 62);+ let (sender, mut receiver) = mpsc::channel(1);++ assert!(matches!(+ handoff_decoded(&sender, malformed).await,+ HandoffOutcome::MalformedRecord(EventBrokerError::Serialization(_))+ ));+ assert!(matches!(+ handoff_decoded(&sender, missing_key).await,+ HandoffOutcome::MalformedRecord(EventBrokerError::MissingMessageKey)+ ));+ match handoff_decoded(&sender, unsupported).await {+ HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {+ topic,+ expected,+ actual,+ }) => {+ assert_eq!(topic, MacroEmailTopic::TOPIC_STR);+ assert_eq!(expected, 1);+ assert_eq!(actual, 2);+ }+ outcome => panic!("expected malformed email record, got {outcome:?}"),+ }+ assert!(matches!(+ receiver.try_recv(),+ Err(mpsc::error::TryRecvError::Empty)+ ));+}+#[tokio::test]async fn malformed_and_unsupported_chat_messages_are_commit_safe() {let malformed = TestMessage {diff --git a/services/search_processing_service/src/process/email/mod.rs b/services/search_processing_service/src/process/email/mod.rsindex 46831b422b..04324042f5 100644--- a/services/search_processing_service/src/process/email/mod.rs+++ b/services/search_processing_service/src/process/email/mod.rs@@ -1,2 +1,2 @@-pub mod remove;-pub mod upsert;+pub(crate) mod remove;+pub(crate) mod upsert;diff --git a/services/search_processing_service/src/process/email/remove.rs b/services/search_processing_service/src/process/email/remove.rsindex e68424e47f..67ea5513e9 100644--- a/services/search_processing_service/src/process/email/remove.rs+++ b/services/search_processing_service/src/process/email/remove.rs@@ -1,12 +1,13 @@use opensearch_client::OpensearchClient;-use sqs_client::search::email::{EmailLinkMessage, EmailMessage};+use uuid::Uuid;pub async fn process_remove_message(opensearch_client: &OpensearchClient,- remove_message: &EmailMessage,+ message_id: Uuid,+ index_override: Option<&str>,) -> anyhow::Result<()> {opensearch_client- .delete_email_message_by_id(remove_message.message_id.as_str())+ .delete_email_message_by_id(&message_id.to_string(), index_override).await?;Ok(())@@ -14,10 +15,10 @@ pub async fn process_remove_message(pub async fn process_remove_messages_by_link_id(opensearch_client: &OpensearchClient,- remove_link_message: &EmailLinkMessage,+ link_id: Uuid,) -> anyhow::Result<()> {opensearch_client- .delete_email_messages_by_link_id(remove_link_message.link_id.as_str())+ .delete_email_messages_by_link_id(&link_id.to_string()).await?;Ok(())diff --git a/services/search_processing_service/src/process/email/upsert.rs b/services/search_processing_service/src/process/email/upsert.rsindex 97a32fb232..9d50f17339 100644--- a/services/search_processing_service/src/process/email/upsert.rs+++ b/services/search_processing_service/src/process/email/upsert.rs@@ -1,6 +1,6 @@use anyhow::Context;use chrono::Utc;-use models_email::service::label::system_labels;+use models_email::service::{label::system_labels, message::ParsedSearchMessage};use models_properties::EntityType;use opensearch_client::{OpensearchClient, date_format::EpochMillis, upsert::email::UpsertEmailArgs,@@ -9,21 +9,40 @@ use properties::outbound::entity_properties_get_query::{get_entity_properties_for_index, get_entity_properties_for_index_batch,};use sqlx::PgPool;-use sqs_client::search::email::{EmailMessage, EmailThreadBatchMessage, EmailThreadMessage};use uuid::Uuid;use crate::process::properties::to_indexed_properties;+#[cfg(test)]+mod test;++#[derive(Debug, PartialEq, Eq)]+enum ThreadMessageDisposition {+ Delete,+ Upsert,+ SkipMissingContent,+}++fn classify_thread_message(message: &ParsedSearchMessage) -> ThreadMessageDisposition {+ if message.labels.iter().any(|label| {+ label.provider_id == system_labels::SPAM || label.provider_id == system_labels::TRASH+ }) {+ return ThreadMessageDisposition::Delete;+ }++ if message.body_parsed_linkless.is_none() {+ return ThreadMessageDisposition::SkipMissingContent;+ }++ ThreadMessageDisposition::Upsert+}+pub async fn process_upsert_message(opensearch_client: &OpensearchClient,db: &PgPool,- upsert_email_message: &EmailMessage,+ message_id: Uuid,+ owner: &str,) -> anyhow::Result<()> {- let message_id: Uuid = upsert_email_message- .message_id- .parse()- .context("failed to parse message_id as UUID")?;-let message_info =email_db_client::messages::get_parsed_search::get_parsed_search_message_by_id(db,@@ -74,9 +93,9 @@ pub async fn process_upsert_message();let upsert_email_message_args: UpsertEmailArgs = UpsertEmailArgs {- message_id: upsert_email_message.message_id.clone(),+ message_id: message_id.to_string(),link_id: message_info.link_id.to_string(),- user_id: upsert_email_message.macro_user_id.clone(),+ user_id: owner.to_string(),thread_id: message_info.thread_db_id.to_string(),subject: message_info.subject,sender: message_info@@ -141,17 +160,14 @@ pub async fn process_upsert_message(pub async fn process_upsert_thread_message(opensearch_client: &OpensearchClient,db: &PgPool,- upsert_email_thread_message: &EmailThreadMessage,+ thread_id: Uuid,+ owner: &str,+ index_override: Option<&str>,) -> anyhow::Result<()> {let mut message_offset = 0;// Max is 100let message_limit = 100;- let thread_id: Uuid = upsert_email_thread_message- .thread_id- .parse()- .context("failed to parse thread_id as UUID")?;-let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;// A full index overwrites each doc, so thread properties must ride along@@ -178,150 +194,29 @@ pub async fn process_upsert_thread_message(break;}+ let mut delete_message_ids = Vec::new();let mut upsert_email_message_args = Vec::new();for message in messages {- // don't insert spam or trash messages- if message.labels.iter().any(|label| {- label.provider_id == system_labels::SPAM- || label.provider_id == system_labels::TRASH- }) {- continue;- }-- if let Some(content) = message.body_parsed_linkless {- let sent_at_millis = message- .internal_date_ts- .map(|date| EpochMillis::new(date.timestamp_millis()))- .transpose()?;-- let updated_at_millis = message- .internal_date_ts- .map(|date| EpochMillis::new(date.timestamp_millis()))- .transpose()?- .unwrap_or(now_millis);-- upsert_email_message_args.push(UpsertEmailArgs {- message_id: message.db_id.to_string(),- link_id: message.link_id.to_string(),- user_id: upsert_email_thread_message.macro_user_id.clone(),- thread_id: upsert_email_thread_message.thread_id.clone(),- subject: message.subject,- sender: message- .from- .as_ref()- .map(|f| f.email.to_lowercase())- .unwrap_or_default(),- sender_name: message.from.as_ref().and_then(|f| f.name.clone()),- reply_to: message.reply_to.map(|r| r.to_lowercase()),- recipients: message- .to- .iter()- .map(|to| to.email.to_lowercase())- .collect(),- recipient_names: message.to.iter().filter_map(|to| to.name.clone()).collect(),- cc: message- .cc- .iter()- .map(|cc| cc.email.to_lowercase())- .collect(),- cc_names: message.cc.iter().filter_map(|cc| cc.name.clone()).collect(),- bcc: message- .bcc- .iter()- .map(|bcc| bcc.email.to_lowercase())- .collect(),- bcc_names: message- .bcc- .iter()- .filter_map(|bcc| bcc.name.clone())- .collect(),- labels: message- .labels- .iter()- .map(|label| label.name.clone())- .collect(),- content,- updated_at_millis,- sent_at_millis,- properties: properties.clone(),- });- } else {- tracing::warn!("no content found for email message");- }- }-- if !upsert_email_message_args.is_empty() {- let result = opensearch_client- .bulk_upsert_email_messages(- &upsert_email_message_args,- upsert_email_thread_message.index_override.as_deref(),- )- .await?;-- if result.failed > 0 {- tracing::warn!(- failed = result.failed,- errors = ?result.errors,- "some email messages failed to upsert"- );+ match classify_thread_message(&message) {+ ThreadMessageDisposition::Delete => {+ delete_message_ids.push(message.db_id);+ continue;+ }+ ThreadMessageDisposition::SkipMissingContent => {+ tracing::warn!(message_id = %message.db_id, "no content found for email message");+ continue;+ }+ ThreadMessageDisposition::Upsert => {}}- }-- // Update offset- message_offset += message_limit;- }-- Ok(())-}--pub async fn process_upsert_thread_batch_message(- opensearch_client: &OpensearchClient,- db: &PgPool,- batch_message: &EmailThreadBatchMessage,-) -> anyhow::Result<()> {- let thread_ids: Vec<Uuid> = batch_message- .thread_ids- .iter()- .map(|id| id.parse().context("failed to parse thread_id as UUID"))- .collect::<anyhow::Result<Vec<_>>>()?;-- let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;-- let messages =- email_db_client::messages::get_parsed_search::get_parsed_search_messages_by_thread_ids(- db,- &thread_ids,- )- .await- .context("failed to get batch thread messages")?;-- // A full index overwrites each doc, so thread properties must ride along- // or a reindex would drop them.- let thread_id_strings: Vec<String> = thread_ids.iter().map(Uuid::to_string).collect();- let properties_by_thread =- get_entity_properties_for_index_batch(db, &thread_id_strings, EntityType::Thread)- .await- .context("failed to fetch thread properties for search index")?- .into_iter()- .map(|(thread_id, properties)| (thread_id, to_indexed_properties(properties)))- .collect::<std::collections::HashMap<_, _>>();-- let mut upsert_email_message_args = Vec::new();-- for message in messages {- if message.labels.iter().any(|label| {- label.provider_id == system_labels::SPAM || label.provider_id == system_labels::TRASH- }) {- continue;- }- if let Some(content) = message.body_parsed_linkless {+ let content = message+ .body_parsed_linkless+ .context("expected content for upsertable email message")?;let sent_at_millis = message.internal_date_ts.map(|date| EpochMillis::new(date.timestamp_millis())).transpose()?;-let updated_at_millis = message.internal_date_ts.map(|date| EpochMillis::new(date.timestamp_millis()))@@ -331,8 +226,8 @@ pub async fn process_upsert_thread_batch_message(upsert_email_message_args.push(UpsertEmailArgs {message_id: message.db_id.to_string(),link_id: message.link_id.to_string(),- user_id: batch_message.macro_user_id.clone(),- thread_id: message.thread_db_id.to_string(),+ user_id: owner.to_string(),+ thread_id: thread_id.to_string(),subject: message.subject,sender: message.from@@ -371,22 +266,152 @@ pub async fn process_upsert_thread_batch_message(content,updated_at_millis,sent_at_millis,- properties: properties_by_thread- .get(&message.thread_db_id.to_string())- .cloned()- .unwrap_or_default(),+ properties: properties.clone(),});- } else {- tracing::warn!("no content found for email message");}++ for message_id in delete_message_ids {+ opensearch_client+ .delete_email_message_by_id(&message_id.to_string(), index_override)+ .await?;+ }++ if !upsert_email_message_args.is_empty() {+ let result = opensearch_client+ .bulk_upsert_email_messages(&upsert_email_message_args, index_override)+ .await?;++ if result.failed > 0 {+ tracing::warn!(+ failed = result.failed,+ errors = ?result.errors,+ "some email messages failed to upsert"+ );+ }+ }++ // Update offset+ message_offset += message_limit;+ }++ Ok(())+}++pub async fn process_upsert_thread_batch_message(+ opensearch_client: &OpensearchClient,+ db: &PgPool,+ thread_ids: &[Uuid],+ owner: &str,+ index_override: Option<&str>,+) -> anyhow::Result<()> {+ let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;++ let messages =+ email_db_client::messages::get_parsed_search::get_parsed_search_messages_by_thread_ids(+ db, thread_ids,+ )+ .await+ .context("failed to get batch thread messages")?;++ // A full index overwrites each doc, so thread properties must ride along+ // or a reindex would drop them.+ let thread_id_strings: Vec<String> = thread_ids.iter().map(Uuid::to_string).collect();+ let properties_by_thread =+ get_entity_properties_for_index_batch(db, &thread_id_strings, EntityType::Thread)+ .await+ .context("failed to fetch thread properties for search index")?+ .into_iter()+ .map(|(thread_id, properties)| (thread_id, to_indexed_properties(properties)))+ .collect::<std::collections::HashMap<_, _>>();++ let mut delete_message_ids = Vec::new();+ let mut upsert_email_message_args = Vec::new();++ for message in messages {+ match classify_thread_message(&message) {+ ThreadMessageDisposition::Delete => {+ delete_message_ids.push(message.db_id);+ continue;+ }+ ThreadMessageDisposition::SkipMissingContent => {+ tracing::warn!(message_id = %message.db_id, "no content found for email message");+ continue;+ }+ ThreadMessageDisposition::Upsert => {}+ }++ let content = message+ .body_parsed_linkless+ .context("expected content for upsertable email message")?;+ let sent_at_millis = message+ .internal_date_ts+ .map(|date| EpochMillis::new(date.timestamp_millis()))+ .transpose()?;+ let updated_at_millis = message+ .internal_date_ts+ .map(|date| EpochMillis::new(date.timestamp_millis()))+ .transpose()?+ .unwrap_or(now_millis);++ upsert_email_message_args.push(UpsertEmailArgs {+ message_id: message.db_id.to_string(),+ link_id: message.link_id.to_string(),+ user_id: owner.to_string(),+ thread_id: message.thread_db_id.to_string(),+ subject: message.subject,+ sender: message+ .from+ .as_ref()+ .map(|f| f.email.to_lowercase())+ .unwrap_or_default(),+ sender_name: message.from.as_ref().and_then(|f| f.name.clone()),+ reply_to: message.reply_to.map(|r| r.to_lowercase()),+ recipients: message+ .to+ .iter()+ .map(|to| to.email.to_lowercase())+ .collect(),+ recipient_names: message.to.iter().filter_map(|to| to.name.clone()).collect(),+ cc: message+ .cc+ .iter()+ .map(|cc| cc.email.to_lowercase())+ .collect(),+ cc_names: message.cc.iter().filter_map(|cc| cc.name.clone()).collect(),+ bcc: message+ .bcc+ .iter()+ .map(|bcc| bcc.email.to_lowercase())+ .collect(),+ bcc_names: message+ .bcc+ .iter()+ .filter_map(|bcc| bcc.name.clone())+ .collect(),+ labels: message+ .labels+ .iter()+ .map(|label| label.name.clone())+ .collect(),+ content,+ updated_at_millis,+ sent_at_millis,+ properties: properties_by_thread+ .get(&message.thread_db_id.to_string())+ .cloned()+ .unwrap_or_default(),+ });+ }++ for message_id in delete_message_ids {+ opensearch_client+ .delete_email_message_by_id(&message_id.to_string(), index_override)+ .await?;}if !upsert_email_message_args.is_empty() {let result = opensearch_client- .bulk_upsert_email_messages(- &upsert_email_message_args,- batch_message.index_override.as_deref(),- )+ .bulk_upsert_email_messages(&upsert_email_message_args, index_override).await?;if result.failed > 0 {diff --git a/services/search_processing_service/src/process/email/upsert/test.rs b/services/search_processing_service/src/process/email/upsert/test.rsnew file mode 100644index 0000000000..41e7aa8d83--- /dev/null+++ b/services/search_processing_service/src/process/email/upsert/test.rs@@ -0,0 +1,49 @@+use models_email::service::{label::LabelInfo, message::ParsedSearchMessage};++use super::*;++fn message(label: Option<&str>, content: Option<&str>) -> ParsedSearchMessage {+ ParsedSearchMessage {+ db_id: Uuid::new_v4(),+ link_id: Uuid::new_v4(),+ thread_db_id: Uuid::new_v4(),+ subject: None,+ from: None,+ reply_to: None,+ to: Vec::new(),+ cc: Vec::new(),+ bcc: Vec::new(),+ labels: label+ .map(|provider_id| {+ vec![LabelInfo {+ provider_id: provider_id.to_string(),+ name: provider_id.to_string(),+ }]+ })+ .unwrap_or_default(),+ body_parsed_linkless: content.map(str::to_string),+ internal_date_ts: None,+ }+}++#[test]+fn classifies_mixed_thread_messages() {+ let messages = [+ message(Some(system_labels::SPAM), Some("spam")),+ message(Some(system_labels::TRASH), None),+ message(Some(system_labels::INBOX), None),+ message(Some(system_labels::INBOX), Some("ordinary")),+ ];++ let dispositions: Vec<_> = messages.iter().map(classify_thread_message).collect();++ assert_eq!(+ dispositions,+ vec![+ ThreadMessageDisposition::Delete,+ ThreadMessageDisposition::Delete,+ ThreadMessageDisposition::SkipMissingContent,+ ThreadMessageDisposition::Upsert,+ ]+ );+}diff --git a/services/search_processing_service/src/process/mod.rs b/services/search_processing_service/src/process/mod.rsindex 9e62159a80..f996a3124b 100644--- a/services/search_processing_service/src/process/mod.rs+++ b/services/search_processing_service/src/process/mod.rs@@ -3,7 +3,7 @@ pub(crate) mod channel;pub(crate) mod chat;pub mod context;pub(crate) mod document;-mod email;+pub(crate) mod email;pub(crate) mod project;pub(crate) mod properties;mod user;@@ -57,29 +57,25 @@ pub async fn process_message().await?;}- SearchQueueMessage::RemoveEmailLink(message) => {- email::remove::process_remove_messages_by_link_id(&ctx.opensearch_client, &message)- .await?;- }- SearchQueueMessage::ExtractEmailThreadMessage(message) => {- email::upsert::process_upsert_thread_message(&ctx.opensearch_client, &ctx.db, &message)- .await?;- }SearchQueueMessage::ExtractEmailThreadBatch(message) => {+ let thread_ids = message+ .thread_ids+ .iter()+ .map(|thread_id| {+ thread_id+ .parse::<Uuid>()+ .context("failed to parse thread_id as UUID")+ })+ .collect::<anyhow::Result<Vec<_>>>()?;email::upsert::process_upsert_thread_batch_message(&ctx.opensearch_client,&ctx.db,- &message,+ &thread_ids,+ &message.macro_user_id,+ message.index_override.as_deref(),).await?;}- SearchQueueMessage::RemoveEmailMessage(message) => {- email::remove::process_remove_message(&ctx.opensearch_client, &message).await?;- }- SearchQueueMessage::ExtractEmailMessage(message) => {- email::upsert::process_upsert_message(&ctx.opensearch_client, &ctx.db, &message)- .await?;- }SearchQueueMessage::ExtractDocumentText(message) => {document::process_extract_text_message(&ctx.opensearch_client,