Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(search_processing_service): email kafka (#5340)

Will Hutchinson8394d21d6938
diff --git a/Cargo.lock b/Cargo.lock
index 96e0baf1b1..927d3ab3b3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -12975,6 +12975,7 @@ dependencies = [
"document_sub_type",
"documents",
"doppler-config",
+ "email",
"email_db_client",
"ensure_exists",
"futures",
diff --git a/crates/email/src/domain/events.rs b/crates/email/src/domain/events.rs
index 6397d0209e..bb491bd366 100644
--- a/crates/email/src/domain/events.rs
+++ b/crates/email/src/domain/events.rs
@@ -8,9 +8,10 @@
//! account lifecycle events are ordered against the inbox's content events,
//! and per-thread order is preserved (a thread belongs to exactly one link).
//!
-//! Payloads carry RFC-822 *header* level data only (subject, from, to/cc
-//! addresses) — never message bodies, snippets, bcc addresses, or attachment
-//! names. Consumers needing content must read back through the API.
+//! When payloads carry message data, it is limited to RFC-822 *header* level
+//! data (subject, from, to/cc addresses) — never message bodies, snippets, bcc
+//! addresses, or attachment names. Consumers needing content, including
+//! consumers of reindex notifications, must read back through the API.
//!
//! Each variant documents its owning emission site: a logical change is
//! published from the point closest to the committed Macro-DB write.
@@ -84,6 +85,14 @@ pub enum SendCancelReason {
ThreadTrashed,
}
+/// Why a set of threads was asked to be reindexed for search.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ThreadsReindexReason {
+ /// Contact display names changed; sender/recipient names in the index are stale.
+ ContactsChanged,
+}
+
/// Metadata for [`EmailTopicEvent::LinkConnected`].
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LinkConnectedMetadata {
@@ -160,6 +169,25 @@ pub struct MessageReceivedMetadata {
pub received_at: Option<DateTime<Utc>>,
}
+/// Metadata for [`EmailTopicEvent::MessageDraftSynced`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct MessageDraftSyncedMetadata {
+ /// The link (inbox) containing the draft.
+ pub link_id: Uuid,
+ /// The Macro user who owns the link.
+ pub owner: MacroUserIdStr<'static>,
+ /// Macro's message row id.
+ pub message_id: Uuid,
+ /// Provider (Gmail) message id.
+ pub provider_message_id: String,
+ /// Macro's thread row id.
+ pub thread_id: Uuid,
+ /// Provider (Gmail) thread id.
+ pub provider_thread_id: String,
+ /// Whether the draft is currently spam or trash.
+ pub is_spam_or_trash: bool,
+}
+
/// Metadata for [`EmailTopicEvent::MessageSent`].
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct MessageSentMetadata {
@@ -315,6 +343,24 @@ pub struct ThreadStarredMetadata {
pub origin: EmailEventOrigin,
}
+/// Metadata for [`EmailTopicEvent::ThreadSpamChanged`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ThreadSpamChangedMetadata {
+ /// The link (inbox) containing the thread.
+ pub link_id: Uuid,
+ /// The Macro user who owns the link.
+ pub owner: MacroUserIdStr<'static>,
+ /// The authenticated user who changed the state; `None` for
+ /// provider-initiated changes.
+ pub actor: Option<MacroUserIdStr<'static>>,
+ /// Macro's thread row id.
+ pub thread_id: Uuid,
+ /// New state: `true` = marked spam, `false` = unmarked.
+ pub spam: bool,
+ /// Where the change originated.
+ pub origin: EmailEventOrigin,
+}
+
/// Metadata for [`EmailTopicEvent::ThreadProjectChanged`].
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ThreadProjectChangedMetadata {
@@ -335,10 +381,10 @@ pub struct ThreadProjectChangedMetadata {
/// Metadata for [`EmailTopicEvent::ThreadLabelsUpdated`].
///
/// Carries **user label** changes only; system-label changes (INBOX, TRASH,
-/// UNREAD, STARRED) are published as the semantic
+/// UNREAD, STARRED, SPAM) are published as the semantic
/// [`EmailTopicEvent::ThreadArchived`] / [`EmailTopicEvent::ThreadTrashed`] /
-/// [`EmailTopicEvent::ThreadRead`] / [`EmailTopicEvent::ThreadStarred`]
-/// events instead.
+/// [`EmailTopicEvent::ThreadRead`] / [`EmailTopicEvent::ThreadStarred`] /
+/// [`EmailTopicEvent::ThreadSpamChanged`] events instead.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ThreadLabelsUpdatedMetadata {
/// The link (inbox) containing the thread.
@@ -358,6 +404,30 @@ pub struct ThreadLabelsUpdatedMetadata {
pub origin: EmailEventOrigin,
}
+/// Metadata for [`EmailTopicEvent::ThreadBackfilled`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ThreadBackfilledMetadata {
+ /// The link (inbox) containing the thread.
+ pub link_id: Uuid,
+ /// The Macro user who owns the link.
+ pub owner: MacroUserIdStr<'static>,
+ /// Macro's thread row id.
+ pub thread_id: Uuid,
+}
+
+/// Metadata for [`EmailTopicEvent::ThreadsReindexRequested`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ThreadsReindexRequestedMetadata {
+ /// The link (inbox) containing the threads.
+ pub link_id: Uuid,
+ /// The Macro user who owns the link.
+ pub owner: MacroUserIdStr<'static>,
+ /// Thread row ids to reindex. Producers must chunk to at most 50 ids.
+ pub thread_ids: Vec<Uuid>,
+ /// Why the reindex was requested.
+ pub reason: ThreadsReindexReason,
+}
+
/// Events that can be published to [`MacroEmailTopic`].
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "event_type", content = "metadata")]
@@ -381,6 +451,10 @@ pub enum EmailTopicEvent {
/// siblings are persisted silently (they were not just received).
#[serde(rename = "email.message_received")]
MessageReceived(MessageReceivedMetadata),
+ /// A draft was persisted or updated while syncing from the provider.
+ /// Emitted on every synced draft update because draft bodies are mutable.
+ #[serde(rename = "email.message_draft_synced")]
+ MessageDraftSynced(MessageDraftSyncedMetadata),
/// A message was accepted by the provider. Emitted from the scheduled
/// send worker after the provider accepts (`origin = user_action`), or
/// from the inbox-sync worker when a message sent from another client is
@@ -421,6 +495,11 @@ pub enum EmailTopicEvent {
/// inbox-sync STARRED-label diff (`origin = provider_sync`).
#[serde(rename = "email.thread_starred")]
ThreadStarred(ThreadStarredMetadata),
+ /// A thread was marked or unmarked spam. Emitted from the inbox-sync
+ /// SPAM-label diff (`origin = provider_sync`) or the domain label path
+ /// (`origin = user_action`).
+ #[serde(rename = "email.thread_spam_changed")]
+ ThreadSpamChanged(ThreadSpamChangedMetadata),
/// A thread was assigned to or removed from a project. Emitted from the
/// email domain service's thread-project update.
#[serde(rename = "email.thread_project_changed")]
@@ -431,6 +510,15 @@ pub enum EmailTopicEvent {
/// semantic thread events instead.
#[serde(rename = "email.thread_labels_updated")]
ThreadLabelsUpdated(ThreadLabelsUpdatedMetadata),
+ /// All messages of a backfilled thread are persisted and thread metadata
+ /// is final. Emitted from the backfill worker's update-metadata step.
+ #[serde(rename = "email.thread_backfilled")]
+ ThreadBackfilled(ThreadBackfilledMetadata),
+ /// A bounded batch of threads should be re-derived by read-back consumers
+ /// (for example, the search index). Emitted from contact sync when contact
+ /// names change. Producers must include at most 50 thread ids.
+ #[serde(rename = "email.threads_reindex_requested")]
+ ThreadsReindexRequested(ThreadsReindexRequestedMetadata),
}
impl TopicEvent for EmailTopicEvent {
@@ -478,6 +566,14 @@ impl EmailMacroEvent {
)
}
+ /// Build a message-draft-synced event keyed by the link id.
+ pub fn message_draft_synced(metadata: MessageDraftSyncedMetadata) -> Self {
+ Self::new(
+ metadata.link_id.to_string(),
+ EmailTopicEvent::MessageDraftSynced(metadata),
+ )
+ }
+
/// Build a message-sent event keyed by the link id.
pub fn message_sent(metadata: MessageSentMetadata) -> Self {
Self::new(
@@ -542,6 +638,14 @@ impl EmailMacroEvent {
)
}
+ /// Build a thread-spam-changed event keyed by the link id.
+ pub fn thread_spam_changed(metadata: ThreadSpamChangedMetadata) -> Self {
+ Self::new(
+ metadata.link_id.to_string(),
+ EmailTopicEvent::ThreadSpamChanged(metadata),
+ )
+ }
+
/// Build a thread-project-changed event keyed by the link id.
pub fn thread_project_changed(metadata: ThreadProjectChangedMetadata) -> Self {
Self::new(
@@ -558,10 +662,27 @@ impl EmailMacroEvent {
)
}
+ /// Build a thread-backfilled event keyed by the link id.
+ pub fn thread_backfilled(metadata: ThreadBackfilledMetadata) -> Self {
+ Self::new(
+ metadata.link_id.to_string(),
+ EmailTopicEvent::ThreadBackfilled(metadata),
+ )
+ }
+
+ /// Build a threads-reindex-requested event keyed by the link id.
+ pub fn threads_reindex_requested(metadata: ThreadsReindexRequestedMetadata) -> Self {
+ Self::new(
+ metadata.link_id.to_string(),
+ EmailTopicEvent::ThreadsReindexRequested(metadata),
+ )
+ }
+
/// Map a thread-scoped label change onto the semantic event for that
- /// label: INBOX/TRASH/UNREAD/STARRED become the dedicated thread-state
- /// events, SPAM/IMPORTANT/SENT/DRAFT publish nothing (`None`), and any
- /// other (user) label becomes a `thread_labels_updated` diff.
+ /// label: INBOX/TRASH/UNREAD/STARRED/SPAM become dedicated thread-state
+ /// events, IMPORTANT/SENT/DRAFT and category labels publish nothing
+ /// (`None`), and any other (user) label becomes a
+ /// `thread_labels_updated` diff.
pub fn thread_label_change(
link_id: Uuid,
owner: MacroUserIdStr<'static>,
@@ -606,10 +727,15 @@ impl EmailMacroEvent {
archived: !added,
origin,
}),
- system_labels::SPAM
- | system_labels::IMPORTANT
- | system_labels::SENT
- | system_labels::DRAFT => return None,
+ system_labels::SPAM => Self::thread_spam_changed(ThreadSpamChangedMetadata {
+ link_id,
+ owner,
+ actor,
+ thread_id,
+ spam: added,
+ origin,
+ }),
+ system_labels::IMPORTANT | system_labels::SENT | system_labels::DRAFT => return None,
// Gmail category tabs (CATEGORY_PERSONAL/SOCIAL/...) are
// provider system labels, not user labels.
l if l.starts_with("CATEGORY_") => return None,
diff --git a/crates/email/src/domain/events/test.rs b/crates/email/src/domain/events/test.rs
index 0ba97ce0ec..aed9ae312f 100644
--- a/crates/email/src/domain/events/test.rs
+++ b/crates/email/src/domain/events/test.rs
@@ -14,6 +14,41 @@ fn timestamp() -> DateTime<Utc> {
.with_timezone(&Utc)
}
+fn new_contract_events(link_id: Uuid) -> Vec<EmailMacroEvent> {
+ let owner = user_id("macro|[email protected]");
+
+ vec![
+ EmailMacroEvent::message_draft_synced(MessageDraftSyncedMetadata {
+ link_id,
+ owner: owner.clone(),
+ message_id: Uuid::new_v4(),
+ provider_message_id: "gmail-draft-1".to_string(),
+ thread_id: Uuid::new_v4(),
+ provider_thread_id: "gmail-thread-1".to_string(),
+ is_spam_or_trash: false,
+ }),
+ EmailMacroEvent::thread_backfilled(ThreadBackfilledMetadata {
+ link_id,
+ owner: owner.clone(),
+ thread_id: Uuid::new_v4(),
+ }),
+ EmailMacroEvent::threads_reindex_requested(ThreadsReindexRequestedMetadata {
+ link_id,
+ owner: owner.clone(),
+ thread_ids: vec![Uuid::new_v4(), Uuid::new_v4()],
+ reason: ThreadsReindexReason::ContactsChanged,
+ }),
+ EmailMacroEvent::thread_spam_changed(ThreadSpamChangedMetadata {
+ link_id,
+ owner,
+ actor: None,
+ thread_id: Uuid::new_v4(),
+ spam: true,
+ origin: EmailEventOrigin::ProviderSync,
+ }),
+ ]
+}
+
#[test]
fn message_received_wire_shape() {
let event = Event::with_event_id(
@@ -63,6 +98,43 @@ fn message_received_wire_shape() {
);
}
+#[test]
+fn message_draft_synced_wire_shape_preserves_spam_or_trash_state() {
+ for is_spam_or_trash in [false, true] {
+ let event = Event::with_event_id(
+ Uuid::nil(),
+ EmailTopicEvent::MessageDraftSynced(MessageDraftSyncedMetadata {
+ link_id: Uuid::nil(),
+ owner: user_id("macro|[email protected]"),
+ message_id: Uuid::nil(),
+ provider_message_id: "gmail-draft-1".to_string(),
+ thread_id: Uuid::nil(),
+ provider_thread_id: "gmail-thread-1".to_string(),
+ is_spam_or_trash,
+ }),
+ );
+
+ let value = serde_json::to_value(&event).expect("serializable");
+ assert_eq!(
+ value,
+ json!({
+ "event_id": "00000000-0000-0000-0000-000000000000",
+ "schema_version": 1,
+ "event_type": "email.message_draft_synced",
+ "metadata": {
+ "link_id": "00000000-0000-0000-0000-000000000000",
+ "owner": "macro|[email protected]",
+ "message_id": "00000000-0000-0000-0000-000000000000",
+ "provider_message_id": "gmail-draft-1",
+ "thread_id": "00000000-0000-0000-0000-000000000000",
+ "provider_thread_id": "gmail-thread-1",
+ "is_spam_or_trash": is_spam_or_trash,
+ },
+ })
+ );
+ }
+}
+
#[test]
fn thread_archived_wire_shape_with_origin() {
let event = Event::with_event_id(
@@ -114,6 +186,19 @@ fn decode_round_trips() {
assert_eq!(decoded.event(), original.event());
}
+#[test]
+fn new_contract_events_decode_round_trip() {
+ let link_id = Uuid::new_v4();
+
+ for original in new_contract_events(link_id) {
+ let payload = serde_json::to_vec(original.event()).expect("serializable");
+ let decoded = EmailMacroEvent::decode(original.key(), &payload).expect("decodable payload");
+
+ assert_eq!(decoded.key(), link_id.to_string());
+ assert_eq!(decoded.event(), original.event());
+ }
+}
+
#[test]
fn events_are_keyed_by_link_id() {
let link_id = Uuid::new_v4();
@@ -126,6 +211,23 @@ fn events_are_keyed_by_link_id() {
assert_eq!(event.key(), link_id.to_string());
}
+#[test]
+fn new_contract_event_constructors_are_keyed_by_link_id() {
+ let link_id = Uuid::new_v4();
+
+ for event in new_contract_events(link_id) {
+ assert_eq!(event.key(), link_id.to_string());
+ }
+}
+
+#[test]
+fn threads_reindex_reason_uses_snake_case() {
+ let value = serde_json::to_value(ThreadsReindexReason::ContactsChanged)
+ .expect("reason is serializable");
+
+ assert_eq!(value, json!("contacts_changed"));
+}
+
#[test]
fn event_type_strings_follow_dot_convention() {
let link_id = Uuid::nil();
@@ -181,6 +283,18 @@ fn event_type_strings_follow_dot_convention() {
}),
"email.message_received",
),
+ (
+ EmailMacroEvent::message_draft_synced(MessageDraftSyncedMetadata {
+ link_id,
+ owner: owner.clone(),
+ message_id: Uuid::nil(),
+ provider_message_id: "m".to_string(),
+ thread_id: Uuid::nil(),
+ provider_thread_id: "t".to_string(),
+ is_spam_or_trash: false,
+ }),
+ "email.message_draft_synced",
+ ),
(
EmailMacroEvent::message_sent(MessageSentMetadata {
link_id,
@@ -275,6 +389,17 @@ fn event_type_strings_follow_dot_convention() {
}),
"email.thread_starred",
),
+ (
+ EmailMacroEvent::thread_spam_changed(ThreadSpamChangedMetadata {
+ link_id,
+ owner: owner.clone(),
+ actor: actor.clone(),
+ thread_id: Uuid::nil(),
+ spam: true,
+ origin: EmailEventOrigin::UserAction,
+ }),
+ "email.thread_spam_changed",
+ ),
(
EmailMacroEvent::thread_project_changed(ThreadProjectChangedMetadata {
link_id,
@@ -302,6 +427,23 @@ fn event_type_strings_follow_dot_convention() {
}),
"email.thread_labels_updated",
),
+ (
+ EmailMacroEvent::thread_backfilled(ThreadBackfilledMetadata {
+ link_id,
+ owner: owner.clone(),
+ thread_id: Uuid::nil(),
+ }),
+ "email.thread_backfilled",
+ ),
+ (
+ EmailMacroEvent::threads_reindex_requested(ThreadsReindexRequestedMetadata {
+ link_id,
+ owner,
+ thread_ids: vec![Uuid::nil()],
+ reason: ThreadsReindexReason::ContactsChanged,
+ }),
+ "email.threads_reindex_requested",
+ ),
];
for (event, expected_type) in cases {
@@ -349,6 +491,8 @@ fn thread_label_change_maps_system_labels_to_semantic_events() {
("TRASH", false, "email.thread_trashed", "trashed", false),
("INBOX", true, "email.thread_archived", "archived", false),
("INBOX", false, "email.thread_archived", "archived", true),
+ ("SPAM", true, "email.thread_spam_changed", "spam", true),
+ ("SPAM", false, "email.thread_spam_changed", "spam", false),
];
for (label, added, expected_type, state_field, expected_state) in cases {
@@ -365,7 +509,7 @@ fn thread_label_change_maps_system_labels_to_semantic_events() {
#[test]
fn thread_label_change_skips_unpublished_system_labels() {
- for label in ["SPAM", "IMPORTANT", "SENT", "DRAFT"] {
+ for label in ["IMPORTANT", "SENT", "DRAFT"] {
assert!(
label_change(system_label(label), true).is_none(),
"{label} add should not publish"
diff --git a/crates/opensearch_client/src/delete/email.rs b/crates/opensearch_client/src/delete/email.rs
index 0f97c57728..d697a31c88 100644
--- a/crates/opensearch_client/src/delete/email.rs
+++ b/crates/opensearch_client/src/delete/email.rs
@@ -2,6 +2,13 @@ use models_opensearch::SearchIndex;
use crate::{Result, error::OpensearchClientError};
+#[cfg(test)]
+mod test;
+
+fn resolve_destination(index_override: Option<&str>) -> &str {
+ index_override.unwrap_or(SearchIndex::Emails.as_ref())
+}
+
/// Deletes all email messages with the specified link_id
#[tracing::instrument(skip(client))]
pub async fn delete_email_by_link_id(client: &opensearch::OpenSearch, link_id: &str) -> Result<()> {
@@ -58,6 +65,7 @@ pub async fn delete_email_by_link_id(client: &opensearch::OpenSearch, link_id: &
pub async fn delete_email_message_by_id(
client: &opensearch::OpenSearch,
message_id: &str,
+ index_override: Option<&str>,
) -> Result<()> {
let query = serde_json::json!({
"query": {
@@ -67,10 +75,9 @@ pub async fn delete_email_message_by_id(
},
});
+ let index = resolve_destination(index_override);
let response = client
- .delete_by_query(opensearch::DeleteByQueryParts::Index(&[
- SearchIndex::Emails.as_ref(),
- ]))
+ .delete_by_query(opensearch::DeleteByQueryParts::Index(&[index]))
.body(query)
.refresh(true) // Ensure the index reflects changes immediately
.send()
diff --git a/crates/opensearch_client/src/delete/email/test.rs b/crates/opensearch_client/src/delete/email/test.rs
new file mode 100644
index 0000000000..09ef960726
--- /dev/null
+++ b/crates/opensearch_client/src/delete/email/test.rs
@@ -0,0 +1,17 @@
+use super::*;
+
+#[test]
+fn resolves_default_email_destination() {
+ assert_eq!(
+ resolve_destination(None),
+ models_opensearch::SearchIndex::Emails.as_ref()
+ );
+}
+
+#[test]
+fn resolves_overridden_email_destination() {
+ assert_eq!(
+ resolve_destination(Some("emails_backfill")),
+ "emails_backfill"
+ );
+}
diff --git a/crates/opensearch_client/src/email.rs b/crates/opensearch_client/src/email.rs
index d913d188d7..8f17ee22de 100644
--- a/crates/opensearch_client/src/email.rs
+++ b/crates/opensearch_client/src/email.rs
@@ -32,8 +32,12 @@ impl OpensearchClient {
}
/// Deletes a particular email message
- pub async fn delete_email_message_by_id(&self, message_id: &str) -> Result<()> {
- delete::email::delete_email_message_by_id(&self.inner, message_id).await
+ pub async fn delete_email_message_by_id(
+ &self,
+ message_id: &str,
+ index_override: Option<&str>,
+ ) -> Result<()> {
+ delete::email::delete_email_message_by_id(&self.inner, message_id, index_override).await
}
/// Deletes all email messages with the specified link_id
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rs
index 08fab0ca4b..db835ed05e 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs
@@ -225,33 +225,57 @@ fn patches_from_chat_event(event: &ChatTopicEvent) -> Vec<SoupRealtimePatch> {
}
fn patches_from_email_event(event: &EmailTopicEvent) -> Vec<SoupRealtimePatch> {
- if let EmailTopicEvent::ThreadTrashed(metadata) = event
- && metadata.trashed
- {
- return vec![delete(EntityType::EmailThread, metadata.thread_id)];
- }
+ let updated_thread = |thread_id| vec![update(EntityType::EmailThread, thread_id)];
+ let deleted_thread = |thread_id| vec![delete(EntityType::EmailThread, thread_id)];
- let thread_id = match event {
- EmailTopicEvent::MessageReceived(metadata) if !metadata.is_spam_or_trash => {
- metadata.thread_id
- }
- EmailTopicEvent::MessageSent(metadata) => metadata.thread_id,
- EmailTopicEvent::MessageDeleted(metadata) => metadata.thread_id,
- EmailTopicEvent::MessageSendQueued(metadata) => metadata.thread_id,
- EmailTopicEvent::MessageSendCancelled(metadata) => metadata.thread_id,
- EmailTopicEvent::ThreadArchived(metadata) => metadata.thread_id,
- EmailTopicEvent::ThreadTrashed(metadata) if !metadata.trashed => metadata.thread_id,
- EmailTopicEvent::ThreadRead(metadata) => metadata.thread_id,
- EmailTopicEvent::ThreadStarred(metadata) => metadata.thread_id,
- EmailTopicEvent::ThreadProjectChanged(metadata) => metadata.thread_id,
- EmailTopicEvent::ThreadLabelsUpdated(metadata) => metadata.thread_id,
+ match event {
+ EmailTopicEvent::MessageReceived(metadata) => {
+ if metadata.is_spam_or_trash {
+ Vec::new()
+ } else {
+ updated_thread(metadata.thread_id)
+ }
+ }
+ EmailTopicEvent::MessageDraftSynced(metadata) => {
+ if metadata.is_spam_or_trash {
+ Vec::new()
+ } else {
+ updated_thread(metadata.thread_id)
+ }
+ }
+ EmailTopicEvent::MessageSent(metadata) => updated_thread(metadata.thread_id),
+ EmailTopicEvent::MessageDeleted(metadata) => updated_thread(metadata.thread_id),
+ EmailTopicEvent::MessageSendQueued(metadata) => updated_thread(metadata.thread_id),
+ EmailTopicEvent::MessageSendCancelled(metadata) => updated_thread(metadata.thread_id),
+ EmailTopicEvent::ThreadArchived(metadata) => updated_thread(metadata.thread_id),
+ EmailTopicEvent::ThreadTrashed(metadata) => {
+ if metadata.trashed {
+ deleted_thread(metadata.thread_id)
+ } else {
+ updated_thread(metadata.thread_id)
+ }
+ }
+ EmailTopicEvent::ThreadRead(metadata) => updated_thread(metadata.thread_id),
+ EmailTopicEvent::ThreadStarred(metadata) => updated_thread(metadata.thread_id),
+ EmailTopicEvent::ThreadSpamChanged(metadata) => {
+ if metadata.spam {
+ deleted_thread(metadata.thread_id)
+ } else {
+ updated_thread(metadata.thread_id)
+ }
+ }
+ EmailTopicEvent::ThreadProjectChanged(metadata) => updated_thread(metadata.thread_id),
+ EmailTopicEvent::ThreadLabelsUpdated(metadata) => updated_thread(metadata.thread_id),
+ EmailTopicEvent::ThreadBackfilled(metadata) => updated_thread(metadata.thread_id),
+ EmailTopicEvent::ThreadsReindexRequested(metadata) => metadata
+ .thread_ids
+ .iter()
+ .map(|thread_id| update(EntityType::EmailThread, thread_id))
+ .collect(),
EmailTopicEvent::LinkConnected(_)
| EmailTopicEvent::LinkDisconnected(_)
- | EmailTopicEvent::LinkReauthRequired(_)
- | EmailTopicEvent::MessageReceived(_)
- | EmailTopicEvent::ThreadTrashed(_) => return Vec::new(),
- };
- vec![update(EntityType::EmailThread, thread_id)]
+ | EmailTopicEvent::LinkReauthRequired(_) => Vec::new(),
+ }
}
fn channel_and_thread_entities(
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
index c44cf68b5e..1443fe44fd 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
@@ -15,7 +15,9 @@ use documents::domain::events::{
DocumentUpdatedMetadata, InteractionReason,
};
use email::domain::events::{
- EmailEventOrigin, EmailTopicEvent, ThreadReadMetadata, ThreadTrashedMetadata,
+ EmailEventOrigin, EmailTopicEvent, MessageDraftSyncedMetadata, ThreadBackfilledMetadata,
+ ThreadReadMetadata, ThreadSpamChangedMetadata, ThreadTrashedMetadata, ThreadsReindexReason,
+ ThreadsReindexRequestedMetadata,
};
use macro_event_broker::{Event, EventBrokerError, MacroEventCollection as _, MessageParts};
use macro_user_id::user_id::MacroUserIdStr;
@@ -378,6 +380,89 @@ fn email_state_events_map_to_updated_or_deleted_patches() {
));
}
+#[test]
+fn new_email_events_map_to_realtime_thread_patches() {
+ let link_id = Uuid::now_v7();
+ let thread_id = Uuid::now_v7();
+ let second_thread_id = Uuid::now_v7();
+
+ let visible_draft = EmailTopicEvent::MessageDraftSynced(MessageDraftSyncedMetadata {
+ link_id,
+ owner: user(),
+ message_id: Uuid::now_v7(),
+ provider_message_id: "message-id".to_string(),
+ thread_id,
+ provider_thread_id: "thread-id".to_string(),
+ is_spam_or_trash: false,
+ });
+ let hidden_draft = EmailTopicEvent::MessageDraftSynced(MessageDraftSyncedMetadata {
+ link_id,
+ owner: user(),
+ message_id: Uuid::now_v7(),
+ provider_message_id: "hidden-message-id".to_string(),
+ thread_id,
+ provider_thread_id: "thread-id".to_string(),
+ is_spam_or_trash: true,
+ });
+ let marked_spam = EmailTopicEvent::ThreadSpamChanged(ThreadSpamChangedMetadata {
+ link_id,
+ owner: user(),
+ actor: Some(user()),
+ thread_id,
+ spam: true,
+ origin: EmailEventOrigin::UserAction,
+ });
+ let restored_from_spam = EmailTopicEvent::ThreadSpamChanged(ThreadSpamChangedMetadata {
+ link_id,
+ owner: user(),
+ actor: Some(user()),
+ thread_id,
+ spam: false,
+ origin: EmailEventOrigin::UserAction,
+ });
+ let backfilled = EmailTopicEvent::ThreadBackfilled(ThreadBackfilledMetadata {
+ link_id,
+ owner: user(),
+ thread_id,
+ });
+ let reindex_requested =
+ EmailTopicEvent::ThreadsReindexRequested(ThreadsReindexRequestedMetadata {
+ link_id,
+ owner: user(),
+ thread_ids: vec![thread_id, second_thread_id],
+ reason: ThreadsReindexReason::ContactsChanged,
+ });
+
+ assert!(matches!(
+ patches_from_email_event(&visible_draft)[0].patch,
+ Patch::Updated(_)
+ ));
+ assert!(patches_from_email_event(&hidden_draft).is_empty());
+ assert!(matches!(
+ patches_from_email_event(&marked_spam)[0].patch,
+ Patch::Deleted(_)
+ ));
+ assert!(matches!(
+ patches_from_email_event(&restored_from_spam)[0].patch,
+ Patch::Updated(_)
+ ));
+ assert!(matches!(
+ patches_from_email_event(&backfilled)[0].patch,
+ Patch::Updated(_)
+ ));
+
+ let reindex_patches = patches_from_email_event(&reindex_requested);
+ assert_eq!(reindex_patches.len(), 2);
+ assert_eq!(
+ patch_entity(&reindex_patches[0]).entity_id,
+ thread_id.to_string()
+ );
+ assert_eq!(
+ patch_entity(&reindex_patches[1]).entity_id,
+ second_thread_id.to_string()
+ );
+}
+
#[test]
fn attachment_events_use_only_metadata_available_on_the_existing_event() {
let channel_id = Uuid::now_v7();
diff --git a/crates/sqs_client/src/search/email.rs b/crates/sqs_client/src/search/email.rs
index 2982d928f1..b7afa62bec 100644
--- a/crates/sqs_client/src/search/email.rs
+++ b/crates/sqs_client/src/search/email.rs
@@ -1,22 +1,3 @@
-#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]
-pub struct EmailMessage {
- /// The message id
- pub message_id: String,
- /// The macro user id of the user who the message is for
- pub macro_user_id: String,
-}
-
-#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]
-pub struct EmailThreadMessage {
- /// The thread id
- pub thread_id: String,
- /// The macro user id of the user who the message is for
- pub macro_user_id: String,
- /// Optional override for the target OpenSearch index (e.g. "emails_v2" for migration backfills)
- #[serde(skip_serializing_if = "Option::is_none")]
- pub index_override: Option<String>,
-}
-
#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]
pub struct EmailThreadBatchMessage {
/// The thread ids to process
@@ -27,11 +8,3 @@ pub struct EmailThreadBatchMessage {
#[serde(skip_serializing_if = "Option::is_none")]
pub index_override: Option<String>,
}
-
-#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]
-pub struct EmailLinkMessage {
- /// The link id
- pub link_id: String,
- /// The macro user id associated with the link
- pub macro_user_id: String,
-}
diff --git a/crates/sqs_client/src/search/mod.rs b/crates/sqs_client/src/search/mod.rs
index a3bd10ffbe..30850b9a72 100644
--- a/crates/sqs_client/src/search/mod.rs
+++ b/crates/sqs_client/src/search/mod.rs
@@ -5,7 +5,7 @@ use crate::{
channel::ChannelMessageUpdate,
chat::ChatMessage,
document::SearchExtractorMessage,
- email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},
+ email::EmailThreadBatchMessage,
project::UpsertProject,
},
};
@@ -74,11 +74,7 @@ pub enum SearchQueueMessage {
// Chat
/// SQS backfill work-queue contract for reconciling a chat message.
ChatMessage(ChatMessage),
- // Email
- ExtractEmailMessage(EmailMessage),
- RemoveEmailMessage(EmailMessage),
- ExtractEmailThreadMessage(EmailThreadMessage),
- RemoveEmailLink(EmailLinkMessage),
+ // Email backfills
ExtractEmailThreadBatch(EmailThreadBatchMessage),
// Channel
ChannelMessageUpdate(ChannelMessageUpdate),
@@ -99,13 +95,9 @@ impl PrimaryId for SearchQueueMessage {
SearchQueueMessage::ExtractSync(message) => message.document_id.clone(),
// The message id keeps entries unique within an SQS batch.
SearchQueueMessage::ChatMessage(message) => message.message_id.clone(),
- SearchQueueMessage::ExtractEmailMessage(message)
- | SearchQueueMessage::RemoveEmailMessage(message) => message.message_id.clone(),
- SearchQueueMessage::ExtractEmailThreadMessage(message) => message.thread_id.clone(),
SearchQueueMessage::ExtractEmailThreadBatch(message) => {
message.thread_ids.first().cloned().unwrap_or_default()
}
- SearchQueueMessage::RemoveEmailLink(message) => message.link_id.clone(),
SearchQueueMessage::ChannelMessageUpdate(message) => message.message_id.clone(),
SearchQueueMessage::CallRecord(message) => message.call_id.clone(),
SearchQueueMessage::RemoveCallRecord(message) => format!(
@@ -128,12 +120,8 @@ impl SearchQueueMessage {
SearchQueueMessage::ExtractSync(_) => Operation::ExtractSync,
// Chat
SearchQueueMessage::ChatMessage(_) => Operation::ExtractText,
- // Email
- SearchQueueMessage::ExtractEmailMessage(_) => Operation::ExtractText,
- SearchQueueMessage::RemoveEmailMessage(_) => Operation::Remove,
- SearchQueueMessage::ExtractEmailThreadMessage(_) => Operation::ExtractText,
+ // Email backfills
SearchQueueMessage::ExtractEmailThreadBatch(_) => Operation::ExtractText,
- SearchQueueMessage::RemoveEmailLink(_) => Operation::Remove,
// Channels
SearchQueueMessage::ChannelMessageUpdate(_) => Operation::ExtractText,
// Calls
diff --git a/services/email_service/Cargo.toml b/services/email_service/Cargo.toml
index a2cda81b6e..8b3f78c2b5 100644
--- a/services/email_service/Cargo.toml
+++ b/services/email_service/Cargo.toml
@@ -158,7 +158,6 @@ sqlx-core = { workspace = true }
sqs_client = { path = "../../crates/sqs_client", default-features = false, features = [
"email",
"gmail",
- "search",
"sfs_delete",
"sfs_uploader"
] }
diff --git a/services/email_service/src/backfill_init_service.rs b/services/email_service/src/backfill_init_service.rs
index d2bb1be3a3..426ae22cef 100644
--- a/services/email_service/src/backfill_init_service.rs
+++ b/services/email_service/src/backfill_init_service.rs
@@ -91,6 +91,7 @@ async fn initialize_backfill(
&ctx.db,
&ctx.gmail_client,
&ctx.sqs_client,
+ &ctx.macro_event_broker,
access_token,
)
.await
diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
index 773a7f0e2d..9a1c2d1e49 100644
--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
@@ -75,7 +75,6 @@ async fn main() -> anyhow::Result<()> {
let gmail_inbox_sync_retry_queue = macro_queues::GmailInboxSyncRetryQueue::new();
let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
let gmail_ops_retry_queue = macro_queues::GmailOpsRetryQueue::new();
- let search_event_queue = macro_queues::SearchEventQueue::new();
let backfill_queue = macro_queues::EmailBackfillQueue::new();
let crm_cleanup_queue = macro_queues::EmailCrmCleanupQueue::new();
let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
@@ -90,7 +89,6 @@ async fn main() -> anyhow::Result<()> {
.gmail_inbox_sync_retry_queue(&gmail_inbox_sync_retry_queue)
.gmail_ops_queue(&gmail_ops_queue)
.gmail_ops_retry_queue(&gmail_ops_retry_queue)
- .search_event_queue(&search_event_queue)
.email_backfill_queue(&backfill_queue)
.email_crm_cleanup_queue(&crm_cleanup_queue)
.email_scheduled_queue(&email_scheduled_queue)
diff --git a/services/email_service/src/main.rs b/services/email_service/src/main.rs
index 98a3486f5a..93cf3c2a19 100644
--- a/services/email_service/src/main.rs
+++ b/services/email_service/src/main.rs
@@ -66,7 +66,6 @@ async fn main() -> anyhow::Result<()> {
let gmail_inbox_sync_queue = macro_queues::GmailInboxSyncQueue::new();
let gmail_inbox_sync_retry_queue = macro_queues::GmailInboxSyncRetryQueue::new();
let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
- let search_event_queue = macro_queues::SearchEventQueue::new();
let backfill_queue = macro_queues::EmailBackfillQueue::new();
let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
let sfs_uploader_queue = macro_queues::SfsUploaderQueue::new();
@@ -75,7 +74,6 @@ async fn main() -> anyhow::Result<()> {
.gmail_inbox_sync_queue(&gmail_inbox_sync_queue)
.gmail_inbox_sync_retry_queue(&gmail_inbox_sync_retry_queue)
.gmail_ops_queue(&gmail_ops_queue)
- .search_event_queue(&search_event_queue)
.email_backfill_queue(&backfill_queue)
.email_scheduled_queue(&email_scheduled_queue)
.sfs_uploader_queue(&sfs_uploader_queue)
diff --git a/services/email_service/src/pubsub/backfill/update_metadata.rs b/services/email_service/src/pubsub/backfill/update_metadata.rs
index 7a165abcee..47ec8bfe83 100644
--- a/services/email_service/src/pubsub/backfill/update_metadata.rs
+++ b/services/email_service/src/pubsub/backfill/update_metadata.rs
@@ -1,10 +1,10 @@
use crate::pubsub::backfill::increment_counters::incr_completed_threads;
use crate::pubsub::context::PubSubContext;
+use crate::pubsub::util::publish_email_event;
+use email::domain::events::{EmailMacroEvent, ThreadBackfilledMetadata};
use models_email::email::service::backfill::{JobScopedPayload, UpdateMetadataPayload};
use models_email::email::service::link;
use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
-use sqs_client::search::SearchQueueMessage;
-use sqs_client::search::email::EmailThreadMessage;
/// This step is invoked by BackfillMessage once all messages in a thread have been backfilled.
/// Updates the thread metadata in the database, the replying_to_id values of its messages, and
@@ -80,22 +80,16 @@ pub async fn update_thread_metadata(
incr_completed_threads(ctx, link, scope.job_id).await?;
- // notify search-service about the new thread
- let search_message = SearchQueueMessage::ExtractEmailThreadMessage(EmailThreadMessage {
- thread_id: p.thread_db_id.clone().to_string(),
- macro_user_id: link.macro_id.to_string(),
- index_override: None,
- });
-
- ctx.sqs_client
- .send_message_to_search_event_queue(search_message)
- .await
- .map_err(|e| {
- ProcessingError::NonRetryable(DetailedError {
- reason: FailureReason::SqsEnqueueFailed,
- source: e.context("Failed to send message to search extractor queue"),
- })
- })?;
+ // Best-effort by design: publication failures are logged and dropped so a committed
+ // backfill is not retried after its completed-thread counters have advanced.
+ publish_email_event(
+ &ctx.macro_event_broker,
+ &EmailMacroEvent::thread_backfilled(ThreadBackfilledMetadata {
+ link_id: link.id,
+ owner: link.macro_id.clone(),
+ thread_id: p.thread_db_id,
+ }),
+ );
Ok(())
}
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs
index 4f0b07bc05..ce1c0d9063 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs
@@ -8,8 +8,6 @@ use models_email::api::refresh::RefreshEmailEvent;
use models_email::email::service::link;
use models_email::gmail::inbox_sync::DeleteMessagePayload;
use models_email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
-use sqs_client::search::SearchQueueMessage;
-use sqs_client::search::email::EmailMessage;
use std::result;
use uuid::Uuid;
@@ -134,19 +132,5 @@ pub async fn delete_message(
)
.await;
- // send message to search text extractor queue
- let _ = ctx
- .sqs_client
- .bulk_send_message_to_search_event_queue(vec![SearchQueueMessage::RemoveEmailMessage(
- EmailMessage {
- message_id: message.db_id.to_string(),
- macro_user_id: link.macro_id.to_string(),
- },
- )])
- .await
- .inspect_err(
- |e| tracing::error!(error = ?e, "failed to send message to search extractor queue"),
- );
-
Ok(())
}
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/mod.rs b/services/email_service/src/pubsub/inbox_sync/operations/mod.rs
index 694cca1d37..c50c3c3df8 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/mod.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/mod.rs
@@ -1,5 +1,4 @@
pub(in crate::pubsub) mod delete_message;
pub(in crate::pubsub) mod gmail_message;
-pub(in crate::pubsub) mod shared;
pub(in crate::pubsub) mod update_labels;
pub(in crate::pubsub) mod upsert_message;
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/shared.rs b/services/email_service/src/pubsub/inbox_sync/operations/shared.rs
deleted file mode 100644
index 491d8faba4..0000000000
--- a/services/email_service/src/pubsub/inbox_sync/operations/shared.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-use crate::pubsub::context::PubSubContext;
-use models_email::service::link;
-use models_email::service::pubsub::ProcessingError;
-use sqs_client::search::SearchQueueMessage;
-use sqs_client::search::email::EmailMessage;
-use std::result;
-use uuid::Uuid;
-
-/// Notify search about a message change. If the message is spam or trash, it will be removed from
-/// search. Otherwise, it will be upserted.
-#[tracing::instrument(skip(ctx, link, message_db_id))]
-pub async fn notify_search(
- ctx: &PubSubContext,
- link: &link::Link,
- message_db_id: Uuid,
- is_spam_or_trash: bool,
-) -> result::Result<(), ProcessingError> {
- let message = if is_spam_or_trash {
- SearchQueueMessage::RemoveEmailMessage(EmailMessage {
- message_id: message_db_id.to_string(),
- macro_user_id: link.macro_id.to_string(),
- })
- } else {
- SearchQueueMessage::ExtractEmailMessage(EmailMessage {
- message_id: message_db_id.to_string(),
- macro_user_id: link.macro_id.to_string(),
- })
- };
-
- ctx.sqs_client
- .send_message_to_search_event_queue(message)
- .await
- .inspect_err(
- |e| tracing::error!(error = ?e, "failed to send message to search extractor queue"),
- )
- .ok();
-
- Ok(())
-}
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs b/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs
index 94eedaa6a2..8d6d07dc5d 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/update_labels.rs
@@ -1,5 +1,4 @@
use crate::pubsub::context::PubSubContext;
-use crate::pubsub::inbox_sync::operations::shared::notify_search;
use crate::pubsub::inbox_sync::process;
use crate::pubsub::inbox_sync::process::check_gmail_rate_limit_inbox_sync;
use crate::pubsub::util::{
@@ -7,11 +6,13 @@ use crate::pubsub::util::{
};
use email::domain::events::{
EmailEventOrigin, EmailMacroEvent, LabelRef, ThreadArchivedMetadata,
- ThreadLabelsUpdatedMetadata, ThreadReadMetadata, ThreadStarredMetadata, ThreadTrashedMetadata,
+ ThreadLabelsUpdatedMetadata, ThreadReadMetadata, ThreadSpamChangedMetadata,
+ ThreadStarredMetadata, ThreadTrashedMetadata,
};
use email_db_client::labels::delete::delete_db_message_labels;
use email_db_client::labels::insert;
use email_db_client::threads::update::update_thread_metadata;
+use macro_user_id::user_id::MacroUserIdStr;
use models_email::api::refresh::RefreshEmailEvent;
use models_email::email::service::link;
use models_email::gmail::inbox_sync::{
@@ -24,6 +25,9 @@ use sqlx::PgPool;
use std::result;
use uuid::Uuid;
+#[cfg(test)]
+mod test;
+
/// Compares a db message's labels to the gmail message's labels, adding/removing as necessary
#[tracing::instrument(skip(ctx))]
pub async fn update_labels(
@@ -159,9 +163,6 @@ pub async fn update_labels(
}
if has_label_changes {
- // Publish semantic macro.email events before the fallible search
- // notify: a notify failure retries the whole op, and on retry the
- // label diff is empty so the events would be lost.
publish_label_diff_events(
ctx,
link,
@@ -172,13 +173,6 @@ pub async fn update_labels(
)
.await;
- let is_spam_or_trash = gmail_message_labels.iter().any(|label| {
- label == service::label::system_labels::SPAM
- || label == service::label::system_labels::TRASH
- });
-
- notify_search(ctx, link, db_message.db_id, is_spam_or_trash).await?;
-
// tell FE to refresh user's inbox
cg_refresh_email(
&ctx.connection_gateway_client,
@@ -192,8 +186,8 @@ pub async fn update_labels(
}
/// System labels that never appear in a `thread_labels_updated` event:
-/// INBOX/TRASH/UNREAD/STARRED map to dedicated thread-state events, and
-/// SPAM/IMPORTANT/SENT/DRAFT changes are not published at all.
+/// INBOX/TRASH/UNREAD/STARRED/SPAM map to dedicated thread-state events, and
+/// IMPORTANT/SENT/DRAFT changes are not published.
const NON_USER_LABELS: [&str; 8] = [
service::label::system_labels::INBOX,
service::label::system_labels::SPAM,
@@ -205,6 +199,36 @@ const NON_USER_LABELS: [&str; 8] = [
service::label::system_labels::DRAFT,
];
+fn build_provider_spam_changed_metadata(
+ link_id: Uuid,
+ owner: &MacroUserIdStr<'static>,
+ thread_id: Uuid,
+ labels_to_add: &[String],
+ labels_to_delete: &[String],
+) -> Option<ThreadSpamChangedMetadata> {
+ let spam_label = service::label::system_labels::SPAM;
+ let spam = if labels_to_add.iter().any(|label| label == spam_label) {
+ true
+ } else if labels_to_delete.iter().any(|label| label == spam_label) {
+ false
+ } else {
+ return None;
+ };
+
+ Some(ThreadSpamChangedMetadata {
+ link_id,
+ owner: owner.clone(),
+ actor: None,
+ thread_id,
+ spam,
+ origin: EmailEventOrigin::ProviderSync,
+ })
+}
+
+fn is_user_label(label: &str) -> bool {
+ !NON_USER_LABELS.contains(&label) && !label.starts_with("CATEGORY_")
+}
+
/// Publish semantic macro.email events for a provider-side label diff, all
/// with `origin = provider_sync`. Changes made through Macro update the DB
/// optimistically, so by the time they echo back through Gmail history sync
@@ -272,6 +296,16 @@ async fn publish_label_diff_events(
}));
}
+ if let Some(metadata) = build_provider_spam_changed_metadata(
+ link.id,
+ &link.macro_id,
+ thread_db_id,
+ labels_to_add,
+ labels_to_delete,
+ ) {
+ events.push(EmailMacroEvent::thread_spam_changed(metadata));
+ }
+
if added(service::label::system_labels::UNREAD)
|| removed(service::label::system_labels::UNREAD)
{
@@ -301,13 +335,9 @@ async fn publish_label_diff_events(
}));
}
- // Gmail category tabs (CATEGORY_PERSONAL/SOCIAL/...) are provider system
- // labels, not user labels.
- let is_user_label = |l: &str| !NON_USER_LABELS.contains(&l) && !l.starts_with("CATEGORY_");
-
let added_user: Vec<LabelRef> = labels_to_add
.iter()
- .filter(|l| is_user_label(l.as_str()))
+ .filter(|label| is_user_label(label.as_str()))
.map(|l| LabelRef {
label_id: None,
provider_label_id: l.clone(),
@@ -316,14 +346,14 @@ async fn publish_label_diff_events(
.collect();
let removed_user: Vec<LabelRef> = db_message_labels
.iter()
- .filter(|l| {
- labels_to_delete.contains(&l.provider_label_id)
- && is_user_label(l.provider_label_id.as_str())
+ .filter(|label| {
+ labels_to_delete.contains(&label.provider_label_id)
+ && is_user_label(label.provider_label_id.as_str())
})
- .map(|l| LabelRef {
- label_id: Some(l.id),
- provider_label_id: l.provider_label_id.clone(),
- name: Some(l.name.clone()),
+ .map(|label| LabelRef {
+ label_id: Some(label.id),
+ provider_label_id: label.provider_label_id.clone(),
+ name: Some(label.name.clone()),
})
.collect();
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs
new file mode 100644
index 0000000000..24cba89e10
--- /dev/null
+++ b/services/email_service/src/pubsub/inbox_sync/operations/update_labels/test.rs
@@ -0,0 +1,72 @@
+use super::*;
+
+fn owner() -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).unwrap()
+}
+
+#[test]
+fn builds_provider_origin_spam_addition() {
+ let link_id = Uuid::from_u128(1);
+ let thread_id = Uuid::from_u128(2);
+ let owner = owner();
+ let labels_to_add = vec![service::label::system_labels::SPAM.to_string()];
+
+ let metadata =
+ build_provider_spam_changed_metadata(link_id, &owner, thread_id, &labels_to_add, &[]);
+
+ assert_eq!(
+ metadata,
+ Some(ThreadSpamChangedMetadata {
+ link_id,
+ owner,
+ actor: None,
+ thread_id,
+ spam: true,
+ origin: EmailEventOrigin::ProviderSync,
+ })
+ );
+}
+
+#[test]
+fn builds_provider_origin_spam_removal() {
+ let link_id = Uuid::from_u128(1);
+ let thread_id = Uuid::from_u128(2);
+ let owner = owner();
+ let labels_to_delete = vec![service::label::system_labels::SPAM.to_string()];
+
+ let metadata =
+ build_provider_spam_changed_metadata(link_id, &owner, thread_id, &[], &labels_to_delete);
+
+ assert_eq!(
+ metadata,
+ Some(ThreadSpamChangedMetadata {
+ link_id,
+ owner,
+ actor: None,
+ thread_id,
+ spam: false,
+ origin: EmailEventOrigin::ProviderSync,
+ })
+ );
+}
+
+#[test]
+fn ignores_diffs_without_spam_changes() {
+ let labels_to_add = vec![service::label::system_labels::STARRED.to_string()];
+
+ assert_eq!(
+ build_provider_spam_changed_metadata(
+ Uuid::from_u128(1),
+ &owner(),
+ Uuid::from_u128(2),
+ &labels_to_add,
+ &[],
+ ),
+ None
+ );
+}
+
+#[test]
+fn spam_remains_excluded_from_user_label_events() {
+ assert!(!is_user_label(service::label::system_labels::SPAM));
+}
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
index 4e26a21a10..ccf9bc298e 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message.rs
@@ -1,6 +1,5 @@
use crate::convert::{map_message_resource_to_service, map_thread_resources_to_service};
use crate::pubsub::context::PubSubContext;
-use crate::pubsub::inbox_sync::operations::shared::notify_search;
use crate::pubsub::inbox_sync::process;
use crate::pubsub::inbox_sync::process::check_gmail_rate_limit_inbox_sync;
use crate::pubsub::util::{
@@ -12,7 +11,8 @@ use crate::util::upload_attachment::{UploadAttachmentContext, upload_attachment}
use contacts::domain::models::messages::ContactConnection;
use contacts::domain::ports::ContactsIngress;
use email::domain::events::{
- EmailEventOrigin, EmailMacroEvent, MessageReceivedMetadata, MessageSentMetadata,
+ EmailEventOrigin, EmailMacroEvent, MessageDraftSyncedMetadata, MessageReceivedMetadata,
+ MessageSentMetadata,
};
use email::domain::models::{PreviewCursorQuery, PreviewView, PreviewViewStandardLabel};
use email::domain::ports::EmailRepo;
@@ -44,6 +44,37 @@ use uuid::Uuid;
#[cfg(test)]
mod test;
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum MessageSyncEventKind {
+ DraftSynced,
+ Received,
+ Sent,
+}
+
+fn select_message_sync_event(
+ existing_message_was_draft: Option<bool>,
+ is_draft: bool,
+ is_sent: bool,
+) -> Option<MessageSyncEventKind> {
+ if is_draft {
+ return Some(MessageSyncEventKind::DraftSynced);
+ }
+
+ if existing_message_was_draft == Some(true) && is_sent {
+ return Some(MessageSyncEventKind::Sent);
+ }
+
+ if existing_message_was_draft.is_some() {
+ return None;
+ }
+
+ if is_sent {
+ Some(MessageSyncEventKind::Sent)
+ } else {
+ Some(MessageSyncEventKind::Received)
+ }
+}
+
// upsert a message into the db. could be a new message or an existing one that had changes
#[tracing::instrument(skip(ctx))]
pub async fn upsert_message(
@@ -174,20 +205,24 @@ pub async fn upsert_message(
})
})?;
- // before upserting, figure out if the message is new so we can send a notification for it if so
- let message_already_exists = email_db_client::messages::get::message_exists_by_provider_id(
- &ctx.db,
- &payload.provider_message_id,
- link.id,
- )
- .await
- .map_err(|e| {
- ProcessingError::NonRetryable(DetailedError {
- reason: FailureReason::DatabaseQueryFailed,
- source: e
- .context("Failed to check whether provider_message_id already exists".to_string()),
- })
- })?;
+ // Snapshot the previous draft state before the upsert. Provider drafts are mutable,
+ // and this state distinguishes a draft-to-sent transition from an unchanged message.
+ let existing_message =
+ email_db_client::messages::get_simple_messages::get_simple_message_by_provider_and_link(
+ &ctx.db,
+ &payload.provider_message_id,
+ &link.id,
+ )
+ .await
+ .map_err(|e| {
+ ProcessingError::NonRetryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e
+ .context("Failed to fetch existing message before provider upsert".to_string()),
+ })
+ })?;
+ let existing_message_was_draft = existing_message.as_ref().map(|message| message.is_draft);
+ let message_already_exists = existing_message.is_some();
let is_new_thread = !thread_provider_to_db_map.contains_key(&provider_thread_id);
@@ -233,17 +268,44 @@ pub async fn upsert_message(
})
})?;
- // Publish to the macro.email topic immediately after the committed
- // insert, BEFORE the fallible side-effect steps below: if any of them
- // fails, the SQS retry finds the message already existing and would
- // never emit. Drafts are skipped — a draft synced from the provider is
- // not a received or sent message.
- if !message_already_exists && !is_draft {
- let event = if is_sent {
- // Sent from another client and first observed via sync; sends
- // performed through Macro are published by the scheduled-send
- // worker and already exist here.
- EmailMacroEvent::message_sent(MessageSentMetadata {
+ // Publish to the macro.email topic immediately after the committed insert.
+ // Drafts publish after every sync because their bodies are mutable. Existing
+ // immutable messages remain suppressed except when a previously synced provider
+ // draft becomes sent.
+ if let Some(event_kind) =
+ select_message_sync_event(existing_message_was_draft, is_draft, is_sent)
+ {
+ let event = match event_kind {
+ MessageSyncEventKind::DraftSynced => {
+ EmailMacroEvent::message_draft_synced(MessageDraftSyncedMetadata {
+ link_id: link.id,
+ owner: link.macro_id.clone(),
+ message_id: message_db_id,
+ provider_message_id: payload.provider_message_id.clone(),
+ thread_id: thread_db_id,
+ provider_thread_id: provider_thread_id.clone(),
+ is_spam_or_trash,
+ })
+ }
+ MessageSyncEventKind::Received => {
+ EmailMacroEvent::message_received(MessageReceivedMetadata {
+ link_id: link.id,
+ owner: link.macro_id.clone(),
+ message_id: message_db_id,
+ provider_message_id: payload.provider_message_id.clone(),
+ thread_id: thread_db_id,
+ provider_thread_id: provider_thread_id.clone(),
+ is_new_thread,
+ subject: event_subject,
+ from_email: event_from.as_ref().map(|contact| contact.email.clone()),
+ from_name: event_from.as_ref().and_then(|contact| contact.name.clone()),
+ to_emails: event_to_emails,
+ attachment_count: message_attachment_count as u32,
+ is_spam_or_trash,
+ received_at: event_received_at,
+ })
+ }
+ MessageSyncEventKind::Sent => EmailMacroEvent::message_sent(MessageSentMetadata {
link_id: link.id,
owner: link.macro_id.clone(),
actor: None,
@@ -256,24 +318,7 @@ pub async fn upsert_message(
cc_emails: event_cc_emails,
origin: EmailEventOrigin::ProviderSync,
sent_at: event_sent_at.unwrap_or_else(chrono::Utc::now),
- })
- } else {
- EmailMacroEvent::message_received(MessageReceivedMetadata {
- link_id: link.id,
- owner: link.macro_id.clone(),
- message_id: message_db_id,
- provider_message_id: payload.provider_message_id.clone(),
- thread_id: thread_db_id,
- provider_thread_id: provider_thread_id.clone(),
- is_new_thread,
- subject: event_subject,
- from_email: event_from.as_ref().map(|c| c.email.clone()),
- from_name: event_from.as_ref().and_then(|c| c.name.clone()),
- to_emails: event_to_emails,
- attachment_count: message_attachment_count as u32,
- is_spam_or_trash,
- received_at: event_received_at,
- })
+ }),
};
publish_email_event(&ctx.macro_event_broker, &event);
}
@@ -329,8 +374,6 @@ pub async fn upsert_message(
enqueue_populate_crm_contacts(ctx, link.id, &self_email, crm_recipients, is_sent).await?;
}
- notify_search(ctx, link, message_db_id, is_spam_or_trash).await?;
-
// trigger FE inbox refresh
cg_refresh_email(
&ctx.connection_gateway_client,
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs
index cc0f92f975..126d7dac23 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/upsert_message/test.rs
@@ -49,3 +49,49 @@ fn skips_unparseable_primaries_keeping_valid_ones() {
HashSet::from([owner, id("macro|[email protected]")])
);
}
+
+#[test]
+fn selects_draft_sync_for_new_draft() {
+ assert_eq!(
+ select_message_sync_event(None, true, false),
+ Some(MessageSyncEventKind::DraftSynced)
+ );
+}
+
+#[test]
+fn selects_draft_sync_for_draft_edit() {
+ assert_eq!(
+ select_message_sync_event(Some(true), true, false),
+ Some(MessageSyncEventKind::DraftSynced)
+ );
+}
+
+#[test]
+fn selects_sent_for_provider_draft_to_sent_transition() {
+ assert_eq!(
+ select_message_sync_event(Some(true), false, true),
+ Some(MessageSyncEventKind::Sent)
+ );
+}
+
+#[test]
+fn selects_sent_for_new_sent_message() {
+ assert_eq!(
+ select_message_sync_event(None, false, true),
+ Some(MessageSyncEventKind::Sent)
+ );
+}
+
+#[test]
+fn selects_received_for_new_received_message() {
+ assert_eq!(
+ select_message_sync_event(None, false, false),
+ Some(MessageSyncEventKind::Received)
+ );
+}
+
+#[test]
+fn suppresses_existing_immutable_non_drafts() {
+ assert_eq!(select_message_sync_event(Some(false), false, false), None);
+ assert_eq!(select_message_sync_event(Some(false), false, true), None);
+}
diff --git a/services/email_service/src/pubsub/link_manager/process.rs b/services/email_service/src/pubsub/link_manager/process.rs
index f1e9b2e5f5..906443ec5f 100644
--- a/services/email_service/src/pubsub/link_manager/process.rs
+++ b/services/email_service/src/pubsub/link_manager/process.rs
@@ -18,8 +18,6 @@ use models_email::service::cache::TokenCacheKey;
use models_email::service::link::{Link, UserProvider};
use notification::domain::models::SendNotificationRequestBuilder;
use notification::domain::service::NotificationIngress;
-use sqs_client::search::SearchQueueMessage;
-use sqs_client::search::email::EmailLinkMessage;
use sqs_worker::cleanup_message;
use std::time::Duration;
use tokio_retry::RetryIf;
@@ -204,6 +202,7 @@ async fn handle_refresh(
&ctx.db,
&ctx.gmail_client,
&ctx.sqs_client,
+ &ctx.macro_event_broker,
gmail_access_token,
)
.await
@@ -369,18 +368,6 @@ async fn handle_delete(
})
.ok();
- // inform search of deletion so it can wipe the email records from OS
- ctx.sqs_client
- .send_message_to_search_event_queue(SearchQueueMessage::RemoveEmailLink(EmailLinkMessage {
- link_id: link.id.to_string(),
- macro_user_id: link.macro_id.to_string(),
- }))
- .await
- .inspect_err(|e| {
- tracing::error!(error=?e, "failed to send message to search extractor queue");
- })
- .ok();
-
// Tear down CRM rows this link contributed to the user's team before
// the big cascading link delete fires. Best-effort: a failure here
// would only leave orphan `crm_contacts`/`crm_companies` rows behind
diff --git a/services/email_service/src/util/sync_contacts.rs b/services/email_service/src/util/sync_contacts.rs
index e9913436d1..594d2d8360 100644
--- a/services/email_service/src/util/sync_contacts.rs
+++ b/services/email_service/src/util/sync_contacts.rs
@@ -1,24 +1,33 @@
use crate::convert::map_person_to_contact;
+use crate::pubsub::util::publish_email_event;
use anyhow::{Context, anyhow};
+use email::domain::events::{
+ EmailMacroEvent, ThreadsReindexReason, ThreadsReindexRequestedMetadata,
+};
use futures::{StreamExt, stream};
use gmail_client::GmailClient;
+use macro_event_broker::MacroEventBroker;
+use macro_user_id::user_id::MacroUserIdStr;
use models_email::service::contact::Contact;
use models_email::service::link::Link;
use models_email::service::pubsub::SFSUploaderMessage;
use models_email::service::sync_token::SyncTokens;
use sqlx::PgPool;
use sqs_client::SQS;
-use sqs_client::search::SearchQueueMessage;
-use sqs_client::search::email::EmailThreadBatchMessage;
use std::collections::HashSet;
use std::time::Instant;
+use uuid::Uuid;
+
+#[cfg(test)]
+mod test;
/// Syncs user's contacts with gmail
-pub async fn sync_contacts(
+pub async fn sync_contacts<B: MacroEventBroker>(
link: &Link,
db: &PgPool,
gmail_client: &GmailClient,
sqs_client: &SQS,
+ macro_event_broker: &B,
gmail_access_token: &str,
) -> anyhow::Result<()> {
// 1. Get existing sync tokens from our DB
@@ -37,7 +46,7 @@ pub async fn sync_contacts(
// 3. If we received any new/updated contacts, process and store them.
if !new_contacts.is_empty() {
- process_and_store_contacts(db, sqs_client, link, new_contacts).await?;
+ process_and_store_contacts(db, sqs_client, macro_event_broker, link, new_contacts).await?;
}
// 4. Store the new sync tokens in our DB
@@ -129,9 +138,10 @@ async fn fetch_new_contacts_from_google(
}
/// Handles processing (SFS uploads) and database storage for a list of contacts.
-async fn process_and_store_contacts(
+async fn process_and_store_contacts<B: MacroEventBroker>(
db: &PgPool,
sqs_client: &SQS,
+ macro_event_broker: &B,
link: &Link,
contacts: Vec<Contact>,
) -> anyhow::Result<()> {
@@ -177,7 +187,8 @@ async fn process_and_store_contacts(
);
if !changed_contact_ids.is_empty() {
- reindex_threads_for_changed_contacts(db, sqs_client, link, &changed_contact_ids).await;
+ reindex_threads_for_changed_contacts(db, macro_event_broker, link, &changed_contact_ids)
+ .await;
}
if cfg!(not(feature = "sfs_map")) {
@@ -215,12 +226,12 @@ async fn process_and_store_contacts(
const REINDEX_BATCH_SIZE: usize = 50;
-#[tracing::instrument(skip(db, sqs_client, link, changed_contact_ids))]
-async fn reindex_threads_for_changed_contacts(
+#[tracing::instrument(skip(db, macro_event_broker, link, changed_contact_ids))]
+async fn reindex_threads_for_changed_contacts<B: MacroEventBroker>(
db: &PgPool,
- sqs_client: &SQS,
+ macro_event_broker: &B,
link: &Link,
- changed_contact_ids: &[uuid::Uuid],
+ changed_contact_ids: &[Uuid],
) {
let thread_ids = match email_db_client::threads::get::get_thread_ids_by_contact_ids(
db,
@@ -239,8 +250,6 @@ async fn reindex_threads_for_changed_contacts(
return;
}
- let macro_user_id = link.macro_id.to_string();
-
tracing::info!(
num_threads = thread_ids.len(),
num_changed_contacts = changed_contact_ids.len(),
@@ -248,21 +257,24 @@ async fn reindex_threads_for_changed_contacts(
"Re-indexing threads for contacts with name changes"
);
- let messages: Vec<SearchQueueMessage> = thread_ids
- .chunks(REINDEX_BATCH_SIZE)
- .map(|chunk| {
- SearchQueueMessage::ExtractEmailThreadBatch(EmailThreadBatchMessage {
- thread_ids: chunk.iter().map(|id| id.to_string()).collect(),
- macro_user_id: macro_user_id.clone(),
- index_override: None,
- })
- })
- .collect();
+ publish_thread_reindex_batches(macro_event_broker, link.id, &link.macro_id, &thread_ids);
+}
- if let Err(e) = sqs_client
- .bulk_send_message_to_search_event_queue(messages)
- .await
- {
- tracing::error!(error=?e, link_id=%link.id, "Failed to enqueue search re-index messages for contact name changes");
+fn publish_thread_reindex_batches<B: MacroEventBroker>(
+ macro_event_broker: &B,
+ link_id: Uuid,
+ owner: &MacroUserIdStr<'static>,
+ thread_ids: &[Uuid],
+) {
+ for thread_ids in thread_ids.chunks(REINDEX_BATCH_SIZE) {
+ publish_email_event(
+ macro_event_broker,
+ &EmailMacroEvent::threads_reindex_requested(ThreadsReindexRequestedMetadata {
+ link_id,
+ owner: owner.clone(),
+ thread_ids: thread_ids.to_vec(),
+ reason: ThreadsReindexReason::ContactsChanged,
+ }),
+ );
}
}
diff --git a/services/email_service/src/util/sync_contacts/test.rs b/services/email_service/src/util/sync_contacts/test.rs
new file mode 100644
index 0000000000..ef1f55ca08
--- /dev/null
+++ b/services/email_service/src/util/sync_contacts/test.rs
@@ -0,0 +1,133 @@
+use std::sync::Mutex;
+
+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};
+use serde_json::{Value, json};
+
+use super::*;
+
+#[derive(Clone, Debug)]
+struct PublishedEvent {
+ topic: String,
+ key: String,
+ envelope: Value,
+}
+
+#[derive(Default)]
+struct RecordingEventBroker {
+ published: Mutex<Vec<PublishedEvent>>,
+}
+
+impl MacroEventBroker for RecordingEventBroker {
+ fn send_event<E: MacroEvent + ?Sized>(
+ &self,
+ event: &E,
+ ) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {
+ self.published
+ .lock()
+ .expect("published events lock should not be poisoned")
+ .push(PublishedEvent {
+ topic: event.topic().to_string(),
+ key: event.key().to_string(),
+ envelope: serde_json::to_value(event.event())?,
+ });
+
+ Ok(tokio::spawn(async { Ok(()) }))
+ }
+}
+
+fn owner() -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid owner")
+}
+
+fn thread_ids(count: usize) -> Vec<Uuid> {
+ (1..=count)
+ .map(|value| Uuid::from_u128(value as u128))
+ .collect()
+}
+
+fn assert_published_batches(thread_count: usize, expected_batch_sizes: &[usize]) {
+ let broker = RecordingEventBroker::default();
+ let link_id = Uuid::from_u128(u128::MAX);
+ let owner = owner();
+ let thread_ids = thread_ids(thread_count);
+
+ publish_thread_reindex_batches(&broker, link_id, &owner, &thread_ids);
+
+ let published = broker
+ .published
+ .lock()
+ .expect("published events lock should not be poisoned");
+ let actual_batch_sizes: Vec<usize> = published
+ .iter()
+ .map(|event| {
+ event.envelope["metadata"]["thread_ids"]
+ .as_array()
+ .expect("thread_ids should be an array")
+ .len()
+ })
+ .collect();
+ assert_eq!(actual_batch_sizes, expected_batch_sizes);
+
+ let published_thread_ids: Vec<Uuid> = published
+ .iter()
+ .flat_map(|event| {
+ assert_eq!(event.topic, "macro.email");
+ assert_eq!(event.key, link_id.to_string());
+ assert_eq!(event.envelope["schema_version"], json!(1));
+ assert_eq!(
+ event.envelope["event_type"],
+ json!("email.threads_reindex_requested")
+ );
+ assert_eq!(
+ event.envelope["metadata"]["link_id"],
+ json!(link_id.to_string())
+ );
+ assert_eq!(
+ event.envelope["metadata"]["owner"],
+ json!(owner.to_string())
+ );
+ assert_eq!(
+ event.envelope["metadata"]["reason"],
+ json!("contacts_changed")
+ );
+
+ let batch = event.envelope["metadata"]["thread_ids"]
+ .as_array()
+ .expect("thread_ids should be an array");
+ assert!(!batch.is_empty());
+ assert!(batch.len() <= REINDEX_BATCH_SIZE);
+
+ batch.iter().map(|thread_id| {
+ Uuid::parse_str(thread_id.as_str().expect("thread id should be a string"))
+ .expect("thread id should be a UUID")
+ })
+ })
+ .collect();
+
+ assert_eq!(published_thread_ids, thread_ids);
+}
+
+#[tokio::test]
+async fn publishes_no_batches_for_zero_threads() {
+ assert_published_batches(0, &[]);
+}
+
+#[tokio::test]
+async fn publishes_one_batch_for_one_thread() {
+ assert_published_batches(1, &[1]);
+}
+
+#[tokio::test]
+async fn publishes_one_full_batch_for_fifty_threads() {
+ assert_published_batches(50, &[50]);
+}
+
+#[tokio::test]
+async fn publishes_a_second_batch_for_fifty_one_threads() {
+ assert_published_batches(51, &[50, 1]);
+}
+
+#[tokio::test]
+async fn publishes_ordered_bounded_batches_for_larger_inputs() {
+ assert_published_batches(137, &[50, 50, 37]);
+}
diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.toml
index 841d2e9b5b..ca3cf93794 100644
--- a/services/search_processing_service/Cargo.toml
+++ b/services/search_processing_service/Cargo.toml
@@ -12,6 +12,7 @@ processing = [
"dep:channels",
"dep:chat",
"dep:documents",
+ "dep:email",
"dep:kafka_util",
"dep:macro_event_broker",
"dep:macro_event_topics",
@@ -51,6 +52,7 @@ channels = { path = "../../crates/channels", default-features = false, optional
chat = { path = "../../crates/chat", default-features = false, optional = true }
chrono = { workspace = true }
comms_db_client = { path = "../../crates/comms_db_client" }
+email = { path = "../../crates/email", default-features = false, optional = true }
email_db_client = { path = "../../crates/email_db_client" }
futures = { workspace = true }
http-body-util = { workspace = true }
diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rs
index 289d4e4439..59cc633a27 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs
@@ -7,8 +7,8 @@
//! offsets are already committed.
//!
//! Per-entity event mapping and processing live in the [`call`], [`channel`],
-//! [`chat`], [`document`], [`project`], and [`property`] submodules; this module
-//! owns the poll loop, worker, retry policy, and commit semantics.
+//! [`chat`], [`document`], [`email`], [`project`], and [`property`] submodules;
+//! this module owns the poll loop, worker, retry policy, and commit semantics.
#![allow(clippy::enum_variant_names)]
@@ -17,6 +17,7 @@ mod channel;
mod chat;
mod context;
mod document;
+mod email;
mod project;
mod property;
#[cfg(test)]
@@ -26,6 +27,7 @@ use std::{future::Future, time::Duration};
use ::call::domain::events::CallMacroEvent;
use ::chat::domain::events::ChatMacroEvent;
+use ::email::domain::events::EmailMacroEvent;
use channels::domain::broker_events::ChannelMacroEvent;
use documents::domain::events::DocumentMacroEvent;
use kafka_util::{GroupName, KafkaEventConsumer};
@@ -42,7 +44,7 @@ use tokio_retry::{Retry, strategy::ExponentialBackoff};
use self::{
call::process_call_event, channel::process_channel_event, chat::process_chat_event,
- document::process_document_event, project::process_project_event,
+ document::process_document_event, email::process_email_event, project::process_project_event,
property::process_property_event,
};
@@ -66,6 +68,7 @@ macro_event_broker::declare_topics!(
ChannelMacroEvent,
ChatMacroEvent,
DocumentMacroEvent,
+ EmailMacroEvent,
ProjectMacroEvent,
PropertyMacroEvent,
);
@@ -190,6 +193,9 @@ async fn process_event(
DeclaredMacroEvent::DocumentMacroEvent(event) => {
process_document_event(context, event, partition, offset).await
}
+ DeclaredMacroEvent::EmailMacroEvent(event) => {
+ process_email_event(db, opensearch_client, event, partition, offset).await
+ }
DeclaredMacroEvent::ProjectMacroEvent(event) => {
process_project_event(db, opensearch_client, event, partition, offset).await
}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/email.rs b/services/search_processing_service/src/inbound/kafka_consumer/email.rs
new file mode 100644
index 0000000000..b812cead3f
--- /dev/null
+++ b/services/search_processing_service/src/inbound/kafka_consumer/email.rs
@@ -0,0 +1,286 @@
+//! Maps email lifecycle events to search-index actions and processes them.
+
+use ::email::domain::events::{EmailMacroEvent, EmailTopicEvent};
+use macro_event_broker::MacroEvent as _;
+use opensearch_client::OpensearchClient;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};
+use crate::process::email::{remove, upsert};
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) enum EmailIndexAction {
+ UpsertMessage {
+ message_id: Uuid,
+ owner: String,
+ },
+ RemoveMessage {
+ message_id: Uuid,
+ },
+ ReindexThread {
+ thread_id: Uuid,
+ owner: String,
+ },
+ ReindexThreads {
+ thread_ids: Vec<Uuid>,
+ owner: String,
+ },
+ RemoveLink {
+ link_id: Uuid,
+ },
+ Ignore,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) struct EmailEventDescription {
+ pub(super) action: EmailIndexAction,
+ pub(super) link_id: Uuid,
+ pub(super) event_type: &'static str,
+}
+
+pub(super) fn describe_email_event(event: &EmailTopicEvent) -> EmailEventDescription {
+ match event {
+ EmailTopicEvent::LinkConnected(metadata) => EmailEventDescription {
+ action: EmailIndexAction::Ignore,
+ link_id: metadata.link_id,
+ event_type: "email.link_connected",
+ },
+ EmailTopicEvent::LinkDisconnected(metadata) => EmailEventDescription {
+ action: EmailIndexAction::RemoveLink {
+ link_id: metadata.link_id,
+ },
+ link_id: metadata.link_id,
+ event_type: "email.link_disconnected",
+ },
+ EmailTopicEvent::LinkReauthRequired(metadata) => EmailEventDescription {
+ action: EmailIndexAction::Ignore,
+ link_id: metadata.link_id,
+ event_type: "email.link_reauth_required",
+ },
+ EmailTopicEvent::MessageReceived(metadata) => EmailEventDescription {
+ action: if metadata.is_spam_or_trash {
+ EmailIndexAction::RemoveMessage {
+ message_id: metadata.message_id,
+ }
+ } else {
+ EmailIndexAction::UpsertMessage {
+ message_id: metadata.message_id,
+ owner: metadata.owner.to_string(),
+ }
+ },
+ link_id: metadata.link_id,
+ event_type: "email.message_received",
+ },
+ EmailTopicEvent::MessageDraftSynced(metadata) => EmailEventDescription {
+ action: if metadata.is_spam_or_trash {
+ EmailIndexAction::RemoveMessage {
+ message_id: metadata.message_id,
+ }
+ } else {
+ EmailIndexAction::UpsertMessage {
+ message_id: metadata.message_id,
+ owner: metadata.owner.to_string(),
+ }
+ },
+ link_id: metadata.link_id,
+ event_type: "email.message_draft_synced",
+ },
+ EmailTopicEvent::MessageSent(metadata) => EmailEventDescription {
+ action: EmailIndexAction::UpsertMessage {
+ message_id: metadata.message_id,
+ owner: metadata.owner.to_string(),
+ },
+ link_id: metadata.link_id,
+ event_type: "email.message_sent",
+ },
+ EmailTopicEvent::MessageDeleted(metadata) => EmailEventDescription {
+ action: EmailIndexAction::RemoveMessage {
+ message_id: metadata.message_id,
+ },
+ link_id: metadata.link_id,
+ event_type: "email.message_deleted",
+ },
+ EmailTopicEvent::MessageSendQueued(metadata) => EmailEventDescription {
+ action: EmailIndexAction::Ignore,
+ link_id: metadata.link_id,
+ event_type: "email.message_send_queued",
+ },
+ EmailTopicEvent::MessageSendCancelled(metadata) => EmailEventDescription {
+ action: EmailIndexAction::Ignore,
+ link_id: metadata.link_id,
+ event_type: "email.message_send_cancelled",
+ },
+ EmailTopicEvent::ThreadArchived(metadata) => EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: metadata.thread_id,
+ owner: metadata.owner.to_string(),
+ },
+ link_id: metadata.link_id,
+ event_type: "email.thread_archived",
+ },
+ EmailTopicEvent::ThreadTrashed(metadata) => EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: metadata.thread_id,
+ owner: metadata.owner.to_string(),
+ },
+ link_id: metadata.link_id,
+ event_type: "email.thread_trashed",
+ },
+ EmailTopicEvent::ThreadRead(metadata) => EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: metadata.thread_id,
+ owner: metadata.owner.to_string(),
+ },
+ link_id: metadata.link_id,
+ event_type: "email.thread_read",
+ },
+ EmailTopicEvent::ThreadStarred(metadata) => EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: metadata.thread_id,
+ owner: metadata.owner.to_string(),
+ },
+ link_id: metadata.link_id,
+ event_type: "email.thread_starred",
+ },
+ EmailTopicEvent::ThreadSpamChanged(metadata) => EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: metadata.thread_id,
+ owner: metadata.owner.to_string(),
+ },
+ link_id: metadata.link_id,
+ event_type: "email.thread_spam_changed",
+ },
+ EmailTopicEvent::ThreadProjectChanged(metadata) => EmailEventDescription {
+ action: EmailIndexAction::Ignore,
+ link_id: metadata.link_id,
+ event_type: "email.thread_project_changed",
+ },
+ EmailTopicEvent::ThreadLabelsUpdated(metadata) => EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: metadata.thread_id,
+ owner: metadata.owner.to_string(),
+ },
+ link_id: metadata.link_id,
+ event_type: "email.thread_labels_updated",
+ },
+ EmailTopicEvent::ThreadBackfilled(metadata) => EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: metadata.thread_id,
+ owner: metadata.owner.to_string(),
+ },
+ link_id: metadata.link_id,
+ event_type: "email.thread_backfilled",
+ },
+ EmailTopicEvent::ThreadsReindexRequested(metadata) => EmailEventDescription {
+ action: EmailIndexAction::ReindexThreads {
+ thread_ids: metadata.thread_ids.clone(),
+ owner: metadata.owner.to_string(),
+ },
+ link_id: metadata.link_id,
+ event_type: "email.threads_reindex_requested",
+ },
+ }
+}
+
+async fn process_email_index_action(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ action: EmailIndexAction,
+) -> anyhow::Result<()> {
+ match action {
+ EmailIndexAction::UpsertMessage { message_id, owner } => {
+ upsert::process_upsert_message(opensearch_client, db, message_id, &owner).await
+ }
+ EmailIndexAction::RemoveMessage { message_id } => {
+ remove::process_remove_message(opensearch_client, message_id, None).await
+ }
+ EmailIndexAction::ReindexThread { thread_id, owner } => {
+ upsert::process_upsert_thread_message(opensearch_client, db, thread_id, &owner, None)
+ .await
+ }
+ EmailIndexAction::ReindexThreads { thread_ids, owner } => {
+ upsert::process_upsert_thread_batch_message(
+ opensearch_client,
+ db,
+ &thread_ids,
+ &owner,
+ None,
+ )
+ .await
+ }
+ EmailIndexAction::RemoveLink { link_id } => {
+ remove::process_remove_messages_by_link_id(opensearch_client, link_id).await
+ }
+ EmailIndexAction::Ignore => Ok(()),
+ }
+}
+
+pub(super) async fn process_email_event(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ event: &EmailMacroEvent,
+ partition: i32,
+ offset: i64,
+) -> EventOutcome {
+ let description = describe_email_event(&event.event().event);
+ if description.action == EmailIndexAction::Ignore {
+ tracing::trace!(
+ link_id = %description.link_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ "ignoring email event without a search-index action"
+ );
+ return EventOutcome::Ignored;
+ }
+
+ let result = retry_processing(|attempt| {
+ let action = description.action.clone();
+ async move {
+ tracing::trace!(
+ link_id = %description.link_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ "processing email search-index event"
+ );
+ process_email_index_action(db, opensearch_client, action)
+ .await
+ .inspect_err(|error| {
+ if attempt < MAX_PROCESSING_ATTEMPTS {
+ let retry_delay =
+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
+ tracing::warn!(
+ error = ?error,
+ link_id = %description.link_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ delay_secs = retry_delay.as_secs(),
+ "email search-index processing failed, retrying"
+ );
+ }
+ })
+ }
+ })
+ .await;
+
+ match result {
+ Ok(()) => EventOutcome::Indexed,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ link_id = %description.link_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempts = MAX_PROCESSING_ATTEMPTS,
+ "dropping email event after processing retries were exhausted"
+ );
+ EventOutcome::Dropped
+ }
+ }
+}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
index cf8daa25e0..2809077124 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
@@ -11,6 +11,15 @@ use ::chat::domain::events::{
ChatMessageRole, ChatMessageSentMetadata, ChatPermanentlyDeletedMetadata, ChatRestoredMetadata,
ChatTopicEvent, ChatUpdatedMetadata,
};
+use ::email::domain::events::{
+ EmailEventOrigin, EmailTopicEvent, LinkConnectedMetadata, LinkDisconnectReason,
+ LinkDisconnectedMetadata, LinkReauthRequiredMetadata, MessageDeletedMetadata,
+ MessageDraftSyncedMetadata, MessageReceivedMetadata, MessageSendCancelledMetadata,
+ MessageSendQueuedMetadata, MessageSentMetadata, SendCancelReason, ThreadArchivedMetadata,
+ ThreadBackfilledMetadata, ThreadLabelsUpdatedMetadata, ThreadProjectChangedMetadata,
+ ThreadReadMetadata, ThreadSpamChangedMetadata, ThreadStarredMetadata, ThreadTrashedMetadata,
+ ThreadsReindexReason, ThreadsReindexRequestedMetadata,
+};
use channels::domain::{
broker_events::{
ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,
@@ -30,8 +39,8 @@ use documents::domain::events::{
};
use macro_event_broker::{Event, EventBrokerError, MacroEvent as _, MessageParts};
use macro_event_topics::{
- MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroDocumentsTopic, MacroProjectsTopic,
- MacroPropertiesTopic, Topic as _,
+ MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroDocumentsTopic, MacroEmailTopic,
+ MacroProjectsTopic, MacroPropertiesTopic, Topic as _,
};
use macro_user_id::user_id::MacroUserIdStr;
use model::document::FileType;
@@ -57,6 +66,7 @@ use super::{
DocumentEventDescription, DocumentIndexAction, describe_document_event,
stored_extractor_message, sync_extractor_message,
},
+ email::{EmailEventDescription, EmailIndexAction, describe_email_event},
project::{
ProjectEventDescription, ProjectIndexAction, collect_project_ids, describe_project_event,
},
@@ -80,6 +90,9 @@ const PROPERTY_ENTITY_ID: &str = "property-entity-id";
const PROPERTY_DEFINITION_ID: Uuid = Uuid::from_u128(4);
const PROPERTY_OPTION_ID: Uuid = Uuid::from_u128(5);
const ENTITY_PROPERTY_ID: Uuid = Uuid::from_u128(6);
+const EMAIL_LINK_ID: Uuid = Uuid::from_u128(7);
+const EMAIL_THREAD_ID: Uuid = Uuid::from_u128(8);
+const SECOND_EMAIL_THREAD_ID: Uuid = Uuid::from_u128(9);
struct TestMessage {
topic: &'static str,
@@ -178,6 +191,334 @@ fn encoded_message<E: serde::Serialize>(
}
}
+fn received_email_event(is_spam_or_trash: bool) -> EmailTopicEvent {
+ EmailTopicEvent::MessageReceived(MessageReceivedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ message_id: MESSAGE_ID,
+ provider_message_id: "provider-message-id".to_string(),
+ thread_id: EMAIL_THREAD_ID,
+ provider_thread_id: "provider-thread-id".to_string(),
+ is_new_thread: true,
+ subject: Some("Subject".to_string()),
+ from_email: Some("[email protected]".to_string()),
+ from_name: Some("Sender".to_string()),
+ to_emails: vec!["[email protected]".to_string()],
+ attachment_count: 1,
+ is_spam_or_trash,
+ received_at: Some(Utc::now()),
+ })
+}
+
+fn draft_email_event(is_spam_or_trash: bool) -> EmailTopicEvent {
+ EmailTopicEvent::MessageDraftSynced(MessageDraftSyncedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ message_id: MESSAGE_ID,
+ provider_message_id: "provider-message-id".to_string(),
+ thread_id: EMAIL_THREAD_ID,
+ provider_thread_id: "provider-thread-id".to_string(),
+ is_spam_or_trash,
+ })
+}
+
+fn email_event_cases() -> Vec<(EmailTopicEvent, EmailEventDescription)> {
+ let owner = "macro|[email protected]".to_string();
+
+ vec![
+ (
+ EmailTopicEvent::LinkConnected(LinkConnectedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ email_address: "[email protected]".to_string(),
+ provider: "GMAIL".to_string(),
+ is_primary: true,
+ connected_at: Utc::now(),
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::Ignore,
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.link_connected",
+ },
+ ),
+ (
+ EmailTopicEvent::LinkDisconnected(LinkDisconnectedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ email_address: "[email protected]".to_string(),
+ reason: LinkDisconnectReason::ManuallyDisabled,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::RemoveLink {
+ link_id: EMAIL_LINK_ID,
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.link_disconnected",
+ },
+ ),
+ (
+ EmailTopicEvent::LinkReauthRequired(LinkReauthRequiredMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ email_address: "[email protected]".to_string(),
+ observed_at: Utc::now(),
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::Ignore,
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.link_reauth_required",
+ },
+ ),
+ (
+ received_email_event(false),
+ EmailEventDescription {
+ action: EmailIndexAction::UpsertMessage {
+ message_id: MESSAGE_ID,
+ owner: owner.clone(),
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.message_received",
+ },
+ ),
+ (
+ draft_email_event(true),
+ EmailEventDescription {
+ action: EmailIndexAction::RemoveMessage {
+ message_id: MESSAGE_ID,
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.message_draft_synced",
+ },
+ ),
+ (
+ EmailTopicEvent::MessageSent(MessageSentMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ actor: Some(user_id()),
+ message_id: MESSAGE_ID,
+ provider_message_id: "provider-message-id".to_string(),
+ thread_id: EMAIL_THREAD_ID,
+ provider_thread_id: "provider-thread-id".to_string(),
+ subject: Some("Subject".to_string()),
+ to_emails: vec!["[email protected]".to_string()],
+ cc_emails: vec![],
+ origin: EmailEventOrigin::UserAction,
+ sent_at: Utc::now(),
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::UpsertMessage {
+ message_id: MESSAGE_ID,
+ owner: owner.clone(),
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.message_sent",
+ },
+ ),
+ (
+ EmailTopicEvent::MessageDeleted(MessageDeletedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ message_id: MESSAGE_ID,
+ provider_message_id: "provider-message-id".to_string(),
+ thread_id: EMAIL_THREAD_ID,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::RemoveMessage {
+ message_id: MESSAGE_ID,
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.message_deleted",
+ },
+ ),
+ (
+ EmailTopicEvent::MessageSendQueued(MessageSendQueuedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ actor: Some(user_id()),
+ message_id: MESSAGE_ID,
+ thread_id: EMAIL_THREAD_ID,
+ scheduled_send_at: Utc::now(),
+ is_scheduled: false,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::Ignore,
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.message_send_queued",
+ },
+ ),
+ (
+ EmailTopicEvent::MessageSendCancelled(MessageSendCancelledMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ actor: Some(user_id()),
+ message_id: MESSAGE_ID,
+ thread_id: EMAIL_THREAD_ID,
+ reason: SendCancelReason::Undo,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::Ignore,
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.message_send_cancelled",
+ },
+ ),
+ (
+ EmailTopicEvent::ThreadArchived(ThreadArchivedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ actor: Some(user_id()),
+ thread_id: EMAIL_THREAD_ID,
+ archived: true,
+ origin: EmailEventOrigin::UserAction,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: EMAIL_THREAD_ID,
+ owner: owner.clone(),
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.thread_archived",
+ },
+ ),
+ (
+ EmailTopicEvent::ThreadTrashed(ThreadTrashedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ actor: None,
+ thread_id: EMAIL_THREAD_ID,
+ trashed: true,
+ origin: EmailEventOrigin::ProviderSync,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: EMAIL_THREAD_ID,
+ owner: owner.clone(),
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.thread_trashed",
+ },
+ ),
+ (
+ EmailTopicEvent::ThreadRead(ThreadReadMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ actor: None,
+ thread_id: EMAIL_THREAD_ID,
+ is_read: true,
+ origin: EmailEventOrigin::ProviderSync,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: EMAIL_THREAD_ID,
+ owner: owner.clone(),
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.thread_read",
+ },
+ ),
+ (
+ EmailTopicEvent::ThreadStarred(ThreadStarredMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ actor: None,
+ thread_id: EMAIL_THREAD_ID,
+ starred: true,
+ origin: EmailEventOrigin::ProviderSync,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: EMAIL_THREAD_ID,
+ owner: owner.clone(),
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.thread_starred",
+ },
+ ),
+ (
+ EmailTopicEvent::ThreadSpamChanged(ThreadSpamChangedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ actor: None,
+ thread_id: EMAIL_THREAD_ID,
+ spam: true,
+ origin: EmailEventOrigin::ProviderSync,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: EMAIL_THREAD_ID,
+ owner: owner.clone(),
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.thread_spam_changed",
+ },
+ ),
+ (
+ EmailTopicEvent::ThreadProjectChanged(ThreadProjectChangedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ actor: user_id(),
+ thread_id: EMAIL_THREAD_ID,
+ previous_project_id: Some("old-project".to_string()),
+ project_id: Some("new-project".to_string()),
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::Ignore,
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.thread_project_changed",
+ },
+ ),
+ (
+ EmailTopicEvent::ThreadLabelsUpdated(ThreadLabelsUpdatedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ actor: None,
+ thread_id: EMAIL_THREAD_ID,
+ added: vec![],
+ removed: vec![],
+ origin: EmailEventOrigin::ProviderSync,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: EMAIL_THREAD_ID,
+ owner: owner.clone(),
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.thread_labels_updated",
+ },
+ ),
+ (
+ EmailTopicEvent::ThreadBackfilled(ThreadBackfilledMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ thread_id: EMAIL_THREAD_ID,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::ReindexThread {
+ thread_id: EMAIL_THREAD_ID,
+ owner: owner.clone(),
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.thread_backfilled",
+ },
+ ),
+ (
+ EmailTopicEvent::ThreadsReindexRequested(ThreadsReindexRequestedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ thread_ids: vec![EMAIL_THREAD_ID, SECOND_EMAIL_THREAD_ID],
+ reason: ThreadsReindexReason::ContactsChanged,
+ }),
+ EmailEventDescription {
+ action: EmailIndexAction::ReindexThreads {
+ thread_ids: vec![EMAIL_THREAD_ID, SECOND_EMAIL_THREAD_ID],
+ owner,
+ },
+ link_id: EMAIL_LINK_ID,
+ event_type: "email.threads_reindex_requested",
+ },
+ ),
+ ]
+}
+
fn chat_event_cases() -> Vec<(ChatTopicEvent, ChatEventDescription<'static>)> {
let owner = user_id();
@@ -891,6 +1232,7 @@ fn subscribes_to_declared_search_processing_topics_with_durable_group() {
assert!(topics.contains(&MacroChannelsTopic::TOPIC_STR));
assert!(topics.contains(&MacroChatsTopic::TOPIC_STR));
assert!(topics.contains(&MacroDocumentsTopic::TOPIC_STR));
+ assert!(topics.contains(&MacroEmailTopic::TOPIC_STR));
assert!(topics.contains(&MacroProjectsTopic::TOPIC_STR));
assert!(topics.contains(&MacroPropertiesTopic::TOPIC_STR));
}
@@ -950,6 +1292,35 @@ fn maps_all_call_lifecycle_events_to_index_actions() {
);
}
+#[test]
+fn maps_all_email_lifecycle_events_to_index_actions() {
+ let cases = email_event_cases();
+ assert_eq!(cases.len(), 18);
+
+ for (event, expected) in cases {
+ let serialized = serde_json::to_value(&event).expect("serializable email event");
+ assert_eq!(serialized["event_type"], expected.event_type);
+ assert_eq!(describe_email_event(&event), expected);
+ }
+}
+
+#[test]
+fn email_message_sync_actions_remove_spam_or_trash_and_upsert_other_messages() {
+ assert_eq!(
+ describe_email_event(&received_email_event(true)).action,
+ EmailIndexAction::RemoveMessage {
+ message_id: MESSAGE_ID,
+ }
+ );
+ assert_eq!(
+ describe_email_event(&draft_email_event(false)).action,
+ EmailIndexAction::UpsertMessage {
+ message_id: MESSAGE_ID,
+ owner: "macro|[email protected]".to_string(),
+ }
+ );
+}
+
#[test]
fn maps_all_chat_lifecycle_events_to_index_actions() {
let cases = chat_event_cases();
@@ -1078,6 +1449,23 @@ fn project_id_collection_is_stable_and_drops_missing_or_empty_ids() {
);
}
+#[test]
+fn email_envelope_decodes_round_trip() {
+ let event = draft_email_event(false);
+ let message = encoded_message(
+ MacroEmailTopic::TOPIC_STR,
+ EMAIL_LINK_ID,
+ Event::new(event.clone()),
+ );
+
+ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable email event");
+ let DeclaredMacroEvent::EmailMacroEvent(decoded_event) = decoded else {
+ panic!("expected email event");
+ };
+ assert_eq!(decoded_event.key(), EMAIL_LINK_ID.to_string());
+ assert_eq!(decoded_event.event().event, event);
+}
+
#[test]
fn chat_envelope_decodes_round_trip() {
let event = ChatTopicEvent::MessageSent(ChatMessageSentMetadata {
@@ -1243,6 +1631,61 @@ fn exact_macro_documents_envelopes_decode_into_document_events() {
}
}
+#[tokio::test]
+async fn malformed_missing_key_and_unsupported_email_messages_are_commit_safe() {
+ let malformed = TestMessage {
+ topic: MacroEmailTopic::TOPIC_STR,
+ key: Some(EMAIL_LINK_ID.to_string()),
+ payload: Some(b"not json".to_vec()),
+ };
+ let malformed = attach_event_coordinates(DeclaredMacroEvent::decode(&malformed), 7, 60);
+
+ let missing_key = TestMessage {
+ topic: MacroEmailTopic::TOPIC_STR,
+ key: None,
+ payload: encoded_message(
+ MacroEmailTopic::TOPIC_STR,
+ EMAIL_LINK_ID,
+ Event::new(draft_email_event(false)),
+ )
+ .payload,
+ };
+ let missing_key = attach_event_coordinates(DeclaredMacroEvent::decode(&missing_key), 7, 61);
+
+ let unsupported = encoded_message(
+ MacroEmailTopic::TOPIC_STR,
+ EMAIL_LINK_ID,
+ Event::with_schema_version(draft_email_event(false), 2),
+ );
+ let unsupported = attach_event_coordinates(DeclaredMacroEvent::decode(&unsupported), 7, 62);
+ let (sender, mut receiver) = mpsc::channel(1);
+
+ assert!(matches!(
+ handoff_decoded(&sender, malformed).await,
+ HandoffOutcome::MalformedRecord(EventBrokerError::Serialization(_))
+ ));
+ assert!(matches!(
+ handoff_decoded(&sender, missing_key).await,
+ HandoffOutcome::MalformedRecord(EventBrokerError::MissingMessageKey)
+ ));
+ match handoff_decoded(&sender, unsupported).await {
+ HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {
+ topic,
+ expected,
+ actual,
+ }) => {
+ assert_eq!(topic, MacroEmailTopic::TOPIC_STR);
+ assert_eq!(expected, 1);
+ assert_eq!(actual, 2);
+ }
+ outcome => panic!("expected malformed email record, got {outcome:?}"),
+ }
+ assert!(matches!(
+ receiver.try_recv(),
+ Err(mpsc::error::TryRecvError::Empty)
+ ));
+}
+
#[tokio::test]
async fn malformed_and_unsupported_chat_messages_are_commit_safe() {
let malformed = TestMessage {
diff --git a/services/search_processing_service/src/process/email/mod.rs b/services/search_processing_service/src/process/email/mod.rs
index 46831b422b..04324042f5 100644
--- a/services/search_processing_service/src/process/email/mod.rs
+++ b/services/search_processing_service/src/process/email/mod.rs
@@ -1,2 +1,2 @@
-pub mod remove;
-pub mod upsert;
+pub(crate) mod remove;
+pub(crate) mod upsert;
diff --git a/services/search_processing_service/src/process/email/remove.rs b/services/search_processing_service/src/process/email/remove.rs
index e68424e47f..67ea5513e9 100644
--- a/services/search_processing_service/src/process/email/remove.rs
+++ b/services/search_processing_service/src/process/email/remove.rs
@@ -1,12 +1,13 @@
use opensearch_client::OpensearchClient;
-use sqs_client::search::email::{EmailLinkMessage, EmailMessage};
+use uuid::Uuid;
pub async fn process_remove_message(
opensearch_client: &OpensearchClient,
- remove_message: &EmailMessage,
+ message_id: Uuid,
+ index_override: Option<&str>,
) -> anyhow::Result<()> {
opensearch_client
- .delete_email_message_by_id(remove_message.message_id.as_str())
+ .delete_email_message_by_id(&message_id.to_string(), index_override)
.await?;
Ok(())
@@ -14,10 +15,10 @@ pub async fn process_remove_message(
pub async fn process_remove_messages_by_link_id(
opensearch_client: &OpensearchClient,
- remove_link_message: &EmailLinkMessage,
+ link_id: Uuid,
) -> anyhow::Result<()> {
opensearch_client
- .delete_email_messages_by_link_id(remove_link_message.link_id.as_str())
+ .delete_email_messages_by_link_id(&link_id.to_string())
.await?;
Ok(())
diff --git a/services/search_processing_service/src/process/email/upsert.rs b/services/search_processing_service/src/process/email/upsert.rs
index 97a32fb232..9d50f17339 100644
--- a/services/search_processing_service/src/process/email/upsert.rs
+++ b/services/search_processing_service/src/process/email/upsert.rs
@@ -1,6 +1,6 @@
use anyhow::Context;
use chrono::Utc;
-use models_email::service::label::system_labels;
+use models_email::service::{label::system_labels, message::ParsedSearchMessage};
use models_properties::EntityType;
use opensearch_client::{
OpensearchClient, date_format::EpochMillis, upsert::email::UpsertEmailArgs,
@@ -9,21 +9,40 @@ use properties::outbound::entity_properties_get_query::{
get_entity_properties_for_index, get_entity_properties_for_index_batch,
};
use sqlx::PgPool;
-use sqs_client::search::email::{EmailMessage, EmailThreadBatchMessage, EmailThreadMessage};
use uuid::Uuid;
use crate::process::properties::to_indexed_properties;
+#[cfg(test)]
+mod test;
+
+#[derive(Debug, PartialEq, Eq)]
+enum ThreadMessageDisposition {
+ Delete,
+ Upsert,
+ SkipMissingContent,
+}
+
+fn classify_thread_message(message: &ParsedSearchMessage) -> ThreadMessageDisposition {
+ if message.labels.iter().any(|label| {
+ label.provider_id == system_labels::SPAM || label.provider_id == system_labels::TRASH
+ }) {
+ return ThreadMessageDisposition::Delete;
+ }
+
+ if message.body_parsed_linkless.is_none() {
+ return ThreadMessageDisposition::SkipMissingContent;
+ }
+
+ ThreadMessageDisposition::Upsert
+}
+
pub async fn process_upsert_message(
opensearch_client: &OpensearchClient,
db: &PgPool,
- upsert_email_message: &EmailMessage,
+ message_id: Uuid,
+ owner: &str,
) -> anyhow::Result<()> {
- let message_id: Uuid = upsert_email_message
- .message_id
- .parse()
- .context("failed to parse message_id as UUID")?;
-
let message_info =
email_db_client::messages::get_parsed_search::get_parsed_search_message_by_id(
db,
@@ -74,9 +93,9 @@ pub async fn process_upsert_message(
);
let upsert_email_message_args: UpsertEmailArgs = UpsertEmailArgs {
- message_id: upsert_email_message.message_id.clone(),
+ message_id: message_id.to_string(),
link_id: message_info.link_id.to_string(),
- user_id: upsert_email_message.macro_user_id.clone(),
+ user_id: owner.to_string(),
thread_id: message_info.thread_db_id.to_string(),
subject: message_info.subject,
sender: message_info
@@ -141,17 +160,14 @@ pub async fn process_upsert_message(
pub async fn process_upsert_thread_message(
opensearch_client: &OpensearchClient,
db: &PgPool,
- upsert_email_thread_message: &EmailThreadMessage,
+ thread_id: Uuid,
+ owner: &str,
+ index_override: Option<&str>,
) -> anyhow::Result<()> {
let mut message_offset = 0;
// Max is 100
let message_limit = 100;
- let thread_id: Uuid = upsert_email_thread_message
- .thread_id
- .parse()
- .context("failed to parse thread_id as UUID")?;
-
let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
// A full index overwrites each doc, so thread properties must ride along
@@ -178,150 +194,29 @@ pub async fn process_upsert_thread_message(
break;
}
+ let mut delete_message_ids = Vec::new();
let mut upsert_email_message_args = Vec::new();
for message in messages {
- // don't insert spam or trash messages
- if message.labels.iter().any(|label| {
- label.provider_id == system_labels::SPAM
- || label.provider_id == system_labels::TRASH
- }) {
- continue;
- }
-
- if let Some(content) = message.body_parsed_linkless {
- let sent_at_millis = message
- .internal_date_ts
- .map(|date| EpochMillis::new(date.timestamp_millis()))
- .transpose()?;
-
- let updated_at_millis = message
- .internal_date_ts
- .map(|date| EpochMillis::new(date.timestamp_millis()))
- .transpose()?
- .unwrap_or(now_millis);
-
- upsert_email_message_args.push(UpsertEmailArgs {
- message_id: message.db_id.to_string(),
- link_id: message.link_id.to_string(),
- user_id: upsert_email_thread_message.macro_user_id.clone(),
- thread_id: upsert_email_thread_message.thread_id.clone(),
- subject: message.subject,
- sender: message
- .from
- .as_ref()
- .map(|f| f.email.to_lowercase())
- .unwrap_or_default(),
- sender_name: message.from.as_ref().and_then(|f| f.name.clone()),
- reply_to: message.reply_to.map(|r| r.to_lowercase()),
- recipients: message
- .to
- .iter()
- .map(|to| to.email.to_lowercase())
- .collect(),
- recipient_names: message.to.iter().filter_map(|to| to.name.clone()).collect(),
- cc: message
- .cc
- .iter()
- .map(|cc| cc.email.to_lowercase())
- .collect(),
- cc_names: message.cc.iter().filter_map(|cc| cc.name.clone()).collect(),
- bcc: message
- .bcc
- .iter()
- .map(|bcc| bcc.email.to_lowercase())
- .collect(),
- bcc_names: message
- .bcc
- .iter()
- .filter_map(|bcc| bcc.name.clone())
- .collect(),
- labels: message
- .labels
- .iter()
- .map(|label| label.name.clone())
- .collect(),
- content,
- updated_at_millis,
- sent_at_millis,
- properties: properties.clone(),
- });
- } else {
- tracing::warn!("no content found for email message");
- }
- }
-
- if !upsert_email_message_args.is_empty() {
- let result = opensearch_client
- .bulk_upsert_email_messages(
- &upsert_email_message_args,
- upsert_email_thread_message.index_override.as_deref(),
- )
- .await?;
-
- if result.failed > 0 {
- tracing::warn!(
- failed = result.failed,
- errors = ?result.errors,
- "some email messages failed to upsert"
- );
+ match classify_thread_message(&message) {
+ ThreadMessageDisposition::Delete => {
+ delete_message_ids.push(message.db_id);
+ continue;
+ }
+ ThreadMessageDisposition::SkipMissingContent => {
+ tracing::warn!(message_id = %message.db_id, "no content found for email message");
+ continue;
+ }
+ ThreadMessageDisposition::Upsert => {}
}
- }
-
- // Update offset
- message_offset += message_limit;
- }
-
- Ok(())
-}
-
-pub async fn process_upsert_thread_batch_message(
- opensearch_client: &OpensearchClient,
- db: &PgPool,
- batch_message: &EmailThreadBatchMessage,
-) -> anyhow::Result<()> {
- let thread_ids: Vec<Uuid> = batch_message
- .thread_ids
- .iter()
- .map(|id| id.parse().context("failed to parse thread_id as UUID"))
- .collect::<anyhow::Result<Vec<_>>>()?;
-
- let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
-
- let messages =
- email_db_client::messages::get_parsed_search::get_parsed_search_messages_by_thread_ids(
- db,
- &thread_ids,
- )
- .await
- .context("failed to get batch thread messages")?;
-
- // A full index overwrites each doc, so thread properties must ride along
- // or a reindex would drop them.
- let thread_id_strings: Vec<String> = thread_ids.iter().map(Uuid::to_string).collect();
- let properties_by_thread =
- get_entity_properties_for_index_batch(db, &thread_id_strings, EntityType::Thread)
- .await
- .context("failed to fetch thread properties for search index")?
- .into_iter()
- .map(|(thread_id, properties)| (thread_id, to_indexed_properties(properties)))
- .collect::<std::collections::HashMap<_, _>>();
-
- let mut upsert_email_message_args = Vec::new();
-
- for message in messages {
- if message.labels.iter().any(|label| {
- label.provider_id == system_labels::SPAM || label.provider_id == system_labels::TRASH
- }) {
- continue;
- }
- if let Some(content) = message.body_parsed_linkless {
+ let content = message
+ .body_parsed_linkless
+ .context("expected content for upsertable email message")?;
let sent_at_millis = message
.internal_date_ts
.map(|date| EpochMillis::new(date.timestamp_millis()))
.transpose()?;
-
let updated_at_millis = message
.internal_date_ts
.map(|date| EpochMillis::new(date.timestamp_millis()))
@@ -331,8 +226,8 @@ pub async fn process_upsert_thread_batch_message(
upsert_email_message_args.push(UpsertEmailArgs {
message_id: message.db_id.to_string(),
link_id: message.link_id.to_string(),
- user_id: batch_message.macro_user_id.clone(),
- thread_id: message.thread_db_id.to_string(),
+ user_id: owner.to_string(),
+ thread_id: thread_id.to_string(),
subject: message.subject,
sender: message
.from
@@ -371,22 +266,152 @@ pub async fn process_upsert_thread_batch_message(
content,
updated_at_millis,
sent_at_millis,
- properties: properties_by_thread
- .get(&message.thread_db_id.to_string())
- .cloned()
- .unwrap_or_default(),
+ properties: properties.clone(),
});
- } else {
- tracing::warn!("no content found for email message");
}
+
+ for message_id in delete_message_ids {
+ opensearch_client
+ .delete_email_message_by_id(&message_id.to_string(), index_override)
+ .await?;
+ }
+
+ if !upsert_email_message_args.is_empty() {
+ let result = opensearch_client
+ .bulk_upsert_email_messages(&upsert_email_message_args, index_override)
+ .await?;
+
+ if result.failed > 0 {
+ tracing::warn!(
+ failed = result.failed,
+ errors = ?result.errors,
+ "some email messages failed to upsert"
+ );
+ }
+ }
+
+ // Update offset
+ message_offset += message_limit;
+ }
+
+ Ok(())
+}
+
+pub async fn process_upsert_thread_batch_message(
+ opensearch_client: &OpensearchClient,
+ db: &PgPool,
+ thread_ids: &[Uuid],
+ owner: &str,
+ index_override: Option<&str>,
+) -> anyhow::Result<()> {
+ let now_millis = EpochMillis::new(Utc::now().timestamp_millis())?;
+
+ let messages =
+ email_db_client::messages::get_parsed_search::get_parsed_search_messages_by_thread_ids(
+ db, thread_ids,
+ )
+ .await
+ .context("failed to get batch thread messages")?;
+
+ // A full index overwrites each doc, so thread properties must ride along
+ // or a reindex would drop them.
+ let thread_id_strings: Vec<String> = thread_ids.iter().map(Uuid::to_string).collect();
+ let properties_by_thread =
+ get_entity_properties_for_index_batch(db, &thread_id_strings, EntityType::Thread)
+ .await
+ .context("failed to fetch thread properties for search index")?
+ .into_iter()
+ .map(|(thread_id, properties)| (thread_id, to_indexed_properties(properties)))
+ .collect::<std::collections::HashMap<_, _>>();
+
+ let mut delete_message_ids = Vec::new();
+ let mut upsert_email_message_args = Vec::new();
+
+ for message in messages {
+ match classify_thread_message(&message) {
+ ThreadMessageDisposition::Delete => {
+ delete_message_ids.push(message.db_id);
+ continue;
+ }
+ ThreadMessageDisposition::SkipMissingContent => {
+ tracing::warn!(message_id = %message.db_id, "no content found for email message");
+ continue;
+ }
+ ThreadMessageDisposition::Upsert => {}
+ }
+
+ let content = message
+ .body_parsed_linkless
+ .context("expected content for upsertable email message")?;
+ let sent_at_millis = message
+ .internal_date_ts
+ .map(|date| EpochMillis::new(date.timestamp_millis()))
+ .transpose()?;
+ let updated_at_millis = message
+ .internal_date_ts
+ .map(|date| EpochMillis::new(date.timestamp_millis()))
+ .transpose()?
+ .unwrap_or(now_millis);
+
+ upsert_email_message_args.push(UpsertEmailArgs {
+ message_id: message.db_id.to_string(),
+ link_id: message.link_id.to_string(),
+ user_id: owner.to_string(),
+ thread_id: message.thread_db_id.to_string(),
+ subject: message.subject,
+ sender: message
+ .from
+ .as_ref()
+ .map(|f| f.email.to_lowercase())
+ .unwrap_or_default(),
+ sender_name: message.from.as_ref().and_then(|f| f.name.clone()),
+ reply_to: message.reply_to.map(|r| r.to_lowercase()),
+ recipients: message
+ .to
+ .iter()
+ .map(|to| to.email.to_lowercase())
+ .collect(),
+ recipient_names: message.to.iter().filter_map(|to| to.name.clone()).collect(),
+ cc: message
+ .cc
+ .iter()
+ .map(|cc| cc.email.to_lowercase())
+ .collect(),
+ cc_names: message.cc.iter().filter_map(|cc| cc.name.clone()).collect(),
+ bcc: message
+ .bcc
+ .iter()
+ .map(|bcc| bcc.email.to_lowercase())
+ .collect(),
+ bcc_names: message
+ .bcc
+ .iter()
+ .filter_map(|bcc| bcc.name.clone())
+ .collect(),
+ labels: message
+ .labels
+ .iter()
+ .map(|label| label.name.clone())
+ .collect(),
+ content,
+ updated_at_millis,
+ sent_at_millis,
+ properties: properties_by_thread
+ .get(&message.thread_db_id.to_string())
+ .cloned()
+ .unwrap_or_default(),
+ });
+ }
+
+ for message_id in delete_message_ids {
+ opensearch_client
+ .delete_email_message_by_id(&message_id.to_string(), index_override)
+ .await?;
}
if !upsert_email_message_args.is_empty() {
let result = opensearch_client
- .bulk_upsert_email_messages(
- &upsert_email_message_args,
- batch_message.index_override.as_deref(),
- )
+ .bulk_upsert_email_messages(&upsert_email_message_args, index_override)
.await?;
if result.failed > 0 {
diff --git a/services/search_processing_service/src/process/email/upsert/test.rs b/services/search_processing_service/src/process/email/upsert/test.rs
new file mode 100644
index 0000000000..41e7aa8d83
--- /dev/null
+++ b/services/search_processing_service/src/process/email/upsert/test.rs
@@ -0,0 +1,49 @@
+use models_email::service::{label::LabelInfo, message::ParsedSearchMessage};
+
+use super::*;
+
+fn message(label: Option<&str>, content: Option<&str>) -> ParsedSearchMessage {
+ ParsedSearchMessage {
+ db_id: Uuid::new_v4(),
+ link_id: Uuid::new_v4(),
+ thread_db_id: Uuid::new_v4(),
+ subject: None,
+ from: None,
+ reply_to: None,
+ to: Vec::new(),
+ cc: Vec::new(),
+ bcc: Vec::new(),
+ labels: label
+ .map(|provider_id| {
+ vec![LabelInfo {
+ provider_id: provider_id.to_string(),
+ name: provider_id.to_string(),
+ }]
+ })
+ .unwrap_or_default(),
+ body_parsed_linkless: content.map(str::to_string),
+ internal_date_ts: None,
+ }
+}
+
+#[test]
+fn classifies_mixed_thread_messages() {
+ let messages = [
+ message(Some(system_labels::SPAM), Some("spam")),
+ message(Some(system_labels::TRASH), None),
+ message(Some(system_labels::INBOX), None),
+ message(Some(system_labels::INBOX), Some("ordinary")),
+ ];
+
+ let dispositions: Vec<_> = messages.iter().map(classify_thread_message).collect();
+
+ assert_eq!(
+ dispositions,
+ vec![
+ ThreadMessageDisposition::Delete,
+ ThreadMessageDisposition::Delete,
+ ThreadMessageDisposition::SkipMissingContent,
+ ThreadMessageDisposition::Upsert,
+ ]
+ );
+}
diff --git a/services/search_processing_service/src/process/mod.rs b/services/search_processing_service/src/process/mod.rs
index 9e62159a80..f996a3124b 100644
--- a/services/search_processing_service/src/process/mod.rs
+++ b/services/search_processing_service/src/process/mod.rs
@@ -3,7 +3,7 @@ pub(crate) mod channel;
pub(crate) mod chat;
pub mod context;
pub(crate) mod document;
-mod email;
+pub(crate) mod email;
pub(crate) mod project;
pub(crate) mod properties;
mod user;
@@ -57,29 +57,25 @@ pub async fn process_message(
)
.await?;
}
- SearchQueueMessage::RemoveEmailLink(message) => {
- email::remove::process_remove_messages_by_link_id(&ctx.opensearch_client, &message)
- .await?;
- }
- SearchQueueMessage::ExtractEmailThreadMessage(message) => {
- email::upsert::process_upsert_thread_message(&ctx.opensearch_client, &ctx.db, &message)
- .await?;
- }
SearchQueueMessage::ExtractEmailThreadBatch(message) => {
+ let thread_ids = message
+ .thread_ids
+ .iter()
+ .map(|thread_id| {
+ thread_id
+ .parse::<Uuid>()
+ .context("failed to parse thread_id as UUID")
+ })
+ .collect::<anyhow::Result<Vec<_>>>()?;
email::upsert::process_upsert_thread_batch_message(
&ctx.opensearch_client,
&ctx.db,
- &message,
+ &thread_ids,
+ &message.macro_user_id,
+ message.index_override.as_deref(),
)
.await?;
}
- SearchQueueMessage::RemoveEmailMessage(message) => {
- email::remove::process_remove_message(&ctx.opensearch_client, &message).await?;
- }
- SearchQueueMessage::ExtractEmailMessage(message) => {
- email::upsert::process_upsert_message(&ctx.opensearch_client, &ctx.db, &message)
- .await?;
- }
SearchQueueMessage::ExtractDocumentText(message) => {
document::process_extract_text_message(
&ctx.opensearch_client,