Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(search_processing_service): parallelize workers (#5421)

Will Hutchinson261948d5a742
diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rs
index 607d374cab..6ac6eec0c7 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs
@@ -1,10 +1,13 @@
//! Kafka consumer for broker events that update the search index.
//!
-//! The poll loop hands decoded events to one bounded, sequential worker and
-//! commits each offset immediately after that handoff. Malformed records are
-//! committed without a handoff so they cannot wedge a partition. Processing is
-//! retried in-process; exhausted events are logged and dropped because their
-//! offsets are already committed.
+//! The poll loop shards each decoded event by its ordering key onto one of a
+//! fixed pool of bounded, sequential workers and commits the offset
+//! immediately after that handoff. Events sharing an ordering key — the
+//! entity whose search documents they mutate — stay on one worker in
+//! partition order; unrelated events process in parallel. Malformed records
+//! are committed without a handoff so they cannot wedge a partition.
+//! Processing is retried in-process; exhausted events are logged and dropped
+//! because their offsets are already committed.
//!
//! Per-entity event mapping and processing live in the [`call`], [`channel`],
//! [`chat`], [`document`], [`email`], [`project`], and [`property`] submodules;
@@ -23,7 +26,13 @@ mod property;
#[cfg(test)]
mod test;
-use std::{future::Future, time::Duration};
+use std::{
+ borrow::Cow,
+ collections::hash_map::DefaultHasher,
+ future::Future,
+ hash::{Hash as _, Hasher as _},
+ time::Duration,
+};
use ::call::domain::events::CallMacroEvent;
use ::chat::domain::events::ChatMacroEvent;
@@ -31,7 +40,9 @@ use ::email::domain::events::EmailMacroEvent;
use channels::domain::broker_events::ChannelMacroEvent;
use documents::domain::events::DocumentMacroEvent;
use kafka_util::{GroupName, KafkaEventConsumer};
-use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};
+use macro_event_broker::{
+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection, MacroEventConsumerService,
+};
use projects::domain::events::ProjectMacroEvent;
use properties::domain::events::PropertyMacroEvent;
use rdkafka::{
@@ -39,12 +50,16 @@ use rdkafka::{
message::{BorrowedMessage, Message as _},
};
use rootcause::prelude::{Report, ResultExt as _};
-use tokio::sync::mpsc;
+use tokio::{sync::mpsc, task::JoinHandle};
use tokio_retry::{Retry, strategy::ExponentialBackoff};
use self::{
- call::process_call_event, channel::process_channel_event, chat::process_chat_event,
- document::process_document_event, email::process_email_event, project::process_project_event,
+ call::process_call_event,
+ channel::process_channel_event,
+ chat::process_chat_event,
+ document::process_document_event,
+ email::{email_ordering_key, process_email_event},
+ project::process_project_event,
property::process_property_event,
};
@@ -73,8 +88,16 @@ macro_event_broker::declare_topics!(
PropertyMacroEvent,
);
-/// Maximum number of decoded events waiting for the sequential worker.
-const CHANNEL_CAPACITY: usize = 128;
+/// Number of sequential workers events are sharded across by ordering key.
+///
+/// Each in-flight event holds a database connection and an OpenSearch
+/// request, so this also bounds the consumer's downstream concurrency.
+const WORKER_COUNT: usize = 10;
+/// Maximum number of decoded events waiting for each sequential worker.
+///
+/// Offsets are committed at handoff, so `WORKER_COUNT * WORKER_CHANNEL_CAPACITY`
+/// also bounds how many committed-but-unprocessed events a crash can drop.
+const WORKER_CHANNEL_CAPACITY: usize = 16;
/// Delay before polling Kafka again after a receive error.
const RECEIVE_ERROR_RETRY_DELAY: Duration = Duration::from_secs(1);
/// Total processing attempts before an event is dropped.
@@ -143,18 +166,75 @@ where
retry_processing_with_strategy(processing_retry_strategy(), operation).await
}
-async fn handoff_decoded<T, E>(
- sender: &mpsc::Sender<T>,
- decoded: Result<T, E>,
-) -> HandoffOutcome<E> {
- let event = match decoded {
- Ok(event) => event,
- Err(error) => return HandoffOutcome::MalformedRecord(error),
- };
-
- match sender.send(event).await {
- Ok(()) => HandoffOutcome::HandedOff,
- Err(_) => HandoffOutcome::WorkerClosed,
+/// Sharding key that serializes all events mutating the same search documents.
+///
+/// Every topic's producer key is already the entity its index actions target
+/// (call, channel, chat, document, project, or property-entity id), except
+/// email events, which are produced with a link-scoped key and instead shard
+/// by thread so one inbox's backlog can spread across the worker pool.
+fn ordering_key(event: &DeclaredMacroEvent) -> Cow<'_, str> {
+ match event {
+ DeclaredMacroEvent::CallMacroEvent(event) => Cow::Borrowed(event.key()),
+ DeclaredMacroEvent::ChannelMacroEvent(event) => Cow::Borrowed(event.key()),
+ DeclaredMacroEvent::ChatMacroEvent(event) => Cow::Borrowed(event.key()),
+ DeclaredMacroEvent::DocumentMacroEvent(event) => Cow::Borrowed(event.key()),
+ DeclaredMacroEvent::EmailMacroEvent(event) => email_ordering_key(event),
+ DeclaredMacroEvent::ProjectMacroEvent(event) => Cow::Borrowed(event.key()),
+ DeclaredMacroEvent::PropertyMacroEvent(event) => Cow::Borrowed(event.key()),
+ }
+}
+
+/// Fixed pool of sequential workers, sharded by [`ordering_key`].
+///
+/// Events with the same ordering key always land on the same worker's FIFO
+/// channel, so they process in handoff (partition) order; events with
+/// different keys may process concurrently.
+struct WorkerPool {
+ senders: Vec<mpsc::Sender<ReceivedEvent>>,
+}
+
+impl WorkerPool {
+ fn new(senders: Vec<mpsc::Sender<ReceivedEvent>>) -> Self {
+ assert!(
+ !senders.is_empty(),
+ "worker pool requires at least one worker"
+ );
+ Self { senders }
+ }
+
+ /// Spawns [`WORKER_COUNT`] workers sharing `context` and returns their
+ /// join handles alongside the pool. Dropping the pool closes every worker
+ /// channel, letting the workers drain and exit.
+ fn spawn(context: &KafkaProcessingContext) -> (Self, Vec<JoinHandle<()>>) {
+ let (senders, workers) = (0..WORKER_COUNT)
+ .map(|_| {
+ let (sender, receiver) = mpsc::channel(WORKER_CHANNEL_CAPACITY);
+ (
+ sender,
+ tokio::spawn(run_event_worker(context.clone(), receiver)),
+ )
+ })
+ .unzip();
+ (Self::new(senders), workers)
+ }
+
+ fn shard_index(&self, key: &str) -> usize {
+ let mut hasher = DefaultHasher::new();
+ key.hash(&mut hasher);
+ (hasher.finish() % self.senders.len() as u64) as usize
+ }
+
+ async fn handoff<E>(&self, decoded: Result<ReceivedEvent, E>) -> HandoffOutcome<E> {
+ let event = match decoded {
+ Ok(event) => event,
+ Err(error) => return HandoffOutcome::MalformedRecord(error),
+ };
+
+ let shard = self.shard_index(ordering_key(&event.event).as_ref());
+ match self.senders[shard].send(event).await {
+ Ok(()) => HandoffOutcome::HandedOff,
+ Err(_) => HandoffOutcome::WorkerClosed,
+ }
}
}
@@ -241,7 +321,7 @@ fn commit_logged(consumer: &SearchProcessingKafkaConsumer, message: &BorrowedMes
async fn poll_events(
consumer: &SearchProcessingKafkaConsumer,
- events: mpsc::Sender<ReceivedEvent>,
+ workers: WorkerPool,
shutdown: impl Future<Output = ()> + Send,
) -> Result<(), Report> {
let mut shutdown = std::pin::pin!(shutdown);
@@ -266,7 +346,7 @@ async fn poll_events(
kafka_message.offset(),
);
- match handoff_decoded(&events, decoded).await {
+ match workers.handoff(decoded).await {
HandoffOutcome::HandedOff => {}
HandoffOutcome::MalformedRecord(error) => tracing::error!(
error = ?error,
@@ -301,12 +381,13 @@ async fn poll_events(
Ok(())
}
-/// Runs the bounded broker event poll-loop/worker pair until `shutdown` resolves.
+/// Runs the bounded broker event poll loop and worker pool until `shutdown`
+/// resolves.
///
/// The caller is responsible for supervising this function. A closed worker
/// channel returns an error without committing the current Kafka message. On
-/// normal shutdown the sender is dropped and all buffered events are processed
-/// before this function returns.
+/// normal shutdown the senders are dropped and all buffered events are
+/// processed before this function returns.
#[tracing::instrument(skip(context, shutdown), fields(brokers), err)]
pub(crate) async fn run_event_consumer(
brokers: &str,
@@ -324,12 +405,21 @@ pub(crate) async fn run_event_consumer(
"search processing event consumer listening"
);
- let (events_tx, events_rx) = mpsc::channel(CHANNEL_CAPACITY);
- let worker = tokio::spawn(run_event_worker(context, events_rx));
- let poll_result = poll_events(&consumer, events_tx, shutdown).await;
- let worker_result = worker.await;
+ let (pool, workers) = WorkerPool::spawn(&context);
+ // Passing the pool by value drops every sender when polling stops, which
+ // closes the worker channels so the workers drain and exit.
+ let poll_result = poll_events(&consumer, pool, shutdown).await;
+
+ let mut worker_failure = None;
+ for worker in workers {
+ if let Err(error) = worker.await {
+ worker_failure = Some(error);
+ }
+ }
poll_result?;
- worker_result.map_err(|error| rootcause::report!(error))?;
+ if let Some(error) = worker_failure {
+ Err(rootcause::report!(error))?;
+ }
Ok(())
}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/email.rs b/services/search_processing_service/src/inbound/kafka_consumer/email.rs
index b812cead3f..20d60ff02a 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/email.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/email.rs
@@ -1,5 +1,7 @@
//! Maps email lifecycle events to search-index actions and processes them.
+use std::borrow::Cow;
+
use ::email::domain::events::{EmailMacroEvent, EmailTopicEvent};
use macro_event_broker::MacroEvent as _;
use opensearch_client::OpensearchClient;
@@ -39,6 +41,39 @@ pub(super) struct EmailEventDescription {
pub(super) event_type: &'static str,
}
+/// Worker-sharding key for an email event.
+///
+/// Email events are produced with a link-scoped (whole inbox) Kafka key, so
+/// thread- and message-scoped events shard by thread id instead: one inbox's
+/// backfill spreads across the worker pool while events for a single thread
+/// stay ordered. Link-scoped events keep the link-id producer key, so their
+/// ordering against a buffered thread event (e.g. a link removal racing a
+/// message upsert) is not guaranteed — an accepted risk of the same order as
+/// this consumer's existing commit-at-handoff drop window.
+pub(super) fn email_ordering_key(event: &EmailMacroEvent) -> Cow<'_, str> {
+ let thread_id = match &event.event().event {
+ EmailTopicEvent::LinkConnected(_)
+ | EmailTopicEvent::LinkDisconnected(_)
+ | EmailTopicEvent::LinkReauthRequired(_)
+ | EmailTopicEvent::ThreadsReindexRequested(_) => return Cow::Borrowed(event.key()),
+ EmailTopicEvent::MessageReceived(metadata) => metadata.thread_id,
+ EmailTopicEvent::MessageDraftSynced(metadata) => metadata.thread_id,
+ EmailTopicEvent::MessageSent(metadata) => metadata.thread_id,
+ EmailTopicEvent::MessageDeleted(metadata) => metadata.thread_id,
+ EmailTopicEvent::MessageSendQueued(metadata) => metadata.thread_id,
+ EmailTopicEvent::MessageSendCancelled(metadata) => metadata.thread_id,
+ EmailTopicEvent::ThreadArchived(metadata) => metadata.thread_id,
+ EmailTopicEvent::ThreadTrashed(metadata) => metadata.thread_id,
+ EmailTopicEvent::ThreadRead(metadata) => metadata.thread_id,
+ EmailTopicEvent::ThreadStarred(metadata) => metadata.thread_id,
+ EmailTopicEvent::ThreadSpamChanged(metadata) => metadata.thread_id,
+ EmailTopicEvent::ThreadProjectChanged(metadata) => metadata.thread_id,
+ EmailTopicEvent::ThreadLabelsUpdated(metadata) => metadata.thread_id,
+ EmailTopicEvent::ThreadBackfilled(metadata) => metadata.thread_id,
+ };
+ Cow::Owned(thread_id.to_string())
+}
+
pub(super) fn describe_email_event(event: &EmailTopicEvent) -> EmailEventDescription {
match event {
EmailTopicEvent::LinkConnected(metadata) => EmailEventDescription {
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
index 30f8db0b49..d99dfd3797 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
@@ -12,13 +12,14 @@ use ::chat::domain::events::{
ChatTopicEvent, ChatUpdatedMetadata,
};
use ::email::domain::events::{
- EmailEventOrigin, EmailTopicEvent, LinkConnectedMetadata, LinkDisconnectReason,
- LinkDisconnectedMetadata, LinkReauthRequiredMetadata, MessageDeletedMetadata,
- MessageDraftSyncedMetadata, MessageReceivedMetadata, MessageSendCancelledMetadata,
- MessageSendQueuedMetadata, MessageSentMetadata, SendCancelReason, ThreadArchivedMetadata,
- ThreadBackfilledMetadata, ThreadLabelsUpdatedMetadata, ThreadProjectChangedMetadata,
- ThreadReadMetadata, ThreadSpamChangedMetadata, ThreadStarredMetadata, ThreadTrashedMetadata,
- ThreadsReindexReason, ThreadsReindexRequestedMetadata,
+ EmailEventOrigin, EmailMacroEvent, EmailTopicEvent, LinkConnectedMetadata,
+ LinkDisconnectReason, LinkDisconnectedMetadata, LinkReauthRequiredMetadata,
+ MessageDeletedMetadata, MessageDraftSyncedMetadata, MessageReceivedMetadata,
+ MessageSendCancelledMetadata, MessageSendQueuedMetadata, MessageSentMetadata, SendCancelReason,
+ ThreadArchivedMetadata, ThreadBackfilledMetadata, ThreadLabelsUpdatedMetadata,
+ ThreadProjectChangedMetadata, ThreadReadMetadata, ThreadSpamChangedMetadata,
+ ThreadStarredMetadata, ThreadTrashedMetadata, ThreadsReindexReason,
+ ThreadsReindexRequestedMetadata,
};
use channels::domain::{
broker_events::{
@@ -118,6 +119,27 @@ fn channel_sender() -> ChannelSender<'static> {
ChannelSender::try_from("macro|[email protected]".to_string()).expect("valid channel sender")
}
+/// Builds a [`WorkerPool`] whose worker channels are captured as receivers.
+fn test_pool(workers: usize, capacity: usize) -> (WorkerPool, Vec<mpsc::Receiver<ReceivedEvent>>) {
+ let (senders, receivers) = (0..workers).map(|_| mpsc::channel(capacity)).unzip();
+ (WorkerPool::new(senders), receivers)
+}
+
+fn received_thread_backfilled_event(thread_id: Uuid, offset: i64) -> ReceivedEvent {
+ ReceivedEvent {
+ event: DeclaredMacroEvent::EmailMacroEvent(EmailMacroEvent::new(
+ EMAIL_LINK_ID.to_string(),
+ EmailTopicEvent::ThreadBackfilled(ThreadBackfilledMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ thread_id,
+ }),
+ )),
+ partition: 0,
+ offset,
+ }
+}
+
fn user_id() -> MacroUserIdStr<'static> {
MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")
}
@@ -1658,17 +1680,17 @@ async fn malformed_missing_key_and_unsupported_email_messages_are_commit_safe()
Event::with_schema_version(draft_email_event(false), 2),
);
let unsupported = attach_event_coordinates(DeclaredMacroEvent::decode(&unsupported), 7, 62);
- let (sender, mut receiver) = mpsc::channel(1);
+ let (pool, mut receivers) = test_pool(1, 1);
assert!(matches!(
- handoff_decoded(&sender, malformed).await,
+ pool.handoff(malformed).await,
HandoffOutcome::MalformedRecord(EventBrokerError::Serialization(_))
));
assert!(matches!(
- handoff_decoded(&sender, missing_key).await,
+ pool.handoff(missing_key).await,
HandoffOutcome::MalformedRecord(EventBrokerError::MissingMessageKey)
));
- match handoff_decoded(&sender, unsupported).await {
+ match pool.handoff(unsupported).await {
HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {
topic,
expected,
@@ -1681,7 +1703,7 @@ async fn malformed_missing_key_and_unsupported_email_messages_are_commit_safe()
outcome => panic!("expected malformed email record, got {outcome:?}"),
}
assert!(matches!(
- receiver.try_recv(),
+ receivers[0].try_recv(),
Err(mpsc::error::TryRecvError::Empty)
));
}
@@ -1694,10 +1716,10 @@ async fn malformed_and_unsupported_chat_messages_are_commit_safe() {
payload: Some(b"not json".to_vec()),
};
let malformed = attach_event_coordinates(DeclaredMacroEvent::decode(&malformed), 5, 40);
- let (sender, mut receiver) = mpsc::channel(1);
+ let (pool, mut receivers) = test_pool(1, 1);
assert!(matches!(
- handoff_decoded(&sender, malformed).await,
+ pool.handoff(malformed).await,
HandoffOutcome::MalformedRecord(EventBrokerError::Serialization(_))
));
@@ -1711,7 +1733,7 @@ async fn malformed_and_unsupported_chat_messages_are_commit_safe() {
Event::with_schema_version(unsupported_event, 2),
);
let unsupported = attach_event_coordinates(DeclaredMacroEvent::decode(&unsupported), 5, 41);
- match handoff_decoded(&sender, unsupported).await {
+ match pool.handoff(unsupported).await {
HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {
topic,
expected,
@@ -1725,7 +1747,7 @@ async fn malformed_and_unsupported_chat_messages_are_commit_safe() {
}
assert!(matches!(
- receiver.try_recv(),
+ receivers[0].try_recv(),
Err(mpsc::error::TryRecvError::Empty)
));
}
@@ -1745,9 +1767,9 @@ async fn unsupported_project_schema_message_is_commit_safe() {
Event::with_schema_version(event, 2),
);
let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 4, 30);
- let (sender, mut receiver) = mpsc::channel(1);
+ let (pool, mut receivers) = test_pool(1, 1);
- match handoff_decoded(&sender, decoded).await {
+ match pool.handoff(decoded).await {
HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {
topic,
expected,
@@ -1760,7 +1782,7 @@ async fn unsupported_project_schema_message_is_commit_safe() {
outcome => panic!("expected malformed project record, got {outcome:?}"),
}
assert!(matches!(
- receiver.try_recv(),
+ receivers[0].try_recv(),
Err(mpsc::error::TryRecvError::Empty)
));
}
@@ -1778,9 +1800,9 @@ async fn unsupported_property_schema_message_is_commit_safe() {
Event::with_schema_version(event, 2),
);
let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 6, 50);
- let (sender, mut receiver) = mpsc::channel(1);
+ let (pool, mut receivers) = test_pool(1, 1);
- match handoff_decoded(&sender, decoded).await {
+ match pool.handoff(decoded).await {
HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {
topic,
expected,
@@ -1793,7 +1815,7 @@ async fn unsupported_property_schema_message_is_commit_safe() {
outcome => panic!("expected malformed property record, got {outcome:?}"),
}
assert!(matches!(
- receiver.try_recv(),
+ receivers[0].try_recv(),
Err(mpsc::error::TryRecvError::Empty)
));
}
@@ -1810,9 +1832,9 @@ async fn unsupported_channel_schema_message_is_commit_safe() {
Event::with_schema_version(event, 2),
);
let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 2, 20);
- let (sender, mut receiver) = mpsc::channel(1);
+ let (pool, mut receivers) = test_pool(1, 1);
- match handoff_decoded(&sender, decoded).await {
+ match pool.handoff(decoded).await {
HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {
topic,
expected,
@@ -1825,7 +1847,7 @@ async fn unsupported_channel_schema_message_is_commit_safe() {
outcome => panic!("expected malformed channel record, got {outcome:?}"),
}
assert!(matches!(
- receiver.try_recv(),
+ receivers[0].try_recv(),
Err(mpsc::error::TryRecvError::Empty)
));
}
@@ -1871,15 +1893,15 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()
})
));
- let (sender, mut receiver) = mpsc::channel(1);
+ let (pool, mut receivers) = test_pool(1, 1);
for decoded in [malformed, missing_key, unsupported] {
assert!(matches!(
- handoff_decoded(&sender, decoded).await,
+ pool.handoff(decoded).await,
HandoffOutcome::MalformedRecord(_)
));
}
assert!(matches!(
- receiver.try_recv(),
+ receivers[0].try_recv(),
Err(mpsc::error::TryRecvError::Empty)
));
}
@@ -1893,14 +1915,14 @@ async fn successful_handoff_carries_event_partition_and_offset() {
Event::new(event.clone()),
);
let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 3, 42);
- let (sender, mut receiver) = mpsc::channel(1);
+ let (pool, mut receivers) = test_pool(1, 1);
assert!(matches!(
- handoff_decoded(&sender, decoded).await,
+ pool.handoff(decoded).await,
HandoffOutcome::HandedOff
));
- let received = receiver.recv().await.expect("handed-off event");
+ let received = receivers[0].recv().await.expect("handed-off event");
assert_eq!(received.partition, 3);
assert_eq!(received.offset, 42);
let DeclaredMacroEvent::CallMacroEvent(received_event) = received.event else {
@@ -1917,43 +1939,150 @@ async fn closed_worker_channel_leaves_the_current_message_uncommitted() {
Event::new(archived_event()),
);
let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 3, 42);
- let (sender, receiver) = mpsc::channel(1);
- drop(receiver);
+ let (pool, receivers) = test_pool(1, 1);
+ drop(receivers);
assert!(matches!(
- handoff_decoded(&sender, decoded).await,
+ pool.handoff(decoded).await,
HandoffOutcome::WorkerClosed
));
}
#[tokio::test]
async fn bounded_handoff_blocks_when_full_and_preserves_order() {
- let (sender, mut receiver) = mpsc::channel(1);
+ let (pool, mut receivers) = test_pool(1, 1);
+ let first = received_thread_backfilled_event(EMAIL_THREAD_ID, 1);
+ let second = received_thread_backfilled_event(EMAIL_THREAD_ID, 2);
+
assert!(matches!(
- handoff_decoded(&sender, Ok::<_, ()>(1)).await,
+ pool.handoff(Ok::<_, EventBrokerError>(first)).await,
HandoffOutcome::HandedOff
));
let blocked_handoff =
- tokio::spawn(async move { handoff_decoded(&sender, Ok::<_, ()>(2)).await });
+ tokio::spawn(async move { pool.handoff(Ok::<_, EventBrokerError>(second)).await });
tokio::task::yield_now().await;
assert!(
!blocked_handoff.is_finished(),
"handoff must wait while the bounded channel is full"
);
- assert_eq!(receiver.recv().await, Some(1));
+ assert_eq!(receivers[0].recv().await.expect("first event").offset, 1);
assert!(matches!(
blocked_handoff.await.expect("handoff task did not panic"),
HandoffOutcome::HandedOff
));
- assert_eq!(receiver.recv().await, Some(2));
+ assert_eq!(receivers[0].recv().await.expect("second event").offset, 2);
+}
+
+#[tokio::test]
+async fn events_with_the_same_ordering_key_stay_on_one_worker_in_order() {
+ let (pool, mut receivers) = test_pool(WORKER_COUNT, WORKER_CHANNEL_CAPACITY);
+ for offset in 0..8 {
+ let event = received_thread_backfilled_event(EMAIL_THREAD_ID, offset);
+ assert!(matches!(
+ pool.handoff(Ok::<_, EventBrokerError>(event)).await,
+ HandoffOutcome::HandedOff
+ ));
+ }
+ drop(pool);
+
+ let mut busy_workers = 0;
+ for receiver in &mut receivers {
+ let mut offsets = Vec::new();
+ while let Some(event) = receiver.recv().await {
+ offsets.push(event.offset);
+ }
+ if offsets.is_empty() {
+ continue;
+ }
+ busy_workers += 1;
+ assert_eq!(offsets, (0..8).collect::<Vec<_>>());
+ }
+ assert_eq!(busy_workers, 1, "one key must map to exactly one worker");
+}
+
+#[tokio::test]
+async fn events_with_distinct_ordering_keys_spread_across_workers() {
+ let (pool, mut receivers) = test_pool(WORKER_COUNT, 128);
+ for index in 0..100 {
+ let thread_id = Uuid::from_u128(1_000 + index);
+ let event = received_thread_backfilled_event(thread_id, index as i64);
+ assert!(matches!(
+ pool.handoff(Ok::<_, EventBrokerError>(event)).await,
+ HandoffOutcome::HandedOff
+ ));
+ }
+ drop(pool);
+
+ let busy_workers = receivers
+ .iter_mut()
+ .map(|receiver| receiver.try_recv().is_ok())
+ .filter(|received| *received)
+ .count();
+ assert!(
+ busy_workers > 1,
+ "distinct ordering keys must spread across workers, got {busy_workers}"
+ );
+}
+
+#[test]
+fn email_events_shard_by_thread_and_link_scoped_events_by_producer_key() {
+ let thread_scoped = DeclaredMacroEvent::EmailMacroEvent(EmailMacroEvent::new(
+ EMAIL_LINK_ID.to_string(),
+ received_email_event(false),
+ ));
+ assert_eq!(ordering_key(&thread_scoped), EMAIL_THREAD_ID.to_string());
+
+ let link_scoped = DeclaredMacroEvent::EmailMacroEvent(EmailMacroEvent::new(
+ EMAIL_LINK_ID.to_string(),
+ EmailTopicEvent::LinkDisconnected(LinkDisconnectedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ email_address: "[email protected]".to_string(),
+ reason: LinkDisconnectReason::ManuallyDisabled,
+ }),
+ ));
+ assert_eq!(ordering_key(&link_scoped), EMAIL_LINK_ID.to_string());
+
+ let multi_thread = DeclaredMacroEvent::EmailMacroEvent(EmailMacroEvent::new(
+ EMAIL_LINK_ID.to_string(),
+ EmailTopicEvent::ThreadsReindexRequested(ThreadsReindexRequestedMetadata {
+ link_id: EMAIL_LINK_ID,
+ owner: user_id(),
+ thread_ids: vec![EMAIL_THREAD_ID, SECOND_EMAIL_THREAD_ID],
+ reason: ThreadsReindexReason::ContactsChanged,
+ }),
+ ));
+ assert_eq!(ordering_key(&multi_thread), EMAIL_LINK_ID.to_string());
+}
+
+#[test]
+fn non_email_events_shard_by_their_producer_key() {
+ let message = encoded_message(
+ MacroCallsTopic::TOPIC_STR,
+ CALL_ID,
+ Event::new(archived_event()),
+ );
+ let call = DeclaredMacroEvent::decode(&message).expect("decodable call event");
+ assert_eq!(ordering_key(&call), CALL_ID.to_string());
+
+ let message = encoded_message(
+ MacroDocumentsTopic::TOPIC_STR,
+ DOCUMENT_ID,
+ Event::new(DocumentTopicEvent::Purged(DocumentPurgedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ })),
+ );
+ let document = DeclaredMacroEvent::decode(&message).expect("decodable document event");
+ assert_eq!(ordering_key(&document), DOCUMENT_ID);
}
#[test]
-fn production_channel_and_retry_bounds_match_the_delivery_contract() {
- let (sender, _receiver) = mpsc::channel::<ReceivedEvent>(CHANNEL_CAPACITY);
- assert_eq!(sender.max_capacity(), 128);
+fn production_worker_pool_and_retry_bounds_match_the_delivery_contract() {
+ assert_eq!(WORKER_COUNT, 10);
+ let (sender, _receiver) = mpsc::channel::<ReceivedEvent>(WORKER_CHANNEL_CAPACITY);
+ assert_eq!(sender.max_capacity(), 16);
assert_eq!(
processing_retry_strategy().collect::<Vec<_>>(),
[Duration::from_secs(1), Duration::from_secs(2)]