Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(search_processing_service): parallelize workers (#5421)
Will Hutchinson261948d5a742
diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rsindex 607d374cab..6ac6eec0c7 100644--- a/services/search_processing_service/src/inbound/kafka_consumer.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs@@ -1,10 +1,13 @@//! Kafka consumer for broker events that update the search index.//!-//! The poll loop hands decoded events to one bounded, sequential worker and-//! commits each offset immediately after that handoff. Malformed records are-//! committed without a handoff so they cannot wedge a partition. Processing is-//! retried in-process; exhausted events are logged and dropped because their-//! offsets are already committed.+//! The poll loop shards each decoded event by its ordering key onto one of a+//! fixed pool of bounded, sequential workers and commits the offset+//! immediately after that handoff. Events sharing an ordering key — the+//! entity whose search documents they mutate — stay on one worker in+//! partition order; unrelated events process in parallel. Malformed records+//! are committed without a handoff so they cannot wedge a partition.+//! Processing is retried in-process; exhausted events are logged and dropped+//! because their offsets are already committed.//!//! Per-entity event mapping and processing live in the [`call`], [`channel`],//! [`chat`], [`document`], [`email`], [`project`], and [`property`] submodules;@@ -23,7 +26,13 @@ mod property;#[cfg(test)]mod test;-use std::{future::Future, time::Duration};+use std::{+ borrow::Cow,+ collections::hash_map::DefaultHasher,+ future::Future,+ hash::{Hash as _, Hasher as _},+ time::Duration,+};use ::call::domain::events::CallMacroEvent;use ::chat::domain::events::ChatMacroEvent;@@ -31,7 +40,9 @@ use ::email::domain::events::EmailMacroEvent;use channels::domain::broker_events::ChannelMacroEvent;use documents::domain::events::DocumentMacroEvent;use kafka_util::{GroupName, KafkaEventConsumer};-use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};+use macro_event_broker::{+ KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection, MacroEventConsumerService,+};use projects::domain::events::ProjectMacroEvent;use properties::domain::events::PropertyMacroEvent;use rdkafka::{@@ -39,12 +50,16 @@ use rdkafka::{message::{BorrowedMessage, Message as _},};use rootcause::prelude::{Report, ResultExt as _};-use tokio::sync::mpsc;+use tokio::{sync::mpsc, task::JoinHandle};use tokio_retry::{Retry, strategy::ExponentialBackoff};use self::{- call::process_call_event, channel::process_channel_event, chat::process_chat_event,- document::process_document_event, email::process_email_event, project::process_project_event,+ call::process_call_event,+ channel::process_channel_event,+ chat::process_chat_event,+ document::process_document_event,+ email::{email_ordering_key, process_email_event},+ project::process_project_event,property::process_property_event,};@@ -73,8 +88,16 @@ macro_event_broker::declare_topics!(PropertyMacroEvent,);-/// Maximum number of decoded events waiting for the sequential worker.-const CHANNEL_CAPACITY: usize = 128;+/// Number of sequential workers events are sharded across by ordering key.+///+/// Each in-flight event holds a database connection and an OpenSearch+/// request, so this also bounds the consumer's downstream concurrency.+const WORKER_COUNT: usize = 10;+/// Maximum number of decoded events waiting for each sequential worker.+///+/// Offsets are committed at handoff, so `WORKER_COUNT * WORKER_CHANNEL_CAPACITY`+/// also bounds how many committed-but-unprocessed events a crash can drop.+const WORKER_CHANNEL_CAPACITY: usize = 16;/// Delay before polling Kafka again after a receive error.const RECEIVE_ERROR_RETRY_DELAY: Duration = Duration::from_secs(1);/// Total processing attempts before an event is dropped.@@ -143,18 +166,75 @@ whereretry_processing_with_strategy(processing_retry_strategy(), operation).await}-async fn handoff_decoded<T, E>(- sender: &mpsc::Sender<T>,- decoded: Result<T, E>,-) -> HandoffOutcome<E> {- let event = match decoded {- Ok(event) => event,- Err(error) => return HandoffOutcome::MalformedRecord(error),- };-- match sender.send(event).await {- Ok(()) => HandoffOutcome::HandedOff,- Err(_) => HandoffOutcome::WorkerClosed,+/// Sharding key that serializes all events mutating the same search documents.+///+/// Every topic's producer key is already the entity its index actions target+/// (call, channel, chat, document, project, or property-entity id), except+/// email events, which are produced with a link-scoped key and instead shard+/// by thread so one inbox's backlog can spread across the worker pool.+fn ordering_key(event: &DeclaredMacroEvent) -> Cow<'_, str> {+ match event {+ DeclaredMacroEvent::CallMacroEvent(event) => Cow::Borrowed(event.key()),+ DeclaredMacroEvent::ChannelMacroEvent(event) => Cow::Borrowed(event.key()),+ DeclaredMacroEvent::ChatMacroEvent(event) => Cow::Borrowed(event.key()),+ DeclaredMacroEvent::DocumentMacroEvent(event) => Cow::Borrowed(event.key()),+ DeclaredMacroEvent::EmailMacroEvent(event) => email_ordering_key(event),+ DeclaredMacroEvent::ProjectMacroEvent(event) => Cow::Borrowed(event.key()),+ DeclaredMacroEvent::PropertyMacroEvent(event) => Cow::Borrowed(event.key()),+ }+}++/// Fixed pool of sequential workers, sharded by [`ordering_key`].+///+/// Events with the same ordering key always land on the same worker's FIFO+/// channel, so they process in handoff (partition) order; events with+/// different keys may process concurrently.+struct WorkerPool {+ senders: Vec<mpsc::Sender<ReceivedEvent>>,+}++impl WorkerPool {+ fn new(senders: Vec<mpsc::Sender<ReceivedEvent>>) -> Self {+ assert!(+ !senders.is_empty(),+ "worker pool requires at least one worker"+ );+ Self { senders }+ }++ /// Spawns [`WORKER_COUNT`] workers sharing `context` and returns their+ /// join handles alongside the pool. Dropping the pool closes every worker+ /// channel, letting the workers drain and exit.+ fn spawn(context: &KafkaProcessingContext) -> (Self, Vec<JoinHandle<()>>) {+ let (senders, workers) = (0..WORKER_COUNT)+ .map(|_| {+ let (sender, receiver) = mpsc::channel(WORKER_CHANNEL_CAPACITY);+ (+ sender,+ tokio::spawn(run_event_worker(context.clone(), receiver)),+ )+ })+ .unzip();+ (Self::new(senders), workers)+ }++ fn shard_index(&self, key: &str) -> usize {+ let mut hasher = DefaultHasher::new();+ key.hash(&mut hasher);+ (hasher.finish() % self.senders.len() as u64) as usize+ }++ async fn handoff<E>(&self, decoded: Result<ReceivedEvent, E>) -> HandoffOutcome<E> {+ let event = match decoded {+ Ok(event) => event,+ Err(error) => return HandoffOutcome::MalformedRecord(error),+ };++ let shard = self.shard_index(ordering_key(&event.event).as_ref());+ match self.senders[shard].send(event).await {+ Ok(()) => HandoffOutcome::HandedOff,+ Err(_) => HandoffOutcome::WorkerClosed,+ }}}@@ -241,7 +321,7 @@ fn commit_logged(consumer: &SearchProcessingKafkaConsumer, message: &BorrowedMesasync fn poll_events(consumer: &SearchProcessingKafkaConsumer,- events: mpsc::Sender<ReceivedEvent>,+ workers: WorkerPool,shutdown: impl Future<Output = ()> + Send,) -> Result<(), Report> {let mut shutdown = std::pin::pin!(shutdown);@@ -266,7 +346,7 @@ async fn poll_events(kafka_message.offset(),);- match handoff_decoded(&events, decoded).await {+ match workers.handoff(decoded).await {HandoffOutcome::HandedOff => {}HandoffOutcome::MalformedRecord(error) => tracing::error!(error = ?error,@@ -301,12 +381,13 @@ async fn poll_events(Ok(())}-/// Runs the bounded broker event poll-loop/worker pair until `shutdown` resolves.+/// Runs the bounded broker event poll loop and worker pool until `shutdown`+/// resolves.////// The caller is responsible for supervising this function. A closed worker/// channel returns an error without committing the current Kafka message. On-/// normal shutdown the sender is dropped and all buffered events are processed-/// before this function returns.+/// normal shutdown the senders are dropped and all buffered events are+/// processed before this function returns.#[tracing::instrument(skip(context, shutdown), fields(brokers), err)]pub(crate) async fn run_event_consumer(brokers: &str,@@ -324,12 +405,21 @@ pub(crate) async fn run_event_consumer("search processing event consumer listening");- let (events_tx, events_rx) = mpsc::channel(CHANNEL_CAPACITY);- let worker = tokio::spawn(run_event_worker(context, events_rx));- let poll_result = poll_events(&consumer, events_tx, shutdown).await;- let worker_result = worker.await;+ let (pool, workers) = WorkerPool::spawn(&context);+ // Passing the pool by value drops every sender when polling stops, which+ // closes the worker channels so the workers drain and exit.+ let poll_result = poll_events(&consumer, pool, shutdown).await;++ let mut worker_failure = None;+ for worker in workers {+ if let Err(error) = worker.await {+ worker_failure = Some(error);+ }+ }poll_result?;- worker_result.map_err(|error| rootcause::report!(error))?;+ if let Some(error) = worker_failure {+ Err(rootcause::report!(error))?;+ }Ok(())}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/email.rs b/services/search_processing_service/src/inbound/kafka_consumer/email.rsindex b812cead3f..20d60ff02a 100644--- a/services/search_processing_service/src/inbound/kafka_consumer/email.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer/email.rs@@ -1,5 +1,7 @@//! Maps email lifecycle events to search-index actions and processes them.+use std::borrow::Cow;+use ::email::domain::events::{EmailMacroEvent, EmailTopicEvent};use macro_event_broker::MacroEvent as _;use opensearch_client::OpensearchClient;@@ -39,6 +41,39 @@ pub(super) struct EmailEventDescription {pub(super) event_type: &'static str,}+/// Worker-sharding key for an email event.+///+/// Email events are produced with a link-scoped (whole inbox) Kafka key, so+/// thread- and message-scoped events shard by thread id instead: one inbox's+/// backfill spreads across the worker pool while events for a single thread+/// stay ordered. Link-scoped events keep the link-id producer key, so their+/// ordering against a buffered thread event (e.g. a link removal racing a+/// message upsert) is not guaranteed — an accepted risk of the same order as+/// this consumer's existing commit-at-handoff drop window.+pub(super) fn email_ordering_key(event: &EmailMacroEvent) -> Cow<'_, str> {+ let thread_id = match &event.event().event {+ EmailTopicEvent::LinkConnected(_)+ | EmailTopicEvent::LinkDisconnected(_)+ | EmailTopicEvent::LinkReauthRequired(_)+ | EmailTopicEvent::ThreadsReindexRequested(_) => return Cow::Borrowed(event.key()),+ EmailTopicEvent::MessageReceived(metadata) => metadata.thread_id,+ EmailTopicEvent::MessageDraftSynced(metadata) => metadata.thread_id,+ EmailTopicEvent::MessageSent(metadata) => metadata.thread_id,+ EmailTopicEvent::MessageDeleted(metadata) => metadata.thread_id,+ EmailTopicEvent::MessageSendQueued(metadata) => metadata.thread_id,+ EmailTopicEvent::MessageSendCancelled(metadata) => metadata.thread_id,+ EmailTopicEvent::ThreadArchived(metadata) => metadata.thread_id,+ EmailTopicEvent::ThreadTrashed(metadata) => metadata.thread_id,+ EmailTopicEvent::ThreadRead(metadata) => metadata.thread_id,+ EmailTopicEvent::ThreadStarred(metadata) => metadata.thread_id,+ EmailTopicEvent::ThreadSpamChanged(metadata) => metadata.thread_id,+ EmailTopicEvent::ThreadProjectChanged(metadata) => metadata.thread_id,+ EmailTopicEvent::ThreadLabelsUpdated(metadata) => metadata.thread_id,+ EmailTopicEvent::ThreadBackfilled(metadata) => metadata.thread_id,+ };+ Cow::Owned(thread_id.to_string())+}+pub(super) fn describe_email_event(event: &EmailTopicEvent) -> EmailEventDescription {match event {EmailTopicEvent::LinkConnected(metadata) => EmailEventDescription {diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rsindex 30f8db0b49..d99dfd3797 100644--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs@@ -12,13 +12,14 @@ use ::chat::domain::events::{ChatTopicEvent, ChatUpdatedMetadata,};use ::email::domain::events::{- EmailEventOrigin, EmailTopicEvent, LinkConnectedMetadata, LinkDisconnectReason,- LinkDisconnectedMetadata, LinkReauthRequiredMetadata, MessageDeletedMetadata,- MessageDraftSyncedMetadata, MessageReceivedMetadata, MessageSendCancelledMetadata,- MessageSendQueuedMetadata, MessageSentMetadata, SendCancelReason, ThreadArchivedMetadata,- ThreadBackfilledMetadata, ThreadLabelsUpdatedMetadata, ThreadProjectChangedMetadata,- ThreadReadMetadata, ThreadSpamChangedMetadata, ThreadStarredMetadata, ThreadTrashedMetadata,- ThreadsReindexReason, ThreadsReindexRequestedMetadata,+ EmailEventOrigin, EmailMacroEvent, EmailTopicEvent, LinkConnectedMetadata,+ LinkDisconnectReason, LinkDisconnectedMetadata, LinkReauthRequiredMetadata,+ MessageDeletedMetadata, MessageDraftSyncedMetadata, MessageReceivedMetadata,+ MessageSendCancelledMetadata, MessageSendQueuedMetadata, MessageSentMetadata, SendCancelReason,+ ThreadArchivedMetadata, ThreadBackfilledMetadata, ThreadLabelsUpdatedMetadata,+ ThreadProjectChangedMetadata, ThreadReadMetadata, ThreadSpamChangedMetadata,+ ThreadStarredMetadata, ThreadTrashedMetadata, ThreadsReindexReason,+ ThreadsReindexRequestedMetadata,};use channels::domain::{broker_events::{@@ -118,6 +119,27 @@ fn channel_sender() -> ChannelSender<'static> {ChannelSender::try_from("macro|[email protected]".to_string()).expect("valid channel sender")}+/// Builds a [`WorkerPool`] whose worker channels are captured as receivers.+fn test_pool(workers: usize, capacity: usize) -> (WorkerPool, Vec<mpsc::Receiver<ReceivedEvent>>) {+ let (senders, receivers) = (0..workers).map(|_| mpsc::channel(capacity)).unzip();+ (WorkerPool::new(senders), receivers)+}++fn received_thread_backfilled_event(thread_id: Uuid, offset: i64) -> ReceivedEvent {+ ReceivedEvent {+ event: DeclaredMacroEvent::EmailMacroEvent(EmailMacroEvent::new(+ EMAIL_LINK_ID.to_string(),+ EmailTopicEvent::ThreadBackfilled(ThreadBackfilledMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ thread_id,+ }),+ )),+ partition: 0,+ offset,+ }+}+fn user_id() -> MacroUserIdStr<'static> {MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")}@@ -1658,17 +1680,17 @@ async fn malformed_missing_key_and_unsupported_email_messages_are_commit_safe()Event::with_schema_version(draft_email_event(false), 2),);let unsupported = attach_event_coordinates(DeclaredMacroEvent::decode(&unsupported), 7, 62);- let (sender, mut receiver) = mpsc::channel(1);+ let (pool, mut receivers) = test_pool(1, 1);assert!(matches!(- handoff_decoded(&sender, malformed).await,+ pool.handoff(malformed).await,HandoffOutcome::MalformedRecord(EventBrokerError::Serialization(_))));assert!(matches!(- handoff_decoded(&sender, missing_key).await,+ pool.handoff(missing_key).await,HandoffOutcome::MalformedRecord(EventBrokerError::MissingMessageKey)));- match handoff_decoded(&sender, unsupported).await {+ match pool.handoff(unsupported).await {HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {topic,expected,@@ -1681,7 +1703,7 @@ async fn malformed_missing_key_and_unsupported_email_messages_are_commit_safe()outcome => panic!("expected malformed email record, got {outcome:?}"),}assert!(matches!(- receiver.try_recv(),+ receivers[0].try_recv(),Err(mpsc::error::TryRecvError::Empty)));}@@ -1694,10 +1716,10 @@ async fn malformed_and_unsupported_chat_messages_are_commit_safe() {payload: Some(b"not json".to_vec()),};let malformed = attach_event_coordinates(DeclaredMacroEvent::decode(&malformed), 5, 40);- let (sender, mut receiver) = mpsc::channel(1);+ let (pool, mut receivers) = test_pool(1, 1);assert!(matches!(- handoff_decoded(&sender, malformed).await,+ pool.handoff(malformed).await,HandoffOutcome::MalformedRecord(EventBrokerError::Serialization(_))));@@ -1711,7 +1733,7 @@ async fn malformed_and_unsupported_chat_messages_are_commit_safe() {Event::with_schema_version(unsupported_event, 2),);let unsupported = attach_event_coordinates(DeclaredMacroEvent::decode(&unsupported), 5, 41);- match handoff_decoded(&sender, unsupported).await {+ match pool.handoff(unsupported).await {HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {topic,expected,@@ -1725,7 +1747,7 @@ async fn malformed_and_unsupported_chat_messages_are_commit_safe() {}assert!(matches!(- receiver.try_recv(),+ receivers[0].try_recv(),Err(mpsc::error::TryRecvError::Empty)));}@@ -1745,9 +1767,9 @@ async fn unsupported_project_schema_message_is_commit_safe() {Event::with_schema_version(event, 2),);let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 4, 30);- let (sender, mut receiver) = mpsc::channel(1);+ let (pool, mut receivers) = test_pool(1, 1);- match handoff_decoded(&sender, decoded).await {+ match pool.handoff(decoded).await {HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {topic,expected,@@ -1760,7 +1782,7 @@ async fn unsupported_project_schema_message_is_commit_safe() {outcome => panic!("expected malformed project record, got {outcome:?}"),}assert!(matches!(- receiver.try_recv(),+ receivers[0].try_recv(),Err(mpsc::error::TryRecvError::Empty)));}@@ -1778,9 +1800,9 @@ async fn unsupported_property_schema_message_is_commit_safe() {Event::with_schema_version(event, 2),);let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 6, 50);- let (sender, mut receiver) = mpsc::channel(1);+ let (pool, mut receivers) = test_pool(1, 1);- match handoff_decoded(&sender, decoded).await {+ match pool.handoff(decoded).await {HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {topic,expected,@@ -1793,7 +1815,7 @@ async fn unsupported_property_schema_message_is_commit_safe() {outcome => panic!("expected malformed property record, got {outcome:?}"),}assert!(matches!(- receiver.try_recv(),+ receivers[0].try_recv(),Err(mpsc::error::TryRecvError::Empty)));}@@ -1810,9 +1832,9 @@ async fn unsupported_channel_schema_message_is_commit_safe() {Event::with_schema_version(event, 2),);let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 2, 20);- let (sender, mut receiver) = mpsc::channel(1);+ let (pool, mut receivers) = test_pool(1, 1);- match handoff_decoded(&sender, decoded).await {+ match pool.handoff(decoded).await {HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {topic,expected,@@ -1825,7 +1847,7 @@ async fn unsupported_channel_schema_message_is_commit_safe() {outcome => panic!("expected malformed channel record, got {outcome:?}"),}assert!(matches!(- receiver.try_recv(),+ receivers[0].try_recv(),Err(mpsc::error::TryRecvError::Empty)));}@@ -1871,15 +1893,15 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()})));- let (sender, mut receiver) = mpsc::channel(1);+ let (pool, mut receivers) = test_pool(1, 1);for decoded in [malformed, missing_key, unsupported] {assert!(matches!(- handoff_decoded(&sender, decoded).await,+ pool.handoff(decoded).await,HandoffOutcome::MalformedRecord(_)));}assert!(matches!(- receiver.try_recv(),+ receivers[0].try_recv(),Err(mpsc::error::TryRecvError::Empty)));}@@ -1893,14 +1915,14 @@ async fn successful_handoff_carries_event_partition_and_offset() {Event::new(event.clone()),);let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 3, 42);- let (sender, mut receiver) = mpsc::channel(1);+ let (pool, mut receivers) = test_pool(1, 1);assert!(matches!(- handoff_decoded(&sender, decoded).await,+ pool.handoff(decoded).await,HandoffOutcome::HandedOff));- let received = receiver.recv().await.expect("handed-off event");+ let received = receivers[0].recv().await.expect("handed-off event");assert_eq!(received.partition, 3);assert_eq!(received.offset, 42);let DeclaredMacroEvent::CallMacroEvent(received_event) = received.event else {@@ -1917,43 +1939,150 @@ async fn closed_worker_channel_leaves_the_current_message_uncommitted() {Event::new(archived_event()),);let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 3, 42);- let (sender, receiver) = mpsc::channel(1);- drop(receiver);+ let (pool, receivers) = test_pool(1, 1);+ drop(receivers);assert!(matches!(- handoff_decoded(&sender, decoded).await,+ pool.handoff(decoded).await,HandoffOutcome::WorkerClosed));}#[tokio::test]async fn bounded_handoff_blocks_when_full_and_preserves_order() {- let (sender, mut receiver) = mpsc::channel(1);+ let (pool, mut receivers) = test_pool(1, 1);+ let first = received_thread_backfilled_event(EMAIL_THREAD_ID, 1);+ let second = received_thread_backfilled_event(EMAIL_THREAD_ID, 2);+assert!(matches!(- handoff_decoded(&sender, Ok::<_, ()>(1)).await,+ pool.handoff(Ok::<_, EventBrokerError>(first)).await,HandoffOutcome::HandedOff));let blocked_handoff =- tokio::spawn(async move { handoff_decoded(&sender, Ok::<_, ()>(2)).await });+ tokio::spawn(async move { pool.handoff(Ok::<_, EventBrokerError>(second)).await });tokio::task::yield_now().await;assert!(!blocked_handoff.is_finished(),"handoff must wait while the bounded channel is full");- assert_eq!(receiver.recv().await, Some(1));+ assert_eq!(receivers[0].recv().await.expect("first event").offset, 1);assert!(matches!(blocked_handoff.await.expect("handoff task did not panic"),HandoffOutcome::HandedOff));- assert_eq!(receiver.recv().await, Some(2));+ assert_eq!(receivers[0].recv().await.expect("second event").offset, 2);+}++#[tokio::test]+async fn events_with_the_same_ordering_key_stay_on_one_worker_in_order() {+ let (pool, mut receivers) = test_pool(WORKER_COUNT, WORKER_CHANNEL_CAPACITY);+ for offset in 0..8 {+ let event = received_thread_backfilled_event(EMAIL_THREAD_ID, offset);+ assert!(matches!(+ pool.handoff(Ok::<_, EventBrokerError>(event)).await,+ HandoffOutcome::HandedOff+ ));+ }+ drop(pool);++ let mut busy_workers = 0;+ for receiver in &mut receivers {+ let mut offsets = Vec::new();+ while let Some(event) = receiver.recv().await {+ offsets.push(event.offset);+ }+ if offsets.is_empty() {+ continue;+ }+ busy_workers += 1;+ assert_eq!(offsets, (0..8).collect::<Vec<_>>());+ }+ assert_eq!(busy_workers, 1, "one key must map to exactly one worker");+}++#[tokio::test]+async fn events_with_distinct_ordering_keys_spread_across_workers() {+ let (pool, mut receivers) = test_pool(WORKER_COUNT, 128);+ for index in 0..100 {+ let thread_id = Uuid::from_u128(1_000 + index);+ let event = received_thread_backfilled_event(thread_id, index as i64);+ assert!(matches!(+ pool.handoff(Ok::<_, EventBrokerError>(event)).await,+ HandoffOutcome::HandedOff+ ));+ }+ drop(pool);++ let busy_workers = receivers+ .iter_mut()+ .map(|receiver| receiver.try_recv().is_ok())+ .filter(|received| *received)+ .count();+ assert!(+ busy_workers > 1,+ "distinct ordering keys must spread across workers, got {busy_workers}"+ );+}++#[test]+fn email_events_shard_by_thread_and_link_scoped_events_by_producer_key() {+ let thread_scoped = DeclaredMacroEvent::EmailMacroEvent(EmailMacroEvent::new(+ EMAIL_LINK_ID.to_string(),+ received_email_event(false),+ ));+ assert_eq!(ordering_key(&thread_scoped), EMAIL_THREAD_ID.to_string());++ let link_scoped = DeclaredMacroEvent::EmailMacroEvent(EmailMacroEvent::new(+ EMAIL_LINK_ID.to_string(),+ EmailTopicEvent::LinkDisconnected(LinkDisconnectedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ email_address: "[email protected]".to_string(),+ reason: LinkDisconnectReason::ManuallyDisabled,+ }),+ ));+ assert_eq!(ordering_key(&link_scoped), EMAIL_LINK_ID.to_string());++ let multi_thread = DeclaredMacroEvent::EmailMacroEvent(EmailMacroEvent::new(+ EMAIL_LINK_ID.to_string(),+ EmailTopicEvent::ThreadsReindexRequested(ThreadsReindexRequestedMetadata {+ link_id: EMAIL_LINK_ID,+ owner: user_id(),+ thread_ids: vec![EMAIL_THREAD_ID, SECOND_EMAIL_THREAD_ID],+ reason: ThreadsReindexReason::ContactsChanged,+ }),+ ));+ assert_eq!(ordering_key(&multi_thread), EMAIL_LINK_ID.to_string());+}++#[test]+fn non_email_events_shard_by_their_producer_key() {+ let message = encoded_message(+ MacroCallsTopic::TOPIC_STR,+ CALL_ID,+ Event::new(archived_event()),+ );+ let call = DeclaredMacroEvent::decode(&message).expect("decodable call event");+ assert_eq!(ordering_key(&call), CALL_ID.to_string());++ let message = encoded_message(+ MacroDocumentsTopic::TOPIC_STR,+ DOCUMENT_ID,+ Event::new(DocumentTopicEvent::Purged(DocumentPurgedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ })),+ );+ let document = DeclaredMacroEvent::decode(&message).expect("decodable document event");+ assert_eq!(ordering_key(&document), DOCUMENT_ID);}#[test]-fn production_channel_and_retry_bounds_match_the_delivery_contract() {- let (sender, _receiver) = mpsc::channel::<ReceivedEvent>(CHANNEL_CAPACITY);- assert_eq!(sender.max_capacity(), 128);+fn production_worker_pool_and_retry_bounds_match_the_delivery_contract() {+ assert_eq!(WORKER_COUNT, 10);+ let (sender, _receiver) = mpsc::channel::<ReceivedEvent>(WORKER_CHANNEL_CAPACITY);+ assert_eq!(sender.max_capacity(), 16);assert_eq!(processing_retry_strategy().collect::<Vec<_>>(),[Duration::from_secs(1), Duration::from_secs(2)]