Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(soup-realtime): backpressure fanout and commit only after delivery (#6874)

Sean Aye4129c38ce7d6
diff --git a/crates/soup_realtime/src/domain/ports.rs b/crates/soup_realtime/src/domain/ports.rs
index 4c7595b53c..ec86034ac0 100644
--- a/crates/soup_realtime/src/domain/ports.rs
+++ b/crates/soup_realtime/src/domain/ports.rs
@@ -11,7 +11,13 @@ use super::models::{Patch, SoupRealtimeMessage, SoupRealtimePatch};
/// Inbound use-case port driven by entity update transports.
pub trait SoupRealtimeService: Send + Sync + 'static {
/// Publishes one entity patch to every current accessor of its access source.
- fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report>;
+ ///
+ /// Waits for capacity and returns success only after all publications succeed.
+ /// Failures may occur after partial delivery, so retries can produce duplicates.
+ fn notify_users(
+ &self,
+ patch: SoupRealtimePatch,
+ ) -> impl Future<Output = Result<(), Report>> + Send;
}
/// Receives recipient-targeted Soup patches.
diff --git a/crates/soup_realtime/src/domain/service.rs b/crates/soup_realtime/src/domain/service.rs
index 529d896140..d2fa886038 100644
--- a/crates/soup_realtime/src/domain/service.rs
+++ b/crates/soup_realtime/src/domain/service.rs
@@ -10,8 +10,9 @@ use futures::{StreamExt as _, stream};
use macro_user_id::user_id::MacroUserIdStr;
use model_entity::Entity;
use rootcause::prelude::{Report, ResultExt as _};
-use tokio::task::JoinHandle;
+use tokio::{sync::oneshot, task::JoinHandle};
use tokio_retry::{Retry, strategy::ExponentialBackoff};
+use tracing::Instrument as _;
use super::{
models::{Patch, SoupRealtimeMessage, SoupRealtimePatch},
@@ -113,24 +114,35 @@ const PUBLISH_CONCURRENCY: usize = 16;
/// Domain service that expands entity access and fans out lightweight patches.
pub struct SoupRealtimeServiceImpl {
- sender: tokio::sync::mpsc::Sender<SoupRealtimePatch>,
+ sender: tokio::sync::mpsc::Sender<PendingPatch>,
_bg: JoinHandle<()>,
}
+struct PendingPatch {
+ patch: SoupRealtimePatch,
+ completed: oneshot::Sender<Result<(), Report>>,
+ span: tracing::Span,
+}
+
struct Worker<A, P> {
access_expander: A,
publisher: P,
- rx: tokio::sync::mpsc::Receiver<SoupRealtimePatch>,
+ rx: tokio::sync::mpsc::Receiver<PendingPatch>,
}
impl<A: UserAccessExpander, P: SoupRealtimePublisher> Worker<A, P> {
- async fn run(&mut self) -> () {
- while let Some(msg) = self.rx.recv().await {
- let _ = self.process_one(msg).await;
+ async fn run(&mut self) {
+ while let Some(pending) = self.rx.recv().await {
+ let result = self
+ .process_one(pending.patch)
+ .instrument(pending.span)
+ .await;
+ // A cancelled caller leaves its source event uncommitted for replay.
+ let _ = pending.completed.send(result);
}
}
- #[tracing::instrument(err, skip(self))]
+ #[tracing::instrument(err, skip(self), fields(recipient_count))]
async fn process_one(&self, msg: SoupRealtimePatch) -> Result<(), Report> {
let mut users = self
.access_expander
@@ -196,7 +208,19 @@ impl SoupRealtimeServiceImpl {
impl SoupRealtimeService for SoupRealtimeServiceImpl {
#[tracing::instrument(skip(self), err)]
- fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {
- Ok(self.sender.try_send(patch)?)
+ async fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {
+ let (completed, result) = oneshot::channel();
+ self.sender
+ .send(PendingPatch {
+ patch,
+ completed,
+ span: tracing::Span::current(),
+ })
+ .await
+ .context("realtime Soup worker unavailable")?;
+
+ result
+ .await
+ .context("realtime Soup worker stopped before completing patch")?
}
}
diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rs
index 922b6c59ff..c9498ec7a8 100644
--- a/crates/soup_realtime/src/domain/service/test.rs
+++ b/crates/soup_realtime/src/domain/service/test.rs
@@ -70,6 +70,133 @@ impl SoupRealtimePublisher for FakePublisher {
}
}
+struct GatedPublisher {
+ messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,
+ permits: Arc<tokio::sync::Semaphore>,
+}
+
+impl SoupRealtimePublisher for GatedPublisher {
+ async fn publish(&self, message: SoupRealtimeMessage) -> Result<(), Report> {
+ self.messages.lock().expect("messages lock").push(message);
+ self.permits
+ .acquire()
+ .await
+ .expect("gate remains open")
+ .forget();
+ Ok(())
+ }
+}
+
+struct GatedHarness {
+ service: Arc<SoupRealtimeServiceImpl>,
+ messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,
+ permits: Arc<tokio::sync::Semaphore>,
+}
+
+fn gated_service() -> GatedHarness {
+ let messages = Arc::new(Mutex::new(Vec::new()));
+ let permits = Arc::new(tokio::sync::Semaphore::new(0));
+ let service = SoupRealtimeServiceImpl::new(
+ FakeAccessExpander {
+ users: vec![user("one")],
+ fail: false,
+ calls: Arc::new(AtomicUsize::new(0)),
+ entities: Arc::new(Mutex::new(Vec::new())),
+ },
+ GatedPublisher {
+ messages: messages.clone(),
+ permits: permits.clone(),
+ },
+ );
+ GatedHarness {
+ service: Arc::new(service),
+ messages,
+ permits,
+ }
+}
+
+#[tokio::test]
+async fn full_queue_waits_for_capacity_and_acknowledges_only_completed_publications() {
+ let GatedHarness {
+ service,
+ messages,
+ permits,
+ } = gated_service();
+ let request_count = BACKGROUND_CHANNEL_SIZE + 2;
+ let mut tasks = tokio::task::JoinSet::new();
+ for _ in 0..request_count {
+ let service = Arc::clone(&service);
+ tasks.spawn(async move {
+ service
+ .notify_users(SoupRealtimePatch::for_entity(updated_document()))
+ .await
+ });
+ }
+
+ wait_until(|| {
+ service.sender.capacity() == 0 && !messages.lock().expect("messages lock").is_empty()
+ })
+ .await;
+ assert!(
+ tasks.try_join_next().is_none(),
+ "no enqueue is acknowledged as delivery"
+ );
+ assert_eq!(messages.lock().expect("messages lock").len(), 1);
+
+ permits.add_permits(request_count);
+ tokio::time::timeout(Duration::from_secs(5), async {
+ while let Some(result) = tasks.join_next().await {
+ result
+ .expect("notification task joins")
+ .expect("patch is published");
+ }
+ })
+ .await
+ .expect("burst drains without dropping patches");
+ assert_eq!(messages.lock().expect("messages lock").len(), request_count);
+}
+
+#[tokio::test]
+async fn worker_stopping_during_publication_returns_an_error() {
+ let GatedHarness {
+ service, messages, ..
+ } = gated_service();
+ let task = tokio::spawn({
+ let service = Arc::clone(&service);
+ async move {
+ service
+ .notify_users(SoupRealtimePatch::for_entity(updated_document()))
+ .await
+ }
+ });
+ wait_until(|| !messages.lock().expect("messages lock").is_empty()).await;
+ service._bg.abort();
+
+ let error = task
+ .await
+ .expect("notification task joins")
+ .expect_err("worker stopped");
+ assert!(
+ error
+ .to_string()
+ .contains("worker stopped before completing patch")
+ );
+}
+
+#[tokio::test]
+async fn closed_worker_queue_returns_an_error() {
+ let harness = harness(Vec::new(), false, HashSet::new());
+ harness.service._bg.abort();
+ wait_until(|| harness.service.sender.is_closed()).await;
+
+ let error = harness
+ .service
+ .notify_users(SoupRealtimePatch::for_entity(updated_document()))
+ .await
+ .expect_err("worker unavailable");
+ assert!(error.to_string().contains("worker unavailable"));
+}
+
struct Harness {
service: SoupRealtimeServiceImpl,
access_calls: Arc<AtomicUsize>,
@@ -191,9 +318,10 @@ async fn zero_users_skips_publication() {
harness
.service
.notify_users(SoupRealtimePatch::for_entity(updated_document()))
- .expect("patch is queued");
+ .await
+ .expect("patch is processed");
- wait_until(|| harness.access_calls.load(Ordering::SeqCst) == 1).await;
+ assert_eq!(harness.access_calls.load(Ordering::SeqCst), 1);
assert!(harness.messages.lock().expect("messages lock").is_empty());
}
@@ -206,17 +334,8 @@ async fn recipient_expansion_can_use_a_different_entity_from_the_patch() {
harness
.service
.notify_users(SoupRealtimePatch::new(updated_document(), channel.clone()))
- .expect("patch is queued");
-
- wait_until(|| {
- harness
- .access_entities
- .lock()
- .expect("access entities lock")
- .len()
- == 1
- })
- .await;
+ .await
+ .expect("patch is published");
assert_eq!(
harness
@@ -240,9 +359,8 @@ async fn updated_and_deleted_patches_are_published_unchanged() {
harness
.service
.notify_users(SoupRealtimePatch::for_entity(patch.clone()))
- .expect("patch is queued");
-
- wait_until(|| harness.messages.lock().expect("messages lock").len() == 1).await;
+ .await
+ .expect("patch is published");
let messages = harness.messages.lock().expect("messages lock");
assert_eq!(
messages.as_slice(),
@@ -260,26 +378,33 @@ async fn duplicate_accessors_are_deduplicated() {
harness
.service
.notify_users(SoupRealtimePatch::for_entity(updated_document()))
- .expect("patch is queued");
+ .await
+ .expect("patch is published");
- wait_until(|| harness.messages.lock().expect("messages lock").len() == 2).await;
+ assert_eq!(harness.messages.lock().expect("messages lock").len(), 2);
}
#[tokio::test]
-async fn access_failure_prevents_publication() {
+async fn access_failure_is_returned_and_prevents_publication() {
let harness = harness(vec![user("one")], true, HashSet::new());
- harness
+ let error = harness
.service
.notify_users(SoupRealtimePatch::for_entity(updated_document()))
- .expect("patch is queued");
+ .await
+ .expect_err("access failure reaches the caller");
- wait_until(|| harness.access_calls.load(Ordering::SeqCst) == 1).await;
+ assert!(
+ error
+ .to_string()
+ .contains("failed to expand current user access")
+ );
+ assert_eq!(harness.access_calls.load(Ordering::SeqCst), 1);
assert!(harness.messages.lock().expect("messages lock").is_empty());
}
#[tokio::test]
-async fn publisher_failure_does_not_block_other_messages() {
+async fn publisher_failure_is_returned_after_attempting_all_recipients() {
let users = [user("one"), user("two"), user("three")];
let harness = harness(
users.to_vec(),
@@ -287,10 +412,16 @@ async fn publisher_failure_does_not_block_other_messages() {
HashSet::from([users[1].as_ref().to_string()]),
);
- harness
+ let error = harness
.service
.notify_users(SoupRealtimePatch::for_entity(updated_document()))
- .expect("patch is queued");
+ .await
+ .expect_err("publication failure reaches the caller");
- wait_until(|| harness.messages.lock().expect("messages lock").len() == 3).await;
+ assert!(
+ error
+ .to_string()
+ .contains("1 realtime Soup publication(s) failed")
+ );
+ assert_eq!(harness.messages.lock().expect("messages lock").len(), 3);
}
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rs
index a6e0d5b9a2..11cef443b8 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs
@@ -1,13 +1,15 @@
//! Kafka consumer for entity events that change realtime Soup items.
//!
-//! Delivery is at least once: offsets are committed only after successful
-//! domain processing. Malformed and recognized-but-irrelevant events are
-//! poison/ignored records and are committed so they cannot wedge a partition.
+//! Delivery is best effort with bounded retries: offsets are committed after
+//! successful domain processing or a logged terminal failure. Exhausted retries
+//! discard the rest of that event so it cannot wedge a partition. Malformed and
+//! recognized-but-irrelevant events are also committed. Cancellation leaves the
+//! in-flight event uncommitted; partial delivery and replay can produce duplicates.
#[cfg(test)]
mod test;
-use std::future::Future;
+use std::{future::Future, time::Duration};
use crate::domain::{
models::{Patch, SoupRealtimePatch},
@@ -32,6 +34,8 @@ use properties::domain::events::{PropertyMacroEvent, PropertyTopicEvent};
use rdkafka::consumer::CommitMode;
use rdkafka::message::{BorrowedMessage, Message as _};
use rootcause::prelude::{Report, ResultExt as _};
+use tokio_retry::{Retry, strategy::ExponentialBackoff};
+use tracing::Instrument as _;
/// Consumer group used for Soup-affecting entity event offsets.
struct SoupRealtimeConsumerGroup;
@@ -473,33 +477,78 @@ fn patches_from_event(event: &DeclaredMacroEvent) -> Vec<SoupRealtimePatch> {
}
}
+/// Total publication attempts per patch before discarding its source event.
+const MAX_NOTIFY_ATTEMPTS: usize = 5;
+
+/// Retries after one, two, four, and eight seconds.
+fn notify_retry_strategy() -> impl Iterator<Item = Duration> {
+ ExponentialBackoff::from_millis(2)
+ .factor(500)
+ .take(MAX_NOTIFY_ATTEMPTS - 1)
+}
+
/// Commit-safe outcome after processing one entity event.
enum EventOutcome {
- /// Every affected item was successfully sent through the domain service.
+ /// Every affected item was successfully published through the domain service.
Notified,
/// A recognized event does not change a Soup-visible entity.
Ignored,
+ /// A patch exhausted its retries; the remaining event was logged and discarded.
+ Dropped,
}
-#[tracing::instrument(skip(service, event), err)]
-fn process_event<S: SoupRealtimeService>(
+#[tracing::instrument(skip(service, event, commit))]
+async fn process_event<S: SoupRealtimeService>(
service: &S,
event: &DeclaredMacroEvent,
-) -> Result<EventOutcome, Report> {
+ commit: impl FnOnce(),
+) -> EventOutcome {
let patches = patches_from_event(event);
- if patches.is_empty() {
+ let mut outcome = if patches.is_empty() {
tracing::trace!("ignoring event without a Soup patch");
- return Ok(EventOutcome::Ignored);
- }
+ EventOutcome::Ignored
+ } else {
+ EventOutcome::Notified
+ };
- for patch in patches {
- service.notify_users(patch)?;
+ let patch_count = patches.len();
+ for (index, patch) in patches.into_iter().enumerate() {
+ let result = Retry::start(notify_retry_strategy(), || {
+ service.notify_users(patch.clone())
+ })
+ .await
+ .inspect_err(|error| {
+ tracing::error!(
+ error = ?error,
+ patch = ?patch,
+ attempts = MAX_NOTIFY_ATTEMPTS,
+ skipped_patch_count = patch_count - index - 1,
+ "discarding realtime Soup source event after exhausting notification retries"
+ );
+ });
+ if result.is_err() {
+ outcome = EventOutcome::Dropped;
+ break;
+ }
}
- Ok(EventOutcome::Notified)
+
+ commit();
+ outcome
}
fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage<'_>) {
- let _ = consumer.inner().commit_message(message, CommitMode::Async);
+ let _ = consumer
+ .inner()
+ .commit_message(message, CommitMode::Async)
+ .inspect_err(|error| {
+ tracing::error!(
+ error = ?error,
+ topic = message.topic(),
+ partition = message.partition(),
+ offset = message.offset(),
+ "failed to commit realtime Soup source offset"
+ );
+ });
}
impl SoupRealtimeServiceImpl {
@@ -507,9 +556,12 @@ impl SoupRealtimeServiceImpl {
///
/// The consumer subscribes to every existing entity topic with events that
/// can change a Soup item under the `soup-realtime` group. It commits malformed
- /// and recognized-but-ignored events, and commits affecting events only after
- /// [`SoupRealtimeService`] succeeds. Exhausted service retries return without
- /// committing so a future supervisor restart can redeliver the record.
+ /// and recognized-but-ignored events immediately. Affecting events are committed
+ /// after every patch succeeds or a patch exhausts its five publication attempts.
+ /// Exhaustion logs an error and discards the remaining event before committing,
+ /// intentionally preferring consumer progress over delivery during persistent
+ /// failures. Shutdown during processing or retry backoff leaves the event
+ /// uncommitted for replay.
#[tracing::instrument(skip(self, shutdown), fields(brokers), err)]
pub async fn run_entity_update_consumer(
&self,
@@ -537,14 +589,17 @@ impl SoupRealtimeServiceImpl {
result = consumer.recv() => {
let Ok(message) = result else { continue; };
let kafka_message = message.inner();
- let _message_span = tracing::info_span!(
+ let span = tracing::info_span!(
"realtime_soup_source_event",
topic = kafka_message.topic(),
partition = kafka_message.partition(),
offset = kafka_message.offset(),
- )
- .entered();
- let event = match message.decode_payload() {
+ );
+ let decoded = {
+ let _guard = span.enter();
+ message.decode_payload()
+ };
+ let event = match decoded {
Ok(event) => event,
Err(_) => {
commit_logged(&consumer, kafka_message);
@@ -552,9 +607,15 @@ impl SoupRealtimeServiceImpl {
}
};
- let _ = process_event(self,&event);
-
- commit_logged(&consumer, kafka_message);
+ // Do not advance until delivery succeeds or the event reaches
+ // the logged terminal-drop outcome after bounded retries.
+ tokio::select! {
+ biased;
+ _ = &mut shutdown => break,
+ _ = process_event(self, &event, || {
+ commit_logged(&consumer, kafka_message);
+ }).instrument(span) => {}
+ }
}
}
}
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
index 9099e9041f..f9a50c62d6 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
@@ -1,4 +1,5 @@
-use std::sync::atomic::{AtomicU32, Ordering};
+use std::collections::HashSet;
+use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use channels::domain::{
@@ -84,16 +85,16 @@ fn patch_entity(patch: &SoupRealtimePatch) -> &Entity<'static> {
#[derive(Clone)]
struct FlakyService {
- attempts: Arc<AtomicU32>,
- failures: u32,
+ attempts: Arc<AtomicUsize>,
+ failures: HashSet<usize>,
patches: Arc<Mutex<Vec<SoupRealtimePatch>>>,
}
impl SoupRealtimeService for FlakyService {
- fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {
+ async fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {
self.patches.lock().expect("patches lock").push(patch);
let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1;
- if attempt <= self.failures {
+ if self.failures.contains(&attempt) {
Err(rootcause::report!("temporary fan-out failure"))
} else {
Ok(())
@@ -101,10 +102,10 @@ impl SoupRealtimeService for FlakyService {
}
}
-fn flaky_service(failures: u32) -> FlakyService {
+fn flaky_service(failures: usize) -> FlakyService {
FlakyService {
- attempts: Arc::new(AtomicU32::new(0)),
- failures,
+ attempts: Arc::new(AtomicUsize::new(0)),
+ failures: (1..=failures).collect(),
patches: Arc::new(Mutex::new(Vec::new())),
}
}
@@ -611,18 +612,23 @@ fn deleting_a_root_channel_message_deletes_its_thread_patch() {
assert_eq!(patches[1].access_source.entity_type, EntityType::Channel);
}
-#[test]
-fn updated_payload_maps_to_document_patch() {
+#[tokio::test]
+async fn updated_payload_maps_to_document_patch_and_commits() {
let event = DeclaredMacroEvent::DocumentMacroEvent(DocumentMacroEvent::with_event(
DOCUMENT_ID,
updated_event(),
));
let service = flaky_service(0);
+ let commits = AtomicUsize::new(0);
assert!(matches!(
- process_event(&service, &event).expect("processing succeeds"),
+ process_event(&service, &event, || {
+ commits.fetch_add(1, Ordering::SeqCst);
+ })
+ .await,
EventOutcome::Notified
));
+ assert_eq!(commits.load(Ordering::SeqCst), 1);
let patches = service.patches.lock().expect("patches lock");
assert_eq!(patches.len(), 1);
@@ -644,16 +650,221 @@ fn malformed_and_unknown_events_are_rejected_by_the_declared_collection() {
assert!(decode_payload(payload).is_err());
}
-#[test]
-fn service_failure_is_returned_without_retry() {
+#[tokio::test(start_paused = true)]
+async fn transient_service_failures_are_retried_before_committing() {
+ let event = DeclaredMacroEvent::DocumentMacroEvent(DocumentMacroEvent::with_event(
+ DOCUMENT_ID,
+ updated_event(),
+ ));
+ let service = flaky_service(MAX_NOTIFY_ATTEMPTS - 1);
+ let commits = AtomicUsize::new(0);
+ let started = tokio::time::Instant::now();
+
+ let outcome = process_event(&service, &event, || {
+ assert_eq!(service.attempts.load(Ordering::SeqCst), MAX_NOTIFY_ATTEMPTS);
+ commits.fetch_add(1, Ordering::SeqCst);
+ })
+ .await;
+
+ assert!(matches!(outcome, EventOutcome::Notified));
+ assert_eq!(commits.load(Ordering::SeqCst), 1);
+ assert_eq!(started.elapsed(), Duration::from_secs(15));
+}
+
+#[tokio::test(start_paused = true)]
+async fn exhausted_service_retries_drop_and_commit_the_event() {
let event = DeclaredMacroEvent::DocumentMacroEvent(DocumentMacroEvent::with_event(
DOCUMENT_ID,
updated_event(),
));
- let service = flaky_service(1);
+ let service = flaky_service(MAX_NOTIFY_ATTEMPTS);
+ let commits = AtomicUsize::new(0);
+ let started = tokio::time::Instant::now();
+
+ let outcome = process_event(&service, &event, || {
+ assert_eq!(service.attempts.load(Ordering::SeqCst), MAX_NOTIFY_ATTEMPTS);
+ assert_eq!(started.elapsed(), Duration::from_secs(15));
+ commits.fetch_add(1, Ordering::SeqCst);
+ })
+ .await;
+
+ assert!(matches!(outcome, EventOutcome::Dropped));
+ assert_eq!(commits.load(Ordering::SeqCst), 1);
+}
- assert!(process_event(&service, &event).is_err());
+#[tokio::test(start_paused = true)]
+async fn dropped_event_does_not_prevent_later_events_from_being_processed() {
+ let first_event = reindex_event(vec![Uuid::now_v7()]);
+ let next_event = reindex_event(vec![Uuid::now_v7()]);
+ let service = flaky_service(MAX_NOTIFY_ATTEMPTS);
+ let committed_offsets = Mutex::new(Vec::new());
+
+ let first_outcome = process_event(&service, &first_event, || {
+ assert_eq!(service.attempts.load(Ordering::SeqCst), MAX_NOTIFY_ATTEMPTS);
+ committed_offsets.lock().expect("offsets lock").push(0);
+ })
+ .await;
+ assert!(matches!(first_outcome, EventOutcome::Dropped));
+
+ let next_outcome = process_event(&service, &next_event, || {
+ assert_eq!(
+ service.attempts.load(Ordering::SeqCst),
+ MAX_NOTIFY_ATTEMPTS + 1
+ );
+ committed_offsets.lock().expect("offsets lock").push(1);
+ })
+ .await;
+ assert!(matches!(next_outcome, EventOutcome::Notified));
+ assert_eq!(*committed_offsets.lock().expect("offsets lock"), vec![0, 1]);
+}
+
+fn reindex_event(thread_ids: Vec<Uuid>) -> DeclaredMacroEvent {
+ DeclaredMacroEvent::EmailMacroEvent(EmailMacroEvent::threads_reindex_requested(
+ ThreadsReindexRequestedMetadata {
+ link_id: Uuid::now_v7(),
+ owner: user(),
+ thread_ids,
+ reason: ThreadsReindexReason::ContactsChanged,
+ },
+ ))
+}
+
+#[tokio::test(start_paused = true)]
+async fn multi_patch_event_retries_only_the_failed_patch_before_committing() {
+ let thread_ids = vec![Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()];
+ let event = reindex_event(thread_ids.clone());
+ let mut service = flaky_service(0);
+ service.failures.insert(2);
+ let commits = AtomicUsize::new(0);
+
+ let outcome = process_event(&service, &event, || {
+ assert_eq!(service.attempts.load(Ordering::SeqCst), 4);
+ commits.fetch_add(1, Ordering::SeqCst);
+ })
+ .await;
+
+ assert!(matches!(outcome, EventOutcome::Notified));
+ let patches = service.patches.lock().expect("patches lock");
+ let attempts: Vec<_> = patches
+ .iter()
+ .map(|patch| patch_entity(patch).entity_id.as_ref())
+ .collect();
+ let expected: Vec<_> = [thread_ids[0], thread_ids[1], thread_ids[1], thread_ids[2]]
+ .into_iter()
+ .map(|id| id.to_string())
+ .collect();
+ assert_eq!(attempts, expected);
+ assert_eq!(commits.load(Ordering::SeqCst), 1);
+}
+
+#[tokio::test(start_paused = true)]
+async fn partial_event_failure_discards_remaining_patches_and_commits() {
+ let thread_ids = vec![Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()];
+ let event = reindex_event(thread_ids.clone());
+ let mut service = flaky_service(0);
+ service.failures = (2..=MAX_NOTIFY_ATTEMPTS + 1).collect();
+ let commits = AtomicUsize::new(0);
+
+ let outcome = process_event(&service, &event, || {
+ assert_eq!(
+ service.attempts.load(Ordering::SeqCst),
+ MAX_NOTIFY_ATTEMPTS + 1
+ );
+ commits.fetch_add(1, Ordering::SeqCst);
+ })
+ .await;
+
+ assert!(matches!(outcome, EventOutcome::Dropped));
+ assert_eq!(commits.load(Ordering::SeqCst), 1);
+ let patches = service.patches.lock().expect("patches lock");
+ assert_eq!(
+ patch_entity(&patches[0]).entity_id,
+ thread_ids[0].to_string()
+ );
+ assert!(
+ patches[1..]
+ .iter()
+ .all(|patch| { patch_entity(patch).entity_id == thread_ids[1].to_string() })
+ );
+}
+
+#[tokio::test]
+async fn ignored_event_commits_without_notifying() {
+ let event = reindex_event(Vec::new());
+ let service = flaky_service(0);
+ let commits = AtomicUsize::new(0);
+
+ let outcome = process_event(&service, &event, || {
+ commits.fetch_add(1, Ordering::SeqCst);
+ })
+ .await;
+
+ assert!(matches!(outcome, EventOutcome::Ignored));
+ assert_eq!(service.attempts.load(Ordering::SeqCst), 0);
+ assert_eq!(commits.load(Ordering::SeqCst), 1);
+}
+
+struct GatedService(tokio::sync::Semaphore);
+
+impl SoupRealtimeService for GatedService {
+ async fn notify_users(&self, _patch: SoupRealtimePatch) -> Result<(), Report> {
+ self.0.acquire().await.expect("gate remains open").forget();
+ Ok(())
+ }
+}
+
+#[tokio::test]
+async fn offset_is_committed_only_after_every_patch_finishes() {
+ let event = reindex_event(vec![Uuid::now_v7(), Uuid::now_v7()]);
+ let service = GatedService(tokio::sync::Semaphore::new(0));
+ let commits = AtomicUsize::new(0);
+ let process = process_event(&service, &event, || {
+ commits.fetch_add(1, Ordering::SeqCst);
+ });
+ tokio::pin!(process);
+
+ assert!(futures::poll!(&mut process).is_pending());
+ assert_eq!(commits.load(Ordering::SeqCst), 0);
+ service.0.add_permits(1);
+ assert!(futures::poll!(&mut process).is_pending());
+ assert_eq!(commits.load(Ordering::SeqCst), 0);
+ service.0.add_permits(1);
+ assert!(matches!(process.await, EventOutcome::Notified));
+ assert_eq!(commits.load(Ordering::SeqCst), 1);
+}
+
+#[tokio::test]
+async fn cancellation_during_publication_leaves_the_event_uncommitted() {
+ let event = reindex_event(vec![Uuid::now_v7()]);
+ let service = GatedService(tokio::sync::Semaphore::new(0));
+ let commits = AtomicUsize::new(0);
+ {
+ let process = process_event(&service, &event, || {
+ commits.fetch_add(1, Ordering::SeqCst);
+ });
+ tokio::pin!(process);
+ assert!(futures::poll!(&mut process).is_pending());
+ }
+ service.0.add_permits(1);
+ assert_eq!(commits.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test(start_paused = true)]
+async fn cancellation_during_retry_backoff_leaves_the_event_uncommitted() {
+ let event = reindex_event(vec![Uuid::now_v7()]);
+ let service = flaky_service(MAX_NOTIFY_ATTEMPTS);
+ let commits = AtomicUsize::new(0);
+ {
+ let process = process_event(&service, &event, || {
+ commits.fetch_add(1, Ordering::SeqCst);
+ });
+ tokio::pin!(process);
+ assert!(futures::poll!(&mut process).is_pending());
+ assert_eq!(service.attempts.load(Ordering::SeqCst), 1);
+ assert_eq!(commits.load(Ordering::SeqCst), 0);
+ }
+ tokio::time::advance(Duration::from_secs(15)).await;
assert_eq!(service.attempts.load(Ordering::SeqCst), 1);
- assert_eq!(service.patches.lock().expect("patches lock").len(), 1);
+ assert_eq!(commits.load(Ordering::SeqCst), 0);
}