Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(soup-realtime): backpressure fanout and commit only after delivery (#6874)
Sean Aye4129c38ce7d6
diff --git a/crates/soup_realtime/src/domain/ports.rs b/crates/soup_realtime/src/domain/ports.rsindex 4c7595b53c..ec86034ac0 100644--- a/crates/soup_realtime/src/domain/ports.rs+++ b/crates/soup_realtime/src/domain/ports.rs@@ -11,7 +11,13 @@ use super::models::{Patch, SoupRealtimeMessage, SoupRealtimePatch};/// Inbound use-case port driven by entity update transports.pub trait SoupRealtimeService: Send + Sync + 'static {/// Publishes one entity patch to every current accessor of its access source.- fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report>;+ ///+ /// Waits for capacity and returns success only after all publications succeed.+ /// Failures may occur after partial delivery, so retries can produce duplicates.+ fn notify_users(+ &self,+ patch: SoupRealtimePatch,+ ) -> impl Future<Output = Result<(), Report>> + Send;}/// Receives recipient-targeted Soup patches.diff --git a/crates/soup_realtime/src/domain/service.rs b/crates/soup_realtime/src/domain/service.rsindex 529d896140..d2fa886038 100644--- a/crates/soup_realtime/src/domain/service.rs+++ b/crates/soup_realtime/src/domain/service.rs@@ -10,8 +10,9 @@ use futures::{StreamExt as _, stream};use macro_user_id::user_id::MacroUserIdStr;use model_entity::Entity;use rootcause::prelude::{Report, ResultExt as _};-use tokio::task::JoinHandle;+use tokio::{sync::oneshot, task::JoinHandle};use tokio_retry::{Retry, strategy::ExponentialBackoff};+use tracing::Instrument as _;use super::{models::{Patch, SoupRealtimeMessage, SoupRealtimePatch},@@ -113,24 +114,35 @@ const PUBLISH_CONCURRENCY: usize = 16;/// Domain service that expands entity access and fans out lightweight patches.pub struct SoupRealtimeServiceImpl {- sender: tokio::sync::mpsc::Sender<SoupRealtimePatch>,+ sender: tokio::sync::mpsc::Sender<PendingPatch>,_bg: JoinHandle<()>,}+struct PendingPatch {+ patch: SoupRealtimePatch,+ completed: oneshot::Sender<Result<(), Report>>,+ span: tracing::Span,+}+struct Worker<A, P> {access_expander: A,publisher: P,- rx: tokio::sync::mpsc::Receiver<SoupRealtimePatch>,+ rx: tokio::sync::mpsc::Receiver<PendingPatch>,}impl<A: UserAccessExpander, P: SoupRealtimePublisher> Worker<A, P> {- async fn run(&mut self) -> () {- while let Some(msg) = self.rx.recv().await {- let _ = self.process_one(msg).await;+ async fn run(&mut self) {+ while let Some(pending) = self.rx.recv().await {+ let result = self+ .process_one(pending.patch)+ .instrument(pending.span)+ .await;+ // A cancelled caller leaves its source event uncommitted for replay.+ let _ = pending.completed.send(result);}}- #[tracing::instrument(err, skip(self))]+ #[tracing::instrument(err, skip(self), fields(recipient_count))]async fn process_one(&self, msg: SoupRealtimePatch) -> Result<(), Report> {let mut users = self.access_expander@@ -196,7 +208,19 @@ impl SoupRealtimeServiceImpl {impl SoupRealtimeService for SoupRealtimeServiceImpl {#[tracing::instrument(skip(self), err)]- fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {- Ok(self.sender.try_send(patch)?)+ async fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {+ let (completed, result) = oneshot::channel();+ self.sender+ .send(PendingPatch {+ patch,+ completed,+ span: tracing::Span::current(),+ })+ .await+ .context("realtime Soup worker unavailable")?;++ result+ .await+ .context("realtime Soup worker stopped before completing patch")?}}diff --git a/crates/soup_realtime/src/domain/service/test.rs b/crates/soup_realtime/src/domain/service/test.rsindex 922b6c59ff..c9498ec7a8 100644--- a/crates/soup_realtime/src/domain/service/test.rs+++ b/crates/soup_realtime/src/domain/service/test.rs@@ -70,6 +70,133 @@ impl SoupRealtimePublisher for FakePublisher {}}+struct GatedPublisher {+ messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,+ permits: Arc<tokio::sync::Semaphore>,+}++impl SoupRealtimePublisher for GatedPublisher {+ async fn publish(&self, message: SoupRealtimeMessage) -> Result<(), Report> {+ self.messages.lock().expect("messages lock").push(message);+ self.permits+ .acquire()+ .await+ .expect("gate remains open")+ .forget();+ Ok(())+ }+}++struct GatedHarness {+ service: Arc<SoupRealtimeServiceImpl>,+ messages: Arc<Mutex<Vec<SoupRealtimeMessage>>>,+ permits: Arc<tokio::sync::Semaphore>,+}++fn gated_service() -> GatedHarness {+ let messages = Arc::new(Mutex::new(Vec::new()));+ let permits = Arc::new(tokio::sync::Semaphore::new(0));+ let service = SoupRealtimeServiceImpl::new(+ FakeAccessExpander {+ users: vec![user("one")],+ fail: false,+ calls: Arc::new(AtomicUsize::new(0)),+ entities: Arc::new(Mutex::new(Vec::new())),+ },+ GatedPublisher {+ messages: messages.clone(),+ permits: permits.clone(),+ },+ );+ GatedHarness {+ service: Arc::new(service),+ messages,+ permits,+ }+}++#[tokio::test]+async fn full_queue_waits_for_capacity_and_acknowledges_only_completed_publications() {+ let GatedHarness {+ service,+ messages,+ permits,+ } = gated_service();+ let request_count = BACKGROUND_CHANNEL_SIZE + 2;+ let mut tasks = tokio::task::JoinSet::new();+ for _ in 0..request_count {+ let service = Arc::clone(&service);+ tasks.spawn(async move {+ service+ .notify_users(SoupRealtimePatch::for_entity(updated_document()))+ .await+ });+ }++ wait_until(|| {+ service.sender.capacity() == 0 && !messages.lock().expect("messages lock").is_empty()+ })+ .await;+ assert!(+ tasks.try_join_next().is_none(),+ "no enqueue is acknowledged as delivery"+ );+ assert_eq!(messages.lock().expect("messages lock").len(), 1);++ permits.add_permits(request_count);+ tokio::time::timeout(Duration::from_secs(5), async {+ while let Some(result) = tasks.join_next().await {+ result+ .expect("notification task joins")+ .expect("patch is published");+ }+ })+ .await+ .expect("burst drains without dropping patches");+ assert_eq!(messages.lock().expect("messages lock").len(), request_count);+}++#[tokio::test]+async fn worker_stopping_during_publication_returns_an_error() {+ let GatedHarness {+ service, messages, ..+ } = gated_service();+ let task = tokio::spawn({+ let service = Arc::clone(&service);+ async move {+ service+ .notify_users(SoupRealtimePatch::for_entity(updated_document()))+ .await+ }+ });+ wait_until(|| !messages.lock().expect("messages lock").is_empty()).await;+ service._bg.abort();++ let error = task+ .await+ .expect("notification task joins")+ .expect_err("worker stopped");+ assert!(+ error+ .to_string()+ .contains("worker stopped before completing patch")+ );+}++#[tokio::test]+async fn closed_worker_queue_returns_an_error() {+ let harness = harness(Vec::new(), false, HashSet::new());+ harness.service._bg.abort();+ wait_until(|| harness.service.sender.is_closed()).await;++ let error = harness+ .service+ .notify_users(SoupRealtimePatch::for_entity(updated_document()))+ .await+ .expect_err("worker unavailable");+ assert!(error.to_string().contains("worker unavailable"));+}+struct Harness {service: SoupRealtimeServiceImpl,access_calls: Arc<AtomicUsize>,@@ -191,9 +318,10 @@ async fn zero_users_skips_publication() {harness.service.notify_users(SoupRealtimePatch::for_entity(updated_document()))- .expect("patch is queued");+ .await+ .expect("patch is processed");- wait_until(|| harness.access_calls.load(Ordering::SeqCst) == 1).await;+ assert_eq!(harness.access_calls.load(Ordering::SeqCst), 1);assert!(harness.messages.lock().expect("messages lock").is_empty());}@@ -206,17 +334,8 @@ async fn recipient_expansion_can_use_a_different_entity_from_the_patch() {harness.service.notify_users(SoupRealtimePatch::new(updated_document(), channel.clone()))- .expect("patch is queued");-- wait_until(|| {- harness- .access_entities- .lock()- .expect("access entities lock")- .len()- == 1- })- .await;+ .await+ .expect("patch is published");assert_eq!(harness@@ -240,9 +359,8 @@ async fn updated_and_deleted_patches_are_published_unchanged() {harness.service.notify_users(SoupRealtimePatch::for_entity(patch.clone()))- .expect("patch is queued");-- wait_until(|| harness.messages.lock().expect("messages lock").len() == 1).await;+ .await+ .expect("patch is published");let messages = harness.messages.lock().expect("messages lock");assert_eq!(messages.as_slice(),@@ -260,26 +378,33 @@ async fn duplicate_accessors_are_deduplicated() {harness.service.notify_users(SoupRealtimePatch::for_entity(updated_document()))- .expect("patch is queued");+ .await+ .expect("patch is published");- wait_until(|| harness.messages.lock().expect("messages lock").len() == 2).await;+ assert_eq!(harness.messages.lock().expect("messages lock").len(), 2);}#[tokio::test]-async fn access_failure_prevents_publication() {+async fn access_failure_is_returned_and_prevents_publication() {let harness = harness(vec![user("one")], true, HashSet::new());- harness+ let error = harness.service.notify_users(SoupRealtimePatch::for_entity(updated_document()))- .expect("patch is queued");+ .await+ .expect_err("access failure reaches the caller");- wait_until(|| harness.access_calls.load(Ordering::SeqCst) == 1).await;+ assert!(+ error+ .to_string()+ .contains("failed to expand current user access")+ );+ assert_eq!(harness.access_calls.load(Ordering::SeqCst), 1);assert!(harness.messages.lock().expect("messages lock").is_empty());}#[tokio::test]-async fn publisher_failure_does_not_block_other_messages() {+async fn publisher_failure_is_returned_after_attempting_all_recipients() {let users = [user("one"), user("two"), user("three")];let harness = harness(users.to_vec(),@@ -287,10 +412,16 @@ async fn publisher_failure_does_not_block_other_messages() {HashSet::from([users[1].as_ref().to_string()]),);- harness+ let error = harness.service.notify_users(SoupRealtimePatch::for_entity(updated_document()))- .expect("patch is queued");+ .await+ .expect_err("publication failure reaches the caller");- wait_until(|| harness.messages.lock().expect("messages lock").len() == 3).await;+ assert!(+ error+ .to_string()+ .contains("1 realtime Soup publication(s) failed")+ );+ assert_eq!(harness.messages.lock().expect("messages lock").len(), 3);}diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rsindex a6e0d5b9a2..11cef443b8 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs@@ -1,13 +1,15 @@//! Kafka consumer for entity events that change realtime Soup items.//!-//! Delivery is at least once: offsets are committed only after successful-//! domain processing. Malformed and recognized-but-irrelevant events are-//! poison/ignored records and are committed so they cannot wedge a partition.+//! Delivery is best effort with bounded retries: offsets are committed after+//! successful domain processing or a logged terminal failure. Exhausted retries+//! discard the rest of that event so it cannot wedge a partition. Malformed and+//! recognized-but-irrelevant events are also committed. Cancellation leaves the+//! in-flight event uncommitted; partial delivery and replay can produce duplicates.#[cfg(test)]mod test;-use std::future::Future;+use std::{future::Future, time::Duration};use crate::domain::{models::{Patch, SoupRealtimePatch},@@ -32,6 +34,8 @@ use properties::domain::events::{PropertyMacroEvent, PropertyTopicEvent};use rdkafka::consumer::CommitMode;use rdkafka::message::{BorrowedMessage, Message as _};use rootcause::prelude::{Report, ResultExt as _};+use tokio_retry::{Retry, strategy::ExponentialBackoff};+use tracing::Instrument as _;/// Consumer group used for Soup-affecting entity event offsets.struct SoupRealtimeConsumerGroup;@@ -473,33 +477,78 @@ fn patches_from_event(event: &DeclaredMacroEvent) -> Vec<SoupRealtimePatch> {}}+/// Total publication attempts per patch before discarding its source event.+const MAX_NOTIFY_ATTEMPTS: usize = 5;++/// Retries after one, two, four, and eight seconds.+fn notify_retry_strategy() -> impl Iterator<Item = Duration> {+ ExponentialBackoff::from_millis(2)+ .factor(500)+ .take(MAX_NOTIFY_ATTEMPTS - 1)+}+/// Commit-safe outcome after processing one entity event.enum EventOutcome {- /// Every affected item was successfully sent through the domain service.+ /// Every affected item was successfully published through the domain service.Notified,/// A recognized event does not change a Soup-visible entity.Ignored,+ /// A patch exhausted its retries; the remaining event was logged and discarded.+ Dropped,}-#[tracing::instrument(skip(service, event), err)]-fn process_event<S: SoupRealtimeService>(+#[tracing::instrument(skip(service, event, commit))]+async fn process_event<S: SoupRealtimeService>(service: &S,event: &DeclaredMacroEvent,-) -> Result<EventOutcome, Report> {+ commit: impl FnOnce(),+) -> EventOutcome {let patches = patches_from_event(event);- if patches.is_empty() {+ let mut outcome = if patches.is_empty() {tracing::trace!("ignoring event without a Soup patch");- return Ok(EventOutcome::Ignored);- }+ EventOutcome::Ignored+ } else {+ EventOutcome::Notified+ };- for patch in patches {- service.notify_users(patch)?;+ let patch_count = patches.len();+ for (index, patch) in patches.into_iter().enumerate() {+ let result = Retry::start(notify_retry_strategy(), || {+ service.notify_users(patch.clone())+ })+ .await+ .inspect_err(|error| {+ tracing::error!(+ error = ?error,+ patch = ?patch,+ attempts = MAX_NOTIFY_ATTEMPTS,+ skipped_patch_count = patch_count - index - 1,+ "discarding realtime Soup source event after exhausting notification retries"+ );+ });+ if result.is_err() {+ outcome = EventOutcome::Dropped;+ break;+ }}- Ok(EventOutcome::Notified)++ commit();+ outcome}fn commit_logged(consumer: &SoupRealtimeKafkaConsumer, message: &BorrowedMessage<'_>) {- let _ = consumer.inner().commit_message(message, CommitMode::Async);+ let _ = consumer+ .inner()+ .commit_message(message, CommitMode::Async)+ .inspect_err(|error| {+ tracing::error!(+ error = ?error,+ topic = message.topic(),+ partition = message.partition(),+ offset = message.offset(),+ "failed to commit realtime Soup source offset"+ );+ });}impl SoupRealtimeServiceImpl {@@ -507,9 +556,12 @@ impl SoupRealtimeServiceImpl {////// The consumer subscribes to every existing entity topic with events that/// can change a Soup item under the `soup-realtime` group. It commits malformed- /// and recognized-but-ignored events, and commits affecting events only after- /// [`SoupRealtimeService`] succeeds. Exhausted service retries return without- /// committing so a future supervisor restart can redeliver the record.+ /// and recognized-but-ignored events immediately. Affecting events are committed+ /// after every patch succeeds or a patch exhausts its five publication attempts.+ /// Exhaustion logs an error and discards the remaining event before committing,+ /// intentionally preferring consumer progress over delivery during persistent+ /// failures. Shutdown during processing or retry backoff leaves the event+ /// uncommitted for replay.#[tracing::instrument(skip(self, shutdown), fields(brokers), err)]pub async fn run_entity_update_consumer(&self,@@ -537,14 +589,17 @@ impl SoupRealtimeServiceImpl {result = consumer.recv() => {let Ok(message) = result else { continue; };let kafka_message = message.inner();- let _message_span = tracing::info_span!(+ let span = tracing::info_span!("realtime_soup_source_event",topic = kafka_message.topic(),partition = kafka_message.partition(),offset = kafka_message.offset(),- )- .entered();- let event = match message.decode_payload() {+ );+ let decoded = {+ let _guard = span.enter();+ message.decode_payload()+ };+ let event = match decoded {Ok(event) => event,Err(_) => {commit_logged(&consumer, kafka_message);@@ -552,9 +607,15 @@ impl SoupRealtimeServiceImpl {}};- let _ = process_event(self,&event);-- commit_logged(&consumer, kafka_message);+ // Do not advance until delivery succeeds or the event reaches+ // the logged terminal-drop outcome after bounded retries.+ tokio::select! {+ biased;+ _ = &mut shutdown => break,+ _ = process_event(self, &event, || {+ commit_logged(&consumer, kafka_message);+ }).instrument(span) => {}+ }}}}diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rsindex 9099e9041f..f9a50c62d6 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs@@ -1,4 +1,5 @@-use std::sync::atomic::{AtomicU32, Ordering};+use std::collections::HashSet;+use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::{Arc, Mutex};use channels::domain::{@@ -84,16 +85,16 @@ fn patch_entity(patch: &SoupRealtimePatch) -> &Entity<'static> {#[derive(Clone)]struct FlakyService {- attempts: Arc<AtomicU32>,- failures: u32,+ attempts: Arc<AtomicUsize>,+ failures: HashSet<usize>,patches: Arc<Mutex<Vec<SoupRealtimePatch>>>,}impl SoupRealtimeService for FlakyService {- fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {+ async fn notify_users(&self, patch: SoupRealtimePatch) -> Result<(), Report> {self.patches.lock().expect("patches lock").push(patch);let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1;- if attempt <= self.failures {+ if self.failures.contains(&attempt) {Err(rootcause::report!("temporary fan-out failure"))} else {Ok(())@@ -101,10 +102,10 @@ impl SoupRealtimeService for FlakyService {}}-fn flaky_service(failures: u32) -> FlakyService {+fn flaky_service(failures: usize) -> FlakyService {FlakyService {- attempts: Arc::new(AtomicU32::new(0)),- failures,+ attempts: Arc::new(AtomicUsize::new(0)),+ failures: (1..=failures).collect(),patches: Arc::new(Mutex::new(Vec::new())),}}@@ -611,18 +612,23 @@ fn deleting_a_root_channel_message_deletes_its_thread_patch() {assert_eq!(patches[1].access_source.entity_type, EntityType::Channel);}-#[test]-fn updated_payload_maps_to_document_patch() {+#[tokio::test]+async fn updated_payload_maps_to_document_patch_and_commits() {let event = DeclaredMacroEvent::DocumentMacroEvent(DocumentMacroEvent::with_event(DOCUMENT_ID,updated_event(),));let service = flaky_service(0);+ let commits = AtomicUsize::new(0);assert!(matches!(- process_event(&service, &event).expect("processing succeeds"),+ process_event(&service, &event, || {+ commits.fetch_add(1, Ordering::SeqCst);+ })+ .await,EventOutcome::Notified));+ assert_eq!(commits.load(Ordering::SeqCst), 1);let patches = service.patches.lock().expect("patches lock");assert_eq!(patches.len(), 1);@@ -644,16 +650,221 @@ fn malformed_and_unknown_events_are_rejected_by_the_declared_collection() {assert!(decode_payload(payload).is_err());}-#[test]-fn service_failure_is_returned_without_retry() {+#[tokio::test(start_paused = true)]+async fn transient_service_failures_are_retried_before_committing() {+ let event = DeclaredMacroEvent::DocumentMacroEvent(DocumentMacroEvent::with_event(+ DOCUMENT_ID,+ updated_event(),+ ));+ let service = flaky_service(MAX_NOTIFY_ATTEMPTS - 1);+ let commits = AtomicUsize::new(0);+ let started = tokio::time::Instant::now();++ let outcome = process_event(&service, &event, || {+ assert_eq!(service.attempts.load(Ordering::SeqCst), MAX_NOTIFY_ATTEMPTS);+ commits.fetch_add(1, Ordering::SeqCst);+ })+ .await;++ assert!(matches!(outcome, EventOutcome::Notified));+ assert_eq!(commits.load(Ordering::SeqCst), 1);+ assert_eq!(started.elapsed(), Duration::from_secs(15));+}++#[tokio::test(start_paused = true)]+async fn exhausted_service_retries_drop_and_commit_the_event() {let event = DeclaredMacroEvent::DocumentMacroEvent(DocumentMacroEvent::with_event(DOCUMENT_ID,updated_event(),));- let service = flaky_service(1);+ let service = flaky_service(MAX_NOTIFY_ATTEMPTS);+ let commits = AtomicUsize::new(0);+ let started = tokio::time::Instant::now();++ let outcome = process_event(&service, &event, || {+ assert_eq!(service.attempts.load(Ordering::SeqCst), MAX_NOTIFY_ATTEMPTS);+ assert_eq!(started.elapsed(), Duration::from_secs(15));+ commits.fetch_add(1, Ordering::SeqCst);+ })+ .await;++ assert!(matches!(outcome, EventOutcome::Dropped));+ assert_eq!(commits.load(Ordering::SeqCst), 1);+}- assert!(process_event(&service, &event).is_err());+#[tokio::test(start_paused = true)]+async fn dropped_event_does_not_prevent_later_events_from_being_processed() {+ let first_event = reindex_event(vec![Uuid::now_v7()]);+ let next_event = reindex_event(vec![Uuid::now_v7()]);+ let service = flaky_service(MAX_NOTIFY_ATTEMPTS);+ let committed_offsets = Mutex::new(Vec::new());++ let first_outcome = process_event(&service, &first_event, || {+ assert_eq!(service.attempts.load(Ordering::SeqCst), MAX_NOTIFY_ATTEMPTS);+ committed_offsets.lock().expect("offsets lock").push(0);+ })+ .await;+ assert!(matches!(first_outcome, EventOutcome::Dropped));++ let next_outcome = process_event(&service, &next_event, || {+ assert_eq!(+ service.attempts.load(Ordering::SeqCst),+ MAX_NOTIFY_ATTEMPTS + 1+ );+ committed_offsets.lock().expect("offsets lock").push(1);+ })+ .await;+ assert!(matches!(next_outcome, EventOutcome::Notified));+ assert_eq!(*committed_offsets.lock().expect("offsets lock"), vec![0, 1]);+}++fn reindex_event(thread_ids: Vec<Uuid>) -> DeclaredMacroEvent {+ DeclaredMacroEvent::EmailMacroEvent(EmailMacroEvent::threads_reindex_requested(+ ThreadsReindexRequestedMetadata {+ link_id: Uuid::now_v7(),+ owner: user(),+ thread_ids,+ reason: ThreadsReindexReason::ContactsChanged,+ },+ ))+}++#[tokio::test(start_paused = true)]+async fn multi_patch_event_retries_only_the_failed_patch_before_committing() {+ let thread_ids = vec![Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()];+ let event = reindex_event(thread_ids.clone());+ let mut service = flaky_service(0);+ service.failures.insert(2);+ let commits = AtomicUsize::new(0);++ let outcome = process_event(&service, &event, || {+ assert_eq!(service.attempts.load(Ordering::SeqCst), 4);+ commits.fetch_add(1, Ordering::SeqCst);+ })+ .await;++ assert!(matches!(outcome, EventOutcome::Notified));+ let patches = service.patches.lock().expect("patches lock");+ let attempts: Vec<_> = patches+ .iter()+ .map(|patch| patch_entity(patch).entity_id.as_ref())+ .collect();+ let expected: Vec<_> = [thread_ids[0], thread_ids[1], thread_ids[1], thread_ids[2]]+ .into_iter()+ .map(|id| id.to_string())+ .collect();+ assert_eq!(attempts, expected);+ assert_eq!(commits.load(Ordering::SeqCst), 1);+}++#[tokio::test(start_paused = true)]+async fn partial_event_failure_discards_remaining_patches_and_commits() {+ let thread_ids = vec![Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()];+ let event = reindex_event(thread_ids.clone());+ let mut service = flaky_service(0);+ service.failures = (2..=MAX_NOTIFY_ATTEMPTS + 1).collect();+ let commits = AtomicUsize::new(0);++ let outcome = process_event(&service, &event, || {+ assert_eq!(+ service.attempts.load(Ordering::SeqCst),+ MAX_NOTIFY_ATTEMPTS + 1+ );+ commits.fetch_add(1, Ordering::SeqCst);+ })+ .await;++ assert!(matches!(outcome, EventOutcome::Dropped));+ assert_eq!(commits.load(Ordering::SeqCst), 1);+ let patches = service.patches.lock().expect("patches lock");+ assert_eq!(+ patch_entity(&patches[0]).entity_id,+ thread_ids[0].to_string()+ );+ assert!(+ patches[1..]+ .iter()+ .all(|patch| { patch_entity(patch).entity_id == thread_ids[1].to_string() })+ );+}++#[tokio::test]+async fn ignored_event_commits_without_notifying() {+ let event = reindex_event(Vec::new());+ let service = flaky_service(0);+ let commits = AtomicUsize::new(0);++ let outcome = process_event(&service, &event, || {+ commits.fetch_add(1, Ordering::SeqCst);+ })+ .await;++ assert!(matches!(outcome, EventOutcome::Ignored));+ assert_eq!(service.attempts.load(Ordering::SeqCst), 0);+ assert_eq!(commits.load(Ordering::SeqCst), 1);+}++struct GatedService(tokio::sync::Semaphore);++impl SoupRealtimeService for GatedService {+ async fn notify_users(&self, _patch: SoupRealtimePatch) -> Result<(), Report> {+ self.0.acquire().await.expect("gate remains open").forget();+ Ok(())+ }+}++#[tokio::test]+async fn offset_is_committed_only_after_every_patch_finishes() {+ let event = reindex_event(vec![Uuid::now_v7(), Uuid::now_v7()]);+ let service = GatedService(tokio::sync::Semaphore::new(0));+ let commits = AtomicUsize::new(0);+ let process = process_event(&service, &event, || {+ commits.fetch_add(1, Ordering::SeqCst);+ });+ tokio::pin!(process);++ assert!(futures::poll!(&mut process).is_pending());+ assert_eq!(commits.load(Ordering::SeqCst), 0);+ service.0.add_permits(1);+ assert!(futures::poll!(&mut process).is_pending());+ assert_eq!(commits.load(Ordering::SeqCst), 0);+ service.0.add_permits(1);+ assert!(matches!(process.await, EventOutcome::Notified));+ assert_eq!(commits.load(Ordering::SeqCst), 1);+}++#[tokio::test]+async fn cancellation_during_publication_leaves_the_event_uncommitted() {+ let event = reindex_event(vec![Uuid::now_v7()]);+ let service = GatedService(tokio::sync::Semaphore::new(0));+ let commits = AtomicUsize::new(0);+ {+ let process = process_event(&service, &event, || {+ commits.fetch_add(1, Ordering::SeqCst);+ });+ tokio::pin!(process);+ assert!(futures::poll!(&mut process).is_pending());+ }+ service.0.add_permits(1);+ assert_eq!(commits.load(Ordering::SeqCst), 0);+}++#[tokio::test(start_paused = true)]+async fn cancellation_during_retry_backoff_leaves_the_event_uncommitted() {+ let event = reindex_event(vec![Uuid::now_v7()]);+ let service = flaky_service(MAX_NOTIFY_ATTEMPTS);+ let commits = AtomicUsize::new(0);+ {+ let process = process_event(&service, &event, || {+ commits.fetch_add(1, Ordering::SeqCst);+ });+ tokio::pin!(process);+ assert!(futures::poll!(&mut process).is_pending());+ assert_eq!(service.attempts.load(Ordering::SeqCst), 1);+ assert_eq!(commits.load(Ordering::SeqCst), 0);+ }+ tokio::time::advance(Duration::from_secs(15)).await;assert_eq!(service.attempts.load(Ordering::SeqCst), 1);- assert_eq!(service.patches.lock().expect("patches lock").len(), 1);+ assert_eq!(commits.load(Ordering::SeqCst), 0);}