Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(kafka): use tokio tasktracker for kafka event broker (#5206)
Will Hutchinsoned112e00b5cf
diff --git a/Cargo.lock b/Cargo.lockindex bd0c4f8389..89a003cfae 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -245,6 +245,7 @@ dependencies = ["system_properties","teams","tokio",+ "tokio-util","tracing","utoipa","uuid",@@ -1043,6 +1044,7 @@ dependencies = ["thiserror 2.0.18","time","tokio",+ "tokio-util","tower 0.5.3","tower-cookies","tower-http",@@ -4336,6 +4338,7 @@ dependencies = ["task_dedup","thiserror 2.0.18","tokio",+ "tokio-util","tower 0.5.3","tower-http","tracing",@@ -4980,6 +4983,7 @@ dependencies = ["thiserror 2.0.18","tokio","tokio-retry",+ "tokio-util","tower 0.5.3","tower-http","tracing",@@ -8424,6 +8428,7 @@ dependencies = ["rootcause","rootcause-tracing","serde_json",+ "tokio","tracing","tracing-opentelemetry","tracing-panic",@@ -8468,6 +8473,7 @@ dependencies = ["serde_json","thiserror 2.0.18","tokio",+ "tokio-util","tracing","uuid","workspace-hack",@@ -8837,6 +8843,7 @@ dependencies = ["sqs_client","sync_service_client","tokio",+ "tokio-util","tracing","workspace-hack",]@@ -8915,6 +8922,7 @@ dependencies = ["sync_service_client","thiserror 2.0.18","tokio",+ "tokio-util","tracing","utoipa","workspace-hack",@@ -12481,6 +12489,7 @@ dependencies = ["serde_json","sqlx","tokio",+ "tokio-util","tracing","utoipa","utoipa-swagger-ui",diff --git a/crates/ai_tools/Cargo.toml b/crates/ai_tools/Cargo.tomlindex a6d00a8c80..916de28414 100644--- a/crates/ai_tools/Cargo.toml+++ b/crates/ai_tools/Cargo.toml@@ -80,6 +80,7 @@ strum.workspace = truesync_service_client = { path = "../sync_service_client" }system_properties = { path = "../system_properties" }tokio.workspace = true+tokio-util = { workspace = true, features = ["rt"] }tracing.workspace = trueutoipa.workspace = trueuuid.workspace = truediff --git a/crates/ai_tools/src/build_context.rs b/crates/ai_tools/src/build_context.rsindex a711c2c424..8662891fca 100644--- a/crates/ai_tools/src/build_context.rs+++ b/crates/ai_tools/src/build_context.rs@@ -47,6 +47,7 @@ use soup::domain::service::SoupImpl;use soup::outbound::pg_soup_repo::PgSoupRepo;use std::sync::Arc;use sync_service_client::SyncServiceClient;+use tokio_util::task::TaskTracker;env_var! {struct ToolContextEnvVars {@@ -97,9 +98,14 @@ maybe_env_var! {/// - `ENABLE_EMAIL_SCHEDULED_QUEUE`/// - `ENABLE_GMAIL_OPS_QUEUE` (if disabled, thread-label updates can't enqueue Gmail sync ops)/// - `ENABLE_NOTIFICATION_QUEUE` (if disabled, notification status updates skip push clearing)-#[tracing::instrument(skip(pool), err)]+///+/// `event_task_tracker` tracks event publishes started by the context. Callers+/// must retain the original tracker, pass a clone here, and close and drain the+/// original after the host stops broker-backed work.+#[tracing::instrument(skip(pool, event_task_tracker), err)]pub async fn build_tool_service_context_from_env(pool: sqlx::PgPool,+ event_task_tracker: TaskTracker,) -> anyhow::Result<ToolServiceContext> {let env = ToolContextEnvVars::new()?;let maybe_env = ToolContextMaybeEnvVars::new();@@ -254,6 +260,7 @@ pub async fn build_tool_service_context_from_env(let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(macro_event_broker::KafkaEventPublisher::new(env.kafka_brokers.as_ref()).context("failed to create kafka event publisher")?,+ event_task_tracker,);// Channel messages sent by AI tools dispatch the same side effects as the// document-storage channel API (realtime, notifications, search indexing,diff --git a/crates/ai_tools/src/tool_context.rs b/crates/ai_tools/src/tool_context.rsindex 2538fe14d4..78f47e85cc 100644--- a/crates/ai_tools/src/tool_context.rs+++ b/crates/ai_tools/src/tool_context.rs@@ -47,6 +47,7 @@ use system_properties::{SystemPropertiesServiceImpl,};use teams::{inbound::toolset::TeamToolContext, outbound::team_repo::TeamRepositoryImpl};+use tokio_util::task::TaskTracker;pub use ai_toolset::RequestContext;@@ -81,6 +82,10 @@ pub type ToolEmailService = EmailServiceImpl<ToolEamService,>;+/// Event broker used by AI tools, with spawned publish tasks tracked for+/// graceful shutdown by the hosting process.+pub type ToolEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;+/// Type alias for the send-capable email service implementation used by user/// tools. Carries the real event broker: these tools mutate email state, and/// the Gmail-echo suppression means events skipped here are never recovered.@@ -90,7 +95,7 @@ pub type ToolUserEmailService = EmailServiceImpl<sqs_client::SQS,ToolCrmService,ToolEamService,- macro_event_broker::MacroEventBrokerService<macro_event_broker::KafkaEventPublisher>,+ ToolEventBroker,>;/// Type alias for the channel list service implementation.@@ -148,7 +153,7 @@ pub struct ChannelSideEffectClients {/// search-event queues.pub sqs: aws_sdk_sqs::Client,/// Broker publishing channel events to the `macro.channels` topic.- pub macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,+ pub macro_event_broker: ToolEventBroker,}/// Build the channel AI tool context dispatching the same side effects as the@@ -591,7 +596,7 @@ pub type ToolDocumentService = documents::domain::service::DocumentServiceImpl<NoOpConnectionService,ToolEntityAccessManagementService,ToolForeignEntityService,- macro_event_broker::MacroEventBrokerService<macro_event_broker::KafkaEventPublisher>,+ ToolEventBroker,>;/// Type alias for the entity access service implementationdiff --git a/crates/macro_entrypoint/Cargo.toml b/crates/macro_entrypoint/Cargo.tomlindex 3639cb39f3..db4c09fbc3 100644--- a/crates/macro_entrypoint/Cargo.toml+++ b/crates/macro_entrypoint/Cargo.toml@@ -13,6 +13,7 @@ opentelemetry-otlp = { workspace = true }rootcause = { workspace = true }rootcause-tracing = { workspace = true }serde_json = { workspace = true }+tokio = { workspace = true, features = ["signal"] }tracing = { workspace = true }tracing-opentelemetry = { workspace = true }tracing-panic = "0.1"@@ -23,3 +24,4 @@ macro_env_var = { path = "../macro_env_var" }[dev-dependencies]opentelemetry_sdk = { workspace = true, features = ["testing"] }+tokio = { workspace = true, features = ["time"] }diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rsindex 9329d5a2a9..7922a41e63 100644--- a/crates/macro_entrypoint/src/lib.rs+++ b/crates/macro_entrypoint/src/lib.rs@@ -3,10 +3,13 @@//! This is used to provide consistent behaviour with e.g. tracing configurationsmod datadog_fmt;+mod shutdown;#[cfg(test)]mod test;+pub use shutdown::shutdown_signal;+use macro_env::Environment;use macro_env_var::{env_vars, maybe_env_vars};use opentelemetry::trace::TracerProvider as _;diff --git a/crates/macro_entrypoint/src/shutdown.rs b/crates/macro_entrypoint/src/shutdown.rsnew file mode 100644index 0000000000..d601c192c6--- /dev/null+++ b/crates/macro_entrypoint/src/shutdown.rs@@ -0,0 +1,58 @@+use std::{+ future::{Future, pending},+ io,+};++#[cfg(test)]+mod test;++async fn wait_for_signal_listener<F>(listener: F, signal: &'static str)+where+ F: Future<Output = io::Result<()>>,+{+ match listener.await {+ Ok(()) => {}+ Err(error) => {+ tracing::error!(error=?error, signal, "failed to install shutdown signal listener");+ pending::<()>().await;+ }+ }+}++async fn wait_for_shutdown_signals<CtrlC, Terminate>(ctrl_c: CtrlC, terminate: Terminate)+where+ CtrlC: Future<Output = io::Result<()>>,+ Terminate: Future<Output = io::Result<()>>,+{+ tokio::select! {+ _ = wait_for_signal_listener(ctrl_c, "Ctrl+C") => {}+ _ = wait_for_signal_listener(terminate, "SIGTERM") => {}+ }+}++#[cfg(unix)]+async fn terminate_signal() -> io::Result<()> {+ let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;++ match terminate.recv().await {+ Some(()) => Ok(()),+ None => pending().await,+ }+}++/// Waits for Ctrl+C on all platforms or SIGTERM on Unix.+///+/// A listener installation error is logged and leaves that listener pending so it cannot be+/// mistaken for a shutdown signal.+pub async fn shutdown_signal() {+ let ctrl_c = tokio::signal::ctrl_c();++ #[cfg(unix)]+ let terminate = terminate_signal();++ #[cfg(not(unix))]+ let terminate = pending::<io::Result<()>>();++ wait_for_shutdown_signals(ctrl_c, terminate).await;+ tracing::info!("shutdown signal received");+}diff --git a/crates/macro_entrypoint/src/shutdown/test.rs b/crates/macro_entrypoint/src/shutdown/test.rsnew file mode 100644index 0000000000..12b3bcf246--- /dev/null+++ b/crates/macro_entrypoint/src/shutdown/test.rs@@ -0,0 +1,55 @@+use super::wait_for_shutdown_signals;++use std::{future::pending, io, time::Duration};+use tokio::{task::yield_now, time::timeout};++#[tokio::test]+async fn failed_signal_listener_does_not_resolve_shutdown() {+ let ctrl_c = async { Err(io::Error::other("listener installation failed")) };+ let terminate = pending::<io::Result<()>>();++ let result = timeout(+ Duration::from_millis(10),+ wait_for_shutdown_signals(ctrl_c, terminate),+ )+ .await;++ assert!(+ result.is_err(),+ "a listener error must not trigger shutdown"+ );+}++#[tokio::test]+async fn successful_signal_resolves_shutdown() {+ let ctrl_c = async { Ok(()) };+ let terminate = pending::<io::Result<()>>();++ let result = timeout(+ Duration::from_millis(100),+ wait_for_shutdown_signals(ctrl_c, terminate),+ )+ .await;++ assert!(result.is_ok(), "an actual signal must trigger shutdown");+}++#[tokio::test]+async fn successful_signal_resolves_after_other_listener_fails() {+ let ctrl_c = async { Err(io::Error::other("listener installation failed")) };+ let terminate = async {+ yield_now().await;+ Ok(())+ };++ let result = timeout(+ Duration::from_millis(100),+ wait_for_shutdown_signals(ctrl_c, terminate),+ )+ .await;++ assert!(+ result.is_ok(),+ "a listener error must not mask a later signal"+ );+}diff --git a/crates/macro_event_broker/Cargo.toml b/crates/macro_event_broker/Cargo.tomlindex 78cab6a390..f1229ebddc 100644--- a/crates/macro_event_broker/Cargo.toml+++ b/crates/macro_event_broker/Cargo.toml@@ -6,7 +6,7 @@ version = "0.1.0"[features]default = ["outbound"]-outbound = ["dep:kafka_util", "dep:rdkafka"]+outbound = ["dep:kafka_util", "dep:rdkafka", "dep:tokio-util"][dependencies]kafka_util = { path = "../kafka_util", optional = true }@@ -15,7 +15,8 @@ rdkafka = { workspace = true, optional = true }serde = { workspace = true }serde_json = { workspace = true }thiserror = { workspace = true }-tokio = { workspace = true, features = ["macros", "time"] }+tokio = { workspace = true, features = ["macros", "rt", "time"] }+tokio-util = { workspace = true, features = ["rt"], optional = true }tracing = { workspace = true }uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/macro_event_broker/examples/example_event.rs b/crates/macro_event_broker/examples/example_event.rsindex 3f75f41038..163311f92a 100644--- a/crates/macro_event_broker/examples/example_event.rs+++ b/crates/macro_event_broker/examples/example_event.rs@@ -1,6 +1,6 @@use macro_event_broker::{- Event, EventBrokerError, EventPublisher, MacroEvent, MacroEventBroker, MacroEventBrokerService,- TopicEvent,+ Event, EventBrokerError, EventPublisher, GlobalSpawner, MacroEvent, MacroEventBroker,+ MacroEventBrokerService, TopicEvent,};use macro_event_topics::{MacroExampleTopic, Topic};use serde::{Deserialize, Serialize};@@ -121,7 +121,7 @@ impl ExampleConsumerEvent {#[tokio::main]pub async fn main() -> Result<(), EventBrokerError> {- let service = MacroEventBrokerService::new(ExampleEventPublisher);+ let service = MacroEventBrokerService::new(ExampleEventPublisher, GlobalSpawner);let event = ExampleMacroEvent::created("example-123",ExampleCreatedMetadata {diff --git a/crates/macro_event_broker/src/domain/ports.rs b/crates/macro_event_broker/src/domain/ports.rsindex f9886ed7b4..2ab980d125 100644--- a/crates/macro_event_broker/src/domain/ports.rs+++ b/crates/macro_event_broker/src/domain/ports.rs@@ -34,6 +34,15 @@ pub trait EventPublisher: Send + Sync + 'static {) -> impl Future<Output = Result<(), EventBrokerError>> + Send;}+/// Outbound port for spawning the background task that performs an event publish.+pub trait Spawner: Send + Sync + 'static {+ /// Spawns `future` onto the async runtime and returns a handle to the task.+ fn spawn<F>(&self, future: F) -> tokio::task::JoinHandle<F::Output>+ where+ F: Future + Send + 'static,+ F::Output: Send + 'static;+}+/// A collection of [`MacroEvent`] types that can decode a broker message.pub trait MacroEventCollection: Sized {/// Decodes an event from the supplied broker message.diff --git a/crates/macro_event_broker/src/domain/service.rs b/crates/macro_event_broker/src/domain/service.rsindex 748f3fe3b1..a9b5be68dd 100644--- a/crates/macro_event_broker/src/domain/service.rs+++ b/crates/macro_event_broker/src/domain/service.rs@@ -11,7 +11,9 @@ use tracing::Instrument as _;use crate::MessageWrapper;use crate::domain::models::{EventBrokerError, MacroEvent, TopicEvent};-use crate::domain::ports::{EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection};+use crate::domain::ports::{+ EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection, Spawner,+};const PUBLISH_TIMEOUT: Duration = Duration::from_secs(6);@@ -58,28 +60,35 @@ where}/// Orchestrates serializing events and handing them to an [`EventPublisher`].-pub struct MacroEventBrokerService<P: EventPublisher> {+pub struct MacroEventBrokerService<P: EventPublisher, S: Spawner> {publisher: Arc<P>,+ spawner: S,}-impl<P: EventPublisher> Clone for MacroEventBrokerService<P> {+impl<P, S> Clone for MacroEventBrokerService<P, S>+where+ P: EventPublisher,+ S: Spawner + Clone,+{fn clone(&self) -> Self {Self {publisher: Arc::clone(&self.publisher),+ spawner: self.spawner.clone(),}}}-impl<P: EventPublisher> MacroEventBrokerService<P> {- /// Create a new service backed by the given outbound publisher.- pub fn new(publisher: P) -> Self {+impl<P: EventPublisher, S: Spawner> MacroEventBrokerService<P, S> {+ /// Creates a new service backed by the given outbound publisher and task spawner.+ pub fn new(publisher: P, spawner: S) -> Self {Self {publisher: Arc::new(publisher),+ spawner,}}}-impl<P: EventPublisher> MacroEventBroker for MacroEventBrokerService<P> {+impl<P: EventPublisher, S: Spawner> MacroEventBroker for MacroEventBrokerService<P, S> {#[tracing::instrument(err, skip(self, event), fields(topic = event.topic(), key = %event.key()))]fn send_event<E: MacroEvent + ?Sized>(&self,@@ -91,7 +100,7 @@ impl<P: EventPublisher> MacroEventBroker for MacroEventBrokerService<P> {let publisher = Arc::clone(&self.publisher);let span = tracing::Span::current();- let handle = tokio::spawn(+ let handle = self.spawner.spawn(async move {tokio::time::timeout(PUBLISH_TIMEOUT,diff --git a/crates/macro_event_broker/src/domain/service/test.rs b/crates/macro_event_broker/src/domain/service/test.rsindex a1e2ccfd4c..fc252c272e 100644--- a/crates/macro_event_broker/src/domain/service/test.rs+++ b/crates/macro_event_broker/src/domain/service/test.rs@@ -1,17 +1,22 @@use std::future::pending;use std::sync::{Arc, Mutex,- atomic::{AtomicBool, Ordering},+ atomic::{AtomicBool, AtomicUsize, Ordering},};use std::time::Duration;use macro_event_topics::{MacroExampleTopic, Topic};use serde::{Deserialize, Serialize};+use tokio::sync::Notify;+use tokio_util::task::TaskTracker;use uuid::Uuid;use super::*;use crate::domain::models::{Event, EventBrokerError, MacroEvent, MessageWrapper, TopicEvent};-use crate::domain::ports::{EventConsumer, EventPublisher, MacroEventBroker, MessageParts};+use crate::domain::ports::{+ EventConsumer, EventPublisher, MacroEventBroker, MessageParts, Spawner,+};+use crate::outbound::spawner::GlobalSpawner;/// A captured publish call: topic, key, and raw payload bytes.#[derive(Debug, Clone, PartialEq, Eq)]@@ -38,6 +43,21 @@ impl EventPublisher for RecordingPublisher {}}+struct RecordingSpawner {+ spawn_count: Arc<AtomicUsize>,+}++impl Spawner for RecordingSpawner {+ fn spawn<F>(&self, future: F) -> tokio::task::JoinHandle<F::Output>+ where+ F: Future + Send + 'static,+ F::Output: Send + 'static,+ {+ self.spawn_count.fetch_add(1, Ordering::SeqCst);+ tokio::spawn(future)+ }+}+struct PendingPublisher {started: Arc<AtomicBool>,}@@ -49,6 +69,19 @@ impl EventPublisher for PendingPublisher {}}+struct GatedPublisher {+ started: Arc<AtomicBool>,+ release: Arc<Notify>,+}++impl EventPublisher for GatedPublisher {+ async fn publish<T: Topic>(&self, _key: &str, _payload: &[u8]) -> Result<(), EventBrokerError> {+ self.started.store(true, Ordering::SeqCst);+ self.release.notified().await;+ Ok(())+ }+}+struct PublishDropGuard {dropped: Arc<AtomicBool>,}@@ -244,9 +277,15 @@ impl MacroEvent for UnserializableMacroEvent {}}+fn unserializable_event() -> UnserializableMacroEvent {+ UnserializableMacroEvent {+ event: Event::with_event_id(Uuid::from_u128(2), UnserializableTopicEvent),+ }+}+#[tokio::test]async fn dispatch_serializes_and_routes() {- let service = MacroEventBrokerService::new(RecordingPublisher::default());+ let service = MacroEventBrokerService::new(RecordingPublisher::default(), GlobalSpawner);let event = example_event();let expected_payload = serde_json::to_vec(event.event()).unwrap();@@ -268,9 +307,12 @@ async fn dispatch_serializes_and_routes() {#[tokio::test]async fn dispatch_returns_before_publish_completes() {let started = Arc::new(AtomicBool::new(false));- let service = MacroEventBrokerService::new(PendingPublisher {- started: Arc::clone(&started),- });+ let service = MacroEventBrokerService::new(+ PendingPublisher {+ started: Arc::clone(&started),+ },+ GlobalSpawner,+ );let handle = service.send_event(&example_event())@@ -287,10 +329,13 @@ async fn dispatch_returns_before_publish_completes() {async fn dispatch_cancels_publish_at_six_second_timeout() {let started = Arc::new(AtomicBool::new(false));let dropped = Arc::new(AtomicBool::new(false));- let service = MacroEventBrokerService::new(HangingPublisher {- started: Arc::clone(&started),- dropped: Arc::clone(&dropped),- });+ let service = MacroEventBrokerService::new(+ HangingPublisher {+ started: Arc::clone(&started),+ dropped: Arc::clone(&dropped),+ },+ GlobalSpawner,+ );let handle = service.send_event(&example_event())@@ -317,9 +362,12 @@ async fn dispatch_cancels_publish_at_six_second_timeout() {#[tokio::test]async fn dispatch_returns_publisher_failure_from_task() {let attempted = Arc::new(AtomicBool::new(false));- let service = MacroEventBrokerService::new(FailingPublisher {- attempted: Arc::clone(&attempted),- });+ let service = MacroEventBrokerService::new(+ FailingPublisher {+ attempted: Arc::clone(&attempted),+ },+ GlobalSpawner,+ );let error = service.send_event(&example_event())@@ -332,15 +380,123 @@ async fn dispatch_returns_publisher_failure_from_task() {assert!(attempted.load(Ordering::SeqCst));}+#[tokio::test]+async fn dispatch_uses_injected_spawner_once() {+ let spawn_count = Arc::new(AtomicUsize::new(0));+ let service = MacroEventBrokerService::new(+ RecordingPublisher::default(),+ RecordingSpawner {+ spawn_count: Arc::clone(&spawn_count),+ },+ );++ service+ .send_event(&example_event())+ .expect("dispatch should succeed")+ .await+ .expect("publish task should complete")+ .expect("publish should succeed");++ assert_eq!(spawn_count.load(Ordering::SeqCst), 1);+}++#[tokio::test]+async fn task_tracker_waits_for_unfinished_publish() {+ let started = Arc::new(AtomicBool::new(false));+ let release = Arc::new(Notify::new());+ let tracker = TaskTracker::new();+ let service = MacroEventBrokerService::new(+ GatedPublisher {+ started: Arc::clone(&started),+ release: Arc::clone(&release),+ },+ tracker.clone(),+ );++ let handle = service+ .send_event(&example_event())+ .expect("dispatch should succeed");+ tokio::task::yield_now().await;+ assert!(started.load(Ordering::SeqCst));++ tracker.close();+ let wait = tracker.wait();+ tokio::pin!(wait);+ tokio::select! {+ () = &mut wait => panic!("tracker should wait for the unfinished publish"),+ () = tokio::task::yield_now() => {}+ }++ release.notify_one();+ handle+ .await+ .expect("publish task should complete")+ .expect("publish should succeed");+ wait.await;+}++#[tokio::test(start_paused = true)]+async fn task_tracker_drains_hanging_publish_after_timeout() {+ let started = Arc::new(AtomicBool::new(false));+ let dropped = Arc::new(AtomicBool::new(false));+ let tracker = TaskTracker::new();+ let service = MacroEventBrokerService::new(+ HangingPublisher {+ started: Arc::clone(&started),+ dropped: Arc::clone(&dropped),+ },+ tracker.clone(),+ );++ let handle = service+ .send_event(&example_event())+ .expect("dispatch should succeed");+ tokio::task::yield_now().await;+ assert!(started.load(Ordering::SeqCst));++ tracker.close();+ tokio::time::advance(Duration::from_secs(5)).await;+ tokio::task::yield_now().await;+ assert!(!dropped.load(Ordering::SeqCst));++ tokio::time::advance(Duration::from_secs(1)).await;+ tracker.wait().await;+ let error = handle+ .await+ .expect("publish task should complete")+ .unwrap_err();++ assert!(matches!(+ error,+ EventBrokerError::PublishTimeout { timeout } if timeout == PUBLISH_TIMEOUT+ ));+ assert!(dropped.load(Ordering::SeqCst));+}+#[tokio::test]async fn dispatch_returns_serialization_failure_without_publishing() {- let service = MacroEventBrokerService::new(RecordingPublisher::default());- let event = UnserializableMacroEvent {- event: Event::with_event_id(Uuid::from_u128(2), UnserializableTopicEvent),- };+ let service = MacroEventBrokerService::new(RecordingPublisher::default(), GlobalSpawner);++ let error = service.send_event(&unserializable_event()).unwrap_err();- let error = service.send_event(&event).unwrap_err();assert!(matches!(error, EventBrokerError::Serialization(_)));tokio::task::yield_now().await;assert!(service.publisher.calls.lock().unwrap().is_empty());}++#[tokio::test]+async fn serialization_failure_does_not_invoke_spawner_or_publisher() {+ let spawn_count = Arc::new(AtomicUsize::new(0));+ let service = MacroEventBrokerService::new(+ RecordingPublisher::default(),+ RecordingSpawner {+ spawn_count: Arc::clone(&spawn_count),+ },+ );++ let error = service.send_event(&unserializable_event()).unwrap_err();++ assert!(matches!(error, EventBrokerError::Serialization(_)));+ assert_eq!(spawn_count.load(Ordering::SeqCst), 0);+ assert!(service.publisher.calls.lock().unwrap().is_empty());+}diff --git a/crates/macro_event_broker/src/lib.rs b/crates/macro_event_broker/src/lib.rsindex 6b2ffc2f16..fe63f4064c 100644--- a/crates/macro_event_broker/src/lib.rs+++ b/crates/macro_event_broker/src/lib.rs@@ -5,13 +5,14 @@//! [`TopicEvent`](domain::models::TopicEvent) enums, the//! [`MacroEvent`](domain::models::MacroEvent) event abstraction, the inbound//! [`MacroEventBroker`](domain::ports::MacroEventBroker) API, the outbound-//! [`EventPublisher`](domain::ports::EventPublisher) port, the producing+//! [`EventPublisher`](domain::ports::EventPublisher) and+//! [`Spawner`](domain::ports::Spawner) ports, the producing//! [`MacroEventBrokerService`](domain::service::MacroEventBrokerService), and the//! typed [`MacroEventConsumerService`](domain::service::MacroEventConsumerService),//! which receives messages through the [`EventConsumer`](domain::ports::EventConsumer) port.//! Kafka topic definitions live in the `macro_event_topics` crate. Shared Kafka//! producer and consumer transports live in `kafka_util`, while the [`outbound`]-//! layer adapts its producer to [`EventPublisher`](domain::ports::EventPublisher).+//! layer provides Kafka and Tokio adapters for the domain ports./// Domain layer: models, ports, and service.pub mod domain;@@ -23,7 +24,7 @@ pub use macro_event_topics::{};pub use domain::ports::{- EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection, MessageParts,+ EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection, MessageParts, Spawner,};pub use domain::service::{MacroEventBrokerService, MacroEventConsumerService, NoopMacroEventBroker,@@ -31,6 +32,7 @@ pub use domain::service::{#[cfg(feature = "outbound")]pub use outbound::{kafka_event_consumer::KafkaConsumerAdapter, kafka_event_publisher::KafkaEventPublisher,+ spawner::GlobalSpawner,};/// Outbound adapters for the macro event broker's required ports.diff --git a/crates/macro_event_broker/src/outbound/mod.rs b/crates/macro_event_broker/src/outbound/mod.rsindex 4fef313236..1202e04a34 100644--- a/crates/macro_event_broker/src/outbound/mod.rs+++ b/crates/macro_event_broker/src/outbound/mod.rs@@ -4,3 +4,6 @@ pub mod kafka_event_consumer;/// Kafka adapter implementing the [`EventPublisher`](crate::domain::ports::EventPublisher) port.#[cfg(feature = "outbound")]pub mod kafka_event_publisher;+/// Tokio adapters implementing the [`Spawner`](crate::domain::ports::Spawner) port.+#[cfg(feature = "outbound")]+pub mod spawner;diff --git a/crates/macro_event_broker/src/outbound/spawner.rs b/crates/macro_event_broker/src/outbound/spawner.rsnew file mode 100644index 0000000000..c3829600eb--- /dev/null+++ b/crates/macro_event_broker/src/outbound/spawner.rs@@ -0,0 +1,30 @@+//! Tokio task-spawning adapters.++use tokio::task::JoinHandle;+use tokio_util::task::TaskTracker;++use crate::domain::ports::Spawner;++/// A [`Spawner`] that uses the current Tokio runtime without tracking tasks.+#[derive(Debug, Clone, Copy, Default)]+pub struct GlobalSpawner;++impl Spawner for GlobalSpawner {+ fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>+ where+ F: Future + Send + 'static,+ F::Output: Send + 'static,+ {+ tokio::spawn(future)+ }+}++impl Spawner for TaskTracker {+ fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>+ where+ F: Future + Send + 'static,+ F::Output: Send + 'static,+ {+ TaskTracker::spawn(self, future)+ }+}diff --git a/crates/memory/Cargo.toml b/crates/memory/Cargo.tomlindex 2ba8738e4e..780f3d82b9 100644--- a/crates/memory/Cargo.toml+++ b/crates/memory/Cargo.toml@@ -28,6 +28,7 @@ serde_json = { workspace = true }sqlx = { workspace = true }thiserror = { workspace = true }tokio = { workspace = true }+tokio-util = { workspace = true, features = ["rt"] }tracing = { workspace = true }utoipa = { workspace = true }diff --git a/crates/memory/src/main.rs b/crates/memory/src/main.rsindex d8f4a7854f..992b09a1da 100644--- a/crates/memory/src/main.rs+++ b/crates/memory/src/main.rs@@ -7,6 +7,10 @@ use memory::config::Config;use memory::domain::{MemoryService, service::MemoryServiceImpl};use memory::outbound::pg_memory_repo::PgMemoryRepo;use sqlx::postgres::PgPoolOptions;+use std::time::Duration;+use tokio_util::task::TaskTracker;++const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);#[tokio::main]#[tracing::instrument(err)]@@ -14,21 +18,36 @@ async fn main() -> anyhow::Result<()> {let config = Config::from_env().context("failed to load memory configuration")?;macro_entrypoint::MacroEntrypoint::new(config.environment).init();+ let user = MacroUserIdStr::try_from(config.user_id.clone())+ .context("USER_ID must be a valid Macro user id")?;+let pool = PgPoolOptions::new().max_connections(5).connect(&config.database_url).await?;- let tool_context = build_tool_service_context_from_env(pool.clone()).await?;+ let event_broker_tracker = TaskTracker::new();+ let tool_context =+ build_tool_service_context_from_env(pool.clone(), event_broker_tracker.clone()).await?;let tools = all_tools();let memory_repo = PgMemoryRepo::new(pool.clone());let memory_service = MemoryServiceImpl::new(pool, memory_repo, tool_context, tools);- let user = MacroUserIdStr::try_from(config.user_id.clone())- .context("USER_ID must be a valid Macro user id")?;-tracing::info!("Generating memory for {user}...");- match memory_service.get_or_generate_memory(user).await? {+ let memory_result = memory_service.get_or_generate_memory(user).await;++ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => tracing::warn!(+ error = ?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ ),+ }++ match memory_result? {Some(memory) => println!("{memory}"),None => println!("No memory yet, generation triggered in background"),}diff --git a/crates/soup_realtime/src/outbound/kafka_publisher/test.rs b/crates/soup_realtime/src/outbound/kafka_publisher/test.rsindex a585ef2010..36854ec445 100644--- a/crates/soup_realtime/src/outbound/kafka_publisher/test.rs+++ b/crates/soup_realtime/src/outbound/kafka_publisher/test.rs@@ -2,7 +2,7 @@ use std::sync::{Arc, Mutex};use chrono::{DateTime, Utc};use macro_event_broker::{- EventBrokerError, EventPublisher, MacroEvent, MacroEventBrokerService, Topic,+ EventBrokerError, EventPublisher, GlobalSpawner, MacroEvent, MacroEventBrokerService, Topic,};use macro_user_id::user_id::MacroUserIdStr;use models_soup::{document::SoupDocument, item::SoupItem};@@ -75,10 +75,13 @@ fn message() -> SoupRealtimeMessage {#[tokio::test]async fn publishes_typed_recipient_keyed_event_to_soup_topic() {let records = Arc::new(Mutex::new(Vec::new()));- let broker = MacroEventBrokerService::new(RecordingPublisher {- records: records.clone(),- fail: false,- });+ let broker = MacroEventBrokerService::new(+ RecordingPublisher {+ records: records.clone(),+ fail: false,+ },+ GlobalSpawner,+ );let adapter = KafkaSoupRealtimePublisher::new(broker);adapter.publish(message()).await.expect("publish succeeds");@@ -111,10 +114,13 @@ async fn publishes_typed_recipient_keyed_event_to_soup_topic() {#[tokio::test]async fn propagates_delivery_failures_from_event_broker_service() {- let broker = MacroEventBrokerService::new(RecordingPublisher {- records: Arc::new(Mutex::new(Vec::new())),- fail: true,- });+ let broker = MacroEventBrokerService::new(+ RecordingPublisher {+ records: Arc::new(Mutex::new(Vec::new())),+ fail: true,+ },+ GlobalSpawner,+ );let adapter = KafkaSoupRealtimePublisher::new(broker);adapterdiff --git a/services/authentication_service/Cargo.toml b/services/authentication_service/Cargo.tomlindex 0b9744a887..8e7743065e 100644--- a/services/authentication_service/Cargo.toml+++ b/services/authentication_service/Cargo.toml@@ -135,7 +135,8 @@ strum = { workspace = true }teams = { path = "../../crates/teams" }thiserror = { workspace = true }time = { workspace = true }-tokio = { workspace = true }+tokio = { workspace = true, features = ["signal"] }+tokio-util = { workspace = true, features = ["rt"] }tower = { workspace = true }tower-cookies = { version = "0.11.0" }tower-http = { workspace = true }diff --git a/services/authentication_service/src/api/context.rs b/services/authentication_service/src/api/context.rsindex b2ffa8d3fe..dce6734514 100644--- a/services/authentication_service/src/api/context.rs+++ b/services/authentication_service/src/api/context.rs@@ -49,8 +49,11 @@ use roles_and_permissions::{domain::service::UserRolesAndPermissionsServiceImpl, outbound::pgpool::MacroDB,};use sqlx::PgPool;+use tokio_util::task::TaskTracker;pub(crate) type NotificationIngressType = SqsNotificationIngress<SqsQueue>;+pub(crate) type AuthenticationEventBroker =+ MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;pub(crate) type ChannelServiceType = ChannelServiceImpl<PgChannelsRepo,@@ -61,7 +64,7 @@ pub(crate) type ChannelServiceType = ChannelServiceImpl<NotificationChannelSender<NotificationIngressType>,SqsChannelSearchIndexer,ContactsChannelDispatcher<SqsContactsIngress<SqsContactsQueue>>,- MacroEventBrokerService<KafkaEventPublisher>,+ AuthenticationEventBroker,>,>,PgChannelReferenceSharePermissions<EntityAccessServiceType>,@@ -79,7 +82,7 @@ pub(crate) type TeamsServiceType = teams::domain::team_service::TeamServiceImpl<teams::outbound::contacts_enqueuer::ContactsIngressEnqueuer<SqsContactsIngress<SqsContactsQueue>,>,- MacroEventBrokerService<KafkaEventPublisher>,+ AuthenticationEventBroker,>;pub(crate) type RateLimiter = RateLimitServiceImpl<RedisRateLimitAdapter<redis::Client>>;diff --git a/services/authentication_service/src/api/mod.rs b/services/authentication_service/src/api/mod.rsindex 9fc2cda505..cf5c192aa0 100644--- a/services/authentication_service/src/api/mod.rs+++ b/services/authentication_service/src/api/mod.rs@@ -70,6 +70,7 @@ pub async fn setup_and_serve(state: ApiContext, port: usize) -> anyhow::Result<(listener,app.into_make_service_with_connect_info::<SocketAddr>(),)+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal()).await.context("error starting service")}diff --git a/services/authentication_service/src/main.rs b/services/authentication_service/src/main.rsindex af26bc5a16..9f7209966f 100644--- a/services/authentication_service/src/main.rs+++ b/services/authentication_service/src/main.rs@@ -72,7 +72,8 @@ use crate::api::context::{ApiContext, AuthorizationService, MacroApiTokenContext, MacroApiTokenExpirySeconds,MacroApiTokenIssuer, MacroApiTokenPrivateSecretKey, StripeWebhookSecretKey,};-use std::sync::Arc;+use std::{sync::Arc, time::Duration};+use tokio_util::task::TaskTracker;mod api;mod config;@@ -322,9 +323,11 @@ async fn main() -> anyhow::Result<()> {});let contacts_enqueuer = ContactsIngressEnqueuer::new(contacts_ingress.clone());let team_analytics = AnalyticsClientTeamAnalytics::new(analytics_client.clone());+ let event_broker_tracker = TaskTracker::new();let macro_event_broker = MacroEventBrokerService::new(KafkaEventPublisher::new(config.kafka_brokers.as_ref()).context("failed to create kafka event publisher")?,+ event_broker_tracker.clone(),);let entity_access_service_impl = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(db.clone()),@@ -393,7 +396,7 @@ async fn main() -> anyhow::Result<()> {notification_ingress: notification_ingress_service.clone(),};- api::setup_and_serve(+ let server_result = api::setup_and_serve(ApiContext {db,github_link_service: Arc::new(github_link_service_impl),@@ -447,10 +450,26 @@ async fn main() -> anyhow::Result<()> {},config.port,)- .await?;- Ok(())+ .await;++ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => {+ tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ );+ }+ }++ server_result}+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);+// SAFETY: this is not a secret valueconst IOS_DEVELOPMENT_TEAM_ID: &str = "TY74Q77JBD";// SAFETY: this is not a secret valuediff --git a/services/document_cognition_service/Cargo.toml b/services/document_cognition_service/Cargo.tomlindex 65d91dcbb9..0534e8b949 100644--- a/services/document_cognition_service/Cargo.toml+++ b/services/document_cognition_service/Cargo.toml@@ -129,8 +129,8 @@ strum = { workspace = true }sync_service_client = { path = "../../crates/sync_service_client" }thiserror = { workspace = true }time = { workspace = true }-tokio = { workspace = true }-tokio-util = { workspace = true }+tokio = { workspace = true, features = ["signal"] }+tokio-util = { workspace = true, features = ["rt"] }tower = { workspace = true, features = ["util"] }tower-http = { workspace = true, features = ["limit", "timeout"] }tracing = { workspace = true }diff --git a/services/document_cognition_service/src/api/context/mod.rs b/services/document_cognition_service/src/api/context/mod.rsindex 9aa79f1588..23d61b3c77 100644--- a/services/document_cognition_service/src/api/context/mod.rs+++ b/services/document_cognition_service/src/api/context/mod.rs@@ -59,13 +59,15 @@ pub type DcsAttachmentProvider = AttachmentProvider<StaticFileAttachmentService<CdnStaticFileRepo>,>;-/// Type alias for the message service wired to concrete DCS services.-pub type DcsMessageService = MessageServiceImpl<- PgChatRepo,- DcsAttachmentProvider,- macro_event_broker::MacroEventBrokerService<macro_event_broker::KafkaEventPublisher>,+/// Kafka-backed event broker with publish tasks tracked for graceful shutdown.+pub type DcsEventBroker = macro_event_broker::MacroEventBrokerService<+ macro_event_broker::KafkaEventPublisher,+ tokio_util::task::TaskTracker,>;+/// Type alias for the message service wired to concrete DCS services.+pub type DcsMessageService = MessageServiceImpl<PgChatRepo, DcsAttachmentProvider, DcsEventBroker>;+#[cfg(test)]mod test;#[cfg(test)]@@ -145,8 +147,7 @@ pub struct ApiContext {pub import_service: Arc<DcsImportService>,pub onboarding_service: Arc<DcsOnboardingService>,/// Kafka-backed macro event broker for publishing domain events.- pub macro_event_broker:- macro_event_broker::MacroEventBrokerService<macro_event_broker::KafkaEventPublisher>,+ pub macro_event_broker: DcsEventBroker,}impl FromRef<ApiContext>diff --git a/services/document_cognition_service/src/api/context/test.rs b/services/document_cognition_service/src/api/context/test.rsindex 848e2d0ad8..4e80a6da86 100644--- a/services/document_cognition_service/src/api/context/test.rs+++ b/services/document_cognition_service/src/api/context/test.rs@@ -7,6 +7,7 @@ use stream::domain::{ItemId, ItemStream, Result as StreamResult, StreamEvent, StreamId, StreamRepo,};use tokio::sync::broadcast::{self, Receiver};+use tokio_util::task::TaskTracker;pub struct MockConnectionRepo;@@ -257,6 +258,7 @@ pub async fn test_api_context(pool: sqlx::Pool<sqlx::Postgres>) -> std::sync::Arlet macro_event_broker = macro_event_broker::MacroEventBrokerService::new(macro_event_broker::KafkaEventPublisher::new("localhost:9092").expect("kafka producer config is valid"),+ TaskTracker::new(),);let document_service = documents::domain::service::DocumentServiceImpl::new(diff --git a/services/document_cognition_service/src/api/mod.rs b/services/document_cognition_service/src/api/mod.rsindex 8269558031..3ae5728ecb 100644--- a/services/document_cognition_service/src/api/mod.rs+++ b/services/document_cognition_service/src/api/mod.rs@@ -65,6 +65,7 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {"document cognition service is up and running");axum::serve(listener, app.into_make_service())+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal()).await.context("error starting service")}diff --git a/services/document_cognition_service/src/main.rs b/services/document_cognition_service/src/main.rsindex 042dde5277..85fd706fc6 100644--- a/services/document_cognition_service/src/main.rs+++ b/services/document_cognition_service/src/main.rs@@ -46,9 +46,10 @@ use notification::outbound::websocket::ConnectionGatewayClient;use readonly_pool::ReadOnlyPool;use search_service_client::SearchServiceClient;use sqlx::postgres::PgPoolOptions;-use std::sync::Arc;+use std::{sync::Arc, time::Duration};use stream::outbound::redis_pg::RedisPostgresStreamRepo;use sync_service_client::SyncServiceClient;+use tokio_util::task::TaskTracker;mod api;mod config;@@ -299,9 +300,11 @@ async fn main() -> anyhow::Result<()> {import::outbound::document_properties::DocumentPropertiesApplicator::new(properties_service.clone(),);+ let event_broker_tracker = TaskTracker::new();let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref()).context("failed to create kafka event publisher")?,+ event_broker_tracker.clone(),);let document_service = DocumentServiceImpl::new(document_repo,@@ -564,7 +567,7 @@ async fn main() -> anyhow::Result<()> {),ai_projections_service_impl.clone(),);- tokio::spawn(async move {+ let ai_projection_worker_task = tokio::spawn(async move {ai_projection_worker.poll().await;});@@ -627,7 +630,7 @@ async fn main() -> anyhow::Result<()> {),);- api::setup_and_serve(ApiContext {+ let api_result = api::setup_and_serve(ApiContext {db: db.clone(),email_service_client_external,sqs_client: Arc::new(sqs_client),@@ -664,6 +667,36 @@ async fn main() -> anyhow::Result<()> {macro_event_broker: macro_event_broker.clone(),}).await- .context("failed to setup and serve api")?;- Ok(())+ .context("failed to setup and serve api");++ ai_projection_worker_task.abort();+ match ai_projection_worker_task.await {+ Err(error) if error.is_cancelled() => {+ tracing::info!("ai projection worker stopped");+ }+ Err(error) => {+ tracing::error!(error = ?error, "ai projection worker exited unexpectedly");+ }+ Ok(()) => {+ tracing::error!(+ error = "worker exited naturally",+ "ai projection worker exited unexpectedly"+ );+ }+ }++ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ ),+ }++ api_result}++const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.tomlindex 6bb533f996..b124994698 100644--- a/services/document_storage_service/Cargo.toml+++ b/services/document_storage_service/Cargo.toml@@ -186,7 +186,8 @@ sync_service_hex = { path = "../../crates/sync_service", package = "sync_servicesystem_properties = { path = "../../crates/system_properties" }task_dedup = { path = "../../crates/task_dedup" }thiserror = { workspace = true }-tokio = { workspace = true }+tokio = { workspace = true, features = ["signal"] }+tokio-util = { workspace = true, features = ["rt"] }tower = { workspace = true }tower-http = { workspace = true, features = ["compression-gzip"] }tracing = { workspace = true }diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rsindex 960c06e9d5..a28690430b 100644--- a/services/document_storage_service/src/api/context.rs+++ b/services/document_storage_service/src/api/context.rs@@ -105,6 +105,7 @@ use system_properties::{PgSystemPropertiesRepository, StatusOption, SystemPropertiesService as _,SystemPropertiesServiceImpl,};+use tokio_util::task::TaskTracker;use webhook::{domain::service::WebhookServiceImpl,inbound::axum_router::WebhookRouterState as MacroWebhookRouterState,@@ -114,6 +115,9 @@ use webhook::{},};+/// Event broker shared by DSS services.+pub(crate) type DssEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;+/// CRM service for DSS — no-op resolver since DSS doesn't populate.pub(crate) type DssCrmService = crm::domain::service::CrmServiceImpl<crm::outbound::companies_repo::CompaniesRepositoryImpl,@@ -256,7 +260,7 @@ pub(crate) type DocumentService = DocumentServiceImpl<ConnectionServiceImpl<EntityAccessService, ConnectionGatewayImpl>,EntityAccessManagementService,ForeignEntityServiceImpl<PgForeignEntityRepo>,- MacroEventBrokerService<KafkaEventPublisher>,+ DssEventBroker,>;/// Type alias for the authorization service.@@ -274,7 +278,7 @@ pub(crate) type ProjectService = ProjectServiceImpl<ShaCountAdapter,EntityAccessManagementService,SqsProjectSearchIndexer,- MacroEventBrokerService<KafkaEventPublisher>,+ DssEventBroker,>;/// Type alias for the projects router state.@@ -299,7 +303,7 @@ pub(crate) type DssChannelService = ChannelServiceImpl<NotificationChannelSender<NotificationIngressType>,SqsChannelSearchIndexer,ContactsChannelDispatcher<SqsContactsIngress<SqsContactsQueue>>,- MacroEventBrokerService<KafkaEventPublisher>,+ DssEventBroker,>,>,PgChannelReferenceSharePermissions<EntityAccessService>,@@ -311,8 +315,7 @@ pub(crate) type DssChannelsState =ChannelsRouterState<DssChannelService, EntityAccessService, AuthorizationService>;/// Type alias for the bots service wired into DSS.-pub(crate) type DssBotService =- BotServiceImpl<PgBotsRepo, MacroEventBrokerService<KafkaEventPublisher>>;+pub(crate) type DssBotService = BotServiceImpl<PgBotsRepo, DssEventBroker>;/// Type alias for the bots router state.pub(crate) type DssBotsState =@@ -392,11 +395,8 @@ pub(crate) type CalWebhookServiceType = CalWebhookServiceImpl<AnalyticsClientSinpub(crate) type DssCalWebhookState = CalWebhookRouterState<CalWebhookServiceType>;/// Type alias for the product webhook service.-pub(crate) type DssWebhookService = WebhookServiceImpl<- PgWebhookRepo,- ReqwestWebhookValidationClient,- MacroEventBrokerService<KafkaEventPublisher>,->;+pub(crate) type DssWebhookService =+ WebhookServiceImpl<PgWebhookRepo, ReqwestWebhookValidationClient, DssEventBroker>;/// Type alias for the product webhook rate limiter.pub(crate) type DssWebhookRateLimiter =diff --git a/services/document_storage_service/src/api/mod.rs b/services/document_storage_service/src/api/mod.rsindex c04d541d47..12f15e21f3 100644--- a/services/document_storage_service/src/api/mod.rs+++ b/services/document_storage_service/src/api/mod.rs@@ -76,6 +76,7 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {&state.config.port);axum::serve(listener, app.into_make_service())+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal()).await.context("error starting service")}diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rsindex 48aef0cce6..74dd70d095 100644--- a/services/document_storage_service/src/main.rs+++ b/services/document_storage_service/src/main.rs@@ -118,7 +118,7 @@ use soup_realtime::{},};use sqlx::postgres::PgPoolOptions;-use std::sync::Arc;+use std::{sync::Arc, time::Duration};use sync_service_client::SyncServiceClient;use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};use task_dedup::{@@ -130,6 +130,7 @@ use task_dedup::{postgres::{PgTaskMatchRepo, PgTaskVectorDb},},};+use tokio_util::{sync::CancellationToken, task::TaskTracker};mod api;mod config;@@ -266,9 +267,13 @@ async fn main() -> anyhow::Result<()> {JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client).await?;+ let consumer_cancellation_token = CancellationToken::new();+ let consumer_tracker = TaskTracker::new();+ let event_broker_tracker = TaskTracker::new();let macro_event_broker = MacroEventBrokerService::new(KafkaEventPublisher::new(config.kafka_brokers.as_ref()).context("failed to create kafka event publisher")?,+ event_broker_tracker.clone(),);let bots_repo = PgBotsRepo::new(db.clone());let bots_service = BotServiceImpl::new(bots_repo, macro_event_broker.clone());@@ -730,22 +735,39 @@ async fn main() -> anyhow::Result<()> {});let webhook_consumer_brokers = config.kafka_brokers.as_ref().to_string();- tokio::spawn(async move {- loop {- tracing::info!("starting webhook event consumer");- let result = webhook::inbound::kafka_consumer::run_webhook_event_consumer(- &webhook_consumer_brokers,- webhook_ingestion_service.clone(),- std::future::pending::<()>(),- )- .await;- match result {- Ok(()) => tracing::error!("webhook event consumer exited unexpectedly"),- Err(error) => {- tracing::error!(error = ?error, "webhook event consumer exited unexpectedly");+ consumer_tracker.spawn({+ let cancellation_token = consumer_cancellation_token.clone();+ async move {+ loop {+ if cancellation_token.is_cancelled() {+ break;+ }++ tracing::info!("starting webhook event consumer");+ let result = webhook::inbound::kafka_consumer::run_webhook_event_consumer(+ &webhook_consumer_brokers,+ webhook_ingestion_service.clone(),+ cancellation_token.cancelled(),+ )+ .await;++ if cancellation_token.is_cancelled() {+ break;+ }++ match result {+ Ok(()) => tracing::error!("webhook event consumer exited unexpectedly"),+ Err(error) => {+ tracing::error!(error = ?error, "webhook event consumer exited unexpectedly");+ }+ }++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}}}- tokio::time::sleep(std::time::Duration::from_secs(5)).await;}});@@ -819,9 +841,10 @@ async fn main() -> anyhow::Result<()> {// Wire Macro AI to react to mentions. The router posts replies through the// channel service we just built and runs the agent loop in-process with the// same pre-configured toolset used by other AI hosts.- let mut macro_agent_tool_context = ai_tools::build_tool_service_context_from_env(db.clone())- .await- .context("failed to build Macro agent tool context")?;+ let mut macro_agent_tool_context =+ ai_tools::build_tool_service_context_from_env(db.clone(), event_broker_tracker.clone())+ .await+ .context("failed to build Macro agent tool context")?;// Wire the agent's SendChannelMessage tool to this service's own// side-effect pipeline so agent-posted messages share the exact instance// used by the HTTP API, including the in-process bot trigger sender (the@@ -865,28 +888,49 @@ async fn main() -> anyhow::Result<()> {anyhow::anyhow!("failed to create realtime Soup topic consumer: {error:?}")})?,));- tokio::spawn({+ consumer_tracker.spawn({let soup_realtime_service = Arc::clone(&soup_realtime_service);+ let cancellation_token = consumer_cancellation_token.clone();async move {loop {- let _ = soup_realtime_service.run().await.inspect_err(|error| {+ let result = tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ result = soup_realtime_service.run() => result,+ };++ if cancellation_token.is_cancelled() {+ break;+ }++ let _ = result.inspect_err(|error| {tracing::error!(error = ?error,"realtime Soup subscription consumer stopped");});- tokio::time::sleep(std::time::Duration::from_secs(1)).await;++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}+ }}}});- tokio::spawn({+ consumer_tracker.spawn({let brokers = config.kafka_brokers.as_ref().to_string();let entity_access_service = entity_access_service.as_ref().clone();let soup_pool = readonly_pool::ReadOnlyPool(readonly_db.clone());let macro_event_broker = macro_event_broker.clone();+ let cancellation_token = consumer_cancellation_token.clone();async move {loop {+ if cancellation_token.is_cancelled() {+ break;+ }+let fanout_service = SoupRealtimeServiceImpl::new(EntityAccessExpander::new(entity_access_service.clone()),SoupRepoItemReader::new(PgSoupRepo::new(soup_pool.clone())),@@ -894,8 +938,13 @@ async fn main() -> anyhow::Result<()> {);tracing::info!("starting realtime Soup document consumer");let result = fanout_service- .run_document_update_consumer(&brokers, std::future::pending::<()>())+ .run_document_update_consumer(&brokers, cancellation_token.cancelled()).await;++ if cancellation_token.is_cancelled() {+ break;+ }+match result {Ok(()) => {tracing::error!("realtime Soup document consumer exited unexpectedly")@@ -905,7 +954,12 @@ async fn main() -> anyhow::Result<()> {"realtime Soup document consumer exited unexpectedly"),}- tokio::time::sleep(std::time::Duration::from_secs(5)).await;++ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => break,+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}+ }}}});@@ -1024,7 +1078,28 @@ async fn main() -> anyhow::Result<()> {});}- api::setup_and_serve(api_context).await?;+ let server_result = api::setup_and_serve(api_context).await;++ tracing::info!("stopping event consumers");+ consumer_cancellation_token.cancel();+ consumer_tracker.close();+ consumer_tracker.wait().await;+ tracing::info!("event consumers stopped");++ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => {+ tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ );+ }+ }- Ok(())+ server_result}++const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);diff --git a/services/email_service/Cargo.toml b/services/email_service/Cargo.tomlindex 38f6817c11..a92a19f379 100644--- a/services/email_service/Cargo.toml+++ b/services/email_service/Cargo.toml@@ -166,6 +166,7 @@ strum_macros = { workspace = true }system_properties = { path = "../../crates/system_properties" }thiserror = { workspace = true }tokio = { workspace = true, features = ["signal"] }+tokio-util = { workspace = true, features = ["rt"] }tokio-retry.workspace = truetower = { workspace = true }tower-http = { workspace = true, features = [diff --git a/services/email_service/src/api/context.rs b/services/email_service/src/api/context.rsindex 56a6a9408f..767e11fedf 100644--- a/services/email_service/src/api/context.rs+++ b/services/email_service/src/api/context.rs@@ -23,11 +23,13 @@ use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use static_file_service_client::StaticFileServiceClient;use std::sync::Arc;use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};+use tokio_util::task::TaskTracker;pub(crate) type AuthorizationService = MacroAuthorizationServiceImpl<MacroAuthJwtValidator>;pub(crate) type EmailEntityAccessService = EntityAccessServiceImpl<PgAccessRepository>;pub(crate) type EmailEntityAccessManagementService =EntityAccessManagementServiceImpl<entity_access_management::outbound::PgRepository>;+pub(crate) type EmailEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;pub(crate) type EmailSvc = EmailServiceImpl<EmailPgRepo,FrecencyQueryServiceImpl<FrecencyPgStorage>,@@ -37,7 +39,7 @@ pub(crate) type EmailSvc = EmailServiceImpl<crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,>,EmailEntityAccessManagementService,- MacroEventBrokerService<KafkaEventPublisher>,+ EmailEventBroker,>;#[derive(Clone, FromRef)]@@ -60,5 +62,5 @@ pub(crate) struct ApiContext {pub email_thread_state:EmailThreadRouterState<EmailSvc, EmailEntityAccessService, AuthorizationService>,pub gmail_token_state: GmailTokenState<GmailTokenProviderImpl>,- pub macro_event_broker: Arc<MacroEventBrokerService<KafkaEventPublisher>>,+ pub macro_event_broker: Arc<EmailEventBroker>,}diff --git a/services/email_service/src/api/mod.rs b/services/email_service/src/api/mod.rsindex 074344ea43..c638bc0be2 100644--- a/services/email_service/src/api/mod.rs+++ b/services/email_service/src/api/mod.rs@@ -42,6 +42,7 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {port);axum::serve(listener, app.into_make_service())+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal()).await.context("error starting service")}diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rsindex 0a4a3405c1..4706b7d5f2 100644--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs@@ -3,7 +3,7 @@ use anyhow::Context;use document_storage_service_client::DocumentStorageServiceClient;use email_service::config::Config;use email_service::pubsub::CrmMetadataResolver;-use macro_entrypoint::MacroEntrypoint;+use macro_entrypoint::{MacroEntrypoint, shutdown_signal};use macro_env::Environment;use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use macro_service_urls::{@@ -14,7 +14,11 @@ use notification::outbound::queue::SqsQueue;use sqlx::postgres::PgPoolOptions;use static_file_service_client::StaticFileServiceClient;use std::sync::Arc;+use std::time::Duration;use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};+use tokio_util::{sync::CancellationToken, task::TaskTracker};++const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);#[tokio::main]#[tracing::instrument(err)]@@ -94,9 +98,13 @@ async fn main() -> anyhow::Result<()> {.sfs_delete_queue(&sfs_delete_queue).email_link_manager_queue(&link_manager_queue);+ let worker_cancellation_token = CancellationToken::new();+ let worker_tracker = TaskTracker::new();+ let event_broker_tracker = TaskTracker::new();let macro_event_broker = MacroEventBrokerService::new(KafkaEventPublisher::new(config.kafka_brokers.as_ref()).context("failed to create kafka event publisher")?,+ event_broker_tracker.clone(),);let contacts_ingress = Arc::new(contacts::domain::service::SqsContactsIngress {@@ -120,6 +128,7 @@ async fn main() -> anyhow::Result<()> {config.queue_wait_time_seconds,);+ #[cfg(feature = "sfs_map")]let sfs_uploader_workers = (0..config.sfs_uploader_workers).map(|_| {sqs_worker::SQSWorker::new(@@ -131,6 +140,7 @@ async fn main() -> anyhow::Result<()> {}).collect::<Vec<_>>();+ #[cfg(feature = "sfs_delete")]let sfs_delete_worker = sqs_worker::SQSWorker::new(aws_sdk_sqs::Client::new(&gmail_queue_aws_config),sfs_delete_queue.to_string(),@@ -323,8 +333,9 @@ async fn main() -> anyhow::Result<()> {let system_properties_service_inbox_sync = system_properties_service.clone();let crm_service_inbox_sync = crm_service.clone();let macro_event_broker_inbox_sync = macro_event_broker.clone();- tokio::spawn(async move {- email_service::pubsub::inbox_sync::worker::run_worker(+ let cancellation_token = worker_cancellation_token.clone();+ worker_tracker.spawn(async move {+ email_service::pubsub::inbox_sync::worker::run_worker_with_cancellation(db_inbox_sync,worker,sqs_client_inbox_sync,@@ -341,6 +352,7 @@ async fn main() -> anyhow::Result<()> {macro_event_broker_inbox_sync,config.notifications_enabled,false,+ cancellation_token,).await;});@@ -365,8 +377,9 @@ async fn main() -> anyhow::Result<()> {let system_properties_service_inbox_sync = system_properties_service.clone();let crm_service_inbox_sync = crm_service.clone();let macro_event_broker_inbox_sync = macro_event_broker.clone();- tokio::spawn(async move {- email_service::pubsub::inbox_sync::worker::run_worker(+ let cancellation_token = worker_cancellation_token.clone();+ worker_tracker.spawn(async move {+ email_service::pubsub::inbox_sync::worker::run_worker_with_cancellation(db_inbox_sync,worker,sqs_client_inbox_sync,@@ -383,6 +396,7 @@ async fn main() -> anyhow::Result<()> {macro_event_broker_inbox_sync,config.notifications_enabled,true,+ cancellation_token,).await;});@@ -399,8 +413,9 @@ async fn main() -> anyhow::Result<()> {let gmail_client_gmail_ops = gmail_client.clone();let auth_service_client_gmail_ops = auth_service_client.clone();let redis_client_gmail_ops = redis_client.clone();- tokio::spawn(async move {- email_service::pubsub::gmail_ops::worker::run_worker(+ let cancellation_token = worker_cancellation_token.clone();+ worker_tracker.spawn(async move {+ email_service::pubsub::gmail_ops::worker::run_worker_with_cancellation(db_gmail_ops,worker,sqs_client_gmail_ops,@@ -408,6 +423,7 @@ async fn main() -> anyhow::Result<()> {auth_service_client_gmail_ops,redis_client_gmail_ops,false,+ cancellation_token,).await;});@@ -424,8 +440,9 @@ async fn main() -> anyhow::Result<()> {let gmail_client_gmail_ops = gmail_client.clone();let auth_service_client_gmail_ops = auth_service_client.clone();let redis_client_gmail_ops = redis_client.clone();- tokio::spawn(async move {- email_service::pubsub::gmail_ops::worker::run_worker(+ let cancellation_token = worker_cancellation_token.clone();+ worker_tracker.spawn(async move {+ email_service::pubsub::gmail_ops::worker::run_worker_with_cancellation(db_gmail_ops,worker,sqs_client_gmail_ops,@@ -433,6 +450,7 @@ async fn main() -> anyhow::Result<()> {auth_service_client_gmail_ops,redis_client_gmail_ops,true,+ cancellation_token,).await;});@@ -457,8 +475,9 @@ async fn main() -> anyhow::Result<()> {let system_properties_service_backfill = system_properties_service.clone();let crm_service_backfill = crm_service_backfill.clone();let macro_event_broker_backfill = macro_event_broker.clone();- tokio::spawn(async move {- email_service::pubsub::backfill::worker::run_worker(+ let cancellation_token = worker_cancellation_token.clone();+ worker_tracker.spawn(async move {+ email_service::pubsub::backfill::worker::run_worker_with_cancellation(db_backfill,worker,sqs_client_backfill,@@ -474,6 +493,7 @@ async fn main() -> anyhow::Result<()> {crm_service_backfill,macro_event_broker_backfill,config.notifications_enabled,+ cancellation_token,).await;});@@ -489,12 +509,14 @@ async fn main() -> anyhow::Result<()> {let db_crm_cleanup = db_backfill.clone();let sqs_client_crm_cleanup = sqs_client.clone();let crm_service_crm_cleanup = crm_service_backfill.clone();- tokio::spawn(async move {- email_service::pubsub::crm_cleanup::worker::run_worker(+ let cancellation_token = worker_cancellation_token.clone();+ worker_tracker.spawn(async move {+ email_service::pubsub::crm_cleanup::worker::run_worker_with_cancellation(db_crm_cleanup,worker,sqs_client_crm_cleanup,crm_service_crm_cleanup,+ cancellation_token,).await;});@@ -513,9 +535,10 @@ async fn main() -> anyhow::Result<()> {let connection_gateway_client_link_manager = connection_gateway_client.clone();let notification_ingress_service_link_manager = notification_ingress_service.clone();let macro_event_broker_link_manager = macro_event_broker.clone();+ let cancellation_token = worker_cancellation_token.clone();// daily link_manager operations for user contacts and inbox subscriptions- tokio::spawn(async move {- email_service::pubsub::link_manager::worker::run_worker(+ worker_tracker.spawn(async move {+ email_service::pubsub::link_manager::worker::run_worker_with_cancellation(link_manager_worker,db_link_manager,gmail_client_link_manager,@@ -526,6 +549,7 @@ async fn main() -> anyhow::Result<()> {connection_gateway_client_link_manager,notification_ingress_service_link_manager,macro_event_broker_link_manager,+ cancellation_token,).await;});@@ -537,9 +561,10 @@ async fn main() -> anyhow::Result<()> {let s3_client_scheduled = s3_client.clone();let attachment_bucket_scheduled = config.attachment_bucket.to_string();let macro_event_broker_scheduled = macro_event_broker.clone();+ let cancellation_token = worker_cancellation_token.clone();// send scheduled emails- tokio::spawn(async move {- email_service::pubsub::scheduled::worker::run_worker(+ worker_tracker.spawn(async move {+ email_service::pubsub::scheduled::worker::run_worker_with_cancellation(scheduled_worker,db_scheduled,gmail_client_scheduled,@@ -548,22 +573,26 @@ async fn main() -> anyhow::Result<()> {s3_client_scheduled,attachment_bucket_scheduled,macro_event_broker_scheduled,+ cancellation_token,).await;});- if cfg!(feature = "sfs_map") {+ #[cfg(feature = "sfs_map")]+ {for worker in sfs_uploader_workers {let db_sfs_uploader = db.clone();let sfs_client_sfs_uploader = sfs_client.clone();let connection_gateway_client_sfs_uploader = connection_gateway_client.clone();+ let cancellation_token = worker_cancellation_token.clone();// upload user contact images to sfs from contact sync- tokio::spawn(async move {- email_service::pubsub::sfs_uploader::worker::run_worker(+ worker_tracker.spawn(async move {+ email_service::pubsub::sfs_uploader::worker::run_worker_with_cancellation(worker,db_sfs_uploader,sfs_client_sfs_uploader,connection_gateway_client_sfs_uploader,+ cancellation_token,).await;});@@ -574,15 +603,18 @@ async fn main() -> anyhow::Result<()> {);}- if cfg!(feature = "sfs_delete") {+ #[cfg(feature = "sfs_delete")]+ {let db_sfs_delete = db.clone();let sfs_client_sfs_delete = sfs_client.clone();+ let cancellation_token = worker_cancellation_token.clone();// delete orphaned sfs attachments- tokio::spawn(async move {- email_service::pubsub::sfs_deleter::worker::run_worker(+ worker_tracker.spawn(async move {+ email_service::pubsub::sfs_deleter::worker::run_worker_with_cancellation(sfs_delete_worker,db_sfs_delete,sfs_client_sfs_delete,+ cancellation_token,).await;});@@ -591,18 +623,23 @@ async fn main() -> anyhow::Result<()> {tracing::info!("All workers started successfully");- // Wait for shutdown signal (SIGTERM from ECS or SIGINT from Ctrl+C)- tokio::select! {- _ = tokio::signal::ctrl_c() => {- tracing::info!("Received SIGINT (Ctrl+C)");- }- _ = async {- let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())- .expect("failed to install SIGTERM handler");- term.recv().await- } => {- tracing::info!("Received SIGTERM");- }+ shutdown_signal().await;++ worker_cancellation_token.cancel();+ worker_tracker.close();+ tracing::info!("Waiting for email workers to stop");+ worker_tracker.wait().await;+ tracing::info!("Email workers stopped");++ event_broker_tracker.close();+ tracing::info!("Waiting for event broker publishes to drain");+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("Event broker publishes drained"),+ Err(error) => tracing::warn!(+ error = ?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "Timed out waiting for event broker publishes to drain"+ ),}tracing::info!("Shutdown signal received, exiting gracefully...");diff --git a/services/email_service/src/main.rs b/services/email_service/src/main.rsindex 0c22b357bc..1ff7c65314 100644--- a/services/email_service/src/main.rs+++ b/services/email_service/src/main.rs@@ -20,8 +20,9 @@ use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use macro_service_urls::{AuthServiceUrl, DocumentStorageServiceUrl, StaticFileServiceUrl};use sqlx::postgres::PgPoolOptions;use static_file_service_client::StaticFileServiceClient;-use std::sync::Arc;+use std::{sync::Arc, time::Duration};use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};+use tokio_util::task::TaskTracker;mod api;mod utils;@@ -143,9 +144,11 @@ async fn main() -> anyhow::Result<()> {crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,);+ let event_broker_tracker = TaskTracker::new();let macro_event_broker = MacroEventBrokerService::new(KafkaEventPublisher::new(config.kafka_brokers.as_ref()).context("failed to create kafka event publisher")?,+ event_broker_tracker.clone(),);let email_service = EmailRouterState::new(EmailServiceImpl::new(@@ -179,7 +182,7 @@ async fn main() -> anyhow::Result<()> {redis_conn,auth_service_client.clone(),));- api::setup_and_serve(ApiContext {+ let api_result = api::setup_and_serve(ApiContext {db,internal_api_key: config.internal_api_key.clone(),config: Arc::new(config),@@ -199,6 +202,22 @@ async fn main() -> anyhow::Result<()> {gmail_token_state,macro_event_broker: Arc::new(macro_event_broker),})- .await?;- Ok(())+ .await;++ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => {+ tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ );+ }+ }++ api_result}++const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);diff --git a/services/email_service/src/pubsub/backfill/worker.rs b/services/email_service/src/pubsub/backfill/worker.rsindex 40a1e08b73..57472939ac 100644--- a/services/email_service/src/pubsub/backfill/worker.rs+++ b/services/email_service/src/pubsub/backfill/worker.rs@@ -1,5 +1,8 @@use crate::pubsub::backfill::process;-use crate::pubsub::context::{CrmServiceType, NotificationIngressType, PubSubContext};+use crate::pubsub::context::{+ CrmServiceType, NotificationIngressType, PubSubContext, PubSubEventBroker,+};+use crate::pubsub::worker_lifecycle::run_until_cancelled;use crate::util::redis::RedisClient;use authentication_service_client::AuthServiceClient;use connection_gateway_client::client::ConnectionGatewayClient;@@ -7,10 +10,10 @@ use contacts::domain::service::SqsContactsIngress;use contacts::outbound::ingress::SqsContactsQueue;use document_storage_service_client::DocumentStorageServiceClient;use futures::StreamExt;-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use static_file_service_client::StaticFileServiceClient;use std::sync::Arc;use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};+use tokio_util::sync::CancellationToken;/// method that ingests sqs messages and calls the process function for each#[expect(clippy::too_many_arguments, reason = "too annoying to fix right now")]@@ -28,8 +31,51 @@ pub async fn run_worker(dss_client: DocumentStorageServiceClient,system_properties_service: Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,crm_service: CrmServiceType,- macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,+ macro_event_broker: PubSubEventBroker,notifications_enabled: bool,+) {+ run_worker_with_cancellation(+ db,+ worker,+ sqs_client,+ contacts_ingress,+ gmail_client,+ auth_service_client,+ redis_client,+ notification_ingress_service,+ sfs_client,+ connection_gateway_client,+ dss_client,+ system_properties_service,+ crm_service,+ macro_event_broker,+ notifications_enabled,+ CancellationToken::new(),+ )+ .await;+}++/// Ingests SQS messages until cancellation is requested.+///+/// A batch already returned by SQS is fully processed before shutdown.+#[expect(clippy::too_many_arguments, reason = "too annoying to fix right now")]+pub async fn run_worker_with_cancellation(+ db: sqlx::Pool<sqlx::Postgres>,+ worker: sqs_worker::SQSWorker,+ sqs_client: sqs_client::SQS,+ contacts_ingress: Arc<SqsContactsIngress<SqsContactsQueue>>,+ gmail_client: gmail_client::GmailClient,+ auth_service_client: AuthServiceClient,+ redis_client: RedisClient,+ notification_ingress_service: Arc<NotificationIngressType>,+ sfs_client: StaticFileServiceClient,+ connection_gateway_client: ConnectionGatewayClient,+ dss_client: DocumentStorageServiceClient,+ system_properties_service: Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,+ crm_service: CrmServiceType,+ macro_event_broker: PubSubEventBroker,+ notifications_enabled: bool,+ cancellation_token: CancellationToken,) {let ctx = PubSubContext {db,@@ -54,9 +100,19 @@ pub async fn run_worker(let worker_result = tokio::spawn({let ctx = ctx.clone();let worker = worker.clone();+ let cancellation_token = cancellation_token.clone();async move {loop {- match worker.receive_messages().await {+ let Some(receive_result) = run_until_cancelled(+ &cancellation_token,+ worker.receive_messages(),+ )+ .await+ else {+ return;+ };++ match receive_result {Ok(messages) => {if messages.is_empty() {continue;@@ -106,6 +162,10 @@ pub async fn run_worker(}).await;+ if cancellation_token.is_cancelled() {+ return;+ }+match worker_result {Ok(_) => {// This should never be hit@@ -118,6 +178,14 @@ pub async fn run_worker(// Add a delay before restarting to avoid rapid restart loopstracing::info!("WORKER RESTARTING...");- tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ if run_until_cancelled(+ &cancellation_token,+ tokio::time::sleep(std::time::Duration::from_secs(5)),+ )+ .await+ .is_none()+ {+ return;+ }}}diff --git a/services/email_service/src/pubsub/context.rs b/services/email_service/src/pubsub/context.rsindex 9540fbb53b..7b23e5ac51 100644--- a/services/email_service/src/pubsub/context.rs+++ b/services/email_service/src/pubsub/context.rs@@ -18,6 +18,10 @@ use sqlx::PgPool;use static_file_service_client::StaticFileServiceClient;use std::sync::Arc;use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};+use tokio_util::task::TaskTracker;++/// The event broker used by pubsub workers, with publish tasks tracked for graceful shutdown.+pub type PubSubEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;/// The concrete notification ingress service type.pub type NotificationIngressType = SqsNotificationIngress<SqsQueue>;@@ -70,7 +74,7 @@ pub struct PubSubContext {pub dss_client: DocumentStorageServiceClient,pub system_properties_service: Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,pub crm_service: CrmServiceType,- pub macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,+ pub macro_event_broker: PubSubEventBroker,pub notifications_enabled: bool,pub retry_worker: bool,}diff --git a/services/email_service/src/pubsub/crm_cleanup/worker.rs b/services/email_service/src/pubsub/crm_cleanup/worker.rsindex a8b68c7b64..31d62ad514 100644--- a/services/email_service/src/pubsub/crm_cleanup/worker.rs+++ b/services/email_service/src/pubsub/crm_cleanup/worker.rs@@ -1,6 +1,8 @@use crate::pubsub::context::CrmServiceType;use crate::pubsub::crm_cleanup::process;+use crate::pubsub::worker_lifecycle::run_until_cancelled;use futures::StreamExt;+use tokio_util::sync::CancellationToken;/// Shared dependencies for the CRM cleanup handlers. Much smaller than the/// backfill `PubSubContext` — this worker only talks to the DB, SQS, and the@@ -19,6 +21,26 @@ pub async fn run_worker(worker: sqs_worker::SQSWorker,sqs_client: sqs_client::SQS,crm_service: CrmServiceType,+) {+ run_worker_with_cancellation(+ db,+ worker,+ sqs_client,+ crm_service,+ CancellationToken::new(),+ )+ .await;+}++/// Ingests CRM cleanup messages until cancellation is requested.+///+/// A batch already returned by SQS is fully processed before shutdown.+pub async fn run_worker_with_cancellation(+ db: sqlx::Pool<sqlx::Postgres>,+ worker: sqs_worker::SQSWorker,+ sqs_client: sqs_client::SQS,+ crm_service: CrmServiceType,+ cancellation_token: CancellationToken,) {let ctx = CrmCleanupContext {db,@@ -31,9 +53,19 @@ pub async fn run_worker(let worker_result = tokio::spawn({let ctx = ctx.clone();let worker = worker.clone();+ let cancellation_token = cancellation_token.clone();async move {loop {- match worker.receive_messages().await {+ let Some(receive_result) = run_until_cancelled(+ &cancellation_token,+ worker.receive_messages(),+ )+ .await+ else {+ return;+ };++ match receive_result {Ok(messages) => {if messages.is_empty() {continue;@@ -68,6 +100,10 @@ pub async fn run_worker(}).await;+ if cancellation_token.is_cancelled() {+ return;+ }+match worker_result {Ok(_) => {// This should never be hit@@ -80,6 +116,14 @@ pub async fn run_worker(// Add a delay before restarting to avoid rapid restart loopstracing::info!("CRM CLEANUP WORKER RESTARTING...");- tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ if run_until_cancelled(+ &cancellation_token,+ tokio::time::sleep(std::time::Duration::from_secs(5)),+ )+ .await+ .is_none()+ {+ return;+ }}}diff --git a/services/email_service/src/pubsub/gmail_ops/worker.rs b/services/email_service/src/pubsub/gmail_ops/worker.rsindex ce199db1e2..7075a51ba3 100644--- a/services/email_service/src/pubsub/gmail_ops/worker.rs+++ b/services/email_service/src/pubsub/gmail_ops/worker.rs@@ -1,9 +1,11 @@use crate::pubsub::gmail_ops::process;+use crate::pubsub::worker_lifecycle::run_until_cancelled;use crate::util::redis::RedisClient;use authentication_service_client::AuthServiceClient;use futures::StreamExt;use gmail_client::GmailClient;use sqlx::PgPool;+use tokio_util::sync::CancellationToken;/// Context for the Gmail operations worker. Simpler than PubSubContext since/// these operations only need Gmail API access and DB for reverts.@@ -27,6 +29,33 @@ pub async fn run_worker(auth_service_client: AuthServiceClient,redis_client: RedisClient,retry_worker: bool,+) {+ run_worker_with_cancellation(+ db,+ worker,+ sqs_client,+ gmail_client,+ auth_service_client,+ redis_client,+ retry_worker,+ CancellationToken::new(),+ )+ .await;+}++/// Ingests Gmail operations messages until cancellation is requested.+///+/// A batch already returned by SQS is fully processed before shutdown.+#[allow(clippy::too_many_arguments)]+pub async fn run_worker_with_cancellation(+ db: PgPool,+ worker: sqs_worker::SQSWorker,+ sqs_client: sqs_client::SQS,+ gmail_client: GmailClient,+ auth_service_client: AuthServiceClient,+ redis_client: RedisClient,+ retry_worker: bool,+ cancellation_token: CancellationToken,) {let ctx = GmailOpsContext {db,@@ -42,9 +71,19 @@ pub async fn run_worker(let worker_result = tokio::spawn({let ctx = ctx.clone();let worker = worker.clone();+ let cancellation_token = cancellation_token.clone();async move {loop {- match worker.receive_messages().await {+ let Some(receive_result) = run_until_cancelled(+ &cancellation_token,+ worker.receive_messages(),+ )+ .await+ else {+ return;+ };++ match receive_result {Ok(messages) => {if messages.is_empty() {continue;@@ -91,6 +130,10 @@ pub async fn run_worker(}).await;+ if cancellation_token.is_cancelled() {+ return;+ }+match worker_result {Ok(_) => {tracing::error!("gmail ops worker exited successfully?");@@ -101,6 +144,14 @@ pub async fn run_worker(}tracing::info!("GMAIL OPS WORKER RESTARTING...");- tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ if run_until_cancelled(+ &cancellation_token,+ tokio::time::sleep(std::time::Duration::from_secs(5)),+ )+ .await+ .is_none()+ {+ return;+ }}}diff --git a/services/email_service/src/pubsub/inbox_sync/worker.rs b/services/email_service/src/pubsub/inbox_sync/worker.rsindex 22bd611d3d..302020a089 100644--- a/services/email_service/src/pubsub/inbox_sync/worker.rs+++ b/services/email_service/src/pubsub/inbox_sync/worker.rs@@ -1,5 +1,8 @@-use crate::pubsub::context::{CrmServiceType, NotificationIngressType, PubSubContext};+use crate::pubsub::context::{+ CrmServiceType, NotificationIngressType, PubSubContext, PubSubEventBroker,+};use crate::pubsub::inbox_sync::process;+use crate::pubsub::worker_lifecycle::run_until_cancelled;use crate::util::redis::RedisClient;use authentication_service_client::AuthServiceClient;use connection_gateway_client::client::ConnectionGatewayClient;@@ -7,10 +10,10 @@ use contacts::domain::service::SqsContactsIngress;use contacts::outbound::ingress::SqsContactsQueue;use document_storage_service_client::DocumentStorageServiceClient;use futures::StreamExt;-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use static_file_service_client::StaticFileServiceClient;use std::sync::Arc;use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};+use tokio_util::sync::CancellationToken;/// method that ingests sqs messages and calls the process function for each#[expect(clippy::too_many_arguments, reason = "too annoying to fix right now")]@@ -28,9 +31,54 @@ pub async fn run_worker(dss_client: DocumentStorageServiceClient,system_properties_service: Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,crm_service: CrmServiceType,- macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,+ macro_event_broker: PubSubEventBroker,notifications_enabled: bool,retry_worker: bool,+) {+ run_worker_with_cancellation(+ db,+ worker,+ sqs_client,+ contacts_ingress,+ gmail_client,+ auth_service_client,+ redis_client,+ notification_ingress_service,+ sfs_client,+ connection_gateway_client,+ dss_client,+ system_properties_service,+ crm_service,+ macro_event_broker,+ notifications_enabled,+ retry_worker,+ CancellationToken::new(),+ )+ .await;+}++/// Ingests SQS messages until cancellation is requested.+///+/// A batch already returned by SQS is fully processed before shutdown.+#[expect(clippy::too_many_arguments, reason = "too annoying to fix right now")]+pub async fn run_worker_with_cancellation(+ db: sqlx::Pool<sqlx::Postgres>,+ worker: sqs_worker::SQSWorker,+ sqs_client: sqs_client::SQS,+ contacts_ingress: Arc<SqsContactsIngress<SqsContactsQueue>>,+ gmail_client: gmail_client::GmailClient,+ auth_service_client: AuthServiceClient,+ redis_client: RedisClient,+ notification_ingress_service: Arc<NotificationIngressType>,+ sfs_client: StaticFileServiceClient,+ connection_gateway_client: ConnectionGatewayClient,+ dss_client: DocumentStorageServiceClient,+ system_properties_service: Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,+ crm_service: CrmServiceType,+ macro_event_broker: PubSubEventBroker,+ notifications_enabled: bool,+ retry_worker: bool,+ cancellation_token: CancellationToken,) {let ctx = PubSubContext {db,@@ -55,9 +103,19 @@ pub async fn run_worker(let worker_result = tokio::spawn({let ctx = ctx.clone();let worker = worker.clone();+ let cancellation_token = cancellation_token.clone();async move {loop {- match worker.receive_messages().await {+ let Some(receive_result) = run_until_cancelled(+ &cancellation_token,+ worker.receive_messages(),+ )+ .await+ else {+ return;+ };++ match receive_result {Ok(messages) => {if messages.is_empty() {continue;@@ -107,6 +165,10 @@ pub async fn run_worker(}).await;+ if cancellation_token.is_cancelled() {+ return;+ }+match worker_result {Ok(_) => {// This should never be hit@@ -119,6 +181,14 @@ pub async fn run_worker(// Add a delay before restarting to avoid rapid restart loopstracing::info!("WORKER RESTARTING...");- tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ if run_until_cancelled(+ &cancellation_token,+ tokio::time::sleep(std::time::Duration::from_secs(5)),+ )+ .await+ .is_none()+ {+ return;+ }}}diff --git a/services/email_service/src/pubsub/link_manager/context.rs b/services/email_service/src/pubsub/link_manager/context.rsindex 015205b3ec..dd2cfb0599 100644--- a/services/email_service/src/pubsub/link_manager/context.rs+++ b/services/email_service/src/pubsub/link_manager/context.rs@@ -1,9 +1,8 @@-use crate::pubsub::context::{CrmServiceType, NotificationIngressType};+use crate::pubsub::context::{CrmServiceType, NotificationIngressType, PubSubEventBroker};use crate::util::redis::RedisClient;use authentication_service_client::AuthServiceClient;use connection_gateway_client::client::ConnectionGatewayClient;use gmail_client::GmailClient;-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use sqlx::PgPool;use sqs_client::SQS;use std::sync::Arc;@@ -19,5 +18,5 @@ pub struct LinkManagerContext {pub crm_service: CrmServiceType,pub connection_gateway_client: ConnectionGatewayClient,pub notification_ingress_service: Arc<NotificationIngressType>,- pub macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,+ pub macro_event_broker: PubSubEventBroker,}diff --git a/services/email_service/src/pubsub/link_manager/worker.rs b/services/email_service/src/pubsub/link_manager/worker.rsindex 85fe694943..9fbcbafd34 100644--- a/services/email_service/src/pubsub/link_manager/worker.rs+++ b/services/email_service/src/pubsub/link_manager/worker.rs@@ -1,14 +1,15 @@-use crate::pubsub::context::{CrmServiceType, NotificationIngressType};+use crate::pubsub::context::{CrmServiceType, NotificationIngressType, PubSubEventBroker};use crate::pubsub::link_manager::context::LinkManagerContext;use crate::pubsub::link_manager::process;+use crate::pubsub::worker_lifecycle::run_until_cancelled;use crate::util::redis::RedisClient;use authentication_service_client::AuthServiceClient;use connection_gateway_client::client::ConnectionGatewayClient;use futures::StreamExt;-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use sqlx::PgPool;use sqs_client::SQS;use std::sync::Arc;+use tokio_util::sync::CancellationToken;/// method that ingests sqs messages and calls the process function for each#[allow(clippy::too_many_arguments)]@@ -22,7 +23,40 @@ pub async fn run_worker(crm_service: CrmServiceType,connection_gateway_client: ConnectionGatewayClient,notification_ingress_service: Arc<NotificationIngressType>,- macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,+ macro_event_broker: PubSubEventBroker,+) {+ run_worker_with_cancellation(+ worker,+ db,+ gmail_client,+ auth_service_client,+ redis_client,+ sqs_client,+ crm_service,+ connection_gateway_client,+ notification_ingress_service,+ macro_event_broker,+ CancellationToken::new(),+ )+ .await;+}++/// Ingests SQS messages until cancellation is requested.+///+/// A batch already returned by SQS is fully processed before shutdown.+#[allow(clippy::too_many_arguments)]+pub async fn run_worker_with_cancellation(+ worker: sqs_worker::SQSWorker,+ db: PgPool,+ gmail_client: gmail_client::GmailClient,+ auth_service_client: AuthServiceClient,+ redis_client: RedisClient,+ sqs_client: SQS,+ crm_service: CrmServiceType,+ connection_gateway_client: ConnectionGatewayClient,+ notification_ingress_service: Arc<NotificationIngressType>,+ macro_event_broker: PubSubEventBroker,+ cancellation_token: CancellationToken,) {let ctx = LinkManagerContext {db,@@ -40,9 +74,19 @@ pub async fn run_worker(let worker_result = tokio::spawn({let ctx = ctx.clone();let worker = worker.clone();+ let cancellation_token = cancellation_token.clone();async move {loop {- match worker.receive_messages().await {+ let Some(receive_result) = run_until_cancelled(+ &cancellation_token,+ worker.receive_messages(),+ )+ .await+ else {+ return;+ };++ match receive_result {Ok(messages) => {if messages.is_empty() {continue;@@ -92,6 +136,10 @@ pub async fn run_worker(}).await;+ if cancellation_token.is_cancelled() {+ return;+ }+match worker_result {Ok(_) => {// This should never be hit@@ -104,6 +152,14 @@ pub async fn run_worker(// Add a delay before restarting to avoid rapid restart loopstracing::info!("WORKER RESTARTING...");- tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ if run_until_cancelled(+ &cancellation_token,+ tokio::time::sleep(std::time::Duration::from_secs(5)),+ )+ .await+ .is_none()+ {+ return;+ }}}diff --git a/services/email_service/src/pubsub/mod.rs b/services/email_service/src/pubsub/mod.rsindex f43ab8a123..fd70404efe 100644--- a/services/email_service/src/pubsub/mod.rs+++ b/services/email_service/src/pubsub/mod.rs@@ -12,6 +12,7 @@ pub mod scheduled;pub mod sfs_deleter;pub mod sfs_uploader;pub(crate) mod util;+pub(crate) mod worker_lifecycle;/// Log-and-drop publisher for `macro.email` events, re-exported for the/// HTTP binary's handlers.pub use util::publish_email_event;diff --git a/services/email_service/src/pubsub/scheduled/context.rs b/services/email_service/src/pubsub/scheduled/context.rsindex 48fb821478..44cf774ba5 100644--- a/services/email_service/src/pubsub/scheduled/context.rs+++ b/services/email_service/src/pubsub/scheduled/context.rs@@ -1,7 +1,7 @@+use crate::pubsub::context::PubSubEventBroker;use crate::util::redis::RedisClient;use authentication_service_client::AuthServiceClient;use gmail_client::GmailClient;-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use sqlx::PgPool;#[derive(Clone)]@@ -13,5 +13,5 @@ pub struct ScheduledContext {pub redis_client: RedisClient,pub s3_client: s3_client::S3,pub attachment_bucket: String,- pub macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,+ pub macro_event_broker: PubSubEventBroker,}diff --git a/services/email_service/src/pubsub/scheduled/worker.rs b/services/email_service/src/pubsub/scheduled/worker.rsindex 51faca5d71..c0d6e72aef 100644--- a/services/email_service/src/pubsub/scheduled/worker.rs+++ b/services/email_service/src/pubsub/scheduled/worker.rs@@ -1,10 +1,12 @@+use crate::pubsub::context::PubSubEventBroker;use crate::pubsub::scheduled::context::ScheduledContext;use crate::pubsub::scheduled::process;+use crate::pubsub::worker_lifecycle::run_until_cancelled;use crate::util::redis::RedisClient;use authentication_service_client::AuthServiceClient;use futures::StreamExt;-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use sqlx::PgPool;+use tokio_util::sync::CancellationToken;/// method that ingests sqs messages and calls the process function for each#[expect(clippy::too_many_arguments, reason = "matches the other workers")]@@ -16,7 +18,36 @@ pub async fn run_worker(redis_client: RedisClient,s3_client: s3_client::S3,attachment_bucket: String,- macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,+ macro_event_broker: PubSubEventBroker,+) {+ run_worker_with_cancellation(+ worker,+ db,+ gmail_client,+ auth_service_client,+ redis_client,+ s3_client,+ attachment_bucket,+ macro_event_broker,+ CancellationToken::new(),+ )+ .await;+}++/// Ingests SQS messages until cancellation is requested.+///+/// A batch already returned by SQS is fully processed before shutdown.+#[expect(clippy::too_many_arguments, reason = "matches the other workers")]+pub async fn run_worker_with_cancellation(+ worker: sqs_worker::SQSWorker,+ db: PgPool,+ gmail_client: gmail_client::GmailClient,+ auth_service_client: AuthServiceClient,+ redis_client: RedisClient,+ s3_client: s3_client::S3,+ attachment_bucket: String,+ macro_event_broker: PubSubEventBroker,+ cancellation_token: CancellationToken,) {let ctx = ScheduledContext {db,@@ -32,9 +63,19 @@ pub async fn run_worker(let worker_result = tokio::spawn({let ctx = ctx.clone();let worker = worker.clone();+ let cancellation_token = cancellation_token.clone();async move {loop {- match worker.receive_messages().await {+ let Some(receive_result) = run_until_cancelled(+ &cancellation_token,+ worker.receive_messages(),+ )+ .await+ else {+ return;+ };++ match receive_result {Ok(messages) => {if messages.is_empty() {continue;@@ -84,6 +125,10 @@ pub async fn run_worker(}).await;+ if cancellation_token.is_cancelled() {+ return;+ }+match worker_result {Ok(_) => {// This should never be hit@@ -96,6 +141,14 @@ pub async fn run_worker(// Add a delay before restarting to avoid rapid restart loopstracing::info!("WORKER RESTARTING...");- tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ if run_until_cancelled(+ &cancellation_token,+ tokio::time::sleep(std::time::Duration::from_secs(5)),+ )+ .await+ .is_none()+ {+ return;+ }}}diff --git a/services/email_service/src/pubsub/sfs_deleter/worker.rs b/services/email_service/src/pubsub/sfs_deleter/worker.rsindex bfaaee1390..ccc1472818 100644--- a/services/email_service/src/pubsub/sfs_deleter/worker.rs+++ b/services/email_service/src/pubsub/sfs_deleter/worker.rs@@ -1,14 +1,28 @@use crate::pubsub::sfs_deleter::context::SFSDeleteContext;use crate::pubsub::sfs_deleter::process;+use crate::pubsub::worker_lifecycle::run_until_cancelled;use futures::StreamExt;use sqlx::PgPool;use static_file_service_client::StaticFileServiceClient;+use tokio_util::sync::CancellationToken;/// method that ingests sqs messages and calls the process function for eachpub async fn run_worker(worker: sqs_worker::SQSWorker,db: PgPool,sfs_client: StaticFileServiceClient,+) {+ run_worker_with_cancellation(worker, db, sfs_client, CancellationToken::new()).await;+}++/// Ingests SFS deleter messages until cancellation is requested.+///+/// A batch already returned by SQS is fully processed before shutdown.+pub async fn run_worker_with_cancellation(+ worker: sqs_worker::SQSWorker,+ db: PgPool,+ sfs_client: StaticFileServiceClient,+ cancellation_token: CancellationToken,) {let ctx = SFSDeleteContext {db,@@ -19,9 +33,19 @@ pub async fn run_worker(let worker_result = tokio::spawn({let ctx = ctx.clone();let worker = worker.clone();+ let cancellation_token = cancellation_token.clone();async move {loop {- match worker.receive_messages().await {+ let Some(receive_result) = run_until_cancelled(+ &cancellation_token,+ worker.receive_messages(),+ )+ .await+ else {+ return;+ };++ match receive_result {Ok(messages) => {if messages.is_empty() {continue;@@ -71,6 +95,10 @@ pub async fn run_worker(}).await;+ if cancellation_token.is_cancelled() {+ return;+ }+match worker_result {Ok(_) => {// This should never be hit@@ -83,6 +111,14 @@ pub async fn run_worker(// Add a delay before restarting to avoid rapid restart loopstracing::info!("WORKER RESTARTING...");- tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ if run_until_cancelled(+ &cancellation_token,+ tokio::time::sleep(std::time::Duration::from_secs(5)),+ )+ .await+ .is_none()+ {+ return;+ }}}diff --git a/services/email_service/src/pubsub/sfs_uploader/worker.rs b/services/email_service/src/pubsub/sfs_uploader/worker.rsindex 3a5c0615bc..be9e136255 100644--- a/services/email_service/src/pubsub/sfs_uploader/worker.rs+++ b/services/email_service/src/pubsub/sfs_uploader/worker.rs@@ -1,9 +1,11 @@use crate::pubsub::sfs_uploader::context::SFSUploaderContext;use crate::pubsub::sfs_uploader::process;+use crate::pubsub::worker_lifecycle::run_until_cancelled;use connection_gateway_client::client::ConnectionGatewayClient;use futures::StreamExt;use sqlx::PgPool;use static_file_service_client::StaticFileServiceClient;+use tokio_util::sync::CancellationToken;/// method that ingests sqs messages and calls the process function for eachpub async fn run_worker(@@ -11,6 +13,26 @@ pub async fn run_worker(db: PgPool,sfs_client: StaticFileServiceClient,connection_gateway_client: ConnectionGatewayClient,+) {+ run_worker_with_cancellation(+ worker,+ db,+ sfs_client,+ connection_gateway_client,+ CancellationToken::new(),+ )+ .await;+}++/// Ingests SFS uploader messages until cancellation is requested.+///+/// A batch already returned by SQS is fully processed before shutdown.+pub async fn run_worker_with_cancellation(+ worker: sqs_worker::SQSWorker,+ db: PgPool,+ sfs_client: StaticFileServiceClient,+ connection_gateway_client: ConnectionGatewayClient,+ cancellation_token: CancellationToken,) {let ctx = SFSUploaderContext {db,@@ -22,9 +44,19 @@ pub async fn run_worker(let worker_result = tokio::spawn({let ctx = ctx.clone();let worker = worker.clone();+ let cancellation_token = cancellation_token.clone();async move {loop {- match worker.receive_messages().await {+ let Some(receive_result) = run_until_cancelled(+ &cancellation_token,+ worker.receive_messages(),+ )+ .await+ else {+ return;+ };++ match receive_result {Ok(messages) => {if messages.is_empty() {continue;@@ -74,6 +106,10 @@ pub async fn run_worker(}).await;+ if cancellation_token.is_cancelled() {+ return;+ }+match worker_result {Ok(_) => {// This should never be hit@@ -86,6 +122,14 @@ pub async fn run_worker(// Add a delay before restarting to avoid rapid restart loopstracing::info!("WORKER RESTARTING...");- tokio::time::sleep(std::time::Duration::from_secs(5)).await;+ if run_until_cancelled(+ &cancellation_token,+ tokio::time::sleep(std::time::Duration::from_secs(5)),+ )+ .await+ .is_none()+ {+ return;+ }}}diff --git a/services/email_service/src/pubsub/worker_lifecycle.rs b/services/email_service/src/pubsub/worker_lifecycle.rsnew file mode 100644index 0000000000..7307c85879--- /dev/null+++ b/services/email_service/src/pubsub/worker_lifecycle.rs@@ -0,0 +1,23 @@+use std::future::Future;+use tokio_util::sync::CancellationToken;++#[cfg(test)]+mod test;++/// Waits for an operation unless worker shutdown is requested first.+///+/// Cancellation is polled first so an already-cancelled worker does not begin+/// another receive or restart wait.+pub(crate) async fn run_until_cancelled<F>(+ cancellation_token: &CancellationToken,+ operation: F,+) -> Option<F::Output>+where+ F: Future,+{+ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => None,+ output = operation => Some(output),+ }+}diff --git a/services/email_service/src/pubsub/worker_lifecycle/test.rs b/services/email_service/src/pubsub/worker_lifecycle/test.rsnew file mode 100644index 0000000000..d9cc851d35--- /dev/null+++ b/services/email_service/src/pubsub/worker_lifecycle/test.rs@@ -0,0 +1,48 @@+use super::run_until_cancelled;+use std::future::{pending, poll_fn, ready};+use std::sync::atomic::{AtomicBool, Ordering};+use std::task::Poll;+use std::time::Duration;+use tokio_util::sync::CancellationToken;++#[tokio::test]+async fn pending_work_is_interrupted_by_cancellation() {+ let cancellation_token = CancellationToken::new();+ let worker_token = cancellation_token.clone();+ let worker =+ tokio::spawn(async move { run_until_cancelled(&worker_token, pending::<()>()).await });++ tokio::task::yield_now().await;+ cancellation_token.cancel();++ let result = tokio::time::timeout(Duration::from_secs(1), worker)+ .await+ .expect("worker should stop after cancellation")+ .expect("worker task should not panic");+ assert_eq!(result, None);+}++#[tokio::test]+async fn completed_work_is_returned() {+ let cancellation_token = CancellationToken::new();++ let result = run_until_cancelled(&cancellation_token, ready(42)).await;++ assert_eq!(result, Some(42));+}++#[tokio::test]+async fn already_cancelled_token_wins_over_ready_work() {+ let cancellation_token = CancellationToken::new();+ cancellation_token.cancel();+ let operation_was_polled = AtomicBool::new(false);+ let operation = poll_fn(|_| {+ operation_was_polled.store(true, Ordering::Relaxed);+ Poll::Ready(42)+ });++ let result = run_until_cancelled(&cancellation_token, operation).await;++ assert_eq!(result, None);+ assert!(!operation_was_polled.load(Ordering::Relaxed));+}diff --git a/services/mcp_service/Cargo.toml b/services/mcp_service/Cargo.tomlindex e8a7207e81..fa9b5c814a 100644--- a/services/mcp_service/Cargo.toml+++ b/services/mcp_service/Cargo.toml@@ -71,7 +71,8 @@ sqs_client = { path = "../../crates/sqs_client", default-features = false, featu"search",] }sync_service_client = { path = "../../crates/sync_service_client" }-tokio = { workspace = true }+tokio = { workspace = true, features = ["signal"] }+tokio-util = { workspace = true, features = ["rt"] }tracing = { workspace = true }macro_service_urls = { path = "../../crates/macro_service_urls" }diff --git a/services/mcp_service/src/context.rs b/services/mcp_service/src/context.rsindex 04e18f4d37..7a6c6a89ad 100644--- a/services/mcp_service/src/context.rs+++ b/services/mcp_service/src/context.rs@@ -44,6 +44,7 @@ use soup::domain::service::SoupImpl;use soup::outbound::pg_soup_repo::PgSoupRepo;use sqlx::{PgPool, postgres::PgPoolOptions};use sync_service_client::SyncServiceClient;+use tokio_util::task::TaskTracker;use crate::config::Config;@@ -55,7 +56,21 @@ pub struct McpContext {pub mcp_public_host: String,}-pub async fn build_context(config: &Config) -> anyhow::Result<McpContext> {+struct ToolContextBuildArgs<'a> {+ config: &'a Config,+ db: &'a PgPool,+ secretsmanager_client: &'a secretsmanager_client::SecretsManager,+ sqs_client: sqs_client::SQS,+ queue_aws_client: aws_sdk_sqs::Client,+ document_storage_service_auth_key: String,+ sync_service_auth_key: String,+ event_task_tracker: TaskTracker,+}++pub async fn build_context(+ config: &Config,+ event_task_tracker: TaskTracker,+) -> anyhow::Result<McpContext> {let db = PgPoolOptions::new().min_connections(3).max_connections(10)@@ -89,15 +104,19 @@ pub async fn build_context(config: &Config) -> anyhow::Result<McpContext> {.await.context("failed to load sync service auth key")?;- let tool_context = build_tool_context(+ let tool_context = build_tool_context(ToolContextBuildArgs {config,- &db,- &secretsmanager_client,+ db: &db,+ secretsmanager_client: &secretsmanager_client,sqs_client,queue_aws_client,- config.document_storage_service_auth_key.as_ref().to_owned(),- sync_service_auth_key.as_ref().to_owned(),- )+ document_storage_service_auth_key: config+ .document_storage_service_auth_key+ .as_ref()+ .to_owned(),+ sync_service_auth_key: sync_service_auth_key.as_ref().to_owned(),+ event_task_tracker,+ }).await?;let auth_proxy = build_auth_proxy(config, &secretsmanager_client).await?;@@ -116,15 +135,18 @@ pub async fn build_context(config: &Config) -> anyhow::Result<McpContext> {})}-async fn build_tool_context(- config: &Config,- db: &PgPool,- secretsmanager_client: &secretsmanager_client::SecretsManager,- sqs_client: sqs_client::SQS,- queue_aws_client: aws_sdk_sqs::Client,- document_storage_service_auth_key: String,- sync_service_auth_key: String,-) -> anyhow::Result<ToolServiceContext> {+async fn build_tool_context(args: ToolContextBuildArgs<'_>) -> anyhow::Result<ToolServiceContext> {+ let ToolContextBuildArgs {+ config,+ db,+ secretsmanager_client,+ sqs_client,+ queue_aws_client,+ document_storage_service_auth_key,+ sync_service_auth_key,+ event_task_tracker,+ } = args;+let dss_url = DocumentStorageServiceUrl::new()?.to_string();let sync_service_url = SyncServiceUrl::new()?.to_string();let lexical_service_url = LexicalServiceUrl::new()?.to_string();@@ -224,6 +246,7 @@ async fn build_tool_context(let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref()).context("failed to create kafka event publisher")?,+ event_task_tracker,);let document_service = documents::domain::service::DocumentServiceImpl {repo: document_repo,diff --git a/services/mcp_service/src/main.rs b/services/mcp_service/src/main.rsindex 218b9560d4..e91c60d824 100644--- a/services/mcp_service/src/main.rs+++ b/services/mcp_service/src/main.rs@@ -18,9 +18,11 @@ use rmcp::transport::streamable_http_server::{};use std::sync::Arc;use tokio::time::Duration;+use tokio_util::task::TaskTracker;use tool_service::AuthenticatedToolService;const AUTH_PROXY_CLEANUP_INTERVAL: Duration = Duration::from_secs(60);+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);#[tokio::main]#[tracing::instrument(err)]@@ -33,7 +35,8 @@ async fn main() -> anyhow::Result<()> {// responses.let item_base_url = config.app_base_url.as_ref().to_string();- let context = build_context(&config).await?;+ let event_broker_tracker = TaskTracker::new();+ let context = build_context(&config, event_broker_tracker.clone()).await?;// Create the MCP service with authenticated tool handlerlet mcp_service = StreamableHttpService::new(@@ -80,9 +83,23 @@ async fn main() -> anyhow::Result<()> {tracing::info!("MCP server listening on http://{addr}/mcp");- axum::serve(listener, app)+ let server_result = axum::serve(listener, app)+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal()).await- .context("MCP server error")?;+ .context("MCP server error");- Ok(())+ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => {+ tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ );+ }+ }++ server_result}diff --git a/services/scheduled_action/Cargo.toml b/services/scheduled_action/Cargo.tomlindex 36f0f6b302..eff35e6b72 100644--- a/services/scheduled_action/Cargo.toml+++ b/services/scheduled_action/Cargo.toml@@ -38,7 +38,8 @@ chrono = { workspace = true}chrono-tz = { version = "0.10.4", features = ["serde"] }agent = { path = "../../crates/agent" }ai_usage = { path = "../../crates/ai_usage" }-tokio = { workspace = true }+tokio = { workspace = true, features = ["signal"] }+tokio-util = { workspace = true, features = ["rt"] }futures = { workspace = true }tracing = { workspace = true }axum = { workspace = true }diff --git a/services/scheduled_action/src/bins/service.rs b/services/scheduled_action/src/bins/service.rsindex cc039558c5..be9fbe9301 100644--- a/services/scheduled_action/src/bins/service.rs+++ b/services/scheduled_action/src/bins/service.rs@@ -1,5 +1,5 @@#![recursion_limit = "256"]-use std::sync::Arc;+use std::{sync::Arc, time::Duration};use ai_tools::build_tool_service_context_from_env;use anyhow::{Context, Result};@@ -22,13 +22,18 @@ use scheduled_action::inbound::axum_router::{};use scheduled_action::outbound::conn_gateway_live_updates::ConnGatewayLiveUpdates;use scheduled_action::outbound::inprocess_executor::InProcessExecutor;-use scheduled_action::outbound::pg_polling_dispatcher::PgPollingDispatcher;+use scheduled_action::outbound::pg_polling_dispatcher::{+ PgPollingDispatcher, PgPollingDispatcherLifecycle,+};use scheduled_action::outbound::pg_scheduled_action_repo::PgScheduledActionRepo;use scheduled_action::swagger::ApiDoc;use sqlx::postgres::PgPoolOptions;+use tokio_util::{sync::CancellationToken, task::TaskTracker};use utoipa::OpenApi;use utoipa_swagger_ui::SwaggerUi;+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);+#[tokio::main]#[tracing::instrument(err)]async fn main() -> Result<()> {@@ -44,9 +49,11 @@ async fn main() -> Result<()> {.await.context("failed to connect to macrodb")?;- let tool_context = build_tool_service_context_from_env(db.clone())- .await- .context("failed to build tool service context")?;+ let event_broker_tracker = TaskTracker::new();+ let tool_context =+ build_tool_service_context_from_env(db.clone(), event_broker_tracker.clone())+ .await+ .context("failed to build tool service context")?;let aws_config = macro_aws_config::get_macro_aws_config().await;let notification_ingress = Arc::new(SqsNotificationIngress {@@ -87,7 +94,14 @@ async fn main() -> Result<()> {live_updates,));- let dispatcher = PgPollingDispatcher::new(Arc::clone(&repo), dispatcher_executor);+ let dispatcher_cancellation_token = CancellationToken::new();+ let dispatcher_tracker = TaskTracker::new();+ let dispatcher_lifecycle = PgPollingDispatcherLifecycle::new(+ dispatcher_cancellation_token.clone(),+ dispatcher_tracker.clone(),+ );+ let dispatcher = PgPollingDispatcher::new(Arc::clone(&repo), dispatcher_executor)+ .with_lifecycle(dispatcher_lifecycle);let (dispatcher_tx, _execution_rx) = dispatcher.begin_dispatch_loop();let service = Arc::new(ScheduledActionServiceImpl::new(@@ -130,8 +144,29 @@ async fn main() -> Result<()> {tracing::info!("scheduled_action service listening on {addr}");- axum::serve(listener, router.into_make_service())+ let server_result = axum::serve(listener, router.into_make_service())+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal()).await- .context("server closed")?;- unreachable!();+ .context("server closed");++ tracing::info!("stopping scheduled action dispatcher");+ dispatcher_cancellation_token.cancel();+ dispatcher_tracker.close();+ dispatcher_tracker.wait().await;+ tracing::info!("scheduled action dispatcher stopped");++ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => {+ tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ );+ }+ }++ server_result}diff --git a/services/scheduled_action/src/outbound/pg_polling_dispatcher.rs b/services/scheduled_action/src/outbound/pg_polling_dispatcher.rsindex 86b549447d..b097803255 100644--- a/services/scheduled_action/src/outbound/pg_polling_dispatcher.rs+++ b/services/scheduled_action/src/outbound/pg_polling_dispatcher.rs@@ -3,6 +3,11 @@ use std::time::{Duration, Instant};use chrono::Utc;use tokio::sync::mpsc::{Receiver, Sender};+use tokio_util::sync::CancellationToken;+use tokio_util::task::TaskTracker;++#[cfg(test)]+mod test;use crate::domain::models::{DispatchEvent, InProgressExecution};use crate::domain::ports::{@@ -36,6 +41,33 @@ const BATCH_MIN_DURATION: Duration = Duration::from_secs(30);pub struct PgPollingDispatcher<Rpo: ScheduledActionRepo, Exe: ScheduledActionExecutor> {repo: Arc<Rpo>,executor: Exe,+ lifecycle: PgPollingDispatcherLifecycle,+}++/// Cooperative shutdown configuration for [`PgPollingDispatcher`].+///+/// Retain clones of the token and tracker to cancel the dispatcher and wait+/// for both of its background tasks to finish.+#[derive(Clone)]+pub struct PgPollingDispatcherLifecycle {+ cancellation_token: CancellationToken,+ task_tracker: TaskTracker,+}++impl PgPollingDispatcherLifecycle {+ /// Creates lifecycle configuration from caller-owned shutdown handles.+ pub fn new(cancellation_token: CancellationToken, task_tracker: TaskTracker) -> Self {+ Self {+ cancellation_token,+ task_tracker,+ }+ }+}++impl Default for PgPollingDispatcherLifecycle {+ fn default() -> Self {+ Self::new(CancellationToken::new(), TaskTracker::new())+ }}impl<Rpo, Exe> PgPollingDispatcher<Rpo, Exe>@@ -44,7 +76,19 @@ whereExe: ScheduledActionExecutor,{pub fn new(repo: Arc<Rpo>, executor: Exe) -> Self {- Self { repo, executor }+ Self {+ repo,+ executor,+ lifecycle: PgPollingDispatcherLifecycle::default(),+ }+ }++ /// Configures handles for cooperatively stopping and joining dispatcher+ /// tasks. Without this opt-in, the dispatcher retains its previous+ /// process-lifetime behavior.+ pub fn with_lifecycle(mut self, lifecycle: PgPollingDispatcherLifecycle) -> Self {+ self.lifecycle = lifecycle;+ self}}@@ -57,17 +101,46 @@ wherelet (tx, mut rx) = tokio::sync::mpsc::channel::<DispatchEvent>(BUFFER_SIZE);let (extx, exrx) = tokio::sync::mpsc::channel::<InProgressExecution>(BUFFER_SIZE);+ let PgPollingDispatcher {+ repo,+ executor,+ lifecycle,+ } = self;+ let cancellation_token = lifecycle.cancellation_token;+// Drain dispatch events — the polling loop pulls fresh state from the// DB each tick, so create/update/delete notifications are not needed.// We still must drain so service-side sends don't block on a full// channel.- tokio::spawn(async move { while rx.recv().await.is_some() {} });+ let drain_cancellation_token = cancellation_token.clone();+ lifecycle.task_tracker.spawn(async move {+ loop {+ tokio::select! {+ biased;+ _ = drain_cancellation_token.cancelled() => break,+ event = rx.recv() => {+ if event.is_none() {+ break;+ }+ }+ }+ }+ });- tokio::spawn(async move {+ lifecycle.task_tracker.spawn(async move {loop {+ if cancellation_token.is_cancelled() {+ return;+ }+let batch_start = Instant::now();+ let poll_result = tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => return,+ result = repo.get_next_unclaimed_actions(BATCH_SIZE) => result,+ };- match self.repo.get_next_unclaimed_actions(BATCH_SIZE).await {+ match poll_result {Ok(candidates) => {let now = Utc::now();for action in candidates {@@ -83,7 +156,10 @@ where// running. If a peer instance claimed it between// our pull and this call, the claim fails and we// skip — that's the multi-instance contract.- match self.executor.execute_action(action).await {+ if cancellation_token.is_cancelled() {+ return;+ }+ match executor.execute_action(action).await {Ok(execution) => {let _ = extx.send(execution).await;}@@ -107,7 +183,11 @@ wherelet elapsed = batch_start.elapsed();if elapsed < BATCH_MIN_DURATION {- tokio::time::sleep(BATCH_MIN_DURATION - elapsed).await;+ tokio::select! {+ biased;+ _ = cancellation_token.cancelled() => return,+ _ = tokio::time::sleep(BATCH_MIN_DURATION - elapsed) => {}+ }}}});diff --git a/services/scheduled_action/src/outbound/pg_polling_dispatcher/test.rs b/services/scheduled_action/src/outbound/pg_polling_dispatcher/test.rsnew file mode 100644index 0000000000..2e3729244d--- /dev/null+++ b/services/scheduled_action/src/outbound/pg_polling_dispatcher/test.rs@@ -0,0 +1,260 @@+use std::sync::Arc;+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+use std::time::Duration;++use anyhow::Result;+use chrono::{DateTime, Duration as ChronoDuration, Utc};+use macro_user_id::user_id::MacroUserIdStr;+use macro_uuid::{Uuid, generate_uuid_v7};+use serde_json::json;+use tokio::sync::Semaphore;+use tokio_util::sync::CancellationToken;+use tokio_util::task::TaskTracker;++use super::{PgPollingDispatcher, PgPollingDispatcherLifecycle};+use crate::domain::models::{+ ActionExecutionRecord, ActionKind, InProgressExecution, Schedule, ScheduledAction,+};+use crate::domain::ports::{+ ScheduledActionDispatcher, ScheduledActionExecutor, ScheduledActionRepo,+};++struct FakeRepository {+ candidates: Vec<ScheduledAction>,+ cancellation_token: Option<CancellationToken>,+ poll_started: Semaphore,+}++impl FakeRepository {+ fn returning(candidates: Vec<ScheduledAction>) -> Self {+ Self {+ candidates,+ cancellation_token: None,+ poll_started: Semaphore::new(0),+ }+ }++ fn cancelling_before_return(+ candidates: Vec<ScheduledAction>,+ cancellation_token: CancellationToken,+ ) -> Self {+ Self {+ candidates,+ cancellation_token: Some(cancellation_token),+ poll_started: Semaphore::new(0),+ }+ }++ async fn wait_for_poll(&self) {+ self.poll_started+ .acquire()+ .await+ .expect("poll semaphore should remain open")+ .forget();+ }+}++impl ScheduledActionRepo for FakeRepository {+ async fn create_action(&self, action: ScheduledAction) -> Result<ScheduledAction> {+ Ok(action)+ }++ async fn get_actions(&self, _user_id: MacroUserIdStr<'static>) -> Result<Vec<ScheduledAction>> {+ Ok(Vec::new())+ }++ async fn get_next_unclaimed_actions(&self, _limit: i64) -> Result<Vec<ScheduledAction>> {+ self.poll_started.add_permits(1);+ if let Some(cancellation_token) = &self.cancellation_token {+ cancellation_token.cancel();+ }+ Ok(self.candidates.clone())+ }++ async fn update_action(&self, action: ScheduledAction) -> Result<ScheduledAction> {+ Ok(action)+ }++ async fn delete_action(+ &self,+ _id: &Uuid,+ _macro_user_id: MacroUserIdStr<'static>,+ ) -> Result<()> {+ Ok(())+ }++ async fn claim_action(&self, _id: &Uuid) -> Result<()> {+ Ok(())+ }++ async fn release_action(&self, _id: &Uuid) -> Result<()> {+ Ok(())+ }++ async fn create_execution_record(&self, _record: ActionExecutionRecord) -> Result<()> {+ Ok(())+ }++ async fn get_execution_records(&self, _action_id: &Uuid) -> Result<Vec<ActionExecutionRecord>> {+ Ok(Vec::new())+ }++ async fn update_next_run_at(&self, _id: &Uuid) -> Result<()> {+ Ok(())+ }++ async fn update_last_executed(&self, _id: &Uuid, _executed_at: DateTime<Utc>) -> Result<()> {+ Ok(())+ }+}++struct RecordingExecutor {+ execution_count: Arc<AtomicUsize>,+}++impl ScheduledActionExecutor for RecordingExecutor {+ async fn execute_action(&self, action: ScheduledAction) -> Result<InProgressExecution> {+ self.execution_count.fetch_add(1, Ordering::SeqCst);+ Ok(InProgressExecution {+ action_id: action.id.expect("test action should have an id"),+ chat_id: None,+ })+ }+}++struct GatedExecutor {+ execution_count: Arc<AtomicUsize>,+ first_execution_started: Arc<Semaphore>,+ release_first_execution: Arc<Semaphore>,+ first_execution_finished: Arc<AtomicBool>,+}++impl ScheduledActionExecutor for GatedExecutor {+ async fn execute_action(&self, action: ScheduledAction) -> Result<InProgressExecution> {+ let execution_index = self.execution_count.fetch_add(1, Ordering::SeqCst);+ if execution_index == 0 {+ self.first_execution_started.add_permits(1);+ self.release_first_execution+ .acquire()+ .await+ .expect("execution semaphore should remain open")+ .forget();+ self.first_execution_finished.store(true, Ordering::SeqCst);+ }++ Ok(InProgressExecution {+ action_id: action.id.expect("test action should have an id"),+ chat_id: None,+ })+ }+}++fn due_action() -> ScheduledAction {+ let now = Utc::now();+ ScheduledAction {+ id: Some(generate_uuid_v7()),+ owner: MacroUserIdStr::parse_from_str("macro|[email protected]")+ .expect("test owner should be valid"),+ name: "test action".to_string(),+ schedule: Schedule::from_cron("0 * * * * *".to_string())+ .expect("test schedule should be valid"),+ kind: ActionKind::Agent,+ created_at: now,+ updated_at: now,+ timezone: chrono_tz::UTC,+ task: json!({}),+ claimed: None,+ next_run_at: now - ChronoDuration::seconds(1),+ enabled: true,+ }+}++fn lifecycle() -> (CancellationToken, TaskTracker, PgPollingDispatcherLifecycle) {+ let cancellation_token = CancellationToken::new();+ let tracker = TaskTracker::new();+ let lifecycle = PgPollingDispatcherLifecycle::new(cancellation_token.clone(), tracker.clone());+ (cancellation_token, tracker, lifecycle)+}++async fn wait_for_tracker(tracker: &TaskTracker) {+ tokio::time::timeout(Duration::from_secs(1), tracker.wait())+ .await+ .expect("dispatcher tasks should stop after cancellation");+}++#[tokio::test]+async fn cancellation_interrupts_polling_delay_and_dispatch_event_drain() {+ let repository = Arc::new(FakeRepository::returning(Vec::new()));+ let executor = RecordingExecutor {+ execution_count: Arc::new(AtomicUsize::new(0)),+ };+ let (cancellation_token, tracker, lifecycle) = lifecycle();+ let dispatcher =+ PgPollingDispatcher::new(Arc::clone(&repository), executor).with_lifecycle(lifecycle);++ let (_dispatch_tx, _execution_rx) = dispatcher.begin_dispatch_loop();+ repository.wait_for_poll().await;++ cancellation_token.cancel();+ tracker.close();+ wait_for_tracker(&tracker).await;+}++#[tokio::test]+async fn cancellation_during_repository_poll_prevents_candidate_execution() {+ let (cancellation_token, tracker, lifecycle) = lifecycle();+ let repository = Arc::new(FakeRepository::cancelling_before_return(+ vec![due_action()],+ cancellation_token.clone(),+ ));+ let execution_count = Arc::new(AtomicUsize::new(0));+ let executor = RecordingExecutor {+ execution_count: Arc::clone(&execution_count),+ };+ let dispatcher = PgPollingDispatcher::new(repository, executor).with_lifecycle(lifecycle);++ let (_dispatch_tx, _execution_rx) = dispatcher.begin_dispatch_loop();+ tracker.close();+ wait_for_tracker(&tracker).await;++ assert_eq!(execution_count.load(Ordering::SeqCst), 0);+}++#[tokio::test]+async fn cancellation_allows_started_execution_to_finish_without_starting_another() {+ let repository = Arc::new(FakeRepository::returning(vec![due_action(), due_action()]));+ let execution_count = Arc::new(AtomicUsize::new(0));+ let first_execution_started = Arc::new(Semaphore::new(0));+ let release_first_execution = Arc::new(Semaphore::new(0));+ let first_execution_finished = Arc::new(AtomicBool::new(false));+ let executor = GatedExecutor {+ execution_count: Arc::clone(&execution_count),+ first_execution_started: Arc::clone(&first_execution_started),+ release_first_execution: Arc::clone(&release_first_execution),+ first_execution_finished: Arc::clone(&first_execution_finished),+ };+ let (cancellation_token, tracker, lifecycle) = lifecycle();+ let dispatcher = PgPollingDispatcher::new(repository, executor).with_lifecycle(lifecycle);++ let (_dispatch_tx, _execution_rx) = dispatcher.begin_dispatch_loop();+ first_execution_started+ .acquire()+ .await+ .expect("execution semaphore should remain open")+ .forget();++ cancellation_token.cancel();+ tracker.close();+ let wait = tracker.wait();+ tokio::pin!(wait);+ tokio::select! {+ () = &mut wait => panic!("tracker should wait for the in-flight execution"),+ () = tokio::task::yield_now() => {}+ }++ release_first_execution.add_permits(1);+ wait.await;++ assert!(first_execution_finished.load(Ordering::SeqCst));+ assert_eq!(execution_count.load(Ordering::SeqCst), 1);+}