Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(kafka): use tokio tasktracker for kafka event broker (#5206)

Will Hutchinsoned112e00b5cf
diff --git a/Cargo.lock b/Cargo.lock
index bd0c4f8389..89a003cfae 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -245,6 +245,7 @@ dependencies = [
"system_properties",
"teams",
"tokio",
+ "tokio-util",
"tracing",
"utoipa",
"uuid",
@@ -1043,6 +1044,7 @@ dependencies = [
"thiserror 2.0.18",
"time",
"tokio",
+ "tokio-util",
"tower 0.5.3",
"tower-cookies",
"tower-http",
@@ -4336,6 +4338,7 @@ dependencies = [
"task_dedup",
"thiserror 2.0.18",
"tokio",
+ "tokio-util",
"tower 0.5.3",
"tower-http",
"tracing",
@@ -4980,6 +4983,7 @@ dependencies = [
"thiserror 2.0.18",
"tokio",
"tokio-retry",
+ "tokio-util",
"tower 0.5.3",
"tower-http",
"tracing",
@@ -8424,6 +8428,7 @@ dependencies = [
"rootcause",
"rootcause-tracing",
"serde_json",
+ "tokio",
"tracing",
"tracing-opentelemetry",
"tracing-panic",
@@ -8468,6 +8473,7 @@ dependencies = [
"serde_json",
"thiserror 2.0.18",
"tokio",
+ "tokio-util",
"tracing",
"uuid",
"workspace-hack",
@@ -8837,6 +8843,7 @@ dependencies = [
"sqs_client",
"sync_service_client",
"tokio",
+ "tokio-util",
"tracing",
"workspace-hack",
]
@@ -8915,6 +8922,7 @@ dependencies = [
"sync_service_client",
"thiserror 2.0.18",
"tokio",
+ "tokio-util",
"tracing",
"utoipa",
"workspace-hack",
@@ -12481,6 +12489,7 @@ dependencies = [
"serde_json",
"sqlx",
"tokio",
+ "tokio-util",
"tracing",
"utoipa",
"utoipa-swagger-ui",
diff --git a/crates/ai_tools/Cargo.toml b/crates/ai_tools/Cargo.toml
index a6d00a8c80..916de28414 100644
--- a/crates/ai_tools/Cargo.toml
+++ b/crates/ai_tools/Cargo.toml
@@ -80,6 +80,7 @@ strum.workspace = true
sync_service_client = { path = "../sync_service_client" }
system_properties = { path = "../system_properties" }
tokio.workspace = true
+tokio-util = { workspace = true, features = ["rt"] }
tracing.workspace = true
utoipa.workspace = true
uuid.workspace = true
diff --git a/crates/ai_tools/src/build_context.rs b/crates/ai_tools/src/build_context.rs
index a711c2c424..8662891fca 100644
--- a/crates/ai_tools/src/build_context.rs
+++ b/crates/ai_tools/src/build_context.rs
@@ -47,6 +47,7 @@ use soup::domain::service::SoupImpl;
use soup::outbound::pg_soup_repo::PgSoupRepo;
use std::sync::Arc;
use sync_service_client::SyncServiceClient;
+use tokio_util::task::TaskTracker;
env_var! {
struct ToolContextEnvVars {
@@ -97,9 +98,14 @@ maybe_env_var! {
/// - `ENABLE_EMAIL_SCHEDULED_QUEUE`
/// - `ENABLE_GMAIL_OPS_QUEUE` (if disabled, thread-label updates can't enqueue Gmail sync ops)
/// - `ENABLE_NOTIFICATION_QUEUE` (if disabled, notification status updates skip push clearing)
-#[tracing::instrument(skip(pool), err)]
+///
+/// `event_task_tracker` tracks event publishes started by the context. Callers
+/// must retain the original tracker, pass a clone here, and close and drain the
+/// original after the host stops broker-backed work.
+#[tracing::instrument(skip(pool, event_task_tracker), err)]
pub async fn build_tool_service_context_from_env(
pool: sqlx::PgPool,
+ event_task_tracker: TaskTracker,
) -> anyhow::Result<ToolServiceContext> {
let env = ToolContextEnvVars::new()?;
let maybe_env = ToolContextMaybeEnvVars::new();
@@ -254,6 +260,7 @@ pub async fn build_tool_service_context_from_env(
let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
macro_event_broker::KafkaEventPublisher::new(env.kafka_brokers.as_ref())
.context("failed to create kafka event publisher")?,
+ event_task_tracker,
);
// Channel messages sent by AI tools dispatch the same side effects as the
// document-storage channel API (realtime, notifications, search indexing,
diff --git a/crates/ai_tools/src/tool_context.rs b/crates/ai_tools/src/tool_context.rs
index 2538fe14d4..78f47e85cc 100644
--- a/crates/ai_tools/src/tool_context.rs
+++ b/crates/ai_tools/src/tool_context.rs
@@ -47,6 +47,7 @@ use system_properties::{
SystemPropertiesServiceImpl,
};
use teams::{inbound::toolset::TeamToolContext, outbound::team_repo::TeamRepositoryImpl};
+use tokio_util::task::TaskTracker;
pub use ai_toolset::RequestContext;
@@ -81,6 +82,10 @@ pub type ToolEmailService = EmailServiceImpl<
ToolEamService,
>;
+/// Event broker used by AI tools, with spawned publish tasks tracked for
+/// graceful shutdown by the hosting process.
+pub type ToolEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;
+
/// Type alias for the send-capable email service implementation used by user
/// tools. Carries the real event broker: these tools mutate email state, and
/// the Gmail-echo suppression means events skipped here are never recovered.
@@ -90,7 +95,7 @@ pub type ToolUserEmailService = EmailServiceImpl<
sqs_client::SQS,
ToolCrmService,
ToolEamService,
- macro_event_broker::MacroEventBrokerService<macro_event_broker::KafkaEventPublisher>,
+ ToolEventBroker,
>;
/// Type alias for the channel list service implementation.
@@ -148,7 +153,7 @@ pub struct ChannelSideEffectClients {
/// search-event queues.
pub sqs: aws_sdk_sqs::Client,
/// Broker publishing channel events to the `macro.channels` topic.
- pub macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,
+ pub macro_event_broker: ToolEventBroker,
}
/// Build the channel AI tool context dispatching the same side effects as the
@@ -591,7 +596,7 @@ pub type ToolDocumentService = documents::domain::service::DocumentServiceImpl<
NoOpConnectionService,
ToolEntityAccessManagementService,
ToolForeignEntityService,
- macro_event_broker::MacroEventBrokerService<macro_event_broker::KafkaEventPublisher>,
+ ToolEventBroker,
>;
/// Type alias for the entity access service implementation
diff --git a/crates/macro_entrypoint/Cargo.toml b/crates/macro_entrypoint/Cargo.toml
index 3639cb39f3..db4c09fbc3 100644
--- a/crates/macro_entrypoint/Cargo.toml
+++ b/crates/macro_entrypoint/Cargo.toml
@@ -13,6 +13,7 @@ opentelemetry-otlp = { workspace = true }
rootcause = { workspace = true }
rootcause-tracing = { workspace = true }
serde_json = { workspace = true }
+tokio = { workspace = true, features = ["signal"] }
tracing = { workspace = true }
tracing-opentelemetry = { workspace = true }
tracing-panic = "0.1"
@@ -23,3 +24,4 @@ macro_env_var = { path = "../macro_env_var" }
[dev-dependencies]
opentelemetry_sdk = { workspace = true, features = ["testing"] }
+tokio = { workspace = true, features = ["time"] }
diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rs
index 9329d5a2a9..7922a41e63 100644
--- a/crates/macro_entrypoint/src/lib.rs
+++ b/crates/macro_entrypoint/src/lib.rs
@@ -3,10 +3,13 @@
//! This is used to provide consistent behaviour with e.g. tracing configurations
mod datadog_fmt;
+mod shutdown;
#[cfg(test)]
mod test;
+pub use shutdown::shutdown_signal;
+
use macro_env::Environment;
use macro_env_var::{env_vars, maybe_env_vars};
use opentelemetry::trace::TracerProvider as _;
diff --git a/crates/macro_entrypoint/src/shutdown.rs b/crates/macro_entrypoint/src/shutdown.rs
new file mode 100644
index 0000000000..d601c192c6
--- /dev/null
+++ b/crates/macro_entrypoint/src/shutdown.rs
@@ -0,0 +1,58 @@
+use std::{
+ future::{Future, pending},
+ io,
+};
+
+#[cfg(test)]
+mod test;
+
+async fn wait_for_signal_listener<F>(listener: F, signal: &'static str)
+where
+ F: Future<Output = io::Result<()>>,
+{
+ match listener.await {
+ Ok(()) => {}
+ Err(error) => {
+ tracing::error!(error=?error, signal, "failed to install shutdown signal listener");
+ pending::<()>().await;
+ }
+ }
+}
+
+async fn wait_for_shutdown_signals<CtrlC, Terminate>(ctrl_c: CtrlC, terminate: Terminate)
+where
+ CtrlC: Future<Output = io::Result<()>>,
+ Terminate: Future<Output = io::Result<()>>,
+{
+ tokio::select! {
+ _ = wait_for_signal_listener(ctrl_c, "Ctrl+C") => {}
+ _ = wait_for_signal_listener(terminate, "SIGTERM") => {}
+ }
+}
+
+#[cfg(unix)]
+async fn terminate_signal() -> io::Result<()> {
+ let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
+
+ match terminate.recv().await {
+ Some(()) => Ok(()),
+ None => pending().await,
+ }
+}
+
+/// Waits for Ctrl+C on all platforms or SIGTERM on Unix.
+///
+/// A listener installation error is logged and leaves that listener pending so it cannot be
+/// mistaken for a shutdown signal.
+pub async fn shutdown_signal() {
+ let ctrl_c = tokio::signal::ctrl_c();
+
+ #[cfg(unix)]
+ let terminate = terminate_signal();
+
+ #[cfg(not(unix))]
+ let terminate = pending::<io::Result<()>>();
+
+ wait_for_shutdown_signals(ctrl_c, terminate).await;
+ tracing::info!("shutdown signal received");
+}
diff --git a/crates/macro_entrypoint/src/shutdown/test.rs b/crates/macro_entrypoint/src/shutdown/test.rs
new file mode 100644
index 0000000000..12b3bcf246
--- /dev/null
+++ b/crates/macro_entrypoint/src/shutdown/test.rs
@@ -0,0 +1,55 @@
+use super::wait_for_shutdown_signals;
+
+use std::{future::pending, io, time::Duration};
+use tokio::{task::yield_now, time::timeout};
+
+#[tokio::test]
+async fn failed_signal_listener_does_not_resolve_shutdown() {
+ let ctrl_c = async { Err(io::Error::other("listener installation failed")) };
+ let terminate = pending::<io::Result<()>>();
+
+ let result = timeout(
+ Duration::from_millis(10),
+ wait_for_shutdown_signals(ctrl_c, terminate),
+ )
+ .await;
+
+ assert!(
+ result.is_err(),
+ "a listener error must not trigger shutdown"
+ );
+}
+
+#[tokio::test]
+async fn successful_signal_resolves_shutdown() {
+ let ctrl_c = async { Ok(()) };
+ let terminate = pending::<io::Result<()>>();
+
+ let result = timeout(
+ Duration::from_millis(100),
+ wait_for_shutdown_signals(ctrl_c, terminate),
+ )
+ .await;
+
+ assert!(result.is_ok(), "an actual signal must trigger shutdown");
+}
+
+#[tokio::test]
+async fn successful_signal_resolves_after_other_listener_fails() {
+ let ctrl_c = async { Err(io::Error::other("listener installation failed")) };
+ let terminate = async {
+ yield_now().await;
+ Ok(())
+ };
+
+ let result = timeout(
+ Duration::from_millis(100),
+ wait_for_shutdown_signals(ctrl_c, terminate),
+ )
+ .await;
+
+ assert!(
+ result.is_ok(),
+ "a listener error must not mask a later signal"
+ );
+}
diff --git a/crates/macro_event_broker/Cargo.toml b/crates/macro_event_broker/Cargo.toml
index 78cab6a390..f1229ebddc 100644
--- a/crates/macro_event_broker/Cargo.toml
+++ b/crates/macro_event_broker/Cargo.toml
@@ -6,7 +6,7 @@ version = "0.1.0"
[features]
default = ["outbound"]
-outbound = ["dep:kafka_util", "dep:rdkafka"]
+outbound = ["dep:kafka_util", "dep:rdkafka", "dep:tokio-util"]
[dependencies]
kafka_util = { path = "../kafka_util", optional = true }
@@ -15,7 +15,8 @@ rdkafka = { workspace = true, optional = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
-tokio = { workspace = true, features = ["macros", "time"] }
+tokio = { workspace = true, features = ["macros", "rt", "time"] }
+tokio-util = { workspace = true, features = ["rt"], optional = true }
tracing = { workspace = true }
uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/macro_event_broker/examples/example_event.rs b/crates/macro_event_broker/examples/example_event.rs
index 3f75f41038..163311f92a 100644
--- a/crates/macro_event_broker/examples/example_event.rs
+++ b/crates/macro_event_broker/examples/example_event.rs
@@ -1,6 +1,6 @@
use macro_event_broker::{
- Event, EventBrokerError, EventPublisher, MacroEvent, MacroEventBroker, MacroEventBrokerService,
- TopicEvent,
+ Event, EventBrokerError, EventPublisher, GlobalSpawner, MacroEvent, MacroEventBroker,
+ MacroEventBrokerService, TopicEvent,
};
use macro_event_topics::{MacroExampleTopic, Topic};
use serde::{Deserialize, Serialize};
@@ -121,7 +121,7 @@ impl ExampleConsumerEvent {
#[tokio::main]
pub async fn main() -> Result<(), EventBrokerError> {
- let service = MacroEventBrokerService::new(ExampleEventPublisher);
+ let service = MacroEventBrokerService::new(ExampleEventPublisher, GlobalSpawner);
let event = ExampleMacroEvent::created(
"example-123",
ExampleCreatedMetadata {
diff --git a/crates/macro_event_broker/src/domain/ports.rs b/crates/macro_event_broker/src/domain/ports.rs
index f9886ed7b4..2ab980d125 100644
--- a/crates/macro_event_broker/src/domain/ports.rs
+++ b/crates/macro_event_broker/src/domain/ports.rs
@@ -34,6 +34,15 @@ pub trait EventPublisher: Send + Sync + 'static {
) -> impl Future<Output = Result<(), EventBrokerError>> + Send;
}
+/// Outbound port for spawning the background task that performs an event publish.
+pub trait Spawner: Send + Sync + 'static {
+ /// Spawns `future` onto the async runtime and returns a handle to the task.
+ fn spawn<F>(&self, future: F) -> tokio::task::JoinHandle<F::Output>
+ where
+ F: Future + Send + 'static,
+ F::Output: Send + 'static;
+}
+
/// A collection of [`MacroEvent`] types that can decode a broker message.
pub trait MacroEventCollection: Sized {
/// Decodes an event from the supplied broker message.
diff --git a/crates/macro_event_broker/src/domain/service.rs b/crates/macro_event_broker/src/domain/service.rs
index 748f3fe3b1..a9b5be68dd 100644
--- a/crates/macro_event_broker/src/domain/service.rs
+++ b/crates/macro_event_broker/src/domain/service.rs
@@ -11,7 +11,9 @@ use tracing::Instrument as _;
use crate::MessageWrapper;
use crate::domain::models::{EventBrokerError, MacroEvent, TopicEvent};
-use crate::domain::ports::{EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection};
+use crate::domain::ports::{
+ EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection, Spawner,
+};
const PUBLISH_TIMEOUT: Duration = Duration::from_secs(6);
@@ -58,28 +60,35 @@ where
}
/// Orchestrates serializing events and handing them to an [`EventPublisher`].
-pub struct MacroEventBrokerService<P: EventPublisher> {
+pub struct MacroEventBrokerService<P: EventPublisher, S: Spawner> {
publisher: Arc<P>,
+ spawner: S,
}
-impl<P: EventPublisher> Clone for MacroEventBrokerService<P> {
+impl<P, S> Clone for MacroEventBrokerService<P, S>
+where
+ P: EventPublisher,
+ S: Spawner + Clone,
+{
fn clone(&self) -> Self {
Self {
publisher: Arc::clone(&self.publisher),
+ spawner: self.spawner.clone(),
}
}
}
-impl<P: EventPublisher> MacroEventBrokerService<P> {
- /// Create a new service backed by the given outbound publisher.
- pub fn new(publisher: P) -> Self {
+impl<P: EventPublisher, S: Spawner> MacroEventBrokerService<P, S> {
+ /// Creates a new service backed by the given outbound publisher and task spawner.
+ pub fn new(publisher: P, spawner: S) -> Self {
Self {
publisher: Arc::new(publisher),
+ spawner,
}
}
}
-impl<P: EventPublisher> MacroEventBroker for MacroEventBrokerService<P> {
+impl<P: EventPublisher, S: Spawner> MacroEventBroker for MacroEventBrokerService<P, S> {
#[tracing::instrument(err, skip(self, event), fields(topic = event.topic(), key = %event.key()))]
fn send_event<E: MacroEvent + ?Sized>(
&self,
@@ -91,7 +100,7 @@ impl<P: EventPublisher> MacroEventBroker for MacroEventBrokerService<P> {
let publisher = Arc::clone(&self.publisher);
let span = tracing::Span::current();
- let handle = tokio::spawn(
+ let handle = self.spawner.spawn(
async move {
tokio::time::timeout(
PUBLISH_TIMEOUT,
diff --git a/crates/macro_event_broker/src/domain/service/test.rs b/crates/macro_event_broker/src/domain/service/test.rs
index a1e2ccfd4c..fc252c272e 100644
--- a/crates/macro_event_broker/src/domain/service/test.rs
+++ b/crates/macro_event_broker/src/domain/service/test.rs
@@ -1,17 +1,22 @@
use std::future::pending;
use std::sync::{
Arc, Mutex,
- atomic::{AtomicBool, Ordering},
+ atomic::{AtomicBool, AtomicUsize, Ordering},
};
use std::time::Duration;
use macro_event_topics::{MacroExampleTopic, Topic};
use serde::{Deserialize, Serialize};
+use tokio::sync::Notify;
+use tokio_util::task::TaskTracker;
use uuid::Uuid;
use super::*;
use crate::domain::models::{Event, EventBrokerError, MacroEvent, MessageWrapper, TopicEvent};
-use crate::domain::ports::{EventConsumer, EventPublisher, MacroEventBroker, MessageParts};
+use crate::domain::ports::{
+ EventConsumer, EventPublisher, MacroEventBroker, MessageParts, Spawner,
+};
+use crate::outbound::spawner::GlobalSpawner;
/// A captured publish call: topic, key, and raw payload bytes.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -38,6 +43,21 @@ impl EventPublisher for RecordingPublisher {
}
}
+struct RecordingSpawner {
+ spawn_count: Arc<AtomicUsize>,
+}
+
+impl Spawner for RecordingSpawner {
+ fn spawn<F>(&self, future: F) -> tokio::task::JoinHandle<F::Output>
+ where
+ F: Future + Send + 'static,
+ F::Output: Send + 'static,
+ {
+ self.spawn_count.fetch_add(1, Ordering::SeqCst);
+ tokio::spawn(future)
+ }
+}
+
struct PendingPublisher {
started: Arc<AtomicBool>,
}
@@ -49,6 +69,19 @@ impl EventPublisher for PendingPublisher {
}
}
+struct GatedPublisher {
+ started: Arc<AtomicBool>,
+ release: Arc<Notify>,
+}
+
+impl EventPublisher for GatedPublisher {
+ async fn publish<T: Topic>(&self, _key: &str, _payload: &[u8]) -> Result<(), EventBrokerError> {
+ self.started.store(true, Ordering::SeqCst);
+ self.release.notified().await;
+ Ok(())
+ }
+}
+
struct PublishDropGuard {
dropped: Arc<AtomicBool>,
}
@@ -244,9 +277,15 @@ impl MacroEvent for UnserializableMacroEvent {
}
}
+fn unserializable_event() -> UnserializableMacroEvent {
+ UnserializableMacroEvent {
+ event: Event::with_event_id(Uuid::from_u128(2), UnserializableTopicEvent),
+ }
+}
+
#[tokio::test]
async fn dispatch_serializes_and_routes() {
- let service = MacroEventBrokerService::new(RecordingPublisher::default());
+ let service = MacroEventBrokerService::new(RecordingPublisher::default(), GlobalSpawner);
let event = example_event();
let expected_payload = serde_json::to_vec(event.event()).unwrap();
@@ -268,9 +307,12 @@ async fn dispatch_serializes_and_routes() {
#[tokio::test]
async fn dispatch_returns_before_publish_completes() {
let started = Arc::new(AtomicBool::new(false));
- let service = MacroEventBrokerService::new(PendingPublisher {
- started: Arc::clone(&started),
- });
+ let service = MacroEventBrokerService::new(
+ PendingPublisher {
+ started: Arc::clone(&started),
+ },
+ GlobalSpawner,
+ );
let handle = service
.send_event(&example_event())
@@ -287,10 +329,13 @@ async fn dispatch_returns_before_publish_completes() {
async fn dispatch_cancels_publish_at_six_second_timeout() {
let started = Arc::new(AtomicBool::new(false));
let dropped = Arc::new(AtomicBool::new(false));
- let service = MacroEventBrokerService::new(HangingPublisher {
- started: Arc::clone(&started),
- dropped: Arc::clone(&dropped),
- });
+ let service = MacroEventBrokerService::new(
+ HangingPublisher {
+ started: Arc::clone(&started),
+ dropped: Arc::clone(&dropped),
+ },
+ GlobalSpawner,
+ );
let handle = service
.send_event(&example_event())
@@ -317,9 +362,12 @@ async fn dispatch_cancels_publish_at_six_second_timeout() {
#[tokio::test]
async fn dispatch_returns_publisher_failure_from_task() {
let attempted = Arc::new(AtomicBool::new(false));
- let service = MacroEventBrokerService::new(FailingPublisher {
- attempted: Arc::clone(&attempted),
- });
+ let service = MacroEventBrokerService::new(
+ FailingPublisher {
+ attempted: Arc::clone(&attempted),
+ },
+ GlobalSpawner,
+ );
let error = service
.send_event(&example_event())
@@ -332,15 +380,123 @@ async fn dispatch_returns_publisher_failure_from_task() {
assert!(attempted.load(Ordering::SeqCst));
}
+#[tokio::test]
+async fn dispatch_uses_injected_spawner_once() {
+ let spawn_count = Arc::new(AtomicUsize::new(0));
+ let service = MacroEventBrokerService::new(
+ RecordingPublisher::default(),
+ RecordingSpawner {
+ spawn_count: Arc::clone(&spawn_count),
+ },
+ );
+
+ service
+ .send_event(&example_event())
+ .expect("dispatch should succeed")
+ .await
+ .expect("publish task should complete")
+ .expect("publish should succeed");
+
+ assert_eq!(spawn_count.load(Ordering::SeqCst), 1);
+}
+
+#[tokio::test]
+async fn task_tracker_waits_for_unfinished_publish() {
+ let started = Arc::new(AtomicBool::new(false));
+ let release = Arc::new(Notify::new());
+ let tracker = TaskTracker::new();
+ let service = MacroEventBrokerService::new(
+ GatedPublisher {
+ started: Arc::clone(&started),
+ release: Arc::clone(&release),
+ },
+ tracker.clone(),
+ );
+
+ let handle = service
+ .send_event(&example_event())
+ .expect("dispatch should succeed");
+ tokio::task::yield_now().await;
+ assert!(started.load(Ordering::SeqCst));
+
+ tracker.close();
+ let wait = tracker.wait();
+ tokio::pin!(wait);
+ tokio::select! {
+ () = &mut wait => panic!("tracker should wait for the unfinished publish"),
+ () = tokio::task::yield_now() => {}
+ }
+
+ release.notify_one();
+ handle
+ .await
+ .expect("publish task should complete")
+ .expect("publish should succeed");
+ wait.await;
+}
+
+#[tokio::test(start_paused = true)]
+async fn task_tracker_drains_hanging_publish_after_timeout() {
+ let started = Arc::new(AtomicBool::new(false));
+ let dropped = Arc::new(AtomicBool::new(false));
+ let tracker = TaskTracker::new();
+ let service = MacroEventBrokerService::new(
+ HangingPublisher {
+ started: Arc::clone(&started),
+ dropped: Arc::clone(&dropped),
+ },
+ tracker.clone(),
+ );
+
+ let handle = service
+ .send_event(&example_event())
+ .expect("dispatch should succeed");
+ tokio::task::yield_now().await;
+ assert!(started.load(Ordering::SeqCst));
+
+ tracker.close();
+ tokio::time::advance(Duration::from_secs(5)).await;
+ tokio::task::yield_now().await;
+ assert!(!dropped.load(Ordering::SeqCst));
+
+ tokio::time::advance(Duration::from_secs(1)).await;
+ tracker.wait().await;
+ let error = handle
+ .await
+ .expect("publish task should complete")
+ .unwrap_err();
+
+ assert!(matches!(
+ error,
+ EventBrokerError::PublishTimeout { timeout } if timeout == PUBLISH_TIMEOUT
+ ));
+ assert!(dropped.load(Ordering::SeqCst));
+}
+
#[tokio::test]
async fn dispatch_returns_serialization_failure_without_publishing() {
- let service = MacroEventBrokerService::new(RecordingPublisher::default());
- let event = UnserializableMacroEvent {
- event: Event::with_event_id(Uuid::from_u128(2), UnserializableTopicEvent),
- };
+ let service = MacroEventBrokerService::new(RecordingPublisher::default(), GlobalSpawner);
+
+ let error = service.send_event(&unserializable_event()).unwrap_err();
- let error = service.send_event(&event).unwrap_err();
assert!(matches!(error, EventBrokerError::Serialization(_)));
tokio::task::yield_now().await;
assert!(service.publisher.calls.lock().unwrap().is_empty());
}
+
+#[tokio::test]
+async fn serialization_failure_does_not_invoke_spawner_or_publisher() {
+ let spawn_count = Arc::new(AtomicUsize::new(0));
+ let service = MacroEventBrokerService::new(
+ RecordingPublisher::default(),
+ RecordingSpawner {
+ spawn_count: Arc::clone(&spawn_count),
+ },
+ );
+
+ let error = service.send_event(&unserializable_event()).unwrap_err();
+
+ assert!(matches!(error, EventBrokerError::Serialization(_)));
+ assert_eq!(spawn_count.load(Ordering::SeqCst), 0);
+ assert!(service.publisher.calls.lock().unwrap().is_empty());
+}
diff --git a/crates/macro_event_broker/src/lib.rs b/crates/macro_event_broker/src/lib.rs
index 6b2ffc2f16..fe63f4064c 100644
--- a/crates/macro_event_broker/src/lib.rs
+++ b/crates/macro_event_broker/src/lib.rs
@@ -5,13 +5,14 @@
//! [`TopicEvent`](domain::models::TopicEvent) enums, the
//! [`MacroEvent`](domain::models::MacroEvent) event abstraction, the inbound
//! [`MacroEventBroker`](domain::ports::MacroEventBroker) API, the outbound
-//! [`EventPublisher`](domain::ports::EventPublisher) port, the producing
+//! [`EventPublisher`](domain::ports::EventPublisher) and
+//! [`Spawner`](domain::ports::Spawner) ports, the producing
//! [`MacroEventBrokerService`](domain::service::MacroEventBrokerService), and the
//! typed [`MacroEventConsumerService`](domain::service::MacroEventConsumerService),
//! which receives messages through the [`EventConsumer`](domain::ports::EventConsumer) port.
//! Kafka topic definitions live in the `macro_event_topics` crate. Shared Kafka
//! producer and consumer transports live in `kafka_util`, while the [`outbound`]
-//! layer adapts its producer to [`EventPublisher`](domain::ports::EventPublisher).
+//! layer provides Kafka and Tokio adapters for the domain ports.
/// Domain layer: models, ports, and service.
pub mod domain;
@@ -23,7 +24,7 @@ pub use macro_event_topics::{
};
pub use domain::ports::{
- EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection, MessageParts,
+ EventConsumer, EventPublisher, MacroEventBroker, MacroEventCollection, MessageParts, Spawner,
};
pub use domain::service::{
MacroEventBrokerService, MacroEventConsumerService, NoopMacroEventBroker,
@@ -31,6 +32,7 @@ pub use domain::service::{
#[cfg(feature = "outbound")]
pub use outbound::{
kafka_event_consumer::KafkaConsumerAdapter, kafka_event_publisher::KafkaEventPublisher,
+ spawner::GlobalSpawner,
};
/// Outbound adapters for the macro event broker's required ports.
diff --git a/crates/macro_event_broker/src/outbound/mod.rs b/crates/macro_event_broker/src/outbound/mod.rs
index 4fef313236..1202e04a34 100644
--- a/crates/macro_event_broker/src/outbound/mod.rs
+++ b/crates/macro_event_broker/src/outbound/mod.rs
@@ -4,3 +4,6 @@ pub mod kafka_event_consumer;
/// Kafka adapter implementing the [`EventPublisher`](crate::domain::ports::EventPublisher) port.
#[cfg(feature = "outbound")]
pub mod kafka_event_publisher;
+/// Tokio adapters implementing the [`Spawner`](crate::domain::ports::Spawner) port.
+#[cfg(feature = "outbound")]
+pub mod spawner;
diff --git a/crates/macro_event_broker/src/outbound/spawner.rs b/crates/macro_event_broker/src/outbound/spawner.rs
new file mode 100644
index 0000000000..c3829600eb
--- /dev/null
+++ b/crates/macro_event_broker/src/outbound/spawner.rs
@@ -0,0 +1,30 @@
+//! Tokio task-spawning adapters.
+
+use tokio::task::JoinHandle;
+use tokio_util::task::TaskTracker;
+
+use crate::domain::ports::Spawner;
+
+/// A [`Spawner`] that uses the current Tokio runtime without tracking tasks.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct GlobalSpawner;
+
+impl Spawner for GlobalSpawner {
+ fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
+ where
+ F: Future + Send + 'static,
+ F::Output: Send + 'static,
+ {
+ tokio::spawn(future)
+ }
+}
+
+impl Spawner for TaskTracker {
+ fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
+ where
+ F: Future + Send + 'static,
+ F::Output: Send + 'static,
+ {
+ TaskTracker::spawn(self, future)
+ }
+}
diff --git a/crates/memory/Cargo.toml b/crates/memory/Cargo.toml
index 2ba8738e4e..780f3d82b9 100644
--- a/crates/memory/Cargo.toml
+++ b/crates/memory/Cargo.toml
@@ -28,6 +28,7 @@ serde_json = { workspace = true }
sqlx = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
+tokio-util = { workspace = true, features = ["rt"] }
tracing = { workspace = true }
utoipa = { workspace = true }
diff --git a/crates/memory/src/main.rs b/crates/memory/src/main.rs
index d8f4a7854f..992b09a1da 100644
--- a/crates/memory/src/main.rs
+++ b/crates/memory/src/main.rs
@@ -7,6 +7,10 @@ use memory::config::Config;
use memory::domain::{MemoryService, service::MemoryServiceImpl};
use memory::outbound::pg_memory_repo::PgMemoryRepo;
use sqlx::postgres::PgPoolOptions;
+use std::time::Duration;
+use tokio_util::task::TaskTracker;
+
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
#[tokio::main]
#[tracing::instrument(err)]
@@ -14,21 +18,36 @@ async fn main() -> anyhow::Result<()> {
let config = Config::from_env().context("failed to load memory configuration")?;
macro_entrypoint::MacroEntrypoint::new(config.environment).init();
+ let user = MacroUserIdStr::try_from(config.user_id.clone())
+ .context("USER_ID must be a valid Macro user id")?;
+
let pool = PgPoolOptions::new()
.max_connections(5)
.connect(&config.database_url)
.await?;
- let tool_context = build_tool_service_context_from_env(pool.clone()).await?;
+ let event_broker_tracker = TaskTracker::new();
+ let tool_context =
+ build_tool_service_context_from_env(pool.clone(), event_broker_tracker.clone()).await?;
let tools = all_tools();
let memory_repo = PgMemoryRepo::new(pool.clone());
let memory_service = MemoryServiceImpl::new(pool, memory_repo, tool_context, tools);
- let user = MacroUserIdStr::try_from(config.user_id.clone())
- .context("USER_ID must be a valid Macro user id")?;
-
tracing::info!("Generating memory for {user}...");
- match memory_service.get_or_generate_memory(user).await? {
+ let memory_result = memory_service.get_or_generate_memory(user).await;
+
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => tracing::warn!(
+ error = ?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ ),
+ }
+
+ match memory_result? {
Some(memory) => println!("{memory}"),
None => println!("No memory yet, generation triggered in background"),
}
diff --git a/crates/soup_realtime/src/outbound/kafka_publisher/test.rs b/crates/soup_realtime/src/outbound/kafka_publisher/test.rs
index a585ef2010..36854ec445 100644
--- a/crates/soup_realtime/src/outbound/kafka_publisher/test.rs
+++ b/crates/soup_realtime/src/outbound/kafka_publisher/test.rs
@@ -2,7 +2,7 @@ use std::sync::{Arc, Mutex};
use chrono::{DateTime, Utc};
use macro_event_broker::{
- EventBrokerError, EventPublisher, MacroEvent, MacroEventBrokerService, Topic,
+ EventBrokerError, EventPublisher, GlobalSpawner, MacroEvent, MacroEventBrokerService, Topic,
};
use macro_user_id::user_id::MacroUserIdStr;
use models_soup::{document::SoupDocument, item::SoupItem};
@@ -75,10 +75,13 @@ fn message() -> SoupRealtimeMessage {
#[tokio::test]
async fn publishes_typed_recipient_keyed_event_to_soup_topic() {
let records = Arc::new(Mutex::new(Vec::new()));
- let broker = MacroEventBrokerService::new(RecordingPublisher {
- records: records.clone(),
- fail: false,
- });
+ let broker = MacroEventBrokerService::new(
+ RecordingPublisher {
+ records: records.clone(),
+ fail: false,
+ },
+ GlobalSpawner,
+ );
let adapter = KafkaSoupRealtimePublisher::new(broker);
adapter.publish(message()).await.expect("publish succeeds");
@@ -111,10 +114,13 @@ async fn publishes_typed_recipient_keyed_event_to_soup_topic() {
#[tokio::test]
async fn propagates_delivery_failures_from_event_broker_service() {
- let broker = MacroEventBrokerService::new(RecordingPublisher {
- records: Arc::new(Mutex::new(Vec::new())),
- fail: true,
- });
+ let broker = MacroEventBrokerService::new(
+ RecordingPublisher {
+ records: Arc::new(Mutex::new(Vec::new())),
+ fail: true,
+ },
+ GlobalSpawner,
+ );
let adapter = KafkaSoupRealtimePublisher::new(broker);
adapter
diff --git a/services/authentication_service/Cargo.toml b/services/authentication_service/Cargo.toml
index 0b9744a887..8e7743065e 100644
--- a/services/authentication_service/Cargo.toml
+++ b/services/authentication_service/Cargo.toml
@@ -135,7 +135,8 @@ strum = { workspace = true }
teams = { path = "../../crates/teams" }
thiserror = { workspace = true }
time = { workspace = true }
-tokio = { workspace = true }
+tokio = { workspace = true, features = ["signal"] }
+tokio-util = { workspace = true, features = ["rt"] }
tower = { workspace = true }
tower-cookies = { version = "0.11.0" }
tower-http = { workspace = true }
diff --git a/services/authentication_service/src/api/context.rs b/services/authentication_service/src/api/context.rs
index b2ffa8d3fe..dce6734514 100644
--- a/services/authentication_service/src/api/context.rs
+++ b/services/authentication_service/src/api/context.rs
@@ -49,8 +49,11 @@ use roles_and_permissions::{
domain::service::UserRolesAndPermissionsServiceImpl, outbound::pgpool::MacroDB,
};
use sqlx::PgPool;
+use tokio_util::task::TaskTracker;
pub(crate) type NotificationIngressType = SqsNotificationIngress<SqsQueue>;
+pub(crate) type AuthenticationEventBroker =
+ MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;
pub(crate) type ChannelServiceType = ChannelServiceImpl<
PgChannelsRepo,
@@ -61,7 +64,7 @@ pub(crate) type ChannelServiceType = ChannelServiceImpl<
NotificationChannelSender<NotificationIngressType>,
SqsChannelSearchIndexer,
ContactsChannelDispatcher<SqsContactsIngress<SqsContactsQueue>>,
- MacroEventBrokerService<KafkaEventPublisher>,
+ AuthenticationEventBroker,
>,
>,
PgChannelReferenceSharePermissions<EntityAccessServiceType>,
@@ -79,7 +82,7 @@ pub(crate) type TeamsServiceType = teams::domain::team_service::TeamServiceImpl<
teams::outbound::contacts_enqueuer::ContactsIngressEnqueuer<
SqsContactsIngress<SqsContactsQueue>,
>,
- MacroEventBrokerService<KafkaEventPublisher>,
+ AuthenticationEventBroker,
>;
pub(crate) type RateLimiter = RateLimitServiceImpl<RedisRateLimitAdapter<redis::Client>>;
diff --git a/services/authentication_service/src/api/mod.rs b/services/authentication_service/src/api/mod.rs
index 9fc2cda505..cf5c192aa0 100644
--- a/services/authentication_service/src/api/mod.rs
+++ b/services/authentication_service/src/api/mod.rs
@@ -70,6 +70,7 @@ pub async fn setup_and_serve(state: ApiContext, port: usize) -> anyhow::Result<(
listener,
app.into_make_service_with_connect_info::<SocketAddr>(),
)
+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal())
.await
.context("error starting service")
}
diff --git a/services/authentication_service/src/main.rs b/services/authentication_service/src/main.rs
index af26bc5a16..9f7209966f 100644
--- a/services/authentication_service/src/main.rs
+++ b/services/authentication_service/src/main.rs
@@ -72,7 +72,8 @@ use crate::api::context::{
ApiContext, AuthorizationService, MacroApiTokenContext, MacroApiTokenExpirySeconds,
MacroApiTokenIssuer, MacroApiTokenPrivateSecretKey, StripeWebhookSecretKey,
};
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
+use tokio_util::task::TaskTracker;
mod api;
mod config;
@@ -322,9 +323,11 @@ async fn main() -> anyhow::Result<()> {
});
let contacts_enqueuer = ContactsIngressEnqueuer::new(contacts_ingress.clone());
let team_analytics = AnalyticsClientTeamAnalytics::new(analytics_client.clone());
+ let event_broker_tracker = TaskTracker::new();
let macro_event_broker = MacroEventBrokerService::new(
KafkaEventPublisher::new(config.kafka_brokers.as_ref())
.context("failed to create kafka event publisher")?,
+ event_broker_tracker.clone(),
);
let entity_access_service_impl = Arc::new(EntityAccessServiceImpl::new(
PgAccessRepository::new(db.clone()),
@@ -393,7 +396,7 @@ async fn main() -> anyhow::Result<()> {
notification_ingress: notification_ingress_service.clone(),
};
- api::setup_and_serve(
+ let server_result = api::setup_and_serve(
ApiContext {
db,
github_link_service: Arc::new(github_link_service_impl),
@@ -447,10 +450,26 @@ async fn main() -> anyhow::Result<()> {
},
config.port,
)
- .await?;
- Ok(())
+ .await;
+
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => {
+ tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ );
+ }
+ }
+
+ server_result
}
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
+
// SAFETY: this is not a secret value
const IOS_DEVELOPMENT_TEAM_ID: &str = "TY74Q77JBD";
// SAFETY: this is not a secret value
diff --git a/services/document_cognition_service/Cargo.toml b/services/document_cognition_service/Cargo.toml
index 65d91dcbb9..0534e8b949 100644
--- a/services/document_cognition_service/Cargo.toml
+++ b/services/document_cognition_service/Cargo.toml
@@ -129,8 +129,8 @@ strum = { workspace = true }
sync_service_client = { path = "../../crates/sync_service_client" }
thiserror = { workspace = true }
time = { workspace = true }
-tokio = { workspace = true }
-tokio-util = { workspace = true }
+tokio = { workspace = true, features = ["signal"] }
+tokio-util = { workspace = true, features = ["rt"] }
tower = { workspace = true, features = ["util"] }
tower-http = { workspace = true, features = ["limit", "timeout"] }
tracing = { workspace = true }
diff --git a/services/document_cognition_service/src/api/context/mod.rs b/services/document_cognition_service/src/api/context/mod.rs
index 9aa79f1588..23d61b3c77 100644
--- a/services/document_cognition_service/src/api/context/mod.rs
+++ b/services/document_cognition_service/src/api/context/mod.rs
@@ -59,13 +59,15 @@ pub type DcsAttachmentProvider = AttachmentProvider<
StaticFileAttachmentService<CdnStaticFileRepo>,
>;
-/// Type alias for the message service wired to concrete DCS services.
-pub type DcsMessageService = MessageServiceImpl<
- PgChatRepo,
- DcsAttachmentProvider,
- macro_event_broker::MacroEventBrokerService<macro_event_broker::KafkaEventPublisher>,
+/// Kafka-backed event broker with publish tasks tracked for graceful shutdown.
+pub type DcsEventBroker = macro_event_broker::MacroEventBrokerService<
+ macro_event_broker::KafkaEventPublisher,
+ tokio_util::task::TaskTracker,
>;
+/// Type alias for the message service wired to concrete DCS services.
+pub type DcsMessageService = MessageServiceImpl<PgChatRepo, DcsAttachmentProvider, DcsEventBroker>;
+
#[cfg(test)]
mod test;
#[cfg(test)]
@@ -145,8 +147,7 @@ pub struct ApiContext {
pub import_service: Arc<DcsImportService>,
pub onboarding_service: Arc<DcsOnboardingService>,
/// Kafka-backed macro event broker for publishing domain events.
- pub macro_event_broker:
- macro_event_broker::MacroEventBrokerService<macro_event_broker::KafkaEventPublisher>,
+ pub macro_event_broker: DcsEventBroker,
}
impl FromRef<ApiContext>
diff --git a/services/document_cognition_service/src/api/context/test.rs b/services/document_cognition_service/src/api/context/test.rs
index 848e2d0ad8..4e80a6da86 100644
--- a/services/document_cognition_service/src/api/context/test.rs
+++ b/services/document_cognition_service/src/api/context/test.rs
@@ -7,6 +7,7 @@ use stream::domain::{
ItemId, ItemStream, Result as StreamResult, StreamEvent, StreamId, StreamRepo,
};
use tokio::sync::broadcast::{self, Receiver};
+use tokio_util::task::TaskTracker;
pub struct MockConnectionRepo;
@@ -257,6 +258,7 @@ pub async fn test_api_context(pool: sqlx::Pool<sqlx::Postgres>) -> std::sync::Ar
let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
macro_event_broker::KafkaEventPublisher::new("localhost:9092")
.expect("kafka producer config is valid"),
+ TaskTracker::new(),
);
let document_service = documents::domain::service::DocumentServiceImpl::new(
diff --git a/services/document_cognition_service/src/api/mod.rs b/services/document_cognition_service/src/api/mod.rs
index 8269558031..3ae5728ecb 100644
--- a/services/document_cognition_service/src/api/mod.rs
+++ b/services/document_cognition_service/src/api/mod.rs
@@ -65,6 +65,7 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {
"document cognition service is up and running"
);
axum::serve(listener, app.into_make_service())
+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal())
.await
.context("error starting service")
}
diff --git a/services/document_cognition_service/src/main.rs b/services/document_cognition_service/src/main.rs
index 042dde5277..85fd706fc6 100644
--- a/services/document_cognition_service/src/main.rs
+++ b/services/document_cognition_service/src/main.rs
@@ -46,9 +46,10 @@ use notification::outbound::websocket::ConnectionGatewayClient;
use readonly_pool::ReadOnlyPool;
use search_service_client::SearchServiceClient;
use sqlx::postgres::PgPoolOptions;
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
use stream::outbound::redis_pg::RedisPostgresStreamRepo;
use sync_service_client::SyncServiceClient;
+use tokio_util::task::TaskTracker;
mod api;
mod config;
@@ -299,9 +300,11 @@ async fn main() -> anyhow::Result<()> {
import::outbound::document_properties::DocumentPropertiesApplicator::new(
properties_service.clone(),
);
+ let event_broker_tracker = TaskTracker::new();
let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())
.context("failed to create kafka event publisher")?,
+ event_broker_tracker.clone(),
);
let document_service = DocumentServiceImpl::new(
document_repo,
@@ -564,7 +567,7 @@ async fn main() -> anyhow::Result<()> {
),
ai_projections_service_impl.clone(),
);
- tokio::spawn(async move {
+ let ai_projection_worker_task = tokio::spawn(async move {
ai_projection_worker.poll().await;
});
@@ -627,7 +630,7 @@ async fn main() -> anyhow::Result<()> {
),
);
- api::setup_and_serve(ApiContext {
+ let api_result = api::setup_and_serve(ApiContext {
db: db.clone(),
email_service_client_external,
sqs_client: Arc::new(sqs_client),
@@ -664,6 +667,36 @@ async fn main() -> anyhow::Result<()> {
macro_event_broker: macro_event_broker.clone(),
})
.await
- .context("failed to setup and serve api")?;
- Ok(())
+ .context("failed to setup and serve api");
+
+ ai_projection_worker_task.abort();
+ match ai_projection_worker_task.await {
+ Err(error) if error.is_cancelled() => {
+ tracing::info!("ai projection worker stopped");
+ }
+ Err(error) => {
+ tracing::error!(error = ?error, "ai projection worker exited unexpectedly");
+ }
+ Ok(()) => {
+ tracing::error!(
+ error = "worker exited naturally",
+ "ai projection worker exited unexpectedly"
+ );
+ }
+ }
+
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ ),
+ }
+
+ api_result
}
+
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.toml
index 6bb533f996..b124994698 100644
--- a/services/document_storage_service/Cargo.toml
+++ b/services/document_storage_service/Cargo.toml
@@ -186,7 +186,8 @@ sync_service_hex = { path = "../../crates/sync_service", package = "sync_service
system_properties = { path = "../../crates/system_properties" }
task_dedup = { path = "../../crates/task_dedup" }
thiserror = { workspace = true }
-tokio = { workspace = true }
+tokio = { workspace = true, features = ["signal"] }
+tokio-util = { workspace = true, features = ["rt"] }
tower = { workspace = true }
tower-http = { workspace = true, features = ["compression-gzip"] }
tracing = { workspace = true }
diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rs
index 960c06e9d5..a28690430b 100644
--- a/services/document_storage_service/src/api/context.rs
+++ b/services/document_storage_service/src/api/context.rs
@@ -105,6 +105,7 @@ use system_properties::{
PgSystemPropertiesRepository, StatusOption, SystemPropertiesService as _,
SystemPropertiesServiceImpl,
};
+use tokio_util::task::TaskTracker;
use webhook::{
domain::service::WebhookServiceImpl,
inbound::axum_router::WebhookRouterState as MacroWebhookRouterState,
@@ -114,6 +115,9 @@ use webhook::{
},
};
+/// Event broker shared by DSS services.
+pub(crate) type DssEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;
+
/// CRM service for DSS — no-op resolver since DSS doesn't populate.
pub(crate) type DssCrmService = crm::domain::service::CrmServiceImpl<
crm::outbound::companies_repo::CompaniesRepositoryImpl,
@@ -256,7 +260,7 @@ pub(crate) type DocumentService = DocumentServiceImpl<
ConnectionServiceImpl<EntityAccessService, ConnectionGatewayImpl>,
EntityAccessManagementService,
ForeignEntityServiceImpl<PgForeignEntityRepo>,
- MacroEventBrokerService<KafkaEventPublisher>,
+ DssEventBroker,
>;
/// Type alias for the authorization service.
@@ -274,7 +278,7 @@ pub(crate) type ProjectService = ProjectServiceImpl<
ShaCountAdapter,
EntityAccessManagementService,
SqsProjectSearchIndexer,
- MacroEventBrokerService<KafkaEventPublisher>,
+ DssEventBroker,
>;
/// Type alias for the projects router state.
@@ -299,7 +303,7 @@ pub(crate) type DssChannelService = ChannelServiceImpl<
NotificationChannelSender<NotificationIngressType>,
SqsChannelSearchIndexer,
ContactsChannelDispatcher<SqsContactsIngress<SqsContactsQueue>>,
- MacroEventBrokerService<KafkaEventPublisher>,
+ DssEventBroker,
>,
>,
PgChannelReferenceSharePermissions<EntityAccessService>,
@@ -311,8 +315,7 @@ pub(crate) type DssChannelsState =
ChannelsRouterState<DssChannelService, EntityAccessService, AuthorizationService>;
/// Type alias for the bots service wired into DSS.
-pub(crate) type DssBotService =
- BotServiceImpl<PgBotsRepo, MacroEventBrokerService<KafkaEventPublisher>>;
+pub(crate) type DssBotService = BotServiceImpl<PgBotsRepo, DssEventBroker>;
/// Type alias for the bots router state.
pub(crate) type DssBotsState =
@@ -392,11 +395,8 @@ pub(crate) type CalWebhookServiceType = CalWebhookServiceImpl<AnalyticsClientSin
pub(crate) type DssCalWebhookState = CalWebhookRouterState<CalWebhookServiceType>;
/// Type alias for the product webhook service.
-pub(crate) type DssWebhookService = WebhookServiceImpl<
- PgWebhookRepo,
- ReqwestWebhookValidationClient,
- MacroEventBrokerService<KafkaEventPublisher>,
->;
+pub(crate) type DssWebhookService =
+ WebhookServiceImpl<PgWebhookRepo, ReqwestWebhookValidationClient, DssEventBroker>;
/// Type alias for the product webhook rate limiter.
pub(crate) type DssWebhookRateLimiter =
diff --git a/services/document_storage_service/src/api/mod.rs b/services/document_storage_service/src/api/mod.rs
index c04d541d47..12f15e21f3 100644
--- a/services/document_storage_service/src/api/mod.rs
+++ b/services/document_storage_service/src/api/mod.rs
@@ -76,6 +76,7 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {
&state.config.port
);
axum::serve(listener, app.into_make_service())
+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal())
.await
.context("error starting service")
}
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index 48aef0cce6..74dd70d095 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -118,7 +118,7 @@ use soup_realtime::{
},
};
use sqlx::postgres::PgPoolOptions;
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
use sync_service_client::SyncServiceClient;
use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
use task_dedup::{
@@ -130,6 +130,7 @@ use task_dedup::{
postgres::{PgTaskMatchRepo, PgTaskVectorDb},
},
};
+use tokio_util::{sync::CancellationToken, task::TaskTracker};
mod api;
mod config;
@@ -266,9 +267,13 @@ async fn main() -> anyhow::Result<()> {
JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
.await?;
+ let consumer_cancellation_token = CancellationToken::new();
+ let consumer_tracker = TaskTracker::new();
+ let event_broker_tracker = TaskTracker::new();
let macro_event_broker = MacroEventBrokerService::new(
KafkaEventPublisher::new(config.kafka_brokers.as_ref())
.context("failed to create kafka event publisher")?,
+ event_broker_tracker.clone(),
);
let bots_repo = PgBotsRepo::new(db.clone());
let bots_service = BotServiceImpl::new(bots_repo, macro_event_broker.clone());
@@ -730,22 +735,39 @@ async fn main() -> anyhow::Result<()> {
});
let webhook_consumer_brokers = config.kafka_brokers.as_ref().to_string();
- tokio::spawn(async move {
- loop {
- tracing::info!("starting webhook event consumer");
- let result = webhook::inbound::kafka_consumer::run_webhook_event_consumer(
- &webhook_consumer_brokers,
- webhook_ingestion_service.clone(),
- std::future::pending::<()>(),
- )
- .await;
- match result {
- Ok(()) => tracing::error!("webhook event consumer exited unexpectedly"),
- Err(error) => {
- tracing::error!(error = ?error, "webhook event consumer exited unexpectedly");
+ consumer_tracker.spawn({
+ let cancellation_token = consumer_cancellation_token.clone();
+ async move {
+ loop {
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ tracing::info!("starting webhook event consumer");
+ let result = webhook::inbound::kafka_consumer::run_webhook_event_consumer(
+ &webhook_consumer_brokers,
+ webhook_ingestion_service.clone(),
+ cancellation_token.cancelled(),
+ )
+ .await;
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ match result {
+ Ok(()) => tracing::error!("webhook event consumer exited unexpectedly"),
+ Err(error) => {
+ tracing::error!(error = ?error, "webhook event consumer exited unexpectedly");
+ }
+ }
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}
}
}
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}
});
@@ -819,9 +841,10 @@ async fn main() -> anyhow::Result<()> {
// Wire Macro AI to react to mentions. The router posts replies through the
// channel service we just built and runs the agent loop in-process with the
// same pre-configured toolset used by other AI hosts.
- let mut macro_agent_tool_context = ai_tools::build_tool_service_context_from_env(db.clone())
- .await
- .context("failed to build Macro agent tool context")?;
+ let mut macro_agent_tool_context =
+ ai_tools::build_tool_service_context_from_env(db.clone(), event_broker_tracker.clone())
+ .await
+ .context("failed to build Macro agent tool context")?;
// Wire the agent's SendChannelMessage tool to this service's own
// side-effect pipeline so agent-posted messages share the exact instance
// used by the HTTP API, including the in-process bot trigger sender (the
@@ -865,28 +888,49 @@ async fn main() -> anyhow::Result<()> {
anyhow::anyhow!("failed to create realtime Soup topic consumer: {error:?}")
})?,
));
- tokio::spawn({
+ consumer_tracker.spawn({
let soup_realtime_service = Arc::clone(&soup_realtime_service);
+ let cancellation_token = consumer_cancellation_token.clone();
async move {
loop {
- let _ = soup_realtime_service.run().await.inspect_err(|error| {
+ let result = tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ result = soup_realtime_service.run() => result,
+ };
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
+ let _ = result.inspect_err(|error| {
tracing::error!(
error = ?error,
"realtime Soup subscription consumer stopped"
);
});
- tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(1)) => {}
+ }
}
}
});
- tokio::spawn({
+ consumer_tracker.spawn({
let brokers = config.kafka_brokers.as_ref().to_string();
let entity_access_service = entity_access_service.as_ref().clone();
let soup_pool = readonly_pool::ReadOnlyPool(readonly_db.clone());
let macro_event_broker = macro_event_broker.clone();
+ let cancellation_token = consumer_cancellation_token.clone();
async move {
loop {
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
let fanout_service = SoupRealtimeServiceImpl::new(
EntityAccessExpander::new(entity_access_service.clone()),
SoupRepoItemReader::new(PgSoupRepo::new(soup_pool.clone())),
@@ -894,8 +938,13 @@ async fn main() -> anyhow::Result<()> {
);
tracing::info!("starting realtime Soup document consumer");
let result = fanout_service
- .run_document_update_consumer(&brokers, std::future::pending::<()>())
+ .run_document_update_consumer(&brokers, cancellation_token.cancelled())
.await;
+
+ if cancellation_token.is_cancelled() {
+ break;
+ }
+
match result {
Ok(()) => {
tracing::error!("realtime Soup document consumer exited unexpectedly")
@@ -905,7 +954,12 @@ async fn main() -> anyhow::Result<()> {
"realtime Soup document consumer exited unexpectedly"
),
}
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => break,
+ _ = tokio::time::sleep(Duration::from_secs(5)) => {}
+ }
}
}
});
@@ -1024,7 +1078,28 @@ async fn main() -> anyhow::Result<()> {
});
}
- api::setup_and_serve(api_context).await?;
+ let server_result = api::setup_and_serve(api_context).await;
+
+ tracing::info!("stopping event consumers");
+ consumer_cancellation_token.cancel();
+ consumer_tracker.close();
+ consumer_tracker.wait().await;
+ tracing::info!("event consumers stopped");
+
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => {
+ tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ );
+ }
+ }
- Ok(())
+ server_result
}
+
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
diff --git a/services/email_service/Cargo.toml b/services/email_service/Cargo.toml
index 38f6817c11..a92a19f379 100644
--- a/services/email_service/Cargo.toml
+++ b/services/email_service/Cargo.toml
@@ -166,6 +166,7 @@ strum_macros = { workspace = true }
system_properties = { path = "../../crates/system_properties" }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["signal"] }
+tokio-util = { workspace = true, features = ["rt"] }
tokio-retry.workspace = true
tower = { workspace = true }
tower-http = { workspace = true, features = [
diff --git a/services/email_service/src/api/context.rs b/services/email_service/src/api/context.rs
index 56a6a9408f..767e11fedf 100644
--- a/services/email_service/src/api/context.rs
+++ b/services/email_service/src/api/context.rs
@@ -23,11 +23,13 @@ use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use static_file_service_client::StaticFileServiceClient;
use std::sync::Arc;
use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
+use tokio_util::task::TaskTracker;
pub(crate) type AuthorizationService = MacroAuthorizationServiceImpl<MacroAuthJwtValidator>;
pub(crate) type EmailEntityAccessService = EntityAccessServiceImpl<PgAccessRepository>;
pub(crate) type EmailEntityAccessManagementService =
EntityAccessManagementServiceImpl<entity_access_management::outbound::PgRepository>;
+pub(crate) type EmailEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;
pub(crate) type EmailSvc = EmailServiceImpl<
EmailPgRepo,
FrecencyQueryServiceImpl<FrecencyPgStorage>,
@@ -37,7 +39,7 @@ pub(crate) type EmailSvc = EmailServiceImpl<
crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
>,
EmailEntityAccessManagementService,
- MacroEventBrokerService<KafkaEventPublisher>,
+ EmailEventBroker,
>;
#[derive(Clone, FromRef)]
@@ -60,5 +62,5 @@ pub(crate) struct ApiContext {
pub email_thread_state:
EmailThreadRouterState<EmailSvc, EmailEntityAccessService, AuthorizationService>,
pub gmail_token_state: GmailTokenState<GmailTokenProviderImpl>,
- pub macro_event_broker: Arc<MacroEventBrokerService<KafkaEventPublisher>>,
+ pub macro_event_broker: Arc<EmailEventBroker>,
}
diff --git a/services/email_service/src/api/mod.rs b/services/email_service/src/api/mod.rs
index 074344ea43..c638bc0be2 100644
--- a/services/email_service/src/api/mod.rs
+++ b/services/email_service/src/api/mod.rs
@@ -42,6 +42,7 @@ pub async fn setup_and_serve(state: ApiContext) -> anyhow::Result<()> {
port
);
axum::serve(listener, app.into_make_service())
+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal())
.await
.context("error starting service")
}
diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
index 0a4a3405c1..4706b7d5f2 100644
--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
@@ -3,7 +3,7 @@ use anyhow::Context;
use document_storage_service_client::DocumentStorageServiceClient;
use email_service::config::Config;
use email_service::pubsub::CrmMetadataResolver;
-use macro_entrypoint::MacroEntrypoint;
+use macro_entrypoint::{MacroEntrypoint, shutdown_signal};
use macro_env::Environment;
use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use macro_service_urls::{
@@ -14,7 +14,11 @@ use notification::outbound::queue::SqsQueue;
use sqlx::postgres::PgPoolOptions;
use static_file_service_client::StaticFileServiceClient;
use std::sync::Arc;
+use std::time::Duration;
use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
+use tokio_util::{sync::CancellationToken, task::TaskTracker};
+
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
#[tokio::main]
#[tracing::instrument(err)]
@@ -94,9 +98,13 @@ async fn main() -> anyhow::Result<()> {
.sfs_delete_queue(&sfs_delete_queue)
.email_link_manager_queue(&link_manager_queue);
+ let worker_cancellation_token = CancellationToken::new();
+ let worker_tracker = TaskTracker::new();
+ let event_broker_tracker = TaskTracker::new();
let macro_event_broker = MacroEventBrokerService::new(
KafkaEventPublisher::new(config.kafka_brokers.as_ref())
.context("failed to create kafka event publisher")?,
+ event_broker_tracker.clone(),
);
let contacts_ingress = Arc::new(contacts::domain::service::SqsContactsIngress {
@@ -120,6 +128,7 @@ async fn main() -> anyhow::Result<()> {
config.queue_wait_time_seconds,
);
+ #[cfg(feature = "sfs_map")]
let sfs_uploader_workers = (0..config.sfs_uploader_workers)
.map(|_| {
sqs_worker::SQSWorker::new(
@@ -131,6 +140,7 @@ async fn main() -> anyhow::Result<()> {
})
.collect::<Vec<_>>();
+ #[cfg(feature = "sfs_delete")]
let sfs_delete_worker = sqs_worker::SQSWorker::new(
aws_sdk_sqs::Client::new(&gmail_queue_aws_config),
sfs_delete_queue.to_string(),
@@ -323,8 +333,9 @@ async fn main() -> anyhow::Result<()> {
let system_properties_service_inbox_sync = system_properties_service.clone();
let crm_service_inbox_sync = crm_service.clone();
let macro_event_broker_inbox_sync = macro_event_broker.clone();
- tokio::spawn(async move {
- email_service::pubsub::inbox_sync::worker::run_worker(
+ let cancellation_token = worker_cancellation_token.clone();
+ worker_tracker.spawn(async move {
+ email_service::pubsub::inbox_sync::worker::run_worker_with_cancellation(
db_inbox_sync,
worker,
sqs_client_inbox_sync,
@@ -341,6 +352,7 @@ async fn main() -> anyhow::Result<()> {
macro_event_broker_inbox_sync,
config.notifications_enabled,
false,
+ cancellation_token,
)
.await;
});
@@ -365,8 +377,9 @@ async fn main() -> anyhow::Result<()> {
let system_properties_service_inbox_sync = system_properties_service.clone();
let crm_service_inbox_sync = crm_service.clone();
let macro_event_broker_inbox_sync = macro_event_broker.clone();
- tokio::spawn(async move {
- email_service::pubsub::inbox_sync::worker::run_worker(
+ let cancellation_token = worker_cancellation_token.clone();
+ worker_tracker.spawn(async move {
+ email_service::pubsub::inbox_sync::worker::run_worker_with_cancellation(
db_inbox_sync,
worker,
sqs_client_inbox_sync,
@@ -383,6 +396,7 @@ async fn main() -> anyhow::Result<()> {
macro_event_broker_inbox_sync,
config.notifications_enabled,
true,
+ cancellation_token,
)
.await;
});
@@ -399,8 +413,9 @@ async fn main() -> anyhow::Result<()> {
let gmail_client_gmail_ops = gmail_client.clone();
let auth_service_client_gmail_ops = auth_service_client.clone();
let redis_client_gmail_ops = redis_client.clone();
- tokio::spawn(async move {
- email_service::pubsub::gmail_ops::worker::run_worker(
+ let cancellation_token = worker_cancellation_token.clone();
+ worker_tracker.spawn(async move {
+ email_service::pubsub::gmail_ops::worker::run_worker_with_cancellation(
db_gmail_ops,
worker,
sqs_client_gmail_ops,
@@ -408,6 +423,7 @@ async fn main() -> anyhow::Result<()> {
auth_service_client_gmail_ops,
redis_client_gmail_ops,
false,
+ cancellation_token,
)
.await;
});
@@ -424,8 +440,9 @@ async fn main() -> anyhow::Result<()> {
let gmail_client_gmail_ops = gmail_client.clone();
let auth_service_client_gmail_ops = auth_service_client.clone();
let redis_client_gmail_ops = redis_client.clone();
- tokio::spawn(async move {
- email_service::pubsub::gmail_ops::worker::run_worker(
+ let cancellation_token = worker_cancellation_token.clone();
+ worker_tracker.spawn(async move {
+ email_service::pubsub::gmail_ops::worker::run_worker_with_cancellation(
db_gmail_ops,
worker,
sqs_client_gmail_ops,
@@ -433,6 +450,7 @@ async fn main() -> anyhow::Result<()> {
auth_service_client_gmail_ops,
redis_client_gmail_ops,
true,
+ cancellation_token,
)
.await;
});
@@ -457,8 +475,9 @@ async fn main() -> anyhow::Result<()> {
let system_properties_service_backfill = system_properties_service.clone();
let crm_service_backfill = crm_service_backfill.clone();
let macro_event_broker_backfill = macro_event_broker.clone();
- tokio::spawn(async move {
- email_service::pubsub::backfill::worker::run_worker(
+ let cancellation_token = worker_cancellation_token.clone();
+ worker_tracker.spawn(async move {
+ email_service::pubsub::backfill::worker::run_worker_with_cancellation(
db_backfill,
worker,
sqs_client_backfill,
@@ -474,6 +493,7 @@ async fn main() -> anyhow::Result<()> {
crm_service_backfill,
macro_event_broker_backfill,
config.notifications_enabled,
+ cancellation_token,
)
.await;
});
@@ -489,12 +509,14 @@ async fn main() -> anyhow::Result<()> {
let db_crm_cleanup = db_backfill.clone();
let sqs_client_crm_cleanup = sqs_client.clone();
let crm_service_crm_cleanup = crm_service_backfill.clone();
- tokio::spawn(async move {
- email_service::pubsub::crm_cleanup::worker::run_worker(
+ let cancellation_token = worker_cancellation_token.clone();
+ worker_tracker.spawn(async move {
+ email_service::pubsub::crm_cleanup::worker::run_worker_with_cancellation(
db_crm_cleanup,
worker,
sqs_client_crm_cleanup,
crm_service_crm_cleanup,
+ cancellation_token,
)
.await;
});
@@ -513,9 +535,10 @@ async fn main() -> anyhow::Result<()> {
let connection_gateway_client_link_manager = connection_gateway_client.clone();
let notification_ingress_service_link_manager = notification_ingress_service.clone();
let macro_event_broker_link_manager = macro_event_broker.clone();
+ let cancellation_token = worker_cancellation_token.clone();
// daily link_manager operations for user contacts and inbox subscriptions
- tokio::spawn(async move {
- email_service::pubsub::link_manager::worker::run_worker(
+ worker_tracker.spawn(async move {
+ email_service::pubsub::link_manager::worker::run_worker_with_cancellation(
link_manager_worker,
db_link_manager,
gmail_client_link_manager,
@@ -526,6 +549,7 @@ async fn main() -> anyhow::Result<()> {
connection_gateway_client_link_manager,
notification_ingress_service_link_manager,
macro_event_broker_link_manager,
+ cancellation_token,
)
.await;
});
@@ -537,9 +561,10 @@ async fn main() -> anyhow::Result<()> {
let s3_client_scheduled = s3_client.clone();
let attachment_bucket_scheduled = config.attachment_bucket.to_string();
let macro_event_broker_scheduled = macro_event_broker.clone();
+ let cancellation_token = worker_cancellation_token.clone();
// send scheduled emails
- tokio::spawn(async move {
- email_service::pubsub::scheduled::worker::run_worker(
+ worker_tracker.spawn(async move {
+ email_service::pubsub::scheduled::worker::run_worker_with_cancellation(
scheduled_worker,
db_scheduled,
gmail_client_scheduled,
@@ -548,22 +573,26 @@ async fn main() -> anyhow::Result<()> {
s3_client_scheduled,
attachment_bucket_scheduled,
macro_event_broker_scheduled,
+ cancellation_token,
)
.await;
});
- if cfg!(feature = "sfs_map") {
+ #[cfg(feature = "sfs_map")]
+ {
for worker in sfs_uploader_workers {
let db_sfs_uploader = db.clone();
let sfs_client_sfs_uploader = sfs_client.clone();
let connection_gateway_client_sfs_uploader = connection_gateway_client.clone();
+ let cancellation_token = worker_cancellation_token.clone();
// upload user contact images to sfs from contact sync
- tokio::spawn(async move {
- email_service::pubsub::sfs_uploader::worker::run_worker(
+ worker_tracker.spawn(async move {
+ email_service::pubsub::sfs_uploader::worker::run_worker_with_cancellation(
worker,
db_sfs_uploader,
sfs_client_sfs_uploader,
connection_gateway_client_sfs_uploader,
+ cancellation_token,
)
.await;
});
@@ -574,15 +603,18 @@ async fn main() -> anyhow::Result<()> {
);
}
- if cfg!(feature = "sfs_delete") {
+ #[cfg(feature = "sfs_delete")]
+ {
let db_sfs_delete = db.clone();
let sfs_client_sfs_delete = sfs_client.clone();
+ let cancellation_token = worker_cancellation_token.clone();
// delete orphaned sfs attachments
- tokio::spawn(async move {
- email_service::pubsub::sfs_deleter::worker::run_worker(
+ worker_tracker.spawn(async move {
+ email_service::pubsub::sfs_deleter::worker::run_worker_with_cancellation(
sfs_delete_worker,
db_sfs_delete,
sfs_client_sfs_delete,
+ cancellation_token,
)
.await;
});
@@ -591,18 +623,23 @@ async fn main() -> anyhow::Result<()> {
tracing::info!("All workers started successfully");
- // Wait for shutdown signal (SIGTERM from ECS or SIGINT from Ctrl+C)
- tokio::select! {
- _ = tokio::signal::ctrl_c() => {
- tracing::info!("Received SIGINT (Ctrl+C)");
- }
- _ = async {
- let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
- .expect("failed to install SIGTERM handler");
- term.recv().await
- } => {
- tracing::info!("Received SIGTERM");
- }
+ shutdown_signal().await;
+
+ worker_cancellation_token.cancel();
+ worker_tracker.close();
+ tracing::info!("Waiting for email workers to stop");
+ worker_tracker.wait().await;
+ tracing::info!("Email workers stopped");
+
+ event_broker_tracker.close();
+ tracing::info!("Waiting for event broker publishes to drain");
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("Event broker publishes drained"),
+ Err(error) => tracing::warn!(
+ error = ?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "Timed out waiting for event broker publishes to drain"
+ ),
}
tracing::info!("Shutdown signal received, exiting gracefully...");
diff --git a/services/email_service/src/main.rs b/services/email_service/src/main.rs
index 0c22b357bc..1ff7c65314 100644
--- a/services/email_service/src/main.rs
+++ b/services/email_service/src/main.rs
@@ -20,8 +20,9 @@ use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use macro_service_urls::{AuthServiceUrl, DocumentStorageServiceUrl, StaticFileServiceUrl};
use sqlx::postgres::PgPoolOptions;
use static_file_service_client::StaticFileServiceClient;
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
+use tokio_util::task::TaskTracker;
mod api;
mod utils;
@@ -143,9 +144,11 @@ async fn main() -> anyhow::Result<()> {
crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
);
+ let event_broker_tracker = TaskTracker::new();
let macro_event_broker = MacroEventBrokerService::new(
KafkaEventPublisher::new(config.kafka_brokers.as_ref())
.context("failed to create kafka event publisher")?,
+ event_broker_tracker.clone(),
);
let email_service = EmailRouterState::new(
EmailServiceImpl::new(
@@ -179,7 +182,7 @@ async fn main() -> anyhow::Result<()> {
redis_conn,
auth_service_client.clone(),
));
- api::setup_and_serve(ApiContext {
+ let api_result = api::setup_and_serve(ApiContext {
db,
internal_api_key: config.internal_api_key.clone(),
config: Arc::new(config),
@@ -199,6 +202,22 @@ async fn main() -> anyhow::Result<()> {
gmail_token_state,
macro_event_broker: Arc::new(macro_event_broker),
})
- .await?;
- Ok(())
+ .await;
+
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => {
+ tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ );
+ }
+ }
+
+ api_result
}
+
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
diff --git a/services/email_service/src/pubsub/backfill/worker.rs b/services/email_service/src/pubsub/backfill/worker.rs
index 40a1e08b73..57472939ac 100644
--- a/services/email_service/src/pubsub/backfill/worker.rs
+++ b/services/email_service/src/pubsub/backfill/worker.rs
@@ -1,5 +1,8 @@
use crate::pubsub::backfill::process;
-use crate::pubsub::context::{CrmServiceType, NotificationIngressType, PubSubContext};
+use crate::pubsub::context::{
+ CrmServiceType, NotificationIngressType, PubSubContext, PubSubEventBroker,
+};
+use crate::pubsub::worker_lifecycle::run_until_cancelled;
use crate::util::redis::RedisClient;
use authentication_service_client::AuthServiceClient;
use connection_gateway_client::client::ConnectionGatewayClient;
@@ -7,10 +10,10 @@ use contacts::domain::service::SqsContactsIngress;
use contacts::outbound::ingress::SqsContactsQueue;
use document_storage_service_client::DocumentStorageServiceClient;
use futures::StreamExt;
-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use static_file_service_client::StaticFileServiceClient;
use std::sync::Arc;
use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
+use tokio_util::sync::CancellationToken;
/// method that ingests sqs messages and calls the process function for each
#[expect(clippy::too_many_arguments, reason = "too annoying to fix right now")]
@@ -28,8 +31,51 @@ pub async fn run_worker(
dss_client: DocumentStorageServiceClient,
system_properties_service: Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,
crm_service: CrmServiceType,
- macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,
+ macro_event_broker: PubSubEventBroker,
notifications_enabled: bool,
+) {
+ run_worker_with_cancellation(
+ db,
+ worker,
+ sqs_client,
+ contacts_ingress,
+ gmail_client,
+ auth_service_client,
+ redis_client,
+ notification_ingress_service,
+ sfs_client,
+ connection_gateway_client,
+ dss_client,
+ system_properties_service,
+ crm_service,
+ macro_event_broker,
+ notifications_enabled,
+ CancellationToken::new(),
+ )
+ .await;
+}
+
+/// Ingests SQS messages until cancellation is requested.
+///
+/// A batch already returned by SQS is fully processed before shutdown.
+#[expect(clippy::too_many_arguments, reason = "too annoying to fix right now")]
+pub async fn run_worker_with_cancellation(
+ db: sqlx::Pool<sqlx::Postgres>,
+ worker: sqs_worker::SQSWorker,
+ sqs_client: sqs_client::SQS,
+ contacts_ingress: Arc<SqsContactsIngress<SqsContactsQueue>>,
+ gmail_client: gmail_client::GmailClient,
+ auth_service_client: AuthServiceClient,
+ redis_client: RedisClient,
+ notification_ingress_service: Arc<NotificationIngressType>,
+ sfs_client: StaticFileServiceClient,
+ connection_gateway_client: ConnectionGatewayClient,
+ dss_client: DocumentStorageServiceClient,
+ system_properties_service: Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,
+ crm_service: CrmServiceType,
+ macro_event_broker: PubSubEventBroker,
+ notifications_enabled: bool,
+ cancellation_token: CancellationToken,
) {
let ctx = PubSubContext {
db,
@@ -54,9 +100,19 @@ pub async fn run_worker(
let worker_result = tokio::spawn({
let ctx = ctx.clone();
let worker = worker.clone();
+ let cancellation_token = cancellation_token.clone();
async move {
loop {
- match worker.receive_messages().await {
+ let Some(receive_result) = run_until_cancelled(
+ &cancellation_token,
+ worker.receive_messages(),
+ )
+ .await
+ else {
+ return;
+ };
+
+ match receive_result {
Ok(messages) => {
if messages.is_empty() {
continue;
@@ -106,6 +162,10 @@ pub async fn run_worker(
})
.await;
+ if cancellation_token.is_cancelled() {
+ return;
+ }
+
match worker_result {
Ok(_) => {
// This should never be hit
@@ -118,6 +178,14 @@ pub async fn run_worker(
// Add a delay before restarting to avoid rapid restart loops
tracing::info!("WORKER RESTARTING...");
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ if run_until_cancelled(
+ &cancellation_token,
+ tokio::time::sleep(std::time::Duration::from_secs(5)),
+ )
+ .await
+ .is_none()
+ {
+ return;
+ }
}
}
diff --git a/services/email_service/src/pubsub/context.rs b/services/email_service/src/pubsub/context.rs
index 9540fbb53b..7b23e5ac51 100644
--- a/services/email_service/src/pubsub/context.rs
+++ b/services/email_service/src/pubsub/context.rs
@@ -18,6 +18,10 @@ use sqlx::PgPool;
use static_file_service_client::StaticFileServiceClient;
use std::sync::Arc;
use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
+use tokio_util::task::TaskTracker;
+
+/// The event broker used by pubsub workers, with publish tasks tracked for graceful shutdown.
+pub type PubSubEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;
/// The concrete notification ingress service type.
pub type NotificationIngressType = SqsNotificationIngress<SqsQueue>;
@@ -70,7 +74,7 @@ pub struct PubSubContext {
pub dss_client: DocumentStorageServiceClient,
pub system_properties_service: Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,
pub crm_service: CrmServiceType,
- pub macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,
+ pub macro_event_broker: PubSubEventBroker,
pub notifications_enabled: bool,
pub retry_worker: bool,
}
diff --git a/services/email_service/src/pubsub/crm_cleanup/worker.rs b/services/email_service/src/pubsub/crm_cleanup/worker.rs
index a8b68c7b64..31d62ad514 100644
--- a/services/email_service/src/pubsub/crm_cleanup/worker.rs
+++ b/services/email_service/src/pubsub/crm_cleanup/worker.rs
@@ -1,6 +1,8 @@
use crate::pubsub::context::CrmServiceType;
use crate::pubsub::crm_cleanup::process;
+use crate::pubsub::worker_lifecycle::run_until_cancelled;
use futures::StreamExt;
+use tokio_util::sync::CancellationToken;
/// Shared dependencies for the CRM cleanup handlers. Much smaller than the
/// backfill `PubSubContext` — this worker only talks to the DB, SQS, and the
@@ -19,6 +21,26 @@ pub async fn run_worker(
worker: sqs_worker::SQSWorker,
sqs_client: sqs_client::SQS,
crm_service: CrmServiceType,
+) {
+ run_worker_with_cancellation(
+ db,
+ worker,
+ sqs_client,
+ crm_service,
+ CancellationToken::new(),
+ )
+ .await;
+}
+
+/// Ingests CRM cleanup messages until cancellation is requested.
+///
+/// A batch already returned by SQS is fully processed before shutdown.
+pub async fn run_worker_with_cancellation(
+ db: sqlx::Pool<sqlx::Postgres>,
+ worker: sqs_worker::SQSWorker,
+ sqs_client: sqs_client::SQS,
+ crm_service: CrmServiceType,
+ cancellation_token: CancellationToken,
) {
let ctx = CrmCleanupContext {
db,
@@ -31,9 +53,19 @@ pub async fn run_worker(
let worker_result = tokio::spawn({
let ctx = ctx.clone();
let worker = worker.clone();
+ let cancellation_token = cancellation_token.clone();
async move {
loop {
- match worker.receive_messages().await {
+ let Some(receive_result) = run_until_cancelled(
+ &cancellation_token,
+ worker.receive_messages(),
+ )
+ .await
+ else {
+ return;
+ };
+
+ match receive_result {
Ok(messages) => {
if messages.is_empty() {
continue;
@@ -68,6 +100,10 @@ pub async fn run_worker(
})
.await;
+ if cancellation_token.is_cancelled() {
+ return;
+ }
+
match worker_result {
Ok(_) => {
// This should never be hit
@@ -80,6 +116,14 @@ pub async fn run_worker(
// Add a delay before restarting to avoid rapid restart loops
tracing::info!("CRM CLEANUP WORKER RESTARTING...");
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ if run_until_cancelled(
+ &cancellation_token,
+ tokio::time::sleep(std::time::Duration::from_secs(5)),
+ )
+ .await
+ .is_none()
+ {
+ return;
+ }
}
}
diff --git a/services/email_service/src/pubsub/gmail_ops/worker.rs b/services/email_service/src/pubsub/gmail_ops/worker.rs
index ce199db1e2..7075a51ba3 100644
--- a/services/email_service/src/pubsub/gmail_ops/worker.rs
+++ b/services/email_service/src/pubsub/gmail_ops/worker.rs
@@ -1,9 +1,11 @@
use crate::pubsub::gmail_ops::process;
+use crate::pubsub::worker_lifecycle::run_until_cancelled;
use crate::util::redis::RedisClient;
use authentication_service_client::AuthServiceClient;
use futures::StreamExt;
use gmail_client::GmailClient;
use sqlx::PgPool;
+use tokio_util::sync::CancellationToken;
/// Context for the Gmail operations worker. Simpler than PubSubContext since
/// these operations only need Gmail API access and DB for reverts.
@@ -27,6 +29,33 @@ pub async fn run_worker(
auth_service_client: AuthServiceClient,
redis_client: RedisClient,
retry_worker: bool,
+) {
+ run_worker_with_cancellation(
+ db,
+ worker,
+ sqs_client,
+ gmail_client,
+ auth_service_client,
+ redis_client,
+ retry_worker,
+ CancellationToken::new(),
+ )
+ .await;
+}
+
+/// Ingests Gmail operations messages until cancellation is requested.
+///
+/// A batch already returned by SQS is fully processed before shutdown.
+#[allow(clippy::too_many_arguments)]
+pub async fn run_worker_with_cancellation(
+ db: PgPool,
+ worker: sqs_worker::SQSWorker,
+ sqs_client: sqs_client::SQS,
+ gmail_client: GmailClient,
+ auth_service_client: AuthServiceClient,
+ redis_client: RedisClient,
+ retry_worker: bool,
+ cancellation_token: CancellationToken,
) {
let ctx = GmailOpsContext {
db,
@@ -42,9 +71,19 @@ pub async fn run_worker(
let worker_result = tokio::spawn({
let ctx = ctx.clone();
let worker = worker.clone();
+ let cancellation_token = cancellation_token.clone();
async move {
loop {
- match worker.receive_messages().await {
+ let Some(receive_result) = run_until_cancelled(
+ &cancellation_token,
+ worker.receive_messages(),
+ )
+ .await
+ else {
+ return;
+ };
+
+ match receive_result {
Ok(messages) => {
if messages.is_empty() {
continue;
@@ -91,6 +130,10 @@ pub async fn run_worker(
})
.await;
+ if cancellation_token.is_cancelled() {
+ return;
+ }
+
match worker_result {
Ok(_) => {
tracing::error!("gmail ops worker exited successfully?");
@@ -101,6 +144,14 @@ pub async fn run_worker(
}
tracing::info!("GMAIL OPS WORKER RESTARTING...");
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ if run_until_cancelled(
+ &cancellation_token,
+ tokio::time::sleep(std::time::Duration::from_secs(5)),
+ )
+ .await
+ .is_none()
+ {
+ return;
+ }
}
}
diff --git a/services/email_service/src/pubsub/inbox_sync/worker.rs b/services/email_service/src/pubsub/inbox_sync/worker.rs
index 22bd611d3d..302020a089 100644
--- a/services/email_service/src/pubsub/inbox_sync/worker.rs
+++ b/services/email_service/src/pubsub/inbox_sync/worker.rs
@@ -1,5 +1,8 @@
-use crate::pubsub::context::{CrmServiceType, NotificationIngressType, PubSubContext};
+use crate::pubsub::context::{
+ CrmServiceType, NotificationIngressType, PubSubContext, PubSubEventBroker,
+};
use crate::pubsub::inbox_sync::process;
+use crate::pubsub::worker_lifecycle::run_until_cancelled;
use crate::util::redis::RedisClient;
use authentication_service_client::AuthServiceClient;
use connection_gateway_client::client::ConnectionGatewayClient;
@@ -7,10 +10,10 @@ use contacts::domain::service::SqsContactsIngress;
use contacts::outbound::ingress::SqsContactsQueue;
use document_storage_service_client::DocumentStorageServiceClient;
use futures::StreamExt;
-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use static_file_service_client::StaticFileServiceClient;
use std::sync::Arc;
use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
+use tokio_util::sync::CancellationToken;
/// method that ingests sqs messages and calls the process function for each
#[expect(clippy::too_many_arguments, reason = "too annoying to fix right now")]
@@ -28,9 +31,54 @@ pub async fn run_worker(
dss_client: DocumentStorageServiceClient,
system_properties_service: Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,
crm_service: CrmServiceType,
- macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,
+ macro_event_broker: PubSubEventBroker,
notifications_enabled: bool,
retry_worker: bool,
+) {
+ run_worker_with_cancellation(
+ db,
+ worker,
+ sqs_client,
+ contacts_ingress,
+ gmail_client,
+ auth_service_client,
+ redis_client,
+ notification_ingress_service,
+ sfs_client,
+ connection_gateway_client,
+ dss_client,
+ system_properties_service,
+ crm_service,
+ macro_event_broker,
+ notifications_enabled,
+ retry_worker,
+ CancellationToken::new(),
+ )
+ .await;
+}
+
+/// Ingests SQS messages until cancellation is requested.
+///
+/// A batch already returned by SQS is fully processed before shutdown.
+#[expect(clippy::too_many_arguments, reason = "too annoying to fix right now")]
+pub async fn run_worker_with_cancellation(
+ db: sqlx::Pool<sqlx::Postgres>,
+ worker: sqs_worker::SQSWorker,
+ sqs_client: sqs_client::SQS,
+ contacts_ingress: Arc<SqsContactsIngress<SqsContactsQueue>>,
+ gmail_client: gmail_client::GmailClient,
+ auth_service_client: AuthServiceClient,
+ redis_client: RedisClient,
+ notification_ingress_service: Arc<NotificationIngressType>,
+ sfs_client: StaticFileServiceClient,
+ connection_gateway_client: ConnectionGatewayClient,
+ dss_client: DocumentStorageServiceClient,
+ system_properties_service: Arc<SystemPropertiesServiceImpl<PgSystemPropertiesRepository>>,
+ crm_service: CrmServiceType,
+ macro_event_broker: PubSubEventBroker,
+ notifications_enabled: bool,
+ retry_worker: bool,
+ cancellation_token: CancellationToken,
) {
let ctx = PubSubContext {
db,
@@ -55,9 +103,19 @@ pub async fn run_worker(
let worker_result = tokio::spawn({
let ctx = ctx.clone();
let worker = worker.clone();
+ let cancellation_token = cancellation_token.clone();
async move {
loop {
- match worker.receive_messages().await {
+ let Some(receive_result) = run_until_cancelled(
+ &cancellation_token,
+ worker.receive_messages(),
+ )
+ .await
+ else {
+ return;
+ };
+
+ match receive_result {
Ok(messages) => {
if messages.is_empty() {
continue;
@@ -107,6 +165,10 @@ pub async fn run_worker(
})
.await;
+ if cancellation_token.is_cancelled() {
+ return;
+ }
+
match worker_result {
Ok(_) => {
// This should never be hit
@@ -119,6 +181,14 @@ pub async fn run_worker(
// Add a delay before restarting to avoid rapid restart loops
tracing::info!("WORKER RESTARTING...");
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ if run_until_cancelled(
+ &cancellation_token,
+ tokio::time::sleep(std::time::Duration::from_secs(5)),
+ )
+ .await
+ .is_none()
+ {
+ return;
+ }
}
}
diff --git a/services/email_service/src/pubsub/link_manager/context.rs b/services/email_service/src/pubsub/link_manager/context.rs
index 015205b3ec..dd2cfb0599 100644
--- a/services/email_service/src/pubsub/link_manager/context.rs
+++ b/services/email_service/src/pubsub/link_manager/context.rs
@@ -1,9 +1,8 @@
-use crate::pubsub::context::{CrmServiceType, NotificationIngressType};
+use crate::pubsub::context::{CrmServiceType, NotificationIngressType, PubSubEventBroker};
use crate::util::redis::RedisClient;
use authentication_service_client::AuthServiceClient;
use connection_gateway_client::client::ConnectionGatewayClient;
use gmail_client::GmailClient;
-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use sqlx::PgPool;
use sqs_client::SQS;
use std::sync::Arc;
@@ -19,5 +18,5 @@ pub struct LinkManagerContext {
pub crm_service: CrmServiceType,
pub connection_gateway_client: ConnectionGatewayClient,
pub notification_ingress_service: Arc<NotificationIngressType>,
- pub macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,
+ pub macro_event_broker: PubSubEventBroker,
}
diff --git a/services/email_service/src/pubsub/link_manager/worker.rs b/services/email_service/src/pubsub/link_manager/worker.rs
index 85fe694943..9fbcbafd34 100644
--- a/services/email_service/src/pubsub/link_manager/worker.rs
+++ b/services/email_service/src/pubsub/link_manager/worker.rs
@@ -1,14 +1,15 @@
-use crate::pubsub::context::{CrmServiceType, NotificationIngressType};
+use crate::pubsub::context::{CrmServiceType, NotificationIngressType, PubSubEventBroker};
use crate::pubsub::link_manager::context::LinkManagerContext;
use crate::pubsub::link_manager::process;
+use crate::pubsub::worker_lifecycle::run_until_cancelled;
use crate::util::redis::RedisClient;
use authentication_service_client::AuthServiceClient;
use connection_gateway_client::client::ConnectionGatewayClient;
use futures::StreamExt;
-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use sqlx::PgPool;
use sqs_client::SQS;
use std::sync::Arc;
+use tokio_util::sync::CancellationToken;
/// method that ingests sqs messages and calls the process function for each
#[allow(clippy::too_many_arguments)]
@@ -22,7 +23,40 @@ pub async fn run_worker(
crm_service: CrmServiceType,
connection_gateway_client: ConnectionGatewayClient,
notification_ingress_service: Arc<NotificationIngressType>,
- macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,
+ macro_event_broker: PubSubEventBroker,
+) {
+ run_worker_with_cancellation(
+ worker,
+ db,
+ gmail_client,
+ auth_service_client,
+ redis_client,
+ sqs_client,
+ crm_service,
+ connection_gateway_client,
+ notification_ingress_service,
+ macro_event_broker,
+ CancellationToken::new(),
+ )
+ .await;
+}
+
+/// Ingests SQS messages until cancellation is requested.
+///
+/// A batch already returned by SQS is fully processed before shutdown.
+#[allow(clippy::too_many_arguments)]
+pub async fn run_worker_with_cancellation(
+ worker: sqs_worker::SQSWorker,
+ db: PgPool,
+ gmail_client: gmail_client::GmailClient,
+ auth_service_client: AuthServiceClient,
+ redis_client: RedisClient,
+ sqs_client: SQS,
+ crm_service: CrmServiceType,
+ connection_gateway_client: ConnectionGatewayClient,
+ notification_ingress_service: Arc<NotificationIngressType>,
+ macro_event_broker: PubSubEventBroker,
+ cancellation_token: CancellationToken,
) {
let ctx = LinkManagerContext {
db,
@@ -40,9 +74,19 @@ pub async fn run_worker(
let worker_result = tokio::spawn({
let ctx = ctx.clone();
let worker = worker.clone();
+ let cancellation_token = cancellation_token.clone();
async move {
loop {
- match worker.receive_messages().await {
+ let Some(receive_result) = run_until_cancelled(
+ &cancellation_token,
+ worker.receive_messages(),
+ )
+ .await
+ else {
+ return;
+ };
+
+ match receive_result {
Ok(messages) => {
if messages.is_empty() {
continue;
@@ -92,6 +136,10 @@ pub async fn run_worker(
})
.await;
+ if cancellation_token.is_cancelled() {
+ return;
+ }
+
match worker_result {
Ok(_) => {
// This should never be hit
@@ -104,6 +152,14 @@ pub async fn run_worker(
// Add a delay before restarting to avoid rapid restart loops
tracing::info!("WORKER RESTARTING...");
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ if run_until_cancelled(
+ &cancellation_token,
+ tokio::time::sleep(std::time::Duration::from_secs(5)),
+ )
+ .await
+ .is_none()
+ {
+ return;
+ }
}
}
diff --git a/services/email_service/src/pubsub/mod.rs b/services/email_service/src/pubsub/mod.rs
index f43ab8a123..fd70404efe 100644
--- a/services/email_service/src/pubsub/mod.rs
+++ b/services/email_service/src/pubsub/mod.rs
@@ -12,6 +12,7 @@ pub mod scheduled;
pub mod sfs_deleter;
pub mod sfs_uploader;
pub(crate) mod util;
+pub(crate) mod worker_lifecycle;
/// Log-and-drop publisher for `macro.email` events, re-exported for the
/// HTTP binary's handlers.
pub use util::publish_email_event;
diff --git a/services/email_service/src/pubsub/scheduled/context.rs b/services/email_service/src/pubsub/scheduled/context.rs
index 48fb821478..44cf774ba5 100644
--- a/services/email_service/src/pubsub/scheduled/context.rs
+++ b/services/email_service/src/pubsub/scheduled/context.rs
@@ -1,7 +1,7 @@
+use crate::pubsub::context::PubSubEventBroker;
use crate::util::redis::RedisClient;
use authentication_service_client::AuthServiceClient;
use gmail_client::GmailClient;
-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use sqlx::PgPool;
#[derive(Clone)]
@@ -13,5 +13,5 @@ pub struct ScheduledContext {
pub redis_client: RedisClient,
pub s3_client: s3_client::S3,
pub attachment_bucket: String,
- pub macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,
+ pub macro_event_broker: PubSubEventBroker,
}
diff --git a/services/email_service/src/pubsub/scheduled/worker.rs b/services/email_service/src/pubsub/scheduled/worker.rs
index 51faca5d71..c0d6e72aef 100644
--- a/services/email_service/src/pubsub/scheduled/worker.rs
+++ b/services/email_service/src/pubsub/scheduled/worker.rs
@@ -1,10 +1,12 @@
+use crate::pubsub::context::PubSubEventBroker;
use crate::pubsub::scheduled::context::ScheduledContext;
use crate::pubsub::scheduled::process;
+use crate::pubsub::worker_lifecycle::run_until_cancelled;
use crate::util::redis::RedisClient;
use authentication_service_client::AuthServiceClient;
use futures::StreamExt;
-use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use sqlx::PgPool;
+use tokio_util::sync::CancellationToken;
/// method that ingests sqs messages and calls the process function for each
#[expect(clippy::too_many_arguments, reason = "matches the other workers")]
@@ -16,7 +18,36 @@ pub async fn run_worker(
redis_client: RedisClient,
s3_client: s3_client::S3,
attachment_bucket: String,
- macro_event_broker: MacroEventBrokerService<KafkaEventPublisher>,
+ macro_event_broker: PubSubEventBroker,
+) {
+ run_worker_with_cancellation(
+ worker,
+ db,
+ gmail_client,
+ auth_service_client,
+ redis_client,
+ s3_client,
+ attachment_bucket,
+ macro_event_broker,
+ CancellationToken::new(),
+ )
+ .await;
+}
+
+/// Ingests SQS messages until cancellation is requested.
+///
+/// A batch already returned by SQS is fully processed before shutdown.
+#[expect(clippy::too_many_arguments, reason = "matches the other workers")]
+pub async fn run_worker_with_cancellation(
+ worker: sqs_worker::SQSWorker,
+ db: PgPool,
+ gmail_client: gmail_client::GmailClient,
+ auth_service_client: AuthServiceClient,
+ redis_client: RedisClient,
+ s3_client: s3_client::S3,
+ attachment_bucket: String,
+ macro_event_broker: PubSubEventBroker,
+ cancellation_token: CancellationToken,
) {
let ctx = ScheduledContext {
db,
@@ -32,9 +63,19 @@ pub async fn run_worker(
let worker_result = tokio::spawn({
let ctx = ctx.clone();
let worker = worker.clone();
+ let cancellation_token = cancellation_token.clone();
async move {
loop {
- match worker.receive_messages().await {
+ let Some(receive_result) = run_until_cancelled(
+ &cancellation_token,
+ worker.receive_messages(),
+ )
+ .await
+ else {
+ return;
+ };
+
+ match receive_result {
Ok(messages) => {
if messages.is_empty() {
continue;
@@ -84,6 +125,10 @@ pub async fn run_worker(
})
.await;
+ if cancellation_token.is_cancelled() {
+ return;
+ }
+
match worker_result {
Ok(_) => {
// This should never be hit
@@ -96,6 +141,14 @@ pub async fn run_worker(
// Add a delay before restarting to avoid rapid restart loops
tracing::info!("WORKER RESTARTING...");
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ if run_until_cancelled(
+ &cancellation_token,
+ tokio::time::sleep(std::time::Duration::from_secs(5)),
+ )
+ .await
+ .is_none()
+ {
+ return;
+ }
}
}
diff --git a/services/email_service/src/pubsub/sfs_deleter/worker.rs b/services/email_service/src/pubsub/sfs_deleter/worker.rs
index bfaaee1390..ccc1472818 100644
--- a/services/email_service/src/pubsub/sfs_deleter/worker.rs
+++ b/services/email_service/src/pubsub/sfs_deleter/worker.rs
@@ -1,14 +1,28 @@
use crate::pubsub::sfs_deleter::context::SFSDeleteContext;
use crate::pubsub::sfs_deleter::process;
+use crate::pubsub::worker_lifecycle::run_until_cancelled;
use futures::StreamExt;
use sqlx::PgPool;
use static_file_service_client::StaticFileServiceClient;
+use tokio_util::sync::CancellationToken;
/// method that ingests sqs messages and calls the process function for each
pub async fn run_worker(
worker: sqs_worker::SQSWorker,
db: PgPool,
sfs_client: StaticFileServiceClient,
+) {
+ run_worker_with_cancellation(worker, db, sfs_client, CancellationToken::new()).await;
+}
+
+/// Ingests SFS deleter messages until cancellation is requested.
+///
+/// A batch already returned by SQS is fully processed before shutdown.
+pub async fn run_worker_with_cancellation(
+ worker: sqs_worker::SQSWorker,
+ db: PgPool,
+ sfs_client: StaticFileServiceClient,
+ cancellation_token: CancellationToken,
) {
let ctx = SFSDeleteContext {
db,
@@ -19,9 +33,19 @@ pub async fn run_worker(
let worker_result = tokio::spawn({
let ctx = ctx.clone();
let worker = worker.clone();
+ let cancellation_token = cancellation_token.clone();
async move {
loop {
- match worker.receive_messages().await {
+ let Some(receive_result) = run_until_cancelled(
+ &cancellation_token,
+ worker.receive_messages(),
+ )
+ .await
+ else {
+ return;
+ };
+
+ match receive_result {
Ok(messages) => {
if messages.is_empty() {
continue;
@@ -71,6 +95,10 @@ pub async fn run_worker(
})
.await;
+ if cancellation_token.is_cancelled() {
+ return;
+ }
+
match worker_result {
Ok(_) => {
// This should never be hit
@@ -83,6 +111,14 @@ pub async fn run_worker(
// Add a delay before restarting to avoid rapid restart loops
tracing::info!("WORKER RESTARTING...");
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ if run_until_cancelled(
+ &cancellation_token,
+ tokio::time::sleep(std::time::Duration::from_secs(5)),
+ )
+ .await
+ .is_none()
+ {
+ return;
+ }
}
}
diff --git a/services/email_service/src/pubsub/sfs_uploader/worker.rs b/services/email_service/src/pubsub/sfs_uploader/worker.rs
index 3a5c0615bc..be9e136255 100644
--- a/services/email_service/src/pubsub/sfs_uploader/worker.rs
+++ b/services/email_service/src/pubsub/sfs_uploader/worker.rs
@@ -1,9 +1,11 @@
use crate::pubsub::sfs_uploader::context::SFSUploaderContext;
use crate::pubsub::sfs_uploader::process;
+use crate::pubsub::worker_lifecycle::run_until_cancelled;
use connection_gateway_client::client::ConnectionGatewayClient;
use futures::StreamExt;
use sqlx::PgPool;
use static_file_service_client::StaticFileServiceClient;
+use tokio_util::sync::CancellationToken;
/// method that ingests sqs messages and calls the process function for each
pub async fn run_worker(
@@ -11,6 +13,26 @@ pub async fn run_worker(
db: PgPool,
sfs_client: StaticFileServiceClient,
connection_gateway_client: ConnectionGatewayClient,
+) {
+ run_worker_with_cancellation(
+ worker,
+ db,
+ sfs_client,
+ connection_gateway_client,
+ CancellationToken::new(),
+ )
+ .await;
+}
+
+/// Ingests SFS uploader messages until cancellation is requested.
+///
+/// A batch already returned by SQS is fully processed before shutdown.
+pub async fn run_worker_with_cancellation(
+ worker: sqs_worker::SQSWorker,
+ db: PgPool,
+ sfs_client: StaticFileServiceClient,
+ connection_gateway_client: ConnectionGatewayClient,
+ cancellation_token: CancellationToken,
) {
let ctx = SFSUploaderContext {
db,
@@ -22,9 +44,19 @@ pub async fn run_worker(
let worker_result = tokio::spawn({
let ctx = ctx.clone();
let worker = worker.clone();
+ let cancellation_token = cancellation_token.clone();
async move {
loop {
- match worker.receive_messages().await {
+ let Some(receive_result) = run_until_cancelled(
+ &cancellation_token,
+ worker.receive_messages(),
+ )
+ .await
+ else {
+ return;
+ };
+
+ match receive_result {
Ok(messages) => {
if messages.is_empty() {
continue;
@@ -74,6 +106,10 @@ pub async fn run_worker(
})
.await;
+ if cancellation_token.is_cancelled() {
+ return;
+ }
+
match worker_result {
Ok(_) => {
// This should never be hit
@@ -86,6 +122,14 @@ pub async fn run_worker(
// Add a delay before restarting to avoid rapid restart loops
tracing::info!("WORKER RESTARTING...");
- tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ if run_until_cancelled(
+ &cancellation_token,
+ tokio::time::sleep(std::time::Duration::from_secs(5)),
+ )
+ .await
+ .is_none()
+ {
+ return;
+ }
}
}
diff --git a/services/email_service/src/pubsub/worker_lifecycle.rs b/services/email_service/src/pubsub/worker_lifecycle.rs
new file mode 100644
index 0000000000..7307c85879
--- /dev/null
+++ b/services/email_service/src/pubsub/worker_lifecycle.rs
@@ -0,0 +1,23 @@
+use std::future::Future;
+use tokio_util::sync::CancellationToken;
+
+#[cfg(test)]
+mod test;
+
+/// Waits for an operation unless worker shutdown is requested first.
+///
+/// Cancellation is polled first so an already-cancelled worker does not begin
+/// another receive or restart wait.
+pub(crate) async fn run_until_cancelled<F>(
+ cancellation_token: &CancellationToken,
+ operation: F,
+) -> Option<F::Output>
+where
+ F: Future,
+{
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => None,
+ output = operation => Some(output),
+ }
+}
diff --git a/services/email_service/src/pubsub/worker_lifecycle/test.rs b/services/email_service/src/pubsub/worker_lifecycle/test.rs
new file mode 100644
index 0000000000..d9cc851d35
--- /dev/null
+++ b/services/email_service/src/pubsub/worker_lifecycle/test.rs
@@ -0,0 +1,48 @@
+use super::run_until_cancelled;
+use std::future::{pending, poll_fn, ready};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::task::Poll;
+use std::time::Duration;
+use tokio_util::sync::CancellationToken;
+
+#[tokio::test]
+async fn pending_work_is_interrupted_by_cancellation() {
+ let cancellation_token = CancellationToken::new();
+ let worker_token = cancellation_token.clone();
+ let worker =
+ tokio::spawn(async move { run_until_cancelled(&worker_token, pending::<()>()).await });
+
+ tokio::task::yield_now().await;
+ cancellation_token.cancel();
+
+ let result = tokio::time::timeout(Duration::from_secs(1), worker)
+ .await
+ .expect("worker should stop after cancellation")
+ .expect("worker task should not panic");
+ assert_eq!(result, None);
+}
+
+#[tokio::test]
+async fn completed_work_is_returned() {
+ let cancellation_token = CancellationToken::new();
+
+ let result = run_until_cancelled(&cancellation_token, ready(42)).await;
+
+ assert_eq!(result, Some(42));
+}
+
+#[tokio::test]
+async fn already_cancelled_token_wins_over_ready_work() {
+ let cancellation_token = CancellationToken::new();
+ cancellation_token.cancel();
+ let operation_was_polled = AtomicBool::new(false);
+ let operation = poll_fn(|_| {
+ operation_was_polled.store(true, Ordering::Relaxed);
+ Poll::Ready(42)
+ });
+
+ let result = run_until_cancelled(&cancellation_token, operation).await;
+
+ assert_eq!(result, None);
+ assert!(!operation_was_polled.load(Ordering::Relaxed));
+}
diff --git a/services/mcp_service/Cargo.toml b/services/mcp_service/Cargo.toml
index e8a7207e81..fa9b5c814a 100644
--- a/services/mcp_service/Cargo.toml
+++ b/services/mcp_service/Cargo.toml
@@ -71,7 +71,8 @@ sqs_client = { path = "../../crates/sqs_client", default-features = false, featu
"search",
] }
sync_service_client = { path = "../../crates/sync_service_client" }
-tokio = { workspace = true }
+tokio = { workspace = true, features = ["signal"] }
+tokio-util = { workspace = true, features = ["rt"] }
tracing = { workspace = true }
macro_service_urls = { path = "../../crates/macro_service_urls" }
diff --git a/services/mcp_service/src/context.rs b/services/mcp_service/src/context.rs
index 04e18f4d37..7a6c6a89ad 100644
--- a/services/mcp_service/src/context.rs
+++ b/services/mcp_service/src/context.rs
@@ -44,6 +44,7 @@ use soup::domain::service::SoupImpl;
use soup::outbound::pg_soup_repo::PgSoupRepo;
use sqlx::{PgPool, postgres::PgPoolOptions};
use sync_service_client::SyncServiceClient;
+use tokio_util::task::TaskTracker;
use crate::config::Config;
@@ -55,7 +56,21 @@ pub struct McpContext {
pub mcp_public_host: String,
}
-pub async fn build_context(config: &Config) -> anyhow::Result<McpContext> {
+struct ToolContextBuildArgs<'a> {
+ config: &'a Config,
+ db: &'a PgPool,
+ secretsmanager_client: &'a secretsmanager_client::SecretsManager,
+ sqs_client: sqs_client::SQS,
+ queue_aws_client: aws_sdk_sqs::Client,
+ document_storage_service_auth_key: String,
+ sync_service_auth_key: String,
+ event_task_tracker: TaskTracker,
+}
+
+pub async fn build_context(
+ config: &Config,
+ event_task_tracker: TaskTracker,
+) -> anyhow::Result<McpContext> {
let db = PgPoolOptions::new()
.min_connections(3)
.max_connections(10)
@@ -89,15 +104,19 @@ pub async fn build_context(config: &Config) -> anyhow::Result<McpContext> {
.await
.context("failed to load sync service auth key")?;
- let tool_context = build_tool_context(
+ let tool_context = build_tool_context(ToolContextBuildArgs {
config,
- &db,
- &secretsmanager_client,
+ db: &db,
+ secretsmanager_client: &secretsmanager_client,
sqs_client,
queue_aws_client,
- config.document_storage_service_auth_key.as_ref().to_owned(),
- sync_service_auth_key.as_ref().to_owned(),
- )
+ document_storage_service_auth_key: config
+ .document_storage_service_auth_key
+ .as_ref()
+ .to_owned(),
+ sync_service_auth_key: sync_service_auth_key.as_ref().to_owned(),
+ event_task_tracker,
+ })
.await?;
let auth_proxy = build_auth_proxy(config, &secretsmanager_client).await?;
@@ -116,15 +135,18 @@ pub async fn build_context(config: &Config) -> anyhow::Result<McpContext> {
})
}
-async fn build_tool_context(
- config: &Config,
- db: &PgPool,
- secretsmanager_client: &secretsmanager_client::SecretsManager,
- sqs_client: sqs_client::SQS,
- queue_aws_client: aws_sdk_sqs::Client,
- document_storage_service_auth_key: String,
- sync_service_auth_key: String,
-) -> anyhow::Result<ToolServiceContext> {
+async fn build_tool_context(args: ToolContextBuildArgs<'_>) -> anyhow::Result<ToolServiceContext> {
+ let ToolContextBuildArgs {
+ config,
+ db,
+ secretsmanager_client,
+ sqs_client,
+ queue_aws_client,
+ document_storage_service_auth_key,
+ sync_service_auth_key,
+ event_task_tracker,
+ } = args;
+
let dss_url = DocumentStorageServiceUrl::new()?.to_string();
let sync_service_url = SyncServiceUrl::new()?.to_string();
let lexical_service_url = LexicalServiceUrl::new()?.to_string();
@@ -224,6 +246,7 @@ async fn build_tool_context(
let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())
.context("failed to create kafka event publisher")?,
+ event_task_tracker,
);
let document_service = documents::domain::service::DocumentServiceImpl {
repo: document_repo,
diff --git a/services/mcp_service/src/main.rs b/services/mcp_service/src/main.rs
index 218b9560d4..e91c60d824 100644
--- a/services/mcp_service/src/main.rs
+++ b/services/mcp_service/src/main.rs
@@ -18,9 +18,11 @@ use rmcp::transport::streamable_http_server::{
};
use std::sync::Arc;
use tokio::time::Duration;
+use tokio_util::task::TaskTracker;
use tool_service::AuthenticatedToolService;
const AUTH_PROXY_CLEANUP_INTERVAL: Duration = Duration::from_secs(60);
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
#[tokio::main]
#[tracing::instrument(err)]
@@ -33,7 +35,8 @@ async fn main() -> anyhow::Result<()> {
// responses.
let item_base_url = config.app_base_url.as_ref().to_string();
- let context = build_context(&config).await?;
+ let event_broker_tracker = TaskTracker::new();
+ let context = build_context(&config, event_broker_tracker.clone()).await?;
// Create the MCP service with authenticated tool handler
let mcp_service = StreamableHttpService::new(
@@ -80,9 +83,23 @@ async fn main() -> anyhow::Result<()> {
tracing::info!("MCP server listening on http://{addr}/mcp");
- axum::serve(listener, app)
+ let server_result = axum::serve(listener, app)
+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal())
.await
- .context("MCP server error")?;
+ .context("MCP server error");
- Ok(())
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => {
+ tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ );
+ }
+ }
+
+ server_result
}
diff --git a/services/scheduled_action/Cargo.toml b/services/scheduled_action/Cargo.toml
index 36f0f6b302..eff35e6b72 100644
--- a/services/scheduled_action/Cargo.toml
+++ b/services/scheduled_action/Cargo.toml
@@ -38,7 +38,8 @@ chrono = { workspace = true}
chrono-tz = { version = "0.10.4", features = ["serde"] }
agent = { path = "../../crates/agent" }
ai_usage = { path = "../../crates/ai_usage" }
-tokio = { workspace = true }
+tokio = { workspace = true, features = ["signal"] }
+tokio-util = { workspace = true, features = ["rt"] }
futures = { workspace = true }
tracing = { workspace = true }
axum = { workspace = true }
diff --git a/services/scheduled_action/src/bins/service.rs b/services/scheduled_action/src/bins/service.rs
index cc039558c5..be9fbe9301 100644
--- a/services/scheduled_action/src/bins/service.rs
+++ b/services/scheduled_action/src/bins/service.rs
@@ -1,5 +1,5 @@
#![recursion_limit = "256"]
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
use ai_tools::build_tool_service_context_from_env;
use anyhow::{Context, Result};
@@ -22,13 +22,18 @@ use scheduled_action::inbound::axum_router::{
};
use scheduled_action::outbound::conn_gateway_live_updates::ConnGatewayLiveUpdates;
use scheduled_action::outbound::inprocess_executor::InProcessExecutor;
-use scheduled_action::outbound::pg_polling_dispatcher::PgPollingDispatcher;
+use scheduled_action::outbound::pg_polling_dispatcher::{
+ PgPollingDispatcher, PgPollingDispatcherLifecycle,
+};
use scheduled_action::outbound::pg_scheduled_action_repo::PgScheduledActionRepo;
use scheduled_action::swagger::ApiDoc;
use sqlx::postgres::PgPoolOptions;
+use tokio_util::{sync::CancellationToken, task::TaskTracker};
use utoipa::OpenApi;
use utoipa_swagger_ui::SwaggerUi;
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
+
#[tokio::main]
#[tracing::instrument(err)]
async fn main() -> Result<()> {
@@ -44,9 +49,11 @@ async fn main() -> Result<()> {
.await
.context("failed to connect to macrodb")?;
- let tool_context = build_tool_service_context_from_env(db.clone())
- .await
- .context("failed to build tool service context")?;
+ let event_broker_tracker = TaskTracker::new();
+ let tool_context =
+ build_tool_service_context_from_env(db.clone(), event_broker_tracker.clone())
+ .await
+ .context("failed to build tool service context")?;
let aws_config = macro_aws_config::get_macro_aws_config().await;
let notification_ingress = Arc::new(SqsNotificationIngress {
@@ -87,7 +94,14 @@ async fn main() -> Result<()> {
live_updates,
));
- let dispatcher = PgPollingDispatcher::new(Arc::clone(&repo), dispatcher_executor);
+ let dispatcher_cancellation_token = CancellationToken::new();
+ let dispatcher_tracker = TaskTracker::new();
+ let dispatcher_lifecycle = PgPollingDispatcherLifecycle::new(
+ dispatcher_cancellation_token.clone(),
+ dispatcher_tracker.clone(),
+ );
+ let dispatcher = PgPollingDispatcher::new(Arc::clone(&repo), dispatcher_executor)
+ .with_lifecycle(dispatcher_lifecycle);
let (dispatcher_tx, _execution_rx) = dispatcher.begin_dispatch_loop();
let service = Arc::new(ScheduledActionServiceImpl::new(
@@ -130,8 +144,29 @@ async fn main() -> Result<()> {
tracing::info!("scheduled_action service listening on {addr}");
- axum::serve(listener, router.into_make_service())
+ let server_result = axum::serve(listener, router.into_make_service())
+ .with_graceful_shutdown(macro_entrypoint::shutdown_signal())
.await
- .context("server closed")?;
- unreachable!();
+ .context("server closed");
+
+ tracing::info!("stopping scheduled action dispatcher");
+ dispatcher_cancellation_token.cancel();
+ dispatcher_tracker.close();
+ dispatcher_tracker.wait().await;
+ tracing::info!("scheduled action dispatcher stopped");
+
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => {
+ tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ );
+ }
+ }
+
+ server_result
}
diff --git a/services/scheduled_action/src/outbound/pg_polling_dispatcher.rs b/services/scheduled_action/src/outbound/pg_polling_dispatcher.rs
index 86b549447d..b097803255 100644
--- a/services/scheduled_action/src/outbound/pg_polling_dispatcher.rs
+++ b/services/scheduled_action/src/outbound/pg_polling_dispatcher.rs
@@ -3,6 +3,11 @@ use std::time::{Duration, Instant};
use chrono::Utc;
use tokio::sync::mpsc::{Receiver, Sender};
+use tokio_util::sync::CancellationToken;
+use tokio_util::task::TaskTracker;
+
+#[cfg(test)]
+mod test;
use crate::domain::models::{DispatchEvent, InProgressExecution};
use crate::domain::ports::{
@@ -36,6 +41,33 @@ const BATCH_MIN_DURATION: Duration = Duration::from_secs(30);
pub struct PgPollingDispatcher<Rpo: ScheduledActionRepo, Exe: ScheduledActionExecutor> {
repo: Arc<Rpo>,
executor: Exe,
+ lifecycle: PgPollingDispatcherLifecycle,
+}
+
+/// Cooperative shutdown configuration for [`PgPollingDispatcher`].
+///
+/// Retain clones of the token and tracker to cancel the dispatcher and wait
+/// for both of its background tasks to finish.
+#[derive(Clone)]
+pub struct PgPollingDispatcherLifecycle {
+ cancellation_token: CancellationToken,
+ task_tracker: TaskTracker,
+}
+
+impl PgPollingDispatcherLifecycle {
+ /// Creates lifecycle configuration from caller-owned shutdown handles.
+ pub fn new(cancellation_token: CancellationToken, task_tracker: TaskTracker) -> Self {
+ Self {
+ cancellation_token,
+ task_tracker,
+ }
+ }
+}
+
+impl Default for PgPollingDispatcherLifecycle {
+ fn default() -> Self {
+ Self::new(CancellationToken::new(), TaskTracker::new())
+ }
}
impl<Rpo, Exe> PgPollingDispatcher<Rpo, Exe>
@@ -44,7 +76,19 @@ where
Exe: ScheduledActionExecutor,
{
pub fn new(repo: Arc<Rpo>, executor: Exe) -> Self {
- Self { repo, executor }
+ Self {
+ repo,
+ executor,
+ lifecycle: PgPollingDispatcherLifecycle::default(),
+ }
+ }
+
+ /// Configures handles for cooperatively stopping and joining dispatcher
+ /// tasks. Without this opt-in, the dispatcher retains its previous
+ /// process-lifetime behavior.
+ pub fn with_lifecycle(mut self, lifecycle: PgPollingDispatcherLifecycle) -> Self {
+ self.lifecycle = lifecycle;
+ self
}
}
@@ -57,17 +101,46 @@ where
let (tx, mut rx) = tokio::sync::mpsc::channel::<DispatchEvent>(BUFFER_SIZE);
let (extx, exrx) = tokio::sync::mpsc::channel::<InProgressExecution>(BUFFER_SIZE);
+ let PgPollingDispatcher {
+ repo,
+ executor,
+ lifecycle,
+ } = self;
+ let cancellation_token = lifecycle.cancellation_token;
+
// Drain dispatch events — the polling loop pulls fresh state from the
// DB each tick, so create/update/delete notifications are not needed.
// We still must drain so service-side sends don't block on a full
// channel.
- tokio::spawn(async move { while rx.recv().await.is_some() {} });
+ let drain_cancellation_token = cancellation_token.clone();
+ lifecycle.task_tracker.spawn(async move {
+ loop {
+ tokio::select! {
+ biased;
+ _ = drain_cancellation_token.cancelled() => break,
+ event = rx.recv() => {
+ if event.is_none() {
+ break;
+ }
+ }
+ }
+ }
+ });
- tokio::spawn(async move {
+ lifecycle.task_tracker.spawn(async move {
loop {
+ if cancellation_token.is_cancelled() {
+ return;
+ }
+
let batch_start = Instant::now();
+ let poll_result = tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => return,
+ result = repo.get_next_unclaimed_actions(BATCH_SIZE) => result,
+ };
- match self.repo.get_next_unclaimed_actions(BATCH_SIZE).await {
+ match poll_result {
Ok(candidates) => {
let now = Utc::now();
for action in candidates {
@@ -83,7 +156,10 @@ where
// running. If a peer instance claimed it between
// our pull and this call, the claim fails and we
// skip — that's the multi-instance contract.
- match self.executor.execute_action(action).await {
+ if cancellation_token.is_cancelled() {
+ return;
+ }
+ match executor.execute_action(action).await {
Ok(execution) => {
let _ = extx.send(execution).await;
}
@@ -107,7 +183,11 @@ where
let elapsed = batch_start.elapsed();
if elapsed < BATCH_MIN_DURATION {
- tokio::time::sleep(BATCH_MIN_DURATION - elapsed).await;
+ tokio::select! {
+ biased;
+ _ = cancellation_token.cancelled() => return,
+ _ = tokio::time::sleep(BATCH_MIN_DURATION - elapsed) => {}
+ }
}
}
});
diff --git a/services/scheduled_action/src/outbound/pg_polling_dispatcher/test.rs b/services/scheduled_action/src/outbound/pg_polling_dispatcher/test.rs
new file mode 100644
index 0000000000..2e3729244d
--- /dev/null
+++ b/services/scheduled_action/src/outbound/pg_polling_dispatcher/test.rs
@@ -0,0 +1,260 @@
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::time::Duration;
+
+use anyhow::Result;
+use chrono::{DateTime, Duration as ChronoDuration, Utc};
+use macro_user_id::user_id::MacroUserIdStr;
+use macro_uuid::{Uuid, generate_uuid_v7};
+use serde_json::json;
+use tokio::sync::Semaphore;
+use tokio_util::sync::CancellationToken;
+use tokio_util::task::TaskTracker;
+
+use super::{PgPollingDispatcher, PgPollingDispatcherLifecycle};
+use crate::domain::models::{
+ ActionExecutionRecord, ActionKind, InProgressExecution, Schedule, ScheduledAction,
+};
+use crate::domain::ports::{
+ ScheduledActionDispatcher, ScheduledActionExecutor, ScheduledActionRepo,
+};
+
+struct FakeRepository {
+ candidates: Vec<ScheduledAction>,
+ cancellation_token: Option<CancellationToken>,
+ poll_started: Semaphore,
+}
+
+impl FakeRepository {
+ fn returning(candidates: Vec<ScheduledAction>) -> Self {
+ Self {
+ candidates,
+ cancellation_token: None,
+ poll_started: Semaphore::new(0),
+ }
+ }
+
+ fn cancelling_before_return(
+ candidates: Vec<ScheduledAction>,
+ cancellation_token: CancellationToken,
+ ) -> Self {
+ Self {
+ candidates,
+ cancellation_token: Some(cancellation_token),
+ poll_started: Semaphore::new(0),
+ }
+ }
+
+ async fn wait_for_poll(&self) {
+ self.poll_started
+ .acquire()
+ .await
+ .expect("poll semaphore should remain open")
+ .forget();
+ }
+}
+
+impl ScheduledActionRepo for FakeRepository {
+ async fn create_action(&self, action: ScheduledAction) -> Result<ScheduledAction> {
+ Ok(action)
+ }
+
+ async fn get_actions(&self, _user_id: MacroUserIdStr<'static>) -> Result<Vec<ScheduledAction>> {
+ Ok(Vec::new())
+ }
+
+ async fn get_next_unclaimed_actions(&self, _limit: i64) -> Result<Vec<ScheduledAction>> {
+ self.poll_started.add_permits(1);
+ if let Some(cancellation_token) = &self.cancellation_token {
+ cancellation_token.cancel();
+ }
+ Ok(self.candidates.clone())
+ }
+
+ async fn update_action(&self, action: ScheduledAction) -> Result<ScheduledAction> {
+ Ok(action)
+ }
+
+ async fn delete_action(
+ &self,
+ _id: &Uuid,
+ _macro_user_id: MacroUserIdStr<'static>,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ async fn claim_action(&self, _id: &Uuid) -> Result<()> {
+ Ok(())
+ }
+
+ async fn release_action(&self, _id: &Uuid) -> Result<()> {
+ Ok(())
+ }
+
+ async fn create_execution_record(&self, _record: ActionExecutionRecord) -> Result<()> {
+ Ok(())
+ }
+
+ async fn get_execution_records(&self, _action_id: &Uuid) -> Result<Vec<ActionExecutionRecord>> {
+ Ok(Vec::new())
+ }
+
+ async fn update_next_run_at(&self, _id: &Uuid) -> Result<()> {
+ Ok(())
+ }
+
+ async fn update_last_executed(&self, _id: &Uuid, _executed_at: DateTime<Utc>) -> Result<()> {
+ Ok(())
+ }
+}
+
+struct RecordingExecutor {
+ execution_count: Arc<AtomicUsize>,
+}
+
+impl ScheduledActionExecutor for RecordingExecutor {
+ async fn execute_action(&self, action: ScheduledAction) -> Result<InProgressExecution> {
+ self.execution_count.fetch_add(1, Ordering::SeqCst);
+ Ok(InProgressExecution {
+ action_id: action.id.expect("test action should have an id"),
+ chat_id: None,
+ })
+ }
+}
+
+struct GatedExecutor {
+ execution_count: Arc<AtomicUsize>,
+ first_execution_started: Arc<Semaphore>,
+ release_first_execution: Arc<Semaphore>,
+ first_execution_finished: Arc<AtomicBool>,
+}
+
+impl ScheduledActionExecutor for GatedExecutor {
+ async fn execute_action(&self, action: ScheduledAction) -> Result<InProgressExecution> {
+ let execution_index = self.execution_count.fetch_add(1, Ordering::SeqCst);
+ if execution_index == 0 {
+ self.first_execution_started.add_permits(1);
+ self.release_first_execution
+ .acquire()
+ .await
+ .expect("execution semaphore should remain open")
+ .forget();
+ self.first_execution_finished.store(true, Ordering::SeqCst);
+ }
+
+ Ok(InProgressExecution {
+ action_id: action.id.expect("test action should have an id"),
+ chat_id: None,
+ })
+ }
+}
+
+fn due_action() -> ScheduledAction {
+ let now = Utc::now();
+ ScheduledAction {
+ id: Some(generate_uuid_v7()),
+ owner: MacroUserIdStr::parse_from_str("macro|[email protected]")
+ .expect("test owner should be valid"),
+ name: "test action".to_string(),
+ schedule: Schedule::from_cron("0 * * * * *".to_string())
+ .expect("test schedule should be valid"),
+ kind: ActionKind::Agent,
+ created_at: now,
+ updated_at: now,
+ timezone: chrono_tz::UTC,
+ task: json!({}),
+ claimed: None,
+ next_run_at: now - ChronoDuration::seconds(1),
+ enabled: true,
+ }
+}
+
+fn lifecycle() -> (CancellationToken, TaskTracker, PgPollingDispatcherLifecycle) {
+ let cancellation_token = CancellationToken::new();
+ let tracker = TaskTracker::new();
+ let lifecycle = PgPollingDispatcherLifecycle::new(cancellation_token.clone(), tracker.clone());
+ (cancellation_token, tracker, lifecycle)
+}
+
+async fn wait_for_tracker(tracker: &TaskTracker) {
+ tokio::time::timeout(Duration::from_secs(1), tracker.wait())
+ .await
+ .expect("dispatcher tasks should stop after cancellation");
+}
+
+#[tokio::test]
+async fn cancellation_interrupts_polling_delay_and_dispatch_event_drain() {
+ let repository = Arc::new(FakeRepository::returning(Vec::new()));
+ let executor = RecordingExecutor {
+ execution_count: Arc::new(AtomicUsize::new(0)),
+ };
+ let (cancellation_token, tracker, lifecycle) = lifecycle();
+ let dispatcher =
+ PgPollingDispatcher::new(Arc::clone(&repository), executor).with_lifecycle(lifecycle);
+
+ let (_dispatch_tx, _execution_rx) = dispatcher.begin_dispatch_loop();
+ repository.wait_for_poll().await;
+
+ cancellation_token.cancel();
+ tracker.close();
+ wait_for_tracker(&tracker).await;
+}
+
+#[tokio::test]
+async fn cancellation_during_repository_poll_prevents_candidate_execution() {
+ let (cancellation_token, tracker, lifecycle) = lifecycle();
+ let repository = Arc::new(FakeRepository::cancelling_before_return(
+ vec![due_action()],
+ cancellation_token.clone(),
+ ));
+ let execution_count = Arc::new(AtomicUsize::new(0));
+ let executor = RecordingExecutor {
+ execution_count: Arc::clone(&execution_count),
+ };
+ let dispatcher = PgPollingDispatcher::new(repository, executor).with_lifecycle(lifecycle);
+
+ let (_dispatch_tx, _execution_rx) = dispatcher.begin_dispatch_loop();
+ tracker.close();
+ wait_for_tracker(&tracker).await;
+
+ assert_eq!(execution_count.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test]
+async fn cancellation_allows_started_execution_to_finish_without_starting_another() {
+ let repository = Arc::new(FakeRepository::returning(vec![due_action(), due_action()]));
+ let execution_count = Arc::new(AtomicUsize::new(0));
+ let first_execution_started = Arc::new(Semaphore::new(0));
+ let release_first_execution = Arc::new(Semaphore::new(0));
+ let first_execution_finished = Arc::new(AtomicBool::new(false));
+ let executor = GatedExecutor {
+ execution_count: Arc::clone(&execution_count),
+ first_execution_started: Arc::clone(&first_execution_started),
+ release_first_execution: Arc::clone(&release_first_execution),
+ first_execution_finished: Arc::clone(&first_execution_finished),
+ };
+ let (cancellation_token, tracker, lifecycle) = lifecycle();
+ let dispatcher = PgPollingDispatcher::new(repository, executor).with_lifecycle(lifecycle);
+
+ let (_dispatch_tx, _execution_rx) = dispatcher.begin_dispatch_loop();
+ first_execution_started
+ .acquire()
+ .await
+ .expect("execution semaphore should remain open")
+ .forget();
+
+ cancellation_token.cancel();
+ tracker.close();
+ let wait = tracker.wait();
+ tokio::pin!(wait);
+ tokio::select! {
+ () = &mut wait => panic!("tracker should wait for the in-flight execution"),
+ () = tokio::task::yield_now() => {}
+ }
+
+ release_first_execution.add_permits(1);
+ wait.await;
+
+ assert!(first_execution_finished.load(Ordering::SeqCst));
+ assert_eq!(execution_count.load(Ordering::SeqCst), 1);
+}