Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(notifications): close retention observability gaps (#8116)

Sean Aye91e02eb8f904
diff --git a/Cargo.lock b/Cargo.lock
index e45197cedd..ee6ee7fa26 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11709,6 +11709,7 @@ dependencies = [
"rootcause",
"rootcause-tracing",
"serde_json",
+ "thiserror 2.0.18",
"tokio",
"tracing",
"tracing-opentelemetry",
@@ -13195,6 +13196,7 @@ dependencies = [
"thiserror 2.0.18",
"tokio",
"tokio-retry",
+ "tokio-util",
"tower 0.5.3",
"tracing",
"url",
diff --git a/crates/kafka_util/AGENTS.md b/crates/kafka_util/AGENTS.md
new file mode 100644
index 0000000000..80e9a73a0c
--- /dev/null
+++ b/crates/kafka_util/AGENTS.md
@@ -0,0 +1,32 @@
+# kafka_util
+
+## What it is
+
+Shared environment-aware Kafka transports: plaintext locally, MSK IAM in hosted
+environments, with trace propagation and typed grouped/ungrouped consumers.
+
+## Start here
+
+- `src/lib.rs`: group markers, assignment, publish/receive and configuration.
+- `src/msk_iam.rs`: OAuth token refresh; statistics observers must preserve it.
+- `src/statistics.rs`: opt-in callbacks for ungrouped consumers, without changing
+ assignment or commit behavior. Observers must not block or log raw statistics.
+
+## Scope
+
+- Owns transport mechanics, not domain event decoding or retry policy.
+- `Ungrouped` always uses manual assignment with automatic commits/stores off.
+- Used by `macro_event_broker` and other domain Kafka adapters.
+
+## Checks
+
+- `cargo test -p kafka_util` (`SQLX_OFFLINE` unset).
+- The in-process librdkafka mock broker test verifies statistics and uncommitted
+ application offsets; no external Kafka/Docker setup is needed.
+
+## Gotchas
+
+- Statistics callbacks run while polling, not independently of consumption.
+ Health reporters must detect stale samples rather than repeating old zero lag.
+- Consumer-group lag cannot describe an ungrouped consumer; use application
+ positions and explicitly handle unknown offsets and prefetched records.
diff --git a/crates/kafka_util/CLAUDE.md b/crates/kafka_util/CLAUDE.md
new file mode 120000
index 0000000000..47dc3e3d86
--- /dev/null
+++ b/crates/kafka_util/CLAUDE.md
@@ -0,0 +1 @@
+AGENTS.md
\ No newline at end of file
diff --git a/crates/kafka_util/src/lib.rs b/crates/kafka_util/src/lib.rs
index 3251e1d02b..e88bbfb33e 100644
--- a/crates/kafka_util/src/lib.rs
+++ b/crates/kafka_util/src/lib.rs
@@ -15,7 +15,9 @@ use std::time::Duration;
use either::Either;
use macro_env::Environment;
use opentelemetry::propagation::{Extractor, Injector};
-use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer};
+use rdkafka::consumer::{
+ CommitMode, Consumer, ConsumerContext, DefaultConsumerContext, StreamConsumer,
+};
use rdkafka::error::{KafkaError, KafkaResult};
use rdkafka::message::{BorrowedMessage, Header, Headers, Message, OwnedHeaders};
use rdkafka::producer::{FutureProducer, FutureRecord};
@@ -24,8 +26,11 @@ use tracing_opentelemetry::OpenTelemetrySpanExt as _;
use uuid::Uuid;
pub use msk_iam::{MskIamClientContext, configure_sasl_iam};
+pub use statistics::ConsumerStatistics;
+use statistics::StatisticsContext;
mod msk_iam;
+mod statistics;
const UNGROUPED_GROUP_PREFIX: &str = "macro-event-broker-independent";
const MESSAGE_TIMEOUT_MS: &str = "5000";
@@ -145,7 +150,12 @@ pub enum KafkaProducerError {
}
/// Underlying Kafka consumer transport selected from the runtime environment.
-struct ConsumerTransport(Either<StreamConsumer, StreamConsumer<MskIamClientContext>>);
+struct ConsumerTransport(
+ Either<
+ StreamConsumer<StatisticsContext<DefaultConsumerContext>>,
+ StreamConsumer<StatisticsContext<MskIamClientContext>>,
+ >,
+);
/// Underlying Kafka producer transport selected from the runtime environment.
#[derive(Clone)]
@@ -243,14 +253,25 @@ fn ungrouped_config(brokers: &str) -> ClientConfig {
fn create_consumer_from_env<T>(
config: ClientConfig,
+ observer: Option<ConsumerStatistics>,
) -> Result<KafkaEventConsumer<T>, KafkaConsumerError> {
let consumer = match Environment::new_or_prod() {
- Environment::Local => Either::Left(config.create().map_err(KafkaConsumerError::Plaintext)?),
+ Environment::Local => Either::Left(
+ config
+ .create_with_context(StatisticsContext {
+ inner: DefaultConsumerContext,
+ observer,
+ })
+ .map_err(KafkaConsumerError::Plaintext)?,
+ ),
Environment::Develop | Environment::Production => {
let config = configure_sasl_iam(config);
Either::Right(
config
- .create_with_context(MskIamClientContext::from_env())
+ .create_with_context(StatisticsContext {
+ inner: MskIamClientContext::from_env(),
+ observer,
+ })
.map_err(KafkaConsumerError::MskIam)?,
)
}
@@ -398,7 +419,7 @@ impl<T: GroupName> KafkaEventConsumer<T> {
/// Creates a named-group consumer, selecting plaintext or MSK IAM transport
/// from the runtime environment.
pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {
- create_consumer_from_env(grouped_config::<T>(brokers))
+ create_consumer_from_env(grouped_config::<T>(brokers), None)
}
/// Subscribes the consumer to exactly the provided topics.
@@ -422,7 +443,18 @@ impl KafkaEventConsumer<Ungrouped> {
///
/// Call [`Self::assign_topics`] before receiving messages.
pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {
- create_consumer_from_env(ungrouped_config(brokers))
+ create_consumer_from_env(ungrouped_config(brokers), None)
+ }
+
+ /// Creates an independent consumer with statistics sampled every ten seconds.
+ /// The observer does not join a group, store offsets, or issue broker calls.
+ pub fn from_env_with_statistics(
+ brokers: &str,
+ observer: ConsumerStatistics,
+ ) -> Result<Self, KafkaConsumerError> {
+ let mut config = ungrouped_config(brokers);
+ config.set("statistics.interval.ms", "10000");
+ create_consumer_from_env(config, Some(observer))
}
/// Manually assigns every current partition of `topics` at `initial_offset`.
diff --git a/crates/kafka_util/src/statistics.rs b/crates/kafka_util/src/statistics.rs
new file mode 100644
index 0000000000..c8ed7e3001
--- /dev/null
+++ b/crates/kafka_util/src/statistics.rs
@@ -0,0 +1,51 @@
+//! Opt-in statistics delivery without changing assignment or commit behavior.
+
+use std::sync::Arc;
+
+use rdkafka::{
+ client::{ClientContext, OAuthToken},
+ config::RDKafkaLogLevel,
+ consumer::ConsumerContext,
+ error::KafkaError,
+ statistics::Statistics,
+};
+
+/// A nonblocking observer of librdkafka statistics. Called while polling the
+/// consumer; observers must not perform I/O or log entire statistics payloads.
+pub type ConsumerStatistics = Arc<dyn Fn(&Statistics) + Send + Sync>;
+
+pub(crate) struct StatisticsContext<C> {
+ pub(crate) inner: C,
+ pub(crate) observer: Option<ConsumerStatistics>,
+}
+
+impl<C: ClientContext> ClientContext for StatisticsContext<C> {
+ const ENABLE_REFRESH_OAUTH_TOKEN: bool = C::ENABLE_REFRESH_OAUTH_TOKEN;
+
+ fn log(&self, level: RDKafkaLogLevel, facility: &str, message: &str) {
+ self.inner.log(level, facility, message);
+ }
+
+ fn error(&self, error: KafkaError, reason: &str) {
+ self.inner.error(error, reason);
+ }
+
+ fn stats(&self, statistics: Statistics) {
+ if let Some(observer) = &self.observer {
+ observer(&statistics);
+ }
+ }
+
+ fn generate_oauth_token(
+ &self,
+ config: Option<&str>,
+ ) -> Result<OAuthToken, Box<dyn std::error::Error>> {
+ self.inner.generate_oauth_token(config)
+ }
+}
+
+// Both supported inner contexts use the default rebalance/commit callbacks.
+impl<C: ConsumerContext> ConsumerContext for StatisticsContext<C> {}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/kafka_util/src/statistics/test.rs b/crates/kafka_util/src/statistics/test.rs
new file mode 100644
index 0000000000..9338248d2a
--- /dev/null
+++ b/crates/kafka_util/src/statistics/test.rs
@@ -0,0 +1,147 @@
+use super::*;
+use std::sync::{
+ Mutex,
+ atomic::{AtomicUsize, Ordering},
+};
+
+use rdkafka::{
+ ClientConfig,
+ consumer::{Consumer, DefaultConsumerContext, StreamConsumer},
+ mocking::MockCluster,
+ producer::{FutureProducer, FutureRecord},
+ statistics::Statistics,
+};
+
+#[test]
+fn observer_receives_statistics_once_without_default_payload_logging() {
+ let calls = Arc::new(AtomicUsize::new(0));
+ let observed = calls.clone();
+ let context = StatisticsContext {
+ inner: DefaultConsumerContext,
+ observer: Some(Arc::new(move |stats| {
+ assert_eq!(stats.rxmsgs, 12);
+ observed.fetch_add(1, Ordering::SeqCst);
+ })),
+ };
+ context.stats(Statistics {
+ rxmsgs: 12,
+ ..Default::default()
+ });
+ assert_eq!(calls.load(Ordering::SeqCst), 1);
+}
+
+struct AuthContext;
+impl ClientContext for AuthContext {
+ const ENABLE_REFRESH_OAUTH_TOKEN: bool = true;
+ fn generate_oauth_token(
+ &self,
+ config: Option<&str>,
+ ) -> Result<OAuthToken, Box<dyn std::error::Error>> {
+ assert_eq!(config, Some("test-config"));
+ Ok(OAuthToken {
+ token: "test-only".into(),
+ principal_name: "test-principal".into(),
+ lifetime_ms: 123,
+ })
+ }
+}
+
+#[test]
+fn wrapping_statistics_preserves_iam_token_refresh() {
+ assert!(StatisticsContext::<AuthContext>::ENABLE_REFRESH_OAUTH_TOKEN);
+ assert!(!StatisticsContext::<DefaultConsumerContext>::ENABLE_REFRESH_OAUTH_TOKEN);
+ let context = StatisticsContext {
+ inner: AuthContext,
+ observer: None,
+ };
+ let token = context
+ .generate_oauth_token(Some("test-config"))
+ .expect("forwarded auth callback");
+ assert_eq!(token.principal_name, "test-principal");
+ assert_eq!(token.lifetime_ms, 123);
+}
+
+#[tokio::test]
+async fn ungrouped_consumer_reports_application_offsets_without_commits() {
+ let cluster = MockCluster::new(1).expect("mock broker");
+ cluster.create_topic("stats-test", 1, 1).expect("topic");
+ let (send, receive) = tokio::sync::oneshot::channel();
+ let sender = Mutex::new(Some(send));
+ let observer: ConsumerStatistics = Arc::new(move |stats| {
+ let Some(partition) = stats
+ .topics
+ .get("stats-test")
+ .and_then(|t| t.partitions.get(&0))
+ else {
+ return;
+ };
+ if partition.app_offset == 1 && partition.hi_offset >= 1 {
+ if let Some(sender) = sender.lock().expect("sender mutex").take() {
+ let broker_connected = stats
+ .brokers
+ .values()
+ .any(|broker| broker.nodeid == partition.broker && broker.state == "UP");
+ sender
+ .send((partition.clone(), broker_connected))
+ .expect("live receiver");
+ }
+ }
+ });
+ let mut config = crate::ungrouped_config(&cluster.bootstrap_servers());
+ config.set("statistics.interval.ms", "10");
+ assert_eq!(config.get("enable.auto.commit"), Some("false"));
+ assert_eq!(config.get("enable.auto.offset.store"), Some("false"));
+ let consumer: StreamConsumer<_> = config
+ .create_with_context(StatisticsContext {
+ inner: DefaultConsumerContext,
+ observer: Some(observer),
+ })
+ .expect("consumer");
+ let mut assignment = rdkafka::TopicPartitionList::new();
+ assignment
+ .add_partition_offset("stats-test", 0, rdkafka::Offset::Beginning)
+ .expect("assignment");
+ consumer.assign(&assignment).expect("manual assignment");
+ let producer: FutureProducer = ClientConfig::new()
+ .set("bootstrap.servers", cluster.bootstrap_servers())
+ .create()
+ .expect("producer");
+ producer
+ .send(
+ FutureRecord::to("stats-test").key("test").payload("test"),
+ std::time::Duration::from_secs(5),
+ )
+ .await
+ .expect("publish");
+ drop(
+ tokio::time::timeout(std::time::Duration::from_secs(5), consumer.recv())
+ .await
+ .expect("receive deadline")
+ .expect("message"),
+ );
+
+ // Polling services callbacks even while no records are being published.
+ let (sample, broker_connected) =
+ tokio::time::timeout(std::time::Duration::from_secs(5), async {
+ tokio::select! {
+ sample = receive => sample.expect("statistics callback"),
+ _ = consumer.recv() => panic!("no second record was published"),
+ }
+ })
+ .await
+ .expect("statistics deadline");
+ assert_eq!(sample.app_offset, 1);
+ assert_eq!(sample.hi_offset, 1);
+ assert!(
+ sample.desired,
+ "manual assignments must appear in health snapshots"
+ );
+ assert!(
+ broker_connected,
+ "the snapshot must identify its active broker"
+ );
+ assert!(
+ sample.committed_offset < 0,
+ "no durable offset was committed"
+ );
+}
diff --git a/crates/macro_entrypoint/AGENTS.md b/crates/macro_entrypoint/AGENTS.md
index 691c118d0a..a8a873bfb0 100644
--- a/crates/macro_entrypoint/AGENTS.md
+++ b/crates/macro_entrypoint/AGENTS.md
@@ -11,7 +11,9 @@ nearly every Rust service's `main.rs` calls.
- `src/lib.rs`: `MacroEntrypoint::init` — builds the tracing subscriber stack
per `Environment` (local / develop / production), returning
`InitializedEntrypoint` as proof of initialization; call `.shutdown()`
- before exit to flush OTel.
+ before exit to flush OTel. Serverless handlers retain it and await
+ `.force_flush()` after invocation spans close, without shutting down warm
+ providers. The caller must bound flush time and preserve its operation result.
- `src/datadog_fmt.rs`: JSON log format injecting `dd.trace_id`/`dd.span_id`
for prod/develop trace-log correlation.
- `src/grafana.rs`: a second, best-effort span exporter that mirrors spans to
diff --git a/crates/macro_entrypoint/Cargo.toml b/crates/macro_entrypoint/Cargo.toml
index 5cb6a46b88..fea7f6ea0c 100644
--- a/crates/macro_entrypoint/Cargo.toml
+++ b/crates/macro_entrypoint/Cargo.toml
@@ -14,6 +14,7 @@ opentelemetry-otlp = { workspace = true }
rootcause = { workspace = true }
rootcause-tracing = { workspace = true }
serde_json = { workspace = true }
+thiserror = { workspace = true }
tokio = { workspace = true, features = ["signal"] }
tracing = { workspace = true }
tracing-opentelemetry = { workspace = true }
diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rs
index b83ba3ae13..7e24273495 100644
--- a/crates/macro_entrypoint/src/lib.rs
+++ b/crates/macro_entrypoint/src/lib.rs
@@ -122,7 +122,41 @@ pub struct InitializedEntrypoint {
logger_provider: Option<SdkLoggerProvider>,
}
+/// Failure to flush telemetry before a serverless runtime freezes.
+#[derive(Debug, thiserror::Error)]
+pub enum TelemetryFlushError {
+ /// The blocking flush task could not finish.
+ #[error("telemetry flush task failed")]
+ Task(#[from] tokio::task::JoinError),
+ /// The trace exporter could not flush.
+ #[error("trace flush failed")]
+ Traces(#[source] opentelemetry_sdk::error::OTelSdkError),
+ /// The log exporter could not flush.
+ #[error("log flush failed")]
+ Logs(#[source] opentelemetry_sdk::error::OTelSdkError),
+}
+
impl InitializedEntrypoint {
+ /// Flush completed telemetry before a serverless runtime freezes, without
+ /// shutting down providers needed by subsequent warm invocations.
+ ///
+ /// Run the blocking SDK flush off the async executor so gRPC can progress
+ /// even on a single-threaded runtime. Call after invocation spans close.
+ pub async fn force_flush(&self) -> Result<(), TelemetryFlushError> {
+ let tracer = self.tracer_provider.clone();
+ let logger = self.logger_provider.clone();
+ let (traces, logs) = tokio::task::spawn_blocking(move || {
+ (
+ tracer.map(|provider| provider.force_flush()),
+ logger.map(|provider| provider.force_flush()),
+ )
+ })
+ .await?;
+ traces.transpose().map_err(TelemetryFlushError::Traces)?;
+ logs.transpose().map_err(TelemetryFlushError::Logs)?;
+ Ok(())
+ }
+
/// Gracefully shut down the OpenTelemetry providers.
/// This should be called before the application exits to ensure all traces and logs are flushed.
pub fn shutdown(&self) {
diff --git a/crates/macro_entrypoint/src/test.rs b/crates/macro_entrypoint/src/test.rs
index 5598848d41..d8cdc75949 100644
--- a/crates/macro_entrypoint/src/test.rs
+++ b/crates/macro_entrypoint/src/test.rs
@@ -44,6 +44,89 @@ fn test_tracer_provider() -> (InMemorySpanExporter, SdkTracerProvider) {
(exporter, provider)
}
+#[tokio::test]
+async fn force_flush_exports_short_invocations_and_preserves_warm_provider() {
+ use opentelemetry::trace::Tracer as _;
+
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_batch_exporter(exporter.clone())
+ .build();
+ let tracer = provider.tracer("retention-test");
+ let entrypoint = InitializedEntrypoint {
+ tracer_provider: Some(provider),
+ logger_provider: None,
+ };
+
+ tracer.in_span("first-invocation", |_| {});
+ entrypoint
+ .force_flush()
+ .await
+ .expect("first flush succeeds");
+ let spans = exporter.get_finished_spans().expect("exported spans");
+ assert_eq!(spans.len(), 1);
+ assert_eq!(spans[0].name, "first-invocation");
+
+ tracer.in_span("warm-invocation", |_| {});
+ entrypoint.force_flush().await.expect("warm flush succeeds");
+ let spans = exporter.get_finished_spans().expect("exported spans");
+ assert_eq!(spans.len(), 2);
+ assert_eq!(spans[1].name, "warm-invocation");
+ entrypoint.shutdown();
+}
+
+#[tokio::test]
+async fn force_flush_reports_export_failure_instead_of_success() {
+ use opentelemetry::trace::Tracer as _;
+ use opentelemetry_sdk::{
+ error::{OTelSdkError, OTelSdkResult},
+ trace::{BatchConfigBuilder, BatchSpanProcessor, SpanData, SpanExporter},
+ };
+
+ #[derive(Debug)]
+ struct FailingExporter;
+ impl SpanExporter for FailingExporter {
+ async fn export(&self, _batch: Vec<SpanData>) -> OTelSdkResult {
+ Err(OTelSdkError::InternalFailure(
+ "test export failure".to_owned(),
+ ))
+ }
+ }
+ let processor = BatchSpanProcessor::builder(FailingExporter)
+ .with_batch_config(
+ BatchConfigBuilder::default()
+ .with_scheduled_delay(std::time::Duration::from_secs(3600))
+ .build(),
+ )
+ .build();
+ let provider = SdkTracerProvider::builder()
+ .with_span_processor(processor)
+ .build();
+ provider
+ .tracer("test")
+ .in_span("completed-invocation", |_| {});
+ let entrypoint = InitializedEntrypoint {
+ tracer_provider: Some(provider),
+ logger_provider: None,
+ };
+ assert!(matches!(
+ entrypoint.force_flush().await,
+ Err(TelemetryFlushError::Traces(_))
+ ));
+ entrypoint.shutdown();
+}
+
+#[tokio::test]
+async fn force_flush_without_exporters_is_a_noop() {
+ InitializedEntrypoint {
+ tracer_provider: None,
+ logger_provider: None,
+ }
+ .force_flush()
+ .await
+ .expect("no exporter needs flushing");
+}
+
#[test]
fn rootcause_tracing_subscriber_captures_span_chain() {
static INSTALL_ROOTCAUSE_HOOKS: Once = Once::new();
diff --git a/crates/notification/AGENTS.md b/crates/notification/AGENTS.md
index d0463d574a..c75803149d 100644
--- a/crates/notification/AGENTS.md
+++ b/crates/notification/AGENTS.md
@@ -20,22 +20,25 @@ through the same service.
and every port an adapter must implement.
- `src/inbound/http.rs`, `src/inbound/worker.rs`,
`src/inbound/notification_events_listener.rs`: HTTP routes, the queue
- worker, and the Kafka event listener that trigger notification creation.
+ worker, and the PostgreSQL LISTEN adapter that forwards delete events.
- `src/outbound/`: one file per channel/concern — `fanout_realtime.rs` +
`kafka_realtime.rs` (live WebSocket fan-out), `mobile.rs` + `sns_endpoint.rs`
(push), `email.rs` + `digest_batcher.rs` (digest emails), `repository.rs`
- (Postgres via `notification_db_client`), `item_preferences.rs` (snooze/mute).
+ (owned Postgres queries), `item_preferences.rs` (snooze/mute).
+- `outbound/notification_consumer/health.rs`: periodic per-instance Kafka
+ backlog and telemetry-health logs; offsets stay uncommitted.
+- `domain/retention.rs` + `outbound/retention.rs`: bounded 30-day cleanup.
## Scope
- Owns: notification creation, read/unread/snooze state transitions,
per-channel delivery, and device registration.
- Not here: the lifecycle state enum itself and its transition rules →
- `notification_state`; the Postgres queries → `notification_db_client`.
+ `notification_state`.
## Boundaries
-- Depends on: `notification_db_client`, `notification_state`, `rate_limit`,
+- Depends on: `notification_state`, `kafka_util`, `rate_limit`,
`macro_authorization`.
- Used by: most feature crates that need to notify a user (`properties`,
`channels`, `reminders`, `teams`, `messages`, `calendar_events`, ...) and
@@ -52,9 +55,11 @@ through the same service.
## Checks
-- `cargo test -p notification` (`SQLX_OFFLINE` unset; needs local MacroDB).
-- `cargo test -p notification --features redis-tests` for the Redis-backed
- preference paths.
+- `cargo test -p notification --all-features` (`SQLX_OFFLINE` unset; needs local
+ MacroDB and Redis).
+- For test edits, also run `RUSTFLAGS=-Dwarnings cargo check -p notification
+ --all-features --tests` against the live schema. Ordinary clippy does not
+ compile test-only code, while CI rejects its warnings.
## Gotchas
@@ -62,6 +67,8 @@ through the same service.
`schema`); building without the feature a caller needs silently drops the
module instead of failing loudly — check `Cargo.toml` before assuming an
adapter is compiled in.
+- Independent Kafka consumers have no durable group lag. Health logs distinguish
+ stale/unknown measurements from zero; they are not client-delivery ACKs.
- Notification payloads must not be logged on tracing spans (see the
`fix(notification): stop recording notification payloads` history).
diff --git a/crates/notification/Cargo.toml b/crates/notification/Cargo.toml
index 77481d5d01..877d6001cf 100644
--- a/crates/notification/Cargo.toml
+++ b/crates/notification/Cargo.toml
@@ -79,6 +79,7 @@ strum = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["time"] }
tokio-retry = { workspace = true }
+tokio-util = { workspace = true, features = ["rt"] }
tracing = { workspace = true }
url = { workspace = true }
utoipa = { workspace = true, optional = true }
diff --git a/crates/notification/src/inbound/notification_events_listener.rs b/crates/notification/src/inbound/notification_events_listener.rs
index d36b60c61b..8dace43fb9 100644
--- a/crates/notification/src/inbound/notification_events_listener.rs
+++ b/crates/notification/src/inbound/notification_events_listener.rs
@@ -1,5 +1,8 @@
//! Listener for notification database events.
+#[cfg(test)]
+mod test;
+
use std::time::Duration;
use macro_user_id::cowlike::CowLike;
@@ -39,6 +42,11 @@ where
}
}
+ #[tracing::instrument(
+ name = "notification.delete.publish",
+ skip_all,
+ fields(notification_id, recipient_count, published)
+ )]
async fn handle_payload(&self, payload: &str) {
let event = match serde_json::from_str::<NotificationDatabaseEvent>(payload) {
Ok(event) => event,
@@ -53,17 +61,22 @@ where
notification_id,
user_ids,
} => {
+ tracing::Span::current()
+ .record("notification_id", tracing::field::display(notification_id));
let user_ids = user_ids
.into_iter()
.filter_map(|user_id| match MacroUserIdStr::parse_from_str(&user_id) {
Ok(user_id) => Some(user_id.into_owned()),
- Err(err) => {
- tracing::warn!(error = ?err, user_id, "invalid user id in notification delete event");
+ Err(_) => {
+ // User IDs contain email addresses; don't include invalid
+ // values (or parser errors echoing them) in telemetry.
+ tracing::warn!("invalid user id in notification delete event");
None
}
})
.collect::<Vec<_>>();
+ tracing::Span::current().record("recipient_count", user_ids.len());
if user_ids.is_empty() {
return;
}
@@ -75,9 +88,11 @@ where
}),
};
- if let Err(err) = self.realtime.publish_updates(&payload).await {
+ let result = self.realtime.publish_updates(&payload).await;
+ tracing::Span::current().record("published", result.is_ok());
+ result.inspect_err(|err| {
tracing::warn!(error = ?err, "failed to publish notification delete realtime update");
- }
+ }).ok();
}
}
}
diff --git a/crates/notification/src/inbound/notification_events_listener/test.rs b/crates/notification/src/inbound/notification_events_listener/test.rs
new file mode 100644
index 0000000000..0f1a81c75d
--- /dev/null
+++ b/crates/notification/src/inbound/notification_events_listener/test.rs
@@ -0,0 +1,83 @@
+use super::*;
+use rootcause::Report;
+use std::sync::Mutex;
+
+struct UnusedReceiver;
+impl NotificationEventsReceiver for UnusedReceiver {
+ async fn receive(&mut self) -> Result<String, Report> {
+ unreachable!("tests feed the listener directly")
+ }
+}
+
+#[derive(Default)]
+struct Publisher {
+ attempts: Mutex<Vec<(Uuid, usize)>>,
+ fail: bool,
+}
+
+impl NotificationRealtimePublisher for Publisher {
+ async fn publish_updates(&self, payload: &NotificationStatusPayload<'_>) -> Result<(), Report> {
+ let NotificationStatusPayload::NotificationForUsers { users, update } = payload else {
+ panic!("expected recipient-scoped deletion");
+ };
+ let NotificationDelete::Delete { id } = **update;
+ self.attempts
+ .lock()
+ .expect("attempts")
+ .push((id, users.len()));
+ if self.fail {
+ return Err(rootcause::report!("test publication failure"));
+ }
+ Ok(())
+ }
+}
+
+#[tokio::test]
+async fn instrumentation_preserves_valid_recipients_and_skips_invalid_values() {
+ let listener = NotificationEventsListener::new(UnusedReceiver, Publisher::default());
+ listener
+ .handle_payload(
+ r#"{
+ "type":"user_notification_deletes",
+ "notificationId":"00000000-0000-7000-8000-000000000001",
+ "userIds":["macro|[email protected]","invalid"]
+ }"#,
+ )
+ .await;
+ assert_eq!(
+ *listener.realtime.attempts.lock().expect("attempts"),
+ vec![(
+ Uuid::parse_str("00000000-0000-7000-8000-000000000001").expect("id"),
+ 1
+ )]
+ );
+}
+
+#[tokio::test]
+async fn failed_publication_does_not_prevent_processing_the_next_event() {
+ let listener = NotificationEventsListener::new(
+ UnusedReceiver,
+ Publisher {
+ fail: true,
+ ..Default::default()
+ },
+ );
+ for id in [
+ "00000000-0000-7000-8000-000000000001",
+ "00000000-0000-7000-8000-000000000002",
+ ] {
+ listener
+ .handle_payload(
+ &serde_json::json!({
+ "type":"user_notification_deletes", "notificationId":id,
+ "userIds":["macro|[email protected]"]
+ })
+ .to_string(),
+ )
+ .await;
+ }
+ assert_eq!(
+ listener.realtime.attempts.lock().expect("attempts").len(),
+ 2
+ );
+}
diff --git a/crates/notification/src/outbound/notification_consumer.rs b/crates/notification/src/outbound/notification_consumer.rs
index 5b826d075a..df2526f99a 100644
--- a/crates/notification/src/outbound/notification_consumer.rs
+++ b/crates/notification/src/outbound/notification_consumer.rs
@@ -4,9 +4,12 @@
//! assigns all `macro.notifications` partitions without joining a durable consumer group. It does
//! not commit offsets.
+mod health;
#[cfg(test)]
mod test;
+use health::ConsumerHealth;
+
use std::{marker::PhantomData, time::Duration};
use kafka_util::{InitialOffset, KafkaEventConsumer, Ungrouped};
@@ -44,6 +47,7 @@ where
T: Clone + Serialize + DeserializeOwned + Send + Sync + 'static,
{
consumer: NotificationEventConsumer,
+ _health: ConsumerHealth,
payload: PhantomData<fn() -> T>,
}
@@ -54,7 +58,8 @@ where
/// Creates a consumer and assigns every current notification topic partition.
#[tracing::instrument(fields(brokers), err)]
pub fn from_env(brokers: &str) -> Result<Self, Report> {
- let consumer = KafkaEventConsumer::<Ungrouped>::from_env(brokers)
+ let (observer, health) = ConsumerHealth::start();
+ let consumer = KafkaEventConsumer::<Ungrouped>::from_env_with_statistics(brokers, observer)
.context("failed to create independent notification topic consumer")?;
let consumer =
IndependentKafkaConsumer::new(consumer, InitialOffset::Latest, TOPIC_METADATA_TIMEOUT)
@@ -67,6 +72,7 @@ where
Ok(Self {
consumer: NotificationEventConsumer::new(consumer),
+ _health: health,
payload: PhantomData,
})
}
diff --git a/crates/notification/src/outbound/notification_consumer/health.rs b/crates/notification/src/outbound/notification_consumer/health.rs
new file mode 100644
index 0000000000..597323a6ce
--- /dev/null
+++ b/crates/notification/src/outbound/notification_consumer/health.rs
@@ -0,0 +1,219 @@
+//! Per-process telemetry for the deliberately ungrouped notification consumer.
+//! A separate timer reports stale callbacks even if application consumption stalls.
+
+use std::{sync::Arc, time::Duration};
+
+use chrono::Utc;
+use kafka_util::ConsumerStatistics;
+use rdkafka::statistics::Statistics;
+use tokio::{
+ sync::watch,
+ time::{Instant, MissedTickBehavior},
+};
+use tokio_util::{sync::CancellationToken, task::TaskTracker};
+use uuid::Uuid;
+
+const TOPIC: &str = "macro.notifications";
+const REPORT_INTERVAL: Duration = Duration::from_secs(10);
+const STALE_AFTER: Duration = Duration::from_secs(30);
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+struct Backlog {
+ known_partitions: u32,
+ unknown_partitions: u32,
+ max_offset_backlog: u64,
+ total_offset_backlog: u64,
+ prefetched_messages: u64,
+ connected_brokers: usize,
+ disconnected_partitions: u32,
+}
+
+fn backlog(stats: &Statistics) -> Option<Backlog> {
+ let topic = stats.topics.get(TOPIC)?;
+ let mut result = Backlog {
+ connected_brokers: stats
+ .brokers
+ .values()
+ .filter(|b| b.nodeid >= 0 && b.state == "UP")
+ .count(),
+ ..Backlog::default()
+ };
+ for partition in topic
+ .partitions
+ .values()
+ .filter(|p| p.partition >= 0 && p.desired)
+ {
+ if !stats
+ .brokers
+ .values()
+ .any(|broker| broker.nodeid == partition.broker && broker.state == "UP")
+ {
+ // Another partition's healthy connection cannot make this cached
+ // high watermark trustworthy while its own broker is unavailable.
+ result.disconnected_partitions += 1;
+ }
+ // app_offset is next-to-deliver, not committed_offset (which is always
+ // invalid here). Before the first delivery use next-to-fetch plus the
+ // prefetched queue, so an idle latest-start partition can report zero.
+ let (position, queued) = if partition.app_offset >= 0 {
+ (partition.app_offset, 0)
+ } else {
+ (partition.next_offset, partition.fetchq_cnt)
+ };
+ let pending = (position >= 0 && partition.hi_offset >= position && queued >= 0)
+ .then(|| (partition.hi_offset - position).checked_add(queued))
+ .flatten()
+ .and_then(|value| u64::try_from(value).ok());
+ if let Some(pending) = pending {
+ result.known_partitions += 1;
+ result.max_offset_backlog = result.max_offset_backlog.max(pending);
+ result.total_offset_backlog = result.total_offset_backlog.saturating_add(pending);
+ } else {
+ result.unknown_partitions += 1;
+ }
+ if let Ok(queued) = u64::try_from(partition.fetchq_cnt) {
+ result.prefetched_messages = result.prefetched_messages.saturating_add(queued);
+ }
+ }
+ Some(result)
+}
+
+#[derive(Clone, Copy)]
+struct Observation {
+ received_at: Instant,
+ sampled_at: i64,
+ backlog: Option<Backlog>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::AsRefStr)]
+#[strum(serialize_all = "snake_case")]
+enum Health {
+ Starting,
+ Healthy,
+ Unavailable,
+ Disconnected,
+ Stale,
+}
+
+fn health(
+ observation: Option<Observation>,
+ started_at: Instant,
+ now: Instant,
+ unix_now: i64,
+) -> Health {
+ let starting = now.duration_since(started_at) < STALE_AFTER;
+ let Some(observation) = observation else {
+ return if starting {
+ Health::Starting
+ } else {
+ Health::Unavailable
+ };
+ };
+ // librdkafka queues callbacks while recv is stalled. A newly delivered but
+ // old sample is stale too; callback arrival time alone would hide that stall.
+ if now.duration_since(observation.received_at) >= STALE_AFTER
+ || unix_now
+ .saturating_sub(observation.sampled_at)
+ .unsigned_abs()
+ >= STALE_AFTER.as_secs()
+ {
+ return Health::Stale;
+ }
+ let Some(backlog) = observation.backlog else {
+ return if starting {
+ Health::Starting
+ } else {
+ Health::Unavailable
+ };
+ };
+ if backlog.connected_brokers == 0 || backlog.disconnected_partitions > 0 {
+ return if starting {
+ Health::Starting
+ } else {
+ Health::Disconnected
+ };
+ }
+ if backlog.known_partitions == 0 || backlog.unknown_partitions > 0 {
+ return if starting {
+ Health::Starting
+ } else {
+ Health::Unavailable
+ };
+ }
+ Health::Healthy
+}
+
+/// Owns the reporter's lifetime; dropping a consumer also stops its telemetry.
+pub(super) struct ConsumerHealth {
+ cancellation: CancellationToken,
+ tasks: TaskTracker,
+}
+
+impl Drop for ConsumerHealth {
+ fn drop(&mut self) {
+ self.cancellation.cancel();
+ self.tasks.close();
+ }
+}
+
+impl ConsumerHealth {
+ pub(super) fn start() -> (ConsumerStatistics, Self) {
+ let (observed, state) = watch::channel(None);
+ let observer: ConsumerStatistics = Arc::new(move |stats| {
+ observed.send_replace(Some(Observation {
+ received_at: Instant::now(),
+ sampled_at: stats.time,
+ backlog: backlog(stats),
+ }));
+ });
+ let consumer_instance = Uuid::now_v7();
+ let started_at = Instant::now();
+ let cancellation = CancellationToken::new();
+ let stop = cancellation.clone();
+ let tasks = TaskTracker::new();
+ tasks.spawn(async move {
+ let mut interval = tokio::time::interval(REPORT_INTERVAL);
+ interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
+ loop {
+ tokio::select! {
+ biased;
+ _ = stop.cancelled() => break,
+ _ = interval.tick() => {}
+ }
+ let observation = *state.borrow();
+ let status = health(
+ observation,
+ started_at,
+ Instant::now(),
+ Utc::now().timestamp(),
+ );
+ let snapshot = observation.and_then(|o| o.backlog);
+ // Never advertise a stale/partial observation as a zero backlog.
+ let known = snapshot.filter(|_| status == Health::Healthy);
+ tracing::info!(
+ telemetry_event = "notification_consumer_health",
+ %consumer_instance,
+ topic = TOPIC,
+ consumer_health = status.as_ref(),
+ known_partitions = snapshot.map(|s| s.known_partitions),
+ unknown_partitions = snapshot.map(|s| s.unknown_partitions),
+ disconnected_partitions = snapshot.map(|s| s.disconnected_partitions),
+ max_offset_backlog = known.map(|s| s.max_offset_backlog),
+ total_offset_backlog = known.map(|s| s.total_offset_backlog),
+ prefetched_messages = known.map(|s| s.prefetched_messages),
+ "notification consumer health"
+ );
+ }
+ });
+ (
+ observer,
+ Self {
+ cancellation,
+ tasks,
+ },
+ )
+ }
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/notification/src/outbound/notification_consumer/health/test.rs b/crates/notification/src/outbound/notification_consumer/health/test.rs
new file mode 100644
index 0000000000..27f805b5b7
--- /dev/null
+++ b/crates/notification/src/outbound/notification_consumer/health/test.rs
@@ -0,0 +1,343 @@
+use super::*;
+use rdkafka::statistics::{Broker, Partition, Topic};
+
+#[tokio::test]
+async fn dropping_consumer_cancels_and_drains_its_tracked_reporter() {
+ let (observer, health) = ConsumerHealth::start();
+ let cancellation = health.cancellation.clone();
+ let tasks = health.tasks.clone();
+ drop(health);
+ assert!(cancellation.is_cancelled());
+ tasks.wait().await;
+ assert!(tasks.is_empty());
+ // A native callback racing with shutdown is harmless after the receiver closes.
+ observer(&Statistics::default());
+}
+
+#[test]
+fn measures_application_backlog_not_commit_or_fetch_position() {
+ let stats = Statistics {
+ topics: [(
+ TOPIC.to_owned(),
+ Topic {
+ partitions: [(
+ 0,
+ Partition {
+ partition: 0,
+ desired: true,
+ app_offset: 10,
+ next_offset: 100,
+ hi_offset: 100,
+ committed_offset: -1001,
+ stored_offset: -1001,
+ fetchq_cnt: 90,
+ ..Default::default()
+ },
+ )]
+ .into(),
+ ..Default::default()
+ },
+ )]
+ .into(),
+ ..Default::default()
+ };
+ let result = backlog(&stats).expect("topic present");
+ assert_eq!(result.known_partitions, 1);
+ assert_eq!(result.max_offset_backlog, 90);
+ assert_eq!(result.total_offset_backlog, 90);
+ assert_eq!(result.prefetched_messages, 90);
+}
+
+#[test]
+fn initial_latest_position_includes_prefetched_records_and_idle_partitions() {
+ let stats = Statistics {
+ topics: [(
+ TOPIC.to_owned(),
+ Topic {
+ partitions: [
+ (
+ 0,
+ Partition {
+ partition: 0,
+ desired: true,
+ app_offset: -1001,
+ next_offset: 100,
+ hi_offset: 100,
+ fetchq_cnt: 0,
+ ..Default::default()
+ },
+ ),
+ (
+ 1,
+ Partition {
+ partition: 1,
+ desired: true,
+ app_offset: -1001,
+ next_offset: 100,
+ hi_offset: 110,
+ fetchq_cnt: 20,
+ ..Default::default()
+ },
+ ),
+ ]
+ .into(),
+ ..Default::default()
+ },
+ )]
+ .into(),
+ ..Default::default()
+ };
+ let result = backlog(&stats).expect("topic present");
+ assert_eq!(result.known_partitions, 2);
+ assert_eq!(result.unknown_partitions, 0);
+ assert_eq!(result.max_offset_backlog, 30);
+ assert_eq!(result.total_offset_backlog, 30);
+}
+
+#[test]
+fn excludes_unassigned_partitions_and_does_not_turn_unknown_offsets_into_zero() {
+ let stats = Statistics {
+ topics: [(
+ TOPIC.to_owned(),
+ Topic {
+ partitions: [
+ (
+ -1,
+ Partition {
+ partition: -1,
+ desired: true,
+ ..Default::default()
+ },
+ ),
+ (
+ 0,
+ Partition {
+ partition: 0,
+ desired: false,
+ ..Default::default()
+ },
+ ),
+ (
+ 1,
+ Partition {
+ partition: 1,
+ desired: true,
+ app_offset: -1001,
+ next_offset: -1001,
+ hi_offset: -1001,
+ ..Default::default()
+ },
+ ),
+ (
+ 2,
+ Partition {
+ partition: 2,
+ desired: true,
+ app_offset: 100,
+ hi_offset: 50,
+ ..Default::default()
+ },
+ ),
+ ]
+ .into(),
+ ..Default::default()
+ },
+ )]
+ .into(),
+ ..Default::default()
+ };
+ let result = backlog(&stats).expect("topic present");
+ assert_eq!(result.known_partitions, 0);
+ assert_eq!(result.unknown_partitions, 2);
+ assert!(backlog(&Statistics::default()).is_none());
+}
+
+#[test]
+fn broker_connectivity_is_reported_without_bootstrap_or_disconnected_brokers() {
+ let stats = Statistics {
+ topics: [(TOPIC.to_owned(), Topic::default())].into(),
+ brokers: [
+ (
+ "leader".to_owned(),
+ Broker {
+ nodeid: 1,
+ state: "UP".into(),
+ ..Default::default()
+ },
+ ),
+ (
+ "down".to_owned(),
+ Broker {
+ nodeid: 2,
+ state: "DOWN".into(),
+ ..Default::default()
+ },
+ ),
+ (
+ "bootstrap".to_owned(),
+ Broker {
+ nodeid: -1,
+ state: "UP".into(),
+ ..Default::default()
+ },
+ ),
+ ]
+ .into(),
+ ..Default::default()
+ };
+ assert_eq!(backlog(&stats).expect("topic present").connected_brokers, 1);
+}
+
+#[tokio::test(start_paused = true)]
+async fn a_connected_broker_does_not_mask_another_partitions_stale_watermark() {
+ let stats = Statistics {
+ topics: [(
+ TOPIC.to_owned(),
+ Topic {
+ partitions: [
+ (
+ 0,
+ Partition {
+ partition: 0,
+ desired: true,
+ broker: 1,
+ app_offset: 10,
+ hi_offset: 10,
+ ..Default::default()
+ },
+ ),
+ (
+ 1,
+ Partition {
+ partition: 1,
+ desired: true,
+ broker: 2,
+ app_offset: 10,
+ hi_offset: 10,
+ ..Default::default()
+ },
+ ),
+ ]
+ .into(),
+ ..Default::default()
+ },
+ )]
+ .into(),
+ brokers: [
+ (
+ "up".to_owned(),
+ Broker {
+ nodeid: 1,
+ state: "UP".into(),
+ ..Default::default()
+ },
+ ),
+ (
+ "down".to_owned(),
+ Broker {
+ nodeid: 2,
+ state: "DOWN".into(),
+ ..Default::default()
+ },
+ ),
+ ]
+ .into(),
+ ..Default::default()
+ };
+ let snapshot = backlog(&stats).expect("topic present");
+ assert_eq!(snapshot.max_offset_backlog, 0);
+ assert_eq!(snapshot.disconnected_partitions, 1);
+ let started_at = Instant::now();
+ tokio::time::advance(STALE_AFTER).await;
+ let now = Instant::now();
+ let observation = Observation {
+ received_at: now,
+ sampled_at: 130,
+ backlog: Some(snapshot),
+ };
+ assert_eq!(
+ health(Some(observation), started_at, now, 130),
+ Health::Disconnected
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn reports_stalls_and_old_queued_callbacks_instead_of_stale_zero_lag() {
+ let start = Instant::now();
+ assert_eq!(health(None, start, start, 100), Health::Starting);
+ let observation = Observation {
+ received_at: start,
+ sampled_at: 100,
+ backlog: Some(Backlog {
+ connected_brokers: 1,
+ known_partitions: 1,
+ ..Default::default()
+ }),
+ };
+ assert_eq!(
+ health(Some(observation), start, start, 100),
+ Health::Healthy
+ );
+ tokio::time::advance(STALE_AFTER).await;
+ let now = Instant::now();
+ assert_eq!(health(None, start, now, 130), Health::Unavailable);
+ assert_eq!(health(Some(observation), start, now, 130), Health::Stale);
+ assert_eq!(
+ health(
+ Some(Observation {
+ received_at: now,
+ ..observation
+ }),
+ start,
+ now,
+ 130
+ ),
+ Health::Stale
+ );
+ assert_eq!(
+ health(
+ Some(Observation {
+ received_at: now,
+ sampled_at: 130,
+ ..observation
+ }),
+ start,
+ now,
+ 130
+ ),
+ Health::Healthy
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn fresh_callbacks_with_missing_offsets_or_brokers_are_not_healthy() {
+ let start = Instant::now();
+ tokio::time::advance(STALE_AFTER).await;
+ let now = Instant::now();
+ let observation = Observation {
+ received_at: now,
+ sampled_at: 130,
+ backlog: None,
+ };
+ assert_eq!(
+ health(Some(observation), start, now, 130),
+ Health::Unavailable
+ );
+ let disconnected = Observation {
+ backlog: Some(Backlog::default()),
+ ..observation
+ };
+ assert_eq!(
+ health(Some(disconnected), start, now, 130),
+ Health::Disconnected
+ );
+ let unknown = Observation {
+ backlog: Some(Backlog {
+ connected_brokers: 1,
+ unknown_partitions: 1,
+ ..Default::default()
+ }),
+ ..observation
+ };
+ assert_eq!(health(Some(unknown), start, now, 130), Health::Unavailable);
+}
diff --git a/docs/runbooks/notification-retention.md b/docs/runbooks/notification-retention.md
index c77d9c01ca..0609fdd0b2 100644
--- a/docs/runbooks/notification-retention.md
+++ b/docs/runbooks/notification-retention.md
@@ -81,6 +81,80 @@ refetch/reconnect; this change does not add durable tombstones or an outbox.
Watch notification-service/Kafka/connection-gateway load during backfill. Do not
turn off delete events to speed up the purge.
+## Tracing and consumer health
+
+The retention Lambda includes the pinned x86_64 Datadog Extension v101, with an
+OTLP gRPC receiver on `127.0.0.1:4317`. The extension retrieves the **existing**
+Datadog key by secret ARN; its role can read only that key, not unrelated secrets.
+No API key value is copied into the Lambda environment. If that secret uses a
+customer-managed KMS key, verify narrowly scoped decrypt permission before deploy.
+CloudWatch → Firehose remains the log path; extension log shipping is disabled to
+avoid duplicates. Runtime identity is `service:notification-retention`, with
+`DD_ENV` set to the stack.
+
+The handler span is `notification.retention.invoke`, searchable by
+`faas.invocation_id`. After that span closes, the worker flushes completed spans
+with a five-second caller deadline before Lambda can freeze the process. It does
+not shut down the provider between warm invocations. Flush failure logs an error
+but preserves the cleanup result: committed deletion must not be retried just
+because telemetry failed. The existing ten-second invocation headroom covers
+this flush. The extension must be checked live after an approved deployment;
+unit tests or a green Lambda result do not prove Datadog received its traces.
+
+DSS notification consumers intentionally assign all partitions from latest,
+without committing offsets or joining durable consumer groups. Consequently MSK
+`MaxOffsetLag` is absent for `macro.notifications`; don't change delivery semantics
+just to manufacture that metric.
+
+Each consumer now reports `telemetry_event:notification_consumer_health` every
+ten seconds through normal service logs, with an opaque `consumer_instance`:
+
+- `max_offset_backlog` / `total_offset_backlog`: broker high watermark minus the
+ next application-delivery position. Before first delivery, use next fetch
+ position plus prefetched records. Offset gaps mean this is an approximation,
+ not an exact count of outstanding business events.
+- `prefetched_messages`: records buffered locally, not just records still on the
+ broker. This prevents a full fetch queue from looking caught up.
+- `known_partitions`, `unknown_partitions`, `disconnected_partitions`, and
+ `consumer_health`: distinguish
+ valid measurements from missing offsets, disconnected brokers, and startup.
+- A separate reporter detects callbacks older than 30 seconds (including old
+ snapshots delivered late), even when the consuming task is stalled. Stale or
+ partial backlog measurements are omitted, **never** reported as zero. Dropping
+ the consumer stops its reporter.
+
+The manual `infra/stacks/monitoring` stack adds nine log monitors and a rollout
+dashboard. Alerts isolate consumer instances; a healthy process cannot mask a
+lagging one. Retired-instance alerts resolve on missing data, and a separate
+per-environment heartbeat alarm catches loss of all consumer telemetry. Starting
+thresholds are warning 100 / critical 1000 for the five-minute average of maximum
+partition backlog; tune against real traffic before enabling production cleanup.
+
+The `notification.delete.publish` span covers forwarding a database deletion,
+with notification ID, recipient count, and publication outcome, but no recipient
+IDs or payload. A log monitor catches failed forwarding. Consumer backlog begins
+at Kafka: it does **not** measure time queued before PostgreSQL NOTIFY is forwarded,
+or acknowledge client delivery. Existing best-effort loss/reconnect semantics
+remain unchanged; no durable outbox is added by this observability change.
+
+Before the next deletion canary, after approved deployments:
+
+1. Deploy the notification-service stack (extension + worker) and DSS (consumer
+ telemetry). Keep both cleanup schedules disabled and dry-run enabled.
+2. Invoke explicit dry runs and verify the complete invocation and batch/read
+ spans in Datadog, with the correct service/env and no exporter/flush errors.
+3. Confirm health logs from each running DSS instance, including healthy zero
+ backlog when idle; investigate missing/unknown/stale samples rather than
+ accepting an empty chart as healthy.
+4. Validate and deploy the monitoring stack separately. Verify the incoming
+ attributes are parsed as numeric measures (`max_offset_backlog`, `deleted`)
+ and grouping attributes (`consumer_instance`, `consumer_health`) in Datadog;
+ configure missing measures/facets before relying on aggregation charts. The
+ authenticated preview must show nine new monitors and one dashboard, without
+ modifying adopted resources.
+5. Only then approve a bounded dev deletion canary and inspect DB/API health,
+ publisher failures, and consumer backlog draining before considering prod.
+
## Rollout
Production deletion needs explicit approval and a verified backup/PITR recovery
@@ -151,14 +225,23 @@ within 30 days; pagination of currently unbounded consumers remains separate wor
Run from the repository root with local PostgreSQL and `SQLX_OFFLINE` unset:
```bash
-cargo test -p notification -p notification_retention_handler -- --test-threads=8
+cargo test -p kafka_util -p macro_entrypoint -p notification -p notification_retention_handler -- --test-threads=8
+# Only if notification SQL or schema changed:
nix develop --command just prepare_db
just check
+just clippy
\cd infra
-bunx tsc --noEmit --project stacks/notification-service/tsconfig.json
+bun test stacks/notification-service/retention.test.ts
+bun test stacks/monitoring/monitors/notification-retention.test.ts
+bun run check
```
Tests cover cutoff boundaries, parent-vs-recipient age, all states, cascades,
read-only previews, deterministic bounded ordering, retry behavior, skipped locks,
transaction advisory locking, timeout rollback, non-UTC sessions, byte-bounded
multi-recipient delete events, invocation caps, and fail-closed event parsing.
+Observability tests also cover warm-invocation trace flushing and exporter
+failures, a real librdkafka mock broker without offset commits, stale/partial
+consumer statistics, broker disconnects, reporter cancellation, and the Pulumi
+collector/secret/monitor configuration. Hosted Datadog query validation and
+trace delivery still require an authenticated preview and approved deployment.
diff --git a/infra/packages/resources/AGENTS.md b/infra/packages/resources/AGENTS.md
new file mode 100644
index 0000000000..4c0116b109
--- /dev/null
+++ b/infra/packages/resources/AGENTS.md
@@ -0,0 +1,26 @@
+# resources
+
+## What it is
+
+Shared Pulumi AWS building blocks used by service stacks: gateway target groups,
+queues, storage, ECS defaults, alarms, and telemetry sidecars.
+
+## Start here
+
+- `src/index.ts`: exported components and helpers.
+- `src/resources/datadog.ts`: ECS telemetry sidecars and the existing Datadog key.
+ `DATADOG_API_KEY_SECRET_ARN` lets extensions retrieve that same key without
+ copying its value into runtime environment variables.
+
+## Checks
+
+- From `infra/`: `bun run check` and Biome on changed files.
+- Changes to Datadog key wiring: run
+ `bun test stacks/notification-service/retention.test.ts`.
+
+## Gotchas
+
+- Do not create a parallel Datadog secret. Grant only the specific secret ARN
+ to consumers that fetch the key themselves.
+- Reuse the gateway/Doppler patterns in `docs/development/infrastructure.md`;
+ some older exported helpers remain for legacy stacks, not new services.
diff --git a/infra/packages/resources/CLAUDE.md b/infra/packages/resources/CLAUDE.md
new file mode 120000
index 0000000000..47dc3e3d86
--- /dev/null
+++ b/infra/packages/resources/CLAUDE.md
@@ -0,0 +1 @@
+AGENTS.md
\ No newline at end of file
diff --git a/infra/packages/resources/src/index.ts b/infra/packages/resources/src/index.ts
index 05ee70d021..65df32b2aa 100644
--- a/infra/packages/resources/src/index.ts
+++ b/infra/packages/resources/src/index.ts
@@ -3,6 +3,7 @@ export { create_cluster } from './resources/cluster';
export { ALLOWED_ORIGINS } from './resources/cors';
export {
DATADOG_API_KEY,
+ DATADOG_API_KEY_SECRET_ARN,
DatadogServiceEntity,
datadogAgentContainer,
fargateLogRouterSidecarContainer,
diff --git a/infra/packages/resources/src/resources/datadog.ts b/infra/packages/resources/src/resources/datadog.ts
index c604176d08..a208534c53 100644
--- a/infra/packages/resources/src/resources/datadog.ts
+++ b/infra/packages/resources/src/resources/datadog.ts
@@ -9,11 +9,14 @@ import { grafanaTelemetryEnabled, stack } from '../../../shared';
const DATADOG_API_KEY_SECRET_KEY = 'datadog-api-key';
-export const DATADOG_API_KEY = aws.secretsmanager
- .getSecretVersionOutput({
- secretId: DATADOG_API_KEY_SECRET_KEY,
- })
- .apply((secret) => secret.secretString);
+const datadogApiKey = aws.secretsmanager.getSecretVersionOutput({
+ secretId: DATADOG_API_KEY_SECRET_KEY,
+});
+
+export const DATADOG_API_KEY = datadogApiKey.secretString;
+// Extensions fetch the existing key themselves; don't copy its value into
+// Lambda environment variables or grant access to unrelated secrets.
+export const DATADOG_API_KEY_SECRET_ARN = datadogApiKey.arn;
interface DatadogServiceEntityArgs {
serviceName: string;
diff --git a/infra/stacks/monitoring/AGENTS.md b/infra/stacks/monitoring/AGENTS.md
new file mode 100644
index 0000000000..1c394bd6e6
--- /dev/null
+++ b/infra/stacks/monitoring/AGENTS.md
@@ -0,0 +1,24 @@
+# monitoring
+
+## What it is
+
+Org-global Datadog monitors, synthetics, and rollout dashboards. This stack is
+manual and has only a prod stack, even for monitors covering both environments.
+
+## Checks
+
+From `infra/`:
+- `bun test stacks/monitoring/monitors/notification-retention.test.ts`
+- `bun run check`
+- Biome on changed files.
+- An authenticated `pulumi preview` must leave adopted monitors unchanged.
+
+## Gotchas
+
+- Read `README.md` before deployment: imported resources must be adopted first
+ to avoid duplicate alerts. Never deploy without explicit approval.
+- Notification retention adds nine monitors and one dashboard. Deploy DSS health
+ instrumentation first; missing telemetry is deliberately an alert, not zero lag.
+- Do not use native committed-offset lag for the independent notification
+ consumers. Their per-instance log measurements preserve broadcast semantics.
+- `notificationRetentionDashboardUrl` is a stack output, not an app route.
diff --git a/infra/stacks/monitoring/CLAUDE.md b/infra/stacks/monitoring/CLAUDE.md
new file mode 120000
index 0000000000..47dc3e3d86
--- /dev/null
+++ b/infra/stacks/monitoring/CLAUDE.md
@@ -0,0 +1 @@
+AGENTS.md
\ No newline at end of file
diff --git a/infra/stacks/monitoring/README.md b/infra/stacks/monitoring/README.md
index 142af69d73..3a65e55b6a 100644
--- a/infra/stacks/monitoring/README.md
+++ b/infra/stacks/monitoring/README.md
@@ -133,3 +133,28 @@ should show exactly those 15 creates and nothing else.
Resolver errors still return HTTP 200, so the error monitors are proxies. Add
per-operation error and latency monitors once the server emits operation names
and error counts.
+
+## Notification retention coverage
+
+`monitors/notification-retention.ts` adds **nine monitors and one dashboard**:
+per-environment consumer backlog, unhealthy telemetry, missing telemetry, and
+failed delete publication, plus a cross-environment retention trace-flush alarm.
+The dashboard URL is exported as `notificationRetentionDashboardUrl`.
+
+Deploy the notification consumer instrumentation in DSS before these monitors;
+otherwise the missing-telemetry alerts intentionally fire. This monitoring stack
+still requires a separate, explicitly approved manual deployment. The preview
+must not change adopted monitors.
+
+Consumers are intentionally ungrouped and do not commit offsets, so native MSK
+consumer-group lag is not meaningful. The new logs expose broker high watermark
+minus application position, not committed position. Unknown/stale measurements
+are omitted rather than graphed as zero. Alerts are per consumer instance so one
+healthy process cannot mask a lagging one; retired-instance alerts resolve on
+missing data, while a separate monitor catches loss of all telemetry in an env.
+The initial backlog thresholds (warning 100, critical 1000 averaged over five
+minutes) need review against live baselines before production cleanup.
+
+These signals do not acknowledge delivery through PostgreSQL NOTIFY or WebSockets.
+See [the rollout runbook](../../../docs/runbooks/notification-retention.md) for
+limitations and the post-deploy verification gates.
diff --git a/infra/stacks/monitoring/index.ts b/infra/stacks/monitoring/index.ts
index 6ee0ebfae5..f72d16d022 100644
--- a/infra/stacks/monitoring/index.ts
+++ b/infra/stacks/monitoring/index.ts
@@ -20,3 +20,7 @@ import './synthetics';
// Declared here first.
import './monitors/ai-editing';
import './monitors/graphql-soup';
+import { notificationRetentionDashboard } from './monitors/notification-retention';
+
+export const notificationRetentionDashboardUrl =
+ notificationRetentionDashboard.url;
diff --git a/infra/stacks/monitoring/monitors/notification-retention.test.ts b/infra/stacks/monitoring/monitors/notification-retention.test.ts
new file mode 100644
index 0000000000..e213808d66
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/notification-retention.test.ts
@@ -0,0 +1,78 @@
+import { beforeAll, expect, test } from 'bun:test';
+import * as pulumi from '@pulumi/pulumi';
+
+const resources: pulumi.runtime.MockResourceArgs[] = [];
+
+beforeAll(async () => {
+ pulumi.runtime.setMocks(
+ {
+ newResource(args) {
+ resources.push(args);
+ return { id: args.name, state: args.inputs };
+ },
+ call(args) {
+ return args.inputs;
+ },
+ },
+ 'monitoring',
+ 'prod',
+ false
+ );
+ await import('./notification-retention');
+ await pulumi.runtime.waitForRPCs();
+});
+
+function monitor(name: string): pulumi.runtime.MockResourceArgs {
+ const value = resources.find((r) => r.name === name);
+ if (!value) throw new Error(`Missing monitor ${name}`);
+ return value;
+}
+
+test('backlog and stale-health alerts isolate each consumer and retire old instances', () => {
+ for (const env of ['dev', 'prod']) {
+ for (const kind of ['backlog', 'health']) {
+ const inputs = monitor(`notification-consumer-${kind}-${env}`).inputs;
+ expect(inputs.query).toContain(`env:${env}`);
+ expect(inputs.query).toContain('.by("@consumer_instance")');
+ expect(inputs.onMissingData).toBe('resolve');
+ }
+ const backlog = monitor(`notification-consumer-backlog-${env}`).inputs;
+ expect(backlog.query).toContain('@consumer_health:healthy');
+ expect(backlog.query).toContain('@max_offset_backlog');
+ expect(backlog.query).not.toContain('committed');
+ const health = monitor(`notification-consumer-health-${env}`).inputs;
+ expect(health.query).toContain('stale OR unavailable OR disconnected');
+ expect(health.query).not.toContain('starting');
+ }
+});
+
+test('missing telemetry is an alarm, not a healthy zero backlog', () => {
+ for (const env of ['dev', 'prod']) {
+ const inputs = monitor(`notification-consumer-heartbeat-${env}`).inputs;
+ expect(inputs.query).toContain('.last("5m") < 1');
+ expect(inputs.query).not.toContain('.by(');
+ expect(inputs.onMissingData).toBe('default');
+ }
+});
+
+test('trace flush and realtime publication failures are covered separately from Lambda errors', () => {
+ expect(
+ monitor('notification-retention-telemetry-export').inputs.query
+ ).toContain('notification retention telemetry flush');
+ for (const env of ['dev', 'prod']) {
+ expect(
+ monitor(`notification-delete-publication-${env}`).inputs.query
+ ).toContain('failed to publish notification delete realtime update');
+ }
+});
+
+test('dashboard charts only known backlog alongside health and deletion progress', () => {
+ const dashboard = monitor('notification-retention').inputs;
+ const query = dashboard.widgets[0].timeseriesDefinition.requests[0].logQuery;
+ expect(query.searchQuery).toContain('@consumer_health:healthy');
+ expect(query.computeQuery.facet).toBe('@max_offset_backlog');
+ expect(dashboard.widgets).toHaveLength(4);
+ expect(
+ resources.filter((r) => r.type.includes('monitor:Monitor'))
+ ).toHaveLength(9);
+});
diff --git a/infra/stacks/monitoring/monitors/notification-retention.ts b/infra/stacks/monitoring/monitors/notification-retention.ts
new file mode 100644
index 0000000000..70e343f99f
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/notification-retention.ts
@@ -0,0 +1,156 @@
+// Independent notification consumers do not commit Kafka offsets. Native MSK
+// consumer-group lag is therefore absent; these logs report application backlog.
+import * as datadog from '@pulumi/datadog';
+
+const HEALTH = '@telemetry_event:notification_consumer_health';
+const CONTEXT = `Notification-retention rollout: pause scheduled cleanup before investigating.
+Check consumer health/backlog, notification.delete.publish spans, database latency,
+and replica lag. A zero application backlog is not an end-to-end delivery ACK:
+Postgres NOTIFY publication and WebSocket delivery can still fail.
+@webhook-macro-alert-hook`;
+
+for (const environment of ['dev', 'prod']) {
+ const query = `${HEALTH} env:${environment}`;
+ const common = {
+ tags: [
+ `env:${environment}`,
+ 'service:notification-realtime',
+ 'rollout:notification-retention',
+ ],
+ priority: environment === 'prod' ? '2' : '3',
+ includeTags: true,
+ requireFullWindow: false,
+ };
+ new datadog.Monitor(`notification-consumer-backlog-${environment}`, {
+ ...common,
+ name: `[${environment.toUpperCase()}] Notification consumer: application backlog high`,
+ type: 'log alert',
+ query: `logs("${query} @consumer_health:healthy").index("*").rollup("avg", "@max_offset_backlog").by("@consumer_instance").last("5m") > 1000`,
+ message: `One consumer's maximum partition offset backlog averages over {{threshold}} for five minutes. This is not committed-offset lag. The initial threshold is conservative; tune after observing dry-run and canary baselines.\n${CONTEXT}`,
+ monitorThresholds: { critical: '1000', warning: '100' },
+ // A retired process must not leave its old lag alert firing forever.
+ // Stalls in live processes produce explicit health events below.
+ onMissingData: 'resolve',
+ });
+ new datadog.Monitor(`notification-consumer-health-${environment}`, {
+ ...common,
+ name: `[${environment.toUpperCase()}] Notification consumer: telemetry stale or unavailable`,
+ type: 'log alert',
+ query: `logs("${query} @consumer_health:(stale OR unavailable OR disconnected)").index("*").rollup("count").by("@consumer_instance").last("5m") > 0`,
+ message: `A live consumer has stale statistics, unknown offsets, or no connected brokers after its startup grace period. Do not interpret missing/partial telemetry as zero lag.\n${CONTEXT}`,
+ monitorThresholds: { critical: '0' },
+ onMissingData: 'resolve',
+ });
+ new datadog.Monitor(`notification-consumer-heartbeat-${environment}`, {
+ ...common,
+ name: `[${environment.toUpperCase()}] Notification consumer: no telemetry`,
+ type: 'log alert',
+ query: `logs("${query}").index("*").rollup("count").last("5m") < 1`,
+ message: `No notification-consumer health events arrived for five minutes. Verify DSS consumers are running and log ingestion is healthy. Deploy the consumer instrumentation before enabling this monitor.\n${CONTEXT}`,
+ monitorThresholds: { critical: '1' },
+ onMissingData: 'default',
+ });
+ new datadog.Monitor(`notification-delete-publication-${environment}`, {
+ ...common,
+ name: `[${environment.toUpperCase()}] Notification deletion: realtime publication failed`,
+ type: 'log alert',
+ query: `logs("env:${environment} \\"failed to publish notification delete realtime update\\"").index("*").rollup("count").last("5m") > 0`,
+ message: `A database deletion could not be forwarded to realtime. Deletion has already committed; PostgreSQL NOTIFY is best-effort, not a durable retry queue.\n${CONTEXT}`,
+ monitorThresholds: { critical: '0' },
+ onMissingData: 'default',
+ });
+}
+
+// CloudWatch's Lambda Errors stays zero when cleanup succeeds but tracing fails.
+// Match the application diagnostic rather than assuming Firehose sets env tags.
+new datadog.Monitor('notification-retention-telemetry-export', {
+ name: 'Notification retention: trace flush failed',
+ type: 'log alert',
+ query:
+ 'logs("\\"notification retention telemetry flush\\"").index("*").rollup("count").last("15m") > 0',
+ message: `Cleanup may have succeeded while its traces were lost. Check the Datadog extension, its scoped secret access, and OTLP receiver. Do not retry deletion merely to retry telemetry.\n${CONTEXT}`,
+ tags: ['service:notification-retention', 'rollout:notification-retention'],
+ priority: '3',
+ monitorThresholds: { critical: '0' },
+ requireFullWindow: false,
+ onMissingData: 'default',
+});
+
+export const notificationRetentionDashboard = new datadog.Dashboard(
+ 'notification-retention',
+ {
+ title: 'Notification retention rollout',
+ description:
+ 'Application backlog for independent consumers, telemetry health, and retention progress. No committed-offset lag is available. See docs/runbooks/notification-retention.md.',
+ layoutType: 'ordered',
+ widgets: [
+ {
+ timeseriesDefinition: {
+ title: 'Maximum partition offset backlog, per consumer',
+ requests: [
+ {
+ displayType: 'line',
+ logQuery: {
+ index: '*',
+ searchQuery: `${HEALTH} @consumer_health:healthy`,
+ computeQuery: {
+ aggregation: 'max',
+ facet: '@max_offset_backlog',
+ },
+ groupBies: [
+ { facet: 'env', limit: 2 },
+ { facet: '@consumer_instance', limit: 20 },
+ ],
+ },
+ },
+ ],
+ },
+ },
+ {
+ timeseriesDefinition: {
+ title: 'Consumer health events (missing is not zero lag)',
+ requests: [
+ {
+ displayType: 'bars',
+ logQuery: {
+ index: '*',
+ searchQuery: HEALTH,
+ computeQuery: { aggregation: 'count' },
+ groupBies: [
+ { facet: 'env', limit: 2 },
+ { facet: '@consumer_health', limit: 5 },
+ ],
+ },
+ },
+ ],
+ },
+ },
+ {
+ timeseriesDefinition: {
+ title: 'Committed parent deletions reported by completed sweeps',
+ requests: [
+ {
+ displayType: 'bars',
+ logQuery: {
+ index: '*',
+ searchQuery: '"notification retention sweep complete"',
+ computeQuery: { aggregation: 'sum', facet: '@deleted' },
+ },
+ },
+ ],
+ },
+ },
+ {
+ timeseriesDefinition: {
+ title: 'MacroDB CPU during rollout',
+ requests: [
+ {
+ q: 'max:aws.rds.cpuutilization{name:macro-db-dev OR name:macro-db-prod} by {name}',
+ displayType: 'line',
+ },
+ ],
+ },
+ },
+ ],
+ }
+);
diff --git a/infra/stacks/notification-service/AGENTS.md b/infra/stacks/notification-service/AGENTS.md
new file mode 100644
index 0000000000..0eb2136d80
--- /dev/null
+++ b/infra/stacks/notification-service/AGENTS.md
@@ -0,0 +1,24 @@
+# notification-service stack
+
+## What it is
+
+ECS notification delivery, queues/mobile platforms, and the separately scheduled
+notification-retention Lambda. Runtime business policy lives in Rust.
+
+## Checks
+
+From `infra/`:
+- `bun test stacks/notification-service/retention.test.ts`
+- `bun run check`
+- `bunx biome check stacks/notification-service`
+
+## Gotchas
+
+- Keep retention disabled/dry-run until explicitly approved. Deployment does not
+ authorize deletion; `docs/runbooks/notification-retention.md` owns rollout.
+- The retention function needs its pinned x86_64 Datadog extension to receive
+ gRPC OTLP; the ECS sidecar default alone does not work in Lambda.
+- Pass the existing Datadog key ARN, not its value, and scope the role to that key.
+- Extension log shipping is off: the shared Lambda helper already forwards
+ CloudWatch logs through Firehose. Monitoring resources deploy separately from
+ `infra/stacks/monitoring`, not automatically with this service.
diff --git a/infra/stacks/notification-service/CLAUDE.md b/infra/stacks/notification-service/CLAUDE.md
new file mode 120000
index 0000000000..47dc3e3d86
--- /dev/null
+++ b/infra/stacks/notification-service/CLAUDE.md
@@ -0,0 +1 @@
+AGENTS.md
\ No newline at end of file
diff --git a/infra/stacks/notification-service/retention.test.ts b/infra/stacks/notification-service/retention.test.ts
new file mode 100644
index 0000000000..8b66b7c2c2
--- /dev/null
+++ b/infra/stacks/notification-service/retention.test.ts
@@ -0,0 +1,121 @@
+import { beforeAll, expect, mock, test } from 'bun:test';
+import * as pulumi from '@pulumi/pulumi';
+
+const resources: pulumi.runtime.MockResourceArgs[] = [];
+const secretArn =
+ 'arn:aws:secretsmanager:us-east-1:123456789012:secret:datadog-test';
+
+beforeAll(async () => {
+ pulumi.runtime.setAllConfig({ 'aws:region': 'us-east-1' });
+ pulumi.runtime.setMocks(
+ {
+ newResource(args) {
+ resources.push(args);
+ return {
+ id: args.name,
+ state: {
+ ...args.inputs,
+ name: args.inputs.name ?? args.name,
+ arn: `arn:aws:mock:us-east-1:123456789012:${args.name}`,
+ },
+ };
+ },
+ call(args) {
+ return args.inputs;
+ },
+ },
+ 'notification-service',
+ 'dev',
+ false
+ );
+ mock.module('../../packages/shared', () => ({
+ stack: 'dev',
+ CLOUD_TRAIL_SNS_TOPIC_ARN: 'arn:aws:sns:us-east-1:123456789012:alerts',
+ CLOUDWATCH_KINESIS_STREAM_ROLE_ARN: 'arn:aws:iam::123456789012:role/logs',
+ DATADOG_KINESIS_FIREHOSE_STREAM_ARN:
+ 'arn:aws:firehose:us-east-1:123456789012:logs',
+ }));
+ mock.module('../../packages/resources', () => ({
+ DATADOG_API_KEY_SECRET_ARN: pulumi.output(secretArn),
+ }));
+ // Keep the real Lambda resources, but don't depend on the test runner's cwd.
+ mock.module('../../packages/lambda/src/hash', () => ({
+ generateContentHash: () => 'test-source-hash',
+ }));
+ // Pulumi dynamic-provider closure serialization is Node-only; it is unrelated
+ // to the AWS configuration under test.
+ mock.module('../../packages/lambda/src/source_code_hash', () => ({
+ SourceCodeHash: class extends pulumi.ComponentResource {
+ constructor(
+ name: string,
+ args: { sourceCodeHash: string },
+ opts?: pulumi.ComponentResourceOptions
+ ) {
+ super('test:source:Hash', name, args, opts);
+ }
+ },
+ }));
+ const { NotificationRetention } = await import('./retention');
+ new NotificationRetention('retention-test', {
+ databaseUrl: pulumi.secret('test-database'),
+ enabled: false,
+ dryRun: true,
+ vpc: {
+ vpcId: 'vpc-test',
+ publicSubnetIds: [],
+ privateSubnetIds: ['subnet-test'],
+ },
+ tags: {},
+ });
+ await pulumi.runtime.waitForRPCs();
+});
+
+function resource(type: string): pulumi.runtime.MockResourceArgs {
+ const value = resources.find((r) => r.type === type);
+ if (!value) throw new Error(`Missing ${type}`);
+ return value;
+}
+
+test('retention has a pinned OTLP collector and consistent service identity', () => {
+ const fn = resource('aws:lambda/function:Function').inputs;
+ expect(fn.layers).toEqual([
+ 'arn:aws:lambda:us-east-1:464622532012:layer:Datadog-Extension:101',
+ ]);
+ const environment = pulumi.runtime.unwrapRpcSecret(fn.environment);
+ expect(environment.variables).toMatchObject({
+ DD_SERVICE: 'notification-retention',
+ DD_ENV: 'dev',
+ DD_SITE: 'us5.datadoghq.com',
+ DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: '127.0.0.1:4317',
+ DD_SERVERLESS_LOGS_ENABLED: 'false',
+ DD_API_KEY_SECRET_ARN: secretArn,
+ });
+ expect(environment.variables.DD_API_KEY).toBeUndefined();
+ expect(fn.reservedConcurrentExecutions).toBe(1);
+});
+
+test('extension access is scoped to the existing Datadog key, not all secrets', () => {
+ const policy = JSON.parse(
+ resource('aws:iam/rolePolicy:RolePolicy').inputs.policy
+ );
+ expect(policy.Statement).toEqual([
+ {
+ Effect: 'Allow',
+ Action: 'secretsmanager:GetSecretValue',
+ Resource: secretArn,
+ },
+ ]);
+});
+
+test('telemetry does not enable cleanup or duplicate the existing log path', () => {
+ expect(resource('aws:cloudwatch/eventRule:EventRule').inputs.state).toBe(
+ 'DISABLED'
+ );
+ expect(
+ JSON.parse(resource('aws:cloudwatch/eventTarget:EventTarget').inputs.input)
+ ).toEqual({ dry_run: true });
+ expect(
+ resource('aws:cloudwatch/logSubscriptionFilter:LogSubscriptionFilter')
+ .inputs.destinationArn
+ ).toBe('arn:aws:firehose:us-east-1:123456789012:logs');
+});
diff --git a/infra/stacks/notification-service/retention.ts b/infra/stacks/notification-service/retention.ts
index 517ec4b976..99f867597e 100644
--- a/infra/stacks/notification-service/retention.ts
+++ b/infra/stacks/notification-service/retention.ts
@@ -1,15 +1,25 @@
import * as aws from '@pulumi/aws';
import * as pulumi from '@pulumi/pulumi';
import { Lambda } from '../../packages/lambda';
+import { DATADOG_API_KEY_SECRET_ARN } from '../../packages/resources';
import { CLOUD_TRAIL_SNS_TOPIC_ARN, stack } from '../../packages/shared';
const BASE_NAME = 'notification_retention_handler';
const REPO_ROOT = '../../..';
+// Pinned public x86_64 layer, verified in us-east-1. Supports OTLP gRPC.
+const DATADOG_EXTENSION_ARN =
+ 'arn:aws:lambda:us-east-1:464622532012:layer:Datadog-Extension:101';
type RetentionEnvVars = {
DATABASE_URL: pulumi.Output<string> | string;
ENVIRONMENT: string;
RUST_LOG: string;
+ DD_SERVICE: string;
+ DD_ENV: string;
+ DD_SITE: string;
+ DD_API_KEY_SECRET_ARN: pulumi.Output<string>;
+ DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: string;
+ DD_SERVERLESS_LOGS_ENABLED: string;
};
type Args = {
@@ -56,6 +66,23 @@ export class NotificationRetention extends pulumi.ComponentResource {
},
{ parent: this }
);
+ const datadogSecretPolicy = new aws.iam.RolePolicy(
+ `${BASE_NAME}-datadog-key`,
+ {
+ role: role.id,
+ policy: pulumi.jsonStringify({
+ Version: '2012-10-17',
+ Statement: [
+ {
+ Effect: 'Allow',
+ Action: 'secretsmanager:GetSecretValue',
+ Resource: DATADOG_API_KEY_SECRET_ARN,
+ },
+ ],
+ }),
+ },
+ { parent: this }
+ );
this.lambda = new Lambda<RetentionEnvVars>(
`${BASE_NAME}-lambda`,
{
@@ -66,9 +93,17 @@ export class NotificationRetention extends pulumi.ComponentResource {
DATABASE_URL: databaseUrl,
ENVIRONMENT: stack,
RUST_LOG: 'info',
+ DD_SERVICE: 'notification-retention',
+ DD_ENV: stack,
+ DD_SITE: 'us5.datadoghq.com',
+ DD_API_KEY_SECRET_ARN: DATADOG_API_KEY_SECRET_ARN,
+ DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: '127.0.0.1:4317',
+ // Keep the existing CloudWatch → Firehose log path; avoid duplicates.
+ DD_SERVERLESS_LOGS_ENABLED: 'false',
},
role,
vpc,
+ layers: [DATADOG_EXTENSION_ARN],
memorySize: 256,
timeout: 120,
reservedConcurrentExecutions: 1,
@@ -77,6 +112,7 @@ export class NotificationRetention extends pulumi.ComponentResource {
{ parent: this }
).lambda;
+ // Ensure the extension can fetch its key before EventBridge can invoke it.
const rule = new aws.cloudwatch.EventRule(
`${BASE_NAME}-schedule`,
{
@@ -84,7 +120,7 @@ export class NotificationRetention extends pulumi.ComponentResource {
state: enabled ? 'ENABLED' : 'DISABLED',
tags,
},
- { parent: this }
+ { parent: this, dependsOn: [datadogSecretPolicy] }
);
new aws.cloudwatch.EventTarget(
`${BASE_NAME}-target`,
diff --git a/services/notification_retention_handler/AGENTS.md b/services/notification_retention_handler/AGENTS.md
new file mode 100644
index 0000000000..b26ce493f3
--- /dev/null
+++ b/services/notification_retention_handler/AGENTS.md
@@ -0,0 +1,34 @@
+# notification_retention_handler
+
+## What it is
+
+Internal scheduled notification cleanup. The owning notification domain enforces
+30-day retention and bounded transactions; this binary only composes it for Lambda.
+
+## Start here
+
+- `src/main.rs`: one-connection pool, retained telemetry provider, post-invocation
+ flush. Close handler spans before flushing; never shut down warm providers.
+- `src/handler.rs`: explicit dry-run default, invocation deadline and request ID.
+- `infra/stacks/notification-service/retention.ts`: disabled schedule, pinned
+ Datadog extension, scoped key access and nonduplicated log forwarding.
+
+## Checks
+
+- `cargo test -p notification_retention_handler` (`SQLX_OFFLINE` unset).
+- `cargo test -p notification -p macro_entrypoint` for policy and flush behavior.
+- From `infra/`: `bun test stacks/notification-service/retention.test.ts`.
+
+## Gotchas
+
+- `{}` is read-only; `{"dry_run":false}` permanently deletes hosted data.
+- Telemetry failure preserves the operation result; it must not trigger a retry
+ of already-committed deletion. Flush time comes from reserved handler headroom.
+- Unit tests cannot prove the deployed extension exported traces. Verify an
+ explicitly approved dry run in Datadog before production cleanup.
+- The schedule stays disabled until the rollout is explicitly approved.
+
+## Related
+
+- `docs/runbooks/notification-retention.md`
+- `.agents/rules/observability.md`
diff --git a/services/notification_retention_handler/CLAUDE.md b/services/notification_retention_handler/CLAUDE.md
new file mode 120000
index 0000000000..47dc3e3d86
--- /dev/null
+++ b/services/notification_retention_handler/CLAUDE.md
@@ -0,0 +1 @@
+AGENTS.md
\ No newline at end of file
diff --git a/services/notification_retention_handler/src/handler.rs b/services/notification_retention_handler/src/handler.rs
index 1390af3244..2c37664644 100644
--- a/services/notification_retention_handler/src/handler.rs
+++ b/services/notification_retention_handler/src/handler.rs
@@ -29,7 +29,12 @@ impl CleanupEvent {
}
}
-#[tracing::instrument(err, skip(service, event))]
+#[tracing::instrument(
+ name = "notification.retention.invoke",
+ err,
+ skip(service, event),
+ fields(faas.invocation_id = %event.context.request_id, dry_run = event.payload.dry_run)
+)]
pub(crate) async fn handle<R: NotificationRetentionRepository>(
service: &NotificationRetentionService<R>,
event: LambdaEvent<CleanupEvent>,
diff --git a/services/notification_retention_handler/src/main.rs b/services/notification_retention_handler/src/main.rs
index 79fb641ee0..57cfe480ae 100644
--- a/services/notification_retention_handler/src/main.rs
+++ b/services/notification_retention_handler/src/main.rs
@@ -20,7 +20,7 @@ env_vars! {
#[tokio::main]
async fn main() -> Result<(), Error> {
- MacroEntrypoint::default().init();
+ let telemetry = Arc::new(MacroEntrypoint::default().init());
let database_url = DatabaseUrl::new()?;
// Bound preview queries as well as deletes. One maintenance connection per
// process, and reserved Lambda concurrency of one, isolate foreground work.
@@ -41,7 +41,18 @@ async fn main() -> Result<(), Error> {
));
run(service_fn(move |event| {
let service = Arc::clone(&service);
- async move { handler::handle(&service, event).await }
+ let telemetry = Arc::clone(&telemetry);
+ async move {
+ // `handle` must finish (closing its root span) before flushing.
+ // A telemetry outage must not turn a committed cleanup into a retry.
+ let result = handler::handle(&service, event).await;
+ match tokio::time::timeout(Duration::from_secs(5), telemetry.force_flush()).await {
+ Ok(Ok(())) => {}
+ Ok(Err(error)) => tracing::error!(error = ?error, "notification retention telemetry flush failed"),
+ Err(error) => tracing::error!(error = ?error, "notification retention telemetry flush timed out"),
+ }
+ result
+ }
}))
.await
}
diff --git a/services/notification_service/AGENTS.md b/services/notification_service/AGENTS.md
index f20246957e..350bc184fe 100644
--- a/services/notification_service/AGENTS.md
+++ b/services/notification_service/AGENTS.md
@@ -41,6 +41,10 @@ worker that sends email/mobile-push/websocket notifications.
## Gotchas
+- PostgreSQL delete forwarding emits `notification.delete.publish` spans and
+ logs publication failures. The retention Lambda is a separate binary/role;
+ see `services/notification_retention_handler` and its rollout runbook.
+
- Don't log notification payloads on tracing spans (#7892) — they can carry
sensitive content; log ids/types instead.