Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(notifications): close retention observability gaps (#8116)
Sean Aye91e02eb8f904
diff --git a/Cargo.lock b/Cargo.lockindex e45197cedd..ee6ee7fa26 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -11709,6 +11709,7 @@ dependencies = ["rootcause","rootcause-tracing","serde_json",+ "thiserror 2.0.18","tokio","tracing","tracing-opentelemetry",@@ -13195,6 +13196,7 @@ dependencies = ["thiserror 2.0.18","tokio","tokio-retry",+ "tokio-util","tower 0.5.3","tracing","url",diff --git a/crates/kafka_util/AGENTS.md b/crates/kafka_util/AGENTS.mdnew file mode 100644index 0000000000..80e9a73a0c--- /dev/null+++ b/crates/kafka_util/AGENTS.md@@ -0,0 +1,32 @@+# kafka_util++## What it is++Shared environment-aware Kafka transports: plaintext locally, MSK IAM in hosted+environments, with trace propagation and typed grouped/ungrouped consumers.++## Start here++- `src/lib.rs`: group markers, assignment, publish/receive and configuration.+- `src/msk_iam.rs`: OAuth token refresh; statistics observers must preserve it.+- `src/statistics.rs`: opt-in callbacks for ungrouped consumers, without changing+ assignment or commit behavior. Observers must not block or log raw statistics.++## Scope++- Owns transport mechanics, not domain event decoding or retry policy.+- `Ungrouped` always uses manual assignment with automatic commits/stores off.+- Used by `macro_event_broker` and other domain Kafka adapters.++## Checks++- `cargo test -p kafka_util` (`SQLX_OFFLINE` unset).+- The in-process librdkafka mock broker test verifies statistics and uncommitted+ application offsets; no external Kafka/Docker setup is needed.++## Gotchas++- Statistics callbacks run while polling, not independently of consumption.+ Health reporters must detect stale samples rather than repeating old zero lag.+- Consumer-group lag cannot describe an ungrouped consumer; use application+ positions and explicitly handle unknown offsets and prefetched records.diff --git a/crates/kafka_util/CLAUDE.md b/crates/kafka_util/CLAUDE.mdnew file mode 120000index 0000000000..47dc3e3d86--- /dev/null+++ b/crates/kafka_util/CLAUDE.md@@ -0,0 +1 @@+AGENTS.md\ No newline at end of filediff --git a/crates/kafka_util/src/lib.rs b/crates/kafka_util/src/lib.rsindex 3251e1d02b..e88bbfb33e 100644--- a/crates/kafka_util/src/lib.rs+++ b/crates/kafka_util/src/lib.rs@@ -15,7 +15,9 @@ use std::time::Duration;use either::Either;use macro_env::Environment;use opentelemetry::propagation::{Extractor, Injector};-use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, StreamConsumer};+use rdkafka::consumer::{+ CommitMode, Consumer, ConsumerContext, DefaultConsumerContext, StreamConsumer,+};use rdkafka::error::{KafkaError, KafkaResult};use rdkafka::message::{BorrowedMessage, Header, Headers, Message, OwnedHeaders};use rdkafka::producer::{FutureProducer, FutureRecord};@@ -24,8 +26,11 @@ use tracing_opentelemetry::OpenTelemetrySpanExt as _;use uuid::Uuid;pub use msk_iam::{MskIamClientContext, configure_sasl_iam};+pub use statistics::ConsumerStatistics;+use statistics::StatisticsContext;mod msk_iam;+mod statistics;const UNGROUPED_GROUP_PREFIX: &str = "macro-event-broker-independent";const MESSAGE_TIMEOUT_MS: &str = "5000";@@ -145,7 +150,12 @@ pub enum KafkaProducerError {}/// Underlying Kafka consumer transport selected from the runtime environment.-struct ConsumerTransport(Either<StreamConsumer, StreamConsumer<MskIamClientContext>>);+struct ConsumerTransport(+ Either<+ StreamConsumer<StatisticsContext<DefaultConsumerContext>>,+ StreamConsumer<StatisticsContext<MskIamClientContext>>,+ >,+);/// Underlying Kafka producer transport selected from the runtime environment.#[derive(Clone)]@@ -243,14 +253,25 @@ fn ungrouped_config(brokers: &str) -> ClientConfig {fn create_consumer_from_env<T>(config: ClientConfig,+ observer: Option<ConsumerStatistics>,) -> Result<KafkaEventConsumer<T>, KafkaConsumerError> {let consumer = match Environment::new_or_prod() {- Environment::Local => Either::Left(config.create().map_err(KafkaConsumerError::Plaintext)?),+ Environment::Local => Either::Left(+ config+ .create_with_context(StatisticsContext {+ inner: DefaultConsumerContext,+ observer,+ })+ .map_err(KafkaConsumerError::Plaintext)?,+ ),Environment::Develop | Environment::Production => {let config = configure_sasl_iam(config);Either::Right(config- .create_with_context(MskIamClientContext::from_env())+ .create_with_context(StatisticsContext {+ inner: MskIamClientContext::from_env(),+ observer,+ }).map_err(KafkaConsumerError::MskIam)?,)}@@ -398,7 +419,7 @@ impl<T: GroupName> KafkaEventConsumer<T> {/// Creates a named-group consumer, selecting plaintext or MSK IAM transport/// from the runtime environment.pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {- create_consumer_from_env(grouped_config::<T>(brokers))+ create_consumer_from_env(grouped_config::<T>(brokers), None)}/// Subscribes the consumer to exactly the provided topics.@@ -422,7 +443,18 @@ impl KafkaEventConsumer<Ungrouped> {////// Call [`Self::assign_topics`] before receiving messages.pub fn from_env(brokers: &str) -> Result<Self, KafkaConsumerError> {- create_consumer_from_env(ungrouped_config(brokers))+ create_consumer_from_env(ungrouped_config(brokers), None)+ }++ /// Creates an independent consumer with statistics sampled every ten seconds.+ /// The observer does not join a group, store offsets, or issue broker calls.+ pub fn from_env_with_statistics(+ brokers: &str,+ observer: ConsumerStatistics,+ ) -> Result<Self, KafkaConsumerError> {+ let mut config = ungrouped_config(brokers);+ config.set("statistics.interval.ms", "10000");+ create_consumer_from_env(config, Some(observer))}/// Manually assigns every current partition of `topics` at `initial_offset`.diff --git a/crates/kafka_util/src/statistics.rs b/crates/kafka_util/src/statistics.rsnew file mode 100644index 0000000000..c8ed7e3001--- /dev/null+++ b/crates/kafka_util/src/statistics.rs@@ -0,0 +1,51 @@+//! Opt-in statistics delivery without changing assignment or commit behavior.++use std::sync::Arc;++use rdkafka::{+ client::{ClientContext, OAuthToken},+ config::RDKafkaLogLevel,+ consumer::ConsumerContext,+ error::KafkaError,+ statistics::Statistics,+};++/// A nonblocking observer of librdkafka statistics. Called while polling the+/// consumer; observers must not perform I/O or log entire statistics payloads.+pub type ConsumerStatistics = Arc<dyn Fn(&Statistics) + Send + Sync>;++pub(crate) struct StatisticsContext<C> {+ pub(crate) inner: C,+ pub(crate) observer: Option<ConsumerStatistics>,+}++impl<C: ClientContext> ClientContext for StatisticsContext<C> {+ const ENABLE_REFRESH_OAUTH_TOKEN: bool = C::ENABLE_REFRESH_OAUTH_TOKEN;++ fn log(&self, level: RDKafkaLogLevel, facility: &str, message: &str) {+ self.inner.log(level, facility, message);+ }++ fn error(&self, error: KafkaError, reason: &str) {+ self.inner.error(error, reason);+ }++ fn stats(&self, statistics: Statistics) {+ if let Some(observer) = &self.observer {+ observer(&statistics);+ }+ }++ fn generate_oauth_token(+ &self,+ config: Option<&str>,+ ) -> Result<OAuthToken, Box<dyn std::error::Error>> {+ self.inner.generate_oauth_token(config)+ }+}++// Both supported inner contexts use the default rebalance/commit callbacks.+impl<C: ConsumerContext> ConsumerContext for StatisticsContext<C> {}++#[cfg(test)]+mod test;diff --git a/crates/kafka_util/src/statistics/test.rs b/crates/kafka_util/src/statistics/test.rsnew file mode 100644index 0000000000..9338248d2a--- /dev/null+++ b/crates/kafka_util/src/statistics/test.rs@@ -0,0 +1,147 @@+use super::*;+use std::sync::{+ Mutex,+ atomic::{AtomicUsize, Ordering},+};++use rdkafka::{+ ClientConfig,+ consumer::{Consumer, DefaultConsumerContext, StreamConsumer},+ mocking::MockCluster,+ producer::{FutureProducer, FutureRecord},+ statistics::Statistics,+};++#[test]+fn observer_receives_statistics_once_without_default_payload_logging() {+ let calls = Arc::new(AtomicUsize::new(0));+ let observed = calls.clone();+ let context = StatisticsContext {+ inner: DefaultConsumerContext,+ observer: Some(Arc::new(move |stats| {+ assert_eq!(stats.rxmsgs, 12);+ observed.fetch_add(1, Ordering::SeqCst);+ })),+ };+ context.stats(Statistics {+ rxmsgs: 12,+ ..Default::default()+ });+ assert_eq!(calls.load(Ordering::SeqCst), 1);+}++struct AuthContext;+impl ClientContext for AuthContext {+ const ENABLE_REFRESH_OAUTH_TOKEN: bool = true;+ fn generate_oauth_token(+ &self,+ config: Option<&str>,+ ) -> Result<OAuthToken, Box<dyn std::error::Error>> {+ assert_eq!(config, Some("test-config"));+ Ok(OAuthToken {+ token: "test-only".into(),+ principal_name: "test-principal".into(),+ lifetime_ms: 123,+ })+ }+}++#[test]+fn wrapping_statistics_preserves_iam_token_refresh() {+ assert!(StatisticsContext::<AuthContext>::ENABLE_REFRESH_OAUTH_TOKEN);+ assert!(!StatisticsContext::<DefaultConsumerContext>::ENABLE_REFRESH_OAUTH_TOKEN);+ let context = StatisticsContext {+ inner: AuthContext,+ observer: None,+ };+ let token = context+ .generate_oauth_token(Some("test-config"))+ .expect("forwarded auth callback");+ assert_eq!(token.principal_name, "test-principal");+ assert_eq!(token.lifetime_ms, 123);+}++#[tokio::test]+async fn ungrouped_consumer_reports_application_offsets_without_commits() {+ let cluster = MockCluster::new(1).expect("mock broker");+ cluster.create_topic("stats-test", 1, 1).expect("topic");+ let (send, receive) = tokio::sync::oneshot::channel();+ let sender = Mutex::new(Some(send));+ let observer: ConsumerStatistics = Arc::new(move |stats| {+ let Some(partition) = stats+ .topics+ .get("stats-test")+ .and_then(|t| t.partitions.get(&0))+ else {+ return;+ };+ if partition.app_offset == 1 && partition.hi_offset >= 1 {+ if let Some(sender) = sender.lock().expect("sender mutex").take() {+ let broker_connected = stats+ .brokers+ .values()+ .any(|broker| broker.nodeid == partition.broker && broker.state == "UP");+ sender+ .send((partition.clone(), broker_connected))+ .expect("live receiver");+ }+ }+ });+ let mut config = crate::ungrouped_config(&cluster.bootstrap_servers());+ config.set("statistics.interval.ms", "10");+ assert_eq!(config.get("enable.auto.commit"), Some("false"));+ assert_eq!(config.get("enable.auto.offset.store"), Some("false"));+ let consumer: StreamConsumer<_> = config+ .create_with_context(StatisticsContext {+ inner: DefaultConsumerContext,+ observer: Some(observer),+ })+ .expect("consumer");+ let mut assignment = rdkafka::TopicPartitionList::new();+ assignment+ .add_partition_offset("stats-test", 0, rdkafka::Offset::Beginning)+ .expect("assignment");+ consumer.assign(&assignment).expect("manual assignment");+ let producer: FutureProducer = ClientConfig::new()+ .set("bootstrap.servers", cluster.bootstrap_servers())+ .create()+ .expect("producer");+ producer+ .send(+ FutureRecord::to("stats-test").key("test").payload("test"),+ std::time::Duration::from_secs(5),+ )+ .await+ .expect("publish");+ drop(+ tokio::time::timeout(std::time::Duration::from_secs(5), consumer.recv())+ .await+ .expect("receive deadline")+ .expect("message"),+ );++ // Polling services callbacks even while no records are being published.+ let (sample, broker_connected) =+ tokio::time::timeout(std::time::Duration::from_secs(5), async {+ tokio::select! {+ sample = receive => sample.expect("statistics callback"),+ _ = consumer.recv() => panic!("no second record was published"),+ }+ })+ .await+ .expect("statistics deadline");+ assert_eq!(sample.app_offset, 1);+ assert_eq!(sample.hi_offset, 1);+ assert!(+ sample.desired,+ "manual assignments must appear in health snapshots"+ );+ assert!(+ broker_connected,+ "the snapshot must identify its active broker"+ );+ assert!(+ sample.committed_offset < 0,+ "no durable offset was committed"+ );+}diff --git a/crates/macro_entrypoint/AGENTS.md b/crates/macro_entrypoint/AGENTS.mdindex 691c118d0a..a8a873bfb0 100644--- a/crates/macro_entrypoint/AGENTS.md+++ b/crates/macro_entrypoint/AGENTS.md@@ -11,7 +11,9 @@ nearly every Rust service's `main.rs` calls.- `src/lib.rs`: `MacroEntrypoint::init` — builds the tracing subscriber stackper `Environment` (local / develop / production), returning`InitializedEntrypoint` as proof of initialization; call `.shutdown()`- before exit to flush OTel.+ before exit to flush OTel. Serverless handlers retain it and await+ `.force_flush()` after invocation spans close, without shutting down warm+ providers. The caller must bound flush time and preserve its operation result.- `src/datadog_fmt.rs`: JSON log format injecting `dd.trace_id`/`dd.span_id`for prod/develop trace-log correlation.- `src/grafana.rs`: a second, best-effort span exporter that mirrors spans todiff --git a/crates/macro_entrypoint/Cargo.toml b/crates/macro_entrypoint/Cargo.tomlindex 5cb6a46b88..fea7f6ea0c 100644--- a/crates/macro_entrypoint/Cargo.toml+++ b/crates/macro_entrypoint/Cargo.toml@@ -14,6 +14,7 @@ opentelemetry-otlp = { workspace = true }rootcause = { workspace = true }rootcause-tracing = { workspace = true }serde_json = { workspace = true }+thiserror = { workspace = true }tokio = { workspace = true, features = ["signal"] }tracing = { workspace = true }tracing-opentelemetry = { workspace = true }diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rsindex b83ba3ae13..7e24273495 100644--- a/crates/macro_entrypoint/src/lib.rs+++ b/crates/macro_entrypoint/src/lib.rs@@ -122,7 +122,41 @@ pub struct InitializedEntrypoint {logger_provider: Option<SdkLoggerProvider>,}+/// Failure to flush telemetry before a serverless runtime freezes.+#[derive(Debug, thiserror::Error)]+pub enum TelemetryFlushError {+ /// The blocking flush task could not finish.+ #[error("telemetry flush task failed")]+ Task(#[from] tokio::task::JoinError),+ /// The trace exporter could not flush.+ #[error("trace flush failed")]+ Traces(#[source] opentelemetry_sdk::error::OTelSdkError),+ /// The log exporter could not flush.+ #[error("log flush failed")]+ Logs(#[source] opentelemetry_sdk::error::OTelSdkError),+}+impl InitializedEntrypoint {+ /// Flush completed telemetry before a serverless runtime freezes, without+ /// shutting down providers needed by subsequent warm invocations.+ ///+ /// Run the blocking SDK flush off the async executor so gRPC can progress+ /// even on a single-threaded runtime. Call after invocation spans close.+ pub async fn force_flush(&self) -> Result<(), TelemetryFlushError> {+ let tracer = self.tracer_provider.clone();+ let logger = self.logger_provider.clone();+ let (traces, logs) = tokio::task::spawn_blocking(move || {+ (+ tracer.map(|provider| provider.force_flush()),+ logger.map(|provider| provider.force_flush()),+ )+ })+ .await?;+ traces.transpose().map_err(TelemetryFlushError::Traces)?;+ logs.transpose().map_err(TelemetryFlushError::Logs)?;+ Ok(())+ }+/// Gracefully shut down the OpenTelemetry providers./// This should be called before the application exits to ensure all traces and logs are flushed.pub fn shutdown(&self) {diff --git a/crates/macro_entrypoint/src/test.rs b/crates/macro_entrypoint/src/test.rsindex 5598848d41..d8cdc75949 100644--- a/crates/macro_entrypoint/src/test.rs+++ b/crates/macro_entrypoint/src/test.rs@@ -44,6 +44,89 @@ fn test_tracer_provider() -> (InMemorySpanExporter, SdkTracerProvider) {(exporter, provider)}+#[tokio::test]+async fn force_flush_exports_short_invocations_and_preserves_warm_provider() {+ use opentelemetry::trace::Tracer as _;++ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_batch_exporter(exporter.clone())+ .build();+ let tracer = provider.tracer("retention-test");+ let entrypoint = InitializedEntrypoint {+ tracer_provider: Some(provider),+ logger_provider: None,+ };++ tracer.in_span("first-invocation", |_| {});+ entrypoint+ .force_flush()+ .await+ .expect("first flush succeeds");+ let spans = exporter.get_finished_spans().expect("exported spans");+ assert_eq!(spans.len(), 1);+ assert_eq!(spans[0].name, "first-invocation");++ tracer.in_span("warm-invocation", |_| {});+ entrypoint.force_flush().await.expect("warm flush succeeds");+ let spans = exporter.get_finished_spans().expect("exported spans");+ assert_eq!(spans.len(), 2);+ assert_eq!(spans[1].name, "warm-invocation");+ entrypoint.shutdown();+}++#[tokio::test]+async fn force_flush_reports_export_failure_instead_of_success() {+ use opentelemetry::trace::Tracer as _;+ use opentelemetry_sdk::{+ error::{OTelSdkError, OTelSdkResult},+ trace::{BatchConfigBuilder, BatchSpanProcessor, SpanData, SpanExporter},+ };++ #[derive(Debug)]+ struct FailingExporter;+ impl SpanExporter for FailingExporter {+ async fn export(&self, _batch: Vec<SpanData>) -> OTelSdkResult {+ Err(OTelSdkError::InternalFailure(+ "test export failure".to_owned(),+ ))+ }+ }+ let processor = BatchSpanProcessor::builder(FailingExporter)+ .with_batch_config(+ BatchConfigBuilder::default()+ .with_scheduled_delay(std::time::Duration::from_secs(3600))+ .build(),+ )+ .build();+ let provider = SdkTracerProvider::builder()+ .with_span_processor(processor)+ .build();+ provider+ .tracer("test")+ .in_span("completed-invocation", |_| {});+ let entrypoint = InitializedEntrypoint {+ tracer_provider: Some(provider),+ logger_provider: None,+ };+ assert!(matches!(+ entrypoint.force_flush().await,+ Err(TelemetryFlushError::Traces(_))+ ));+ entrypoint.shutdown();+}++#[tokio::test]+async fn force_flush_without_exporters_is_a_noop() {+ InitializedEntrypoint {+ tracer_provider: None,+ logger_provider: None,+ }+ .force_flush()+ .await+ .expect("no exporter needs flushing");+}+#[test]fn rootcause_tracing_subscriber_captures_span_chain() {static INSTALL_ROOTCAUSE_HOOKS: Once = Once::new();diff --git a/crates/notification/AGENTS.md b/crates/notification/AGENTS.mdindex d0463d574a..c75803149d 100644--- a/crates/notification/AGENTS.md+++ b/crates/notification/AGENTS.md@@ -20,22 +20,25 @@ through the same service.and every port an adapter must implement.- `src/inbound/http.rs`, `src/inbound/worker.rs`,`src/inbound/notification_events_listener.rs`: HTTP routes, the queue- worker, and the Kafka event listener that trigger notification creation.+ worker, and the PostgreSQL LISTEN adapter that forwards delete events.- `src/outbound/`: one file per channel/concern — `fanout_realtime.rs` +`kafka_realtime.rs` (live WebSocket fan-out), `mobile.rs` + `sns_endpoint.rs`(push), `email.rs` + `digest_batcher.rs` (digest emails), `repository.rs`- (Postgres via `notification_db_client`), `item_preferences.rs` (snooze/mute).+ (owned Postgres queries), `item_preferences.rs` (snooze/mute).+- `outbound/notification_consumer/health.rs`: periodic per-instance Kafka+ backlog and telemetry-health logs; offsets stay uncommitted.+- `domain/retention.rs` + `outbound/retention.rs`: bounded 30-day cleanup.## Scope- Owns: notification creation, read/unread/snooze state transitions,per-channel delivery, and device registration.- Not here: the lifecycle state enum itself and its transition rules →- `notification_state`; the Postgres queries → `notification_db_client`.+ `notification_state`.## Boundaries-- Depends on: `notification_db_client`, `notification_state`, `rate_limit`,+- Depends on: `notification_state`, `kafka_util`, `rate_limit`,`macro_authorization`.- Used by: most feature crates that need to notify a user (`properties`,`channels`, `reminders`, `teams`, `messages`, `calendar_events`, ...) and@@ -52,9 +55,11 @@ through the same service.## Checks-- `cargo test -p notification` (`SQLX_OFFLINE` unset; needs local MacroDB).-- `cargo test -p notification --features redis-tests` for the Redis-backed- preference paths.+- `cargo test -p notification --all-features` (`SQLX_OFFLINE` unset; needs local+ MacroDB and Redis).+- For test edits, also run `RUSTFLAGS=-Dwarnings cargo check -p notification+ --all-features --tests` against the live schema. Ordinary clippy does not+ compile test-only code, while CI rejects its warnings.## Gotchas@@ -62,6 +67,8 @@ through the same service.`schema`); building without the feature a caller needs silently drops themodule instead of failing loudly — check `Cargo.toml` before assuming anadapter is compiled in.+- Independent Kafka consumers have no durable group lag. Health logs distinguish+ stale/unknown measurements from zero; they are not client-delivery ACKs.- Notification payloads must not be logged on tracing spans (see the`fix(notification): stop recording notification payloads` history).diff --git a/crates/notification/Cargo.toml b/crates/notification/Cargo.tomlindex 77481d5d01..877d6001cf 100644--- a/crates/notification/Cargo.toml+++ b/crates/notification/Cargo.toml@@ -79,6 +79,7 @@ strum = { workspace = true }thiserror = { workspace = true }tokio = { workspace = true, features = ["time"] }tokio-retry = { workspace = true }+tokio-util = { workspace = true, features = ["rt"] }tracing = { workspace = true }url = { workspace = true }utoipa = { workspace = true, optional = true }diff --git a/crates/notification/src/inbound/notification_events_listener.rs b/crates/notification/src/inbound/notification_events_listener.rsindex d36b60c61b..8dace43fb9 100644--- a/crates/notification/src/inbound/notification_events_listener.rs+++ b/crates/notification/src/inbound/notification_events_listener.rs@@ -1,5 +1,8 @@//! Listener for notification database events.+#[cfg(test)]+mod test;+use std::time::Duration;use macro_user_id::cowlike::CowLike;@@ -39,6 +42,11 @@ where}}+ #[tracing::instrument(+ name = "notification.delete.publish",+ skip_all,+ fields(notification_id, recipient_count, published)+ )]async fn handle_payload(&self, payload: &str) {let event = match serde_json::from_str::<NotificationDatabaseEvent>(payload) {Ok(event) => event,@@ -53,17 +61,22 @@ wherenotification_id,user_ids,} => {+ tracing::Span::current()+ .record("notification_id", tracing::field::display(notification_id));let user_ids = user_ids.into_iter().filter_map(|user_id| match MacroUserIdStr::parse_from_str(&user_id) {Ok(user_id) => Some(user_id.into_owned()),- Err(err) => {- tracing::warn!(error = ?err, user_id, "invalid user id in notification delete event");+ Err(_) => {+ // User IDs contain email addresses; don't include invalid+ // values (or parser errors echoing them) in telemetry.+ tracing::warn!("invalid user id in notification delete event");None}}).collect::<Vec<_>>();+ tracing::Span::current().record("recipient_count", user_ids.len());if user_ids.is_empty() {return;}@@ -75,9 +88,11 @@ where}),};- if let Err(err) = self.realtime.publish_updates(&payload).await {+ let result = self.realtime.publish_updates(&payload).await;+ tracing::Span::current().record("published", result.is_ok());+ result.inspect_err(|err| {tracing::warn!(error = ?err, "failed to publish notification delete realtime update");- }+ }).ok();}}}diff --git a/crates/notification/src/inbound/notification_events_listener/test.rs b/crates/notification/src/inbound/notification_events_listener/test.rsnew file mode 100644index 0000000000..0f1a81c75d--- /dev/null+++ b/crates/notification/src/inbound/notification_events_listener/test.rs@@ -0,0 +1,83 @@+use super::*;+use rootcause::Report;+use std::sync::Mutex;++struct UnusedReceiver;+impl NotificationEventsReceiver for UnusedReceiver {+ async fn receive(&mut self) -> Result<String, Report> {+ unreachable!("tests feed the listener directly")+ }+}++#[derive(Default)]+struct Publisher {+ attempts: Mutex<Vec<(Uuid, usize)>>,+ fail: bool,+}++impl NotificationRealtimePublisher for Publisher {+ async fn publish_updates(&self, payload: &NotificationStatusPayload<'_>) -> Result<(), Report> {+ let NotificationStatusPayload::NotificationForUsers { users, update } = payload else {+ panic!("expected recipient-scoped deletion");+ };+ let NotificationDelete::Delete { id } = **update;+ self.attempts+ .lock()+ .expect("attempts")+ .push((id, users.len()));+ if self.fail {+ return Err(rootcause::report!("test publication failure"));+ }+ Ok(())+ }+}++#[tokio::test]+async fn instrumentation_preserves_valid_recipients_and_skips_invalid_values() {+ let listener = NotificationEventsListener::new(UnusedReceiver, Publisher::default());+ listener+ .handle_payload(+ r#"{+ "type":"user_notification_deletes",+ "notificationId":"00000000-0000-7000-8000-000000000001",+ "userIds":["macro|[email protected]","invalid"]+ }"#,+ )+ .await;+ assert_eq!(+ *listener.realtime.attempts.lock().expect("attempts"),+ vec![(+ Uuid::parse_str("00000000-0000-7000-8000-000000000001").expect("id"),+ 1+ )]+ );+}++#[tokio::test]+async fn failed_publication_does_not_prevent_processing_the_next_event() {+ let listener = NotificationEventsListener::new(+ UnusedReceiver,+ Publisher {+ fail: true,+ ..Default::default()+ },+ );+ for id in [+ "00000000-0000-7000-8000-000000000001",+ "00000000-0000-7000-8000-000000000002",+ ] {+ listener+ .handle_payload(+ &serde_json::json!({+ "type":"user_notification_deletes", "notificationId":id,+ "userIds":["macro|[email protected]"]+ })+ .to_string(),+ )+ .await;+ }+ assert_eq!(+ listener.realtime.attempts.lock().expect("attempts").len(),+ 2+ );+}diff --git a/crates/notification/src/outbound/notification_consumer.rs b/crates/notification/src/outbound/notification_consumer.rsindex 5b826d075a..df2526f99a 100644--- a/crates/notification/src/outbound/notification_consumer.rs+++ b/crates/notification/src/outbound/notification_consumer.rs@@ -4,9 +4,12 @@//! assigns all `macro.notifications` partitions without joining a durable consumer group. It does//! not commit offsets.+mod health;#[cfg(test)]mod test;+use health::ConsumerHealth;+use std::{marker::PhantomData, time::Duration};use kafka_util::{InitialOffset, KafkaEventConsumer, Ungrouped};@@ -44,6 +47,7 @@ whereT: Clone + Serialize + DeserializeOwned + Send + Sync + 'static,{consumer: NotificationEventConsumer,+ _health: ConsumerHealth,payload: PhantomData<fn() -> T>,}@@ -54,7 +58,8 @@ where/// Creates a consumer and assigns every current notification topic partition.#[tracing::instrument(fields(brokers), err)]pub fn from_env(brokers: &str) -> Result<Self, Report> {- let consumer = KafkaEventConsumer::<Ungrouped>::from_env(brokers)+ let (observer, health) = ConsumerHealth::start();+ let consumer = KafkaEventConsumer::<Ungrouped>::from_env_with_statistics(brokers, observer).context("failed to create independent notification topic consumer")?;let consumer =IndependentKafkaConsumer::new(consumer, InitialOffset::Latest, TOPIC_METADATA_TIMEOUT)@@ -67,6 +72,7 @@ whereOk(Self {consumer: NotificationEventConsumer::new(consumer),+ _health: health,payload: PhantomData,})}diff --git a/crates/notification/src/outbound/notification_consumer/health.rs b/crates/notification/src/outbound/notification_consumer/health.rsnew file mode 100644index 0000000000..597323a6ce--- /dev/null+++ b/crates/notification/src/outbound/notification_consumer/health.rs@@ -0,0 +1,219 @@+//! Per-process telemetry for the deliberately ungrouped notification consumer.+//! A separate timer reports stale callbacks even if application consumption stalls.++use std::{sync::Arc, time::Duration};++use chrono::Utc;+use kafka_util::ConsumerStatistics;+use rdkafka::statistics::Statistics;+use tokio::{+ sync::watch,+ time::{Instant, MissedTickBehavior},+};+use tokio_util::{sync::CancellationToken, task::TaskTracker};+use uuid::Uuid;++const TOPIC: &str = "macro.notifications";+const REPORT_INTERVAL: Duration = Duration::from_secs(10);+const STALE_AFTER: Duration = Duration::from_secs(30);++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]+struct Backlog {+ known_partitions: u32,+ unknown_partitions: u32,+ max_offset_backlog: u64,+ total_offset_backlog: u64,+ prefetched_messages: u64,+ connected_brokers: usize,+ disconnected_partitions: u32,+}++fn backlog(stats: &Statistics) -> Option<Backlog> {+ let topic = stats.topics.get(TOPIC)?;+ let mut result = Backlog {+ connected_brokers: stats+ .brokers+ .values()+ .filter(|b| b.nodeid >= 0 && b.state == "UP")+ .count(),+ ..Backlog::default()+ };+ for partition in topic+ .partitions+ .values()+ .filter(|p| p.partition >= 0 && p.desired)+ {+ if !stats+ .brokers+ .values()+ .any(|broker| broker.nodeid == partition.broker && broker.state == "UP")+ {+ // Another partition's healthy connection cannot make this cached+ // high watermark trustworthy while its own broker is unavailable.+ result.disconnected_partitions += 1;+ }+ // app_offset is next-to-deliver, not committed_offset (which is always+ // invalid here). Before the first delivery use next-to-fetch plus the+ // prefetched queue, so an idle latest-start partition can report zero.+ let (position, queued) = if partition.app_offset >= 0 {+ (partition.app_offset, 0)+ } else {+ (partition.next_offset, partition.fetchq_cnt)+ };+ let pending = (position >= 0 && partition.hi_offset >= position && queued >= 0)+ .then(|| (partition.hi_offset - position).checked_add(queued))+ .flatten()+ .and_then(|value| u64::try_from(value).ok());+ if let Some(pending) = pending {+ result.known_partitions += 1;+ result.max_offset_backlog = result.max_offset_backlog.max(pending);+ result.total_offset_backlog = result.total_offset_backlog.saturating_add(pending);+ } else {+ result.unknown_partitions += 1;+ }+ if let Ok(queued) = u64::try_from(partition.fetchq_cnt) {+ result.prefetched_messages = result.prefetched_messages.saturating_add(queued);+ }+ }+ Some(result)+}++#[derive(Clone, Copy)]+struct Observation {+ received_at: Instant,+ sampled_at: i64,+ backlog: Option<Backlog>,+}++#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::AsRefStr)]+#[strum(serialize_all = "snake_case")]+enum Health {+ Starting,+ Healthy,+ Unavailable,+ Disconnected,+ Stale,+}++fn health(+ observation: Option<Observation>,+ started_at: Instant,+ now: Instant,+ unix_now: i64,+) -> Health {+ let starting = now.duration_since(started_at) < STALE_AFTER;+ let Some(observation) = observation else {+ return if starting {+ Health::Starting+ } else {+ Health::Unavailable+ };+ };+ // librdkafka queues callbacks while recv is stalled. A newly delivered but+ // old sample is stale too; callback arrival time alone would hide that stall.+ if now.duration_since(observation.received_at) >= STALE_AFTER+ || unix_now+ .saturating_sub(observation.sampled_at)+ .unsigned_abs()+ >= STALE_AFTER.as_secs()+ {+ return Health::Stale;+ }+ let Some(backlog) = observation.backlog else {+ return if starting {+ Health::Starting+ } else {+ Health::Unavailable+ };+ };+ if backlog.connected_brokers == 0 || backlog.disconnected_partitions > 0 {+ return if starting {+ Health::Starting+ } else {+ Health::Disconnected+ };+ }+ if backlog.known_partitions == 0 || backlog.unknown_partitions > 0 {+ return if starting {+ Health::Starting+ } else {+ Health::Unavailable+ };+ }+ Health::Healthy+}++/// Owns the reporter's lifetime; dropping a consumer also stops its telemetry.+pub(super) struct ConsumerHealth {+ cancellation: CancellationToken,+ tasks: TaskTracker,+}++impl Drop for ConsumerHealth {+ fn drop(&mut self) {+ self.cancellation.cancel();+ self.tasks.close();+ }+}++impl ConsumerHealth {+ pub(super) fn start() -> (ConsumerStatistics, Self) {+ let (observed, state) = watch::channel(None);+ let observer: ConsumerStatistics = Arc::new(move |stats| {+ observed.send_replace(Some(Observation {+ received_at: Instant::now(),+ sampled_at: stats.time,+ backlog: backlog(stats),+ }));+ });+ let consumer_instance = Uuid::now_v7();+ let started_at = Instant::now();+ let cancellation = CancellationToken::new();+ let stop = cancellation.clone();+ let tasks = TaskTracker::new();+ tasks.spawn(async move {+ let mut interval = tokio::time::interval(REPORT_INTERVAL);+ interval.set_missed_tick_behavior(MissedTickBehavior::Skip);+ loop {+ tokio::select! {+ biased;+ _ = stop.cancelled() => break,+ _ = interval.tick() => {}+ }+ let observation = *state.borrow();+ let status = health(+ observation,+ started_at,+ Instant::now(),+ Utc::now().timestamp(),+ );+ let snapshot = observation.and_then(|o| o.backlog);+ // Never advertise a stale/partial observation as a zero backlog.+ let known = snapshot.filter(|_| status == Health::Healthy);+ tracing::info!(+ telemetry_event = "notification_consumer_health",+ %consumer_instance,+ topic = TOPIC,+ consumer_health = status.as_ref(),+ known_partitions = snapshot.map(|s| s.known_partitions),+ unknown_partitions = snapshot.map(|s| s.unknown_partitions),+ disconnected_partitions = snapshot.map(|s| s.disconnected_partitions),+ max_offset_backlog = known.map(|s| s.max_offset_backlog),+ total_offset_backlog = known.map(|s| s.total_offset_backlog),+ prefetched_messages = known.map(|s| s.prefetched_messages),+ "notification consumer health"+ );+ }+ });+ (+ observer,+ Self {+ cancellation,+ tasks,+ },+ )+ }+}++#[cfg(test)]+mod test;diff --git a/crates/notification/src/outbound/notification_consumer/health/test.rs b/crates/notification/src/outbound/notification_consumer/health/test.rsnew file mode 100644index 0000000000..27f805b5b7--- /dev/null+++ b/crates/notification/src/outbound/notification_consumer/health/test.rs@@ -0,0 +1,343 @@+use super::*;+use rdkafka::statistics::{Broker, Partition, Topic};++#[tokio::test]+async fn dropping_consumer_cancels_and_drains_its_tracked_reporter() {+ let (observer, health) = ConsumerHealth::start();+ let cancellation = health.cancellation.clone();+ let tasks = health.tasks.clone();+ drop(health);+ assert!(cancellation.is_cancelled());+ tasks.wait().await;+ assert!(tasks.is_empty());+ // A native callback racing with shutdown is harmless after the receiver closes.+ observer(&Statistics::default());+}++#[test]+fn measures_application_backlog_not_commit_or_fetch_position() {+ let stats = Statistics {+ topics: [(+ TOPIC.to_owned(),+ Topic {+ partitions: [(+ 0,+ Partition {+ partition: 0,+ desired: true,+ app_offset: 10,+ next_offset: 100,+ hi_offset: 100,+ committed_offset: -1001,+ stored_offset: -1001,+ fetchq_cnt: 90,+ ..Default::default()+ },+ )]+ .into(),+ ..Default::default()+ },+ )]+ .into(),+ ..Default::default()+ };+ let result = backlog(&stats).expect("topic present");+ assert_eq!(result.known_partitions, 1);+ assert_eq!(result.max_offset_backlog, 90);+ assert_eq!(result.total_offset_backlog, 90);+ assert_eq!(result.prefetched_messages, 90);+}++#[test]+fn initial_latest_position_includes_prefetched_records_and_idle_partitions() {+ let stats = Statistics {+ topics: [(+ TOPIC.to_owned(),+ Topic {+ partitions: [+ (+ 0,+ Partition {+ partition: 0,+ desired: true,+ app_offset: -1001,+ next_offset: 100,+ hi_offset: 100,+ fetchq_cnt: 0,+ ..Default::default()+ },+ ),+ (+ 1,+ Partition {+ partition: 1,+ desired: true,+ app_offset: -1001,+ next_offset: 100,+ hi_offset: 110,+ fetchq_cnt: 20,+ ..Default::default()+ },+ ),+ ]+ .into(),+ ..Default::default()+ },+ )]+ .into(),+ ..Default::default()+ };+ let result = backlog(&stats).expect("topic present");+ assert_eq!(result.known_partitions, 2);+ assert_eq!(result.unknown_partitions, 0);+ assert_eq!(result.max_offset_backlog, 30);+ assert_eq!(result.total_offset_backlog, 30);+}++#[test]+fn excludes_unassigned_partitions_and_does_not_turn_unknown_offsets_into_zero() {+ let stats = Statistics {+ topics: [(+ TOPIC.to_owned(),+ Topic {+ partitions: [+ (+ -1,+ Partition {+ partition: -1,+ desired: true,+ ..Default::default()+ },+ ),+ (+ 0,+ Partition {+ partition: 0,+ desired: false,+ ..Default::default()+ },+ ),+ (+ 1,+ Partition {+ partition: 1,+ desired: true,+ app_offset: -1001,+ next_offset: -1001,+ hi_offset: -1001,+ ..Default::default()+ },+ ),+ (+ 2,+ Partition {+ partition: 2,+ desired: true,+ app_offset: 100,+ hi_offset: 50,+ ..Default::default()+ },+ ),+ ]+ .into(),+ ..Default::default()+ },+ )]+ .into(),+ ..Default::default()+ };+ let result = backlog(&stats).expect("topic present");+ assert_eq!(result.known_partitions, 0);+ assert_eq!(result.unknown_partitions, 2);+ assert!(backlog(&Statistics::default()).is_none());+}++#[test]+fn broker_connectivity_is_reported_without_bootstrap_or_disconnected_brokers() {+ let stats = Statistics {+ topics: [(TOPIC.to_owned(), Topic::default())].into(),+ brokers: [+ (+ "leader".to_owned(),+ Broker {+ nodeid: 1,+ state: "UP".into(),+ ..Default::default()+ },+ ),+ (+ "down".to_owned(),+ Broker {+ nodeid: 2,+ state: "DOWN".into(),+ ..Default::default()+ },+ ),+ (+ "bootstrap".to_owned(),+ Broker {+ nodeid: -1,+ state: "UP".into(),+ ..Default::default()+ },+ ),+ ]+ .into(),+ ..Default::default()+ };+ assert_eq!(backlog(&stats).expect("topic present").connected_brokers, 1);+}++#[tokio::test(start_paused = true)]+async fn a_connected_broker_does_not_mask_another_partitions_stale_watermark() {+ let stats = Statistics {+ topics: [(+ TOPIC.to_owned(),+ Topic {+ partitions: [+ (+ 0,+ Partition {+ partition: 0,+ desired: true,+ broker: 1,+ app_offset: 10,+ hi_offset: 10,+ ..Default::default()+ },+ ),+ (+ 1,+ Partition {+ partition: 1,+ desired: true,+ broker: 2,+ app_offset: 10,+ hi_offset: 10,+ ..Default::default()+ },+ ),+ ]+ .into(),+ ..Default::default()+ },+ )]+ .into(),+ brokers: [+ (+ "up".to_owned(),+ Broker {+ nodeid: 1,+ state: "UP".into(),+ ..Default::default()+ },+ ),+ (+ "down".to_owned(),+ Broker {+ nodeid: 2,+ state: "DOWN".into(),+ ..Default::default()+ },+ ),+ ]+ .into(),+ ..Default::default()+ };+ let snapshot = backlog(&stats).expect("topic present");+ assert_eq!(snapshot.max_offset_backlog, 0);+ assert_eq!(snapshot.disconnected_partitions, 1);+ let started_at = Instant::now();+ tokio::time::advance(STALE_AFTER).await;+ let now = Instant::now();+ let observation = Observation {+ received_at: now,+ sampled_at: 130,+ backlog: Some(snapshot),+ };+ assert_eq!(+ health(Some(observation), started_at, now, 130),+ Health::Disconnected+ );+}++#[tokio::test(start_paused = true)]+async fn reports_stalls_and_old_queued_callbacks_instead_of_stale_zero_lag() {+ let start = Instant::now();+ assert_eq!(health(None, start, start, 100), Health::Starting);+ let observation = Observation {+ received_at: start,+ sampled_at: 100,+ backlog: Some(Backlog {+ connected_brokers: 1,+ known_partitions: 1,+ ..Default::default()+ }),+ };+ assert_eq!(+ health(Some(observation), start, start, 100),+ Health::Healthy+ );+ tokio::time::advance(STALE_AFTER).await;+ let now = Instant::now();+ assert_eq!(health(None, start, now, 130), Health::Unavailable);+ assert_eq!(health(Some(observation), start, now, 130), Health::Stale);+ assert_eq!(+ health(+ Some(Observation {+ received_at: now,+ ..observation+ }),+ start,+ now,+ 130+ ),+ Health::Stale+ );+ assert_eq!(+ health(+ Some(Observation {+ received_at: now,+ sampled_at: 130,+ ..observation+ }),+ start,+ now,+ 130+ ),+ Health::Healthy+ );+}++#[tokio::test(start_paused = true)]+async fn fresh_callbacks_with_missing_offsets_or_brokers_are_not_healthy() {+ let start = Instant::now();+ tokio::time::advance(STALE_AFTER).await;+ let now = Instant::now();+ let observation = Observation {+ received_at: now,+ sampled_at: 130,+ backlog: None,+ };+ assert_eq!(+ health(Some(observation), start, now, 130),+ Health::Unavailable+ );+ let disconnected = Observation {+ backlog: Some(Backlog::default()),+ ..observation+ };+ assert_eq!(+ health(Some(disconnected), start, now, 130),+ Health::Disconnected+ );+ let unknown = Observation {+ backlog: Some(Backlog {+ connected_brokers: 1,+ unknown_partitions: 1,+ ..Default::default()+ }),+ ..observation+ };+ assert_eq!(health(Some(unknown), start, now, 130), Health::Unavailable);+}diff --git a/docs/runbooks/notification-retention.md b/docs/runbooks/notification-retention.mdindex c77d9c01ca..0609fdd0b2 100644--- a/docs/runbooks/notification-retention.md+++ b/docs/runbooks/notification-retention.md@@ -81,6 +81,80 @@ refetch/reconnect; this change does not add durable tombstones or an outbox.Watch notification-service/Kafka/connection-gateway load during backfill. Do notturn off delete events to speed up the purge.+## Tracing and consumer health++The retention Lambda includes the pinned x86_64 Datadog Extension v101, with an+OTLP gRPC receiver on `127.0.0.1:4317`. The extension retrieves the **existing**+Datadog key by secret ARN; its role can read only that key, not unrelated secrets.+No API key value is copied into the Lambda environment. If that secret uses a+customer-managed KMS key, verify narrowly scoped decrypt permission before deploy.+CloudWatch → Firehose remains the log path; extension log shipping is disabled to+avoid duplicates. Runtime identity is `service:notification-retention`, with+`DD_ENV` set to the stack.++The handler span is `notification.retention.invoke`, searchable by+`faas.invocation_id`. After that span closes, the worker flushes completed spans+with a five-second caller deadline before Lambda can freeze the process. It does+not shut down the provider between warm invocations. Flush failure logs an error+but preserves the cleanup result: committed deletion must not be retried just+because telemetry failed. The existing ten-second invocation headroom covers+this flush. The extension must be checked live after an approved deployment;+unit tests or a green Lambda result do not prove Datadog received its traces.++DSS notification consumers intentionally assign all partitions from latest,+without committing offsets or joining durable consumer groups. Consequently MSK+`MaxOffsetLag` is absent for `macro.notifications`; don't change delivery semantics+just to manufacture that metric.++Each consumer now reports `telemetry_event:notification_consumer_health` every+ten seconds through normal service logs, with an opaque `consumer_instance`:++- `max_offset_backlog` / `total_offset_backlog`: broker high watermark minus the+ next application-delivery position. Before first delivery, use next fetch+ position plus prefetched records. Offset gaps mean this is an approximation,+ not an exact count of outstanding business events.+- `prefetched_messages`: records buffered locally, not just records still on the+ broker. This prevents a full fetch queue from looking caught up.+- `known_partitions`, `unknown_partitions`, `disconnected_partitions`, and+ `consumer_health`: distinguish+ valid measurements from missing offsets, disconnected brokers, and startup.+- A separate reporter detects callbacks older than 30 seconds (including old+ snapshots delivered late), even when the consuming task is stalled. Stale or+ partial backlog measurements are omitted, **never** reported as zero. Dropping+ the consumer stops its reporter.++The manual `infra/stacks/monitoring` stack adds nine log monitors and a rollout+dashboard. Alerts isolate consumer instances; a healthy process cannot mask a+lagging one. Retired-instance alerts resolve on missing data, and a separate+per-environment heartbeat alarm catches loss of all consumer telemetry. Starting+thresholds are warning 100 / critical 1000 for the five-minute average of maximum+partition backlog; tune against real traffic before enabling production cleanup.++The `notification.delete.publish` span covers forwarding a database deletion,+with notification ID, recipient count, and publication outcome, but no recipient+IDs or payload. A log monitor catches failed forwarding. Consumer backlog begins+at Kafka: it does **not** measure time queued before PostgreSQL NOTIFY is forwarded,+or acknowledge client delivery. Existing best-effort loss/reconnect semantics+remain unchanged; no durable outbox is added by this observability change.++Before the next deletion canary, after approved deployments:++1. Deploy the notification-service stack (extension + worker) and DSS (consumer+ telemetry). Keep both cleanup schedules disabled and dry-run enabled.+2. Invoke explicit dry runs and verify the complete invocation and batch/read+ spans in Datadog, with the correct service/env and no exporter/flush errors.+3. Confirm health logs from each running DSS instance, including healthy zero+ backlog when idle; investigate missing/unknown/stale samples rather than+ accepting an empty chart as healthy.+4. Validate and deploy the monitoring stack separately. Verify the incoming+ attributes are parsed as numeric measures (`max_offset_backlog`, `deleted`)+ and grouping attributes (`consumer_instance`, `consumer_health`) in Datadog;+ configure missing measures/facets before relying on aggregation charts. The+ authenticated preview must show nine new monitors and one dashboard, without+ modifying adopted resources.+5. Only then approve a bounded dev deletion canary and inspect DB/API health,+ publisher failures, and consumer backlog draining before considering prod.+## RolloutProduction deletion needs explicit approval and a verified backup/PITR recovery@@ -151,14 +225,23 @@ within 30 days; pagination of currently unbounded consumers remains separate worRun from the repository root with local PostgreSQL and `SQLX_OFFLINE` unset:```bash-cargo test -p notification -p notification_retention_handler -- --test-threads=8+cargo test -p kafka_util -p macro_entrypoint -p notification -p notification_retention_handler -- --test-threads=8+# Only if notification SQL or schema changed:nix develop --command just prepare_dbjust check+just clippy\cd infra-bunx tsc --noEmit --project stacks/notification-service/tsconfig.json+bun test stacks/notification-service/retention.test.ts+bun test stacks/monitoring/monitors/notification-retention.test.ts+bun run check```Tests cover cutoff boundaries, parent-vs-recipient age, all states, cascades,read-only previews, deterministic bounded ordering, retry behavior, skipped locks,transaction advisory locking, timeout rollback, non-UTC sessions, byte-boundedmulti-recipient delete events, invocation caps, and fail-closed event parsing.+Observability tests also cover warm-invocation trace flushing and exporter+failures, a real librdkafka mock broker without offset commits, stale/partial+consumer statistics, broker disconnects, reporter cancellation, and the Pulumi+collector/secret/monitor configuration. Hosted Datadog query validation and+trace delivery still require an authenticated preview and approved deployment.diff --git a/infra/packages/resources/AGENTS.md b/infra/packages/resources/AGENTS.mdnew file mode 100644index 0000000000..4c0116b109--- /dev/null+++ b/infra/packages/resources/AGENTS.md@@ -0,0 +1,26 @@+# resources++## What it is++Shared Pulumi AWS building blocks used by service stacks: gateway target groups,+queues, storage, ECS defaults, alarms, and telemetry sidecars.++## Start here++- `src/index.ts`: exported components and helpers.+- `src/resources/datadog.ts`: ECS telemetry sidecars and the existing Datadog key.+ `DATADOG_API_KEY_SECRET_ARN` lets extensions retrieve that same key without+ copying its value into runtime environment variables.++## Checks++- From `infra/`: `bun run check` and Biome on changed files.+- Changes to Datadog key wiring: run+ `bun test stacks/notification-service/retention.test.ts`.++## Gotchas++- Do not create a parallel Datadog secret. Grant only the specific secret ARN+ to consumers that fetch the key themselves.+- Reuse the gateway/Doppler patterns in `docs/development/infrastructure.md`;+ some older exported helpers remain for legacy stacks, not new services.diff --git a/infra/packages/resources/CLAUDE.md b/infra/packages/resources/CLAUDE.mdnew file mode 120000index 0000000000..47dc3e3d86--- /dev/null+++ b/infra/packages/resources/CLAUDE.md@@ -0,0 +1 @@+AGENTS.md\ No newline at end of filediff --git a/infra/packages/resources/src/index.ts b/infra/packages/resources/src/index.tsindex 05ee70d021..65df32b2aa 100644--- a/infra/packages/resources/src/index.ts+++ b/infra/packages/resources/src/index.ts@@ -3,6 +3,7 @@ export { create_cluster } from './resources/cluster';export { ALLOWED_ORIGINS } from './resources/cors';export {DATADOG_API_KEY,+ DATADOG_API_KEY_SECRET_ARN,DatadogServiceEntity,datadogAgentContainer,fargateLogRouterSidecarContainer,diff --git a/infra/packages/resources/src/resources/datadog.ts b/infra/packages/resources/src/resources/datadog.tsindex c604176d08..a208534c53 100644--- a/infra/packages/resources/src/resources/datadog.ts+++ b/infra/packages/resources/src/resources/datadog.ts@@ -9,11 +9,14 @@ import { grafanaTelemetryEnabled, stack } from '../../../shared';const DATADOG_API_KEY_SECRET_KEY = 'datadog-api-key';-export const DATADOG_API_KEY = aws.secretsmanager- .getSecretVersionOutput({- secretId: DATADOG_API_KEY_SECRET_KEY,- })- .apply((secret) => secret.secretString);+const datadogApiKey = aws.secretsmanager.getSecretVersionOutput({+ secretId: DATADOG_API_KEY_SECRET_KEY,+});++export const DATADOG_API_KEY = datadogApiKey.secretString;+// Extensions fetch the existing key themselves; don't copy its value into+// Lambda environment variables or grant access to unrelated secrets.+export const DATADOG_API_KEY_SECRET_ARN = datadogApiKey.arn;interface DatadogServiceEntityArgs {serviceName: string;diff --git a/infra/stacks/monitoring/AGENTS.md b/infra/stacks/monitoring/AGENTS.mdnew file mode 100644index 0000000000..1c394bd6e6--- /dev/null+++ b/infra/stacks/monitoring/AGENTS.md@@ -0,0 +1,24 @@+# monitoring++## What it is++Org-global Datadog monitors, synthetics, and rollout dashboards. This stack is+manual and has only a prod stack, even for monitors covering both environments.++## Checks++From `infra/`:+- `bun test stacks/monitoring/monitors/notification-retention.test.ts`+- `bun run check`+- Biome on changed files.+- An authenticated `pulumi preview` must leave adopted monitors unchanged.++## Gotchas++- Read `README.md` before deployment: imported resources must be adopted first+ to avoid duplicate alerts. Never deploy without explicit approval.+- Notification retention adds nine monitors and one dashboard. Deploy DSS health+ instrumentation first; missing telemetry is deliberately an alert, not zero lag.+- Do not use native committed-offset lag for the independent notification+ consumers. Their per-instance log measurements preserve broadcast semantics.+- `notificationRetentionDashboardUrl` is a stack output, not an app route.diff --git a/infra/stacks/monitoring/CLAUDE.md b/infra/stacks/monitoring/CLAUDE.mdnew file mode 120000index 0000000000..47dc3e3d86--- /dev/null+++ b/infra/stacks/monitoring/CLAUDE.md@@ -0,0 +1 @@+AGENTS.md\ No newline at end of filediff --git a/infra/stacks/monitoring/README.md b/infra/stacks/monitoring/README.mdindex 142af69d73..3a65e55b6a 100644--- a/infra/stacks/monitoring/README.md+++ b/infra/stacks/monitoring/README.md@@ -133,3 +133,28 @@ should show exactly those 15 creates and nothing else.Resolver errors still return HTTP 200, so the error monitors are proxies. Addper-operation error and latency monitors once the server emits operation namesand error counts.++## Notification retention coverage++`monitors/notification-retention.ts` adds **nine monitors and one dashboard**:+per-environment consumer backlog, unhealthy telemetry, missing telemetry, and+failed delete publication, plus a cross-environment retention trace-flush alarm.+The dashboard URL is exported as `notificationRetentionDashboardUrl`.++Deploy the notification consumer instrumentation in DSS before these monitors;+otherwise the missing-telemetry alerts intentionally fire. This monitoring stack+still requires a separate, explicitly approved manual deployment. The preview+must not change adopted monitors.++Consumers are intentionally ungrouped and do not commit offsets, so native MSK+consumer-group lag is not meaningful. The new logs expose broker high watermark+minus application position, not committed position. Unknown/stale measurements+are omitted rather than graphed as zero. Alerts are per consumer instance so one+healthy process cannot mask a lagging one; retired-instance alerts resolve on+missing data, while a separate monitor catches loss of all telemetry in an env.+The initial backlog thresholds (warning 100, critical 1000 averaged over five+minutes) need review against live baselines before production cleanup.++These signals do not acknowledge delivery through PostgreSQL NOTIFY or WebSockets.+See [the rollout runbook](../../../docs/runbooks/notification-retention.md) for+limitations and the post-deploy verification gates.diff --git a/infra/stacks/monitoring/index.ts b/infra/stacks/monitoring/index.tsindex 6ee0ebfae5..f72d16d022 100644--- a/infra/stacks/monitoring/index.ts+++ b/infra/stacks/monitoring/index.ts@@ -20,3 +20,7 @@ import './synthetics';// Declared here first.import './monitors/ai-editing';import './monitors/graphql-soup';+import { notificationRetentionDashboard } from './monitors/notification-retention';++export const notificationRetentionDashboardUrl =+ notificationRetentionDashboard.url;diff --git a/infra/stacks/monitoring/monitors/notification-retention.test.ts b/infra/stacks/monitoring/monitors/notification-retention.test.tsnew file mode 100644index 0000000000..e213808d66--- /dev/null+++ b/infra/stacks/monitoring/monitors/notification-retention.test.ts@@ -0,0 +1,78 @@+import { beforeAll, expect, test } from 'bun:test';+import * as pulumi from '@pulumi/pulumi';++const resources: pulumi.runtime.MockResourceArgs[] = [];++beforeAll(async () => {+ pulumi.runtime.setMocks(+ {+ newResource(args) {+ resources.push(args);+ return { id: args.name, state: args.inputs };+ },+ call(args) {+ return args.inputs;+ },+ },+ 'monitoring',+ 'prod',+ false+ );+ await import('./notification-retention');+ await pulumi.runtime.waitForRPCs();+});++function monitor(name: string): pulumi.runtime.MockResourceArgs {+ const value = resources.find((r) => r.name === name);+ if (!value) throw new Error(`Missing monitor ${name}`);+ return value;+}++test('backlog and stale-health alerts isolate each consumer and retire old instances', () => {+ for (const env of ['dev', 'prod']) {+ for (const kind of ['backlog', 'health']) {+ const inputs = monitor(`notification-consumer-${kind}-${env}`).inputs;+ expect(inputs.query).toContain(`env:${env}`);+ expect(inputs.query).toContain('.by("@consumer_instance")');+ expect(inputs.onMissingData).toBe('resolve');+ }+ const backlog = monitor(`notification-consumer-backlog-${env}`).inputs;+ expect(backlog.query).toContain('@consumer_health:healthy');+ expect(backlog.query).toContain('@max_offset_backlog');+ expect(backlog.query).not.toContain('committed');+ const health = monitor(`notification-consumer-health-${env}`).inputs;+ expect(health.query).toContain('stale OR unavailable OR disconnected');+ expect(health.query).not.toContain('starting');+ }+});++test('missing telemetry is an alarm, not a healthy zero backlog', () => {+ for (const env of ['dev', 'prod']) {+ const inputs = monitor(`notification-consumer-heartbeat-${env}`).inputs;+ expect(inputs.query).toContain('.last("5m") < 1');+ expect(inputs.query).not.toContain('.by(');+ expect(inputs.onMissingData).toBe('default');+ }+});++test('trace flush and realtime publication failures are covered separately from Lambda errors', () => {+ expect(+ monitor('notification-retention-telemetry-export').inputs.query+ ).toContain('notification retention telemetry flush');+ for (const env of ['dev', 'prod']) {+ expect(+ monitor(`notification-delete-publication-${env}`).inputs.query+ ).toContain('failed to publish notification delete realtime update');+ }+});++test('dashboard charts only known backlog alongside health and deletion progress', () => {+ const dashboard = monitor('notification-retention').inputs;+ const query = dashboard.widgets[0].timeseriesDefinition.requests[0].logQuery;+ expect(query.searchQuery).toContain('@consumer_health:healthy');+ expect(query.computeQuery.facet).toBe('@max_offset_backlog');+ expect(dashboard.widgets).toHaveLength(4);+ expect(+ resources.filter((r) => r.type.includes('monitor:Monitor'))+ ).toHaveLength(9);+});diff --git a/infra/stacks/monitoring/monitors/notification-retention.ts b/infra/stacks/monitoring/monitors/notification-retention.tsnew file mode 100644index 0000000000..70e343f99f--- /dev/null+++ b/infra/stacks/monitoring/monitors/notification-retention.ts@@ -0,0 +1,156 @@+// Independent notification consumers do not commit Kafka offsets. Native MSK+// consumer-group lag is therefore absent; these logs report application backlog.+import * as datadog from '@pulumi/datadog';++const HEALTH = '@telemetry_event:notification_consumer_health';+const CONTEXT = `Notification-retention rollout: pause scheduled cleanup before investigating.+Check consumer health/backlog, notification.delete.publish spans, database latency,+and replica lag. A zero application backlog is not an end-to-end delivery ACK:+Postgres NOTIFY publication and WebSocket delivery can still fail.+@webhook-macro-alert-hook`;++for (const environment of ['dev', 'prod']) {+ const query = `${HEALTH} env:${environment}`;+ const common = {+ tags: [+ `env:${environment}`,+ 'service:notification-realtime',+ 'rollout:notification-retention',+ ],+ priority: environment === 'prod' ? '2' : '3',+ includeTags: true,+ requireFullWindow: false,+ };+ new datadog.Monitor(`notification-consumer-backlog-${environment}`, {+ ...common,+ name: `[${environment.toUpperCase()}] Notification consumer: application backlog high`,+ type: 'log alert',+ query: `logs("${query} @consumer_health:healthy").index("*").rollup("avg", "@max_offset_backlog").by("@consumer_instance").last("5m") > 1000`,+ message: `One consumer's maximum partition offset backlog averages over {{threshold}} for five minutes. This is not committed-offset lag. The initial threshold is conservative; tune after observing dry-run and canary baselines.\n${CONTEXT}`,+ monitorThresholds: { critical: '1000', warning: '100' },+ // A retired process must not leave its old lag alert firing forever.+ // Stalls in live processes produce explicit health events below.+ onMissingData: 'resolve',+ });+ new datadog.Monitor(`notification-consumer-health-${environment}`, {+ ...common,+ name: `[${environment.toUpperCase()}] Notification consumer: telemetry stale or unavailable`,+ type: 'log alert',+ query: `logs("${query} @consumer_health:(stale OR unavailable OR disconnected)").index("*").rollup("count").by("@consumer_instance").last("5m") > 0`,+ message: `A live consumer has stale statistics, unknown offsets, or no connected brokers after its startup grace period. Do not interpret missing/partial telemetry as zero lag.\n${CONTEXT}`,+ monitorThresholds: { critical: '0' },+ onMissingData: 'resolve',+ });+ new datadog.Monitor(`notification-consumer-heartbeat-${environment}`, {+ ...common,+ name: `[${environment.toUpperCase()}] Notification consumer: no telemetry`,+ type: 'log alert',+ query: `logs("${query}").index("*").rollup("count").last("5m") < 1`,+ message: `No notification-consumer health events arrived for five minutes. Verify DSS consumers are running and log ingestion is healthy. Deploy the consumer instrumentation before enabling this monitor.\n${CONTEXT}`,+ monitorThresholds: { critical: '1' },+ onMissingData: 'default',+ });+ new datadog.Monitor(`notification-delete-publication-${environment}`, {+ ...common,+ name: `[${environment.toUpperCase()}] Notification deletion: realtime publication failed`,+ type: 'log alert',+ query: `logs("env:${environment} \\"failed to publish notification delete realtime update\\"").index("*").rollup("count").last("5m") > 0`,+ message: `A database deletion could not be forwarded to realtime. Deletion has already committed; PostgreSQL NOTIFY is best-effort, not a durable retry queue.\n${CONTEXT}`,+ monitorThresholds: { critical: '0' },+ onMissingData: 'default',+ });+}++// CloudWatch's Lambda Errors stays zero when cleanup succeeds but tracing fails.+// Match the application diagnostic rather than assuming Firehose sets env tags.+new datadog.Monitor('notification-retention-telemetry-export', {+ name: 'Notification retention: trace flush failed',+ type: 'log alert',+ query:+ 'logs("\\"notification retention telemetry flush\\"").index("*").rollup("count").last("15m") > 0',+ message: `Cleanup may have succeeded while its traces were lost. Check the Datadog extension, its scoped secret access, and OTLP receiver. Do not retry deletion merely to retry telemetry.\n${CONTEXT}`,+ tags: ['service:notification-retention', 'rollout:notification-retention'],+ priority: '3',+ monitorThresholds: { critical: '0' },+ requireFullWindow: false,+ onMissingData: 'default',+});++export const notificationRetentionDashboard = new datadog.Dashboard(+ 'notification-retention',+ {+ title: 'Notification retention rollout',+ description:+ 'Application backlog for independent consumers, telemetry health, and retention progress. No committed-offset lag is available. See docs/runbooks/notification-retention.md.',+ layoutType: 'ordered',+ widgets: [+ {+ timeseriesDefinition: {+ title: 'Maximum partition offset backlog, per consumer',+ requests: [+ {+ displayType: 'line',+ logQuery: {+ index: '*',+ searchQuery: `${HEALTH} @consumer_health:healthy`,+ computeQuery: {+ aggregation: 'max',+ facet: '@max_offset_backlog',+ },+ groupBies: [+ { facet: 'env', limit: 2 },+ { facet: '@consumer_instance', limit: 20 },+ ],+ },+ },+ ],+ },+ },+ {+ timeseriesDefinition: {+ title: 'Consumer health events (missing is not zero lag)',+ requests: [+ {+ displayType: 'bars',+ logQuery: {+ index: '*',+ searchQuery: HEALTH,+ computeQuery: { aggregation: 'count' },+ groupBies: [+ { facet: 'env', limit: 2 },+ { facet: '@consumer_health', limit: 5 },+ ],+ },+ },+ ],+ },+ },+ {+ timeseriesDefinition: {+ title: 'Committed parent deletions reported by completed sweeps',+ requests: [+ {+ displayType: 'bars',+ logQuery: {+ index: '*',+ searchQuery: '"notification retention sweep complete"',+ computeQuery: { aggregation: 'sum', facet: '@deleted' },+ },+ },+ ],+ },+ },+ {+ timeseriesDefinition: {+ title: 'MacroDB CPU during rollout',+ requests: [+ {+ q: 'max:aws.rds.cpuutilization{name:macro-db-dev OR name:macro-db-prod} by {name}',+ displayType: 'line',+ },+ ],+ },+ },+ ],+ }+);diff --git a/infra/stacks/notification-service/AGENTS.md b/infra/stacks/notification-service/AGENTS.mdnew file mode 100644index 0000000000..0eb2136d80--- /dev/null+++ b/infra/stacks/notification-service/AGENTS.md@@ -0,0 +1,24 @@+# notification-service stack++## What it is++ECS notification delivery, queues/mobile platforms, and the separately scheduled+notification-retention Lambda. Runtime business policy lives in Rust.++## Checks++From `infra/`:+- `bun test stacks/notification-service/retention.test.ts`+- `bun run check`+- `bunx biome check stacks/notification-service`++## Gotchas++- Keep retention disabled/dry-run until explicitly approved. Deployment does not+ authorize deletion; `docs/runbooks/notification-retention.md` owns rollout.+- The retention function needs its pinned x86_64 Datadog extension to receive+ gRPC OTLP; the ECS sidecar default alone does not work in Lambda.+- Pass the existing Datadog key ARN, not its value, and scope the role to that key.+- Extension log shipping is off: the shared Lambda helper already forwards+ CloudWatch logs through Firehose. Monitoring resources deploy separately from+ `infra/stacks/monitoring`, not automatically with this service.diff --git a/infra/stacks/notification-service/CLAUDE.md b/infra/stacks/notification-service/CLAUDE.mdnew file mode 120000index 0000000000..47dc3e3d86--- /dev/null+++ b/infra/stacks/notification-service/CLAUDE.md@@ -0,0 +1 @@+AGENTS.md\ No newline at end of filediff --git a/infra/stacks/notification-service/retention.test.ts b/infra/stacks/notification-service/retention.test.tsnew file mode 100644index 0000000000..8b66b7c2c2--- /dev/null+++ b/infra/stacks/notification-service/retention.test.ts@@ -0,0 +1,121 @@+import { beforeAll, expect, mock, test } from 'bun:test';+import * as pulumi from '@pulumi/pulumi';++const resources: pulumi.runtime.MockResourceArgs[] = [];+const secretArn =+ 'arn:aws:secretsmanager:us-east-1:123456789012:secret:datadog-test';++beforeAll(async () => {+ pulumi.runtime.setAllConfig({ 'aws:region': 'us-east-1' });+ pulumi.runtime.setMocks(+ {+ newResource(args) {+ resources.push(args);+ return {+ id: args.name,+ state: {+ ...args.inputs,+ name: args.inputs.name ?? args.name,+ arn: `arn:aws:mock:us-east-1:123456789012:${args.name}`,+ },+ };+ },+ call(args) {+ return args.inputs;+ },+ },+ 'notification-service',+ 'dev',+ false+ );+ mock.module('../../packages/shared', () => ({+ stack: 'dev',+ CLOUD_TRAIL_SNS_TOPIC_ARN: 'arn:aws:sns:us-east-1:123456789012:alerts',+ CLOUDWATCH_KINESIS_STREAM_ROLE_ARN: 'arn:aws:iam::123456789012:role/logs',+ DATADOG_KINESIS_FIREHOSE_STREAM_ARN:+ 'arn:aws:firehose:us-east-1:123456789012:logs',+ }));+ mock.module('../../packages/resources', () => ({+ DATADOG_API_KEY_SECRET_ARN: pulumi.output(secretArn),+ }));+ // Keep the real Lambda resources, but don't depend on the test runner's cwd.+ mock.module('../../packages/lambda/src/hash', () => ({+ generateContentHash: () => 'test-source-hash',+ }));+ // Pulumi dynamic-provider closure serialization is Node-only; it is unrelated+ // to the AWS configuration under test.+ mock.module('../../packages/lambda/src/source_code_hash', () => ({+ SourceCodeHash: class extends pulumi.ComponentResource {+ constructor(+ name: string,+ args: { sourceCodeHash: string },+ opts?: pulumi.ComponentResourceOptions+ ) {+ super('test:source:Hash', name, args, opts);+ }+ },+ }));+ const { NotificationRetention } = await import('./retention');+ new NotificationRetention('retention-test', {+ databaseUrl: pulumi.secret('test-database'),+ enabled: false,+ dryRun: true,+ vpc: {+ vpcId: 'vpc-test',+ publicSubnetIds: [],+ privateSubnetIds: ['subnet-test'],+ },+ tags: {},+ });+ await pulumi.runtime.waitForRPCs();+});++function resource(type: string): pulumi.runtime.MockResourceArgs {+ const value = resources.find((r) => r.type === type);+ if (!value) throw new Error(`Missing ${type}`);+ return value;+}++test('retention has a pinned OTLP collector and consistent service identity', () => {+ const fn = resource('aws:lambda/function:Function').inputs;+ expect(fn.layers).toEqual([+ 'arn:aws:lambda:us-east-1:464622532012:layer:Datadog-Extension:101',+ ]);+ const environment = pulumi.runtime.unwrapRpcSecret(fn.environment);+ expect(environment.variables).toMatchObject({+ DD_SERVICE: 'notification-retention',+ DD_ENV: 'dev',+ DD_SITE: 'us5.datadoghq.com',+ DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: '127.0.0.1:4317',+ DD_SERVERLESS_LOGS_ENABLED: 'false',+ DD_API_KEY_SECRET_ARN: secretArn,+ });+ expect(environment.variables.DD_API_KEY).toBeUndefined();+ expect(fn.reservedConcurrentExecutions).toBe(1);+});++test('extension access is scoped to the existing Datadog key, not all secrets', () => {+ const policy = JSON.parse(+ resource('aws:iam/rolePolicy:RolePolicy').inputs.policy+ );+ expect(policy.Statement).toEqual([+ {+ Effect: 'Allow',+ Action: 'secretsmanager:GetSecretValue',+ Resource: secretArn,+ },+ ]);+});++test('telemetry does not enable cleanup or duplicate the existing log path', () => {+ expect(resource('aws:cloudwatch/eventRule:EventRule').inputs.state).toBe(+ 'DISABLED'+ );+ expect(+ JSON.parse(resource('aws:cloudwatch/eventTarget:EventTarget').inputs.input)+ ).toEqual({ dry_run: true });+ expect(+ resource('aws:cloudwatch/logSubscriptionFilter:LogSubscriptionFilter')+ .inputs.destinationArn+ ).toBe('arn:aws:firehose:us-east-1:123456789012:logs');+});diff --git a/infra/stacks/notification-service/retention.ts b/infra/stacks/notification-service/retention.tsindex 517ec4b976..99f867597e 100644--- a/infra/stacks/notification-service/retention.ts+++ b/infra/stacks/notification-service/retention.ts@@ -1,15 +1,25 @@import * as aws from '@pulumi/aws';import * as pulumi from '@pulumi/pulumi';import { Lambda } from '../../packages/lambda';+import { DATADOG_API_KEY_SECRET_ARN } from '../../packages/resources';import { CLOUD_TRAIL_SNS_TOPIC_ARN, stack } from '../../packages/shared';const BASE_NAME = 'notification_retention_handler';const REPO_ROOT = '../../..';+// Pinned public x86_64 layer, verified in us-east-1. Supports OTLP gRPC.+const DATADOG_EXTENSION_ARN =+ 'arn:aws:lambda:us-east-1:464622532012:layer:Datadog-Extension:101';type RetentionEnvVars = {DATABASE_URL: pulumi.Output<string> | string;ENVIRONMENT: string;RUST_LOG: string;+ DD_SERVICE: string;+ DD_ENV: string;+ DD_SITE: string;+ DD_API_KEY_SECRET_ARN: pulumi.Output<string>;+ DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: string;+ DD_SERVERLESS_LOGS_ENABLED: string;};type Args = {@@ -56,6 +66,23 @@ export class NotificationRetention extends pulumi.ComponentResource {},{ parent: this });+ const datadogSecretPolicy = new aws.iam.RolePolicy(+ `${BASE_NAME}-datadog-key`,+ {+ role: role.id,+ policy: pulumi.jsonStringify({+ Version: '2012-10-17',+ Statement: [+ {+ Effect: 'Allow',+ Action: 'secretsmanager:GetSecretValue',+ Resource: DATADOG_API_KEY_SECRET_ARN,+ },+ ],+ }),+ },+ { parent: this }+ );this.lambda = new Lambda<RetentionEnvVars>(`${BASE_NAME}-lambda`,{@@ -66,9 +93,17 @@ export class NotificationRetention extends pulumi.ComponentResource {DATABASE_URL: databaseUrl,ENVIRONMENT: stack,RUST_LOG: 'info',+ DD_SERVICE: 'notification-retention',+ DD_ENV: stack,+ DD_SITE: 'us5.datadoghq.com',+ DD_API_KEY_SECRET_ARN: DATADOG_API_KEY_SECRET_ARN,+ DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: '127.0.0.1:4317',+ // Keep the existing CloudWatch → Firehose log path; avoid duplicates.+ DD_SERVERLESS_LOGS_ENABLED: 'false',},role,vpc,+ layers: [DATADOG_EXTENSION_ARN],memorySize: 256,timeout: 120,reservedConcurrentExecutions: 1,@@ -77,6 +112,7 @@ export class NotificationRetention extends pulumi.ComponentResource {{ parent: this }).lambda;+ // Ensure the extension can fetch its key before EventBridge can invoke it.const rule = new aws.cloudwatch.EventRule(`${BASE_NAME}-schedule`,{@@ -84,7 +120,7 @@ export class NotificationRetention extends pulumi.ComponentResource {state: enabled ? 'ENABLED' : 'DISABLED',tags,},- { parent: this }+ { parent: this, dependsOn: [datadogSecretPolicy] });new aws.cloudwatch.EventTarget(`${BASE_NAME}-target`,diff --git a/services/notification_retention_handler/AGENTS.md b/services/notification_retention_handler/AGENTS.mdnew file mode 100644index 0000000000..b26ce493f3--- /dev/null+++ b/services/notification_retention_handler/AGENTS.md@@ -0,0 +1,34 @@+# notification_retention_handler++## What it is++Internal scheduled notification cleanup. The owning notification domain enforces+30-day retention and bounded transactions; this binary only composes it for Lambda.++## Start here++- `src/main.rs`: one-connection pool, retained telemetry provider, post-invocation+ flush. Close handler spans before flushing; never shut down warm providers.+- `src/handler.rs`: explicit dry-run default, invocation deadline and request ID.+- `infra/stacks/notification-service/retention.ts`: disabled schedule, pinned+ Datadog extension, scoped key access and nonduplicated log forwarding.++## Checks++- `cargo test -p notification_retention_handler` (`SQLX_OFFLINE` unset).+- `cargo test -p notification -p macro_entrypoint` for policy and flush behavior.+- From `infra/`: `bun test stacks/notification-service/retention.test.ts`.++## Gotchas++- `{}` is read-only; `{"dry_run":false}` permanently deletes hosted data.+- Telemetry failure preserves the operation result; it must not trigger a retry+ of already-committed deletion. Flush time comes from reserved handler headroom.+- Unit tests cannot prove the deployed extension exported traces. Verify an+ explicitly approved dry run in Datadog before production cleanup.+- The schedule stays disabled until the rollout is explicitly approved.++## Related++- `docs/runbooks/notification-retention.md`+- `.agents/rules/observability.md`diff --git a/services/notification_retention_handler/CLAUDE.md b/services/notification_retention_handler/CLAUDE.mdnew file mode 120000index 0000000000..47dc3e3d86--- /dev/null+++ b/services/notification_retention_handler/CLAUDE.md@@ -0,0 +1 @@+AGENTS.md\ No newline at end of filediff --git a/services/notification_retention_handler/src/handler.rs b/services/notification_retention_handler/src/handler.rsindex 1390af3244..2c37664644 100644--- a/services/notification_retention_handler/src/handler.rs+++ b/services/notification_retention_handler/src/handler.rs@@ -29,7 +29,12 @@ impl CleanupEvent {}}-#[tracing::instrument(err, skip(service, event))]+#[tracing::instrument(+ name = "notification.retention.invoke",+ err,+ skip(service, event),+ fields(faas.invocation_id = %event.context.request_id, dry_run = event.payload.dry_run)+)]pub(crate) async fn handle<R: NotificationRetentionRepository>(service: &NotificationRetentionService<R>,event: LambdaEvent<CleanupEvent>,diff --git a/services/notification_retention_handler/src/main.rs b/services/notification_retention_handler/src/main.rsindex 79fb641ee0..57cfe480ae 100644--- a/services/notification_retention_handler/src/main.rs+++ b/services/notification_retention_handler/src/main.rs@@ -20,7 +20,7 @@ env_vars! {#[tokio::main]async fn main() -> Result<(), Error> {- MacroEntrypoint::default().init();+ let telemetry = Arc::new(MacroEntrypoint::default().init());let database_url = DatabaseUrl::new()?;// Bound preview queries as well as deletes. One maintenance connection per// process, and reserved Lambda concurrency of one, isolate foreground work.@@ -41,7 +41,18 @@ async fn main() -> Result<(), Error> {));run(service_fn(move |event| {let service = Arc::clone(&service);- async move { handler::handle(&service, event).await }+ let telemetry = Arc::clone(&telemetry);+ async move {+ // `handle` must finish (closing its root span) before flushing.+ // A telemetry outage must not turn a committed cleanup into a retry.+ let result = handler::handle(&service, event).await;+ match tokio::time::timeout(Duration::from_secs(5), telemetry.force_flush()).await {+ Ok(Ok(())) => {}+ Ok(Err(error)) => tracing::error!(error = ?error, "notification retention telemetry flush failed"),+ Err(error) => tracing::error!(error = ?error, "notification retention telemetry flush timed out"),+ }+ result+ }})).await}diff --git a/services/notification_service/AGENTS.md b/services/notification_service/AGENTS.mdindex f20246957e..350bc184fe 100644--- a/services/notification_service/AGENTS.md+++ b/services/notification_service/AGENTS.md@@ -41,6 +41,10 @@ worker that sends email/mobile-push/websocket notifications.## Gotchas+- PostgreSQL delete forwarding emits `notification.delete.publish` spans and+ logs publication failures. The retention Lambda is a separate binary/role;+ see `services/notification_retention_handler` and its rollout runbook.+- Don't log notification payloads on tracing spans (#7892) — they can carrysensitive content; log ids/types instead.