Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/feat/notification retention (#8083)
Sean Aye37077ec462fb
diff --git a/.github/services-config.json b/.github/services-config.jsonindex 800913ccc9..ea1df00cab 100644--- a/.github/services-config.json+++ b/.github/services-config.json@@ -185,11 +185,15 @@},"notification-service": {"source_paths": [- "services/notification_service/**"+ "services/notification_service/**",+ "services/notification_retention_handler/**"],"stack_path": "infra/stacks/notification-service/**","deploy_binaries": ["notification_service"+ ],+ "deploy_lambdas": [+ "notification_retention_handler"]},"organization-retention": {diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 706f35c2ff..6e5ec4093f 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -10199,6 +10199,38 @@"crates/remote_env_var","crates/workspace-hack"],+ "notification_retention_handler": [+ "crates/ai_toolset",+ "crates/bot_id",+ "crates/bot_token",+ "crates/broadcast",+ "crates/cowlike",+ "crates/genai_telemetry",+ "crates/harness_id",+ "crates/harness_token",+ "crates/kafka_util",+ "crates/last_online_tracker",+ "crates/macro_auth",+ "crates/macro_authorization",+ "crates/macro_entrypoint",+ "crates/macro_env",+ "crates/macro_env_var",+ "crates/macro_event_broker",+ "crates/macro_event_topics",+ "crates/macro_service_urls",+ "crates/macro_user_id",+ "crates/macro_uuid",+ "crates/model-entity",+ "crates/model-error-response",+ "crates/model_user",+ "crates/models_pagination",+ "crates/notification",+ "crates/notification_state",+ "crates/rate_limit",+ "crates/remote_env_var",+ "crates/workspace-hack",+ "services/notification_retention_handler"+ ],"notification_sandbox": ["crates/ai_toolset","crates/bot_id",diff --git a/.sqlx/query-370764911f1740f87020c23f75698124a953a3ee175b20eafbb522bf7cf8d1bf.json b/.sqlx/query-370764911f1740f87020c23f75698124a953a3ee175b20eafbb522bf7cf8d1bf.jsonnew file mode 100644index 0000000000..d24dbdcc50--- /dev/null+++ b/.sqlx/query-370764911f1740f87020c23f75698124a953a3ee175b20eafbb522bf7cf8d1bf.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT pg_try_advisory_xact_lock($1) AS \"acquired!\"",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "acquired!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Int8"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "370764911f1740f87020c23f75698124a953a3ee175b20eafbb522bf7cf8d1bf"+}diff --git a/.sqlx/query-6af068223ccebea85f085701b94b10b7f61c6912dea741ba74d717c4f4f2af87.json b/.sqlx/query-6af068223ccebea85f085701b94b10b7f61c6912dea741ba74d717c4f4f2af87.jsonnew file mode 100644index 0000000000..709dbb331f--- /dev/null+++ b/.sqlx/query-6af068223ccebea85f085701b94b10b7f61c6912dea741ba74d717c4f4f2af87.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH expired AS (\n SELECT id FROM notification\n WHERE created_at < $1\n ORDER BY created_at, id\n LIMIT $2\n FOR UPDATE SKIP LOCKED\n )\n DELETE FROM notification n\n USING expired e\n WHERE n.id = e.id\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Timestamp",+ "Int8"+ ]+ },+ "nullable": []+ },+ "hash": "6af068223ccebea85f085701b94b10b7f61c6912dea741ba74d717c4f4f2af87"+}diff --git a/.sqlx/query-78ae885ce1b7d7df2dff41d4699d8f73ca0a9279d4167e82c0e603d7d2d82884.json b/.sqlx/query-78ae885ce1b7d7df2dff41d4699d8f73ca0a9279d4167e82c0e603d7d2d82884.jsonnew file mode 100644index 0000000000..3721c454fb--- /dev/null+++ b/.sqlx/query-78ae885ce1b7d7df2dff41d4699d8f73ca0a9279d4167e82c0e603d7d2d82884.json@@ -0,0 +1,20 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT clock_timestamp() AS \"now!\"",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "now!",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ null+ ]+ },+ "hash": "78ae885ce1b7d7df2dff41d4699d8f73ca0a9279d4167e82c0e603d7d2d82884"+}diff --git a/.sqlx/query-b3379073b89935dd1359532395b8a04a9873349b5ae80066e024ca19ef5ebba8.json b/.sqlx/query-b3379073b89935dd1359532395b8a04a9873349b5ae80066e024ca19ef5ebba8.jsonnew file mode 100644index 0000000000..55ca8a5b43--- /dev/null+++ b/.sqlx/query-b3379073b89935dd1359532395b8a04a9873349b5ae80066e024ca19ef5ebba8.json@@ -0,0 +1,29 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT count(*) AS \"count!\", min(created_at) AS oldest\n FROM (\n SELECT created_at FROM notification\n WHERE created_at < $1\n ORDER BY created_at, id\n LIMIT $2\n ) expired\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "count!",+ "type_info": "Int8"+ },+ {+ "ordinal": 1,+ "name": "oldest",+ "type_info": "Timestamp"+ }+ ],+ "parameters": {+ "Left": [+ "Timestamp",+ "Int8"+ ]+ },+ "nullable": [+ null,+ null+ ]+ },+ "hash": "b3379073b89935dd1359532395b8a04a9873349b5ae80066e024ca19ef5ebba8"+}diff --git a/.sqlx/query-d9946d44ba3ecfd34d8e9419a4918e0cee4d99c52737f63b62043c82b60030e5.json b/.sqlx/query-d9946d44ba3ecfd34d8e9419a4918e0cee4d99c52737f63b62043c82b60030e5.jsonnew file mode 100644index 0000000000..b30b311b52--- /dev/null+++ b/.sqlx/query-d9946d44ba3ecfd34d8e9419a4918e0cee4d99c52737f63b62043c82b60030e5.json@@ -0,0 +1,12 @@+{+ "db_name": "PostgreSQL",+ "query": "SET LOCAL statement_timeout = '5s'",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": []+ },+ "nullable": []+ },+ "hash": "d9946d44ba3ecfd34d8e9419a4918e0cee4d99c52737f63b62043c82b60030e5"+}diff --git a/.sqlx/query-e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533.json b/.sqlx/query-e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533.jsonnew file mode 100644index 0000000000..fb106d1b14--- /dev/null+++ b/.sqlx/query-e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533.json@@ -0,0 +1,12 @@+{+ "db_name": "PostgreSQL",+ "query": "SET LOCAL lock_timeout = '1s'",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": []+ },+ "nullable": []+ },+ "hash": "e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533"+}diff --git a/Cargo.lock b/Cargo.lockindex 590a95a19b..fce86cd057 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -13215,6 +13215,22 @@ dependencies = ["workspace-hack",]+[[package]]+name = "notification_retention_handler"+version = "0.1.0"+dependencies = [+ "lambda_runtime",+ "macro_entrypoint",+ "macro_env_var",+ "notification",+ "serde",+ "serde_json",+ "sqlx",+ "tokio",+ "tracing",+ "workspace-hack",+]+[[package]]name = "notification_sandbox"version = "0.1.0"diff --git a/Cargo.toml b/Cargo.tomlindex 5058eae091..9314d2ae2f 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -152,6 +152,7 @@ members = ["services/image_proxy_service","services/mcp_auth_proxy","services/mcp_service",+ "services/notification_retention_handler","services/notification_service","services/organization_retention_handler","services/organization_retention_trigger",diff --git a/crates/macro_db_client/migrations/20261015151242_notification_retention_index.down.sql b/crates/macro_db_client/migrations/20261015151242_notification_retention_index.down.sqlnew file mode 100644index 0000000000..468e14cc85--- /dev/null+++ b/crates/macro_db_client/migrations/20261015151242_notification_retention_index.down.sql@@ -0,0 +1,2 @@+-- no-transaction+DROP INDEX CONCURRENTLY IF EXISTS idx_notification_retention;diff --git a/crates/macro_db_client/migrations/20261015151242_notification_retention_index.up.sql b/crates/macro_db_client/migrations/20261015151242_notification_retention_index.up.sqlnew file mode 100644index 0000000000..80ec32f9c2--- /dev/null+++ b/crates/macro_db_client/migrations/20261015151242_notification_retention_index.up.sql@@ -0,0 +1,5 @@+-- no-transaction+-- One statement: SQLx sends nontransactional migrations as one query.+-- Retention scans oldest-first without sorting the notification history.+CREATE INDEX CONCURRENTLY idx_notification_retention+ ON notification (created_at, id);diff --git a/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.down.sql b/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.down.sqlnew file mode 100644index 0000000000..ab6bff7c0a--- /dev/null+++ b/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.down.sql@@ -0,0 +1,29 @@+-- Restore the previous unchunked event publisher. This does not restore any+-- notifications deleted by the worker; retention deletion is irreversible.+CREATE OR REPLACE FUNCTION notify_user_notification_deletes()+RETURNS TRIGGER AS $$+DECLARE+ affected_user_ids JSONB;+BEGIN+ SELECT jsonb_agg(user_id)+ INTO affected_user_ids+ FROM user_notification+ WHERE notification_id = OLD.id;++ IF affected_user_ids IS NOT NULL THEN+ BEGIN+ PERFORM pg_notify(+ 'notification_events',+ jsonb_build_object(+ 'type', 'user_notification_deletes',+ 'notificationId', OLD.id,+ 'userIds', affected_user_ids+ )::TEXT+ );+ EXCEPTION WHEN OTHERS THEN+ RAISE WARNING 'pg_notify failed: %', SQLERRM;+ END;+ END IF;+ RETURN OLD;+END;+$$ LANGUAGE plpgsql;diff --git a/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.up.sql b/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.up.sqlnew file mode 100644index 0000000000..7803ca7e49--- /dev/null+++ b/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.up.sql@@ -0,0 +1,49 @@+-- Preserve the existing event shape while keeping every NOTIFY payload under+-- PostgreSQL's 8000-byte limit. Bulk retention must not silently lose updates+-- for notifications addressed to many users. No history is deleted here.+CREATE OR REPLACE FUNCTION notify_user_notification_deletes()+RETURNS TRIGGER AS $$+DECLARE+ recipient TEXT;+ affected_user_ids JSONB := '[]'::JSONB;+ recipient_json JSONB;+BEGIN+ FOR recipient IN+ SELECT user_id FROM user_notification WHERE notification_id = OLD.id+ LOOP+ recipient_json := jsonb_build_array(recipient);+ -- Leave ample room for the fixed event envelope (type and UUID).+ IF octet_length(recipient_json::TEXT) > 7000 THEN+ RAISE WARNING 'notification delete recipient exceeds NOTIFY payload budget';+ CONTINUE;+ END IF;++ IF octet_length((affected_user_ids || recipient_json)::TEXT) > 7000 THEN+ BEGIN+ PERFORM pg_notify('notification_events', jsonb_build_object(+ 'type', 'user_notification_deletes',+ 'notificationId', OLD.id,+ 'userIds', affected_user_ids+ )::TEXT);+ EXCEPTION WHEN OTHERS THEN+ RAISE WARNING 'pg_notify failed: %', SQLERRM;+ END;+ affected_user_ids := '[]'::JSONB;+ END IF;+ affected_user_ids := affected_user_ids || recipient_json;+ END LOOP;++ IF jsonb_array_length(affected_user_ids) > 0 THEN+ BEGIN+ PERFORM pg_notify('notification_events', jsonb_build_object(+ 'type', 'user_notification_deletes',+ 'notificationId', OLD.id,+ 'userIds', affected_user_ids+ )::TEXT);+ EXCEPTION WHEN OTHERS THEN+ RAISE WARNING 'pg_notify failed: %', SQLERRM;+ END;+ END IF;+ RETURN OLD;+END;+$$ LANGUAGE plpgsql;diff --git a/crates/notification/src/domain.rs b/crates/notification/src/domain.rsindex db2c301ff3..887d102049 100644--- a/crates/notification/src/domain.rs+++ b/crates/notification/src/domain.rs@@ -1,4 +1,5 @@pub mod item_preferences;pub mod models;pub mod ports;+pub mod retention;pub mod service;diff --git a/crates/notification/src/domain/retention.rs b/crates/notification/src/domain/retention.rsnew file mode 100644index 0000000000..1683c52f0e--- /dev/null+++ b/crates/notification/src/domain/retention.rs@@ -0,0 +1,188 @@+//! Bounded, internal maintenance of notification history.+//!+//! Retention applies to the parent notification's creation time, independently+//! of recipient state. The scheduler is the authorization boundary: this use+//! case is deliberately not exposed by a user-facing API.++use std::{num::NonZeroU16, time::Duration};++use chrono::{DateTime, NaiveDateTime, TimeDelta, Utc};+use rootcause::{Report, bail};+use tokio::time::Instant;++/// Notifications live for thirty complete UTC days.+pub const RETENTION: TimeDelta = TimeDelta::days(30);+const BATCH_SIZE: NonZeroU16 = NonZeroU16::new(100).unwrap();+const MAX_BATCHES: u16 = 100;+const MAX_RUN_TIME: Duration = Duration::from_secs(100);+const BATCH_HEADROOM: Duration = Duration::from_secs(15);+const BATCH_PAUSE: Duration = Duration::from_millis(250);++/// Explicitly distinguish a read-only preview from destructive cleanup.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum RetentionMode {+ /// Inspect at most one batch without taking deletion locks or deleting rows.+ DryRun,+ /// Delete expired notifications and their dependent rows.+ Delete,+}++/// A bounded preview; the count is a sample, not the total backlog.+#[derive(Debug, Default)]+pub struct RetentionPreview {+ /// Number of expired parents in the sampled batch.+ pub sampled_expired: u64,+ /// Creation time of the oldest expired parent, if any.+ pub oldest_expired_at: Option<NaiveDateTime>,+}++/// Outcome of one independently committed deletion transaction.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum RetentionBatch {+ /// Number of deleted parents (not the number of cascaded recipient rows).+ Deleted(u64),+ /// Another cleanup transaction holds the maintenance lock.+ Busy,+ /// The transaction rolled back after a statement or lock timeout.+ TimedOut,+}++/// Storage capabilities needed by the internal retention use case.+pub trait NotificationRetentionRepository: Send + Sync {+ /// Use the database clock, not the worker's wall clock, for the cutoff.+ fn current_time(&self) -> impl Future<Output = Result<DateTime<Utc>, Report>> + Send;++ /// Inspect only the oldest `limit` expired parents using an ordered index.+ fn preview(+ &self,+ cutoff: NaiveDateTime,+ limit: NonZeroU16,+ ) -> impl Future<Output = Result<RetentionPreview, Report>> + Send;++ /// Atomically delete at most `limit` parents strictly older than `cutoff`.+ /// Preserve cascade/notification triggers, skip locked parents, and release+ /// the maintenance lock on commit or rollback, including on cancellation.+ fn delete_batch(+ &self,+ cutoff: NaiveDateTime,+ limit: NonZeroU16,+ ) -> impl Future<Output = Result<RetentionBatch, Report>> + Send;+}++/// Why an invocation stopped. A zero-row batch may mean rows were locked;+/// it is not proof that the backlog has been completely drained.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum RetentionStop {+ /// Read-only preview completed.+ DryRun,+ /// No currently available expired parents.+ NoCandidates,+ /// Another cleanup invocation is working.+ Busy,+ /// Leave enough time to finish the current invocation safely.+ TimeBudget,+ /// The per-invocation batch cap was reached.+ BatchLimit,+}++/// Summary suitable for operational logs; contains no recipient data.+#[derive(Debug)]+pub struct RetentionReport {+ /// Fixed UTC cutoff shared by every batch in this invocation.+ pub cutoff: NaiveDateTime,+ /// Bounded snapshot taken before deletion.+ pub preview: RetentionPreview,+ /// Number of parent notifications deleted by committed batches.+ pub deleted: u64,+ /// Number of batches attempted, including timeout retries.+ pub batches: u16,+ /// Reason this invocation stopped.+ pub stopped: RetentionStop,+}++/// Enforces retention policy and throttles maintenance work between commits.+pub struct NotificationRetentionService<R> {+ repository: R,+}++impl<R: NotificationRetentionRepository> NotificationRetentionService<R> {+ /// Construct the internal service over its storage port.+ pub fn new(repository: R) -> Self {+ Self { repository }+ }++ /// Run a bounded sweep. Retries are safe: committed deletes disappear from+ /// the next scan, and skipped/rolled-back rows are reconsidered next time.+ #[tracing::instrument(err, skip(self), fields(deleted, batches, stopped))]+ pub async fn run(+ &self,+ mode: RetentionMode,+ time_budget: Duration,+ ) -> Result<RetentionReport, Report> {+ let deadline = Instant::now() + time_budget.min(MAX_RUN_TIME);+ let cutoff = (self.repository.current_time().await? - RETENTION).naive_utc();+ let preview = self.repository.preview(cutoff, BATCH_SIZE).await?;+ let mut report = RetentionReport {+ cutoff,+ preview,+ deleted: 0,+ batches: 0,+ stopped: RetentionStop::DryRun,+ };++ if mode == RetentionMode::Delete {+ let mut batch_size = BATCH_SIZE;+ report.stopped = loop {+ if report.batches >= MAX_BATCHES {+ break RetentionStop::BatchLimit;+ }+ if Instant::now() + BATCH_HEADROOM >= deadline {+ break RetentionStop::TimeBudget;+ }+ report.batches += 1;+ match self.repository.delete_batch(cutoff, batch_size).await? {+ RetentionBatch::Busy => break RetentionStop::Busy,+ RetentionBatch::Deleted(0) => break RetentionStop::NoCandidates,+ RetentionBatch::Deleted(deleted) => {+ report.deleted += deleted;+ tracing::info!(+ deleted,+ batch = report.batches,+ "notification retention batch committed"+ );+ }+ RetentionBatch::TimedOut => {+ let Some(smaller) = NonZeroU16::new(batch_size.get() / 2) else {+ bail!(+ "notification retention timed out deleting a single parent; inspect recipient fanout and locks"+ );+ };+ batch_size = smaller;+ tracing::warn!(+ batch_size = batch_size.get(),+ "notification retention batch timed out; reducing batch size"+ );+ }+ }+ tokio::time::sleep(BATCH_PAUSE).await;+ };+ }++ tracing::Span::current().record("deleted", report.deleted);+ tracing::Span::current().record("batches", report.batches);+ tracing::Span::current().record("stopped", tracing::field::debug(report.stopped));+ tracing::info!(+ %cutoff,+ sampled_expired = report.preview.sampled_expired,+ oldest_expired_at = ?report.preview.oldest_expired_at,+ deleted = report.deleted,+ batches = report.batches,+ stopped = ?report.stopped,+ "notification retention sweep complete"+ );+ Ok(report)+ }+}++#[cfg(test)]+mod test;diff --git a/crates/notification/src/domain/retention/test.rs b/crates/notification/src/domain/retention/test.rsnew file mode 100644index 0000000000..a6fea247a5--- /dev/null+++ b/crates/notification/src/domain/retention/test.rs@@ -0,0 +1,182 @@+use super::*;+use std::{collections::VecDeque, sync::Mutex};++#[derive(Default)]+struct FakeRepository {+ responses: Mutex<VecDeque<RetentionBatch>>,+ calls: Mutex<Vec<(NaiveDateTime, u16)>>,+ delay: Duration,+}++fn now() -> DateTime<Utc> {+ DateTime::parse_from_rfc3339("2026-10-15T12:00:00Z")+ .unwrap()+ .with_timezone(&Utc)+}++impl NotificationRetentionRepository for &FakeRepository {+ async fn current_time(&self) -> Result<DateTime<Utc>, Report> {+ Ok(now())+ }++ async fn preview(+ &self,+ cutoff: NaiveDateTime,+ limit: NonZeroU16,+ ) -> Result<RetentionPreview, Report> {+ assert_eq!(cutoff, (now() - RETENTION).naive_utc());+ assert_eq!(limit, BATCH_SIZE);+ Ok(RetentionPreview {+ sampled_expired: 100,+ oldest_expired_at: Some(cutoff - TimeDelta::days(1)),+ })+ }++ async fn delete_batch(+ &self,+ cutoff: NaiveDateTime,+ limit: NonZeroU16,+ ) -> Result<RetentionBatch, Report> {+ self.calls.lock().unwrap().push((cutoff, limit.get()));+ tokio::time::sleep(self.delay).await;+ Ok(self+ .responses+ .lock()+ .unwrap()+ .pop_front()+ .unwrap_or(RetentionBatch::Deleted(u64::from(limit.get()))))+ }+}++#[tokio::test(start_paused = true)]+async fn dry_run_never_deletes() {+ let repo = FakeRepository::default();+ let report = NotificationRetentionService::new(&repo)+ .run(RetentionMode::DryRun, MAX_RUN_TIME)+ .await+ .unwrap();+ assert_eq!(report.stopped, RetentionStop::DryRun);+ assert_eq!(report.deleted, 0);+ assert_eq!(report.preview.sampled_expired, 100);+ assert!(repo.calls.lock().unwrap().is_empty());+}++#[tokio::test(start_paused = true)]+async fn batches_share_cutoff_and_pause_between_commits() {+ let repo = FakeRepository {+ responses: Mutex::new(VecDeque::from([+ RetentionBatch::Deleted(100),+ RetentionBatch::Deleted(3),+ RetentionBatch::Deleted(0),+ ])),+ ..Default::default()+ };+ let start = Instant::now();+ let report = NotificationRetentionService::new(&repo)+ .run(RetentionMode::Delete, MAX_RUN_TIME)+ .await+ .unwrap();+ assert_eq!(report.deleted, 103);+ assert_eq!(report.batches, 3);+ assert_eq!(report.stopped, RetentionStop::NoCandidates);+ assert!(start.elapsed() >= BATCH_PAUSE * 2);+ assert!(repo.calls.lock().unwrap().iter().all(|(cutoff, limit)| {+ *cutoff == (now() - RETENTION).naive_utc() && *limit == BATCH_SIZE.get()+ }));+}++#[tokio::test(start_paused = true)]+async fn busy_worker_stops_without_retrying() {+ let repo = FakeRepository {+ responses: Mutex::new(VecDeque::from([RetentionBatch::Busy])),+ ..Default::default()+ };+ let report = NotificationRetentionService::new(&repo)+ .run(RetentionMode::Delete, MAX_RUN_TIME)+ .await+ .unwrap();+ assert_eq!(report.stopped, RetentionStop::Busy);+ assert_eq!(report.batches, 1);+ assert_eq!(report.deleted, 0);+}++#[tokio::test(start_paused = true)]+async fn invocation_has_a_hard_batch_cap() {+ let repo = FakeRepository::default();+ let report = NotificationRetentionService::new(&repo)+ .run(RetentionMode::Delete, MAX_RUN_TIME)+ .await+ .unwrap();+ assert_eq!(report.stopped, RetentionStop::BatchLimit);+ assert_eq!(report.batches, MAX_BATCHES);+ assert_eq!(report.deleted, u64::from(MAX_BATCHES) * 100);+}++#[tokio::test(start_paused = true)]+async fn respects_caller_deadline_and_leaves_batch_headroom() {+ let repo = FakeRepository::default();+ let service = NotificationRetentionService::new(&repo);+ let report = service+ .run(RetentionMode::Delete, BATCH_HEADROOM)+ .await+ .unwrap();+ assert_eq!(report.stopped, RetentionStop::TimeBudget);+ assert!(repo.calls.lock().unwrap().is_empty());++ let report = service+ .run(RetentionMode::Delete, BATCH_HEADROOM + BATCH_PAUSE)+ .await+ .unwrap();+ assert_eq!(report.stopped, RetentionStop::TimeBudget);+ assert_eq!(report.batches, 1);+}++#[tokio::test(start_paused = true)]+async fn oversized_time_budget_is_capped() {+ let repo = FakeRepository {+ delay: Duration::from_secs(10),+ ..Default::default()+ };+ let report = NotificationRetentionService::new(&repo)+ .run(RetentionMode::Delete, Duration::from_secs(3600))+ .await+ .unwrap();+ assert_eq!(report.stopped, RetentionStop::TimeBudget);+ assert!(report.batches < 10);+}++#[tokio::test(start_paused = true)]+async fn timeouts_reduce_batch_size_without_counting_rolled_back_deletes() {+ let repo = FakeRepository {+ responses: Mutex::new(VecDeque::from([+ RetentionBatch::TimedOut,+ RetentionBatch::Deleted(50),+ RetentionBatch::Deleted(0),+ ])),+ ..Default::default()+ };+ let report = NotificationRetentionService::new(&repo)+ .run(RetentionMode::Delete, MAX_RUN_TIME)+ .await+ .unwrap();+ assert_eq!(report.deleted, 50);+ assert_eq!(report.batches, 3);+ let sizes: Vec<_> = repo.calls.lock().unwrap().iter().map(|x| x.1).collect();+ assert_eq!(sizes, [100, 50, 50]);+}++#[tokio::test(start_paused = true)]+async fn single_parent_timeout_fails_instead_of_looping_forever() {+ let repo = FakeRepository {+ responses: Mutex::new(VecDeque::from(vec![RetentionBatch::TimedOut; 7])),+ ..Default::default()+ };+ assert!(+ NotificationRetentionService::new(&repo)+ .run(RetentionMode::Delete, MAX_RUN_TIME)+ .await+ .is_err()+ );+ let sizes: Vec<_> = repo.calls.lock().unwrap().iter().map(|x| x.1).collect();+ assert_eq!(sizes, [100, 50, 25, 12, 6, 3, 1]);+}diff --git a/crates/notification/src/outbound.rs b/crates/notification/src/outbound.rsindex 0c376c61b8..2fe7cf3ea3 100644--- a/crates/notification/src/outbound.rs+++ b/crates/notification/src/outbound.rs@@ -28,6 +28,7 @@ pub mod push_notification_event_queue;pub mod queue;pub mod rate_limit;pub mod repository;+pub mod retention;pub mod sns_endpoint;pub mod user_existence_checker;pub mod websocket;diff --git a/crates/notification/src/outbound/retention.rs b/crates/notification/src/outbound/retention.rsnew file mode 100644index 0000000000..982b6dfdd9--- /dev/null+++ b/crates/notification/src/outbound/retention.rs@@ -0,0 +1,123 @@+//! PostgreSQL retention adapter. Only parent notifications are deleted: foreign+//! keys remove recipients/receipts and the existing trigger emits delete events.++use std::num::NonZeroU16;++use chrono::{DateTime, NaiveDateTime, Utc};+use rootcause::Report;+use sqlx::PgPool;++use crate::domain::retention::{NotificationRetentionRepository, RetentionBatch, RetentionPreview};++// Transaction-scoped, so a pooled connection can never retain the lock after+// this operation. The key is local to notification retention ("notifret").+const RETENTION_LOCK: i64 = 0x6e6f_7469_6672_6574;++/// An indexed, bounded retention repository using short transactions.+pub struct PgNotificationRetentionRepository {+ pool: PgPool,+}++impl PgNotificationRetentionRepository {+ /// Construct over a maintenance pool with a short acquisition timeout.+ pub fn new(pool: PgPool) -> Self {+ Self { pool }+ }+}++impl NotificationRetentionRepository for PgNotificationRetentionRepository {+ async fn current_time(&self) -> Result<DateTime<Utc>, Report> {+ Ok(sqlx::query_scalar!(r#"SELECT clock_timestamp() AS "now!""#)+ .fetch_one(&self.pool)+ .await?)+ }++ async fn preview(+ &self,+ cutoff: NaiveDateTime,+ limit: NonZeroU16,+ ) -> Result<RetentionPreview, Report> {+ let row = sqlx::query!(+ r#"+ SELECT count(*) AS "count!", min(created_at) AS oldest+ FROM (+ SELECT created_at FROM notification+ WHERE created_at < $1+ ORDER BY created_at, id+ LIMIT $2+ ) expired+ "#,+ cutoff,+ i64::from(limit.get()),+ )+ .fetch_one(&self.pool)+ .await?;+ Ok(RetentionPreview {+ sampled_expired: row.count as u64,+ oldest_expired_at: row.oldest,+ })+ }++ #[tracing::instrument(err, skip(self))]+ async fn delete_batch(+ &self,+ cutoff: NaiveDateTime,+ limit: NonZeroU16,+ ) -> Result<RetentionBatch, Report> {+ let mut tx = self.pool.begin().await?;+ sqlx::query!("SET LOCAL statement_timeout = '5s'")+ .execute(&mut *tx)+ .await?;+ sqlx::query!("SET LOCAL lock_timeout = '1s'")+ .execute(&mut *tx)+ .await?;+ let acquired = sqlx::query_scalar!(+ r#"SELECT pg_try_advisory_xact_lock($1) AS "acquired!""#,+ RETENTION_LOCK,+ )+ .fetch_one(&mut *tx)+ .await?;+ if !acquired {+ tx.rollback().await?;+ return Ok(RetentionBatch::Busy);+ }++ let result = sqlx::query!(+ r#"+ WITH expired AS (+ SELECT id FROM notification+ WHERE created_at < $1+ ORDER BY created_at, id+ LIMIT $2+ FOR UPDATE SKIP LOCKED+ )+ DELETE FROM notification n+ USING expired e+ WHERE n.id = e.id+ "#,+ cutoff,+ i64::from(limit.get()),+ )+ .execute(&mut *tx)+ .await;+ match result {+ Ok(result) => {+ tx.commit().await?;+ Ok(RetentionBatch::Deleted(result.rows_affected()))+ }+ Err(error) => {+ tx.rollback().await?;+ // A timeout may mean one parent fans out to too many children.+ // The service retries a smaller batch; unrelated errors fail+ // the invocation and surface through the Lambda error alarm.+ match error.as_database_error().and_then(|e| e.code()).as_deref() {+ Some("57014" | "55P03") => Ok(RetentionBatch::TimedOut),+ _ => Err(error.into()),+ }+ }+ }+ }+}++#[cfg(test)]+mod test;diff --git a/crates/notification/src/outbound/retention/test.rs b/crates/notification/src/outbound/retention/test.rsnew file mode 100644index 0000000000..184ebd7abc--- /dev/null+++ b/crates/notification/src/outbound/retention/test.rs@@ -0,0 +1,286 @@+use super::*;+use crate::domain::models::NotificationState;+use chrono::TimeDelta;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::postgres::PgListener;+use std::collections::HashSet;+use uuid::Uuid;++fn cutoff() -> NaiveDateTime {+ DateTime::parse_from_rfc3339("2026-09-15T12:00:00Z")+ .unwrap()+ .naive_utc()+}++fn limit(n: u16) -> NonZeroU16 {+ NonZeroU16::new(n).unwrap()+}++async fn insert(pool: &PgPool, created_at: NaiveDateTime) -> Uuid {+ let id = Uuid::now_v7();+ sqlx::query!(+ "INSERT INTO notification (id, notification_event_type, event_item_id, event_item_type, service_sender, created_at)+ VALUES ($1, 'test', 'retention-document', 'document', 'retention-test', $2)",+ id, created_at,+ )+ .execute(pool)+ .await+ .unwrap();+ id+}++async fn remaining(pool: &PgPool) -> Vec<Uuid> {+ sqlx::query_scalar!("SELECT id FROM notification ORDER BY created_at, id")+ .fetch_all(pool)+ .await+ .unwrap()+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn boundary_is_strict_and_preview_is_read_only_and_bounded(pool: PgPool) {+ let old = insert(&pool, cutoff() - TimeDelta::days(1)).await;+ let just_old = insert(&pool, cutoff() - TimeDelta::microseconds(1)).await;+ let boundary = insert(&pool, cutoff()).await;+ let fresh = insert(&pool, cutoff() + TimeDelta::microseconds(1)).await;+ let repo = PgNotificationRetentionRepository::new(pool.clone());+ let preview = repo.preview(cutoff(), limit(1)).await.unwrap();+ assert_eq!(preview.sampled_expired, 1);+ assert_eq!(+ preview.oldest_expired_at,+ Some(cutoff() - TimeDelta::days(1))+ );+ assert_eq!(remaining(&pool).await, [old, just_old, boundary, fresh]);+ assert_eq!(+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),+ RetentionBatch::Deleted(2)+ );+ assert_eq!(remaining(&pool).await, [boundary, fresh]);+ assert_eq!(+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),+ RetentionBatch::Deleted(0)+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn bounded_oldest_first_batches_resume_on_retry(pool: PgPool) {+ let later = insert(&pool, cutoff() - TimeDelta::seconds(1)).await;+ let oldest_a = insert(&pool, cutoff() - TimeDelta::days(2)).await;+ let oldest_b = insert(&pool, cutoff() - TimeDelta::days(2)).await;+ let repo = PgNotificationRetentionRepository::new(pool.clone());+ assert_eq!(+ repo.delete_batch(cutoff(), limit(1)).await.unwrap(),+ RetentionBatch::Deleted(1)+ );+ assert_eq!(remaining(&pool).await, [oldest_a.max(oldest_b), later]);+ assert_eq!(+ repo.delete_batch(cutoff(), limit(1)).await.unwrap(),+ RetentionBatch::Deleted(1)+ );+ assert_eq!(remaining(&pool).await, [later]);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn parent_age_controls_all_states_and_cascades_to_receipts(pool: PgPool) {+ let old = insert(&pool, cutoff() - TimeDelta::days(1)).await;+ let fresh = insert(&pool, cutoff() + TimeDelta::days(1)).await;+ for (i, state) in [+ NotificationState::Unseen,+ NotificationState::Seen,+ NotificationState::Done,+ ]+ .into_iter()+ .enumerate()+ {+ let user = format!("macro|retention-{i}@test.com");+ for notification in [old, fresh] {+ // A fresh recipient on an expired parent must expire; an old+ // recipient on a fresh parent must survive. Include soft deletes.+ let child_created = if notification == old {+ cutoff() + TimeDelta::days(1)+ } else {+ cutoff() - TimeDelta::days(1)+ };+ let deleted_at = (i == 2).then_some(child_created);+ sqlx::query!(+ "INSERT INTO user_notification (user_id, notification_id, created_at, state, deleted_at)+ VALUES ($1, $2, $3, $4, $5)",+ user, notification, child_created, state as NotificationState, deleted_at,+ ).execute(&pool).await.unwrap();+ let message = format!("{notification}-{i}");+ sqlx::query!(+ "INSERT INTO notification_message_receipt (message_id, user_id, notification_id) VALUES ($1, $2, $3)",+ message, user, notification,+ ).execute(&pool).await.unwrap();+ }+ }+ // Settings with similar names are not notification history.+ sqlx::query!(+ "INSERT INTO user_mute_notification (user_id) VALUES ('macro|[email protected]')"+ )+ .execute(&pool)+ .await+ .unwrap();+ let repo = PgNotificationRetentionRepository::new(pool.clone());+ assert_eq!(+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),+ RetentionBatch::Deleted(1)+ );+ assert_eq!(remaining(&pool).await, [fresh]);+ assert_eq!(+ sqlx::query_scalar!("SELECT notification_id FROM user_notification")+ .fetch_all(&pool)+ .await+ .unwrap(),+ vec![fresh; 3]+ );+ assert_eq!(+ sqlx::query_scalar!("SELECT notification_id FROM notification_message_receipt")+ .fetch_all(&pool)+ .await+ .unwrap(),+ vec![fresh; 3]+ );+ assert_eq!(+ sqlx::query_scalar!("SELECT count(*) FROM user_mute_notification")+ .fetch_one(&pool)+ .await+ .unwrap(),+ Some(1)+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn locked_parents_are_revisited_and_advisory_lock_is_transaction_scoped(pool: PgPool) {+ let oldest = insert(&pool, cutoff() - TimeDelta::days(2)).await;+ let next = insert(&pool, cutoff() - TimeDelta::days(1)).await;+ let repo = PgNotificationRetentionRepository::new(pool.clone());+ let mut tx = pool.begin().await.unwrap();+ sqlx::query!(+ "SELECT id FROM notification WHERE id = $1 FOR UPDATE",+ oldest+ )+ .fetch_one(&mut *tx)+ .await+ .unwrap();+ assert_eq!(+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),+ RetentionBatch::Deleted(1)+ );+ assert_eq!(remaining(&pool).await, [oldest]);+ assert_ne!(oldest, next);+ tx.rollback().await.unwrap();++ let mut tx = pool.begin().await.unwrap();+ assert!(+ sqlx::query_scalar!(+ r#"SELECT pg_try_advisory_xact_lock($1) AS "locked!""#,+ RETENTION_LOCK+ )+ .fetch_one(&mut *tx)+ .await+ .unwrap()+ );+ assert_eq!(+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),+ RetentionBatch::Busy+ );+ tx.rollback().await.unwrap();+ assert_eq!(+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),+ RetentionBatch::Deleted(1)+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn cascade_lock_timeout_rolls_back_entire_batch_and_allows_retry(pool: PgPool) {+ let old = insert(&pool, cutoff() - TimeDelta::days(1)).await;+ sqlx::query!("INSERT INTO user_notification (user_id, notification_id) VALUES ('macro|[email protected]', $1)", old)+ .execute(&pool).await.unwrap();+ let mut tx = pool.begin().await.unwrap();+ sqlx::query!(+ "SELECT notification_id FROM user_notification WHERE notification_id = $1 FOR UPDATE",+ old+ )+ .fetch_one(&mut *tx)+ .await+ .unwrap();+ let repo = PgNotificationRetentionRepository::new(pool.clone());+ assert_eq!(+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),+ RetentionBatch::TimedOut+ );+ assert_eq!(remaining(&pool).await, [old]);+ tx.rollback().await.unwrap();+ assert_eq!(+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),+ RetentionBatch::Deleted(1)+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn deletion_notifies_every_recipient_in_byte_bounded_chunks(pool: PgPool) {+ let old = insert(&pool, cutoff() - TimeDelta::days(1)).await;+ // Multibyte addresses ensure the trigger measures bytes, not characters.+ let users: Vec<_> = (0..500)+ .map(|i| format!("macro|retention-é-{i}@test.com"))+ .collect();+ sqlx::query!(+ "INSERT INTO user_notification (user_id, notification_id) SELECT unnest($1::text[]), $2",+ &users,+ old+ )+ .execute(&pool)+ .await+ .unwrap();+ let mut listener = PgListener::connect_with(&pool).await.unwrap();+ listener.listen("notification_events").await.unwrap();+ let repo = PgNotificationRetentionRepository::new(pool.clone());+ assert_eq!(+ repo.delete_batch(cutoff(), limit(1)).await.unwrap(),+ RetentionBatch::Deleted(1)+ );+ let mut received = HashSet::new();+ let mut events = 0;+ while received.len() < users.len() {+ let event = tokio::time::timeout(std::time::Duration::from_secs(2), listener.recv())+ .await+ .unwrap()+ .unwrap();+ assert!(event.payload().len() < 8000);+ let payload: serde_json::Value = serde_json::from_str(event.payload()).unwrap();+ assert_eq!(payload["type"], "user_notification_deletes");+ assert_eq!(payload["notificationId"], old.to_string());+ for user in payload["userIds"].as_array().unwrap() {+ assert!(received.insert(user.as_str().unwrap().to_owned()));+ }+ events += 1;+ }+ assert!(events > 1);+ assert_eq!(received, users.into_iter().collect());+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn database_clock_uses_utc_even_with_a_non_utc_session(pool: PgPool) {+ let non_utc_pool = sqlx::postgres::PgPoolOptions::new()+ .max_connections(1)+ .connect_with(+ pool.connect_options()+ .as_ref()+ .clone()+ .options([("TimeZone", "Pacific/Auckland")]),+ )+ .await+ .unwrap();+ let repo = PgNotificationRetentionRepository::new(non_utc_pool.clone());+ let now = repo.current_time().await.unwrap();+ assert!((Utc::now() - now).num_seconds().abs() < 5);+ let cutoff = (now - crate::domain::retention::RETENTION).naive_utc();+ insert(&pool, cutoff - TimeDelta::days(1)).await;+ let fresh = insert(&pool, cutoff + TimeDelta::days(1)).await;+ assert_eq!(+ repo.delete_batch(cutoff, limit(100)).await.unwrap(),+ RetentionBatch::Deleted(1)+ );+ assert_eq!(remaining(&pool).await, [fresh]);+ non_utc_pool.close().await;+}diff --git a/docs/notification-retention.md b/docs/notification-retention.mdnew file mode 100644index 0000000000..c77d9c01ca--- /dev/null+++ b/docs/notification-retention.md@@ -0,0 +1,164 @@+# Notification retention++The internal `notification_retention_handler` Lambda removes notifications older+than 30 complete UTC days. Its EventBridge schedule is **disabled** in both dev+and prod, and its event payload defaults to **dry-run**. Deploying the code or+migrations alone does not purge history.++## Semantics++- The cutoff is the database clock at invocation start minus 30 days, converted+ to UTC for the schema's timezone-less `notification.created_at` column.+- Delete only parents with `created_at < cutoff`. Equality and newer timestamps+ survive. Reading, completing, reopening, or sending does not extend retention.+- All recipient states (`unseen`, `seen`, `done`) and soft-deleted recipients+ expire with the parent. The parent age wins even if recipient timestamps differ.+- Foreign keys cascade to `user_notification` and `notification_message_receipt`.+ Messages, documents, other source entities, preferences, unsubscribe records,+ device registrations, and email-send bookkeeping are not removed.+- Old unread notifications disappear. Items whose only feed-placement reason was+ an expired notification may disappear from that feed, not from source storage.+- This is eventual cleanup, normally within a five-minute schedule interval plus+ execution time after expiry once caught up. Backlogs, failures, or locked rows+ can extend that delay. There is no new read-time expiration filter.+- **Replay protection is intentionally out of scope.** Parent-row existence still+ provides notification-ID deduplication. After retention deletes a parent, an+ old event replay can recreate it with a fresh timestamp. Do not assume forever+ idempotency when replaying old ingress events.++## Execution and safety limits++The notification domain owns policy and batching in+`crates/notification/src/domain/retention.rs`. Its outbound adapter owns checked+SQL and transactions in `crates/notification/src/outbound/retention.rs`. The+Lambda only parses an internal event, computes remaining invocation time, and+calls the domain service. There is no HTTP route or user-callable cleanup API.+IAM controls invocation; do not grant end users permission to invoke this Lambda.++Each invocation:++1. Computes one cutoff using the database clock and previews at most 100 expired+ parents. `sampled_expired` is **not** a total backlog count.+2. In deletion mode, selects oldest-first using `(created_at, id)`, at most 100+ parents per transaction. A matching B-tree index avoids a history-wide sort.+3. Acquires a nonblocking transaction-scoped advisory lock, then uses+ `FOR UPDATE SKIP LOCKED`. Another cleanup transaction causes this invocation to+ stop. Locked parents are retried on future invocations; there is no watermark+ that could permanently skip them.+4. Deletes parents, allowing ordinary FK cascades and delete triggers, and commits+ each batch. No recipient payloads are returned to the worker.+5. Pauses 250 ms between batches. Caps work at 100 attempted batches and a+ 100-second invocation budget, leaving 15 seconds of headroom before starting+ another batch. The Lambda timeout is 120 seconds and remaining Lambda time is+ also respected.++The dedicated pool has one connection and a five-second acquisition timeout.+Statements time out after five seconds and lock waits after one second. Idle+transactions time out after ten seconds, so a stalled/frozen worker cannot hold+locks indefinitely. The handler also bounds the entire call by its remaining+invocation budget, including pool/network waits. A delete+statement/lock timeout rolls back that batch; the service halves the next batch+size. A timeout at one parent fails the invocation and raises the error alarm.+Other errors also fail, without undoing previously committed batches. A subsequent+invocation resumes naturally. Lambda concurrency is one; the advisory lock also+protects against other instances of this worker.++Parent limits do not bound recipient/receipt fanout. A single huge notification+can still exceed the timeout and require operator investigation. Never raise+limits or disable constraints/triggers merely to force progress.++## Realtime effects++The existing parent-delete trigger publishes `user_notification_deletes` events+before recipients cascade away. The migration splits recipients into JSON chunks+with a 7000-byte recipient-array budget, leaving room below PostgreSQL's 8000-byte+NOTIFY limit. Existing consumers accept the same event shape for each chunk.+An individual malformed/oversized recipient is warned about and skipped; a+publication failure remains best-effort, as in the previous trigger.++`LISTEN/NOTIFY` is not durable. Disconnected clients/listeners must reconcile on+refetch/reconnect; this change does not add durable tombstones or an outbox.+Watch notification-service/Kafka/connection-gateway load during backfill. Do not+turn off delete events to speed up the purge.++## Rollout++Production deletion needs explicit approval and a verified backup/PITR recovery+procedure. Do not run a historical `DELETE` inside a migration.++1. Apply the additive migrations through the normal approved database workflow.+ The retention index is built concurrently in its own nontransactional+ migration; the other migration replaces the delete-event function without+ changing its contract. Verify the index is valid. If a concurrent build fails,+ inspect its invalid index and repair/retry under the database deployment+ procedure rather than marking the migration successful.+2. Deploy the notification-service stack. Its deployment configuration also builds+ the new Lambda. Keep `notification_retention_enabled: false` and+ `notification_retention_dry_run: true` initially.+3. Inspect age distribution, recipient/receipt fanout, and query plans against+ representative data. Use plain `EXPLAIN` for production deletion plans;+ `EXPLAIN ANALYZE DELETE` actually deletes data.+4. Enable the schedule **in dry-run mode** first:++ ```bash+ \cd infra/stacks/notification-service+ pulumi config set notification_retention_enabled true --stack <dev-or-prod>+ pulumi config set notification_retention_dry_run true --stack <dev-or-prod>+ # Review/apply through the normal deployment workflow.+ ```++5. After approval and reviewing dry-run logs, change+ `notification_retention_dry_run` to `false` and deploy. Drain existing history+ with these same bounded batches; do not substitute a bulk deletion. Initial+ capacity is at most 10,000 parents per invocation (usually less). Verify+ steady-state capacity exceeds incoming expiration volume before enabling prod.+6. Monitor application latency, database CPU/I/O, WAL and replica lag, dead tuples,+ vacuum progress, NOTIFY warnings/queue pressure, and realtime consumer lag.+ Pause the schedule on unacceptable impact. Do not automatically run+ `VACUUM FULL`; ordinary vacuum reclaims space for reuse, not necessarily for+ returning to the filesystem.++A manual invocation with `{}` or `{"dry_run": true}` is read-only.+`{"dry_run": false}` explicitly opts into deletion. Unknown fields, alternate+spellings, and nonboolean settings are rejected. Invocation cannot override the+retention period or batch caps.++## Monitoring and stopping++Infrastructure alarms on Lambda errors, throttles, and no invocations for+15 minutes while the schedule is enabled. Each sweep logs the cutoff,+`sampled_expired`, `oldest_expired_at`, committed parent `deleted` count,+attempted `batches`, and stopping reason. Each committed batch is also logged,+so partial progress remains visible if a later batch fails.++Track `cutoff - oldest_expired_at` as expiration backlog age. During backfill it+should trend downward; once caught up, sustained growth indicates insufficient+capacity, long-lived locks, or a failing high-fanout parent. A zero-row batch can+mean rows were locked, not necessarily that every expired row has been removed.+Do not run a full `COUNT(*)` over the backlog every five minutes.++Disable `notification_retention_enabled` and deploy to stop future scheduled+invocations. An already-running invocation may still commit batches. Emergency+operational controls may also stop future invokes; they do not undo commits.+Disabling the worker or rolling back migrations **cannot restore deleted data**.+Recovery requires the approved backup/PITR process.++Retention bounds age, not cardinality. Busy users can still have many notifications+within 30 days; pagination of currently unbounded consumers remains separate work.++## Local verification++Run from the repository root with local PostgreSQL and `SQLX_OFFLINE` unset:++```bash+cargo test -p notification -p notification_retention_handler -- --test-threads=8+nix develop --command just prepare_db+just check+\cd infra+bunx tsc --noEmit --project stacks/notification-service/tsconfig.json+```++Tests cover cutoff boundaries, parent-vs-recipient age, all states, cascades,+read-only previews, deterministic bounded ordering, retry behavior, skipped locks,+transaction advisory locking, timeout rollback, non-UTC sessions, byte-bounded+multi-recipient delete events, invocation caps, and fail-closed event parsing.diff --git a/infra/stacks/notification-service/Pulumi.dev.yaml b/infra/stacks/notification-service/Pulumi.dev.yamlindex dcdb6f8af4..b0537a4f73 100644--- a/infra/stacks/notification-service/Pulumi.dev.yaml+++ b/infra/stacks/notification-service/Pulumi.dev.yaml@@ -1,6 +1,9 @@config:aws-native:region: us-east-1aws:region: us-east-1+ notification-service:macro_db_secret_key: macro-db-dev+ notification-service:notification_retention_enabled: false+ notification-service:notification_retention_dry_run: truenotification-service:jwt_secret_key: fusionauth-jwt-secret-devnotification-service:fusionauth_client_id: fusionauth-client-id-key-devnotification-service:fusionauth_issuer: fusionauth-dev.macro.comdiff --git a/infra/stacks/notification-service/Pulumi.prod.yaml b/infra/stacks/notification-service/Pulumi.prod.yamlindex 809c1efa52..2c9076cd67 100644--- a/infra/stacks/notification-service/Pulumi.prod.yaml+++ b/infra/stacks/notification-service/Pulumi.prod.yaml@@ -1,6 +1,9 @@config:aws-native:region: us-east-1aws:region: us-east-1+ notification-service:macro_db_secret_key: macro-db-prod+ notification-service:notification_retention_enabled: false+ notification-service:notification_retention_dry_run: truenotification-service:jwt_secret_key: fusionauth-jwt-secret-prodnotification-service:fusionauth_client_id: fusionauth-client-id-key-prodnotification-service:fusionauth_issuer: auth.macro.comdiff --git a/infra/stacks/notification-service/index.ts b/infra/stacks/notification-service/index.tsindex 85642eb31e..1676ade77a 100644--- a/infra/stacks/notification-service/index.ts+++ b/infra/stacks/notification-service/index.ts@@ -11,6 +11,7 @@ import {} from '../../packages/shared';import { get_coparse_api_vpc } from '../../packages/vpc';import { PushNotificationEventHandler } from './push';+import { NotificationRetention } from './retention';import { NotificationService } from './service';const tags = {@@ -63,6 +64,19 @@ const authenticationServiceInternalApiKeyArn: pulumi.Output<string> =export const coparse_api_vpc = get_coparse_api_vpc();+// Fetch as a secret Pulumi output; never export the database URL.+const retentionDatabaseUrl = aws.secretsmanager+ .getSecretVersionOutput({ secretId: config.require('macro_db_secret_key') })+ .apply((secret) => pulumi.secret(secret.secretString));++new NotificationRetention('notification-retention', {+ databaseUrl: retentionDatabaseUrl,+ enabled: config.getBoolean('notification_retention_enabled') ?? false,+ dryRun: config.getBoolean('notification_retention_dry_run') ?? true,+ vpc: coparse_api_vpc,+ tags,+});+const cloudStorageStack = new pulumi.StackReference('cloud-storage-stack', {name: `macro-inc/document-storage/${stack}`,});diff --git a/infra/stacks/notification-service/retention.ts b/infra/stacks/notification-service/retention.tsnew file mode 100644index 0000000000..517ec4b976--- /dev/null+++ b/infra/stacks/notification-service/retention.ts@@ -0,0 +1,158 @@+import * as aws from '@pulumi/aws';+import * as pulumi from '@pulumi/pulumi';+import { Lambda } from '../../packages/lambda';+import { CLOUD_TRAIL_SNS_TOPIC_ARN, stack } from '../../packages/shared';++const BASE_NAME = 'notification_retention_handler';+const REPO_ROOT = '../../..';++type RetentionEnvVars = {+ DATABASE_URL: pulumi.Output<string> | string;+ ENVIRONMENT: string;+ RUST_LOG: string;+};++type Args = {+ databaseUrl: pulumi.Output<string>;+ enabled: boolean;+ dryRun: boolean;+ vpc: {+ vpcId: pulumi.Output<string> | string;+ publicSubnetIds: pulumi.Output<string[]> | string[];+ privateSubnetIds: pulumi.Output<string[]> | string[];+ };+ tags: Record<string, string>;+};++/** Isolated maintenance worker: disabled initially, then preview before deleting. */+export class NotificationRetention extends pulumi.ComponentResource {+ readonly lambda: aws.lambda.Function;++ constructor(+ name: string,+ args: Args,+ opts?: pulumi.ComponentResourceOptions+ ) {+ super('macro:notifications:Retention', name, {}, opts);+ const { databaseUrl, enabled, dryRun, vpc, tags } = args;+ const role = new aws.iam.Role(+ `${BASE_NAME}-role`,+ {+ assumeRolePolicy: JSON.stringify({+ Version: '2012-10-17',+ Statement: [+ {+ Effect: 'Allow',+ Principal: { Service: 'lambda.amazonaws.com' },+ Action: 'sts:AssumeRole',+ },+ ],+ }),+ managedPolicyArns: [+ aws.iam.ManagedPolicy.AWSLambdaBasicExecutionRole,+ aws.iam.ManagedPolicy.AWSLambdaVPCAccessExecutionRole,+ ],+ tags,+ },+ { parent: this }+ );+ this.lambda = new Lambda<RetentionEnvVars>(+ `${BASE_NAME}-lambda`,+ {+ baseName: BASE_NAME,+ handlerBase: `${REPO_ROOT}/services/${BASE_NAME}`,+ zipLocation: `${REPO_ROOT}/target/lambda/${BASE_NAME}/bootstrap.zip`,+ envVars: {+ DATABASE_URL: databaseUrl,+ ENVIRONMENT: stack,+ RUST_LOG: 'info',+ },+ role,+ vpc,+ memorySize: 256,+ timeout: 120,+ reservedConcurrentExecutions: 1,+ tags,+ },+ { parent: this }+ ).lambda;++ const rule = new aws.cloudwatch.EventRule(+ `${BASE_NAME}-schedule`,+ {+ scheduleExpression: 'rate(5 minutes)',+ state: enabled ? 'ENABLED' : 'DISABLED',+ tags,+ },+ { parent: this }+ );+ new aws.cloudwatch.EventTarget(+ `${BASE_NAME}-target`,+ {+ rule: rule.name,+ arn: this.lambda.arn,+ input: JSON.stringify({ dry_run: dryRun }),+ retryPolicy: { maximumEventAgeInSeconds: 60, maximumRetryAttempts: 0 },+ },+ { parent: this }+ );+ new aws.lambda.Permission(+ `${BASE_NAME}-invoke`,+ {+ action: 'lambda:InvokeFunction',+ function: this.lambda.name,+ principal: 'events.amazonaws.com',+ sourceArn: rule.arn,+ },+ { parent: this }+ );+ // The next scheduled sweep retries any outstanding rows. Don't accumulate+ // asynchronous retries of an old invocation while the worker is throttled.+ new aws.lambda.FunctionEventInvokeConfig(+ `${BASE_NAME}-async-policy`,+ {+ functionName: this.lambda.name,+ maximumEventAgeInSeconds: 60,+ maximumRetryAttempts: 0,+ },+ { parent: this }+ );+ for (const metricName of ['Errors', 'Throttles']) {+ new aws.cloudwatch.MetricAlarm(+ `${BASE_NAME}-${metricName.toLowerCase()}`,+ {+ namespace: 'AWS/Lambda',+ metricName,+ dimensions: { FunctionName: this.lambda.name },+ statistic: 'Sum',+ period: 300,+ evaluationPeriods: 1,+ threshold: 1,+ comparisonOperator: 'GreaterThanOrEqualToThreshold',+ treatMissingData: 'notBreaching',+ alarmActions: [CLOUD_TRAIL_SNS_TOPIC_ARN],+ tags,+ },+ { parent: this }+ );+ }+ new aws.cloudwatch.MetricAlarm(+ `${BASE_NAME}-missing-invocations`,+ {+ namespace: 'AWS/Lambda',+ metricName: 'Invocations',+ dimensions: { FunctionName: this.lambda.name },+ statistic: 'Sum',+ period: 900,+ evaluationPeriods: 1,+ threshold: 1,+ comparisonOperator: 'LessThanThreshold',+ treatMissingData: enabled ? 'breaching' : 'notBreaching',+ actionsEnabled: enabled,+ alarmActions: [CLOUD_TRAIL_SNS_TOPIC_ARN],+ tags,+ },+ { parent: this }+ );+ }+}diff --git a/services/notification_retention_handler/Cargo.toml b/services/notification_retention_handler/Cargo.tomlnew file mode 100644index 0000000000..9b628dc42c--- /dev/null+++ b/services/notification_retention_handler/Cargo.toml@@ -0,0 +1,19 @@+[package]+edition = "2024"+name = "notification_retention_handler"+publish = false+version = "0.1.0"++[dependencies]+lambda_runtime = { workspace = true }+macro_entrypoint = { path = "../../crates/macro_entrypoint" }+macro_env_var = { path = "../../crates/macro_env_var" }+notification = { path = "../../crates/notification" }+serde = { workspace = true }+sqlx = { workspace = true }+tokio = { workspace = true }+tracing = { workspace = true }+workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }++[dev-dependencies]+serde_json = { workspace = true }diff --git a/services/notification_retention_handler/justfile b/services/notification_retention_handler/justfilenew file mode 100644index 0000000000..9c5ce2bb17--- /dev/null+++ b/services/notification_retention_handler/justfile@@ -0,0 +1,4 @@+lambda_name := file_name(justfile_directory())++build compiler="":+ SQLX_OFFLINE=true cargo lambda build {{ if compiler == "" { "" } else { "--compiler " + compiler } }} --release --bin {{lambda_name}} --output-format zipdiff --git a/services/notification_retention_handler/src/handler.rs b/services/notification_retention_handler/src/handler.rsnew file mode 100644index 0000000000..1390af3244--- /dev/null+++ b/services/notification_retention_handler/src/handler.rs@@ -0,0 +1,54 @@+//! Thin Lambda adapter. Empty/manual payloads default to a read-only preview.++use std::time::{Duration, SystemTime};++use lambda_runtime::{Error, LambdaEvent};+use notification::domain::retention::{+ NotificationRetentionRepository, NotificationRetentionService, RetentionMode,+};+use serde::Deserialize;++#[derive(Debug, Deserialize)]+#[serde(deny_unknown_fields)]+pub(crate) struct CleanupEvent {+ #[serde(default = "default_dry_run")]+ dry_run: bool,+}++fn default_dry_run() -> bool {+ true+}++impl CleanupEvent {+ fn mode(&self) -> RetentionMode {+ if self.dry_run {+ RetentionMode::DryRun+ } else {+ RetentionMode::Delete+ }+ }+}++#[tracing::instrument(err, skip(service, event))]+pub(crate) async fn handle<R: NotificationRetentionRepository>(+ service: &NotificationRetentionService<R>,+ event: LambdaEvent<CleanupEvent>,+) -> Result<(), Error> {+ let budget = event+ .context+ .deadline()+ .duration_since(SystemTime::now())+ .unwrap_or_default()+ .saturating_sub(Duration::from_secs(10));+ if budget < Duration::from_secs(15) {+ tracing::warn!("not enough invocation time for notification retention; skipping");+ return Ok(());+ }+ // Also bound stalls outside SQL execution (pool acquisition/network I/O).+ // Dropping an in-flight repository future rolls back its transaction.+ tokio::time::timeout(budget, service.run(event.payload.mode(), budget)).await??;+ Ok(())+}++#[cfg(test)]+mod test;diff --git a/services/notification_retention_handler/src/handler/test.rs b/services/notification_retention_handler/src/handler/test.rsnew file mode 100644index 0000000000..d154eec3bc--- /dev/null+++ b/services/notification_retention_handler/src/handler/test.rs@@ -0,0 +1,23 @@+use super::*;++#[test]+fn manual_invocation_is_read_only_unless_explicitly_requested() {+ for payload in ["{}", r#"{"dry_run":true}"#] {+ let event: CleanupEvent = serde_json::from_str(payload).unwrap();+ assert_eq!(event.mode(), RetentionMode::DryRun);+ }+ let event: CleanupEvent = serde_json::from_str(r#"{"dry_run":false}"#).unwrap();+ assert_eq!(event.mode(), RetentionMode::Delete);+}++#[test]+fn invalid_or_unsupported_settings_fail_closed() {+ for payload in [+ r#"{"dry_run":null}"#,+ r#"{"dry_run":"false"}"#,+ r#"{"dryRun":false}"#,+ r#"{"dry_run":false,"retention_days":0}"#,+ ] {+ assert!(serde_json::from_str::<CleanupEvent>(payload).is_err());+ }+}diff --git a/services/notification_retention_handler/src/main.rs b/services/notification_retention_handler/src/main.rsnew file mode 100644index 0000000000..79fb641ee0--- /dev/null+++ b/services/notification_retention_handler/src/main.rs@@ -0,0 +1,47 @@+//! Scheduled notification retention composition root. Invocation is internal;+//! the notification domain owns the policy and all deletion orchestration.++mod handler;++use std::{str::FromStr, sync::Arc, time::Duration};++use lambda_runtime::{Error, run, service_fn};+use macro_entrypoint::MacroEntrypoint;+use macro_env_var::env_vars;+use notification::{+ domain::retention::NotificationRetentionService,+ outbound::retention::PgNotificationRetentionRepository,+};+use sqlx::postgres::{PgConnectOptions, PgPoolOptions};++env_vars! {+ struct DatabaseUrl;+}++#[tokio::main]+async fn main() -> Result<(), Error> {+ MacroEntrypoint::default().init();+ let database_url = DatabaseUrl::new()?;+ // Bound preview queries as well as deletes. One maintenance connection per+ // process, and reserved Lambda concurrency of one, isolate foreground work.+ let options = PgConnectOptions::from_str(database_url.as_ref())?+ .application_name("notification_retention_handler")+ .options([+ ("statement_timeout", "5000"),+ ("lock_timeout", "1000"),+ ("idle_in_transaction_session_timeout", "10000"),+ ]);+ let pool = PgPoolOptions::new()+ .max_connections(1)+ .acquire_timeout(Duration::from_secs(5))+ .connect_with(options)+ .await?;+ let service = Arc::new(NotificationRetentionService::new(+ PgNotificationRetentionRepository::new(pool),+ ));+ run(service_fn(move |event| {+ let service = Arc::clone(&service);+ async move { handler::handle(&service, event).await }+ }))+ .await+}diff --git a/tooling/just/rust.just b/tooling/just/rust.justindex bd63226278..eac2ee67df 100644--- a/tooling/just/rust.just+++ b/tooling/just/rust.just@@ -56,6 +56,7 @@ build_lambdas compiler="":ulimit -n 10240 && just services/deleted_item_poller/build {{compiler}}ulimit -n 10240 && just services/email_refresh_handler/build {{compiler}}ulimit -n 10240 && just services/user_link_cleanup_handler/build {{compiler}}+ ulimit -n 10240 && just services/notification_retention_handler/build {{compiler}}ulimit -n 10240 && just services/email_scheduled_handler/build {{compiler}}ulimit -n 10240 && just services/email_sfs_delete_handler/build {{compiler}}ulimit -n 10240 && just services/image_optimizer/build {{compiler}}