Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/feat/notification retention (#8083)

Sean Aye37077ec462fb
diff --git a/.github/services-config.json b/.github/services-config.json
index 800913ccc9..ea1df00cab 100644
--- a/.github/services-config.json
+++ b/.github/services-config.json
@@ -185,11 +185,15 @@
},
"notification-service": {
"source_paths": [
- "services/notification_service/**"
+ "services/notification_service/**",
+ "services/notification_retention_handler/**"
],
"stack_path": "infra/stacks/notification-service/**",
"deploy_binaries": [
"notification_service"
+ ],
+ "deploy_lambdas": [
+ "notification_retention_handler"
]
},
"organization-retention": {
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 706f35c2ff..6e5ec4093f 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -10199,6 +10199,38 @@
"crates/remote_env_var",
"crates/workspace-hack"
],
+ "notification_retention_handler": [
+ "crates/ai_toolset",
+ "crates/bot_id",
+ "crates/bot_token",
+ "crates/broadcast",
+ "crates/cowlike",
+ "crates/genai_telemetry",
+ "crates/harness_id",
+ "crates/harness_token",
+ "crates/kafka_util",
+ "crates/last_online_tracker",
+ "crates/macro_auth",
+ "crates/macro_authorization",
+ "crates/macro_entrypoint",
+ "crates/macro_env",
+ "crates/macro_env_var",
+ "crates/macro_event_broker",
+ "crates/macro_event_topics",
+ "crates/macro_service_urls",
+ "crates/macro_user_id",
+ "crates/macro_uuid",
+ "crates/model-entity",
+ "crates/model-error-response",
+ "crates/model_user",
+ "crates/models_pagination",
+ "crates/notification",
+ "crates/notification_state",
+ "crates/rate_limit",
+ "crates/remote_env_var",
+ "crates/workspace-hack",
+ "services/notification_retention_handler"
+ ],
"notification_sandbox": [
"crates/ai_toolset",
"crates/bot_id",
diff --git a/.sqlx/query-370764911f1740f87020c23f75698124a953a3ee175b20eafbb522bf7cf8d1bf.json b/.sqlx/query-370764911f1740f87020c23f75698124a953a3ee175b20eafbb522bf7cf8d1bf.json
new file mode 100644
index 0000000000..d24dbdcc50
--- /dev/null
+++ b/.sqlx/query-370764911f1740f87020c23f75698124a953a3ee175b20eafbb522bf7cf8d1bf.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT pg_try_advisory_xact_lock($1) AS \"acquired!\"",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "acquired!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "370764911f1740f87020c23f75698124a953a3ee175b20eafbb522bf7cf8d1bf"
+}
diff --git a/.sqlx/query-6af068223ccebea85f085701b94b10b7f61c6912dea741ba74d717c4f4f2af87.json b/.sqlx/query-6af068223ccebea85f085701b94b10b7f61c6912dea741ba74d717c4f4f2af87.json
new file mode 100644
index 0000000000..709dbb331f
--- /dev/null
+++ b/.sqlx/query-6af068223ccebea85f085701b94b10b7f61c6912dea741ba74d717c4f4f2af87.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH expired AS (\n SELECT id FROM notification\n WHERE created_at < $1\n ORDER BY created_at, id\n LIMIT $2\n FOR UPDATE SKIP LOCKED\n )\n DELETE FROM notification n\n USING expired e\n WHERE n.id = e.id\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Timestamp",
+ "Int8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "6af068223ccebea85f085701b94b10b7f61c6912dea741ba74d717c4f4f2af87"
+}
diff --git a/.sqlx/query-78ae885ce1b7d7df2dff41d4699d8f73ca0a9279d4167e82c0e603d7d2d82884.json b/.sqlx/query-78ae885ce1b7d7df2dff41d4699d8f73ca0a9279d4167e82c0e603d7d2d82884.json
new file mode 100644
index 0000000000..3721c454fb
--- /dev/null
+++ b/.sqlx/query-78ae885ce1b7d7df2dff41d4699d8f73ca0a9279d4167e82c0e603d7d2d82884.json
@@ -0,0 +1,20 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT clock_timestamp() AS \"now!\"",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "now!",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "78ae885ce1b7d7df2dff41d4699d8f73ca0a9279d4167e82c0e603d7d2d82884"
+}
diff --git a/.sqlx/query-b3379073b89935dd1359532395b8a04a9873349b5ae80066e024ca19ef5ebba8.json b/.sqlx/query-b3379073b89935dd1359532395b8a04a9873349b5ae80066e024ca19ef5ebba8.json
new file mode 100644
index 0000000000..55ca8a5b43
--- /dev/null
+++ b/.sqlx/query-b3379073b89935dd1359532395b8a04a9873349b5ae80066e024ca19ef5ebba8.json
@@ -0,0 +1,29 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT count(*) AS \"count!\", min(created_at) AS oldest\n FROM (\n SELECT created_at FROM notification\n WHERE created_at < $1\n ORDER BY created_at, id\n LIMIT $2\n ) expired\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "count!",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "oldest",
+ "type_info": "Timestamp"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Timestamp",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ null,
+ null
+ ]
+ },
+ "hash": "b3379073b89935dd1359532395b8a04a9873349b5ae80066e024ca19ef5ebba8"
+}
diff --git a/.sqlx/query-d9946d44ba3ecfd34d8e9419a4918e0cee4d99c52737f63b62043c82b60030e5.json b/.sqlx/query-d9946d44ba3ecfd34d8e9419a4918e0cee4d99c52737f63b62043c82b60030e5.json
new file mode 100644
index 0000000000..b30b311b52
--- /dev/null
+++ b/.sqlx/query-d9946d44ba3ecfd34d8e9419a4918e0cee4d99c52737f63b62043c82b60030e5.json
@@ -0,0 +1,12 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SET LOCAL statement_timeout = '5s'",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": []
+ },
+ "hash": "d9946d44ba3ecfd34d8e9419a4918e0cee4d99c52737f63b62043c82b60030e5"
+}
diff --git a/.sqlx/query-e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533.json b/.sqlx/query-e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533.json
new file mode 100644
index 0000000000..fb106d1b14
--- /dev/null
+++ b/.sqlx/query-e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533.json
@@ -0,0 +1,12 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SET LOCAL lock_timeout = '1s'",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": []
+ },
+ "hash": "e87a3f76d9caee326b4f36802430f3ea370cd374183c6a0e229aeada18bba533"
+}
diff --git a/Cargo.lock b/Cargo.lock
index 590a95a19b..fce86cd057 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -13215,6 +13215,22 @@ dependencies = [
"workspace-hack",
]
+[[package]]
+name = "notification_retention_handler"
+version = "0.1.0"
+dependencies = [
+ "lambda_runtime",
+ "macro_entrypoint",
+ "macro_env_var",
+ "notification",
+ "serde",
+ "serde_json",
+ "sqlx",
+ "tokio",
+ "tracing",
+ "workspace-hack",
+]
+
[[package]]
name = "notification_sandbox"
version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 5058eae091..9314d2ae2f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -152,6 +152,7 @@ members = [
"services/image_proxy_service",
"services/mcp_auth_proxy",
"services/mcp_service",
+ "services/notification_retention_handler",
"services/notification_service",
"services/organization_retention_handler",
"services/organization_retention_trigger",
diff --git a/crates/macro_db_client/migrations/20261015151242_notification_retention_index.down.sql b/crates/macro_db_client/migrations/20261015151242_notification_retention_index.down.sql
new file mode 100644
index 0000000000..468e14cc85
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261015151242_notification_retention_index.down.sql
@@ -0,0 +1,2 @@
+-- no-transaction
+DROP INDEX CONCURRENTLY IF EXISTS idx_notification_retention;
diff --git a/crates/macro_db_client/migrations/20261015151242_notification_retention_index.up.sql b/crates/macro_db_client/migrations/20261015151242_notification_retention_index.up.sql
new file mode 100644
index 0000000000..80ec32f9c2
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261015151242_notification_retention_index.up.sql
@@ -0,0 +1,5 @@
+-- no-transaction
+-- One statement: SQLx sends nontransactional migrations as one query.
+-- Retention scans oldest-first without sorting the notification history.
+CREATE INDEX CONCURRENTLY idx_notification_retention
+ ON notification (created_at, id);
diff --git a/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.down.sql b/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.down.sql
new file mode 100644
index 0000000000..ab6bff7c0a
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.down.sql
@@ -0,0 +1,29 @@
+-- Restore the previous unchunked event publisher. This does not restore any
+-- notifications deleted by the worker; retention deletion is irreversible.
+CREATE OR REPLACE FUNCTION notify_user_notification_deletes()
+RETURNS TRIGGER AS $$
+DECLARE
+ affected_user_ids JSONB;
+BEGIN
+ SELECT jsonb_agg(user_id)
+ INTO affected_user_ids
+ FROM user_notification
+ WHERE notification_id = OLD.id;
+
+ IF affected_user_ids IS NOT NULL THEN
+ BEGIN
+ PERFORM pg_notify(
+ 'notification_events',
+ jsonb_build_object(
+ 'type', 'user_notification_deletes',
+ 'notificationId', OLD.id,
+ 'userIds', affected_user_ids
+ )::TEXT
+ );
+ EXCEPTION WHEN OTHERS THEN
+ RAISE WARNING 'pg_notify failed: %', SQLERRM;
+ END;
+ END IF;
+ RETURN OLD;
+END;
+$$ LANGUAGE plpgsql;
diff --git a/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.up.sql b/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.up.sql
new file mode 100644
index 0000000000..7803ca7e49
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261015151257_chunk_notification_delete_events.up.sql
@@ -0,0 +1,49 @@
+-- Preserve the existing event shape while keeping every NOTIFY payload under
+-- PostgreSQL's 8000-byte limit. Bulk retention must not silently lose updates
+-- for notifications addressed to many users. No history is deleted here.
+CREATE OR REPLACE FUNCTION notify_user_notification_deletes()
+RETURNS TRIGGER AS $$
+DECLARE
+ recipient TEXT;
+ affected_user_ids JSONB := '[]'::JSONB;
+ recipient_json JSONB;
+BEGIN
+ FOR recipient IN
+ SELECT user_id FROM user_notification WHERE notification_id = OLD.id
+ LOOP
+ recipient_json := jsonb_build_array(recipient);
+ -- Leave ample room for the fixed event envelope (type and UUID).
+ IF octet_length(recipient_json::TEXT) > 7000 THEN
+ RAISE WARNING 'notification delete recipient exceeds NOTIFY payload budget';
+ CONTINUE;
+ END IF;
+
+ IF octet_length((affected_user_ids || recipient_json)::TEXT) > 7000 THEN
+ BEGIN
+ PERFORM pg_notify('notification_events', jsonb_build_object(
+ 'type', 'user_notification_deletes',
+ 'notificationId', OLD.id,
+ 'userIds', affected_user_ids
+ )::TEXT);
+ EXCEPTION WHEN OTHERS THEN
+ RAISE WARNING 'pg_notify failed: %', SQLERRM;
+ END;
+ affected_user_ids := '[]'::JSONB;
+ END IF;
+ affected_user_ids := affected_user_ids || recipient_json;
+ END LOOP;
+
+ IF jsonb_array_length(affected_user_ids) > 0 THEN
+ BEGIN
+ PERFORM pg_notify('notification_events', jsonb_build_object(
+ 'type', 'user_notification_deletes',
+ 'notificationId', OLD.id,
+ 'userIds', affected_user_ids
+ )::TEXT);
+ EXCEPTION WHEN OTHERS THEN
+ RAISE WARNING 'pg_notify failed: %', SQLERRM;
+ END;
+ END IF;
+ RETURN OLD;
+END;
+$$ LANGUAGE plpgsql;
diff --git a/crates/notification/src/domain.rs b/crates/notification/src/domain.rs
index db2c301ff3..887d102049 100644
--- a/crates/notification/src/domain.rs
+++ b/crates/notification/src/domain.rs
@@ -1,4 +1,5 @@
pub mod item_preferences;
pub mod models;
pub mod ports;
+pub mod retention;
pub mod service;
diff --git a/crates/notification/src/domain/retention.rs b/crates/notification/src/domain/retention.rs
new file mode 100644
index 0000000000..1683c52f0e
--- /dev/null
+++ b/crates/notification/src/domain/retention.rs
@@ -0,0 +1,188 @@
+//! Bounded, internal maintenance of notification history.
+//!
+//! Retention applies to the parent notification's creation time, independently
+//! of recipient state. The scheduler is the authorization boundary: this use
+//! case is deliberately not exposed by a user-facing API.
+
+use std::{num::NonZeroU16, time::Duration};
+
+use chrono::{DateTime, NaiveDateTime, TimeDelta, Utc};
+use rootcause::{Report, bail};
+use tokio::time::Instant;
+
+/// Notifications live for thirty complete UTC days.
+pub const RETENTION: TimeDelta = TimeDelta::days(30);
+const BATCH_SIZE: NonZeroU16 = NonZeroU16::new(100).unwrap();
+const MAX_BATCHES: u16 = 100;
+const MAX_RUN_TIME: Duration = Duration::from_secs(100);
+const BATCH_HEADROOM: Duration = Duration::from_secs(15);
+const BATCH_PAUSE: Duration = Duration::from_millis(250);
+
+/// Explicitly distinguish a read-only preview from destructive cleanup.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum RetentionMode {
+ /// Inspect at most one batch without taking deletion locks or deleting rows.
+ DryRun,
+ /// Delete expired notifications and their dependent rows.
+ Delete,
+}
+
+/// A bounded preview; the count is a sample, not the total backlog.
+#[derive(Debug, Default)]
+pub struct RetentionPreview {
+ /// Number of expired parents in the sampled batch.
+ pub sampled_expired: u64,
+ /// Creation time of the oldest expired parent, if any.
+ pub oldest_expired_at: Option<NaiveDateTime>,
+}
+
+/// Outcome of one independently committed deletion transaction.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum RetentionBatch {
+ /// Number of deleted parents (not the number of cascaded recipient rows).
+ Deleted(u64),
+ /// Another cleanup transaction holds the maintenance lock.
+ Busy,
+ /// The transaction rolled back after a statement or lock timeout.
+ TimedOut,
+}
+
+/// Storage capabilities needed by the internal retention use case.
+pub trait NotificationRetentionRepository: Send + Sync {
+ /// Use the database clock, not the worker's wall clock, for the cutoff.
+ fn current_time(&self) -> impl Future<Output = Result<DateTime<Utc>, Report>> + Send;
+
+ /// Inspect only the oldest `limit` expired parents using an ordered index.
+ fn preview(
+ &self,
+ cutoff: NaiveDateTime,
+ limit: NonZeroU16,
+ ) -> impl Future<Output = Result<RetentionPreview, Report>> + Send;
+
+ /// Atomically delete at most `limit` parents strictly older than `cutoff`.
+ /// Preserve cascade/notification triggers, skip locked parents, and release
+ /// the maintenance lock on commit or rollback, including on cancellation.
+ fn delete_batch(
+ &self,
+ cutoff: NaiveDateTime,
+ limit: NonZeroU16,
+ ) -> impl Future<Output = Result<RetentionBatch, Report>> + Send;
+}
+
+/// Why an invocation stopped. A zero-row batch may mean rows were locked;
+/// it is not proof that the backlog has been completely drained.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum RetentionStop {
+ /// Read-only preview completed.
+ DryRun,
+ /// No currently available expired parents.
+ NoCandidates,
+ /// Another cleanup invocation is working.
+ Busy,
+ /// Leave enough time to finish the current invocation safely.
+ TimeBudget,
+ /// The per-invocation batch cap was reached.
+ BatchLimit,
+}
+
+/// Summary suitable for operational logs; contains no recipient data.
+#[derive(Debug)]
+pub struct RetentionReport {
+ /// Fixed UTC cutoff shared by every batch in this invocation.
+ pub cutoff: NaiveDateTime,
+ /// Bounded snapshot taken before deletion.
+ pub preview: RetentionPreview,
+ /// Number of parent notifications deleted by committed batches.
+ pub deleted: u64,
+ /// Number of batches attempted, including timeout retries.
+ pub batches: u16,
+ /// Reason this invocation stopped.
+ pub stopped: RetentionStop,
+}
+
+/// Enforces retention policy and throttles maintenance work between commits.
+pub struct NotificationRetentionService<R> {
+ repository: R,
+}
+
+impl<R: NotificationRetentionRepository> NotificationRetentionService<R> {
+ /// Construct the internal service over its storage port.
+ pub fn new(repository: R) -> Self {
+ Self { repository }
+ }
+
+ /// Run a bounded sweep. Retries are safe: committed deletes disappear from
+ /// the next scan, and skipped/rolled-back rows are reconsidered next time.
+ #[tracing::instrument(err, skip(self), fields(deleted, batches, stopped))]
+ pub async fn run(
+ &self,
+ mode: RetentionMode,
+ time_budget: Duration,
+ ) -> Result<RetentionReport, Report> {
+ let deadline = Instant::now() + time_budget.min(MAX_RUN_TIME);
+ let cutoff = (self.repository.current_time().await? - RETENTION).naive_utc();
+ let preview = self.repository.preview(cutoff, BATCH_SIZE).await?;
+ let mut report = RetentionReport {
+ cutoff,
+ preview,
+ deleted: 0,
+ batches: 0,
+ stopped: RetentionStop::DryRun,
+ };
+
+ if mode == RetentionMode::Delete {
+ let mut batch_size = BATCH_SIZE;
+ report.stopped = loop {
+ if report.batches >= MAX_BATCHES {
+ break RetentionStop::BatchLimit;
+ }
+ if Instant::now() + BATCH_HEADROOM >= deadline {
+ break RetentionStop::TimeBudget;
+ }
+ report.batches += 1;
+ match self.repository.delete_batch(cutoff, batch_size).await? {
+ RetentionBatch::Busy => break RetentionStop::Busy,
+ RetentionBatch::Deleted(0) => break RetentionStop::NoCandidates,
+ RetentionBatch::Deleted(deleted) => {
+ report.deleted += deleted;
+ tracing::info!(
+ deleted,
+ batch = report.batches,
+ "notification retention batch committed"
+ );
+ }
+ RetentionBatch::TimedOut => {
+ let Some(smaller) = NonZeroU16::new(batch_size.get() / 2) else {
+ bail!(
+ "notification retention timed out deleting a single parent; inspect recipient fanout and locks"
+ );
+ };
+ batch_size = smaller;
+ tracing::warn!(
+ batch_size = batch_size.get(),
+ "notification retention batch timed out; reducing batch size"
+ );
+ }
+ }
+ tokio::time::sleep(BATCH_PAUSE).await;
+ };
+ }
+
+ tracing::Span::current().record("deleted", report.deleted);
+ tracing::Span::current().record("batches", report.batches);
+ tracing::Span::current().record("stopped", tracing::field::debug(report.stopped));
+ tracing::info!(
+ %cutoff,
+ sampled_expired = report.preview.sampled_expired,
+ oldest_expired_at = ?report.preview.oldest_expired_at,
+ deleted = report.deleted,
+ batches = report.batches,
+ stopped = ?report.stopped,
+ "notification retention sweep complete"
+ );
+ Ok(report)
+ }
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/notification/src/domain/retention/test.rs b/crates/notification/src/domain/retention/test.rs
new file mode 100644
index 0000000000..a6fea247a5
--- /dev/null
+++ b/crates/notification/src/domain/retention/test.rs
@@ -0,0 +1,182 @@
+use super::*;
+use std::{collections::VecDeque, sync::Mutex};
+
+#[derive(Default)]
+struct FakeRepository {
+ responses: Mutex<VecDeque<RetentionBatch>>,
+ calls: Mutex<Vec<(NaiveDateTime, u16)>>,
+ delay: Duration,
+}
+
+fn now() -> DateTime<Utc> {
+ DateTime::parse_from_rfc3339("2026-10-15T12:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc)
+}
+
+impl NotificationRetentionRepository for &FakeRepository {
+ async fn current_time(&self) -> Result<DateTime<Utc>, Report> {
+ Ok(now())
+ }
+
+ async fn preview(
+ &self,
+ cutoff: NaiveDateTime,
+ limit: NonZeroU16,
+ ) -> Result<RetentionPreview, Report> {
+ assert_eq!(cutoff, (now() - RETENTION).naive_utc());
+ assert_eq!(limit, BATCH_SIZE);
+ Ok(RetentionPreview {
+ sampled_expired: 100,
+ oldest_expired_at: Some(cutoff - TimeDelta::days(1)),
+ })
+ }
+
+ async fn delete_batch(
+ &self,
+ cutoff: NaiveDateTime,
+ limit: NonZeroU16,
+ ) -> Result<RetentionBatch, Report> {
+ self.calls.lock().unwrap().push((cutoff, limit.get()));
+ tokio::time::sleep(self.delay).await;
+ Ok(self
+ .responses
+ .lock()
+ .unwrap()
+ .pop_front()
+ .unwrap_or(RetentionBatch::Deleted(u64::from(limit.get()))))
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn dry_run_never_deletes() {
+ let repo = FakeRepository::default();
+ let report = NotificationRetentionService::new(&repo)
+ .run(RetentionMode::DryRun, MAX_RUN_TIME)
+ .await
+ .unwrap();
+ assert_eq!(report.stopped, RetentionStop::DryRun);
+ assert_eq!(report.deleted, 0);
+ assert_eq!(report.preview.sampled_expired, 100);
+ assert!(repo.calls.lock().unwrap().is_empty());
+}
+
+#[tokio::test(start_paused = true)]
+async fn batches_share_cutoff_and_pause_between_commits() {
+ let repo = FakeRepository {
+ responses: Mutex::new(VecDeque::from([
+ RetentionBatch::Deleted(100),
+ RetentionBatch::Deleted(3),
+ RetentionBatch::Deleted(0),
+ ])),
+ ..Default::default()
+ };
+ let start = Instant::now();
+ let report = NotificationRetentionService::new(&repo)
+ .run(RetentionMode::Delete, MAX_RUN_TIME)
+ .await
+ .unwrap();
+ assert_eq!(report.deleted, 103);
+ assert_eq!(report.batches, 3);
+ assert_eq!(report.stopped, RetentionStop::NoCandidates);
+ assert!(start.elapsed() >= BATCH_PAUSE * 2);
+ assert!(repo.calls.lock().unwrap().iter().all(|(cutoff, limit)| {
+ *cutoff == (now() - RETENTION).naive_utc() && *limit == BATCH_SIZE.get()
+ }));
+}
+
+#[tokio::test(start_paused = true)]
+async fn busy_worker_stops_without_retrying() {
+ let repo = FakeRepository {
+ responses: Mutex::new(VecDeque::from([RetentionBatch::Busy])),
+ ..Default::default()
+ };
+ let report = NotificationRetentionService::new(&repo)
+ .run(RetentionMode::Delete, MAX_RUN_TIME)
+ .await
+ .unwrap();
+ assert_eq!(report.stopped, RetentionStop::Busy);
+ assert_eq!(report.batches, 1);
+ assert_eq!(report.deleted, 0);
+}
+
+#[tokio::test(start_paused = true)]
+async fn invocation_has_a_hard_batch_cap() {
+ let repo = FakeRepository::default();
+ let report = NotificationRetentionService::new(&repo)
+ .run(RetentionMode::Delete, MAX_RUN_TIME)
+ .await
+ .unwrap();
+ assert_eq!(report.stopped, RetentionStop::BatchLimit);
+ assert_eq!(report.batches, MAX_BATCHES);
+ assert_eq!(report.deleted, u64::from(MAX_BATCHES) * 100);
+}
+
+#[tokio::test(start_paused = true)]
+async fn respects_caller_deadline_and_leaves_batch_headroom() {
+ let repo = FakeRepository::default();
+ let service = NotificationRetentionService::new(&repo);
+ let report = service
+ .run(RetentionMode::Delete, BATCH_HEADROOM)
+ .await
+ .unwrap();
+ assert_eq!(report.stopped, RetentionStop::TimeBudget);
+ assert!(repo.calls.lock().unwrap().is_empty());
+
+ let report = service
+ .run(RetentionMode::Delete, BATCH_HEADROOM + BATCH_PAUSE)
+ .await
+ .unwrap();
+ assert_eq!(report.stopped, RetentionStop::TimeBudget);
+ assert_eq!(report.batches, 1);
+}
+
+#[tokio::test(start_paused = true)]
+async fn oversized_time_budget_is_capped() {
+ let repo = FakeRepository {
+ delay: Duration::from_secs(10),
+ ..Default::default()
+ };
+ let report = NotificationRetentionService::new(&repo)
+ .run(RetentionMode::Delete, Duration::from_secs(3600))
+ .await
+ .unwrap();
+ assert_eq!(report.stopped, RetentionStop::TimeBudget);
+ assert!(report.batches < 10);
+}
+
+#[tokio::test(start_paused = true)]
+async fn timeouts_reduce_batch_size_without_counting_rolled_back_deletes() {
+ let repo = FakeRepository {
+ responses: Mutex::new(VecDeque::from([
+ RetentionBatch::TimedOut,
+ RetentionBatch::Deleted(50),
+ RetentionBatch::Deleted(0),
+ ])),
+ ..Default::default()
+ };
+ let report = NotificationRetentionService::new(&repo)
+ .run(RetentionMode::Delete, MAX_RUN_TIME)
+ .await
+ .unwrap();
+ assert_eq!(report.deleted, 50);
+ assert_eq!(report.batches, 3);
+ let sizes: Vec<_> = repo.calls.lock().unwrap().iter().map(|x| x.1).collect();
+ assert_eq!(sizes, [100, 50, 50]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn single_parent_timeout_fails_instead_of_looping_forever() {
+ let repo = FakeRepository {
+ responses: Mutex::new(VecDeque::from(vec![RetentionBatch::TimedOut; 7])),
+ ..Default::default()
+ };
+ assert!(
+ NotificationRetentionService::new(&repo)
+ .run(RetentionMode::Delete, MAX_RUN_TIME)
+ .await
+ .is_err()
+ );
+ let sizes: Vec<_> = repo.calls.lock().unwrap().iter().map(|x| x.1).collect();
+ assert_eq!(sizes, [100, 50, 25, 12, 6, 3, 1]);
+}
diff --git a/crates/notification/src/outbound.rs b/crates/notification/src/outbound.rs
index 0c376c61b8..2fe7cf3ea3 100644
--- a/crates/notification/src/outbound.rs
+++ b/crates/notification/src/outbound.rs
@@ -28,6 +28,7 @@ pub mod push_notification_event_queue;
pub mod queue;
pub mod rate_limit;
pub mod repository;
+pub mod retention;
pub mod sns_endpoint;
pub mod user_existence_checker;
pub mod websocket;
diff --git a/crates/notification/src/outbound/retention.rs b/crates/notification/src/outbound/retention.rs
new file mode 100644
index 0000000000..982b6dfdd9
--- /dev/null
+++ b/crates/notification/src/outbound/retention.rs
@@ -0,0 +1,123 @@
+//! PostgreSQL retention adapter. Only parent notifications are deleted: foreign
+//! keys remove recipients/receipts and the existing trigger emits delete events.
+
+use std::num::NonZeroU16;
+
+use chrono::{DateTime, NaiveDateTime, Utc};
+use rootcause::Report;
+use sqlx::PgPool;
+
+use crate::domain::retention::{NotificationRetentionRepository, RetentionBatch, RetentionPreview};
+
+// Transaction-scoped, so a pooled connection can never retain the lock after
+// this operation. The key is local to notification retention ("notifret").
+const RETENTION_LOCK: i64 = 0x6e6f_7469_6672_6574;
+
+/// An indexed, bounded retention repository using short transactions.
+pub struct PgNotificationRetentionRepository {
+ pool: PgPool,
+}
+
+impl PgNotificationRetentionRepository {
+ /// Construct over a maintenance pool with a short acquisition timeout.
+ pub fn new(pool: PgPool) -> Self {
+ Self { pool }
+ }
+}
+
+impl NotificationRetentionRepository for PgNotificationRetentionRepository {
+ async fn current_time(&self) -> Result<DateTime<Utc>, Report> {
+ Ok(sqlx::query_scalar!(r#"SELECT clock_timestamp() AS "now!""#)
+ .fetch_one(&self.pool)
+ .await?)
+ }
+
+ async fn preview(
+ &self,
+ cutoff: NaiveDateTime,
+ limit: NonZeroU16,
+ ) -> Result<RetentionPreview, Report> {
+ let row = sqlx::query!(
+ r#"
+ SELECT count(*) AS "count!", min(created_at) AS oldest
+ FROM (
+ SELECT created_at FROM notification
+ WHERE created_at < $1
+ ORDER BY created_at, id
+ LIMIT $2
+ ) expired
+ "#,
+ cutoff,
+ i64::from(limit.get()),
+ )
+ .fetch_one(&self.pool)
+ .await?;
+ Ok(RetentionPreview {
+ sampled_expired: row.count as u64,
+ oldest_expired_at: row.oldest,
+ })
+ }
+
+ #[tracing::instrument(err, skip(self))]
+ async fn delete_batch(
+ &self,
+ cutoff: NaiveDateTime,
+ limit: NonZeroU16,
+ ) -> Result<RetentionBatch, Report> {
+ let mut tx = self.pool.begin().await?;
+ sqlx::query!("SET LOCAL statement_timeout = '5s'")
+ .execute(&mut *tx)
+ .await?;
+ sqlx::query!("SET LOCAL lock_timeout = '1s'")
+ .execute(&mut *tx)
+ .await?;
+ let acquired = sqlx::query_scalar!(
+ r#"SELECT pg_try_advisory_xact_lock($1) AS "acquired!""#,
+ RETENTION_LOCK,
+ )
+ .fetch_one(&mut *tx)
+ .await?;
+ if !acquired {
+ tx.rollback().await?;
+ return Ok(RetentionBatch::Busy);
+ }
+
+ let result = sqlx::query!(
+ r#"
+ WITH expired AS (
+ SELECT id FROM notification
+ WHERE created_at < $1
+ ORDER BY created_at, id
+ LIMIT $2
+ FOR UPDATE SKIP LOCKED
+ )
+ DELETE FROM notification n
+ USING expired e
+ WHERE n.id = e.id
+ "#,
+ cutoff,
+ i64::from(limit.get()),
+ )
+ .execute(&mut *tx)
+ .await;
+ match result {
+ Ok(result) => {
+ tx.commit().await?;
+ Ok(RetentionBatch::Deleted(result.rows_affected()))
+ }
+ Err(error) => {
+ tx.rollback().await?;
+ // A timeout may mean one parent fans out to too many children.
+ // The service retries a smaller batch; unrelated errors fail
+ // the invocation and surface through the Lambda error alarm.
+ match error.as_database_error().and_then(|e| e.code()).as_deref() {
+ Some("57014" | "55P03") => Ok(RetentionBatch::TimedOut),
+ _ => Err(error.into()),
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/notification/src/outbound/retention/test.rs b/crates/notification/src/outbound/retention/test.rs
new file mode 100644
index 0000000000..184ebd7abc
--- /dev/null
+++ b/crates/notification/src/outbound/retention/test.rs
@@ -0,0 +1,286 @@
+use super::*;
+use crate::domain::models::NotificationState;
+use chrono::TimeDelta;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::postgres::PgListener;
+use std::collections::HashSet;
+use uuid::Uuid;
+
+fn cutoff() -> NaiveDateTime {
+ DateTime::parse_from_rfc3339("2026-09-15T12:00:00Z")
+ .unwrap()
+ .naive_utc()
+}
+
+fn limit(n: u16) -> NonZeroU16 {
+ NonZeroU16::new(n).unwrap()
+}
+
+async fn insert(pool: &PgPool, created_at: NaiveDateTime) -> Uuid {
+ let id = Uuid::now_v7();
+ sqlx::query!(
+ "INSERT INTO notification (id, notification_event_type, event_item_id, event_item_type, service_sender, created_at)
+ VALUES ($1, 'test', 'retention-document', 'document', 'retention-test', $2)",
+ id, created_at,
+ )
+ .execute(pool)
+ .await
+ .unwrap();
+ id
+}
+
+async fn remaining(pool: &PgPool) -> Vec<Uuid> {
+ sqlx::query_scalar!("SELECT id FROM notification ORDER BY created_at, id")
+ .fetch_all(pool)
+ .await
+ .unwrap()
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn boundary_is_strict_and_preview_is_read_only_and_bounded(pool: PgPool) {
+ let old = insert(&pool, cutoff() - TimeDelta::days(1)).await;
+ let just_old = insert(&pool, cutoff() - TimeDelta::microseconds(1)).await;
+ let boundary = insert(&pool, cutoff()).await;
+ let fresh = insert(&pool, cutoff() + TimeDelta::microseconds(1)).await;
+ let repo = PgNotificationRetentionRepository::new(pool.clone());
+ let preview = repo.preview(cutoff(), limit(1)).await.unwrap();
+ assert_eq!(preview.sampled_expired, 1);
+ assert_eq!(
+ preview.oldest_expired_at,
+ Some(cutoff() - TimeDelta::days(1))
+ );
+ assert_eq!(remaining(&pool).await, [old, just_old, boundary, fresh]);
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),
+ RetentionBatch::Deleted(2)
+ );
+ assert_eq!(remaining(&pool).await, [boundary, fresh]);
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),
+ RetentionBatch::Deleted(0)
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn bounded_oldest_first_batches_resume_on_retry(pool: PgPool) {
+ let later = insert(&pool, cutoff() - TimeDelta::seconds(1)).await;
+ let oldest_a = insert(&pool, cutoff() - TimeDelta::days(2)).await;
+ let oldest_b = insert(&pool, cutoff() - TimeDelta::days(2)).await;
+ let repo = PgNotificationRetentionRepository::new(pool.clone());
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(1)).await.unwrap(),
+ RetentionBatch::Deleted(1)
+ );
+ assert_eq!(remaining(&pool).await, [oldest_a.max(oldest_b), later]);
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(1)).await.unwrap(),
+ RetentionBatch::Deleted(1)
+ );
+ assert_eq!(remaining(&pool).await, [later]);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn parent_age_controls_all_states_and_cascades_to_receipts(pool: PgPool) {
+ let old = insert(&pool, cutoff() - TimeDelta::days(1)).await;
+ let fresh = insert(&pool, cutoff() + TimeDelta::days(1)).await;
+ for (i, state) in [
+ NotificationState::Unseen,
+ NotificationState::Seen,
+ NotificationState::Done,
+ ]
+ .into_iter()
+ .enumerate()
+ {
+ let user = format!("macro|retention-{i}@test.com");
+ for notification in [old, fresh] {
+ // A fresh recipient on an expired parent must expire; an old
+ // recipient on a fresh parent must survive. Include soft deletes.
+ let child_created = if notification == old {
+ cutoff() + TimeDelta::days(1)
+ } else {
+ cutoff() - TimeDelta::days(1)
+ };
+ let deleted_at = (i == 2).then_some(child_created);
+ sqlx::query!(
+ "INSERT INTO user_notification (user_id, notification_id, created_at, state, deleted_at)
+ VALUES ($1, $2, $3, $4, $5)",
+ user, notification, child_created, state as NotificationState, deleted_at,
+ ).execute(&pool).await.unwrap();
+ let message = format!("{notification}-{i}");
+ sqlx::query!(
+ "INSERT INTO notification_message_receipt (message_id, user_id, notification_id) VALUES ($1, $2, $3)",
+ message, user, notification,
+ ).execute(&pool).await.unwrap();
+ }
+ }
+ // Settings with similar names are not notification history.
+ sqlx::query!(
+ "INSERT INTO user_mute_notification (user_id) VALUES ('macro|[email protected]')"
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ let repo = PgNotificationRetentionRepository::new(pool.clone());
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),
+ RetentionBatch::Deleted(1)
+ );
+ assert_eq!(remaining(&pool).await, [fresh]);
+ assert_eq!(
+ sqlx::query_scalar!("SELECT notification_id FROM user_notification")
+ .fetch_all(&pool)
+ .await
+ .unwrap(),
+ vec![fresh; 3]
+ );
+ assert_eq!(
+ sqlx::query_scalar!("SELECT notification_id FROM notification_message_receipt")
+ .fetch_all(&pool)
+ .await
+ .unwrap(),
+ vec![fresh; 3]
+ );
+ assert_eq!(
+ sqlx::query_scalar!("SELECT count(*) FROM user_mute_notification")
+ .fetch_one(&pool)
+ .await
+ .unwrap(),
+ Some(1)
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn locked_parents_are_revisited_and_advisory_lock_is_transaction_scoped(pool: PgPool) {
+ let oldest = insert(&pool, cutoff() - TimeDelta::days(2)).await;
+ let next = insert(&pool, cutoff() - TimeDelta::days(1)).await;
+ let repo = PgNotificationRetentionRepository::new(pool.clone());
+ let mut tx = pool.begin().await.unwrap();
+ sqlx::query!(
+ "SELECT id FROM notification WHERE id = $1 FOR UPDATE",
+ oldest
+ )
+ .fetch_one(&mut *tx)
+ .await
+ .unwrap();
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),
+ RetentionBatch::Deleted(1)
+ );
+ assert_eq!(remaining(&pool).await, [oldest]);
+ assert_ne!(oldest, next);
+ tx.rollback().await.unwrap();
+
+ let mut tx = pool.begin().await.unwrap();
+ assert!(
+ sqlx::query_scalar!(
+ r#"SELECT pg_try_advisory_xact_lock($1) AS "locked!""#,
+ RETENTION_LOCK
+ )
+ .fetch_one(&mut *tx)
+ .await
+ .unwrap()
+ );
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),
+ RetentionBatch::Busy
+ );
+ tx.rollback().await.unwrap();
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),
+ RetentionBatch::Deleted(1)
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn cascade_lock_timeout_rolls_back_entire_batch_and_allows_retry(pool: PgPool) {
+ let old = insert(&pool, cutoff() - TimeDelta::days(1)).await;
+ sqlx::query!("INSERT INTO user_notification (user_id, notification_id) VALUES ('macro|[email protected]', $1)", old)
+ .execute(&pool).await.unwrap();
+ let mut tx = pool.begin().await.unwrap();
+ sqlx::query!(
+ "SELECT notification_id FROM user_notification WHERE notification_id = $1 FOR UPDATE",
+ old
+ )
+ .fetch_one(&mut *tx)
+ .await
+ .unwrap();
+ let repo = PgNotificationRetentionRepository::new(pool.clone());
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),
+ RetentionBatch::TimedOut
+ );
+ assert_eq!(remaining(&pool).await, [old]);
+ tx.rollback().await.unwrap();
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(100)).await.unwrap(),
+ RetentionBatch::Deleted(1)
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn deletion_notifies_every_recipient_in_byte_bounded_chunks(pool: PgPool) {
+ let old = insert(&pool, cutoff() - TimeDelta::days(1)).await;
+ // Multibyte addresses ensure the trigger measures bytes, not characters.
+ let users: Vec<_> = (0..500)
+ .map(|i| format!("macro|retention-é-{i}@test.com"))
+ .collect();
+ sqlx::query!(
+ "INSERT INTO user_notification (user_id, notification_id) SELECT unnest($1::text[]), $2",
+ &users,
+ old
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ let mut listener = PgListener::connect_with(&pool).await.unwrap();
+ listener.listen("notification_events").await.unwrap();
+ let repo = PgNotificationRetentionRepository::new(pool.clone());
+ assert_eq!(
+ repo.delete_batch(cutoff(), limit(1)).await.unwrap(),
+ RetentionBatch::Deleted(1)
+ );
+ let mut received = HashSet::new();
+ let mut events = 0;
+ while received.len() < users.len() {
+ let event = tokio::time::timeout(std::time::Duration::from_secs(2), listener.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(event.payload().len() < 8000);
+ let payload: serde_json::Value = serde_json::from_str(event.payload()).unwrap();
+ assert_eq!(payload["type"], "user_notification_deletes");
+ assert_eq!(payload["notificationId"], old.to_string());
+ for user in payload["userIds"].as_array().unwrap() {
+ assert!(received.insert(user.as_str().unwrap().to_owned()));
+ }
+ events += 1;
+ }
+ assert!(events > 1);
+ assert_eq!(received, users.into_iter().collect());
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn database_clock_uses_utc_even_with_a_non_utc_session(pool: PgPool) {
+ let non_utc_pool = sqlx::postgres::PgPoolOptions::new()
+ .max_connections(1)
+ .connect_with(
+ pool.connect_options()
+ .as_ref()
+ .clone()
+ .options([("TimeZone", "Pacific/Auckland")]),
+ )
+ .await
+ .unwrap();
+ let repo = PgNotificationRetentionRepository::new(non_utc_pool.clone());
+ let now = repo.current_time().await.unwrap();
+ assert!((Utc::now() - now).num_seconds().abs() < 5);
+ let cutoff = (now - crate::domain::retention::RETENTION).naive_utc();
+ insert(&pool, cutoff - TimeDelta::days(1)).await;
+ let fresh = insert(&pool, cutoff + TimeDelta::days(1)).await;
+ assert_eq!(
+ repo.delete_batch(cutoff, limit(100)).await.unwrap(),
+ RetentionBatch::Deleted(1)
+ );
+ assert_eq!(remaining(&pool).await, [fresh]);
+ non_utc_pool.close().await;
+}
diff --git a/docs/notification-retention.md b/docs/notification-retention.md
new file mode 100644
index 0000000000..c77d9c01ca
--- /dev/null
+++ b/docs/notification-retention.md
@@ -0,0 +1,164 @@
+# Notification retention
+
+The internal `notification_retention_handler` Lambda removes notifications older
+than 30 complete UTC days. Its EventBridge schedule is **disabled** in both dev
+and prod, and its event payload defaults to **dry-run**. Deploying the code or
+migrations alone does not purge history.
+
+## Semantics
+
+- The cutoff is the database clock at invocation start minus 30 days, converted
+ to UTC for the schema's timezone-less `notification.created_at` column.
+- Delete only parents with `created_at < cutoff`. Equality and newer timestamps
+ survive. Reading, completing, reopening, or sending does not extend retention.
+- All recipient states (`unseen`, `seen`, `done`) and soft-deleted recipients
+ expire with the parent. The parent age wins even if recipient timestamps differ.
+- Foreign keys cascade to `user_notification` and `notification_message_receipt`.
+ Messages, documents, other source entities, preferences, unsubscribe records,
+ device registrations, and email-send bookkeeping are not removed.
+- Old unread notifications disappear. Items whose only feed-placement reason was
+ an expired notification may disappear from that feed, not from source storage.
+- This is eventual cleanup, normally within a five-minute schedule interval plus
+ execution time after expiry once caught up. Backlogs, failures, or locked rows
+ can extend that delay. There is no new read-time expiration filter.
+- **Replay protection is intentionally out of scope.** Parent-row existence still
+ provides notification-ID deduplication. After retention deletes a parent, an
+ old event replay can recreate it with a fresh timestamp. Do not assume forever
+ idempotency when replaying old ingress events.
+
+## Execution and safety limits
+
+The notification domain owns policy and batching in
+`crates/notification/src/domain/retention.rs`. Its outbound adapter owns checked
+SQL and transactions in `crates/notification/src/outbound/retention.rs`. The
+Lambda only parses an internal event, computes remaining invocation time, and
+calls the domain service. There is no HTTP route or user-callable cleanup API.
+IAM controls invocation; do not grant end users permission to invoke this Lambda.
+
+Each invocation:
+
+1. Computes one cutoff using the database clock and previews at most 100 expired
+ parents. `sampled_expired` is **not** a total backlog count.
+2. In deletion mode, selects oldest-first using `(created_at, id)`, at most 100
+ parents per transaction. A matching B-tree index avoids a history-wide sort.
+3. Acquires a nonblocking transaction-scoped advisory lock, then uses
+ `FOR UPDATE SKIP LOCKED`. Another cleanup transaction causes this invocation to
+ stop. Locked parents are retried on future invocations; there is no watermark
+ that could permanently skip them.
+4. Deletes parents, allowing ordinary FK cascades and delete triggers, and commits
+ each batch. No recipient payloads are returned to the worker.
+5. Pauses 250 ms between batches. Caps work at 100 attempted batches and a
+ 100-second invocation budget, leaving 15 seconds of headroom before starting
+ another batch. The Lambda timeout is 120 seconds and remaining Lambda time is
+ also respected.
+
+The dedicated pool has one connection and a five-second acquisition timeout.
+Statements time out after five seconds and lock waits after one second. Idle
+transactions time out after ten seconds, so a stalled/frozen worker cannot hold
+locks indefinitely. The handler also bounds the entire call by its remaining
+invocation budget, including pool/network waits. A delete
+statement/lock timeout rolls back that batch; the service halves the next batch
+size. A timeout at one parent fails the invocation and raises the error alarm.
+Other errors also fail, without undoing previously committed batches. A subsequent
+invocation resumes naturally. Lambda concurrency is one; the advisory lock also
+protects against other instances of this worker.
+
+Parent limits do not bound recipient/receipt fanout. A single huge notification
+can still exceed the timeout and require operator investigation. Never raise
+limits or disable constraints/triggers merely to force progress.
+
+## Realtime effects
+
+The existing parent-delete trigger publishes `user_notification_deletes` events
+before recipients cascade away. The migration splits recipients into JSON chunks
+with a 7000-byte recipient-array budget, leaving room below PostgreSQL's 8000-byte
+NOTIFY limit. Existing consumers accept the same event shape for each chunk.
+An individual malformed/oversized recipient is warned about and skipped; a
+publication failure remains best-effort, as in the previous trigger.
+
+`LISTEN/NOTIFY` is not durable. Disconnected clients/listeners must reconcile on
+refetch/reconnect; this change does not add durable tombstones or an outbox.
+Watch notification-service/Kafka/connection-gateway load during backfill. Do not
+turn off delete events to speed up the purge.
+
+## Rollout
+
+Production deletion needs explicit approval and a verified backup/PITR recovery
+procedure. Do not run a historical `DELETE` inside a migration.
+
+1. Apply the additive migrations through the normal approved database workflow.
+ The retention index is built concurrently in its own nontransactional
+ migration; the other migration replaces the delete-event function without
+ changing its contract. Verify the index is valid. If a concurrent build fails,
+ inspect its invalid index and repair/retry under the database deployment
+ procedure rather than marking the migration successful.
+2. Deploy the notification-service stack. Its deployment configuration also builds
+ the new Lambda. Keep `notification_retention_enabled: false` and
+ `notification_retention_dry_run: true` initially.
+3. Inspect age distribution, recipient/receipt fanout, and query plans against
+ representative data. Use plain `EXPLAIN` for production deletion plans;
+ `EXPLAIN ANALYZE DELETE` actually deletes data.
+4. Enable the schedule **in dry-run mode** first:
+
+ ```bash
+ \cd infra/stacks/notification-service
+ pulumi config set notification_retention_enabled true --stack <dev-or-prod>
+ pulumi config set notification_retention_dry_run true --stack <dev-or-prod>
+ # Review/apply through the normal deployment workflow.
+ ```
+
+5. After approval and reviewing dry-run logs, change
+ `notification_retention_dry_run` to `false` and deploy. Drain existing history
+ with these same bounded batches; do not substitute a bulk deletion. Initial
+ capacity is at most 10,000 parents per invocation (usually less). Verify
+ steady-state capacity exceeds incoming expiration volume before enabling prod.
+6. Monitor application latency, database CPU/I/O, WAL and replica lag, dead tuples,
+ vacuum progress, NOTIFY warnings/queue pressure, and realtime consumer lag.
+ Pause the schedule on unacceptable impact. Do not automatically run
+ `VACUUM FULL`; ordinary vacuum reclaims space for reuse, not necessarily for
+ returning to the filesystem.
+
+A manual invocation with `{}` or `{"dry_run": true}` is read-only.
+`{"dry_run": false}` explicitly opts into deletion. Unknown fields, alternate
+spellings, and nonboolean settings are rejected. Invocation cannot override the
+retention period or batch caps.
+
+## Monitoring and stopping
+
+Infrastructure alarms on Lambda errors, throttles, and no invocations for
+15 minutes while the schedule is enabled. Each sweep logs the cutoff,
+`sampled_expired`, `oldest_expired_at`, committed parent `deleted` count,
+attempted `batches`, and stopping reason. Each committed batch is also logged,
+so partial progress remains visible if a later batch fails.
+
+Track `cutoff - oldest_expired_at` as expiration backlog age. During backfill it
+should trend downward; once caught up, sustained growth indicates insufficient
+capacity, long-lived locks, or a failing high-fanout parent. A zero-row batch can
+mean rows were locked, not necessarily that every expired row has been removed.
+Do not run a full `COUNT(*)` over the backlog every five minutes.
+
+Disable `notification_retention_enabled` and deploy to stop future scheduled
+invocations. An already-running invocation may still commit batches. Emergency
+operational controls may also stop future invokes; they do not undo commits.
+Disabling the worker or rolling back migrations **cannot restore deleted data**.
+Recovery requires the approved backup/PITR process.
+
+Retention bounds age, not cardinality. Busy users can still have many notifications
+within 30 days; pagination of currently unbounded consumers remains separate work.
+
+## Local verification
+
+Run from the repository root with local PostgreSQL and `SQLX_OFFLINE` unset:
+
+```bash
+cargo test -p notification -p notification_retention_handler -- --test-threads=8
+nix develop --command just prepare_db
+just check
+\cd infra
+bunx tsc --noEmit --project stacks/notification-service/tsconfig.json
+```
+
+Tests cover cutoff boundaries, parent-vs-recipient age, all states, cascades,
+read-only previews, deterministic bounded ordering, retry behavior, skipped locks,
+transaction advisory locking, timeout rollback, non-UTC sessions, byte-bounded
+multi-recipient delete events, invocation caps, and fail-closed event parsing.
diff --git a/infra/stacks/notification-service/Pulumi.dev.yaml b/infra/stacks/notification-service/Pulumi.dev.yaml
index dcdb6f8af4..b0537a4f73 100644
--- a/infra/stacks/notification-service/Pulumi.dev.yaml
+++ b/infra/stacks/notification-service/Pulumi.dev.yaml
@@ -1,6 +1,9 @@
config:
aws-native:region: us-east-1
aws:region: us-east-1
+ notification-service:macro_db_secret_key: macro-db-dev
+ notification-service:notification_retention_enabled: false
+ notification-service:notification_retention_dry_run: true
notification-service:jwt_secret_key: fusionauth-jwt-secret-dev
notification-service:fusionauth_client_id: fusionauth-client-id-key-dev
notification-service:fusionauth_issuer: fusionauth-dev.macro.com
diff --git a/infra/stacks/notification-service/Pulumi.prod.yaml b/infra/stacks/notification-service/Pulumi.prod.yaml
index 809c1efa52..2c9076cd67 100644
--- a/infra/stacks/notification-service/Pulumi.prod.yaml
+++ b/infra/stacks/notification-service/Pulumi.prod.yaml
@@ -1,6 +1,9 @@
config:
aws-native:region: us-east-1
aws:region: us-east-1
+ notification-service:macro_db_secret_key: macro-db-prod
+ notification-service:notification_retention_enabled: false
+ notification-service:notification_retention_dry_run: true
notification-service:jwt_secret_key: fusionauth-jwt-secret-prod
notification-service:fusionauth_client_id: fusionauth-client-id-key-prod
notification-service:fusionauth_issuer: auth.macro.com
diff --git a/infra/stacks/notification-service/index.ts b/infra/stacks/notification-service/index.ts
index 85642eb31e..1676ade77a 100644
--- a/infra/stacks/notification-service/index.ts
+++ b/infra/stacks/notification-service/index.ts
@@ -11,6 +11,7 @@ import {
} from '../../packages/shared';
import { get_coparse_api_vpc } from '../../packages/vpc';
import { PushNotificationEventHandler } from './push';
+import { NotificationRetention } from './retention';
import { NotificationService } from './service';
const tags = {
@@ -63,6 +64,19 @@ const authenticationServiceInternalApiKeyArn: pulumi.Output<string> =
export const coparse_api_vpc = get_coparse_api_vpc();
+// Fetch as a secret Pulumi output; never export the database URL.
+const retentionDatabaseUrl = aws.secretsmanager
+ .getSecretVersionOutput({ secretId: config.require('macro_db_secret_key') })
+ .apply((secret) => pulumi.secret(secret.secretString));
+
+new NotificationRetention('notification-retention', {
+ databaseUrl: retentionDatabaseUrl,
+ enabled: config.getBoolean('notification_retention_enabled') ?? false,
+ dryRun: config.getBoolean('notification_retention_dry_run') ?? true,
+ vpc: coparse_api_vpc,
+ tags,
+});
+
const cloudStorageStack = new pulumi.StackReference('cloud-storage-stack', {
name: `macro-inc/document-storage/${stack}`,
});
diff --git a/infra/stacks/notification-service/retention.ts b/infra/stacks/notification-service/retention.ts
new file mode 100644
index 0000000000..517ec4b976
--- /dev/null
+++ b/infra/stacks/notification-service/retention.ts
@@ -0,0 +1,158 @@
+import * as aws from '@pulumi/aws';
+import * as pulumi from '@pulumi/pulumi';
+import { Lambda } from '../../packages/lambda';
+import { CLOUD_TRAIL_SNS_TOPIC_ARN, stack } from '../../packages/shared';
+
+const BASE_NAME = 'notification_retention_handler';
+const REPO_ROOT = '../../..';
+
+type RetentionEnvVars = {
+ DATABASE_URL: pulumi.Output<string> | string;
+ ENVIRONMENT: string;
+ RUST_LOG: string;
+};
+
+type Args = {
+ databaseUrl: pulumi.Output<string>;
+ enabled: boolean;
+ dryRun: boolean;
+ vpc: {
+ vpcId: pulumi.Output<string> | string;
+ publicSubnetIds: pulumi.Output<string[]> | string[];
+ privateSubnetIds: pulumi.Output<string[]> | string[];
+ };
+ tags: Record<string, string>;
+};
+
+/** Isolated maintenance worker: disabled initially, then preview before deleting. */
+export class NotificationRetention extends pulumi.ComponentResource {
+ readonly lambda: aws.lambda.Function;
+
+ constructor(
+ name: string,
+ args: Args,
+ opts?: pulumi.ComponentResourceOptions
+ ) {
+ super('macro:notifications:Retention', name, {}, opts);
+ const { databaseUrl, enabled, dryRun, vpc, tags } = args;
+ const role = new aws.iam.Role(
+ `${BASE_NAME}-role`,
+ {
+ assumeRolePolicy: JSON.stringify({
+ Version: '2012-10-17',
+ Statement: [
+ {
+ Effect: 'Allow',
+ Principal: { Service: 'lambda.amazonaws.com' },
+ Action: 'sts:AssumeRole',
+ },
+ ],
+ }),
+ managedPolicyArns: [
+ aws.iam.ManagedPolicy.AWSLambdaBasicExecutionRole,
+ aws.iam.ManagedPolicy.AWSLambdaVPCAccessExecutionRole,
+ ],
+ tags,
+ },
+ { parent: this }
+ );
+ this.lambda = new Lambda<RetentionEnvVars>(
+ `${BASE_NAME}-lambda`,
+ {
+ baseName: BASE_NAME,
+ handlerBase: `${REPO_ROOT}/services/${BASE_NAME}`,
+ zipLocation: `${REPO_ROOT}/target/lambda/${BASE_NAME}/bootstrap.zip`,
+ envVars: {
+ DATABASE_URL: databaseUrl,
+ ENVIRONMENT: stack,
+ RUST_LOG: 'info',
+ },
+ role,
+ vpc,
+ memorySize: 256,
+ timeout: 120,
+ reservedConcurrentExecutions: 1,
+ tags,
+ },
+ { parent: this }
+ ).lambda;
+
+ const rule = new aws.cloudwatch.EventRule(
+ `${BASE_NAME}-schedule`,
+ {
+ scheduleExpression: 'rate(5 minutes)',
+ state: enabled ? 'ENABLED' : 'DISABLED',
+ tags,
+ },
+ { parent: this }
+ );
+ new aws.cloudwatch.EventTarget(
+ `${BASE_NAME}-target`,
+ {
+ rule: rule.name,
+ arn: this.lambda.arn,
+ input: JSON.stringify({ dry_run: dryRun }),
+ retryPolicy: { maximumEventAgeInSeconds: 60, maximumRetryAttempts: 0 },
+ },
+ { parent: this }
+ );
+ new aws.lambda.Permission(
+ `${BASE_NAME}-invoke`,
+ {
+ action: 'lambda:InvokeFunction',
+ function: this.lambda.name,
+ principal: 'events.amazonaws.com',
+ sourceArn: rule.arn,
+ },
+ { parent: this }
+ );
+ // The next scheduled sweep retries any outstanding rows. Don't accumulate
+ // asynchronous retries of an old invocation while the worker is throttled.
+ new aws.lambda.FunctionEventInvokeConfig(
+ `${BASE_NAME}-async-policy`,
+ {
+ functionName: this.lambda.name,
+ maximumEventAgeInSeconds: 60,
+ maximumRetryAttempts: 0,
+ },
+ { parent: this }
+ );
+ for (const metricName of ['Errors', 'Throttles']) {
+ new aws.cloudwatch.MetricAlarm(
+ `${BASE_NAME}-${metricName.toLowerCase()}`,
+ {
+ namespace: 'AWS/Lambda',
+ metricName,
+ dimensions: { FunctionName: this.lambda.name },
+ statistic: 'Sum',
+ period: 300,
+ evaluationPeriods: 1,
+ threshold: 1,
+ comparisonOperator: 'GreaterThanOrEqualToThreshold',
+ treatMissingData: 'notBreaching',
+ alarmActions: [CLOUD_TRAIL_SNS_TOPIC_ARN],
+ tags,
+ },
+ { parent: this }
+ );
+ }
+ new aws.cloudwatch.MetricAlarm(
+ `${BASE_NAME}-missing-invocations`,
+ {
+ namespace: 'AWS/Lambda',
+ metricName: 'Invocations',
+ dimensions: { FunctionName: this.lambda.name },
+ statistic: 'Sum',
+ period: 900,
+ evaluationPeriods: 1,
+ threshold: 1,
+ comparisonOperator: 'LessThanThreshold',
+ treatMissingData: enabled ? 'breaching' : 'notBreaching',
+ actionsEnabled: enabled,
+ alarmActions: [CLOUD_TRAIL_SNS_TOPIC_ARN],
+ tags,
+ },
+ { parent: this }
+ );
+ }
+}
diff --git a/services/notification_retention_handler/Cargo.toml b/services/notification_retention_handler/Cargo.toml
new file mode 100644
index 0000000000..9b628dc42c
--- /dev/null
+++ b/services/notification_retention_handler/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+edition = "2024"
+name = "notification_retention_handler"
+publish = false
+version = "0.1.0"
+
+[dependencies]
+lambda_runtime = { workspace = true }
+macro_entrypoint = { path = "../../crates/macro_entrypoint" }
+macro_env_var = { path = "../../crates/macro_env_var" }
+notification = { path = "../../crates/notification" }
+serde = { workspace = true }
+sqlx = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
+workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }
+
+[dev-dependencies]
+serde_json = { workspace = true }
diff --git a/services/notification_retention_handler/justfile b/services/notification_retention_handler/justfile
new file mode 100644
index 0000000000..9c5ce2bb17
--- /dev/null
+++ b/services/notification_retention_handler/justfile
@@ -0,0 +1,4 @@
+lambda_name := file_name(justfile_directory())
+
+build compiler="":
+ SQLX_OFFLINE=true cargo lambda build {{ if compiler == "" { "" } else { "--compiler " + compiler } }} --release --bin {{lambda_name}} --output-format zip
diff --git a/services/notification_retention_handler/src/handler.rs b/services/notification_retention_handler/src/handler.rs
new file mode 100644
index 0000000000..1390af3244
--- /dev/null
+++ b/services/notification_retention_handler/src/handler.rs
@@ -0,0 +1,54 @@
+//! Thin Lambda adapter. Empty/manual payloads default to a read-only preview.
+
+use std::time::{Duration, SystemTime};
+
+use lambda_runtime::{Error, LambdaEvent};
+use notification::domain::retention::{
+ NotificationRetentionRepository, NotificationRetentionService, RetentionMode,
+};
+use serde::Deserialize;
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct CleanupEvent {
+ #[serde(default = "default_dry_run")]
+ dry_run: bool,
+}
+
+fn default_dry_run() -> bool {
+ true
+}
+
+impl CleanupEvent {
+ fn mode(&self) -> RetentionMode {
+ if self.dry_run {
+ RetentionMode::DryRun
+ } else {
+ RetentionMode::Delete
+ }
+ }
+}
+
+#[tracing::instrument(err, skip(service, event))]
+pub(crate) async fn handle<R: NotificationRetentionRepository>(
+ service: &NotificationRetentionService<R>,
+ event: LambdaEvent<CleanupEvent>,
+) -> Result<(), Error> {
+ let budget = event
+ .context
+ .deadline()
+ .duration_since(SystemTime::now())
+ .unwrap_or_default()
+ .saturating_sub(Duration::from_secs(10));
+ if budget < Duration::from_secs(15) {
+ tracing::warn!("not enough invocation time for notification retention; skipping");
+ return Ok(());
+ }
+ // Also bound stalls outside SQL execution (pool acquisition/network I/O).
+ // Dropping an in-flight repository future rolls back its transaction.
+ tokio::time::timeout(budget, service.run(event.payload.mode(), budget)).await??;
+ Ok(())
+}
+
+#[cfg(test)]
+mod test;
diff --git a/services/notification_retention_handler/src/handler/test.rs b/services/notification_retention_handler/src/handler/test.rs
new file mode 100644
index 0000000000..d154eec3bc
--- /dev/null
+++ b/services/notification_retention_handler/src/handler/test.rs
@@ -0,0 +1,23 @@
+use super::*;
+
+#[test]
+fn manual_invocation_is_read_only_unless_explicitly_requested() {
+ for payload in ["{}", r#"{"dry_run":true}"#] {
+ let event: CleanupEvent = serde_json::from_str(payload).unwrap();
+ assert_eq!(event.mode(), RetentionMode::DryRun);
+ }
+ let event: CleanupEvent = serde_json::from_str(r#"{"dry_run":false}"#).unwrap();
+ assert_eq!(event.mode(), RetentionMode::Delete);
+}
+
+#[test]
+fn invalid_or_unsupported_settings_fail_closed() {
+ for payload in [
+ r#"{"dry_run":null}"#,
+ r#"{"dry_run":"false"}"#,
+ r#"{"dryRun":false}"#,
+ r#"{"dry_run":false,"retention_days":0}"#,
+ ] {
+ assert!(serde_json::from_str::<CleanupEvent>(payload).is_err());
+ }
+}
diff --git a/services/notification_retention_handler/src/main.rs b/services/notification_retention_handler/src/main.rs
new file mode 100644
index 0000000000..79fb641ee0
--- /dev/null
+++ b/services/notification_retention_handler/src/main.rs
@@ -0,0 +1,47 @@
+//! Scheduled notification retention composition root. Invocation is internal;
+//! the notification domain owns the policy and all deletion orchestration.
+
+mod handler;
+
+use std::{str::FromStr, sync::Arc, time::Duration};
+
+use lambda_runtime::{Error, run, service_fn};
+use macro_entrypoint::MacroEntrypoint;
+use macro_env_var::env_vars;
+use notification::{
+ domain::retention::NotificationRetentionService,
+ outbound::retention::PgNotificationRetentionRepository,
+};
+use sqlx::postgres::{PgConnectOptions, PgPoolOptions};
+
+env_vars! {
+ struct DatabaseUrl;
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Error> {
+ MacroEntrypoint::default().init();
+ let database_url = DatabaseUrl::new()?;
+ // Bound preview queries as well as deletes. One maintenance connection per
+ // process, and reserved Lambda concurrency of one, isolate foreground work.
+ let options = PgConnectOptions::from_str(database_url.as_ref())?
+ .application_name("notification_retention_handler")
+ .options([
+ ("statement_timeout", "5000"),
+ ("lock_timeout", "1000"),
+ ("idle_in_transaction_session_timeout", "10000"),
+ ]);
+ let pool = PgPoolOptions::new()
+ .max_connections(1)
+ .acquire_timeout(Duration::from_secs(5))
+ .connect_with(options)
+ .await?;
+ let service = Arc::new(NotificationRetentionService::new(
+ PgNotificationRetentionRepository::new(pool),
+ ));
+ run(service_fn(move |event| {
+ let service = Arc::clone(&service);
+ async move { handler::handle(&service, event).await }
+ }))
+ .await
+}
diff --git a/tooling/just/rust.just b/tooling/just/rust.just
index bd63226278..eac2ee67df 100644
--- a/tooling/just/rust.just
+++ b/tooling/just/rust.just
@@ -56,6 +56,7 @@ build_lambdas compiler="":
ulimit -n 10240 && just services/deleted_item_poller/build {{compiler}}
ulimit -n 10240 && just services/email_refresh_handler/build {{compiler}}
ulimit -n 10240 && just services/user_link_cleanup_handler/build {{compiler}}
+ ulimit -n 10240 && just services/notification_retention_handler/build {{compiler}}
ulimit -n 10240 && just services/email_scheduled_handler/build {{compiler}}
ulimit -n 10240 && just services/email_sfs_delete_handler/build {{compiler}}
ulimit -n 10240 && just services/image_optimizer/build {{compiler}}