Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(email): dedupe CRM depopulation into a nightly cleanup job (#5141)

Evan Hutnik42df6eab9dae
diff --git a/.sqlx/query-3dcbbcd7145c44bc2e282316d6080dae0749c9053f30d7888f5c7d52401eadc8.json b/.sqlx/query-3dcbbcd7145c44bc2e282316d6080dae0749c9053f30d7888f5c7d52401eadc8.json
new file mode 100644
index 0000000000..e96d108554
--- /dev/null
+++ b/.sqlx/query-3dcbbcd7145c44bc2e282316d6080dae0749c9053f30d7888f5c7d52401eadc8.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE crm_cleanup_jobs\n SET dispatched_count = dispatched_count + $1, updated_at = now()\n WHERE id = $2\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "3dcbbcd7145c44bc2e282316d6080dae0749c9053f30d7888f5c7d52401eadc8"
+}
diff --git a/.sqlx/query-4575cb6d19d4c0df01889d43a8e2426fb596e406c2fd316be1db6f0f2c769689.json b/.sqlx/query-4575cb6d19d4c0df01889d43a8e2426fb596e406c2fd316be1db6f0f2c769689.json
new file mode 100644
index 0000000000..a415316796
--- /dev/null
+++ b/.sqlx/query-4575cb6d19d4c0df01889d43a8e2426fb596e406c2fd316be1db6f0f2c769689.json
@@ -0,0 +1,36 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT id, link_id, contact_email\n FROM crm_cleanup_candidates\n WHERE id > $1 AND id <= $2\n ORDER BY id\n LIMIT $3\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "link_id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 2,
+ "name": "contact_email",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Int8",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "4575cb6d19d4c0df01889d43a8e2426fb596e406c2fd316be1db6f0f2c769689"
+}
diff --git a/.sqlx/query-5da3501a72c81a825537d0b11a3f9c0d9c4c87e14febeab9fc5272b2ac582080.json b/.sqlx/query-5da3501a72c81a825537d0b11a3f9c0d9c4c87e14febeab9fc5272b2ac582080.json
new file mode 100644
index 0000000000..8d33d04c02
--- /dev/null
+++ b/.sqlx/query-5da3501a72c81a825537d0b11a3f9c0d9c4c87e14febeab9fc5272b2ac582080.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO crm_cleanup_candidates (link_id, contact_email)\n SELECT $1, unnest($2::text[])\n ON CONFLICT (link_id, contact_email) DO NOTHING\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "TextArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "5da3501a72c81a825537d0b11a3f9c0d9c4c87e14febeab9fc5272b2ac582080"
+}
diff --git a/.sqlx/query-689490b62b78314388ddf6ad540d48aca103b27a8c9e0a22f2f73ff791491fd0.json b/.sqlx/query-689490b62b78314388ddf6ad540d48aca103b27a8c9e0a22f2f73ff791491fd0.json
new file mode 100644
index 0000000000..244ed4480d
--- /dev/null
+++ b/.sqlx/query-689490b62b78314388ddf6ad540d48aca103b27a8c9e0a22f2f73ff791491fd0.json
@@ -0,0 +1,70 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n id,\n status as \"status: CrmCleanupJobStatus\",\n total_candidates,\n dispatched_count,\n max_candidate_id,\n created_at,\n updated_at\n FROM crm_cleanup_jobs\n WHERE id = $1\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "status: CrmCleanupJobStatus",
+ "type_info": {
+ "Custom": {
+ "name": "crm_cleanup_job_status",
+ "kind": {
+ "Enum": [
+ "Init",
+ "InProgress",
+ "Complete",
+ "Failed"
+ ]
+ }
+ }
+ }
+ },
+ {
+ "ordinal": 2,
+ "name": "total_candidates",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 3,
+ "name": "dispatched_count",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 4,
+ "name": "max_candidate_id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 5,
+ "name": "created_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 6,
+ "name": "updated_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "689490b62b78314388ddf6ad540d48aca103b27a8c9e0a22f2f73ff791491fd0"
+}
diff --git a/.sqlx/query-6fc1d293fb45bfd5788378f7dd61517f26adfbd430a833f07e0f63807fb40ae1.json b/.sqlx/query-6fc1d293fb45bfd5788378f7dd61517f26adfbd430a833f07e0f63807fb40ae1.json
new file mode 100644
index 0000000000..f832dd2ee4
--- /dev/null
+++ b/.sqlx/query-6fc1d293fb45bfd5788378f7dd61517f26adfbd430a833f07e0f63807fb40ae1.json
@@ -0,0 +1,72 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO crm_cleanup_jobs (id, total_candidates, max_candidate_id, status)\n VALUES ($1, $2, $3, 'Init')\n ON CONFLICT ((TRUE)) WHERE status IN ('Init', 'InProgress') DO NOTHING\n RETURNING\n id,\n status as \"status: CrmCleanupJobStatus\",\n total_candidates,\n dispatched_count,\n max_candidate_id,\n created_at,\n updated_at\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "status: CrmCleanupJobStatus",
+ "type_info": {
+ "Custom": {
+ "name": "crm_cleanup_job_status",
+ "kind": {
+ "Enum": [
+ "Init",
+ "InProgress",
+ "Complete",
+ "Failed"
+ ]
+ }
+ }
+ }
+ },
+ {
+ "ordinal": 2,
+ "name": "total_candidates",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 3,
+ "name": "dispatched_count",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 4,
+ "name": "max_candidate_id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 5,
+ "name": "created_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 6,
+ "name": "updated_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Int8",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "6fc1d293fb45bfd5788378f7dd61517f26adfbd430a833f07e0f63807fb40ae1"
+}
diff --git a/.sqlx/query-a46293bdc1febd9e7df39651549b5e47e1b142d30ce56b195ae6552796c9da96.json b/.sqlx/query-a46293bdc1febd9e7df39651549b5e47e1b142d30ce56b195ae6552796c9da96.json
new file mode 100644
index 0000000000..76802ec408
--- /dev/null
+++ b/.sqlx/query-a46293bdc1febd9e7df39651549b5e47e1b142d30ce56b195ae6552796c9da96.json
@@ -0,0 +1,27 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE crm_cleanup_jobs\n SET status = $1::crm_cleanup_job_status, updated_at = now()\n WHERE id = $2\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ {
+ "Custom": {
+ "name": "crm_cleanup_job_status",
+ "kind": {
+ "Enum": [
+ "Init",
+ "InProgress",
+ "Complete",
+ "Failed"
+ ]
+ }
+ }
+ },
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "a46293bdc1febd9e7df39651549b5e47e1b142d30ce56b195ae6552796c9da96"
+}
diff --git a/.sqlx/query-d2b6caccc2c37da4d1ca11c10fd78c7304e3e9aa16a80b11d9189fc27b97678d.json b/.sqlx/query-d2b6caccc2c37da4d1ca11c10fd78c7304e3e9aa16a80b11d9189fc27b97678d.json
new file mode 100644
index 0000000000..c96e257731
--- /dev/null
+++ b/.sqlx/query-d2b6caccc2c37da4d1ca11c10fd78c7304e3e9aa16a80b11d9189fc27b97678d.json
@@ -0,0 +1,68 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n id,\n status as \"status: CrmCleanupJobStatus\",\n total_candidates,\n dispatched_count,\n max_candidate_id,\n created_at,\n updated_at\n FROM crm_cleanup_jobs\n WHERE status IN ('Init', 'InProgress')\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "status: CrmCleanupJobStatus",
+ "type_info": {
+ "Custom": {
+ "name": "crm_cleanup_job_status",
+ "kind": {
+ "Enum": [
+ "Init",
+ "InProgress",
+ "Complete",
+ "Failed"
+ ]
+ }
+ }
+ }
+ },
+ {
+ "ordinal": 2,
+ "name": "total_candidates",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 3,
+ "name": "dispatched_count",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 4,
+ "name": "max_candidate_id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 5,
+ "name": "created_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 6,
+ "name": "updated_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "d2b6caccc2c37da4d1ca11c10fd78c7304e3e9aa16a80b11d9189fc27b97678d"
+}
diff --git a/.sqlx/query-ea00eb1f5f38a4397875efe17d1184107ca291aacf93d4d8fbfa3d5d49101ab7.json b/.sqlx/query-ea00eb1f5f38a4397875efe17d1184107ca291aacf93d4d8fbfa3d5d49101ab7.json
new file mode 100644
index 0000000000..568e0db692
--- /dev/null
+++ b/.sqlx/query-ea00eb1f5f38a4397875efe17d1184107ca291aacf93d4d8fbfa3d5d49101ab7.json
@@ -0,0 +1,26 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT MAX(id) as \"max_id\", COUNT(*) as \"count!\"\n FROM crm_cleanup_candidates\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "max_id",
+ "type_info": "Int8"
+ },
+ {
+ "ordinal": 1,
+ "name": "count!",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ null,
+ null
+ ]
+ },
+ "hash": "ea00eb1f5f38a4397875efe17d1184107ca291aacf93d4d8fbfa3d5d49101ab7"
+}
diff --git a/.sqlx/query-f188a83d94141f189fa0cbae17122fcc70a2bffb407d975755541c8b05972e3b.json b/.sqlx/query-f188a83d94141f189fa0cbae17122fcc70a2bffb407d975755541c8b05972e3b.json
new file mode 100644
index 0000000000..51bc9b6c18
--- /dev/null
+++ b/.sqlx/query-f188a83d94141f189fa0cbae17122fcc70a2bffb407d975755541c8b05972e3b.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n DELETE FROM crm_cleanup_candidates\n WHERE link_id = $1 AND contact_email = $2\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "f188a83d94141f189fa0cbae17122fcc70a2bffb407d975755541c8b05972e3b"
+}
diff --git a/crates/email_db_client/src/crm_cleanup/candidates.rs b/crates/email_db_client/src/crm_cleanup/candidates.rs
new file mode 100644
index 0000000000..836a35d932
--- /dev/null
+++ b/crates/email_db_client/src/crm_cleanup/candidates.rs
@@ -0,0 +1,104 @@
+use models_email::email::service::crm_cleanup::CrmCleanupCandidate;
+use sqlx::PgPool;
+use sqlx::types::Uuid;
+
+/// Upserts one cleanup candidate per email for the link. The unique index on
+/// `(link_id, contact_email)` dedupes repeat deletes into a single pending row.
+/// Returns the number of rows actually inserted.
+#[tracing::instrument(skip(pool, contact_emails), err)]
+pub async fn insert_candidates(
+ pool: &PgPool,
+ link_id: Uuid,
+ contact_emails: &[String],
+) -> anyhow::Result<u64> {
+ if contact_emails.is_empty() {
+ return Ok(0);
+ }
+
+ let result = sqlx::query!(
+ r#"
+ INSERT INTO crm_cleanup_candidates (link_id, contact_email)
+ SELECT $1, unnest($2::text[])
+ ON CONFLICT (link_id, contact_email) DO NOTHING
+ "#,
+ link_id,
+ contact_emails
+ )
+ .execute(pool)
+ .await?;
+
+ Ok(result.rows_affected())
+}
+
+/// Fetches one keyset page of candidates: `last_id < id <= max_id`, ordered by
+/// id. Rows deleted behind the cursor just leave gaps the predicate skips, so
+/// pagination never loses its place.
+#[tracing::instrument(skip(pool), err)]
+pub async fn list_candidates_page(
+ pool: &PgPool,
+ last_id: i64,
+ max_id: i64,
+ limit: i64,
+) -> anyhow::Result<Vec<CrmCleanupCandidate>> {
+ let candidates = sqlx::query_as!(
+ CrmCleanupCandidate,
+ r#"
+ SELECT id, link_id, contact_email
+ FROM crm_cleanup_candidates
+ WHERE id > $1 AND id <= $2
+ ORDER BY id
+ LIMIT $3
+ "#,
+ last_id,
+ max_id,
+ limit
+ )
+ .fetch_all(pool)
+ .await?;
+
+ Ok(candidates)
+}
+
+/// Claims (deletes) the candidate row for a pair. Returns `true` when the row
+/// existed. Claiming happens before the depopulate check so a message delete
+/// that lands mid-processing re-inserts a fresh row for the next run instead
+/// of being swallowed.
+#[tracing::instrument(skip(pool), err)]
+pub async fn claim_candidate(
+ pool: &PgPool,
+ link_id: Uuid,
+ contact_email: &str,
+) -> anyhow::Result<bool> {
+ let result = sqlx::query!(
+ r#"
+ DELETE FROM crm_cleanup_candidates
+ WHERE link_id = $1 AND contact_email = $2
+ "#,
+ link_id,
+ contact_email
+ )
+ .execute(pool)
+ .await?;
+
+ Ok(result.rows_affected() > 0)
+}
+
+/// Returns `(max_id, count)` over all candidate rows, or `None` when the table
+/// is empty. Snapshotting `max_id` at job kickoff freezes the job's working
+/// set: rows inserted later get higher ids and wait for the next run.
+#[tracing::instrument(skip(pool), err)]
+pub async fn get_max_id_and_count(pool: &PgPool) -> anyhow::Result<Option<(i64, i64)>> {
+ let row = sqlx::query!(
+ r#"
+ SELECT MAX(id) as "max_id", COUNT(*) as "count!"
+ FROM crm_cleanup_candidates
+ "#
+ )
+ .fetch_one(pool)
+ .await?;
+
+ Ok(row.max_id.map(|max_id| (max_id, row.count)))
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/email_db_client/src/crm_cleanup/candidates/test.rs b/crates/email_db_client/src/crm_cleanup/candidates/test.rs
new file mode 100644
index 0000000000..727c721866
--- /dev/null
+++ b/crates/email_db_client/src/crm_cleanup/candidates/test.rs
@@ -0,0 +1,114 @@
+use super::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::{Pool, Postgres};
+
+async fn insert_link(pool: &Pool<Postgres>, link_id: Uuid) {
+ sqlx::query!(
+ r#"INSERT INTO email_links (id, macro_id, fusionauth_user_id, email_address, provider)
+ VALUES ($1, $2, $2, $3, 'GMAIL')"#,
+ link_id,
+ )
+ .execute(pool)
+ .await
+ .unwrap();
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn insert_candidates_dedupes_pairs(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let link_id = Uuid::new_v4();
+ insert_link(&pool, link_id).await;
+
+ let emails = vec!["[email protected]".to_string(), "[email protected]".to_string()];
+ let inserted = insert_candidates(&pool, link_id, &emails).await?;
+ assert_eq!(inserted, 2);
+
+ // Re-inserting the same pairs (plus one new) only adds the new row.
+ let emails = vec![
+ "[email protected]".to_string(),
+ "[email protected]".to_string(),
+ "[email protected]".to_string(),
+ ];
+ let inserted = insert_candidates(&pool, link_id, &emails).await?;
+ assert_eq!(inserted, 1);
+
+ let (_, count) = get_max_id_and_count(&pool).await?.unwrap();
+ assert_eq!(count, 3);
+
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn insert_candidates_empty_is_noop(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let link_id = Uuid::new_v4();
+ insert_link(&pool, link_id).await;
+
+ let inserted = insert_candidates(&pool, link_id, &[]).await?;
+ assert_eq!(inserted, 0);
+ assert!(get_max_id_and_count(&pool).await?.is_none());
+
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn list_candidates_page_keyset_survives_deletes(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let link_id = Uuid::new_v4();
+ insert_link(&pool, link_id).await;
+
+ let emails: Vec<String> = (0..6).map(|i| format!("c{i}@ext.test")).collect();
+ insert_candidates(&pool, link_id, &emails).await?;
+
+ let (max_id, count) = get_max_id_and_count(&pool).await?.unwrap();
+ assert_eq!(count, 6);
+
+ let first_page = list_candidates_page(&pool, 0, max_id, 3).await?;
+ assert_eq!(first_page.len(), 3);
+ let cursor = first_page.last().unwrap().id;
+
+ // Consumers delete processed rows behind the cursor; the next page is unaffected.
+ for c in &first_page {
+ assert!(claim_candidate(&pool, c.link_id, &c.contact_email).await?);
+ }
+
+ let second_page = list_candidates_page(&pool, cursor, max_id, 3).await?;
+ assert_eq!(second_page.len(), 3);
+ assert!(second_page.iter().all(|c| c.id > cursor && c.id <= max_id));
+
+ let third_page = list_candidates_page(&pool, second_page.last().unwrap().id, max_id, 3).await?;
+ assert!(third_page.is_empty());
+
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn list_candidates_page_respects_max_id_snapshot(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let link_id = Uuid::new_v4();
+ insert_link(&pool, link_id).await;
+
+ insert_candidates(&pool, link_id, &["[email protected]".to_string()]).await?;
+ let (max_id, _) = get_max_id_and_count(&pool).await?.unwrap();
+
+ // A row inserted after the snapshot gets a higher id and is excluded.
+ insert_candidates(&pool, link_id, &["[email protected]".to_string()]).await?;
+
+ let page = list_candidates_page(&pool, 0, max_id, 10).await?;
+ assert_eq!(page.len(), 1);
+ assert_eq!(page[0].contact_email, "[email protected]");
+
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn claim_candidate_is_idempotent(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let link_id = Uuid::new_v4();
+ insert_link(&pool, link_id).await;
+
+ insert_candidates(&pool, link_id, &["[email protected]".to_string()]).await?;
+
+ assert!(claim_candidate(&pool, link_id, "[email protected]").await?);
+ // Second claim (duplicate dispatch) finds nothing and is harmless.
+ assert!(!claim_candidate(&pool, link_id, "[email protected]").await?);
+
+ Ok(())
+}
diff --git a/crates/email_db_client/src/crm_cleanup/job.rs b/crates/email_db_client/src/crm_cleanup/job.rs
new file mode 100644
index 0000000000..060c1537e7
--- /dev/null
+++ b/crates/email_db_client/src/crm_cleanup/job.rs
@@ -0,0 +1,137 @@
+use models_email::email::service::crm_cleanup::{CrmCleanupJob, CrmCleanupJobStatus};
+use sqlx::PgPool;
+use sqlx::types::Uuid;
+
+/// Creates an `Init` cleanup job. Returns `None` when an active
+/// (`Init`/`InProgress`) job already exists — the `uq_active_crm_cleanup_job`
+/// partial unique index makes this atomic, so duplicate kickoffs no-op.
+#[tracing::instrument(skip(pool), err)]
+pub async fn create_job(
+ pool: &PgPool,
+ total_candidates: i64,
+ max_candidate_id: i64,
+) -> anyhow::Result<Option<CrmCleanupJob>> {
+ let id = macro_uuid::generate_uuid_v7();
+
+ let record = sqlx::query_as!(
+ CrmCleanupJob,
+ r#"
+ INSERT INTO crm_cleanup_jobs (id, total_candidates, max_candidate_id, status)
+ VALUES ($1, $2, $3, 'Init')
+ ON CONFLICT ((TRUE)) WHERE status IN ('Init', 'InProgress') DO NOTHING
+ RETURNING
+ id,
+ status as "status: CrmCleanupJobStatus",
+ total_candidates,
+ dispatched_count,
+ max_candidate_id,
+ created_at,
+ updated_at
+ "#,
+ id,
+ total_candidates,
+ max_candidate_id
+ )
+ .fetch_optional(pool)
+ .await?;
+
+ Ok(record)
+}
+
+/// Fetches the active (`Init`/`InProgress`) cleanup job, if any. At most one
+/// exists thanks to the `uq_active_crm_cleanup_job` partial unique index.
+#[tracing::instrument(skip(pool), err)]
+pub async fn get_active_job(pool: &PgPool) -> anyhow::Result<Option<CrmCleanupJob>> {
+ let record = sqlx::query_as!(
+ CrmCleanupJob,
+ r#"
+ SELECT
+ id,
+ status as "status: CrmCleanupJobStatus",
+ total_candidates,
+ dispatched_count,
+ max_candidate_id,
+ created_at,
+ updated_at
+ FROM crm_cleanup_jobs
+ WHERE status IN ('Init', 'InProgress')
+ "#
+ )
+ .fetch_optional(pool)
+ .await?;
+
+ Ok(record)
+}
+
+#[tracing::instrument(skip(pool), err)]
+pub async fn get_job(pool: &PgPool, job_id: Uuid) -> anyhow::Result<Option<CrmCleanupJob>> {
+ let record = sqlx::query_as!(
+ CrmCleanupJob,
+ r#"
+ SELECT
+ id,
+ status as "status: CrmCleanupJobStatus",
+ total_candidates,
+ dispatched_count,
+ max_candidate_id,
+ created_at,
+ updated_at
+ FROM crm_cleanup_jobs
+ WHERE id = $1
+ "#,
+ job_id
+ )
+ .fetch_optional(pool)
+ .await?;
+
+ Ok(record)
+}
+
+#[tracing::instrument(skip(pool), err)]
+pub async fn add_dispatched_count(pool: &PgPool, job_id: Uuid, count: i64) -> anyhow::Result<()> {
+ let result = sqlx::query!(
+ r#"
+ UPDATE crm_cleanup_jobs
+ SET dispatched_count = dispatched_count + $1, updated_at = now()
+ WHERE id = $2
+ "#,
+ count,
+ job_id
+ )
+ .execute(pool)
+ .await?;
+
+ if result.rows_affected() == 0 {
+ anyhow::bail!("No crm cleanup job found with ID: {}", job_id);
+ }
+
+ Ok(())
+}
+
+#[tracing::instrument(skip(pool), err)]
+pub async fn set_job_status(
+ pool: &PgPool,
+ job_id: Uuid,
+ status: CrmCleanupJobStatus,
+) -> anyhow::Result<()> {
+ let result = sqlx::query!(
+ r#"
+ UPDATE crm_cleanup_jobs
+ SET status = $1::crm_cleanup_job_status, updated_at = now()
+ WHERE id = $2
+ "#,
+ status as _,
+ job_id
+ )
+ .execute(pool)
+ .await?;
+
+ if result.rows_affected() == 0 {
+ anyhow::bail!("No crm cleanup job found with ID: {}", job_id);
+ }
+
+ Ok(())
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/email_db_client/src/crm_cleanup/job/test.rs b/crates/email_db_client/src/crm_cleanup/job/test.rs
new file mode 100644
index 0000000000..bf9abb1d2e
--- /dev/null
+++ b/crates/email_db_client/src/crm_cleanup/job/test.rs
@@ -0,0 +1,61 @@
+use super::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::{Pool, Postgres};
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn create_job_dedupes_active_job(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let first = create_job(&pool, 10, 100).await?;
+ assert!(first.is_some(), "first create should insert a job");
+
+ let second = create_job(&pool, 20, 200).await?;
+ assert!(
+ second.is_none(),
+ "second create should no-op while a job is active"
+ );
+
+ // The active job is the first one.
+ let first_id = first.unwrap().id;
+ let active = get_active_job(&pool).await?;
+ assert_eq!(active.map(|j| j.id), Some(first_id));
+
+ // Once the active job finishes, a new one can be created.
+ set_job_status(&pool, first_id, CrmCleanupJobStatus::Complete).await?;
+ assert!(get_active_job(&pool).await?.is_none());
+ let third = create_job(&pool, 20, 200).await?;
+ assert!(third.is_some(), "create should succeed after completion");
+
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn job_counters_and_status_roundtrip(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let job = create_job(&pool, 5, 50).await?.unwrap();
+ assert_eq!(job.total_candidates, 5);
+ assert_eq!(job.max_candidate_id, 50);
+ assert_eq!(job.dispatched_count, 0);
+ assert_eq!(job.status, CrmCleanupJobStatus::Init);
+
+ set_job_status(&pool, job.id, CrmCleanupJobStatus::InProgress).await?;
+ add_dispatched_count(&pool, job.id, 3).await?;
+ add_dispatched_count(&pool, job.id, 2).await?;
+
+ let fetched = get_job(&pool, job.id).await?.unwrap();
+ assert_eq!(fetched.dispatched_count, 5);
+ assert_eq!(fetched.status, CrmCleanupJobStatus::InProgress);
+
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn job_updates_fail_for_unknown_id(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let missing = Uuid::new_v4();
+ assert!(add_dispatched_count(&pool, missing, 1).await.is_err());
+ assert!(
+ set_job_status(&pool, missing, CrmCleanupJobStatus::Failed)
+ .await
+ .is_err()
+ );
+ assert!(get_job(&pool, missing).await?.is_none());
+
+ Ok(())
+}
diff --git a/crates/email_db_client/src/crm_cleanup/mod.rs b/crates/email_db_client/src/crm_cleanup/mod.rs
new file mode 100644
index 0000000000..26abe8d69c
--- /dev/null
+++ b/crates/email_db_client/src/crm_cleanup/mod.rs
@@ -0,0 +1,2 @@
+pub mod candidates;
+pub mod job;
diff --git a/crates/email_db_client/src/lib.rs b/crates/email_db_client/src/lib.rs
index c1bdc26cd7..16eecc673e 100644
--- a/crates/email_db_client/src/lib.rs
+++ b/crates/email_db_client/src/lib.rs
@@ -1,6 +1,7 @@
pub mod attachments;
pub mod backfill;
pub mod contacts;
+pub mod crm_cleanup;
pub mod histories;
pub mod labels;
pub mod links;
diff --git a/crates/macro_db_client/migrations/20260723160259_crm_cleanup_tables.sql b/crates/macro_db_client/migrations/20260723160259_crm_cleanup_tables.sql
new file mode 100644
index 0000000000..4589e6362f
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260723160259_crm_cleanup_tables.sql
@@ -0,0 +1,35 @@
+-- Deferred CRM cleanup: deduped (link_id, contact_email) pairs written on message
+-- delete, swept by a nightly job instead of per-delete depopulate messages.
+CREATE TABLE IF NOT EXISTS crm_cleanup_candidates (
+ id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+ link_id UUID NOT NULL REFERENCES email_links (id) ON DELETE CASCADE,
+ contact_email TEXT NOT NULL,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_crm_cleanup_candidate
+ ON crm_cleanup_candidates (link_id, contact_email);
+
+DO $$
+BEGIN
+ CREATE TYPE crm_cleanup_job_status AS ENUM ('Init', 'InProgress', 'Complete', 'Failed');
+EXCEPTION
+ WHEN duplicate_object THEN NULL;
+END
+$$;
+
+CREATE TABLE IF NOT EXISTS crm_cleanup_jobs (
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ status crm_cleanup_job_status NOT NULL DEFAULT 'Init',
+ total_candidates BIGINT NOT NULL,
+ dispatched_count BIGINT NOT NULL DEFAULT 0,
+ -- MAX(crm_cleanup_candidates.id) at kickoff; the job only processes ids <= this
+ max_candidate_id BIGINT NOT NULL,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- Only one active cleanup job at a time
+CREATE UNIQUE INDEX IF NOT EXISTS uq_active_crm_cleanup_job
+ ON crm_cleanup_jobs ((TRUE))
+ WHERE status IN ('Init', 'InProgress');
diff --git a/crates/macro_queues/src/lib.rs b/crates/macro_queues/src/lib.rs
index fb492e55a0..db30ab6fe0 100644
--- a/crates/macro_queues/src/lib.rs
+++ b/crates/macro_queues/src/lib.rs
@@ -590,6 +590,12 @@ queue! {
dev: "email-service-backfill-queue-dev",
prod: "email-service-backfill-queue-prod",
},
+ /// Queue for the nightly CRM cleanup job (email_service).
+ pub EmailCrmCleanupQueue {
+ local: "email-service-crm-cleanup-queue",
+ dev: "email-service-crm-cleanup-queue-dev",
+ prod: "email-service-crm-cleanup-queue-prod",
+ },
/// Queue for the bulk upload extractor lambda
/// (upload_extractor_lambda_trigger `UPLOAD_EXTRACTOR_QUEUE`).
pub UploadExtractorQueue {
diff --git a/crates/models_email/src/email/service/crm_cleanup.rs b/crates/models_email/src/email/service/crm_cleanup.rs
new file mode 100644
index 0000000000..4885ef18e9
--- /dev/null
+++ b/crates/models_email/src/email/service/crm_cleanup.rs
@@ -0,0 +1,90 @@
+//! Types for the nightly CRM cleanup queue. Message deletions upsert
+//! `(link_id, contact_email)` rows into `crm_cleanup_candidates`; a nightly
+//! EventBridge-triggered job pages through the table and tears down CRM rows
+//! for contacts that no longer have any messages on the link.
+
+#[cfg(test)]
+mod test;
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use strum::{AsRefStr, Display, EnumString};
+use uuid::Uuid;
+
+#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum CrmCleanupOperation {
+ // Nightly kickoff, sent by EventBridge directly to the queue as the
+ // static payload {"operation":"start_job"}. Snapshots the candidate
+ // table, creates the job row, and enqueues the first ListCandidates.
+ StartJob,
+ // Lists one keyset page of candidates (id > last_id, bounded by the job's
+ // max_candidate_id snapshot), publishes a ProcessCandidate per row, then
+ // re-enqueues itself with the new cursor. A short page completes the job.
+ ListCandidates {
+ job_id: Uuid,
+ last_id: i64,
+ },
+ // Claims (deletes) the candidate row, then depopulates the CRM contact if
+ // the link has no remaining messages involving the contact.
+ ProcessCandidate {
+ link_id: Uuid,
+ contact_email: String,
+ },
+}
+
+// the object we send on the crm cleanup pubsub queue
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CrmCleanupPubsubMessage {
+ pub operation: CrmCleanupOperation,
+}
+
+// Enum for crm cleanup job status
+#[derive(
+ Debug,
+ Serialize,
+ Deserialize,
+ sqlx::Type,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ EnumString,
+ AsRefStr,
+ Display,
+)]
+#[sqlx(type_name = "crm_cleanup_job_status", rename_all = "PascalCase")]
+pub enum CrmCleanupJobStatus {
+ Init,
+ InProgress,
+ // Fanout complete: every candidate in the snapshot has been dispatched.
+ // ProcessCandidate messages may still be in flight or retrying.
+ Complete,
+ Failed,
+}
+
+// Struct for the crm_cleanup_jobs table
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct CrmCleanupJob {
+ pub id: Uuid,
+ pub status: CrmCleanupJobStatus,
+ // Candidate row count at kickoff, for observability
+ pub total_candidates: i64,
+ // Number of candidates dispatched as ProcessCandidate messages so far.
+ // Observability only: redelivered lister pages re-count, so this can
+ // exceed total_candidates.
+ pub dispatched_count: i64,
+ // MAX(crm_cleanup_candidates.id) at kickoff; the job only processes ids <= this
+ pub max_candidate_id: i64,
+ pub created_at: DateTime<Utc>,
+ pub updated_at: DateTime<Utc>,
+}
+
+/// A row of `crm_cleanup_candidates`.
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct CrmCleanupCandidate {
+ pub id: i64,
+ pub link_id: Uuid,
+ pub contact_email: String,
+}
diff --git a/crates/models_email/src/email/service/crm_cleanup/test.rs b/crates/models_email/src/email/service/crm_cleanup/test.rs
new file mode 100644
index 0000000000..3ab168c5e6
--- /dev/null
+++ b/crates/models_email/src/email/service/crm_cleanup/test.rs
@@ -0,0 +1,34 @@
+use super::*;
+
+/// The EventBridge rule sends this exact static payload straight to the
+/// queue (see infra/stacks/email-service/index.ts). If the serde shape of
+/// the wrapper or enum changes, the nightly cron silently stops parsing —
+/// this test pins the wire format.
+#[test]
+fn start_job_static_payload_deserializes() {
+ let msg: CrmCleanupPubsubMessage = serde_json::from_str(r#"{"operation":"start_job"}"#)
+ .expect("EventBridge static payload must deserialize");
+ assert_eq!(msg.operation, CrmCleanupOperation::StartJob);
+}
+
+#[test]
+fn operations_round_trip() {
+ let list = CrmCleanupPubsubMessage {
+ operation: CrmCleanupOperation::ListCandidates {
+ job_id: Uuid::new_v4(),
+ last_id: 42,
+ },
+ };
+ let process = CrmCleanupPubsubMessage {
+ operation: CrmCleanupOperation::ProcessCandidate {
+ link_id: Uuid::new_v4(),
+ contact_email: "[email protected]".to_string(),
+ },
+ };
+
+ for msg in [list, process] {
+ let json = serde_json::to_string(&msg).unwrap();
+ let parsed: CrmCleanupPubsubMessage = serde_json::from_str(&json).unwrap();
+ assert_eq!(parsed.operation, msg.operation);
+ }
+}
diff --git a/crates/models_email/src/email/service/mod.rs b/crates/models_email/src/email/service/mod.rs
index 7b6098f65e..c5d081cd21 100644
--- a/crates/models_email/src/email/service/mod.rs
+++ b/crates/models_email/src/email/service/mod.rs
@@ -4,6 +4,7 @@ pub mod backfill;
mod body_parsing;
pub mod cache;
pub mod contact;
+pub mod crm_cleanup;
pub mod label;
pub mod link;
pub mod message;
diff --git a/crates/sqs_client/src/email.rs b/crates/sqs_client/src/email.rs
index e5e5bd63ce..38adcf04f2 100644
--- a/crates/sqs_client/src/email.rs
+++ b/crates/sqs_client/src/email.rs
@@ -1,6 +1,7 @@
use crate::SQS;
use email::domain::ports::EmailMessageEnqueuer;
use models_email::email::service::backfill::BackfillPubsubMessage;
+use models_email::email::service::crm_cleanup::CrmCleanupPubsubMessage;
pub use models_email::email::service::pubsub::LinkManagerMessage;
#[cfg(feature = "sfs_uploader")]
use models_email::service::pubsub::SFSUploaderMessage;
@@ -23,6 +24,11 @@ impl SQS {
self
}
+ pub fn email_crm_cleanup_queue(mut self, email_crm_cleanup_queue: &str) -> Self {
+ self.email_crm_cleanup_queue = Some(email_crm_cleanup_queue.to_string());
+ self
+ }
+
#[cfg(feature = "sfs_uploader")]
pub fn sfs_uploader_queue(mut self, email_sfs_uploader_queue: &str) -> Self {
self.email_sfs_uploader_queue = Some(email_sfs_uploader_queue.to_string());
@@ -60,7 +66,20 @@ impl SQS {
Err(anyhow::anyhow!("email_backfill_queue is not configured"))
}
- /// Sends a message to the email backfill queue
+ /// Sends a message to the email crm cleanup queue
+ #[tracing::instrument(skip(self), err)]
+ pub async fn enqueue_email_crm_cleanup_message(
+ &self,
+ message: CrmCleanupPubsubMessage,
+ ) -> anyhow::Result<()> {
+ if let Some(email_crm_cleanup_queue) = &self.email_crm_cleanup_queue {
+ return enqueue_crm_cleanup_message(&self.inner, email_crm_cleanup_queue, message)
+ .await;
+ }
+ anyhow::bail!("email_crm_cleanup_queue is not configured")
+ }
+
+ /// Sends a message to the email scheduled queue
#[tracing::instrument(skip(self))]
pub async fn enqueue_email_scheduled_message(
&self,
@@ -195,6 +214,24 @@ pub async fn enqueue_backfill_message(
Ok(())
}
+#[tracing::instrument(skip(sqs_client))]
+pub async fn enqueue_crm_cleanup_message(
+ sqs_client: &aws_sdk_sqs::Client,
+ queue_url: &str,
+ message: CrmCleanupPubsubMessage,
+) -> anyhow::Result<()> {
+ let message_str = serde_json::to_string(&message)?;
+
+ sqs_client
+ .send_message()
+ .queue_url(queue_url)
+ .message_body(message_str)
+ .send()
+ .await?;
+
+ Ok(())
+}
+
#[tracing::instrument(skip(sqs_client))]
pub async fn enqueue_scheduled_message(
sqs_client: &aws_sdk_sqs::Client,
diff --git a/crates/sqs_client/src/lib.rs b/crates/sqs_client/src/lib.rs
index 3ca831b2bb..c480e626f2 100644
--- a/crates/sqs_client/src/lib.rs
+++ b/crates/sqs_client/src/lib.rs
@@ -86,6 +86,8 @@ pub struct SQS {
email_scheduled_queue: Option<String>,
#[cfg(feature = "email")]
email_backfill_queue: Option<String>,
+ #[cfg(feature = "email")]
+ email_crm_cleanup_queue: Option<String>,
#[cfg(feature = "search")]
search_event_queue: Option<String>,
#[cfg(feature = "sfs_uploader")]
@@ -126,6 +128,8 @@ impl SQS {
email_scheduled_queue: None,
#[cfg(feature = "email")]
email_backfill_queue: None,
+ #[cfg(feature = "email")]
+ email_crm_cleanup_queue: None,
#[cfg(feature = "search")]
search_event_queue: None,
#[cfg(feature = "sfs_uploader")]
diff --git a/infra/stacks/email-service/index.ts b/infra/stacks/email-service/index.ts
index fd5b291921..74364a2d58 100644
--- a/infra/stacks/email-service/index.ts
+++ b/infra/stacks/email-service/index.ts
@@ -175,6 +175,50 @@ const backfill_queue = new Queue('email-service-backfill', {
export const backfillQueueArn = pulumi.interpolate`${backfill_queue.queue.arn}`;
export const backfillQueueName = pulumi.interpolate`${backfill_queue.queue.name}`;
+const crm_cleanup_queue = new Queue('email-service-crm-cleanup', {
+ tags,
+ maxReceiveCount: 5,
+ visibilityTimeoutSeconds: 60,
+});
+
+export const crmCleanupQueueArn = pulumi.interpolate`${crm_cleanup_queue.queue.arn}`;
+export const crmCleanupQueueName = pulumi.interpolate`${crm_cleanup_queue.queue.name}`;
+
+// Nightly at 08:00 UTC (midnight PT): EventBridge sends a static StartJob
+// payload straight to the crm cleanup queue; the pubsub worker creates the
+// job row and starts paging candidates.
+const crmCleanupRule = new aws.cloudwatch.EventRule('crm-cleanup-rule', {
+ name: `email-crm-cleanup-rule-${stack}`,
+ scheduleExpression: 'cron(0 8 * * ? *)',
+ tags,
+});
+
+new aws.cloudwatch.EventTarget('crm-cleanup-nightly-target', {
+ rule: crmCleanupRule.name,
+ arn: crmCleanupQueueArn,
+ input: JSON.stringify({ operation: 'start_job' }),
+});
+
+new aws.sqs.QueuePolicy('crm-cleanup-queue-policy', {
+ queueUrl: crm_cleanup_queue.queue.url,
+ policy: pulumi
+ .all([crmCleanupQueueArn, crmCleanupRule.arn])
+ .apply(([queueArn, ruleArn]) =>
+ JSON.stringify({
+ Version: '2012-10-17',
+ Statement: [
+ {
+ Effect: 'Allow',
+ Principal: { Service: 'events.amazonaws.com' },
+ Action: 'sqs:SendMessage',
+ Resource: queueArn,
+ Condition: { ArnEquals: { 'aws:SourceArn': ruleArn } },
+ },
+ ],
+ })
+ ),
+});
+
const sfs_uploader_queue = new Queue('email-service-sfs-mapper', {
tags,
maxReceiveCount: 5,
@@ -233,6 +277,7 @@ const queueArns = [
scheduledQueueArn,
searchEventQueueArn,
backfillQueueArn,
+ crmCleanupQueueArn,
sfsUploaderQueueArn,
sfsDeleteQueueArn,
contactsQueueArn,
diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
index 39b3226e16..0a4a3405c1 100644
--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
@@ -73,6 +73,7 @@ async fn main() -> anyhow::Result<()> {
let gmail_ops_retry_queue = macro_queues::GmailOpsRetryQueue::new();
let search_event_queue = macro_queues::SearchEventQueue::new();
let backfill_queue = macro_queues::EmailBackfillQueue::new();
+ let crm_cleanup_queue = macro_queues::EmailCrmCleanupQueue::new();
let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
let sfs_uploader_queue = macro_queues::SfsUploaderQueue::new();
let sfs_delete_queue = macro_queues::SfsDeleteQueue::new();
@@ -87,6 +88,7 @@ async fn main() -> anyhow::Result<()> {
.gmail_ops_retry_queue(&gmail_ops_retry_queue)
.search_event_queue(&search_event_queue)
.email_backfill_queue(&backfill_queue)
+ .email_crm_cleanup_queue(&crm_cleanup_queue)
.email_scheduled_queue(&email_scheduled_queue)
.sfs_uploader_queue(&sfs_uploader_queue)
.sfs_delete_queue(&sfs_delete_queue)
@@ -147,6 +149,17 @@ async fn main() -> anyhow::Result<()> {
})
.collect::<Vec<_>>();
+ let crm_cleanup_workers = (0..config.crm_cleanup_queue_workers)
+ .map(|_| {
+ sqs_worker::SQSWorker::new(
+ aws_sdk_sqs::Client::new(&gmail_queue_aws_config),
+ crm_cleanup_queue.to_string(),
+ config.crm_cleanup_queue_max_messages,
+ config.queue_wait_time_seconds,
+ )
+ })
+ .collect::<Vec<_>>();
+
let gmail_ops_workers = (0..config.gmail_ops_queue_workers)
.map(|_| {
sqs_worker::SQSWorker::new(
@@ -470,6 +483,27 @@ async fn main() -> anyhow::Result<()> {
"backfill workers started"
);
+ // nightly crm cleanup: pages crm_cleanup_candidates and depopulates
+ // contacts whose links no longer have messages with them
+ for worker in crm_cleanup_workers {
+ let db_crm_cleanup = db_backfill.clone();
+ let sqs_client_crm_cleanup = sqs_client.clone();
+ let crm_service_crm_cleanup = crm_service_backfill.clone();
+ tokio::spawn(async move {
+ email_service::pubsub::crm_cleanup::worker::run_worker(
+ db_crm_cleanup,
+ worker,
+ sqs_client_crm_cleanup,
+ crm_service_crm_cleanup,
+ )
+ .await;
+ });
+ }
+ tracing::info!(
+ num_workers = config.crm_cleanup_queue_workers,
+ "crm cleanup workers started"
+ );
+
let db_link_manager = db.clone();
let gmail_client_link_manager = gmail_client.clone();
let auth_service_client_link_manager = auth_service_client.clone();
diff --git a/services/email_service/src/config.rs b/services/email_service/src/config.rs
index f1709224cb..49fa8f5106 100644
--- a/services/email_service/src/config.rs
+++ b/services/email_service/src/config.rs
@@ -72,6 +72,14 @@ pub struct Config {
#[macro_config_default(1)]
pub backfill_queue_max_messages: i32,
+ /// The number of workers we spawn for the nightly crm cleanup queue
+ #[macro_config_default(2)]
+ pub crm_cleanup_queue_workers: i32,
+
+ /// The queue max messages per poll for crm cleanup
+ #[macro_config_default(10)]
+ pub crm_cleanup_queue_max_messages: i32,
+
/// The number of workers we spawn for gmail inbox sync
#[macro_config_default(10)]
pub inbox_sync_queue_workers: i32,
diff --git a/services/email_service/src/pubsub/crm_cleanup/list_candidates.rs b/services/email_service/src/pubsub/crm_cleanup/list_candidates.rs
new file mode 100644
index 0000000000..ff1db31697
--- /dev/null
+++ b/services/email_service/src/pubsub/crm_cleanup/list_candidates.rs
@@ -0,0 +1,119 @@
+use crate::pubsub::crm_cleanup::worker::CrmCleanupContext;
+use email_db_client::crm_cleanup::{candidates, job};
+use models_email::email::service::crm_cleanup::{
+ CrmCleanupJobStatus, CrmCleanupOperation, CrmCleanupPubsubMessage,
+};
+use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
+use uuid::Uuid;
+
+/// Candidates dispatched per ListCandidates message.
+const CANDIDATE_PAGE_SIZE: i64 = 200;
+
+/// Dispatches one keyset page of cleanup candidates as `ProcessCandidate`
+/// messages, then re-enqueues itself with the new cursor. Mirrors the
+/// backfill `list_threads` pattern: each page is its own SQS message, so a
+/// crash resumes from the last acked page. A short page means every id up to
+/// the job's `max_candidate_id` snapshot has been dispatched — the job is
+/// marked `Complete`.
+///
+/// A redelivered page re-publishes ProcessCandidate messages for pairs that
+/// were already dispatched; that's harmless because the consumer's claim and
+/// gate check are both idempotent.
+#[tracing::instrument(skip(ctx), err)]
+pub async fn list_candidates(
+ ctx: &CrmCleanupContext,
+ job_id: Uuid,
+ last_id: i64,
+) -> Result<(), ProcessingError> {
+ let job = job::get_job(&ctx.db, job_id)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to fetch crm cleanup job"),
+ })
+ })?
+ .ok_or_else(|| {
+ ProcessingError::NonRetryable(DetailedError {
+ reason: FailureReason::InvalidData,
+ source: anyhow::anyhow!("Crm cleanup job not found: {job_id}"),
+ })
+ })?;
+
+ match job.status {
+ CrmCleanupJobStatus::Init | CrmCleanupJobStatus::InProgress => {}
+ CrmCleanupJobStatus::Complete | CrmCleanupJobStatus::Failed => {
+ tracing::warn!(job_id = %job_id, status = %job.status, "Stale ListCandidates message for finished job; acking");
+ return Ok(());
+ }
+ }
+
+ let page = candidates::list_candidates_page(
+ &ctx.db,
+ last_id,
+ job.max_candidate_id,
+ CANDIDATE_PAGE_SIZE,
+ )
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to list crm cleanup candidates"),
+ })
+ })?;
+
+ for candidate in &page {
+ ctx.sqs_client
+ .enqueue_email_crm_cleanup_message(CrmCleanupPubsubMessage {
+ operation: CrmCleanupOperation::ProcessCandidate {
+ link_id: candidate.link_id,
+ contact_email: candidate.contact_email.clone(),
+ },
+ })
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::SqsEnqueueFailed,
+ source: e.context("Failed to enqueue ProcessCandidate message"),
+ })
+ })?;
+ }
+
+ // Observability only; may over-count when a page is redelivered mid-dispatch.
+ if !page.is_empty() {
+ job::add_dispatched_count(&ctx.db, job_id, page.len() as i64)
+ .await
+ .inspect_err(|e| tracing::error!(error = ?e, "Failed to bump dispatched_count"))
+ .ok();
+ }
+
+ if page.len() as i64 == CANDIDATE_PAGE_SIZE {
+ let next_cursor = page.last().map(|c| c.id).unwrap_or(last_id);
+ ctx.sqs_client
+ .enqueue_email_crm_cleanup_message(CrmCleanupPubsubMessage {
+ operation: CrmCleanupOperation::ListCandidates {
+ job_id,
+ last_id: next_cursor,
+ },
+ })
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::SqsEnqueueFailed,
+ source: e.context("Failed to re-enqueue ListCandidates message"),
+ })
+ })?;
+ } else {
+ job::set_job_status(&ctx.db, job_id, CrmCleanupJobStatus::Complete)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to mark crm cleanup job complete"),
+ })
+ })?;
+ tracing::info!(job_id = %job_id, dispatched = job.dispatched_count + page.len() as i64, "Crm cleanup job dispatch complete");
+ }
+
+ Ok(())
+}
diff --git a/services/email_service/src/pubsub/crm_cleanup/mod.rs b/services/email_service/src/pubsub/crm_cleanup/mod.rs
new file mode 100644
index 0000000000..53dd334c54
--- /dev/null
+++ b/services/email_service/src/pubsub/crm_cleanup/mod.rs
@@ -0,0 +1,12 @@
+//! Nightly CRM cleanup worker. An EventBridge cron sends a static `StartJob`
+//! payload straight to the queue; the handler creates a `crm_cleanup_jobs`
+//! row and enqueues the first `ListCandidates` message; the lister pages
+//! through `crm_cleanup_candidates` with a keyset cursor, fanning out one
+//! `ProcessCandidate` per pair and re-enqueueing itself until the job's
+//! `max_candidate_id` snapshot is exhausted.
+
+mod list_candidates;
+mod process;
+mod process_candidate;
+mod start_job;
+pub mod worker;
diff --git a/services/email_service/src/pubsub/crm_cleanup/process.rs b/services/email_service/src/pubsub/crm_cleanup/process.rs
new file mode 100644
index 0000000000..39ec2eb42d
--- /dev/null
+++ b/services/email_service/src/pubsub/crm_cleanup/process.rs
@@ -0,0 +1,91 @@
+use crate::pubsub::crm_cleanup::worker::CrmCleanupContext;
+use crate::pubsub::crm_cleanup::{list_candidates, process_candidate, start_job};
+use anyhow::Context;
+use models_email::email::service::crm_cleanup::{
+ CrmCleanupJobStatus, CrmCleanupOperation, CrmCleanupPubsubMessage,
+};
+use models_email::email::service::pubsub::ProcessingError;
+use sqs_worker::cleanup_message;
+
+/// Process a single message from the crm cleanup queue.
+pub async fn process_message(
+ ctx: CrmCleanupContext,
+ message: &aws_sdk_sqs::types::Message,
+) -> anyhow::Result<()> {
+ // Malformed JSON is NOT retryable.
+ let data = match extract_cleanup_message(message) {
+ Ok(data) => data,
+ Err(e) => {
+ tracing::error!(error = %e, "Failed to extract crm cleanup message, this is non-retryable.");
+ if let Err(cleanup_err) = cleanup_message(&ctx.sqs_worker, message).await {
+ tracing::error!(error = %cleanup_err, "Failed to clean up message after extraction error");
+ }
+ return Err(e);
+ }
+ };
+
+ let processing_result = inner_process_message(&ctx, &data).await;
+
+ match processing_result {
+ // Processing success. Clean up the message
+ Ok(()) => {
+ cleanup_message(&ctx.sqs_worker, message).await?;
+ Ok(())
+ }
+
+ // A permanent failure occurred. Clean up the message so it isn't retried;
+ // a dead lister also fails its job so it doesn't dangle as active.
+ Err(ProcessingError::NonRetryable(e)) => {
+ tracing::error!(error = ?e, operation = ?data.operation, "Non-retryable crm cleanup error");
+ if let CrmCleanupOperation::ListCandidates { job_id, .. } = &data.operation
+ && let Err(status_err) = email_db_client::crm_cleanup::job::set_job_status(
+ &ctx.db,
+ *job_id,
+ CrmCleanupJobStatus::Failed,
+ )
+ .await
+ {
+ tracing::error!(error = ?status_err, job_id = %job_id, "Failed to mark crm cleanup job as failed");
+ }
+ cleanup_message(&ctx.sqs_worker, message).await?;
+ Ok(())
+ }
+
+ // A temporary failure occurred. Leave the message for SQS redelivery;
+ // the queue's redrive policy dead-letters poison messages.
+ Err(ProcessingError::Retryable(e)) => {
+ tracing::warn!(error = ?e, operation = ?data.operation, "Retryable crm cleanup error; message will be redelivered");
+ Ok(())
+ }
+ }
+}
+
+#[tracing::instrument(skip(ctx))]
+async fn inner_process_message(
+ ctx: &CrmCleanupContext,
+ data: &CrmCleanupPubsubMessage,
+) -> Result<(), ProcessingError> {
+ match &data.operation {
+ CrmCleanupOperation::StartJob => start_job::start_job(ctx).await,
+ CrmCleanupOperation::ListCandidates { job_id, last_id } => {
+ list_candidates::list_candidates(ctx, *job_id, *last_id).await
+ }
+ CrmCleanupOperation::ProcessCandidate {
+ link_id,
+ contact_email,
+ } => process_candidate::process_candidate(ctx, *link_id, contact_email).await,
+ }
+}
+
+/// Extracts the crm cleanup message from the SQS message body
+#[tracing::instrument(skip(message))]
+fn extract_cleanup_message(
+ message: &aws_sdk_sqs::types::Message,
+) -> anyhow::Result<CrmCleanupPubsubMessage> {
+ let message_body = message.body().context("message body not found")?;
+
+ let cleanup_message: CrmCleanupPubsubMessage = serde_json::from_str(message_body)
+ .context("Failed to deserialize message body to CrmCleanupPubsubMessage")?;
+
+ Ok(cleanup_message)
+}
diff --git a/services/email_service/src/pubsub/crm_cleanup/process_candidate.rs b/services/email_service/src/pubsub/crm_cleanup/process_candidate.rs
new file mode 100644
index 0000000000..e57067c329
--- /dev/null
+++ b/services/email_service/src/pubsub/crm_cleanup/process_candidate.rs
@@ -0,0 +1,99 @@
+use crate::pubsub::crm_cleanup::worker::CrmCleanupContext;
+use crm::domain::service::CrmService;
+use email_db_client::crm_cleanup::candidates;
+use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
+use uuid::Uuid;
+
+/// Processes one cleanup candidate: claims (deletes) the candidate row, then
+/// depopulates the CRM contact if the link no longer has any message (sent or
+/// received) involving `contact_email`.
+///
+/// The claim happens *before* the gate check so that a message delete landing
+/// mid-processing re-inserts a fresh candidate for the next nightly run
+/// instead of being swallowed. We proceed even when the row is already gone —
+/// a redelivery after a mid-processing failure must still run the gate, and a
+/// duplicate dispatch just performs one redundant (cheap) check.
+///
+/// The gate runs outside the CRM crate's advisory lock, so a brand-new sent
+/// message landing between the check and the cascade could see a transient
+/// deleted state; the populate path on that new message re-creates the rows.
+#[tracing::instrument(skip(ctx), err, fields(link_id = %link_id, contact_email = %contact_email))]
+pub async fn process_candidate(
+ ctx: &CrmCleanupContext,
+ link_id: Uuid,
+ contact_email: &str,
+) -> Result<(), ProcessingError> {
+ let claimed = candidates::claim_candidate(&ctx.db, link_id, contact_email)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to claim crm cleanup candidate"),
+ })
+ })?;
+
+ if !claimed {
+ tracing::debug!("Candidate row already claimed (duplicate dispatch or redelivery)");
+ }
+
+ let link = match email_db_client::links::get::fetch_link_by_id(&ctx.db, link_id).await {
+ Ok(Some(link)) => link,
+ Ok(None) => {
+ // Link deleted since the candidate was written; its CRM sources are
+ // torn down by the link-deletion flow.
+ tracing::debug!("Link no longer exists; skipping CRM cleanup");
+ return Ok(());
+ }
+ Err(e) => {
+ return Err(ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to fetch link for crm cleanup"),
+ }));
+ }
+ };
+
+ let team_id = ctx
+ .crm_service
+ .get_team_id_for_user(&link.macro_id.to_string())
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: anyhow::Error::from(e).context("Failed to look up team for link.macro_id"),
+ })
+ })?;
+
+ let Some(team_id) = team_id else {
+ tracing::debug!("User has no team; skipping CRM cleanup");
+ return Ok(());
+ };
+
+ let still_has_any =
+ email_db_client::contacts::get::link_has_any_message_with(&ctx.db, link.id, contact_email)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to check remaining messages with contact"),
+ })
+ })?;
+
+ if still_has_any {
+ tracing::debug!(
+ "Link still has messages with contact (sent or received); skipping depopulation"
+ );
+ return Ok(());
+ }
+
+ ctx.crm_service
+ .depopulate_contact(&team_id, &link.id, contact_email)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: anyhow::Error::from(e).context("Failed to depopulate CRM contact"),
+ })
+ })?;
+
+ Ok(())
+}
diff --git a/services/email_service/src/pubsub/crm_cleanup/start_job.rs b/services/email_service/src/pubsub/crm_cleanup/start_job.rs
new file mode 100644
index 0000000000..db971052c5
--- /dev/null
+++ b/services/email_service/src/pubsub/crm_cleanup/start_job.rs
@@ -0,0 +1,137 @@
+use crate::pubsub::crm_cleanup::worker::CrmCleanupContext;
+use email_db_client::crm_cleanup::{candidates, job};
+use models_email::email::service::crm_cleanup::{
+ CrmCleanupJob, CrmCleanupJobStatus, CrmCleanupOperation, CrmCleanupPubsubMessage,
+};
+use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
+
+/// An active job whose `updated_at` hasn't moved for this long is considered
+/// stranded (e.g. its lister exhausted retries into the DLQ) and gets failed
+/// so it can't block future nights. A healthy job's `updated_at` advances
+/// with every dispatched page, and a full run takes minutes, not hours.
+const STALE_ACTIVE_JOB_MAX_AGE_HOURS: i64 = 12;
+
+/// Nightly kickoff, delivered by EventBridge as a static payload: snapshots
+/// the candidate table, creates the job row, and enqueues the first
+/// `ListCandidates` message.
+///
+/// Idempotent under duplicate fires and redeliveries: `create_job` no-ops
+/// against the one-active-job unique index, and an existing job still in
+/// `Init` (a previous kickoff died before enqueueing its lister) is resumed
+/// rather than skipped. An active job that has sat untouched for
+/// [`STALE_ACTIVE_JOB_MAX_AGE_HOURS`] is failed and replaced, so a job
+/// stranded by retry exhaustion can't block cleanup forever.
+#[tracing::instrument(skip(ctx), err)]
+pub async fn start_job(ctx: &CrmCleanupContext) -> Result<(), ProcessingError> {
+ // Freeze the working set: candidates inserted after this point get higher
+ // ids and wait for the next nightly run.
+ let snapshot = candidates::get_max_id_and_count(&ctx.db)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to snapshot crm cleanup candidates"),
+ })
+ })?;
+
+ let Some((max_id, count)) = snapshot else {
+ tracing::info!("No crm cleanup candidates; skipping job creation");
+ return Ok(());
+ };
+
+ let created = job::create_job(&ctx.db, count, max_id).await.map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to create crm cleanup job"),
+ })
+ })?;
+
+ let cleanup_job: CrmCleanupJob = match created {
+ Some(created_job) => created_job,
+ None => {
+ let active = job::get_active_job(&ctx.db)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to fetch active crm cleanup job"),
+ })
+ })?
+ .ok_or_else(|| {
+ // The active job finished between create and fetch; the
+ // next nightly fire picks up any remaining candidates.
+ ProcessingError::NonRetryable(DetailedError {
+ reason: FailureReason::InvalidData,
+ source: anyhow::anyhow!(
+ "Active crm cleanup job disappeared between create and fetch"
+ ),
+ })
+ })?;
+
+ let age = chrono::Utc::now() - active.updated_at;
+ if age > chrono::Duration::hours(STALE_ACTIVE_JOB_MAX_AGE_HOURS) {
+ // Stranded job (its lister likely DLQ'd): fail it and start
+ // fresh so the one-active-job index doesn't block cleanup.
+ tracing::warn!(job_id = %active.id, status = %active.status, age_hours = age.num_hours(), "Active crm cleanup job is stale; failing it and starting a new one");
+ job::set_job_status(&ctx.db, active.id, CrmCleanupJobStatus::Failed)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to fail stale crm cleanup job"),
+ })
+ })?;
+
+ let fresh = job::create_job(&ctx.db, count, max_id).await.map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to create replacement crm cleanup job"),
+ })
+ })?;
+ let Some(fresh) = fresh else {
+ // A concurrent kickoff won the replacement race.
+ tracing::warn!("Another kickoff created the replacement job; skipping");
+ return Ok(());
+ };
+ fresh
+ } else {
+ match active.status {
+ // A previous kickoff created the job but died before
+ // enqueueing its lister — resume it.
+ CrmCleanupJobStatus::Init => active,
+ _ => {
+ tracing::warn!(job_id = %active.id, status = %active.status, "An active crm cleanup job already exists; skipping kickoff");
+ return Ok(());
+ }
+ }
+ }
+ }
+ };
+
+ tracing::info!(job_id = %cleanup_job.id, total_candidates = cleanup_job.total_candidates, max_candidate_id = cleanup_job.max_candidate_id, "Starting crm cleanup job");
+
+ ctx.sqs_client
+ .enqueue_email_crm_cleanup_message(CrmCleanupPubsubMessage {
+ operation: CrmCleanupOperation::ListCandidates {
+ job_id: cleanup_job.id,
+ last_id: 0,
+ },
+ })
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::SqsEnqueueFailed,
+ source: e.context("Failed to enqueue first ListCandidates message"),
+ })
+ })?;
+
+ // Lister handles Init too, so a failure here only costs observability.
+ job::set_job_status(&ctx.db, cleanup_job.id, CrmCleanupJobStatus::InProgress)
+ .await
+ .inspect_err(
+ |e| tracing::error!(error = ?e, job_id = %cleanup_job.id, "Failed to mark job in progress"),
+ )
+ .ok();
+
+ Ok(())
+}
diff --git a/services/email_service/src/pubsub/crm_cleanup/worker.rs b/services/email_service/src/pubsub/crm_cleanup/worker.rs
new file mode 100644
index 0000000000..a8b68c7b64
--- /dev/null
+++ b/services/email_service/src/pubsub/crm_cleanup/worker.rs
@@ -0,0 +1,85 @@
+use crate::pubsub::context::CrmServiceType;
+use crate::pubsub::crm_cleanup::process;
+use futures::StreamExt;
+
+/// Shared dependencies for the CRM cleanup handlers. Much smaller than the
+/// backfill `PubSubContext` — this worker only talks to the DB, SQS, and the
+/// CRM service.
+#[derive(Clone)]
+pub struct CrmCleanupContext {
+ pub db: sqlx::Pool<sqlx::Postgres>,
+ pub sqs_worker: sqs_worker::SQSWorker,
+ pub sqs_client: sqs_client::SQS,
+ pub crm_service: CrmServiceType,
+}
+
+/// method that ingests sqs messages and calls the process function for each
+pub async fn run_worker(
+ db: sqlx::Pool<sqlx::Postgres>,
+ worker: sqs_worker::SQSWorker,
+ sqs_client: sqs_client::SQS,
+ crm_service: CrmServiceType,
+) {
+ let ctx = CrmCleanupContext {
+ db,
+ sqs_worker: worker.clone(),
+ sqs_client,
+ crm_service,
+ };
+
+ loop {
+ let worker_result = tokio::spawn({
+ let ctx = ctx.clone();
+ let worker = worker.clone();
+ async move {
+ loop {
+ match worker.receive_messages().await {
+ Ok(messages) => {
+ if messages.is_empty() {
+ continue;
+ }
+ let results = futures::stream::iter(messages.iter())
+ .then(|message| {
+ let ctx = ctx.clone();
+ async move {
+ process::process_message(ctx, message).await.map_err(|e| {
+ (
+ message.message_id.clone().unwrap_or_default(),
+ e,
+ )
+ })
+ }
+ })
+ .collect::<Vec<Result<(), (String, anyhow::Error)>>>()
+ .await;
+
+ for (message_id, error) in
+ results.into_iter().filter_map(|result| result.err())
+ {
+ tracing::error!(message_id, error=?error, "error processing crm cleanup message");
+ }
+ }
+ Err(e) => {
+ tracing::error!(error=?e, "error receiving crm cleanup messages");
+ }
+ }
+ }
+ }
+ })
+ .await;
+
+ match worker_result {
+ Ok(_) => {
+ // This should never be hit
+ tracing::error!("crm cleanup worker exited successfully?");
+ }
+ Err(e) => {
+ tracing::error!(error=?e, "crm cleanup worker crashed with error");
+ }
+ }
+
+ // Add a delay before restarting to avoid rapid restart loops
+ tracing::info!("CRM CLEANUP WORKER RESTARTING...");
+ tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+ }
+}
diff --git a/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs b/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs
index 5f9603dd64..4f0b07bc05 100644
--- a/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs
+++ b/services/email_service/src/pubsub/inbox_sync/operations/delete_message.rs
@@ -1,6 +1,6 @@
use crate::pubsub::context::PubSubContext;
use crate::pubsub::util::{
- cg_refresh_email, complete_transaction_with_processing_error, enqueue_depopulate_crm_contacts,
+ cg_refresh_email, complete_transaction_with_processing_error, insert_crm_cleanup_candidates,
publish_email_event,
};
use email::domain::events::{EmailMacroEvent, MessageDeletedMetadata};
@@ -80,17 +80,16 @@ pub async fn delete_message(
sender.into_iter().filter_map(|c| c.email_address).collect()
};
- // Enqueue CRM teardown BEFORE the delete commits, so a transient
- // enqueue failure here doesn't strand the depopulate job after the
- // message row is already gone (SQS retry would then short-circuit
- // at the `None` arm above and never re-enqueue). If enqueue
- // succeeds but the delete below fails, the depopulate consumer's
+ // Record CRM cleanup candidates BEFORE the delete commits, so a
+ // transient insert failure here doesn't strand the teardown after the
+ // message row is already gone (SQS retry would then short-circuit at
+ // the `None` arm above and never re-record). If the insert succeeds
+ // but the delete below fails, the nightly cleanup job's
// `link_has_any_message_with` pre-check sees the message still in
- // place and acks without touching CRM — so the ordering is safe in
- // both directions.
+ // place and skips — so the ordering is safe in both directions.
if !crm_emails.is_empty() {
let self_email = link.email_address.0.as_ref().to_ascii_lowercase();
- enqueue_depopulate_crm_contacts(ctx, link.id, &self_email, crm_emails).await?;
+ insert_crm_cleanup_candidates(ctx, link.id, &self_email, crm_emails).await?;
}
let mut tx = ctx.db.begin().await.map_err(|e| {
diff --git a/services/email_service/src/pubsub/mod.rs b/services/email_service/src/pubsub/mod.rs
index 48aab6124c..f43ab8a123 100644
--- a/services/email_service/src/pubsub/mod.rs
+++ b/services/email_service/src/pubsub/mod.rs
@@ -3,6 +3,7 @@ pub(crate) mod context;
/// The flag-selected CRM metadata resolver, re-exported for the
/// pubsub_workers binary to construct.
pub use context::CrmMetadataResolver;
+pub mod crm_cleanup;
pub mod gmail_ops;
pub mod inbox_sync;
pub mod link_manager;
diff --git a/services/email_service/src/pubsub/util.rs b/services/email_service/src/pubsub/util.rs
index d13918929f..79d903b3a2 100644
--- a/services/email_service/src/pubsub/util.rs
+++ b/services/email_service/src/pubsub/util.rs
@@ -12,8 +12,7 @@ use models_email::api::refresh::RefreshEmailEvent;
#[cfg(test)]
mod test;
use models_email::email::service::backfill::{
- BackfillOperation, BackfillPubsubMessage, DepopulateCrmContactPayload, LinkScopedPayload,
- PopulateCrmContactPayload,
+ BackfillOperation, BackfillPubsubMessage, LinkScopedPayload, PopulateCrmContactPayload,
};
use models_email::email::service::pubsub::{DetailedError, FailureReason, ProcessingError};
use models_email::gmail::operations::GmailApiOperation;
@@ -250,38 +249,28 @@ pub async fn enqueue_populate_crm_contacts(
Ok(())
}
-/// Producer-side fan-out helper for tearing CRM contacts down when a sent
-/// message is deleted. Mirrors [`enqueue_populate_crm_contacts`] and uses
-/// the same [`normalized_non_self_contact_emails`] filter.
-///
-/// Used by `delete_message` in the inbox-sync worker. The consumer
-/// (`depopulate_crm_contact`) re-checks whether the link still has any
-/// sent message to the contact before deleting, so duplicate enqueues
-/// from retries are harmless.
-pub async fn enqueue_depopulate_crm_contacts(
+/// Producer-side dedupe for deferred CRM teardown when a message is deleted.
+/// Uses the same [`normalized_non_self_contact_emails`] filter as the populate
+/// helper, then upserts one `crm_cleanup_candidates` row per distinct contact.
+/// The unique `(link_id, contact_email)` index collapses bulk deletes into a
+/// single pending row; the nightly cleanup job sweeps the table and
+/// depopulates contacts whose links no longer have messages with them.
+pub async fn insert_crm_cleanup_candidates(
ctx: &PubSubContext,
link_id: Uuid,
self_email: &str,
contact_emails: impl IntoIterator<Item = String>,
) -> Result<(), ProcessingError> {
- for contact_email in normalized_non_self_contact_emails(self_email, contact_emails) {
- let ps_message = BackfillPubsubMessage {
- backfill_operation: BackfillOperation::DepopulateCrmContact(LinkScopedPayload {
- link_id,
- payload: DepopulateCrmContactPayload { contact_email },
- }),
- };
+ let emails = normalized_non_self_contact_emails(self_email, contact_emails);
- ctx.sqs_client
- .enqueue_email_backfill_message(ps_message)
- .await
- .map_err(|e| {
- ProcessingError::Retryable(DetailedError {
- reason: FailureReason::SqsEnqueueFailed,
- source: e.context("Failed to enqueue DepopulateCrmContact message"),
- })
- })?;
- }
+ email_db_client::crm_cleanup::candidates::insert_candidates(&ctx.db, link_id, &emails)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: e.context("Failed to insert crm cleanup candidates"),
+ })
+ })?;
Ok(())
}
diff --git a/tooling/just/local_stack.just b/tooling/just/local_stack.just
index d80fa3612e..ced7b3bec4 100644
--- a/tooling/just/local_stack.just
+++ b/tooling/just/local_stack.just
@@ -31,6 +31,7 @@ create_local_queues:
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name push-delivery-queue || true
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name webhook-event-queue.fifo --attributes FifoQueue=true || true
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name email-service-backfill-queue || true
+ aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name email-service-crm-cleanup-queue || true
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name delete-chat-handler-queue || true
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name contacts-queue || true
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name convert-service-queue || true