Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(crm): skip the cleanup message scan when CRM has no contact to remove (#7538)
teo509e12e67dc6
diff --git a/crates/crm/src/domain/companies_repo.rs b/crates/crm/src/domain/companies_repo.rsindex ac5d4740dc..a8a27935ef 100644--- a/crates/crm/src/domain/companies_repo.rs+++ b/crates/crm/src/domain/companies_repo.rs@@ -342,6 +342,17 @@ pub trait CompaniesRepository: Clone + Send + Sync + 'static {email: &str,) -> impl Future<Output = Result<DepopulateContactOutcome, CrmError>> + Send;+ /// Whether [`depopulate_contact`] would find a contact to act on for+ /// `(team_id, domain, email)`: the same lookup, under the same advisory+ /// lock, with nothing deleted. `false` means `depopulate_contact` is a+ /// no-op for every link in the team.+ fn has_depopulate_target(+ &self,+ team_id: &uuid::Uuid,+ domain: &str,+ email: &str,+ ) -> impl Future<Output = Result<bool, CrmError>> + Send;+/// Filters `(link_id, email)` pairs down to those that currently have a/// `crm_contact_sources` row — i.e. the pairs where this link actually/// contributed something [`depopulate_contact`] would tear down.diff --git a/crates/crm/src/domain/service.rs b/crates/crm/src/domain/service.rsindex b5fefb6027..88a5d1d081 100644--- a/crates/crm/src/domain/service.rs+++ b/crates/crm/src/domain/service.rs@@ -134,6 +134,18 @@ pub trait CrmService: Clone + Send + Sync + 'static {email: &str,) -> impl Future<Output = Result<DepopulateContactOutcome, CrmError>> + Send;+ /// Whether [`depopulate_contact`] would find a contact to act on for+ /// `email` in `team_id`; `false` means it is a no-op for every link.+ /// Callers check this before running an expensive gate that only+ /// matters when there is something to tear down. Malformed emails+ /// return `false`, matching `depopulate_contact`. See+ /// [`crate::domain::companies_repo::CompaniesRepository::has_depopulate_target`].+ fn has_depopulate_target(+ &self,+ team_id: &uuid::Uuid,+ email: &str,+ ) -> impl Future<Output = Result<bool, CrmError>> + Send;+/// Filters `(link_id, email)` pairs down to those with a live/// `crm_contact_sources` row — the pairs a teardown would actually touch./// See@@ -512,6 +524,25 @@ fn normalize_company_domain(raw: &str) -> Result<String, CrmError> {Ok(domain)}+/// Splits an observed address into `(trimmed email, domain)` for the+/// depopulate path. `None` (logged) when it has no `@`, an empty part, or+/// more than one `@`.+fn split_depopulate_email(email: &str) -> Option<(&str, &str)> {+ let email = email.trim();+ let Some((local_part, domain)) = email.split_once('@') else {+ tracing::debug!(email, "skipping malformed email (no '@')");+ return None;+ };+ if local_part.is_empty() || domain.is_empty() || domain.contains('@') {+ tracing::debug!(+ email,+ "skipping malformed email (empty part or multiple '@')"+ );+ return None;+ }+ Some((email, domain))+}+/// Implementation of [`CrmService`] backed by a [`CompaniesRepository`]/// and a [`CompanyMetadataResolver`].#[derive(Debug)]@@ -745,26 +776,28 @@ wherelink_id: &uuid::Uuid,email: &str,) -> Result<DepopulateContactOutcome, CrmError> {- let email = email.trim();- let Some((local_part, domain)) = email.split_once('@') else {- tracing::debug!(- email,- "depopulate_contact: skipping malformed email (no '@')"- );+ let Some((email, domain)) = split_depopulate_email(email) else {return Ok(DepopulateContactOutcome::default());};- if local_part.is_empty() || domain.is_empty() || domain.contains('@') {- tracing::debug!(- email,- "depopulate_contact: skipping malformed email (empty part or multiple '@')"- );- return Ok(DepopulateContactOutcome::default());- }self.companies_repository.depopulate_contact(team_id, link_id, domain, email).await}+ #[tracing::instrument(skip(self, email), err)]+ async fn has_depopulate_target(+ &self,+ team_id: &uuid::Uuid,+ email: &str,+ ) -> Result<bool, CrmError> {+ let Some((email, domain)) = split_depopulate_email(email) else {+ return Ok(false);+ };+ self.companies_repository+ .has_depopulate_target(team_id, domain, email)+ .await+ }+#[tracing::instrument(skip(self, pairs), fields(pair_count = pairs.len()), err)]async fn link_contact_pairs_with_sources(&self,@@ -1182,6 +1215,14 @@ impl CrmService for NoOpCrmService {unimplemented!("NoOpCrmService.depopulate_contact")}+ async fn has_depopulate_target(+ &self,+ _team_id: &uuid::Uuid,+ _email: &str,+ ) -> Result<bool, CrmError> {+ unimplemented!("NoOpCrmService.has_depopulate_target")+ }+async fn link_contact_pairs_with_sources(&self,_pairs: &[(uuid::Uuid, String)],diff --git a/crates/crm/src/domain/service/test.rs b/crates/crm/src/domain/service/test.rsindex 661a100344..423d1c145a 100644--- a/crates/crm/src/domain/service/test.rs+++ b/crates/crm/src/domain/service/test.rs@@ -152,6 +152,15 @@ impl CompaniesRepository for StubRepo {unimplemented!()}+ async fn has_depopulate_target(+ &self,+ _team_id: &uuid::Uuid,+ _domain: &str,+ _email: &str,+ ) -> Result<bool, CrmError> {+ unimplemented!()+ }+async fn link_contact_pairs_with_sources(&self,_pairs: &[(uuid::Uuid, String)],@@ -829,6 +838,25 @@ async fn set_company_hidden_requires_admin_role() {}}+#[tokio::test]+async fn malformed_emails_have_no_depopulate_target() {+ let team_id = uuid::Uuid::now_v7();+ for email in [+ "",+ "jane",+ "jane@",+ "@acme.test",+ "jane@@acme.test",+ "a@[email protected]",+ ] {+ let has_target = service()+ .has_depopulate_target(&team_id, email)+ .await+ .unwrap();+ assert!(!has_target, "email {email:?}");+ }+}+#[tokio::test]async fn set_contact_hidden_requires_admin_role() {let err = service()diff --git a/crates/crm/src/inbound/axum_extractors/test.rs b/crates/crm/src/inbound/axum_extractors/test.rsindex d60e24dd89..55a98fde27 100644--- a/crates/crm/src/inbound/axum_extractors/test.rs+++ b/crates/crm/src/inbound/axum_extractors/test.rs@@ -279,6 +279,10 @@ impl CrmService for FakeCrmService {panic!("unexpected depopulate_contact call")}+ async fn has_depopulate_target(&self, _team_id: &Uuid, _email: &str) -> Result<bool, CrmError> {+ panic!("unexpected has_depopulate_target call")+ }+async fn link_contact_pairs_with_sources(&self,_pairs: &[(Uuid, String)],diff --git a/crates/crm/src/outbound/companies_repo.rs b/crates/crm/src/outbound/companies_repo.rsindex 29ff8b32fb..6062852256 100644--- a/crates/crm/src/outbound/companies_repo.rs+++ b/crates/crm/src/outbound/companies_repo.rs@@ -20,6 +20,16 @@ use sqlx::PgPool;use std::collections::BTreeMap;use uuid::Uuid;+/// The contact `depopulate_contact` acts on, with the flags that decide+/// how far its teardown cascades.+struct DepopulateTarget {+ contact_id: Uuid,+ contact_manually_created: bool,+ company_id: Uuid,+ email_sync: bool,+ company_manually_created: bool,+}+/// PostgreSQL-backed [`CompaniesRepository`].#[derive(Clone)]pub struct CompaniesRepositoryImpl {@@ -80,6 +90,43 @@ impl CompaniesRepositoryImpl {Ok(())}+ /// Takes the domain lock, then finds the contact `depopulate_contact`+ /// tears down. Locking before the lookup keeps a concurrent in-flight+ /// populate from committing past it. `None` = nothing to tear down.+ async fn lock_depopulate_target(+ tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,+ team_id: &uuid::Uuid,+ normalized_domain: &str,+ normalized_email: &str,+ ) -> Result<Option<DepopulateTarget>, CrmError> {+ Self::lock_team_domain(tx, team_id, normalized_domain).await?;++ sqlx::query_as!(+ DepopulateTarget,+ r#"+ SELECT+ ct.id AS contact_id,+ ct.manually_created AS "contact_manually_created!",+ co.id AS company_id,+ co.email_sync AS "email_sync!",+ co.manually_created AS "company_manually_created!"+ FROM crm_contacts ct+ JOIN crm_companies co ON co.id = ct.company_id+ JOIN crm_domains d ON d.company_id = co.id+ WHERE co.team_id = $1+ AND LOWER(ct.email) = $2+ AND LOWER(d.domain) = $3+ LIMIT 1+ "#,+ team_id,+ normalized_email,+ normalized_domain,+ )+ .fetch_optional(&mut **tx)+ .await+ .map_err(|e| CrmError::StorageLayerError(e.into()))+ }+/// Team killswitch, read inside the tx (after the advisory lock) so/// a concurrent disable+purge can't race past us. Missing row =/// default false.@@ -712,40 +759,9 @@ impl CompaniesRepository for CompaniesRepositoryImpl {.await.map_err(|e| CrmError::StorageLayerError(e.into()))?;- // Lock BEFORE observing state: a concurrent populate could- // commit rows for a since-deleted sent message otherwise.- sqlx::query!(- r#"SELECT pg_advisory_xact_lock(hashtextextended($1, 0))"#,- format!("{team_id}:{normalized_domain}"),- )- .execute(&mut *tx)- .await- .map_err(|e| CrmError::StorageLayerError(e.into()))?;-- // None here = nothing to tear down.- let Some(row) = sqlx::query!(- r#"- SELECT- ct.id AS contact_id,- ct.manually_created AS "contact_manually_created!",- co.id AS company_id,- co.email_sync AS "email_sync!",- co.manually_created AS "company_manually_created!"- FROM crm_contacts ct- JOIN crm_companies co ON co.id = ct.company_id- JOIN crm_domains d ON d.company_id = co.id- WHERE co.team_id = $1- AND LOWER(ct.email) = $2- AND LOWER(d.domain) = $3- LIMIT 1- "#,- team_id,- normalized_email,- normalized_domain,- )- .fetch_optional(&mut *tx)- .await- .map_err(|e| CrmError::StorageLayerError(e.into()))?+ let Some(row) =+ Self::lock_depopulate_target(&mut tx, team_id, &normalized_domain, &normalized_email)+ .await?else {tx.commit().await@@ -850,6 +866,34 @@ impl CompaniesRepository for CompaniesRepositoryImpl {Ok(outcome)}+ #[tracing::instrument(skip(self, email), err)]+ async fn has_depopulate_target(+ &self,+ team_id: &uuid::Uuid,+ domain: &str,+ email: &str,+ ) -> Result<bool, CrmError> {+ let mut tx = self+ .pool+ .begin()+ .await+ .map_err(|e| CrmError::StorageLayerError(e.into()))?;++ let target = Self::lock_depopulate_target(+ &mut tx,+ team_id,+ &domain.to_ascii_lowercase(),+ &email.to_ascii_lowercase(),+ )+ .await?;++ tx.commit()+ .await+ .map_err(|e| CrmError::StorageLayerError(e.into()))?;++ Ok(target.is_some())+ }+#[tracing::instrument(skip(self, pairs), fields(pair_count = pairs.len()), err)]async fn link_contact_pairs_with_sources(&self,diff --git a/crates/crm/src/outbound/companies_repo/test.rs b/crates/crm/src/outbound/companies_repo/test.rsindex 502bc39b5c..d6898c06b7 100644--- a/crates/crm/src/outbound/companies_repo/test.rs+++ b/crates/crm/src/outbound/companies_repo/test.rs@@ -12,6 +12,7 @@ mod domain_directory;mod get_company_for_team;mod get_contact_by_email_for_team;mod get_contact_for_team;+mod has_depopulate_target;mod link_contact_pairs_with_sources;mod list_companies_for_soup;mod list_contacts_for_company;diff --git a/crates/crm/src/outbound/companies_repo/test/has_depopulate_target.rs b/crates/crm/src/outbound/companies_repo/test/has_depopulate_target.rsnew file mode 100644index 0000000000..416632276d--- /dev/null+++ b/crates/crm/src/outbound/companies_repo/test/has_depopulate_target.rs@@ -0,0 +1,141 @@+use super::helpers::*;+use crate::domain::companies_repo::*;+use crate::outbound::companies_repo::*;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::PgPool;+use std::time::Duration;+use uuid::Uuid;++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn true_only_for_a_contact_on_a_tracked_domain(pool: PgPool) -> anyhow::Result<()> {+ let team_id = Uuid::now_v7();+ let owner_id = "macro|[email protected]";+ seed_team(&pool, team_id, owner_id).await?;+ let other_team_id = Uuid::now_v7();+ seed_team(&pool, other_team_id, "macro|[email protected]").await?;+ let link_id = insert_email_link(&pool, owner_id, "[email protected]").await?;+ let company_id = insert_company(&pool, team_id, true, &["acme.test"]).await?;+ insert_contact_with_source(&pool, company_id, "[email protected]", link_id).await?;++ let repo = CompaniesRepositoryImpl::new(pool.clone());+ assert!(+ repo.has_depopulate_target(&team_id, "acme.test", "[email protected]")+ .await?+ );+ assert!(+ repo.has_depopulate_target(&team_id, "ACME.test", "[email protected]")+ .await?+ );+ // Never in CRM: the common case for a fresh cleanup candidate.+ assert!(+ !repo+ .has_depopulate_target(&team_id, "acme.test", "[email protected]")+ .await?+ );+ assert!(+ !repo+ .has_depopulate_target(&team_id, "example.com", "[email protected]")+ .await?+ );+ assert!(+ !repo+ .has_depopulate_target(&other_team_id, "acme.test", "[email protected]")+ .await?+ );+ Ok(())+}++/// A contact no link sources is still a target: `depopulate_contact`+/// deletes it as an orphan even when the calling link never contributed it.+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn agrees_with_depopulate_contact(pool: PgPool) -> anyhow::Result<()> {+ let team_id = Uuid::now_v7();+ let owner_id = "macro|[email protected]";+ seed_team(&pool, team_id, owner_id).await?;+ let link_id = insert_email_link(&pool, owner_id, "[email protected]").await?;+ let company_id = insert_company(&pool, team_id, true, &["acme.test"]).await?;+ insert_contact(&pool, company_id, "[email protected]").await?;++ let repo = CompaniesRepositoryImpl::new(pool.clone());+ assert!(+ repo.has_depopulate_target(&team_id, "acme.test", "[email protected]")+ .await?+ );+ let outcome = repo+ .depopulate_contact(&team_id, &link_id, "acme.test", "[email protected]")+ .await?;+ assert!(outcome.contact_deleted);+ assert!(outcome.company_deleted);++ assert!(+ !repo+ .has_depopulate_target(&team_id, "acme.test", "[email protected]")+ .await?+ );+ let outcome = repo+ .depopulate_contact(&team_id, &link_id, "acme.test", "[email protected]")+ .await?;+ assert!(!outcome.removed_anything());+ Ok(())+}++/// Without the domain lock the check would read past a populate's+/// uncommitted contact and report `false`, where `depopulate_contact`+/// would wait for the commit and tear the contact down.+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn waits_for_an_in_flight_populate(pool: PgPool) -> anyhow::Result<()> {+ let team_id = Uuid::now_v7();+ seed_team(&pool, team_id, "macro|[email protected]").await?;+ let company_id = insert_company(&pool, team_id, true, &["acme.test"]).await?;++ let mut populate = pool.begin().await?;+ let populate_pid = sqlx::query_scalar!(r#"SELECT pg_backend_pid() AS "pid!""#)+ .fetch_one(&mut *populate)+ .await?;+ sqlx::query!(+ r#"SELECT pg_advisory_xact_lock(hashtextextended($1, 0))"#,+ format!("{team_id}:acme.test"),+ )+ .execute(&mut *populate)+ .await?;+ sqlx::query!(+ r#"INSERT INTO crm_contacts (company_id, email, first_interaction, last_interaction)+ VALUES ($1, '[email protected]', now(), now())"#,+ company_id,+ )+ .execute(&mut *populate)+ .await?;++ let repo = CompaniesRepositoryImpl::new(pool.clone());+ let check = tokio::spawn(async move {+ repo.has_depopulate_target(&team_id, "acme.test", "[email protected]")+ .await+ });++ tokio::time::timeout(Duration::from_secs(10), async {+ loop {+ let waiting = sqlx::query_scalar!(+ r#"SELECT EXISTS (+ SELECT 1 FROM pg_stat_activity+ WHERE datname = current_database() AND $1 = ANY(pg_blocking_pids(pid))+ ) AS "waiting!""#,+ populate_pid,+ )+ .fetch_one(&pool)+ .await?;+ if waiting {+ return anyhow::Ok(());+ }+ assert!(+ !check.is_finished(),+ "check did not wait for the domain lock"+ );+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await??;++ populate.commit().await?;+ assert!(check.await??);+ Ok(())+}diff --git a/crates/macro_db_client/migrations/20261101055617_crm_contacts_lower_email_idx.sql b/crates/macro_db_client/migrations/20261101055617_crm_contacts_lower_email_idx.sqlnew file mode 100644index 0000000000..8231e3eb60--- /dev/null+++ b/crates/macro_db_client/migrations/20261101055617_crm_contacts_lower_email_idx.sql@@ -0,0 +1,11 @@+-- no-transaction+-- The CRM depopulate lookup (crates/crm, `lock_depopulate_target`) and the+-- cleanup lister's `link_contact_pairs_with_sources` match LOWER(ct.email).+-- Without this index the lookup walks the team's companies and the lister+-- walks every source of each candidate link.+-- Keep this a single statement: SQLx batches would create an implicit+-- transaction, which CONCURRENTLY forbids. No IF NOT EXISTS: if an+-- interrupted build leaves an INVALID index, the next run fails on it instead+-- of recording success; drop it with DROP INDEX CONCURRENTLY and re-run.+CREATE INDEX CONCURRENTLY crm_contacts_lower_email_idx+ ON crm_contacts (LOWER(email));diff --git a/crates/soup/src/domain/service/tests/crm_association.rs b/crates/soup/src/domain/service/tests/crm_association.rsindex 1a1de961d8..78784545c9 100644--- a/crates/soup/src/domain/service/tests/crm_association.rs+++ b/crates/soup/src/domain/service/tests/crm_association.rs@@ -80,6 +80,14 @@ impl CrmService for FakeCrm {unimplemented!("FakeCrm.depopulate_contact")}+ async fn has_depopulate_target(+ &self,+ _team_id: &uuid::Uuid,+ _email: &str,+ ) -> Result<bool, CrmError> {+ unimplemented!("FakeCrm.has_depopulate_target")+ }+async fn link_contact_pairs_with_sources(&self,_pairs: &[(uuid::Uuid, String)],diff --git a/services/email_service/src/pubsub/crm_cleanup/process_candidate.rs b/services/email_service/src/pubsub/crm_cleanup/process_candidate.rsindex 682ca8ab5e..7f71a3d56d 100644--- a/services/email_service/src/pubsub/crm_cleanup/process_candidate.rs+++ b/services/email_service/src/pubsub/crm_cleanup/process_candidate.rs@@ -6,7 +6,9 @@ use uuid::Uuid;/// Processes one cleanup candidate: claims (deletes) the candidate row, then/// depopulates the CRM contact if the link no longer has any message (sent or-/// received) involving `contact_email`.+/// received) involving `contact_email`. The message scan only runs when CRM+/// has a contact for `contact_email` to act on; most candidates are addresses+/// CRM never tracked.////// The claim happens *before* the gate check so that a message delete landing/// mid-processing re-inserts a fresh candidate for the next nightly run@@ -68,6 +70,25 @@ pub async fn process_candidate(return Ok(());};+ // Without a target `depopulate_contact` is a no-op, so the message scan+ // below could not change the outcome.+ let has_target = ctx+ .crm_service+ .has_depopulate_target(&team_id, contact_email)+ .await+ .map_err(|e| {+ ProcessingError::Retryable(DetailedError {+ reason: FailureReason::DatabaseQueryFailed,+ source: anyhow::Error::from(e)+ .context("Failed to check for a CRM contact to tear down"),+ })+ })?;++ if !has_target {+ tracing::debug!("No CRM contact to tear down; skipping message scan");+ return Ok(());+ }+let still_has_any =email_db_client::contacts::get::link_has_any_message_with(&ctx.db, link.id, contact_email).await