Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

perf(crm): skip the cleanup message scan when CRM has no contact to remove (#7538)

teo509e12e67dc6
diff --git a/crates/crm/src/domain/companies_repo.rs b/crates/crm/src/domain/companies_repo.rs
index ac5d4740dc..a8a27935ef 100644
--- a/crates/crm/src/domain/companies_repo.rs
+++ b/crates/crm/src/domain/companies_repo.rs
@@ -342,6 +342,17 @@ pub trait CompaniesRepository: Clone + Send + Sync + 'static {
email: &str,
) -> impl Future<Output = Result<DepopulateContactOutcome, CrmError>> + Send;
+ /// Whether [`depopulate_contact`] would find a contact to act on for
+ /// `(team_id, domain, email)`: the same lookup, under the same advisory
+ /// lock, with nothing deleted. `false` means `depopulate_contact` is a
+ /// no-op for every link in the team.
+ fn has_depopulate_target(
+ &self,
+ team_id: &uuid::Uuid,
+ domain: &str,
+ email: &str,
+ ) -> impl Future<Output = Result<bool, CrmError>> + Send;
+
/// Filters `(link_id, email)` pairs down to those that currently have a
/// `crm_contact_sources` row — i.e. the pairs where this link actually
/// contributed something [`depopulate_contact`] would tear down.
diff --git a/crates/crm/src/domain/service.rs b/crates/crm/src/domain/service.rs
index b5fefb6027..88a5d1d081 100644
--- a/crates/crm/src/domain/service.rs
+++ b/crates/crm/src/domain/service.rs
@@ -134,6 +134,18 @@ pub trait CrmService: Clone + Send + Sync + 'static {
email: &str,
) -> impl Future<Output = Result<DepopulateContactOutcome, CrmError>> + Send;
+ /// Whether [`depopulate_contact`] would find a contact to act on for
+ /// `email` in `team_id`; `false` means it is a no-op for every link.
+ /// Callers check this before running an expensive gate that only
+ /// matters when there is something to tear down. Malformed emails
+ /// return `false`, matching `depopulate_contact`. See
+ /// [`crate::domain::companies_repo::CompaniesRepository::has_depopulate_target`].
+ fn has_depopulate_target(
+ &self,
+ team_id: &uuid::Uuid,
+ email: &str,
+ ) -> impl Future<Output = Result<bool, CrmError>> + Send;
+
/// Filters `(link_id, email)` pairs down to those with a live
/// `crm_contact_sources` row — the pairs a teardown would actually touch.
/// See
@@ -512,6 +524,25 @@ fn normalize_company_domain(raw: &str) -> Result<String, CrmError> {
Ok(domain)
}
+/// Splits an observed address into `(trimmed email, domain)` for the
+/// depopulate path. `None` (logged) when it has no `@`, an empty part, or
+/// more than one `@`.
+fn split_depopulate_email(email: &str) -> Option<(&str, &str)> {
+ let email = email.trim();
+ let Some((local_part, domain)) = email.split_once('@') else {
+ tracing::debug!(email, "skipping malformed email (no '@')");
+ return None;
+ };
+ if local_part.is_empty() || domain.is_empty() || domain.contains('@') {
+ tracing::debug!(
+ email,
+ "skipping malformed email (empty part or multiple '@')"
+ );
+ return None;
+ }
+ Some((email, domain))
+}
+
/// Implementation of [`CrmService`] backed by a [`CompaniesRepository`]
/// and a [`CompanyMetadataResolver`].
#[derive(Debug)]
@@ -745,26 +776,28 @@ where
link_id: &uuid::Uuid,
email: &str,
) -> Result<DepopulateContactOutcome, CrmError> {
- let email = email.trim();
- let Some((local_part, domain)) = email.split_once('@') else {
- tracing::debug!(
- email,
- "depopulate_contact: skipping malformed email (no '@')"
- );
+ let Some((email, domain)) = split_depopulate_email(email) else {
return Ok(DepopulateContactOutcome::default());
};
- if local_part.is_empty() || domain.is_empty() || domain.contains('@') {
- tracing::debug!(
- email,
- "depopulate_contact: skipping malformed email (empty part or multiple '@')"
- );
- return Ok(DepopulateContactOutcome::default());
- }
self.companies_repository
.depopulate_contact(team_id, link_id, domain, email)
.await
}
+ #[tracing::instrument(skip(self, email), err)]
+ async fn has_depopulate_target(
+ &self,
+ team_id: &uuid::Uuid,
+ email: &str,
+ ) -> Result<bool, CrmError> {
+ let Some((email, domain)) = split_depopulate_email(email) else {
+ return Ok(false);
+ };
+ self.companies_repository
+ .has_depopulate_target(team_id, domain, email)
+ .await
+ }
+
#[tracing::instrument(skip(self, pairs), fields(pair_count = pairs.len()), err)]
async fn link_contact_pairs_with_sources(
&self,
@@ -1182,6 +1215,14 @@ impl CrmService for NoOpCrmService {
unimplemented!("NoOpCrmService.depopulate_contact")
}
+ async fn has_depopulate_target(
+ &self,
+ _team_id: &uuid::Uuid,
+ _email: &str,
+ ) -> Result<bool, CrmError> {
+ unimplemented!("NoOpCrmService.has_depopulate_target")
+ }
+
async fn link_contact_pairs_with_sources(
&self,
_pairs: &[(uuid::Uuid, String)],
diff --git a/crates/crm/src/domain/service/test.rs b/crates/crm/src/domain/service/test.rs
index 661a100344..423d1c145a 100644
--- a/crates/crm/src/domain/service/test.rs
+++ b/crates/crm/src/domain/service/test.rs
@@ -152,6 +152,15 @@ impl CompaniesRepository for StubRepo {
unimplemented!()
}
+ async fn has_depopulate_target(
+ &self,
+ _team_id: &uuid::Uuid,
+ _domain: &str,
+ _email: &str,
+ ) -> Result<bool, CrmError> {
+ unimplemented!()
+ }
+
async fn link_contact_pairs_with_sources(
&self,
_pairs: &[(uuid::Uuid, String)],
@@ -829,6 +838,25 @@ async fn set_company_hidden_requires_admin_role() {
}
}
+#[tokio::test]
+async fn malformed_emails_have_no_depopulate_target() {
+ let team_id = uuid::Uuid::now_v7();
+ for email in [
+ "",
+ "jane",
+ "jane@",
+ "@acme.test",
+ "jane@@acme.test",
+ ] {
+ let has_target = service()
+ .has_depopulate_target(&team_id, email)
+ .await
+ .unwrap();
+ assert!(!has_target, "email {email:?}");
+ }
+}
+
#[tokio::test]
async fn set_contact_hidden_requires_admin_role() {
let err = service()
diff --git a/crates/crm/src/inbound/axum_extractors/test.rs b/crates/crm/src/inbound/axum_extractors/test.rs
index d60e24dd89..55a98fde27 100644
--- a/crates/crm/src/inbound/axum_extractors/test.rs
+++ b/crates/crm/src/inbound/axum_extractors/test.rs
@@ -279,6 +279,10 @@ impl CrmService for FakeCrmService {
panic!("unexpected depopulate_contact call")
}
+ async fn has_depopulate_target(&self, _team_id: &Uuid, _email: &str) -> Result<bool, CrmError> {
+ panic!("unexpected has_depopulate_target call")
+ }
+
async fn link_contact_pairs_with_sources(
&self,
_pairs: &[(Uuid, String)],
diff --git a/crates/crm/src/outbound/companies_repo.rs b/crates/crm/src/outbound/companies_repo.rs
index 29ff8b32fb..6062852256 100644
--- a/crates/crm/src/outbound/companies_repo.rs
+++ b/crates/crm/src/outbound/companies_repo.rs
@@ -20,6 +20,16 @@ use sqlx::PgPool;
use std::collections::BTreeMap;
use uuid::Uuid;
+/// The contact `depopulate_contact` acts on, with the flags that decide
+/// how far its teardown cascades.
+struct DepopulateTarget {
+ contact_id: Uuid,
+ contact_manually_created: bool,
+ company_id: Uuid,
+ email_sync: bool,
+ company_manually_created: bool,
+}
+
/// PostgreSQL-backed [`CompaniesRepository`].
#[derive(Clone)]
pub struct CompaniesRepositoryImpl {
@@ -80,6 +90,43 @@ impl CompaniesRepositoryImpl {
Ok(())
}
+ /// Takes the domain lock, then finds the contact `depopulate_contact`
+ /// tears down. Locking before the lookup keeps a concurrent in-flight
+ /// populate from committing past it. `None` = nothing to tear down.
+ async fn lock_depopulate_target(
+ tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+ team_id: &uuid::Uuid,
+ normalized_domain: &str,
+ normalized_email: &str,
+ ) -> Result<Option<DepopulateTarget>, CrmError> {
+ Self::lock_team_domain(tx, team_id, normalized_domain).await?;
+
+ sqlx::query_as!(
+ DepopulateTarget,
+ r#"
+ SELECT
+ ct.id AS contact_id,
+ ct.manually_created AS "contact_manually_created!",
+ co.id AS company_id,
+ co.email_sync AS "email_sync!",
+ co.manually_created AS "company_manually_created!"
+ FROM crm_contacts ct
+ JOIN crm_companies co ON co.id = ct.company_id
+ JOIN crm_domains d ON d.company_id = co.id
+ WHERE co.team_id = $1
+ AND LOWER(ct.email) = $2
+ AND LOWER(d.domain) = $3
+ LIMIT 1
+ "#,
+ team_id,
+ normalized_email,
+ normalized_domain,
+ )
+ .fetch_optional(&mut **tx)
+ .await
+ .map_err(|e| CrmError::StorageLayerError(e.into()))
+ }
+
/// Team killswitch, read inside the tx (after the advisory lock) so
/// a concurrent disable+purge can't race past us. Missing row =
/// default false.
@@ -712,40 +759,9 @@ impl CompaniesRepository for CompaniesRepositoryImpl {
.await
.map_err(|e| CrmError::StorageLayerError(e.into()))?;
- // Lock BEFORE observing state: a concurrent populate could
- // commit rows for a since-deleted sent message otherwise.
- sqlx::query!(
- r#"SELECT pg_advisory_xact_lock(hashtextextended($1, 0))"#,
- format!("{team_id}:{normalized_domain}"),
- )
- .execute(&mut *tx)
- .await
- .map_err(|e| CrmError::StorageLayerError(e.into()))?;
-
- // None here = nothing to tear down.
- let Some(row) = sqlx::query!(
- r#"
- SELECT
- ct.id AS contact_id,
- ct.manually_created AS "contact_manually_created!",
- co.id AS company_id,
- co.email_sync AS "email_sync!",
- co.manually_created AS "company_manually_created!"
- FROM crm_contacts ct
- JOIN crm_companies co ON co.id = ct.company_id
- JOIN crm_domains d ON d.company_id = co.id
- WHERE co.team_id = $1
- AND LOWER(ct.email) = $2
- AND LOWER(d.domain) = $3
- LIMIT 1
- "#,
- team_id,
- normalized_email,
- normalized_domain,
- )
- .fetch_optional(&mut *tx)
- .await
- .map_err(|e| CrmError::StorageLayerError(e.into()))?
+ let Some(row) =
+ Self::lock_depopulate_target(&mut tx, team_id, &normalized_domain, &normalized_email)
+ .await?
else {
tx.commit()
.await
@@ -850,6 +866,34 @@ impl CompaniesRepository for CompaniesRepositoryImpl {
Ok(outcome)
}
+ #[tracing::instrument(skip(self, email), err)]
+ async fn has_depopulate_target(
+ &self,
+ team_id: &uuid::Uuid,
+ domain: &str,
+ email: &str,
+ ) -> Result<bool, CrmError> {
+ let mut tx = self
+ .pool
+ .begin()
+ .await
+ .map_err(|e| CrmError::StorageLayerError(e.into()))?;
+
+ let target = Self::lock_depopulate_target(
+ &mut tx,
+ team_id,
+ &domain.to_ascii_lowercase(),
+ &email.to_ascii_lowercase(),
+ )
+ .await?;
+
+ tx.commit()
+ .await
+ .map_err(|e| CrmError::StorageLayerError(e.into()))?;
+
+ Ok(target.is_some())
+ }
+
#[tracing::instrument(skip(self, pairs), fields(pair_count = pairs.len()), err)]
async fn link_contact_pairs_with_sources(
&self,
diff --git a/crates/crm/src/outbound/companies_repo/test.rs b/crates/crm/src/outbound/companies_repo/test.rs
index 502bc39b5c..d6898c06b7 100644
--- a/crates/crm/src/outbound/companies_repo/test.rs
+++ b/crates/crm/src/outbound/companies_repo/test.rs
@@ -12,6 +12,7 @@ mod domain_directory;
mod get_company_for_team;
mod get_contact_by_email_for_team;
mod get_contact_for_team;
+mod has_depopulate_target;
mod link_contact_pairs_with_sources;
mod list_companies_for_soup;
mod list_contacts_for_company;
diff --git a/crates/crm/src/outbound/companies_repo/test/has_depopulate_target.rs b/crates/crm/src/outbound/companies_repo/test/has_depopulate_target.rs
new file mode 100644
index 0000000000..416632276d
--- /dev/null
+++ b/crates/crm/src/outbound/companies_repo/test/has_depopulate_target.rs
@@ -0,0 +1,141 @@
+use super::helpers::*;
+use crate::domain::companies_repo::*;
+use crate::outbound::companies_repo::*;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::PgPool;
+use std::time::Duration;
+use uuid::Uuid;
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn true_only_for_a_contact_on_a_tracked_domain(pool: PgPool) -> anyhow::Result<()> {
+ let team_id = Uuid::now_v7();
+ let owner_id = "macro|[email protected]";
+ seed_team(&pool, team_id, owner_id).await?;
+ let other_team_id = Uuid::now_v7();
+ seed_team(&pool, other_team_id, "macro|[email protected]").await?;
+ let link_id = insert_email_link(&pool, owner_id, "[email protected]").await?;
+ let company_id = insert_company(&pool, team_id, true, &["acme.test"]).await?;
+ insert_contact_with_source(&pool, company_id, "[email protected]", link_id).await?;
+
+ let repo = CompaniesRepositoryImpl::new(pool.clone());
+ assert!(
+ repo.has_depopulate_target(&team_id, "acme.test", "[email protected]")
+ .await?
+ );
+ assert!(
+ repo.has_depopulate_target(&team_id, "ACME.test", "[email protected]")
+ .await?
+ );
+ // Never in CRM: the common case for a fresh cleanup candidate.
+ assert!(
+ !repo
+ .has_depopulate_target(&team_id, "acme.test", "[email protected]")
+ .await?
+ );
+ assert!(
+ !repo
+ .has_depopulate_target(&team_id, "example.com", "[email protected]")
+ .await?
+ );
+ assert!(
+ !repo
+ .has_depopulate_target(&other_team_id, "acme.test", "[email protected]")
+ .await?
+ );
+ Ok(())
+}
+
+/// A contact no link sources is still a target: `depopulate_contact`
+/// deletes it as an orphan even when the calling link never contributed it.
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn agrees_with_depopulate_contact(pool: PgPool) -> anyhow::Result<()> {
+ let team_id = Uuid::now_v7();
+ let owner_id = "macro|[email protected]";
+ seed_team(&pool, team_id, owner_id).await?;
+ let link_id = insert_email_link(&pool, owner_id, "[email protected]").await?;
+ let company_id = insert_company(&pool, team_id, true, &["acme.test"]).await?;
+ insert_contact(&pool, company_id, "[email protected]").await?;
+
+ let repo = CompaniesRepositoryImpl::new(pool.clone());
+ assert!(
+ repo.has_depopulate_target(&team_id, "acme.test", "[email protected]")
+ .await?
+ );
+ let outcome = repo
+ .depopulate_contact(&team_id, &link_id, "acme.test", "[email protected]")
+ .await?;
+ assert!(outcome.contact_deleted);
+ assert!(outcome.company_deleted);
+
+ assert!(
+ !repo
+ .has_depopulate_target(&team_id, "acme.test", "[email protected]")
+ .await?
+ );
+ let outcome = repo
+ .depopulate_contact(&team_id, &link_id, "acme.test", "[email protected]")
+ .await?;
+ assert!(!outcome.removed_anything());
+ Ok(())
+}
+
+/// Without the domain lock the check would read past a populate's
+/// uncommitted contact and report `false`, where `depopulate_contact`
+/// would wait for the commit and tear the contact down.
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn waits_for_an_in_flight_populate(pool: PgPool) -> anyhow::Result<()> {
+ let team_id = Uuid::now_v7();
+ seed_team(&pool, team_id, "macro|[email protected]").await?;
+ let company_id = insert_company(&pool, team_id, true, &["acme.test"]).await?;
+
+ let mut populate = pool.begin().await?;
+ let populate_pid = sqlx::query_scalar!(r#"SELECT pg_backend_pid() AS "pid!""#)
+ .fetch_one(&mut *populate)
+ .await?;
+ sqlx::query!(
+ r#"SELECT pg_advisory_xact_lock(hashtextextended($1, 0))"#,
+ format!("{team_id}:acme.test"),
+ )
+ .execute(&mut *populate)
+ .await?;
+ sqlx::query!(
+ r#"INSERT INTO crm_contacts (company_id, email, first_interaction, last_interaction)
+ VALUES ($1, '[email protected]', now(), now())"#,
+ company_id,
+ )
+ .execute(&mut *populate)
+ .await?;
+
+ let repo = CompaniesRepositoryImpl::new(pool.clone());
+ let check = tokio::spawn(async move {
+ repo.has_depopulate_target(&team_id, "acme.test", "[email protected]")
+ .await
+ });
+
+ tokio::time::timeout(Duration::from_secs(10), async {
+ loop {
+ let waiting = sqlx::query_scalar!(
+ r#"SELECT EXISTS (
+ SELECT 1 FROM pg_stat_activity
+ WHERE datname = current_database() AND $1 = ANY(pg_blocking_pids(pid))
+ ) AS "waiting!""#,
+ populate_pid,
+ )
+ .fetch_one(&pool)
+ .await?;
+ if waiting {
+ return anyhow::Ok(());
+ }
+ assert!(
+ !check.is_finished(),
+ "check did not wait for the domain lock"
+ );
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await??;
+
+ populate.commit().await?;
+ assert!(check.await??);
+ Ok(())
+}
diff --git a/crates/macro_db_client/migrations/20261101055617_crm_contacts_lower_email_idx.sql b/crates/macro_db_client/migrations/20261101055617_crm_contacts_lower_email_idx.sql
new file mode 100644
index 0000000000..8231e3eb60
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261101055617_crm_contacts_lower_email_idx.sql
@@ -0,0 +1,11 @@
+-- no-transaction
+-- The CRM depopulate lookup (crates/crm, `lock_depopulate_target`) and the
+-- cleanup lister's `link_contact_pairs_with_sources` match LOWER(ct.email).
+-- Without this index the lookup walks the team's companies and the lister
+-- walks every source of each candidate link.
+-- Keep this a single statement: SQLx batches would create an implicit
+-- transaction, which CONCURRENTLY forbids. No IF NOT EXISTS: if an
+-- interrupted build leaves an INVALID index, the next run fails on it instead
+-- of recording success; drop it with DROP INDEX CONCURRENTLY and re-run.
+CREATE INDEX CONCURRENTLY crm_contacts_lower_email_idx
+ ON crm_contacts (LOWER(email));
diff --git a/crates/soup/src/domain/service/tests/crm_association.rs b/crates/soup/src/domain/service/tests/crm_association.rs
index 1a1de961d8..78784545c9 100644
--- a/crates/soup/src/domain/service/tests/crm_association.rs
+++ b/crates/soup/src/domain/service/tests/crm_association.rs
@@ -80,6 +80,14 @@ impl CrmService for FakeCrm {
unimplemented!("FakeCrm.depopulate_contact")
}
+ async fn has_depopulate_target(
+ &self,
+ _team_id: &uuid::Uuid,
+ _email: &str,
+ ) -> Result<bool, CrmError> {
+ unimplemented!("FakeCrm.has_depopulate_target")
+ }
+
async fn link_contact_pairs_with_sources(
&self,
_pairs: &[(uuid::Uuid, String)],
diff --git a/services/email_service/src/pubsub/crm_cleanup/process_candidate.rs b/services/email_service/src/pubsub/crm_cleanup/process_candidate.rs
index 682ca8ab5e..7f71a3d56d 100644
--- a/services/email_service/src/pubsub/crm_cleanup/process_candidate.rs
+++ b/services/email_service/src/pubsub/crm_cleanup/process_candidate.rs
@@ -6,7 +6,9 @@ use uuid::Uuid;
/// Processes one cleanup candidate: claims (deletes) the candidate row, then
/// depopulates the CRM contact if the link no longer has any message (sent or
-/// received) involving `contact_email`.
+/// received) involving `contact_email`. The message scan only runs when CRM
+/// has a contact for `contact_email` to act on; most candidates are addresses
+/// CRM never tracked.
///
/// The claim happens *before* the gate check so that a message delete landing
/// mid-processing re-inserts a fresh candidate for the next nightly run
@@ -68,6 +70,25 @@ pub async fn process_candidate(
return Ok(());
};
+ // Without a target `depopulate_contact` is a no-op, so the message scan
+ // below could not change the outcome.
+ let has_target = ctx
+ .crm_service
+ .has_depopulate_target(&team_id, contact_email)
+ .await
+ .map_err(|e| {
+ ProcessingError::Retryable(DetailedError {
+ reason: FailureReason::DatabaseQueryFailed,
+ source: anyhow::Error::from(e)
+ .context("Failed to check for a CRM contact to tear down"),
+ })
+ })?;
+
+ if !has_target {
+ tracing::debug!("No CRM contact to tear down; skipping message scan");
+ return Ok(());
+ }
+
let still_has_any =
email_db_client::contacts::get::link_has_any_message_with(&ctx.db, link.id, contact_email)
.await