Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(crm): keep unresolved company domains pending and enrich them later (#8449)
gbirmandcd643778eef
diff --git a/.sqlx/query-334c6ec59d818a2c7251d13e1b5b04a943f677e749735806a7e137743c5afc7c.json b/.sqlx/query-334c6ec59d818a2c7251d13e1b5b04a943f677e749735806a7e137743c5afc7c.jsonnew file mode 100644index 0000000000..6decab7d71--- /dev/null+++ b/.sqlx/query-334c6ec59d818a2c7251d13e1b5b04a943f677e749735806a7e137743c5afc7c.json@@ -0,0 +1,43 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO crm_domain_directory (\n domain, name, description, icon_url,\n apollo_organization_id, website_url, linkedin_url, twitter_url,\n facebook_url, industry, keywords, technologies,\n estimated_num_employees, annual_revenue, annual_revenue_printed,\n total_funding, total_funding_printed, latest_funding_stage,\n latest_funding_round_date, founded_year, publicly_traded_symbol,\n publicly_traded_exchange, phone, raw_address, street_address,\n city, state, postal_code, country, raw, enriched_at\n )\n VALUES (\n $1, $2, $3, $4,\n $5, $6, $7, $8,\n $9, $10, $11, $12,\n $13, $14, $15,\n $16, $17, $18,\n $19, $20, $21,\n $22, $23, $24, $25,\n $26, $27, $28, $29, $30, now()\n )\n ON CONFLICT (LOWER(domain)) DO UPDATE SET\n name = EXCLUDED.name,\n description = EXCLUDED.description,\n icon_url = EXCLUDED.icon_url,\n apollo_organization_id = EXCLUDED.apollo_organization_id,\n website_url = EXCLUDED.website_url,\n linkedin_url = EXCLUDED.linkedin_url,\n twitter_url = EXCLUDED.twitter_url,\n facebook_url = EXCLUDED.facebook_url,\n industry = EXCLUDED.industry,\n keywords = EXCLUDED.keywords,\n technologies = EXCLUDED.technologies,\n estimated_num_employees = EXCLUDED.estimated_num_employees,\n annual_revenue = EXCLUDED.annual_revenue,\n annual_revenue_printed = EXCLUDED.annual_revenue_printed,\n total_funding = EXCLUDED.total_funding,\n total_funding_printed = EXCLUDED.total_funding_printed,\n latest_funding_stage = EXCLUDED.latest_funding_stage,\n latest_funding_round_date = EXCLUDED.latest_funding_round_date,\n founded_year = EXCLUDED.founded_year,\n publicly_traded_symbol = EXCLUDED.publicly_traded_symbol,\n publicly_traded_exchange = EXCLUDED.publicly_traded_exchange,\n phone = EXCLUDED.phone,\n raw_address = EXCLUDED.raw_address,\n street_address = EXCLUDED.street_address,\n city = EXCLUDED.city,\n state = EXCLUDED.state,\n postal_code = EXCLUDED.postal_code,\n country = EXCLUDED.country,\n raw = EXCLUDED.raw,\n enriched_at = EXCLUDED.enriched_at\n WHERE crm_domain_directory.enriched_at IS NULL\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "TextArray",+ "TextArray",+ "Int4",+ "Int8",+ "Text",+ "Int8",+ "Text",+ "Text",+ "Timestamptz",+ "Int4",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Jsonb"+ ]+ },+ "nullable": []+ },+ "hash": "334c6ec59d818a2c7251d13e1b5b04a943f677e749735806a7e137743c5afc7c"+}diff --git a/.sqlx/query-81e64debfcca3111e88cbdce997a9eb11fb0d1c66828a4361f7de63f9dffa91b.json b/.sqlx/query-81e64debfcca3111e88cbdce997a9eb11fb0d1c66828a4361f7de63f9dffa91b.jsonnew file mode 100644index 0000000000..e022d1c684--- /dev/null+++ b/.sqlx/query-81e64debfcca3111e88cbdce997a9eb11fb0d1c66828a4361f7de63f9dffa91b.json@@ -0,0 +1,24 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH candidate AS (\n SELECT dd.id\n FROM crm_domain_directory dd\n WHERE dd.enriched_at IS NULL\n AND dd.enrichment_attempts < $2\n AND (dd.enrichment_attempted_at IS NULL OR dd.enrichment_attempted_at < $3)\n AND EXISTS (SELECT 1 FROM crm_domains d WHERE d.domain = dd.domain)\n ORDER BY dd.enrichment_attempted_at NULLS FIRST, dd.created_at\n LIMIT $1\n FOR UPDATE OF dd SKIP LOCKED\n )\n UPDATE crm_domain_directory dd\n SET enrichment_attempts = dd.enrichment_attempts + 1,\n enrichment_attempted_at = now()\n FROM candidate c\n WHERE dd.id = c.id\n RETURNING dd.domain\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "domain",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Int8",+ "Int2",+ "Timestamptz"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "81e64debfcca3111e88cbdce997a9eb11fb0d1c66828a4361f7de63f9dffa91b"+}diff --git a/.sqlx/query-c4998096a67401510c7d65a72a503c6df9b9c79085da41b4569854f81f6444dc.json b/.sqlx/query-c4998096a67401510c7d65a72a503c6df9b9c79085da41b4569854f81f6444dc.jsonnew file mode 100644index 0000000000..26288f385c--- /dev/null+++ b/.sqlx/query-c4998096a67401510c7d65a72a503c6df9b9c79085da41b4569854f81f6444dc.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO crm_domain_directory (domain)\n VALUES ($1)\n ON CONFLICT (LOWER(domain)) DO NOTHING\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "c4998096a67401510c7d65a72a503c6df9b9c79085da41b4569854f81f6444dc"+}diff --git a/.sqlx/query-fa0410f4a8b799f01b4a582582d0baafac16a1058e7017d63813abf87b340e4a.json b/.sqlx/query-fa0410f4a8b799f01b4a582582d0baafac16a1058e7017d63813abf87b340e4a.jsondeleted file mode 100644index 1913a02577..0000000000--- a/.sqlx/query-fa0410f4a8b799f01b4a582582d0baafac16a1058e7017d63813abf87b340e4a.json+++ /dev/null@@ -1,43 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n INSERT INTO crm_domain_directory (\n domain, name, description, icon_url,\n apollo_organization_id, website_url, linkedin_url, twitter_url,\n facebook_url, industry, keywords, technologies,\n estimated_num_employees, annual_revenue, annual_revenue_printed,\n total_funding, total_funding_printed, latest_funding_stage,\n latest_funding_round_date, founded_year, publicly_traded_symbol,\n publicly_traded_exchange, phone, raw_address, street_address,\n city, state, postal_code, country, raw, enriched_at\n )\n VALUES (\n $1, $2, $3, $4,\n $5, $6, $7, $8,\n $9, $10, $11, $12,\n $13, $14, $15,\n $16, $17, $18,\n $19, $20, $21,\n $22, $23, $24, $25,\n $26, $27, $28, $29, $30, now()\n )\n ON CONFLICT (LOWER(domain)) DO NOTHING\n ",- "describe": {- "columns": [],- "parameters": {- "Left": [- "Text",- "Text",- "Text",- "Text",- "Text",- "Text",- "Text",- "Text",- "Text",- "Text",- "TextArray",- "TextArray",- "Int4",- "Int8",- "Text",- "Int8",- "Text",- "Text",- "Timestamptz",- "Int4",- "Text",- "Text",- "Text",- "Text",- "Text",- "Text",- "Text",- "Text",- "Text",- "Jsonb"- ]- },- "nullable": []- },- "hash": "fa0410f4a8b799f01b4a582582d0baafac16a1058e7017d63813abf87b340e4a"-}diff --git a/Cargo.lock b/Cargo.lockindex f1364fb425..6bdf84b9b6 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5256,6 +5256,7 @@ dependencies = ["unfurl","utoipa","uuid",+ "wiremock","workspace-hack",]diff --git a/crates/crm/AGENTS.md b/crates/crm/AGENTS.mdindex 3a5cee903f..37ab297d4c 100644--- a/crates/crm/AGENTS.md+++ b/crates/crm/AGENTS.md@@ -57,6 +57,12 @@ tools/HTTP/GraphQL surfaces over them.## Gotchas+- Company names/logos live in the global `crm_domain_directory`. Only the+ email pubsub workers carry a real `CompanyMetadataResolver`; every other+ binary uses `NoOpCompanyMetadataResolver`, which leaves a pending entry+ (NULL `enriched_at`) that `CrmServiceImpl::enrich_pending_domains` resolves+ later with bounded retries. Never store an empty entry for a domain no+ resolver answered for: resolved entries are permanent.- The Sales migration (`SALES_BACKFILL.md`) is forward-only and additive —its down migration refuses to erase completion or user edits; never revertit to undo a bad migration, only the application code that reads it.diff --git a/crates/crm/Cargo.toml b/crates/crm/Cargo.tomlindex 4bcfe50e63..1f94c0062d 100644--- a/crates/crm/Cargo.toml+++ b/crates/crm/Cargo.toml@@ -125,3 +125,4 @@ properties = { path = "../properties", default-features = false, features = [rootcause = { workspace = true }tokio = { workspace = true, features = ["macros"] }tower = { workspace = true }+wiremock = { workspace = true }diff --git a/crates/crm/src/domain/companies_repo.rs b/crates/crm/src/domain/companies_repo.rsindex c789c8ee47..c3805535a4 100644--- a/crates/crm/src/domain/companies_repo.rs+++ b/crates/crm/src/domain/companies_repo.rs@@ -241,24 +241,26 @@ pub trait CompaniesRepository: Clone + Send + Sync + 'static {/// case-insensitively. Returns `Ok(None)` when no row exists for/// the domain — the caller is expected to resolve via/// [`crate::domain::company_metadata_resolver::CompanyMetadataResolver`]- /// and then [`upsert_domain_metadata`] before retrying.+ /// and then [`upsert_domain_metadata`] (or, when the resolver is+ /// unavailable, [`insert_pending_domain`]) before retrying.////// `Some(DomainMetadata { name: None, ... })` is distinct from- /// `None`: it means the domain has been looked up before and the- /// resolver returned nothing useful — the negative-cache entry- /// suppresses further resolver calls.+ /// `None`: the domain already has an entry, either resolved to+ /// nothing useful (the negative cache) or still pending for the+ /// enrichment worker. Either way callers don't resolve it inline.////// [`upsert_domain_metadata`]: CompaniesRepository::upsert_domain_metadata+ /// [`insert_pending_domain`]: CompaniesRepository::insert_pending_domainfn lookup_domain_metadata(&self,domain: &str,) -> impl Future<Output = Result<Option<DomainMetadata>, CrmError>> + Send;- /// Insert `metadata` for `domain` into `crm_domain_directory` with- /// `ON CONFLICT (LOWER(domain)) DO NOTHING`. The directory is a- /// global, first-write-wins cache: a row for `domain` (whether- /// populated or all-NULL) is preserved as-is for the lifetime of- /// the table. `domain` is lower-cased before storage.+ /// Store resolved `metadata` for `domain` in `crm_domain_directory`+ /// and stamp it resolved. The directory is a global cache: a+ /// resolved row (whether populated or all-NULL) is first-write-wins+ /// and preserved as-is; a pending row is filled in. `domain` is+ /// lower-cased before storage.////// Idempotent under concurrent calls — racing producers can both/// resolve the same domain and both call this method; the second@@ -269,6 +271,30 @@ pub trait CompaniesRepository: Clone + Send + Sync + 'static {metadata: &DomainMetadata,) -> impl Future<Output = Result<(), CrmError>> + Send;+ /// Insert a pending `crm_domain_directory` row for `domain` (no+ /// metadata, not resolved) unless the domain already has a row.+ /// Read paths treat it like a negative-cache row; the enrichment+ /// worker claims it through [`claim_pending_domains`].+ ///+ /// [`claim_pending_domains`]: CompaniesRepository::claim_pending_domains+ fn insert_pending_domain(+ &self,+ domain: &str,+ ) -> impl Future<Output = Result<(), CrmError>> + Send;++ /// Claim up to `limit` pending `crm_domain_directory` rows whose+ /// domain some team still tracks, with fewer than `max_attempts`+ /// enrichment attempts and no attempt since `attempted_before`.+ /// Claiming counts an attempt and stamps it now, so concurrent+ /// workers skip the claimed rows until `attempted_before` passes+ /// them. Returns the claimed domains.+ fn claim_pending_domains(+ &self,+ limit: i64,+ max_attempts: i16,+ attempted_before: DateTime<Utc>,+ ) -> impl Future<Output = Result<Vec<String>, CrmError>> + Send;+/// Reverses [`populate_contact`] for one `(link_id, email)`: drops the/// matching `crm_contact_sources` row, then `crm_contacts` if no other/// source rows remain for that contact, then `crm_companies` (cascadingdiff --git a/crates/crm/src/domain/company_metadata_resolver.rs b/crates/crm/src/domain/company_metadata_resolver.rsindex e75f6937bc..eaf8297573 100644--- a/crates/crm/src/domain/company_metadata_resolver.rs+++ b/crates/crm/src/domain/company_metadata_resolver.rs@@ -11,16 +11,24 @@use crate::domain::model::DomainMetadata;+/// Outcome of resolving one domain.+#[derive(Debug, Clone, PartialEq)]+pub enum DomainResolution {+ /// The provider answered. Stored in `crm_domain_directory` as the+ /// domain's permanent entry, even when every field is empty (the+ /// provider knows nothing about the domain — the negative cache).+ Resolved(Box<DomainMetadata>),+ /// No answer this time: the binary has no resolver, the provider is+ /// not configured, or the call failed. The directory entry stays+ /// pending so the enrichment worker retries it.+ Unavailable,+}+/// Resolves company metadata for a single email domain.///-/// Implementations are expected to be best-effort: a missing page,-/// network timeout, or malformed metadata should be surfaced as a-/// [`DomainMetadata`] with all-NULL fields rather than an error, since-/// the caller writes the result into `crm_domain_directory` as a-/// negative cache so the domain isn't re-resolved on the next populate.+/// Implementations are best-effort and never error: a failed call is+/// [`DomainResolution::Unavailable`], logged by the implementation.pub trait CompanyMetadataResolver: Clone + Send + Sync + 'static {- /// Resolve metadata for the given lower-cased `domain`. Returns- /// [`DomainMetadata::default`] (all fields `None`) on any failure;- /// implementations are expected to log internally.- fn resolve(&self, domain: &str) -> impl Future<Output = DomainMetadata> + Send;+ /// Resolve metadata for the given lower-cased `domain`.+ fn resolve(&self, domain: &str) -> impl Future<Output = DomainResolution> + Send;}diff --git a/crates/crm/src/domain/service.rs b/crates/crm/src/domain/service.rsindex 53e76c2026..48f8f1bf98 100644--- a/crates/crm/src/domain/service.rs+++ b/crates/crm/src/domain/service.rs@@ -6,14 +6,14 @@ mod test;use crate::domain::{auth::{CrmCompanyReceipt, CrmContactReceipt, CrmTeamReceipt},companies_repo::{CompaniesRepository, CrmCompanyListSort, CrmCompanySoupCursor},- company_metadata_resolver::CompanyMetadataResolver,+ company_metadata_resolver::{CompanyMetadataResolver, DomainResolution},generic_email_domains::is_generic_email_domain,model::{CrmCompanyForSoup, CrmCompanyWithContacts, CrmContact, CrmError, CrmScopePrecheck,CrmTeamSettings, CrmTeamSettingsPatch, DepopulateContactOutcome,},};-use chrono::{DateTime, Utc};+use chrono::{DateTime, TimeDelta, Utc};use entity_access::domain::models::{EditAccessLevel, MemberTeamRole, ViewAccessLevel};/// The CrmService exposes operations over CRM records (companies, their@@ -65,10 +65,12 @@ pub trait CrmService: Clone + Send + Sync + 'static {/// `crm_domain_directory` has an entry for the email's domain — if/// not, it invokes the/// [`crate::domain::company_metadata_resolver::CompanyMetadataResolver`]- /// and inserts the result (which may be all-NULL on resolver- /// failure — that's the negative cache). The directory write is its- /// own transaction so the populate tx never holds locks across an- /// HTTP fetch.+ /// and inserts the result (which may be all-NULL when the provider+ /// knows nothing about the domain — that's the negative cache), or a+ /// pending entry for [`CrmServiceImpl::enrich_pending_domains`] when+ /// the resolver is unavailable. The directory write is its own+ /// transaction so the populate tx never holds locks across an HTTP+ /// fetch.////// `first_at` / `last_at` are the contact's known activity range/// for this populate. Per-message paths pass the message's@@ -185,7 +187,9 @@ pub trait CrmService: Clone + Send + Sync + 'static {/// Like populate, ensures `crm_domain_directory` has an entry for/// the domain (resolving via the/// [`crate::domain::company_metadata_resolver::CompanyMetadataResolver`]- /// on a miss) so the company still picks up icon/description+ /// on a miss, or leaving it pending for+ /// [`CrmServiceImpl::enrich_pending_domains`] when the binary has no+ /// resolver) so the company still picks up icon/description/// enrichment; the user's `name` is stored on `crm_companies.custom_name`/// (never in the shared directory — it's global across teams) and/// overrides the directory name on every read path.@@ -516,9 +520,10 @@ where{/// The underlying companies repositorycompanies_repository: CR,- /// Resolver consulted only when `crm_domain_directory` has no entry- /// for a given domain. The resolver itself is best-effort — its- /// failures collapse to a negative-cache row in the directory.+ /// Resolver consulted when `crm_domain_directory` has no entry for a+ /// given domain, and for pending entries in+ /// [`CrmServiceImpl::enrich_pending_domains`]. The resolver itself is+ /// best-effort — when it is unavailable the entry stays pending.metadata_resolver: R,}@@ -547,8 +552,101 @@ wheremetadata_resolver,}}++ /// Resolves one batch of pending `crm_domain_directory` entries+ /// whose domain some team still tracks, and returns how many were+ /// claimed. Each entry gets at most [`MAX_ENRICHMENT_ATTEMPTS`]+ /// attempts, spaced [`ENRICHMENT_RETRY_AFTER`] apart; an entry the+ /// resolver keeps failing on stays pending (read paths show the+ /// domain) and is not claimed again. Team-scoped+ /// `crm_companies.custom_name` overrides are never touched — they win+ /// over the directory name on every read path.+ ///+ /// Only meaningful with a real resolver: with+ /// [`crate::outbound::no_op_resolver::NoOpCompanyMetadataResolver`]+ /// every claim would burn an attempt.+ #[tracing::instrument(+ name = "crm.enrich_pending_domains",+ skip(self),+ fields(claimed, resolved),+ err+ )]+ pub async fn enrich_pending_domains(&self) -> Result<usize, CrmError> {+ let domains = self+ .companies_repository+ .claim_pending_domains(+ ENRICHMENT_BATCH_SIZE,+ MAX_ENRICHMENT_ATTEMPTS,+ Utc::now() - ENRICHMENT_RETRY_AFTER,+ )+ .await?;++ let mut resolved = 0;+ for domain in &domains {+ let DomainResolution::Resolved(metadata) = self.metadata_resolver.resolve(domain).await+ else {+ continue;+ };+ // One failed write leaves that entry pending for a later+ // attempt; it must not drop the rest of the batch.+ if self+ .companies_repository+ .upsert_domain_metadata(domain, &metadata)+ .await+ .inspect_err(|error| {+ tracing::warn!(?error, domain, "failed to store enriched domain metadata")+ })+ .is_ok()+ {+ resolved += 1;+ }+ }++ let span = tracing::Span::current();+ span.record("claimed", domains.len());+ span.record("resolved", resolved);+ Ok(domains.len())+ }++ /// Ensures `crm_domain_directory` has an entry for `domain`: resolves+ /// it when the domain has none, storing a pending entry instead when+ /// the resolver is unavailable.+ async fn ensure_domain_directory_entry(&self, domain: &str) -> Result<(), CrmError> {+ if self+ .companies_repository+ .lookup_domain_metadata(domain)+ .await?+ .is_some()+ {+ return Ok(());+ }+ match self.metadata_resolver.resolve(domain).await {+ DomainResolution::Resolved(metadata) => {+ self.companies_repository+ .upsert_domain_metadata(domain, &metadata)+ .await+ }+ DomainResolution::Unavailable => {+ self.companies_repository+ .insert_pending_domain(domain)+ .await+ }+ }+ }}+/// Pending directory entries one [`CrmServiceImpl::enrich_pending_domains`]+/// pass claims. Passes run on a fixed interval, so this caps the resolver+/// (Apollo) call rate.+const ENRICHMENT_BATCH_SIZE: i64 = 10;++/// Enrichment attempts after which a pending directory entry is left alone.+const MAX_ENRICHMENT_ATTEMPTS: i16 = 5;++/// Minimum gap between enrichment attempts for one directory entry; also+/// how long a claim by a worker that died mid-batch blocks other workers.+const ENRICHMENT_RETRY_AFTER: TimeDelta = TimeDelta::hours(6);+impl<CR, R> CrmService for CrmServiceImpl<CR, R>whereCR: CompaniesRepository,@@ -627,17 +725,8 @@ where// directory entry was written when its first sent message// populated; if it isn't tracked, this call will no-op in the// repo anyway.- if is_sent- && self- .companies_repository- .lookup_domain_metadata(domain)- .await?- .is_none()- {- let metadata = self.metadata_resolver.resolve(domain).await;- self.companies_repository- .upsert_domain_metadata(domain, &metadata)- .await?;+ if is_sent {+ self.ensure_domain_directory_entry(domain).await?;}self.companies_repository@@ -725,17 +814,7 @@ where// domain so icon/description enrichment resolves. The user's// name never goes here — the directory is global across teams;// it lands on `crm_companies.custom_name` in the repo instead.- if self- .companies_repository- .lookup_domain_metadata(&domain)- .await?- .is_none()- {- let metadata = self.metadata_resolver.resolve(&domain).await;- self.companies_repository- .upsert_domain_metadata(&domain, &metadata)- .await?;- }+ self.ensure_domain_directory_entry(&domain).await?;self.companies_repository.create_company_for_team(&access.team_id(), &domain, name, Utc::now())diff --git a/crates/crm/src/domain/service/test.rs b/crates/crm/src/domain/service/test.rsindex 7e5f7a8efc..ae86b89009 100644--- a/crates/crm/src/domain/service/test.rs+++ b/crates/crm/src/domain/service/test.rs@@ -120,6 +120,19 @@ impl CompaniesRepository for StubRepo {unimplemented!()}+ async fn insert_pending_domain(&self, _domain: &str) -> Result<(), CrmError> {+ unimplemented!()+ }++ async fn claim_pending_domains(+ &self,+ _limit: i64,+ _max_attempts: i16,+ _attempted_before: DateTime<Utc>,+ ) -> Result<Vec<String>, CrmError> {+ unimplemented!()+ }+async fn depopulate_contact(&self,_team_id: &uuid::Uuid,diff --git a/crates/crm/src/outbound/apollo_resolver.rs b/crates/crm/src/outbound/apollo_resolver.rsindex 7bf0da6e1f..b4b6dec456 100644--- a/crates/crm/src/outbound/apollo_resolver.rs+++ b/crates/crm/src/outbound/apollo_resolver.rs@@ -6,10 +6,12 @@//! onto [`DomainMetadata`]. The full payload — minus our own Apollo//! workspace `account` record — is preserved in [`DomainMetadata::raw`].//!-//! Best-effort: any failure (network error, non-2xx status, malformed-//! body, missing `organization`) is logged and surfaced as-//! [`DomainMetadata::default`] so the caller can persist a negative-cache-//! row and avoid re-resolving the domain.+//! Best-effort: a 2xx response without an `organization` (Apollo knows+//! nothing about the domain) resolves to [`DomainMetadata::default`], the+//! negative cache. A missing API key, network error, non-2xx status+//! (including rate limits), or malformed body is logged and surfaced as+//! [`DomainResolution::Unavailable`], so the domain is retried instead of+//! cached empty.#[cfg(test)]mod test;@@ -19,7 +21,10 @@ use std::time::Duration;use chrono::{DateTime, Utc};use serde_json::Value;-use crate::domain::{company_metadata_resolver::CompanyMetadataResolver, model::DomainMetadata};+use crate::domain::{+ company_metadata_resolver::{CompanyMetadataResolver, DomainResolution},+ model::DomainMetadata,+};/// Default Apollo API base URL.const DEFAULT_BASE_URL: &str = "https://api.apollo.io";@@ -68,12 +73,12 @@ impl ApolloCompanyMetadataResolver {impl CompanyMetadataResolver for ApolloCompanyMetadataResolver {#[tracing::instrument(skip(self))]- async fn resolve(&self, domain: &str) -> DomainMetadata {+ async fn resolve(&self, domain: &str) -> DomainResolution {// No key configured (e.g. local without APOLLO_API_KEY): skip the- // call entirely rather than burning a 401 and caching an empty row.+ // call entirely rather than burning a 401.if self.api_key.is_empty() {tracing::debug!(domain, "apollo enrich: no api key configured; skipping");- return DomainMetadata::default();+ return DomainResolution::Unavailable;}let url = format!("{}/api/v1/organizations/enrich", self.base_url);@@ -91,25 +96,25 @@ impl CompanyMetadataResolver for ApolloCompanyMetadataResolver {Ok(body) => body,Err(e) => {tracing::warn!(error=?e, domain, "apollo enrich: failed to parse body");- return DomainMetadata::default();+ return DomainResolution::Unavailable;}},Ok(resp) => {tracing::warn!(status = %resp.status(), domain, "apollo enrich: non-success status");- return DomainMetadata::default();+ return DomainResolution::Unavailable;}Err(e) => {tracing::warn!(error=?e, domain, "apollo enrich: request failed");- return DomainMetadata::default();+ return DomainResolution::Unavailable;}};let Some(organization) = body.get("organization") else {tracing::warn!(domain, "apollo enrich: response had no organization");- return DomainMetadata::default();+ return DomainResolution::Resolved(Box::default());};- map_organization(organization)+ DomainResolution::Resolved(Box::new(map_organization(organization)))}}diff --git a/crates/crm/src/outbound/apollo_resolver/test.rs b/crates/crm/src/outbound/apollo_resolver/test.rsindex 03eb0fefa2..f1758dddbb 100644--- a/crates/crm/src/outbound/apollo_resolver/test.rs+++ b/crates/crm/src/outbound/apollo_resolver/test.rs@@ -1,6 +1,8 @@use super::*;use crate::domain::company_metadata_resolver::CompanyMetadataResolver;use serde_json::json;+use wiremock::matchers::{method, path, query_param};+use wiremock::{Mock, MockServer, ResponseTemplate};/// The documented Apollo sample payload maps onto every field, and the/// `account` / `account_id` keys are stripped from `raw`.@@ -115,6 +117,54 @@ fn tolerates_mistyped_and_blank_fields() {);}+/// Failures that say nothing about the domain stay retryable; only an+/// answer from Apollo becomes a (possibly empty) directory entry.+#[tokio::test]+async fn only_apollo_answers_resolve() {+ let server = MockServer::start().await;+ Mock::given(method("GET"))+ .and(path("/api/v1/organizations/enrich"))+ .and(query_param("domain", "unknown.com"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({})))+ .mount(&server)+ .await;+ Mock::given(method("GET"))+ .and(path("/api/v1/organizations/enrich"))+ .and(query_param("domain", "acme.com"))+ .respond_with(+ ResponseTemplate::new(200)+ .set_body_json(json!({ "organization": { "name": "Acme Inc." } })),+ )+ .mount(&server)+ .await;+ Mock::given(method("GET"))+ .and(path("/api/v1/organizations/enrich"))+ .and(query_param("domain", "busy.com"))+ .respond_with(ResponseTemplate::new(429))+ .mount(&server)+ .await;+ let resolver = ApolloCompanyMetadataResolver::with_base_url("key".to_string(), server.uri());++ assert_eq!(+ resolver.resolve("unknown.com").await,+ DomainResolution::Resolved(Box::default())+ );+ let DomainResolution::Resolved(acme) = resolver.resolve("acme.com").await else {+ panic!("expected acme.com to resolve");+ };+ assert_eq!(acme.name.as_deref(), Some("Acme Inc."));+ assert_eq!(+ resolver.resolve("busy.com").await,+ DomainResolution::Unavailable+ );+ assert_eq!(+ ApolloCompanyMetadataResolver::with_base_url(String::new(), server.uri())+ .resolve("acme.com")+ .await,+ DomainResolution::Unavailable+ );+}+/// Live smoke test against the real Apollo API. Ignored by default — it/// makes a network call and consumes an Apollo credit. Run with:/// APOLLO_API_KEY=<key> cargo test -p crm apollo_live -- --ignored --nocapture@@ -126,7 +176,9 @@ async fn apollo_live_enrich() {let domain = std::env::var("APOLLO_TEST_DOMAIN").unwrap_or_else(|_| "macro.com".to_string());let resolver = ApolloCompanyMetadataResolver::new(api_key);- let md = resolver.resolve(&domain).await;+ let DomainResolution::Resolved(md) = resolver.resolve(&domain).await else {+ panic!("Apollo was unavailable for {domain} — check API key and credits");+ };println!("DomainMetadata for {domain}:\n{md:#?}");assert!(diff --git a/crates/crm/src/outbound/companies_repo.rs b/crates/crm/src/outbound/companies_repo.rsindex 7f0144a2ed..661655301c 100644--- a/crates/crm/src/outbound/companies_repo.rs+++ b/crates/crm/src/outbound/companies_repo.rs@@ -937,8 +937,9 @@ impl CompaniesRepository for CompaniesRepositoryImpl {metadata: &DomainMetadata,) -> Result<(), CrmError> {let normalized_domain = domain.to_ascii_lowercase();- // First-write-wins. All-empty rows are negative-cache entries that- // suppress future resolver calls; `enriched_at` stamps resolve time.+ // First-write-wins for resolved rows; a pending row (NULL+ // `enriched_at`) is filled in. All-empty resolved rows are+ // negative-cache entries that suppress future resolver calls.sqlx::query!(r#"INSERT INTO crm_domain_directory (@@ -961,7 +962,38 @@ impl CompaniesRepository for CompaniesRepositoryImpl {$22, $23, $24, $25,$26, $27, $28, $29, $30, now())- ON CONFLICT (LOWER(domain)) DO NOTHING+ ON CONFLICT (LOWER(domain)) DO UPDATE SET+ name = EXCLUDED.name,+ description = EXCLUDED.description,+ icon_url = EXCLUDED.icon_url,+ apollo_organization_id = EXCLUDED.apollo_organization_id,+ website_url = EXCLUDED.website_url,+ linkedin_url = EXCLUDED.linkedin_url,+ twitter_url = EXCLUDED.twitter_url,+ facebook_url = EXCLUDED.facebook_url,+ industry = EXCLUDED.industry,+ keywords = EXCLUDED.keywords,+ technologies = EXCLUDED.technologies,+ estimated_num_employees = EXCLUDED.estimated_num_employees,+ annual_revenue = EXCLUDED.annual_revenue,+ annual_revenue_printed = EXCLUDED.annual_revenue_printed,+ total_funding = EXCLUDED.total_funding,+ total_funding_printed = EXCLUDED.total_funding_printed,+ latest_funding_stage = EXCLUDED.latest_funding_stage,+ latest_funding_round_date = EXCLUDED.latest_funding_round_date,+ founded_year = EXCLUDED.founded_year,+ publicly_traded_symbol = EXCLUDED.publicly_traded_symbol,+ publicly_traded_exchange = EXCLUDED.publicly_traded_exchange,+ phone = EXCLUDED.phone,+ raw_address = EXCLUDED.raw_address,+ street_address = EXCLUDED.street_address,+ city = EXCLUDED.city,+ state = EXCLUDED.state,+ postal_code = EXCLUDED.postal_code,+ country = EXCLUDED.country,+ raw = EXCLUDED.raw,+ enriched_at = EXCLUDED.enriched_at+ WHERE crm_domain_directory.enriched_at IS NULL"#,normalized_domain,metadata.name,@@ -1001,6 +1033,61 @@ impl CompaniesRepository for CompaniesRepositoryImpl {Ok(())}+ #[tracing::instrument(skip(self), err)]+ async fn insert_pending_domain(&self, domain: &str) -> Result<(), CrmError> {+ sqlx::query!(+ r#"+ INSERT INTO crm_domain_directory (domain)+ VALUES ($1)+ ON CONFLICT (LOWER(domain)) DO NOTHING+ "#,+ domain.to_ascii_lowercase(),+ )+ .execute(&self.pool)+ .await+ .map_err(|e| CrmError::StorageLayerError(e.into()))?;++ Ok(())+ }++ #[tracing::instrument(skip(self), err)]+ async fn claim_pending_domains(+ &self,+ limit: i64,+ max_attempts: i16,+ attempted_before: DateTime<Utc>,+ ) -> Result<Vec<String>, CrmError> {+ // `crm_domains.domain` and directory domains are both stored+ // lower-cased, so plain equality uses `crm_domains_domain_idx`.+ sqlx::query_scalar!(+ r#"+ WITH candidate AS (+ SELECT dd.id+ FROM crm_domain_directory dd+ WHERE dd.enriched_at IS NULL+ AND dd.enrichment_attempts < $2+ AND (dd.enrichment_attempted_at IS NULL OR dd.enrichment_attempted_at < $3)+ AND EXISTS (SELECT 1 FROM crm_domains d WHERE d.domain = dd.domain)+ ORDER BY dd.enrichment_attempted_at NULLS FIRST, dd.created_at+ LIMIT $1+ FOR UPDATE OF dd SKIP LOCKED+ )+ UPDATE crm_domain_directory dd+ SET enrichment_attempts = dd.enrichment_attempts + 1,+ enrichment_attempted_at = now()+ FROM candidate c+ WHERE dd.id = c.id+ RETURNING dd.domain+ "#,+ limit,+ max_attempts,+ attempted_before,+ )+ .fetch_all(&self.pool)+ .await+ .map_err(|e| CrmError::StorageLayerError(e.into()))+ }+#[tracing::instrument(skip(self), err)]async fn set_email_sync(&self,diff --git a/crates/crm/src/outbound/companies_repo/test.rs b/crates/crm/src/outbound/companies_repo/test.rsindex 2cf612b1b8..502bc39b5c 100644--- a/crates/crm/src/outbound/companies_repo/test.rs+++ b/crates/crm/src/outbound/companies_repo/test.rs@@ -8,6 +8,7 @@ pub(crate) mod helpers;mod create_company;mod create_contact;+mod domain_directory;mod get_company_for_team;mod get_contact_by_email_for_team;mod get_contact_for_team;diff --git a/crates/crm/src/outbound/companies_repo/test/domain_directory.rs b/crates/crm/src/outbound/companies_repo/test/domain_directory.rsnew file mode 100644index 0000000000..88839d7f38--- /dev/null+++ b/crates/crm/src/outbound/companies_repo/test/domain_directory.rs@@ -0,0 +1,189 @@+use super::helpers::*;+use crate::domain::auth::CrmTeamReceipt;+use crate::domain::companies_repo::CompaniesRepository;+use crate::domain::company_metadata_resolver::{CompanyMetadataResolver, DomainResolution};+use crate::domain::model::DomainMetadata;+use crate::domain::service::{CrmService, CrmServiceImpl};+use crate::outbound::companies_repo::CompaniesRepositoryImpl;+use crate::outbound::no_op_resolver::NoOpCompanyMetadataResolver;+use chrono::{DateTime, TimeDelta, Utc};+use entity_access::domain::models::MemberTeamRole;+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use sqlx::PgPool;+use uuid::Uuid;++#[derive(Clone)]+struct FixedResolver(DomainResolution);++impl CompanyMetadataResolver for FixedResolver {+ async fn resolve(&self, _domain: &str) -> DomainResolution {+ self.0.clone()+ }+}++/// `(name, icon_url, enriched_at, enrichment_attempts)` of the domain's+/// directory row.+type DirectoryRow = (Option<String>, Option<String>, Option<DateTime<Utc>>, i16);++async fn fetch_directory(pool: &PgPool, domain: &str) -> sqlx::Result<Option<DirectoryRow>> {+ sqlx::query_as(+ r#"SELECT name, icon_url, enriched_at, enrichment_attempts+ FROM crm_domain_directory WHERE LOWER(domain) = LOWER($1)"#,+ )+ .bind(domain)+ .fetch_optional(pool)+ .await+}++fn acme_metadata() -> DomainMetadata {+ DomainMetadata {+ name: Some("Acme Inc.".to_string()),+ icon_url: Some("https://acme.com/logo.png".to_string()),+ ..Default::default()+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn upsert_fills_pending_entry_but_keeps_resolved_entry(pool: PgPool) -> anyhow::Result<()> {+ let repo = CompaniesRepositoryImpl::new(pool.clone());++ repo.insert_pending_domain("Acme.com").await?;+ let (name, _, enriched_at, _) = fetch_directory(&pool, "acme.com").await?.unwrap();+ assert_eq!(name, None);+ assert_eq!(enriched_at, None);++ repo.upsert_domain_metadata("acme.com", &acme_metadata())+ .await?;+ let (name, icon_url, enriched_at, _) = fetch_directory(&pool, "acme.com").await?.unwrap();+ assert_eq!(name.as_deref(), Some("Acme Inc."));+ assert_eq!(icon_url.as_deref(), Some("https://acme.com/logo.png"));+ assert!(enriched_at.is_some());++ // A resolved entry is first-write-wins: neither a later resolve nor a+ // pending insert replaces it.+ repo.upsert_domain_metadata(+ "acme.com",+ &DomainMetadata {+ name: Some("Other".to_string()),+ ..Default::default()+ },+ )+ .await?;+ repo.insert_pending_domain("acme.com").await?;+ let (name, _, enriched_at, _) = fetch_directory(&pool, "acme.com").await?.unwrap();+ assert_eq!(name.as_deref(), Some("Acme Inc."));+ assert!(enriched_at.is_some());+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn claim_pending_domains_is_bounded(pool: PgPool) -> anyhow::Result<()> {+ let team_id = Uuid::now_v7();+ seed_team(&pool, team_id, "macro|[email protected]").await?;+ insert_company(+ &pool,+ team_id,+ true,+ &["acme.com", "globex.com", "exhausted.com"],+ )+ .await?;++ let repo = CompaniesRepositoryImpl::new(pool.clone());+ repo.insert_pending_domain("acme.com").await?;+ // Pending, but no team tracks the domain any more.+ repo.insert_pending_domain("orphan.com").await?;+ repo.upsert_domain_metadata("globex.com", &DomainMetadata::default())+ .await?;+ repo.insert_pending_domain("exhausted.com").await?;+ sqlx::query(+ "UPDATE crm_domain_directory SET enrichment_attempts = 5 WHERE domain = 'exhausted.com'",+ )+ .execute(&pool)+ .await?;++ let retry_cutoff = Utc::now() - TimeDelta::hours(6);+ let claimed = repo.claim_pending_domains(10, 5, retry_cutoff).await?;+ assert_eq!(claimed, vec!["acme.com".to_string()]);+ let (_, _, _, attempts) = fetch_directory(&pool, "acme.com").await?.unwrap();+ assert_eq!(attempts, 1);++ // The fresh attempt holds the entry until the retry gap passes.+ assert!(+ repo.claim_pending_domains(10, 5, retry_cutoff)+ .await?+ .is_empty()+ );+ let after_gap = Utc::now() + TimeDelta::seconds(1);+ assert_eq!(+ repo.claim_pending_domains(10, 5, after_gap).await?,+ vec!["acme.com".to_string()]+ );+ let (_, _, _, attempts) = fetch_directory(&pool, "acme.com").await?.unwrap();+ assert_eq!(attempts, 2);+ Ok(())+}++/// A company created where no resolver is available (DSS, MCP, agent tools)+/// leaves a pending entry instead of a permanent empty one, and the+/// enrichment worker later fills it for every team tracking the domain —+/// without touching a team's own name for the company.+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn manual_company_is_enriched_later_and_keeps_team_names(pool: PgPool) -> anyhow::Result<()> {+ let team_a = Uuid::now_v7();+ let team_b = Uuid::now_v7();+ seed_team(&pool, team_a, "macro|[email protected]").await?;+ seed_team(&pool, team_b, "macro|[email protected]").await?;+ enable_crm_for_team(&pool, team_a).await?;+ enable_crm_for_team(&pool, team_b).await?;++ let manual = CrmServiceImpl::new(+ CompaniesRepositoryImpl::new(pool.clone()),+ NoOpCompanyMetadataResolver,+ );+ let renamed = manual+ .create_company(+ &CrmTeamReceipt::<MemberTeamRole>::dangerously_internal(team_a),+ "Acme (ours)",+ "acme.com",+ )+ .await?;+ let unnamed = insert_company(&pool, team_b, true, &["acme.com"]).await?;+ let (name, _, enriched_at, _) = fetch_directory(&pool, "acme.com").await?.unwrap();+ assert_eq!(name, None);+ assert_eq!(enriched_at, None);++ let unavailable = CrmServiceImpl::new(+ CompaniesRepositoryImpl::new(pool.clone()),+ FixedResolver(DomainResolution::Unavailable),+ );+ assert_eq!(unavailable.enrich_pending_domains().await?, 1);+ let (_, _, enriched_at, attempts) = fetch_directory(&pool, "acme.com").await?.unwrap();+ assert_eq!(enriched_at, None);+ assert_eq!(attempts, 1);++ sqlx::query("UPDATE crm_domain_directory SET enrichment_attempted_at = NULL")+ .execute(&pool)+ .await?;+ let worker = CrmServiceImpl::new(+ CompaniesRepositoryImpl::new(pool.clone()),+ FixedResolver(DomainResolution::Resolved(Box::new(acme_metadata()))),+ );+ assert_eq!(worker.enrich_pending_domains().await?, 1);+ assert_eq!(worker.enrich_pending_domains().await?, 0);++ let repo = CompaniesRepositoryImpl::new(pool.clone());+ let team_a_company = repo+ .get_company_for_team(&team_a, &renamed.company.id, false)+ .await?+ .unwrap();+ assert_eq!(team_a_company.name.as_deref(), Some("Acme (ours)"));+ let team_b_company = repo+ .get_company_for_team(&team_b, &unnamed, false)+ .await?+ .unwrap();+ assert_eq!(team_b_company.name.as_deref(), Some("Acme Inc."));+ let (_, icon_url, enriched_at, _) = fetch_directory(&pool, "acme.com").await?.unwrap();+ assert_eq!(icon_url.as_deref(), Some("https://acme.com/logo.png"));+ assert!(enriched_at.is_some());+ Ok(())+}diff --git a/crates/crm/src/outbound/no_op_resolver.rs b/crates/crm/src/outbound/no_op_resolver.rsindex d6e57abd7b..d7a418fe6f 100644--- a/crates/crm/src/outbound/no_op_resolver.rs+++ b/crates/crm/src/outbound/no_op_resolver.rs@@ -1,33 +1,23 @@-//! [`CompanyMetadataResolver`] adapter that always returns-//! [`DomainMetadata::default`].+//! [`CompanyMetadataResolver`] adapter that never resolves.//!-//! Used by binaries that need to construct a [`CrmService`] but never-//! call [`populate_contact`] — typically HTTP API servers that only-//! read existing CRM rows. Wiring in a real unfurl-backed resolver in-//! those binaries would force them to carry the reqwest + scraper-//! deps for code paths they never hit.-//!-//! If a binary using this resolver ever *does* call `populate_contact`,-//! the resulting `crm_domain_directory` entries will all have NULL-//! `name`/`description`/`icon_url` fields — i.e. the directory's-//! negative cache fires for every domain. That's a silent loss of-//! company metadata, not a crash, so callers should prefer wiring in-//! the real [`UnfurlCompanyMetadataResolver`] when they participate in-//! the populate path.+//! Used by binaries that construct a [`CrmService`] without carrying a+//! metadata provider — HTTP API servers, MCP, and agent tools. Wiring a+//! real resolver into each of them would force them to carry the reqwest+//! + scraper deps and provider credentials. Companies they create get a+//! pending `crm_domain_directory` entry, which the email pubsub workers'+//! enrichment worker resolves with the real resolver.//!//! [`CrmService`]: crate::domain::service::CrmService-//! [`populate_contact`]: crate::domain::service::CrmService::populate_contact-//! [`UnfurlCompanyMetadataResolver`]: crate::outbound::unfurl_resolver::UnfurlCompanyMetadataResolver-use crate::domain::{company_metadata_resolver::CompanyMetadataResolver, model::DomainMetadata};+use crate::domain::company_metadata_resolver::{CompanyMetadataResolver, DomainResolution};-/// A [`CompanyMetadataResolver`] that always resolves to-/// [`DomainMetadata::default`] (all fields `None`).+/// A [`CompanyMetadataResolver`] that always returns+/// [`DomainResolution::Unavailable`].#[derive(Debug, Default, Clone, Copy)]pub struct NoOpCompanyMetadataResolver;impl CompanyMetadataResolver for NoOpCompanyMetadataResolver {- async fn resolve(&self, _domain: &str) -> DomainMetadata {- DomainMetadata::default()+ async fn resolve(&self, _domain: &str) -> DomainResolution {+ DomainResolution::Unavailable}}diff --git a/crates/crm/src/outbound/unfurl_resolver.rs b/crates/crm/src/outbound/unfurl_resolver.rsindex 937dbaa6c4..60b54e71d0 100644--- a/crates/crm/src/outbound/unfurl_resolver.rs+++ b/crates/crm/src/outbound/unfurl_resolver.rs@@ -17,12 +17,15 @@//!//! Failures (network errors, SSRF-blocked redirects, parse errors,//! non-HTML responses) are logged and surfaced as-//! [`DomainMetadata::default`] so the caller can persist a-//! negative-cache entry and never re-resolve the domain.+//! [`DomainResolution::Unavailable`], so the domain is retried a bounded+//! number of times instead of cached empty on a transient error.use std::sync::Arc;-use crate::domain::{company_metadata_resolver::CompanyMetadataResolver, model::DomainMetadata};+use crate::domain::{+ company_metadata_resolver::{CompanyMetadataResolver, DomainResolution},+ model::DomainMetadata,+};use unfurl::domain::ports::UnfurlService;/// Adapter that resolves [`DomainMetadata`] by unfurling@@ -58,7 +61,7 @@ whereU: UnfurlService,{#[tracing::instrument(skip(self))]- async fn resolve(&self, domain: &str) -> DomainMetadata {+ async fn resolve(&self, domain: &str) -> DomainResolution {let url = format!("https://{domain}");match self.unfurl_service.unfurl(&url).await {Ok(response) => {@@ -73,20 +76,16 @@ where} else {Some(title.to_string())};- DomainMetadata {+ DomainResolution::Resolved(Box::new(DomainMetadata {name,description: response.description,icon_url: response.favicon_url,..Default::default()- }+ }))}Err(e) => {- tracing::warn!(- error=?e,- domain,- "unfurl failed; returning empty DomainMetadata for negative cache"- );- DomainMetadata::default()+ tracing::warn!(error=?e, domain, "unfurl failed; domain metadata unavailable");+ DomainResolution::Unavailable}}}diff --git a/crates/macro_db_client/migrations/20261021155020_crm_domain_directory_pending_enrichment.sql b/crates/macro_db_client/migrations/20261021155020_crm_domain_directory_pending_enrichment.sqlnew file mode 100644index 0000000000..60804306e3--- /dev/null+++ b/crates/macro_db_client/migrations/20261021155020_crm_domain_directory_pending_enrichment.sql@@ -0,0 +1,21 @@+-- A directory entry with NULL enriched_at is pending: no resolver has answered+-- for its domain yet. The email pubsub workers retry pending entries of+-- tracked domains, at most a bounded number of times.+SET LOCAL lock_timeout = '5s';++ALTER TABLE crm_domain_directory+ ADD COLUMN enrichment_attempts SMALLINT NOT NULL DEFAULT 0,+ ADD COLUMN enrichment_attempted_at TIMESTAMPTZ;++-- Entries written before enriched_at existed were resolved by unfurl.+UPDATE crm_domain_directory+SET enriched_at = created_at+WHERE enriched_at IS NULL+ AND (name IS NOT NULL OR description IS NOT NULL OR icon_url IS NOT NULL);++-- Empty entries include the ones manual, API, and agent company creation+-- wrote through the no-op resolver. Make them pending so each gets one more+-- real resolve.+UPDATE crm_domain_directory+SET enriched_at = NULL+WHERE name IS NULL AND description IS NULL AND icon_url IS NULL;diff --git a/crates/macro_db_client/migrations/20261021170901_crm_domain_directory_pending_index.sql b/crates/macro_db_client/migrations/20261021170901_crm_domain_directory_pending_index.sqlnew file mode 100644index 0000000000..763a60c4a8--- /dev/null+++ b/crates/macro_db_client/migrations/20261021170901_crm_domain_directory_pending_index.sql@@ -0,0 +1,8 @@+-- no-transaction+-- Lets the CRM enrichment worker find pending directory entries. Must stay a+-- single statement: sqlx sends no-transaction migrations with multiple+-- statements as one batch, which wraps them in an implicit transaction that+-- CONCURRENTLY forbids.+CREATE INDEX CONCURRENTLY IF NOT EXISTS crm_domain_directory_pending_idx+ ON crm_domain_directory (enrichment_attempted_at NULLS FIRST, created_at)+ WHERE enriched_at IS NULL;diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.mdindex 37d4be3554..6f56eded4a 100644--- a/services/email_service/AGENTS.md+++ b/services/email_service/AGENTS.md@@ -16,6 +16,8 @@ the gateway `/email` prefix.`crm_cleanup`, `worker_lifecycle`.- `src/backfill_outbox.rs`, `src/backfill_init_service.rs`,`src/backfill_completion_service.rs`: Gmail history backfill orchestration.+- `src/crm_enrichment_worker.rs`: periodic pass that resolves pending CRM+ company directory entries (names/logos) left by binaries without a resolver.- `src/bin/`: standalone binaries — `pubsub_workers` (the queue consumerprocess), plus one-off backfill/cleanup scripts.diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rsindex 10558ff6a2..94c1744292 100644--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs@@ -371,7 +371,8 @@ async fn main() -> anyhow::Result<()> {));// The CRM crate's company-metadata resolver is consulted by- // `crm_service.populate_contact` only on `crm_domain_directory` misses.+ // `crm_service.populate_contact` on `crm_domain_directory` misses and by+ // the enrichment worker for pending entries other binaries leave behind.// `USE_APOLLO_CRM_ENRICHMENT` selects Apollo.io vs. the unfurl-backed// resolver; we also fall back to unfurl when the Apollo key can't be// loaded. The resolver is cheap to clone.@@ -409,6 +410,10 @@ async fn main() -> anyhow::Result<()> {crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),metadata_resolver.clone(),);+ worker_tracker.spawn(email_service::crm_enrichment_worker::run(+ crm_service.clone(),+ worker_cancellation_token.clone(),+ ));// Backfill workers run against a dedicated pool to keep their writes// off the primary worker pool. CRM writes are part of the backfilldiff --git a/services/email_service/src/crm_enrichment_worker.rs b/services/email_service/src/crm_enrichment_worker.rsnew file mode 100644index 0000000000..467ab269fb--- /dev/null+++ b/services/email_service/src/crm_enrichment_worker.rs@@ -0,0 +1,28 @@+//! Resolves CRM company directory entries left pending by binaries without a+//! metadata resolver (manual, API, and agent company creation) or by a failed+//! resolve.++use crate::pubsub::context::CrmServiceType;+use std::time::Duration;+use tokio_util::sync::CancellationToken;++/// One enrichment pass per interval keeps the resolver (Apollo) call rate at+/// the service's batch size per minute per worker.+const ENRICHMENT_INTERVAL: Duration = Duration::from_secs(60);++pub async fn run(service: CrmServiceType, cancellation: CancellationToken) {+ let mut interval = tokio::time::interval(ENRICHMENT_INTERVAL);+ loop {+ tokio::select! {+ biased;+ _ = cancellation.cancelled() => return,+ _ = interval.tick() => {},+ }+ // The service span records the error; the next tick retries.+ let _ = tokio::select! {+ biased;+ _ = cancellation.cancelled() => return,+ result = service.enrich_pending_domains() => result,+ };+ }+}diff --git a/services/email_service/src/lib.rs b/services/email_service/src/lib.rsindex 7d31765b44..5587d38888 100644--- a/services/email_service/src/lib.rs+++ b/services/email_service/src/lib.rs@@ -18,6 +18,8 @@ pub mod calendar_request_gate;pub mod calendar_tokens;pub mod composition;pub mod config;+/// Periodic enrichment of pending CRM company directory entries.+pub mod crm_enrichment_worker;/// Durable provider-neutral mailbox worker scheduling.pub mod mailbox_sync_worker;/// Outbound infrastructure adapters for email provider capabilities.diff --git a/services/email_service/src/pubsub/context.rs b/services/email_service/src/pubsub/context.rsindex e0f5b8c7e5..2308cf61ad 100644--- a/services/email_service/src/pubsub/context.rs+++ b/services/email_service/src/pubsub/context.rs@@ -3,8 +3,7 @@ use crate::util::redis::RedisClient;use connection_gateway_client::client::ConnectionGatewayClient;use contacts::domain::service::SqsContactsIngress;use contacts::outbound::ingress::SqsContactsQueue;-use crm::domain::company_metadata_resolver::CompanyMetadataResolver;-use crm::domain::model::DomainMetadata;+use crm::domain::company_metadata_resolver::{CompanyMetadataResolver, DomainResolution};use crm::domain::service::CrmServiceImpl;use crm::outbound::apollo_resolver::ApolloCompanyMetadataResolver;use crm::outbound::companies_repo::CompaniesRepositoryImpl;@@ -44,7 +43,7 @@ pub enum CrmMetadataResolver {}impl CompanyMetadataResolver for CrmMetadataResolver {- async fn resolve(&self, domain: &str) -> DomainMetadata {+ async fn resolve(&self, domain: &str) -> DomainResolution {match self {CrmMetadataResolver::Apollo(r) => r.resolve(domain).await,CrmMetadataResolver::Unfurl(r) => r.resolve(domain).await,@@ -54,8 +53,8 @@ impl CompanyMetadataResolver for CrmMetadataResolver {/// The concrete CRM service type, backed by Postgres and the/// flag-selected [`CrmMetadataResolver`]. The resolver is consulted only-/// on `crm_domain_directory` misses, so it isn't surfaced separately on-/// [`PubSubContext`].+/// on `crm_domain_directory` misses and for pending entries, so it isn't+/// surfaced separately on [`PubSubContext`].pub type CrmServiceType = CrmServiceImpl<CompaniesRepositoryImpl, CrmMetadataResolver>;#[derive(Clone)]