Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(crm): keep unresolved company domains pending and enrich them later (#8449)

gbirmandcd643778eef
diff --git a/.sqlx/query-334c6ec59d818a2c7251d13e1b5b04a943f677e749735806a7e137743c5afc7c.json b/.sqlx/query-334c6ec59d818a2c7251d13e1b5b04a943f677e749735806a7e137743c5afc7c.json
new file mode 100644
index 0000000000..6decab7d71
--- /dev/null
+++ b/.sqlx/query-334c6ec59d818a2c7251d13e1b5b04a943f677e749735806a7e137743c5afc7c.json
@@ -0,0 +1,43 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO crm_domain_directory (\n domain, name, description, icon_url,\n apollo_organization_id, website_url, linkedin_url, twitter_url,\n facebook_url, industry, keywords, technologies,\n estimated_num_employees, annual_revenue, annual_revenue_printed,\n total_funding, total_funding_printed, latest_funding_stage,\n latest_funding_round_date, founded_year, publicly_traded_symbol,\n publicly_traded_exchange, phone, raw_address, street_address,\n city, state, postal_code, country, raw, enriched_at\n )\n VALUES (\n $1, $2, $3, $4,\n $5, $6, $7, $8,\n $9, $10, $11, $12,\n $13, $14, $15,\n $16, $17, $18,\n $19, $20, $21,\n $22, $23, $24, $25,\n $26, $27, $28, $29, $30, now()\n )\n ON CONFLICT (LOWER(domain)) DO UPDATE SET\n name = EXCLUDED.name,\n description = EXCLUDED.description,\n icon_url = EXCLUDED.icon_url,\n apollo_organization_id = EXCLUDED.apollo_organization_id,\n website_url = EXCLUDED.website_url,\n linkedin_url = EXCLUDED.linkedin_url,\n twitter_url = EXCLUDED.twitter_url,\n facebook_url = EXCLUDED.facebook_url,\n industry = EXCLUDED.industry,\n keywords = EXCLUDED.keywords,\n technologies = EXCLUDED.technologies,\n estimated_num_employees = EXCLUDED.estimated_num_employees,\n annual_revenue = EXCLUDED.annual_revenue,\n annual_revenue_printed = EXCLUDED.annual_revenue_printed,\n total_funding = EXCLUDED.total_funding,\n total_funding_printed = EXCLUDED.total_funding_printed,\n latest_funding_stage = EXCLUDED.latest_funding_stage,\n latest_funding_round_date = EXCLUDED.latest_funding_round_date,\n founded_year = EXCLUDED.founded_year,\n publicly_traded_symbol = EXCLUDED.publicly_traded_symbol,\n publicly_traded_exchange = EXCLUDED.publicly_traded_exchange,\n phone = EXCLUDED.phone,\n raw_address = EXCLUDED.raw_address,\n street_address = EXCLUDED.street_address,\n city = EXCLUDED.city,\n state = EXCLUDED.state,\n postal_code = EXCLUDED.postal_code,\n country = EXCLUDED.country,\n raw = EXCLUDED.raw,\n enriched_at = EXCLUDED.enriched_at\n WHERE crm_domain_directory.enriched_at IS NULL\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "TextArray",
+ "TextArray",
+ "Int4",
+ "Int8",
+ "Text",
+ "Int8",
+ "Text",
+ "Text",
+ "Timestamptz",
+ "Int4",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Jsonb"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "334c6ec59d818a2c7251d13e1b5b04a943f677e749735806a7e137743c5afc7c"
+}
diff --git a/.sqlx/query-81e64debfcca3111e88cbdce997a9eb11fb0d1c66828a4361f7de63f9dffa91b.json b/.sqlx/query-81e64debfcca3111e88cbdce997a9eb11fb0d1c66828a4361f7de63f9dffa91b.json
new file mode 100644
index 0000000000..e022d1c684
--- /dev/null
+++ b/.sqlx/query-81e64debfcca3111e88cbdce997a9eb11fb0d1c66828a4361f7de63f9dffa91b.json
@@ -0,0 +1,24 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH candidate AS (\n SELECT dd.id\n FROM crm_domain_directory dd\n WHERE dd.enriched_at IS NULL\n AND dd.enrichment_attempts < $2\n AND (dd.enrichment_attempted_at IS NULL OR dd.enrichment_attempted_at < $3)\n AND EXISTS (SELECT 1 FROM crm_domains d WHERE d.domain = dd.domain)\n ORDER BY dd.enrichment_attempted_at NULLS FIRST, dd.created_at\n LIMIT $1\n FOR UPDATE OF dd SKIP LOCKED\n )\n UPDATE crm_domain_directory dd\n SET enrichment_attempts = dd.enrichment_attempts + 1,\n enrichment_attempted_at = now()\n FROM candidate c\n WHERE dd.id = c.id\n RETURNING dd.domain\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "domain",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Int2",
+ "Timestamptz"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "81e64debfcca3111e88cbdce997a9eb11fb0d1c66828a4361f7de63f9dffa91b"
+}
diff --git a/.sqlx/query-c4998096a67401510c7d65a72a503c6df9b9c79085da41b4569854f81f6444dc.json b/.sqlx/query-c4998096a67401510c7d65a72a503c6df9b9c79085da41b4569854f81f6444dc.json
new file mode 100644
index 0000000000..26288f385c
--- /dev/null
+++ b/.sqlx/query-c4998096a67401510c7d65a72a503c6df9b9c79085da41b4569854f81f6444dc.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO crm_domain_directory (domain)\n VALUES ($1)\n ON CONFLICT (LOWER(domain)) DO NOTHING\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "c4998096a67401510c7d65a72a503c6df9b9c79085da41b4569854f81f6444dc"
+}
diff --git a/.sqlx/query-fa0410f4a8b799f01b4a582582d0baafac16a1058e7017d63813abf87b340e4a.json b/.sqlx/query-fa0410f4a8b799f01b4a582582d0baafac16a1058e7017d63813abf87b340e4a.json
deleted file mode 100644
index 1913a02577..0000000000
--- a/.sqlx/query-fa0410f4a8b799f01b4a582582d0baafac16a1058e7017d63813abf87b340e4a.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n INSERT INTO crm_domain_directory (\n domain, name, description, icon_url,\n apollo_organization_id, website_url, linkedin_url, twitter_url,\n facebook_url, industry, keywords, technologies,\n estimated_num_employees, annual_revenue, annual_revenue_printed,\n total_funding, total_funding_printed, latest_funding_stage,\n latest_funding_round_date, founded_year, publicly_traded_symbol,\n publicly_traded_exchange, phone, raw_address, street_address,\n city, state, postal_code, country, raw, enriched_at\n )\n VALUES (\n $1, $2, $3, $4,\n $5, $6, $7, $8,\n $9, $10, $11, $12,\n $13, $14, $15,\n $16, $17, $18,\n $19, $20, $21,\n $22, $23, $24, $25,\n $26, $27, $28, $29, $30, now()\n )\n ON CONFLICT (LOWER(domain)) DO NOTHING\n ",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "TextArray",
- "TextArray",
- "Int4",
- "Int8",
- "Text",
- "Int8",
- "Text",
- "Text",
- "Timestamptz",
- "Int4",
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "Text",
- "Jsonb"
- ]
- },
- "nullable": []
- },
- "hash": "fa0410f4a8b799f01b4a582582d0baafac16a1058e7017d63813abf87b340e4a"
-}
diff --git a/Cargo.lock b/Cargo.lock
index f1364fb425..6bdf84b9b6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5256,6 +5256,7 @@ dependencies = [
"unfurl",
"utoipa",
"uuid",
+ "wiremock",
"workspace-hack",
]
diff --git a/crates/crm/AGENTS.md b/crates/crm/AGENTS.md
index 3a5cee903f..37ab297d4c 100644
--- a/crates/crm/AGENTS.md
+++ b/crates/crm/AGENTS.md
@@ -57,6 +57,12 @@ tools/HTTP/GraphQL surfaces over them.
## Gotchas
+- Company names/logos live in the global `crm_domain_directory`. Only the
+ email pubsub workers carry a real `CompanyMetadataResolver`; every other
+ binary uses `NoOpCompanyMetadataResolver`, which leaves a pending entry
+ (NULL `enriched_at`) that `CrmServiceImpl::enrich_pending_domains` resolves
+ later with bounded retries. Never store an empty entry for a domain no
+ resolver answered for: resolved entries are permanent.
- The Sales migration (`SALES_BACKFILL.md`) is forward-only and additive —
its down migration refuses to erase completion or user edits; never revert
it to undo a bad migration, only the application code that reads it.
diff --git a/crates/crm/Cargo.toml b/crates/crm/Cargo.toml
index 4bcfe50e63..1f94c0062d 100644
--- a/crates/crm/Cargo.toml
+++ b/crates/crm/Cargo.toml
@@ -125,3 +125,4 @@ properties = { path = "../properties", default-features = false, features = [
rootcause = { workspace = true }
tokio = { workspace = true, features = ["macros"] }
tower = { workspace = true }
+wiremock = { workspace = true }
diff --git a/crates/crm/src/domain/companies_repo.rs b/crates/crm/src/domain/companies_repo.rs
index c789c8ee47..c3805535a4 100644
--- a/crates/crm/src/domain/companies_repo.rs
+++ b/crates/crm/src/domain/companies_repo.rs
@@ -241,24 +241,26 @@ pub trait CompaniesRepository: Clone + Send + Sync + 'static {
/// case-insensitively. Returns `Ok(None)` when no row exists for
/// the domain — the caller is expected to resolve via
/// [`crate::domain::company_metadata_resolver::CompanyMetadataResolver`]
- /// and then [`upsert_domain_metadata`] before retrying.
+ /// and then [`upsert_domain_metadata`] (or, when the resolver is
+ /// unavailable, [`insert_pending_domain`]) before retrying.
///
/// `Some(DomainMetadata { name: None, ... })` is distinct from
- /// `None`: it means the domain has been looked up before and the
- /// resolver returned nothing useful — the negative-cache entry
- /// suppresses further resolver calls.
+ /// `None`: the domain already has an entry, either resolved to
+ /// nothing useful (the negative cache) or still pending for the
+ /// enrichment worker. Either way callers don't resolve it inline.
///
/// [`upsert_domain_metadata`]: CompaniesRepository::upsert_domain_metadata
+ /// [`insert_pending_domain`]: CompaniesRepository::insert_pending_domain
fn lookup_domain_metadata(
&self,
domain: &str,
) -> impl Future<Output = Result<Option<DomainMetadata>, CrmError>> + Send;
- /// Insert `metadata` for `domain` into `crm_domain_directory` with
- /// `ON CONFLICT (LOWER(domain)) DO NOTHING`. The directory is a
- /// global, first-write-wins cache: a row for `domain` (whether
- /// populated or all-NULL) is preserved as-is for the lifetime of
- /// the table. `domain` is lower-cased before storage.
+ /// Store resolved `metadata` for `domain` in `crm_domain_directory`
+ /// and stamp it resolved. The directory is a global cache: a
+ /// resolved row (whether populated or all-NULL) is first-write-wins
+ /// and preserved as-is; a pending row is filled in. `domain` is
+ /// lower-cased before storage.
///
/// Idempotent under concurrent calls — racing producers can both
/// resolve the same domain and both call this method; the second
@@ -269,6 +271,30 @@ pub trait CompaniesRepository: Clone + Send + Sync + 'static {
metadata: &DomainMetadata,
) -> impl Future<Output = Result<(), CrmError>> + Send;
+ /// Insert a pending `crm_domain_directory` row for `domain` (no
+ /// metadata, not resolved) unless the domain already has a row.
+ /// Read paths treat it like a negative-cache row; the enrichment
+ /// worker claims it through [`claim_pending_domains`].
+ ///
+ /// [`claim_pending_domains`]: CompaniesRepository::claim_pending_domains
+ fn insert_pending_domain(
+ &self,
+ domain: &str,
+ ) -> impl Future<Output = Result<(), CrmError>> + Send;
+
+ /// Claim up to `limit` pending `crm_domain_directory` rows whose
+ /// domain some team still tracks, with fewer than `max_attempts`
+ /// enrichment attempts and no attempt since `attempted_before`.
+ /// Claiming counts an attempt and stamps it now, so concurrent
+ /// workers skip the claimed rows until `attempted_before` passes
+ /// them. Returns the claimed domains.
+ fn claim_pending_domains(
+ &self,
+ limit: i64,
+ max_attempts: i16,
+ attempted_before: DateTime<Utc>,
+ ) -> impl Future<Output = Result<Vec<String>, CrmError>> + Send;
+
/// Reverses [`populate_contact`] for one `(link_id, email)`: drops the
/// matching `crm_contact_sources` row, then `crm_contacts` if no other
/// source rows remain for that contact, then `crm_companies` (cascading
diff --git a/crates/crm/src/domain/company_metadata_resolver.rs b/crates/crm/src/domain/company_metadata_resolver.rs
index e75f6937bc..eaf8297573 100644
--- a/crates/crm/src/domain/company_metadata_resolver.rs
+++ b/crates/crm/src/domain/company_metadata_resolver.rs
@@ -11,16 +11,24 @@
use crate::domain::model::DomainMetadata;
+/// Outcome of resolving one domain.
+#[derive(Debug, Clone, PartialEq)]
+pub enum DomainResolution {
+ /// The provider answered. Stored in `crm_domain_directory` as the
+ /// domain's permanent entry, even when every field is empty (the
+ /// provider knows nothing about the domain — the negative cache).
+ Resolved(Box<DomainMetadata>),
+ /// No answer this time: the binary has no resolver, the provider is
+ /// not configured, or the call failed. The directory entry stays
+ /// pending so the enrichment worker retries it.
+ Unavailable,
+}
+
/// Resolves company metadata for a single email domain.
///
-/// Implementations are expected to be best-effort: a missing page,
-/// network timeout, or malformed metadata should be surfaced as a
-/// [`DomainMetadata`] with all-NULL fields rather than an error, since
-/// the caller writes the result into `crm_domain_directory` as a
-/// negative cache so the domain isn't re-resolved on the next populate.
+/// Implementations are best-effort and never error: a failed call is
+/// [`DomainResolution::Unavailable`], logged by the implementation.
pub trait CompanyMetadataResolver: Clone + Send + Sync + 'static {
- /// Resolve metadata for the given lower-cased `domain`. Returns
- /// [`DomainMetadata::default`] (all fields `None`) on any failure;
- /// implementations are expected to log internally.
- fn resolve(&self, domain: &str) -> impl Future<Output = DomainMetadata> + Send;
+ /// Resolve metadata for the given lower-cased `domain`.
+ fn resolve(&self, domain: &str) -> impl Future<Output = DomainResolution> + Send;
}
diff --git a/crates/crm/src/domain/service.rs b/crates/crm/src/domain/service.rs
index 53e76c2026..48f8f1bf98 100644
--- a/crates/crm/src/domain/service.rs
+++ b/crates/crm/src/domain/service.rs
@@ -6,14 +6,14 @@ mod test;
use crate::domain::{
auth::{CrmCompanyReceipt, CrmContactReceipt, CrmTeamReceipt},
companies_repo::{CompaniesRepository, CrmCompanyListSort, CrmCompanySoupCursor},
- company_metadata_resolver::CompanyMetadataResolver,
+ company_metadata_resolver::{CompanyMetadataResolver, DomainResolution},
generic_email_domains::is_generic_email_domain,
model::{
CrmCompanyForSoup, CrmCompanyWithContacts, CrmContact, CrmError, CrmScopePrecheck,
CrmTeamSettings, CrmTeamSettingsPatch, DepopulateContactOutcome,
},
};
-use chrono::{DateTime, Utc};
+use chrono::{DateTime, TimeDelta, Utc};
use entity_access::domain::models::{EditAccessLevel, MemberTeamRole, ViewAccessLevel};
/// The CrmService exposes operations over CRM records (companies, their
@@ -65,10 +65,12 @@ pub trait CrmService: Clone + Send + Sync + 'static {
/// `crm_domain_directory` has an entry for the email's domain — if
/// not, it invokes the
/// [`crate::domain::company_metadata_resolver::CompanyMetadataResolver`]
- /// and inserts the result (which may be all-NULL on resolver
- /// failure — that's the negative cache). The directory write is its
- /// own transaction so the populate tx never holds locks across an
- /// HTTP fetch.
+ /// and inserts the result (which may be all-NULL when the provider
+ /// knows nothing about the domain — that's the negative cache), or a
+ /// pending entry for [`CrmServiceImpl::enrich_pending_domains`] when
+ /// the resolver is unavailable. The directory write is its own
+ /// transaction so the populate tx never holds locks across an HTTP
+ /// fetch.
///
/// `first_at` / `last_at` are the contact's known activity range
/// for this populate. Per-message paths pass the message's
@@ -185,7 +187,9 @@ pub trait CrmService: Clone + Send + Sync + 'static {
/// Like populate, ensures `crm_domain_directory` has an entry for
/// the domain (resolving via the
/// [`crate::domain::company_metadata_resolver::CompanyMetadataResolver`]
- /// on a miss) so the company still picks up icon/description
+ /// on a miss, or leaving it pending for
+ /// [`CrmServiceImpl::enrich_pending_domains`] when the binary has no
+ /// resolver) so the company still picks up icon/description
/// enrichment; the user's `name` is stored on `crm_companies.custom_name`
/// (never in the shared directory — it's global across teams) and
/// overrides the directory name on every read path.
@@ -516,9 +520,10 @@ where
{
/// The underlying companies repository
companies_repository: CR,
- /// Resolver consulted only when `crm_domain_directory` has no entry
- /// for a given domain. The resolver itself is best-effort — its
- /// failures collapse to a negative-cache row in the directory.
+ /// Resolver consulted when `crm_domain_directory` has no entry for a
+ /// given domain, and for pending entries in
+ /// [`CrmServiceImpl::enrich_pending_domains`]. The resolver itself is
+ /// best-effort — when it is unavailable the entry stays pending.
metadata_resolver: R,
}
@@ -547,8 +552,101 @@ where
metadata_resolver,
}
}
+
+ /// Resolves one batch of pending `crm_domain_directory` entries
+ /// whose domain some team still tracks, and returns how many were
+ /// claimed. Each entry gets at most [`MAX_ENRICHMENT_ATTEMPTS`]
+ /// attempts, spaced [`ENRICHMENT_RETRY_AFTER`] apart; an entry the
+ /// resolver keeps failing on stays pending (read paths show the
+ /// domain) and is not claimed again. Team-scoped
+ /// `crm_companies.custom_name` overrides are never touched — they win
+ /// over the directory name on every read path.
+ ///
+ /// Only meaningful with a real resolver: with
+ /// [`crate::outbound::no_op_resolver::NoOpCompanyMetadataResolver`]
+ /// every claim would burn an attempt.
+ #[tracing::instrument(
+ name = "crm.enrich_pending_domains",
+ skip(self),
+ fields(claimed, resolved),
+ err
+ )]
+ pub async fn enrich_pending_domains(&self) -> Result<usize, CrmError> {
+ let domains = self
+ .companies_repository
+ .claim_pending_domains(
+ ENRICHMENT_BATCH_SIZE,
+ MAX_ENRICHMENT_ATTEMPTS,
+ Utc::now() - ENRICHMENT_RETRY_AFTER,
+ )
+ .await?;
+
+ let mut resolved = 0;
+ for domain in &domains {
+ let DomainResolution::Resolved(metadata) = self.metadata_resolver.resolve(domain).await
+ else {
+ continue;
+ };
+ // One failed write leaves that entry pending for a later
+ // attempt; it must not drop the rest of the batch.
+ if self
+ .companies_repository
+ .upsert_domain_metadata(domain, &metadata)
+ .await
+ .inspect_err(|error| {
+ tracing::warn!(?error, domain, "failed to store enriched domain metadata")
+ })
+ .is_ok()
+ {
+ resolved += 1;
+ }
+ }
+
+ let span = tracing::Span::current();
+ span.record("claimed", domains.len());
+ span.record("resolved", resolved);
+ Ok(domains.len())
+ }
+
+ /// Ensures `crm_domain_directory` has an entry for `domain`: resolves
+ /// it when the domain has none, storing a pending entry instead when
+ /// the resolver is unavailable.
+ async fn ensure_domain_directory_entry(&self, domain: &str) -> Result<(), CrmError> {
+ if self
+ .companies_repository
+ .lookup_domain_metadata(domain)
+ .await?
+ .is_some()
+ {
+ return Ok(());
+ }
+ match self.metadata_resolver.resolve(domain).await {
+ DomainResolution::Resolved(metadata) => {
+ self.companies_repository
+ .upsert_domain_metadata(domain, &metadata)
+ .await
+ }
+ DomainResolution::Unavailable => {
+ self.companies_repository
+ .insert_pending_domain(domain)
+ .await
+ }
+ }
+ }
}
+/// Pending directory entries one [`CrmServiceImpl::enrich_pending_domains`]
+/// pass claims. Passes run on a fixed interval, so this caps the resolver
+/// (Apollo) call rate.
+const ENRICHMENT_BATCH_SIZE: i64 = 10;
+
+/// Enrichment attempts after which a pending directory entry is left alone.
+const MAX_ENRICHMENT_ATTEMPTS: i16 = 5;
+
+/// Minimum gap between enrichment attempts for one directory entry; also
+/// how long a claim by a worker that died mid-batch blocks other workers.
+const ENRICHMENT_RETRY_AFTER: TimeDelta = TimeDelta::hours(6);
+
impl<CR, R> CrmService for CrmServiceImpl<CR, R>
where
CR: CompaniesRepository,
@@ -627,17 +725,8 @@ where
// directory entry was written when its first sent message
// populated; if it isn't tracked, this call will no-op in the
// repo anyway.
- if is_sent
- && self
- .companies_repository
- .lookup_domain_metadata(domain)
- .await?
- .is_none()
- {
- let metadata = self.metadata_resolver.resolve(domain).await;
- self.companies_repository
- .upsert_domain_metadata(domain, &metadata)
- .await?;
+ if is_sent {
+ self.ensure_domain_directory_entry(domain).await?;
}
self.companies_repository
@@ -725,17 +814,7 @@ where
// domain so icon/description enrichment resolves. The user's
// name never goes here — the directory is global across teams;
// it lands on `crm_companies.custom_name` in the repo instead.
- if self
- .companies_repository
- .lookup_domain_metadata(&domain)
- .await?
- .is_none()
- {
- let metadata = self.metadata_resolver.resolve(&domain).await;
- self.companies_repository
- .upsert_domain_metadata(&domain, &metadata)
- .await?;
- }
+ self.ensure_domain_directory_entry(&domain).await?;
self.companies_repository
.create_company_for_team(&access.team_id(), &domain, name, Utc::now())
diff --git a/crates/crm/src/domain/service/test.rs b/crates/crm/src/domain/service/test.rs
index 7e5f7a8efc..ae86b89009 100644
--- a/crates/crm/src/domain/service/test.rs
+++ b/crates/crm/src/domain/service/test.rs
@@ -120,6 +120,19 @@ impl CompaniesRepository for StubRepo {
unimplemented!()
}
+ async fn insert_pending_domain(&self, _domain: &str) -> Result<(), CrmError> {
+ unimplemented!()
+ }
+
+ async fn claim_pending_domains(
+ &self,
+ _limit: i64,
+ _max_attempts: i16,
+ _attempted_before: DateTime<Utc>,
+ ) -> Result<Vec<String>, CrmError> {
+ unimplemented!()
+ }
+
async fn depopulate_contact(
&self,
_team_id: &uuid::Uuid,
diff --git a/crates/crm/src/outbound/apollo_resolver.rs b/crates/crm/src/outbound/apollo_resolver.rs
index 7bf0da6e1f..b4b6dec456 100644
--- a/crates/crm/src/outbound/apollo_resolver.rs
+++ b/crates/crm/src/outbound/apollo_resolver.rs
@@ -6,10 +6,12 @@
//! onto [`DomainMetadata`]. The full payload — minus our own Apollo
//! workspace `account` record — is preserved in [`DomainMetadata::raw`].
//!
-//! Best-effort: any failure (network error, non-2xx status, malformed
-//! body, missing `organization`) is logged and surfaced as
-//! [`DomainMetadata::default`] so the caller can persist a negative-cache
-//! row and avoid re-resolving the domain.
+//! Best-effort: a 2xx response without an `organization` (Apollo knows
+//! nothing about the domain) resolves to [`DomainMetadata::default`], the
+//! negative cache. A missing API key, network error, non-2xx status
+//! (including rate limits), or malformed body is logged and surfaced as
+//! [`DomainResolution::Unavailable`], so the domain is retried instead of
+//! cached empty.
#[cfg(test)]
mod test;
@@ -19,7 +21,10 @@ use std::time::Duration;
use chrono::{DateTime, Utc};
use serde_json::Value;
-use crate::domain::{company_metadata_resolver::CompanyMetadataResolver, model::DomainMetadata};
+use crate::domain::{
+ company_metadata_resolver::{CompanyMetadataResolver, DomainResolution},
+ model::DomainMetadata,
+};
/// Default Apollo API base URL.
const DEFAULT_BASE_URL: &str = "https://api.apollo.io";
@@ -68,12 +73,12 @@ impl ApolloCompanyMetadataResolver {
impl CompanyMetadataResolver for ApolloCompanyMetadataResolver {
#[tracing::instrument(skip(self))]
- async fn resolve(&self, domain: &str) -> DomainMetadata {
+ async fn resolve(&self, domain: &str) -> DomainResolution {
// No key configured (e.g. local without APOLLO_API_KEY): skip the
- // call entirely rather than burning a 401 and caching an empty row.
+ // call entirely rather than burning a 401.
if self.api_key.is_empty() {
tracing::debug!(domain, "apollo enrich: no api key configured; skipping");
- return DomainMetadata::default();
+ return DomainResolution::Unavailable;
}
let url = format!("{}/api/v1/organizations/enrich", self.base_url);
@@ -91,25 +96,25 @@ impl CompanyMetadataResolver for ApolloCompanyMetadataResolver {
Ok(body) => body,
Err(e) => {
tracing::warn!(error=?e, domain, "apollo enrich: failed to parse body");
- return DomainMetadata::default();
+ return DomainResolution::Unavailable;
}
},
Ok(resp) => {
tracing::warn!(status = %resp.status(), domain, "apollo enrich: non-success status");
- return DomainMetadata::default();
+ return DomainResolution::Unavailable;
}
Err(e) => {
tracing::warn!(error=?e, domain, "apollo enrich: request failed");
- return DomainMetadata::default();
+ return DomainResolution::Unavailable;
}
};
let Some(organization) = body.get("organization") else {
tracing::warn!(domain, "apollo enrich: response had no organization");
- return DomainMetadata::default();
+ return DomainResolution::Resolved(Box::default());
};
- map_organization(organization)
+ DomainResolution::Resolved(Box::new(map_organization(organization)))
}
}
diff --git a/crates/crm/src/outbound/apollo_resolver/test.rs b/crates/crm/src/outbound/apollo_resolver/test.rs
index 03eb0fefa2..f1758dddbb 100644
--- a/crates/crm/src/outbound/apollo_resolver/test.rs
+++ b/crates/crm/src/outbound/apollo_resolver/test.rs
@@ -1,6 +1,8 @@
use super::*;
use crate::domain::company_metadata_resolver::CompanyMetadataResolver;
use serde_json::json;
+use wiremock::matchers::{method, path, query_param};
+use wiremock::{Mock, MockServer, ResponseTemplate};
/// The documented Apollo sample payload maps onto every field, and the
/// `account` / `account_id` keys are stripped from `raw`.
@@ -115,6 +117,54 @@ fn tolerates_mistyped_and_blank_fields() {
);
}
+/// Failures that say nothing about the domain stay retryable; only an
+/// answer from Apollo becomes a (possibly empty) directory entry.
+#[tokio::test]
+async fn only_apollo_answers_resolve() {
+ let server = MockServer::start().await;
+ Mock::given(method("GET"))
+ .and(path("/api/v1/organizations/enrich"))
+ .and(query_param("domain", "unknown.com"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({})))
+ .mount(&server)
+ .await;
+ Mock::given(method("GET"))
+ .and(path("/api/v1/organizations/enrich"))
+ .and(query_param("domain", "acme.com"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .set_body_json(json!({ "organization": { "name": "Acme Inc." } })),
+ )
+ .mount(&server)
+ .await;
+ Mock::given(method("GET"))
+ .and(path("/api/v1/organizations/enrich"))
+ .and(query_param("domain", "busy.com"))
+ .respond_with(ResponseTemplate::new(429))
+ .mount(&server)
+ .await;
+ let resolver = ApolloCompanyMetadataResolver::with_base_url("key".to_string(), server.uri());
+
+ assert_eq!(
+ resolver.resolve("unknown.com").await,
+ DomainResolution::Resolved(Box::default())
+ );
+ let DomainResolution::Resolved(acme) = resolver.resolve("acme.com").await else {
+ panic!("expected acme.com to resolve");
+ };
+ assert_eq!(acme.name.as_deref(), Some("Acme Inc."));
+ assert_eq!(
+ resolver.resolve("busy.com").await,
+ DomainResolution::Unavailable
+ );
+ assert_eq!(
+ ApolloCompanyMetadataResolver::with_base_url(String::new(), server.uri())
+ .resolve("acme.com")
+ .await,
+ DomainResolution::Unavailable
+ );
+}
+
/// Live smoke test against the real Apollo API. Ignored by default — it
/// makes a network call and consumes an Apollo credit. Run with:
/// APOLLO_API_KEY=<key> cargo test -p crm apollo_live -- --ignored --nocapture
@@ -126,7 +176,9 @@ async fn apollo_live_enrich() {
let domain = std::env::var("APOLLO_TEST_DOMAIN").unwrap_or_else(|_| "macro.com".to_string());
let resolver = ApolloCompanyMetadataResolver::new(api_key);
- let md = resolver.resolve(&domain).await;
+ let DomainResolution::Resolved(md) = resolver.resolve(&domain).await else {
+ panic!("Apollo was unavailable for {domain} — check API key and credits");
+ };
println!("DomainMetadata for {domain}:\n{md:#?}");
assert!(
diff --git a/crates/crm/src/outbound/companies_repo.rs b/crates/crm/src/outbound/companies_repo.rs
index 7f0144a2ed..661655301c 100644
--- a/crates/crm/src/outbound/companies_repo.rs
+++ b/crates/crm/src/outbound/companies_repo.rs
@@ -937,8 +937,9 @@ impl CompaniesRepository for CompaniesRepositoryImpl {
metadata: &DomainMetadata,
) -> Result<(), CrmError> {
let normalized_domain = domain.to_ascii_lowercase();
- // First-write-wins. All-empty rows are negative-cache entries that
- // suppress future resolver calls; `enriched_at` stamps resolve time.
+ // First-write-wins for resolved rows; a pending row (NULL
+ // `enriched_at`) is filled in. All-empty resolved rows are
+ // negative-cache entries that suppress future resolver calls.
sqlx::query!(
r#"
INSERT INTO crm_domain_directory (
@@ -961,7 +962,38 @@ impl CompaniesRepository for CompaniesRepositoryImpl {
$22, $23, $24, $25,
$26, $27, $28, $29, $30, now()
)
- ON CONFLICT (LOWER(domain)) DO NOTHING
+ ON CONFLICT (LOWER(domain)) DO UPDATE SET
+ name = EXCLUDED.name,
+ description = EXCLUDED.description,
+ icon_url = EXCLUDED.icon_url,
+ apollo_organization_id = EXCLUDED.apollo_organization_id,
+ website_url = EXCLUDED.website_url,
+ linkedin_url = EXCLUDED.linkedin_url,
+ twitter_url = EXCLUDED.twitter_url,
+ facebook_url = EXCLUDED.facebook_url,
+ industry = EXCLUDED.industry,
+ keywords = EXCLUDED.keywords,
+ technologies = EXCLUDED.technologies,
+ estimated_num_employees = EXCLUDED.estimated_num_employees,
+ annual_revenue = EXCLUDED.annual_revenue,
+ annual_revenue_printed = EXCLUDED.annual_revenue_printed,
+ total_funding = EXCLUDED.total_funding,
+ total_funding_printed = EXCLUDED.total_funding_printed,
+ latest_funding_stage = EXCLUDED.latest_funding_stage,
+ latest_funding_round_date = EXCLUDED.latest_funding_round_date,
+ founded_year = EXCLUDED.founded_year,
+ publicly_traded_symbol = EXCLUDED.publicly_traded_symbol,
+ publicly_traded_exchange = EXCLUDED.publicly_traded_exchange,
+ phone = EXCLUDED.phone,
+ raw_address = EXCLUDED.raw_address,
+ street_address = EXCLUDED.street_address,
+ city = EXCLUDED.city,
+ state = EXCLUDED.state,
+ postal_code = EXCLUDED.postal_code,
+ country = EXCLUDED.country,
+ raw = EXCLUDED.raw,
+ enriched_at = EXCLUDED.enriched_at
+ WHERE crm_domain_directory.enriched_at IS NULL
"#,
normalized_domain,
metadata.name,
@@ -1001,6 +1033,61 @@ impl CompaniesRepository for CompaniesRepositoryImpl {
Ok(())
}
+ #[tracing::instrument(skip(self), err)]
+ async fn insert_pending_domain(&self, domain: &str) -> Result<(), CrmError> {
+ sqlx::query!(
+ r#"
+ INSERT INTO crm_domain_directory (domain)
+ VALUES ($1)
+ ON CONFLICT (LOWER(domain)) DO NOTHING
+ "#,
+ domain.to_ascii_lowercase(),
+ )
+ .execute(&self.pool)
+ .await
+ .map_err(|e| CrmError::StorageLayerError(e.into()))?;
+
+ Ok(())
+ }
+
+ #[tracing::instrument(skip(self), err)]
+ async fn claim_pending_domains(
+ &self,
+ limit: i64,
+ max_attempts: i16,
+ attempted_before: DateTime<Utc>,
+ ) -> Result<Vec<String>, CrmError> {
+ // `crm_domains.domain` and directory domains are both stored
+ // lower-cased, so plain equality uses `crm_domains_domain_idx`.
+ sqlx::query_scalar!(
+ r#"
+ WITH candidate AS (
+ SELECT dd.id
+ FROM crm_domain_directory dd
+ WHERE dd.enriched_at IS NULL
+ AND dd.enrichment_attempts < $2
+ AND (dd.enrichment_attempted_at IS NULL OR dd.enrichment_attempted_at < $3)
+ AND EXISTS (SELECT 1 FROM crm_domains d WHERE d.domain = dd.domain)
+ ORDER BY dd.enrichment_attempted_at NULLS FIRST, dd.created_at
+ LIMIT $1
+ FOR UPDATE OF dd SKIP LOCKED
+ )
+ UPDATE crm_domain_directory dd
+ SET enrichment_attempts = dd.enrichment_attempts + 1,
+ enrichment_attempted_at = now()
+ FROM candidate c
+ WHERE dd.id = c.id
+ RETURNING dd.domain
+ "#,
+ limit,
+ max_attempts,
+ attempted_before,
+ )
+ .fetch_all(&self.pool)
+ .await
+ .map_err(|e| CrmError::StorageLayerError(e.into()))
+ }
+
#[tracing::instrument(skip(self), err)]
async fn set_email_sync(
&self,
diff --git a/crates/crm/src/outbound/companies_repo/test.rs b/crates/crm/src/outbound/companies_repo/test.rs
index 2cf612b1b8..502bc39b5c 100644
--- a/crates/crm/src/outbound/companies_repo/test.rs
+++ b/crates/crm/src/outbound/companies_repo/test.rs
@@ -8,6 +8,7 @@ pub(crate) mod helpers;
mod create_company;
mod create_contact;
+mod domain_directory;
mod get_company_for_team;
mod get_contact_by_email_for_team;
mod get_contact_for_team;
diff --git a/crates/crm/src/outbound/companies_repo/test/domain_directory.rs b/crates/crm/src/outbound/companies_repo/test/domain_directory.rs
new file mode 100644
index 0000000000..88839d7f38
--- /dev/null
+++ b/crates/crm/src/outbound/companies_repo/test/domain_directory.rs
@@ -0,0 +1,189 @@
+use super::helpers::*;
+use crate::domain::auth::CrmTeamReceipt;
+use crate::domain::companies_repo::CompaniesRepository;
+use crate::domain::company_metadata_resolver::{CompanyMetadataResolver, DomainResolution};
+use crate::domain::model::DomainMetadata;
+use crate::domain::service::{CrmService, CrmServiceImpl};
+use crate::outbound::companies_repo::CompaniesRepositoryImpl;
+use crate::outbound::no_op_resolver::NoOpCompanyMetadataResolver;
+use chrono::{DateTime, TimeDelta, Utc};
+use entity_access::domain::models::MemberTeamRole;
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+#[derive(Clone)]
+struct FixedResolver(DomainResolution);
+
+impl CompanyMetadataResolver for FixedResolver {
+ async fn resolve(&self, _domain: &str) -> DomainResolution {
+ self.0.clone()
+ }
+}
+
+/// `(name, icon_url, enriched_at, enrichment_attempts)` of the domain's
+/// directory row.
+type DirectoryRow = (Option<String>, Option<String>, Option<DateTime<Utc>>, i16);
+
+async fn fetch_directory(pool: &PgPool, domain: &str) -> sqlx::Result<Option<DirectoryRow>> {
+ sqlx::query_as(
+ r#"SELECT name, icon_url, enriched_at, enrichment_attempts
+ FROM crm_domain_directory WHERE LOWER(domain) = LOWER($1)"#,
+ )
+ .bind(domain)
+ .fetch_optional(pool)
+ .await
+}
+
+fn acme_metadata() -> DomainMetadata {
+ DomainMetadata {
+ name: Some("Acme Inc.".to_string()),
+ icon_url: Some("https://acme.com/logo.png".to_string()),
+ ..Default::default()
+ }
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn upsert_fills_pending_entry_but_keeps_resolved_entry(pool: PgPool) -> anyhow::Result<()> {
+ let repo = CompaniesRepositoryImpl::new(pool.clone());
+
+ repo.insert_pending_domain("Acme.com").await?;
+ let (name, _, enriched_at, _) = fetch_directory(&pool, "acme.com").await?.unwrap();
+ assert_eq!(name, None);
+ assert_eq!(enriched_at, None);
+
+ repo.upsert_domain_metadata("acme.com", &acme_metadata())
+ .await?;
+ let (name, icon_url, enriched_at, _) = fetch_directory(&pool, "acme.com").await?.unwrap();
+ assert_eq!(name.as_deref(), Some("Acme Inc."));
+ assert_eq!(icon_url.as_deref(), Some("https://acme.com/logo.png"));
+ assert!(enriched_at.is_some());
+
+ // A resolved entry is first-write-wins: neither a later resolve nor a
+ // pending insert replaces it.
+ repo.upsert_domain_metadata(
+ "acme.com",
+ &DomainMetadata {
+ name: Some("Other".to_string()),
+ ..Default::default()
+ },
+ )
+ .await?;
+ repo.insert_pending_domain("acme.com").await?;
+ let (name, _, enriched_at, _) = fetch_directory(&pool, "acme.com").await?.unwrap();
+ assert_eq!(name.as_deref(), Some("Acme Inc."));
+ assert!(enriched_at.is_some());
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn claim_pending_domains_is_bounded(pool: PgPool) -> anyhow::Result<()> {
+ let team_id = Uuid::now_v7();
+ seed_team(&pool, team_id, "macro|[email protected]").await?;
+ insert_company(
+ &pool,
+ team_id,
+ true,
+ &["acme.com", "globex.com", "exhausted.com"],
+ )
+ .await?;
+
+ let repo = CompaniesRepositoryImpl::new(pool.clone());
+ repo.insert_pending_domain("acme.com").await?;
+ // Pending, but no team tracks the domain any more.
+ repo.insert_pending_domain("orphan.com").await?;
+ repo.upsert_domain_metadata("globex.com", &DomainMetadata::default())
+ .await?;
+ repo.insert_pending_domain("exhausted.com").await?;
+ sqlx::query(
+ "UPDATE crm_domain_directory SET enrichment_attempts = 5 WHERE domain = 'exhausted.com'",
+ )
+ .execute(&pool)
+ .await?;
+
+ let retry_cutoff = Utc::now() - TimeDelta::hours(6);
+ let claimed = repo.claim_pending_domains(10, 5, retry_cutoff).await?;
+ assert_eq!(claimed, vec!["acme.com".to_string()]);
+ let (_, _, _, attempts) = fetch_directory(&pool, "acme.com").await?.unwrap();
+ assert_eq!(attempts, 1);
+
+ // The fresh attempt holds the entry until the retry gap passes.
+ assert!(
+ repo.claim_pending_domains(10, 5, retry_cutoff)
+ .await?
+ .is_empty()
+ );
+ let after_gap = Utc::now() + TimeDelta::seconds(1);
+ assert_eq!(
+ repo.claim_pending_domains(10, 5, after_gap).await?,
+ vec!["acme.com".to_string()]
+ );
+ let (_, _, _, attempts) = fetch_directory(&pool, "acme.com").await?.unwrap();
+ assert_eq!(attempts, 2);
+ Ok(())
+}
+
+/// A company created where no resolver is available (DSS, MCP, agent tools)
+/// leaves a pending entry instead of a permanent empty one, and the
+/// enrichment worker later fills it for every team tracking the domain —
+/// without touching a team's own name for the company.
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn manual_company_is_enriched_later_and_keeps_team_names(pool: PgPool) -> anyhow::Result<()> {
+ let team_a = Uuid::now_v7();
+ let team_b = Uuid::now_v7();
+ seed_team(&pool, team_a, "macro|[email protected]").await?;
+ seed_team(&pool, team_b, "macro|[email protected]").await?;
+ enable_crm_for_team(&pool, team_a).await?;
+ enable_crm_for_team(&pool, team_b).await?;
+
+ let manual = CrmServiceImpl::new(
+ CompaniesRepositoryImpl::new(pool.clone()),
+ NoOpCompanyMetadataResolver,
+ );
+ let renamed = manual
+ .create_company(
+ &CrmTeamReceipt::<MemberTeamRole>::dangerously_internal(team_a),
+ "Acme (ours)",
+ "acme.com",
+ )
+ .await?;
+ let unnamed = insert_company(&pool, team_b, true, &["acme.com"]).await?;
+ let (name, _, enriched_at, _) = fetch_directory(&pool, "acme.com").await?.unwrap();
+ assert_eq!(name, None);
+ assert_eq!(enriched_at, None);
+
+ let unavailable = CrmServiceImpl::new(
+ CompaniesRepositoryImpl::new(pool.clone()),
+ FixedResolver(DomainResolution::Unavailable),
+ );
+ assert_eq!(unavailable.enrich_pending_domains().await?, 1);
+ let (_, _, enriched_at, attempts) = fetch_directory(&pool, "acme.com").await?.unwrap();
+ assert_eq!(enriched_at, None);
+ assert_eq!(attempts, 1);
+
+ sqlx::query("UPDATE crm_domain_directory SET enrichment_attempted_at = NULL")
+ .execute(&pool)
+ .await?;
+ let worker = CrmServiceImpl::new(
+ CompaniesRepositoryImpl::new(pool.clone()),
+ FixedResolver(DomainResolution::Resolved(Box::new(acme_metadata()))),
+ );
+ assert_eq!(worker.enrich_pending_domains().await?, 1);
+ assert_eq!(worker.enrich_pending_domains().await?, 0);
+
+ let repo = CompaniesRepositoryImpl::new(pool.clone());
+ let team_a_company = repo
+ .get_company_for_team(&team_a, &renamed.company.id, false)
+ .await?
+ .unwrap();
+ assert_eq!(team_a_company.name.as_deref(), Some("Acme (ours)"));
+ let team_b_company = repo
+ .get_company_for_team(&team_b, &unnamed, false)
+ .await?
+ .unwrap();
+ assert_eq!(team_b_company.name.as_deref(), Some("Acme Inc."));
+ let (_, icon_url, enriched_at, _) = fetch_directory(&pool, "acme.com").await?.unwrap();
+ assert_eq!(icon_url.as_deref(), Some("https://acme.com/logo.png"));
+ assert!(enriched_at.is_some());
+ Ok(())
+}
diff --git a/crates/crm/src/outbound/no_op_resolver.rs b/crates/crm/src/outbound/no_op_resolver.rs
index d6e57abd7b..d7a418fe6f 100644
--- a/crates/crm/src/outbound/no_op_resolver.rs
+++ b/crates/crm/src/outbound/no_op_resolver.rs
@@ -1,33 +1,23 @@
-//! [`CompanyMetadataResolver`] adapter that always returns
-//! [`DomainMetadata::default`].
+//! [`CompanyMetadataResolver`] adapter that never resolves.
//!
-//! Used by binaries that need to construct a [`CrmService`] but never
-//! call [`populate_contact`] — typically HTTP API servers that only
-//! read existing CRM rows. Wiring in a real unfurl-backed resolver in
-//! those binaries would force them to carry the reqwest + scraper
-//! deps for code paths they never hit.
-//!
-//! If a binary using this resolver ever *does* call `populate_contact`,
-//! the resulting `crm_domain_directory` entries will all have NULL
-//! `name`/`description`/`icon_url` fields — i.e. the directory's
-//! negative cache fires for every domain. That's a silent loss of
-//! company metadata, not a crash, so callers should prefer wiring in
-//! the real [`UnfurlCompanyMetadataResolver`] when they participate in
-//! the populate path.
+//! Used by binaries that construct a [`CrmService`] without carrying a
+//! metadata provider — HTTP API servers, MCP, and agent tools. Wiring a
+//! real resolver into each of them would force them to carry the reqwest
+//! + scraper deps and provider credentials. Companies they create get a
+//! pending `crm_domain_directory` entry, which the email pubsub workers'
+//! enrichment worker resolves with the real resolver.
//!
//! [`CrmService`]: crate::domain::service::CrmService
-//! [`populate_contact`]: crate::domain::service::CrmService::populate_contact
-//! [`UnfurlCompanyMetadataResolver`]: crate::outbound::unfurl_resolver::UnfurlCompanyMetadataResolver
-use crate::domain::{company_metadata_resolver::CompanyMetadataResolver, model::DomainMetadata};
+use crate::domain::company_metadata_resolver::{CompanyMetadataResolver, DomainResolution};
-/// A [`CompanyMetadataResolver`] that always resolves to
-/// [`DomainMetadata::default`] (all fields `None`).
+/// A [`CompanyMetadataResolver`] that always returns
+/// [`DomainResolution::Unavailable`].
#[derive(Debug, Default, Clone, Copy)]
pub struct NoOpCompanyMetadataResolver;
impl CompanyMetadataResolver for NoOpCompanyMetadataResolver {
- async fn resolve(&self, _domain: &str) -> DomainMetadata {
- DomainMetadata::default()
+ async fn resolve(&self, _domain: &str) -> DomainResolution {
+ DomainResolution::Unavailable
}
}
diff --git a/crates/crm/src/outbound/unfurl_resolver.rs b/crates/crm/src/outbound/unfurl_resolver.rs
index 937dbaa6c4..60b54e71d0 100644
--- a/crates/crm/src/outbound/unfurl_resolver.rs
+++ b/crates/crm/src/outbound/unfurl_resolver.rs
@@ -17,12 +17,15 @@
//!
//! Failures (network errors, SSRF-blocked redirects, parse errors,
//! non-HTML responses) are logged and surfaced as
-//! [`DomainMetadata::default`] so the caller can persist a
-//! negative-cache entry and never re-resolve the domain.
+//! [`DomainResolution::Unavailable`], so the domain is retried a bounded
+//! number of times instead of cached empty on a transient error.
use std::sync::Arc;
-use crate::domain::{company_metadata_resolver::CompanyMetadataResolver, model::DomainMetadata};
+use crate::domain::{
+ company_metadata_resolver::{CompanyMetadataResolver, DomainResolution},
+ model::DomainMetadata,
+};
use unfurl::domain::ports::UnfurlService;
/// Adapter that resolves [`DomainMetadata`] by unfurling
@@ -58,7 +61,7 @@ where
U: UnfurlService,
{
#[tracing::instrument(skip(self))]
- async fn resolve(&self, domain: &str) -> DomainMetadata {
+ async fn resolve(&self, domain: &str) -> DomainResolution {
let url = format!("https://{domain}");
match self.unfurl_service.unfurl(&url).await {
Ok(response) => {
@@ -73,20 +76,16 @@ where
} else {
Some(title.to_string())
};
- DomainMetadata {
+ DomainResolution::Resolved(Box::new(DomainMetadata {
name,
description: response.description,
icon_url: response.favicon_url,
..Default::default()
- }
+ }))
}
Err(e) => {
- tracing::warn!(
- error=?e,
- domain,
- "unfurl failed; returning empty DomainMetadata for negative cache"
- );
- DomainMetadata::default()
+ tracing::warn!(error=?e, domain, "unfurl failed; domain metadata unavailable");
+ DomainResolution::Unavailable
}
}
}
diff --git a/crates/macro_db_client/migrations/20261021155020_crm_domain_directory_pending_enrichment.sql b/crates/macro_db_client/migrations/20261021155020_crm_domain_directory_pending_enrichment.sql
new file mode 100644
index 0000000000..60804306e3
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261021155020_crm_domain_directory_pending_enrichment.sql
@@ -0,0 +1,21 @@
+-- A directory entry with NULL enriched_at is pending: no resolver has answered
+-- for its domain yet. The email pubsub workers retry pending entries of
+-- tracked domains, at most a bounded number of times.
+SET LOCAL lock_timeout = '5s';
+
+ALTER TABLE crm_domain_directory
+ ADD COLUMN enrichment_attempts SMALLINT NOT NULL DEFAULT 0,
+ ADD COLUMN enrichment_attempted_at TIMESTAMPTZ;
+
+-- Entries written before enriched_at existed were resolved by unfurl.
+UPDATE crm_domain_directory
+SET enriched_at = created_at
+WHERE enriched_at IS NULL
+ AND (name IS NOT NULL OR description IS NOT NULL OR icon_url IS NOT NULL);
+
+-- Empty entries include the ones manual, API, and agent company creation
+-- wrote through the no-op resolver. Make them pending so each gets one more
+-- real resolve.
+UPDATE crm_domain_directory
+SET enriched_at = NULL
+WHERE name IS NULL AND description IS NULL AND icon_url IS NULL;
diff --git a/crates/macro_db_client/migrations/20261021170901_crm_domain_directory_pending_index.sql b/crates/macro_db_client/migrations/20261021170901_crm_domain_directory_pending_index.sql
new file mode 100644
index 0000000000..763a60c4a8
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261021170901_crm_domain_directory_pending_index.sql
@@ -0,0 +1,8 @@
+-- no-transaction
+-- Lets the CRM enrichment worker find pending directory entries. Must stay a
+-- single statement: sqlx sends no-transaction migrations with multiple
+-- statements as one batch, which wraps them in an implicit transaction that
+-- CONCURRENTLY forbids.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS crm_domain_directory_pending_idx
+ ON crm_domain_directory (enrichment_attempted_at NULLS FIRST, created_at)
+ WHERE enriched_at IS NULL;
diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.md
index 37d4be3554..6f56eded4a 100644
--- a/services/email_service/AGENTS.md
+++ b/services/email_service/AGENTS.md
@@ -16,6 +16,8 @@ the gateway `/email` prefix.
`crm_cleanup`, `worker_lifecycle`.
- `src/backfill_outbox.rs`, `src/backfill_init_service.rs`,
`src/backfill_completion_service.rs`: Gmail history backfill orchestration.
+- `src/crm_enrichment_worker.rs`: periodic pass that resolves pending CRM
+ company directory entries (names/logos) left by binaries without a resolver.
- `src/bin/`: standalone binaries — `pubsub_workers` (the queue consumer
process), plus one-off backfill/cleanup scripts.
diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
index 10558ff6a2..94c1744292 100644
--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
@@ -371,7 +371,8 @@ async fn main() -> anyhow::Result<()> {
));
// The CRM crate's company-metadata resolver is consulted by
- // `crm_service.populate_contact` only on `crm_domain_directory` misses.
+ // `crm_service.populate_contact` on `crm_domain_directory` misses and by
+ // the enrichment worker for pending entries other binaries leave behind.
// `USE_APOLLO_CRM_ENRICHMENT` selects Apollo.io vs. the unfurl-backed
// resolver; we also fall back to unfurl when the Apollo key can't be
// loaded. The resolver is cheap to clone.
@@ -409,6 +410,10 @@ async fn main() -> anyhow::Result<()> {
crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
metadata_resolver.clone(),
);
+ worker_tracker.spawn(email_service::crm_enrichment_worker::run(
+ crm_service.clone(),
+ worker_cancellation_token.clone(),
+ ));
// Backfill workers run against a dedicated pool to keep their writes
// off the primary worker pool. CRM writes are part of the backfill
diff --git a/services/email_service/src/crm_enrichment_worker.rs b/services/email_service/src/crm_enrichment_worker.rs
new file mode 100644
index 0000000000..467ab269fb
--- /dev/null
+++ b/services/email_service/src/crm_enrichment_worker.rs
@@ -0,0 +1,28 @@
+//! Resolves CRM company directory entries left pending by binaries without a
+//! metadata resolver (manual, API, and agent company creation) or by a failed
+//! resolve.
+
+use crate::pubsub::context::CrmServiceType;
+use std::time::Duration;
+use tokio_util::sync::CancellationToken;
+
+/// One enrichment pass per interval keeps the resolver (Apollo) call rate at
+/// the service's batch size per minute per worker.
+const ENRICHMENT_INTERVAL: Duration = Duration::from_secs(60);
+
+pub async fn run(service: CrmServiceType, cancellation: CancellationToken) {
+ let mut interval = tokio::time::interval(ENRICHMENT_INTERVAL);
+ loop {
+ tokio::select! {
+ biased;
+ _ = cancellation.cancelled() => return,
+ _ = interval.tick() => {},
+ }
+ // The service span records the error; the next tick retries.
+ let _ = tokio::select! {
+ biased;
+ _ = cancellation.cancelled() => return,
+ result = service.enrich_pending_domains() => result,
+ };
+ }
+}
diff --git a/services/email_service/src/lib.rs b/services/email_service/src/lib.rs
index 7d31765b44..5587d38888 100644
--- a/services/email_service/src/lib.rs
+++ b/services/email_service/src/lib.rs
@@ -18,6 +18,8 @@ pub mod calendar_request_gate;
pub mod calendar_tokens;
pub mod composition;
pub mod config;
+/// Periodic enrichment of pending CRM company directory entries.
+pub mod crm_enrichment_worker;
/// Durable provider-neutral mailbox worker scheduling.
pub mod mailbox_sync_worker;
/// Outbound infrastructure adapters for email provider capabilities.
diff --git a/services/email_service/src/pubsub/context.rs b/services/email_service/src/pubsub/context.rs
index e0f5b8c7e5..2308cf61ad 100644
--- a/services/email_service/src/pubsub/context.rs
+++ b/services/email_service/src/pubsub/context.rs
@@ -3,8 +3,7 @@ use crate::util::redis::RedisClient;
use connection_gateway_client::client::ConnectionGatewayClient;
use contacts::domain::service::SqsContactsIngress;
use contacts::outbound::ingress::SqsContactsQueue;
-use crm::domain::company_metadata_resolver::CompanyMetadataResolver;
-use crm::domain::model::DomainMetadata;
+use crm::domain::company_metadata_resolver::{CompanyMetadataResolver, DomainResolution};
use crm::domain::service::CrmServiceImpl;
use crm::outbound::apollo_resolver::ApolloCompanyMetadataResolver;
use crm::outbound::companies_repo::CompaniesRepositoryImpl;
@@ -44,7 +43,7 @@ pub enum CrmMetadataResolver {
}
impl CompanyMetadataResolver for CrmMetadataResolver {
- async fn resolve(&self, domain: &str) -> DomainMetadata {
+ async fn resolve(&self, domain: &str) -> DomainResolution {
match self {
CrmMetadataResolver::Apollo(r) => r.resolve(domain).await,
CrmMetadataResolver::Unfurl(r) => r.resolve(domain).await,
@@ -54,8 +53,8 @@ impl CompanyMetadataResolver for CrmMetadataResolver {
/// The concrete CRM service type, backed by Postgres and the
/// flag-selected [`CrmMetadataResolver`]. The resolver is consulted only
-/// on `crm_domain_directory` misses, so it isn't surfaced separately on
-/// [`PubSubContext`].
+/// on `crm_domain_directory` misses and for pending entries, so it isn't
+/// surfaced separately on [`PubSubContext`].
pub type CrmServiceType = CrmServiceImpl<CompaniesRepositoryImpl, CrmMetadataResolver>;
#[derive(Clone)]