Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

refactor(crm): remove Apollo enrichment and legacy access (#9386)

gbirmanf7008ff2c136
diff --git a/Cargo.lock b/Cargo.lock
index 758a67047a..d5e460a464 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5409,7 +5409,6 @@ dependencies = [
"models_properties",
"properties",
"rdkafka",
- "reqwest 0.13.4",
"rootcause",
"schemars 1.2.1",
"serde",
@@ -5423,7 +5422,6 @@ dependencies = [
"unfurl",
"utoipa",
"uuid",
- "wiremock",
"workspace-hack",
]
diff --git a/crates/crm/AGENTS.md b/crates/crm/AGENTS.md
index e6a98b8649..6cc27f4ec2 100644
--- a/crates/crm/AGENTS.md
+++ b/crates/crm/AGENTS.md
@@ -31,7 +31,7 @@ tools/HTTP/GraphQL surfaces over them.
- Cargo features gate what's pulled in: `lookup` and `search` are read-only,
sqlx-only slices for other domains to depend on without pulling in
- `reqwest`/`axum`; pick the narrowest feature for a new caller.
+ `unfurl`/`axum`; pick the narrowest feature for a new caller.
- Depends on: `entity_access` (ports-and-adapters base), `databases` (ports),
`filter_ast`, `item_filters`, `macro_authorization`, `properties`.
- Used by: `email`, `memory`, `ai_tools`, `soup`/`models_soup`, `graphql_crm`,
@@ -87,11 +87,15 @@ tools/HTTP/GraphQL surfaces over them.
them; continue valid rows, but propagate lookup storage and authorization failures.
- Company names/logos live in the global `crm_domain_directory`. Only the
- email pubsub workers carry a real `CompanyMetadataResolver`; every other
+ email pubsub workers carry a real `CompanyMetadataResolver`, using public
+ homepage metadata through `unfurl`; every other
binary uses `NoOpCompanyMetadataResolver`, which leaves a pending entry
(NULL `enriched_at`) that `CrmServiceImpl::enrich_pending_domains` resolves
later with bounded retries. Never store an empty entry for a domain no
resolver answered for: resolved entries are permanent.
+- Apollo enrichment has been removed. Historical provider fields in the
+ directory remain readable; do not clear existing metadata or drop columns
+ as part of provider removal.
- Contacts carry their own `team_id`; `company_id` and `email` are optional.
Scope contact reads by `ct.team_id` and LEFT JOIN the company for its
`hidden` flag and name. Email sync files personal-provider addresses
diff --git a/crates/crm/Cargo.toml b/crates/crm/Cargo.toml
index 23bf0031ff..ae63e1d53d 100644
--- a/crates/crm/Cargo.toml
+++ b/crates/crm/Cargo.toml
@@ -49,14 +49,13 @@ outbound = [
"dep:macro_user_id",
"dep:models_properties",
"dep:properties",
- "dep:reqwest",
"dep:sqlx",
"dep:unfurl",
"ports",
]
ports = ["dep:bot_id", "dep:csv", "dep:databases", "dep:futures", "dep:models_databases", "dep:models_permissions", "dep:models_properties", "dep:macro_user_id", "dep:macro_uuid", "dep:rootcause", "dep:utoipa", "dep:properties", "entity_access/ports"]
# Read-only CRM search (name/domain). Pulls only sqlx + ports — no
-# reqwest/unfurl/axum — so search consumers stay lightweight.
+# unfurl/axum — so search consumers stay lightweight.
search = ["dep:sqlx", "ports"]
[dependencies]
@@ -106,7 +105,6 @@ model-error-response = { path = "../model-error-response", optional = true }
utoipa = { workspace = true, optional = true }
# Outbound dependencies
-reqwest = { workspace = true, optional = true }
sqlx = { workspace = true, optional = true }
unfurl = { path = "../unfurl", features = ["ports"], optional = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
@@ -126,4 +124,3 @@ properties = { path = "../properties", default-features = false, features = [
rootcause = { workspace = true }
tokio = { workspace = true, features = ["macros"] }
tower = { workspace = true }
-wiremock = { workspace = true }
diff --git a/crates/crm/src/domain/model.rs b/crates/crm/src/domain/model.rs
index 40c5cf33cf..67668aaa92 100644
--- a/crates/crm/src/domain/model.rs
+++ b/crates/crm/src/domain/model.rs
@@ -81,25 +81,24 @@ pub struct CrmCompany {
}
/// Cached metadata about a company keyed on its email domain. Resolved
-/// by [`CompanyMetadataResolver`] (the Apollo.io organization-enrichment
-/// adapter) and stored in `crm_domain_directory`.
+/// by [`CompanyMetadataResolver`] and stored in `crm_domain_directory`.
///
/// Every scalar is `Option` and every list defaults empty: the resolver
-/// may succeed with little data or fail entirely, and both collapse to an
-/// all-empty row so the directory doubles as a negative cache.
+/// may succeed with little data, including an all-empty negative-cache row.
+/// Unavailable results stay pending for a bounded retry instead.
///
/// [`CompanyMetadataResolver`]: crate::domain::company_metadata_resolver::CompanyMetadataResolver
#[derive(Debug, Clone, Default, PartialEq)]
pub struct DomainMetadata {
- /// Company display name (Apollo `name`).
+ /// Company display name.
pub name: Option<String>,
- /// Short company description (Apollo `short_description`).
+ /// Short company description.
pub description: Option<String>,
- /// Logo / icon URL (Apollo `logo_url`).
+ /// Logo / icon URL.
pub icon_url: Option<String>,
- /// Apollo's organization id, for re-enrichment / linking.
+ /// Historical Apollo organization id retained for stored-data compatibility.
pub apollo_organization_id: Option<String>,
- /// Canonical company website (Apollo `website_url`).
+ /// Canonical company website.
pub website_url: Option<String>,
/// LinkedIn company URL.
pub linkedin_url: Option<String>,
@@ -107,11 +106,11 @@ pub struct DomainMetadata {
pub twitter_url: Option<String>,
/// Facebook URL.
pub facebook_url: Option<String>,
- /// Primary industry (Apollo `industry`).
+ /// Primary industry.
pub industry: Option<String>,
- /// Free-text keywords / tags (Apollo `keywords`).
+ /// Free-text keywords / tags.
pub keywords: Vec<String>,
- /// Detected technologies (Apollo `technology_names`).
+ /// Detected technologies.
pub technologies: Vec<String>,
/// Estimated headcount.
pub estimated_num_employees: Option<i32>,
@@ -147,8 +146,7 @@ pub struct DomainMetadata {
pub postal_code: Option<String>,
/// HQ country.
pub country: Option<String>,
- /// Full Apollo `organization` payload (minus our workspace `account`),
- /// kept so fields we don't model yet aren't lost.
+ /// Historical provider payload retained alongside the modeled metadata.
pub raw: Option<Value>,
}
diff --git a/crates/crm/src/domain/service.rs b/crates/crm/src/domain/service.rs
index 6618cfb756..66aa404e1d 100644
--- a/crates/crm/src/domain/service.rs
+++ b/crates/crm/src/domain/service.rs
@@ -671,7 +671,7 @@ where
/// Pending directory entries one [`CrmServiceImpl::enrich_pending_domains`]
/// pass claims. Passes run on a fixed interval, so this caps the resolver
-/// (Apollo) call rate.
+/// call rate.
const ENRICHMENT_BATCH_SIZE: i64 = 10;
/// Enrichment attempts after which a pending directory entry is left alone.
diff --git a/crates/crm/src/outbound.rs b/crates/crm/src/outbound.rs
index 0146528440..2da2b3d379 100644
--- a/crates/crm/src/outbound.rs
+++ b/crates/crm/src/outbound.rs
@@ -1,6 +1,3 @@
-/// Apollo.io-backed implementation of the company metadata resolver
-#[cfg(feature = "outbound")]
-pub mod apollo_resolver;
/// Postgres store linking archived call records to CRM records
#[cfg(feature = "call_link")]
pub mod call_link;
diff --git a/crates/crm/src/outbound/apollo_resolver.rs b/crates/crm/src/outbound/apollo_resolver.rs
deleted file mode 100644
index b4b6dec456..0000000000
--- a/crates/crm/src/outbound/apollo_resolver.rs
+++ /dev/null
@@ -1,204 +0,0 @@
-//! [`CompanyMetadataResolver`] adapter backed by the Apollo.io
-//! organization-enrichment API.
-//!
-//! Calls `GET https://api.apollo.io/api/v1/organizations/enrich?domain={domain}`
-//! with an `x-api-key` header and maps the returned `organization` object
-//! onto [`DomainMetadata`]. The full payload — minus our own Apollo
-//! workspace `account` record — is preserved in [`DomainMetadata::raw`].
-//!
-//! Best-effort: a 2xx response without an `organization` (Apollo knows
-//! nothing about the domain) resolves to [`DomainMetadata::default`], the
-//! negative cache. A missing API key, network error, non-2xx status
-//! (including rate limits), or malformed body is logged and surfaced as
-//! [`DomainResolution::Unavailable`], so the domain is retried instead of
-//! cached empty.
-
-#[cfg(test)]
-mod test;
-
-use std::time::Duration;
-
-use chrono::{DateTime, Utc};
-use serde_json::Value;
-
-use crate::domain::{
- company_metadata_resolver::{CompanyMetadataResolver, DomainResolution},
- model::DomainMetadata,
-};
-
-/// Default Apollo API base URL.
-const DEFAULT_BASE_URL: &str = "https://api.apollo.io";
-
-/// Request timeout for Apollo calls — a stalled connection must not block
-/// the populate worker indefinitely.
-const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
-
-/// Build the reqwest client with the request timeout applied.
-fn build_client() -> reqwest::Client {
- reqwest::Client::builder()
- .timeout(REQUEST_TIMEOUT)
- .build()
- .expect("apollo reqwest client should build")
-}
-
-/// Adapter that resolves [`DomainMetadata`] via Apollo.io organization
-/// enrichment. Cheap to [`Clone`] — the inner [`reqwest::Client`] is
-/// reference-counted.
-#[derive(Clone)]
-pub struct ApolloCompanyMetadataResolver {
- client: reqwest::Client,
- api_key: String,
- base_url: String,
-}
-
-impl ApolloCompanyMetadataResolver {
- /// Build a resolver with the given Apollo API key and a fresh HTTP client.
- pub fn new(api_key: String) -> Self {
- Self {
- client: build_client(),
- api_key,
- base_url: DEFAULT_BASE_URL.to_string(),
- }
- }
-
- /// Build a resolver pointed at a custom base URL (used in tests).
- pub fn with_base_url(api_key: String, base_url: String) -> Self {
- Self {
- client: build_client(),
- api_key,
- base_url,
- }
- }
-}
-
-impl CompanyMetadataResolver for ApolloCompanyMetadataResolver {
- #[tracing::instrument(skip(self))]
- async fn resolve(&self, domain: &str) -> DomainResolution {
- // No key configured (e.g. local without APOLLO_API_KEY): skip the
- // call entirely rather than burning a 401.
- if self.api_key.is_empty() {
- tracing::debug!(domain, "apollo enrich: no api key configured; skipping");
- return DomainResolution::Unavailable;
- }
-
- let url = format!("{}/api/v1/organizations/enrich", self.base_url);
- let response = self
- .client
- .get(&url)
- .query(&[("domain", domain)])
- .header("x-api-key", self.api_key.as_str())
- .header(reqwest::header::ACCEPT, "application/json")
- .send()
- .await;
-
- let body = match response {
- Ok(resp) if resp.status().is_success() => match resp.json::<Value>().await {
- Ok(body) => body,
- Err(e) => {
- tracing::warn!(error=?e, domain, "apollo enrich: failed to parse body");
- return DomainResolution::Unavailable;
- }
- },
- Ok(resp) => {
- tracing::warn!(status = %resp.status(), domain, "apollo enrich: non-success status");
- return DomainResolution::Unavailable;
- }
- Err(e) => {
- tracing::warn!(error=?e, domain, "apollo enrich: request failed");
- return DomainResolution::Unavailable;
- }
- };
-
- let Some(organization) = body.get("organization") else {
- tracing::warn!(domain, "apollo enrich: response had no organization");
- return DomainResolution::Resolved(Box::default());
- };
-
- DomainResolution::Resolved(Box::new(map_organization(organization)))
- }
-}
-
-/// Map an Apollo `organization` JSON object onto [`DomainMetadata`].
-/// Tolerant of missing / mistyped fields — anything unreadable stays
-/// `None` / empty. The `account` and `account_id` keys (our own Apollo
-/// workspace record, not company data) are stripped from `raw`.
-fn map_organization(org: &Value) -> DomainMetadata {
- let mut raw = org.clone();
- if let Some(obj) = raw.as_object_mut() {
- obj.remove("account");
- obj.remove("account_id");
- }
-
- DomainMetadata {
- name: str_field(org, "name"),
- description: str_field(org, "short_description"),
- icon_url: str_field(org, "logo_url"),
- apollo_organization_id: str_field(org, "id"),
- website_url: str_field(org, "website_url"),
- linkedin_url: str_field(org, "linkedin_url"),
- twitter_url: str_field(org, "twitter_url"),
- facebook_url: str_field(org, "facebook_url"),
- industry: str_field(org, "industry"),
- keywords: str_array(org, "keywords"),
- technologies: str_array(org, "technology_names"),
- estimated_num_employees: i32_field(org, "estimated_num_employees"),
- annual_revenue: i64_field(org, "annual_revenue"),
- annual_revenue_printed: str_field(org, "annual_revenue_printed"),
- total_funding: i64_field(org, "total_funding"),
- total_funding_printed: str_field(org, "total_funding_printed"),
- latest_funding_stage: str_field(org, "latest_funding_stage"),
- latest_funding_round_date: date_field(org, "latest_funding_round_date"),
- founded_year: i32_field(org, "founded_year"),
- publicly_traded_symbol: str_field(org, "publicly_traded_symbol"),
- publicly_traded_exchange: str_field(org, "publicly_traded_exchange"),
- phone: str_field(org, "phone"),
- raw_address: str_field(org, "raw_address"),
- street_address: str_field(org, "street_address"),
- city: str_field(org, "city"),
- state: str_field(org, "state"),
- postal_code: str_field(org, "postal_code"),
- country: str_field(org, "country"),
- raw: Some(raw),
- }
-}
-
-/// Read a string field; `None` when absent, null, non-string, or
-/// blank after trimming.
-fn str_field(org: &Value, key: &str) -> Option<String> {
- let s = org.get(key)?.as_str()?.trim();
- (!s.is_empty()).then(|| s.to_string())
-}
-
-/// Read an integer field as `i64`. Floats and strings yield `None`.
-fn i64_field(org: &Value, key: &str) -> Option<i64> {
- org.get(key)?.as_i64()
-}
-
-/// Read an integer field as `i32`, dropping out-of-range values.
-fn i32_field(org: &Value, key: &str) -> Option<i32> {
- i32::try_from(org.get(key)?.as_i64()?).ok()
-}
-
-/// Read an array-of-strings field, skipping non-string / blank elements.
-fn str_array(org: &Value, key: &str) -> Vec<String> {
- org.get(key)
- .and_then(Value::as_array)
- .map(|items| {
- items
- .iter()
- .filter_map(Value::as_str)
- .map(str::trim)
- .filter(|s| !s.is_empty())
- .map(String::from)
- .collect()
- })
- .unwrap_or_default()
-}
-
-/// Read an RFC 3339 timestamp field; `None` when absent or unparseable.
-fn date_field(org: &Value, key: &str) -> Option<DateTime<Utc>> {
- let s = org.get(key)?.as_str()?;
- DateTime::parse_from_rfc3339(s)
- .ok()
- .map(|dt| dt.with_timezone(&Utc))
-}
diff --git a/crates/crm/src/outbound/apollo_resolver/test.rs b/crates/crm/src/outbound/apollo_resolver/test.rs
deleted file mode 100644
index f1758dddbb..0000000000
--- a/crates/crm/src/outbound/apollo_resolver/test.rs
+++ /dev/null
@@ -1,188 +0,0 @@
-use super::*;
-use crate::domain::company_metadata_resolver::CompanyMetadataResolver;
-use serde_json::json;
-use wiremock::matchers::{method, path, query_param};
-use wiremock::{Mock, MockServer, ResponseTemplate};
-
-/// The documented Apollo sample payload maps onto every field, and the
-/// `account` / `account_id` keys are stripped from `raw`.
-#[test]
-fn maps_full_apollo_organization() {
- let org = json!({
- "id": "5e66b6381e05b4008c8331b8",
- "name": "Apollo.io",
- "website_url": "http://www.apollo.io",
- "linkedin_url": "http://www.linkedin.com/company/apolloio",
- "twitter_url": "https://twitter.com/meetapollo/",
- "facebook_url": "https://www.facebook.com/MeetApollo",
- "logo_url": "https://example.com/logo.png",
- "founded_year": 2015,
- "industry": "information technology & services",
- "keywords": ["sales engagement", "lead generation"],
- "technology_names": ["AI", "Android"],
- "estimated_num_employees": 1600,
- "annual_revenue": 100000000_i64,
- "annual_revenue_printed": "100M",
- "total_funding": 251200000_i64,
- "total_funding_printed": "251.2M",
- "latest_funding_stage": "Series D",
- "latest_funding_round_date": "2023-08-01T00:00:00.000+00:00",
- "short_description": "Apollo.io combines a buyer database.",
- "raw_address": "415 Mission St, Floor 37, San Francisco, California 94105, US",
- "street_address": "415 Mission St",
- "city": "San Francisco",
- "state": "California",
- "postal_code": "94105-2301",
- "country": "United States",
- "account_id": "63f53afe4ceeca00016bdd37",
- "account": { "id": "63f53afe4ceeca00016bdd37", "team_id": "ours" }
- });
-
- let md = map_organization(&org);
-
- assert_eq!(md.name.as_deref(), Some("Apollo.io"));
- assert_eq!(
- md.description.as_deref(),
- Some("Apollo.io combines a buyer database.")
- );
- assert_eq!(md.icon_url.as_deref(), Some("https://example.com/logo.png"));
- assert_eq!(
- md.apollo_organization_id.as_deref(),
- Some("5e66b6381e05b4008c8331b8")
- );
- assert_eq!(md.website_url.as_deref(), Some("http://www.apollo.io"));
- assert_eq!(
- md.linkedin_url.as_deref(),
- Some("http://www.linkedin.com/company/apolloio")
- );
- assert_eq!(
- md.industry.as_deref(),
- Some("information technology & services")
- );
- assert_eq!(md.keywords, vec!["sales engagement", "lead generation"]);
- assert_eq!(md.technologies, vec!["AI", "Android"]);
- assert_eq!(md.estimated_num_employees, Some(1600));
- assert_eq!(md.annual_revenue, Some(100_000_000));
- assert_eq!(md.annual_revenue_printed.as_deref(), Some("100M"));
- assert_eq!(md.total_funding, Some(251_200_000));
- assert_eq!(md.latest_funding_stage.as_deref(), Some("Series D"));
- assert_eq!(md.founded_year, Some(2015));
- assert_eq!(md.city.as_deref(), Some("San Francisco"));
- assert_eq!(md.country.as_deref(), Some("United States"));
- assert!(md.latest_funding_round_date.is_some());
-
- let raw = md.raw.expect("raw present");
- assert!(raw.get("account").is_none(), "account stripped from raw");
- assert!(
- raw.get("account_id").is_none(),
- "account_id stripped from raw"
- );
- assert_eq!(raw.get("name").and_then(Value::as_str), Some("Apollo.io"));
-}
-
-/// A sparse / empty organization collapses to an all-empty metadata
-/// (negative-cache row), with `raw` still present.
-#[test]
-fn missing_fields_collapse_to_empty() {
- let md = map_organization(&json!({}));
-
- assert_eq!(md.name, None);
- assert_eq!(md.annual_revenue, None);
- assert_eq!(md.latest_funding_round_date, None);
- assert!(md.keywords.is_empty());
- assert!(md.technologies.is_empty());
- assert!(md.raw.is_some());
-}
-
-/// Mistyped fields are tolerated: a string revenue / blank name don't
-/// blow up the whole mapping.
-#[test]
-fn tolerates_mistyped_and_blank_fields() {
- let org = json!({
- "name": " ",
- "annual_revenue": "not a number",
- "estimated_num_employees": 42,
- "keywords": ["ok", "", 123, " spaced "],
- });
-
- let md = map_organization(&org);
-
- assert_eq!(md.name, None, "blank name -> None");
- assert_eq!(md.annual_revenue, None, "string revenue -> None");
- assert_eq!(md.estimated_num_employees, Some(42));
- assert_eq!(
- md.keywords,
- vec!["ok", "spaced"],
- "blank / non-string elements dropped"
- );
-}
-
-/// Failures that say nothing about the domain stay retryable; only an
-/// answer from Apollo becomes a (possibly empty) directory entry.
-#[tokio::test]
-async fn only_apollo_answers_resolve() {
- let server = MockServer::start().await;
- Mock::given(method("GET"))
- .and(path("/api/v1/organizations/enrich"))
- .and(query_param("domain", "unknown.com"))
- .respond_with(ResponseTemplate::new(200).set_body_json(json!({})))
- .mount(&server)
- .await;
- Mock::given(method("GET"))
- .and(path("/api/v1/organizations/enrich"))
- .and(query_param("domain", "acme.com"))
- .respond_with(
- ResponseTemplate::new(200)
- .set_body_json(json!({ "organization": { "name": "Acme Inc." } })),
- )
- .mount(&server)
- .await;
- Mock::given(method("GET"))
- .and(path("/api/v1/organizations/enrich"))
- .and(query_param("domain", "busy.com"))
- .respond_with(ResponseTemplate::new(429))
- .mount(&server)
- .await;
- let resolver = ApolloCompanyMetadataResolver::with_base_url("key".to_string(), server.uri());
-
- assert_eq!(
- resolver.resolve("unknown.com").await,
- DomainResolution::Resolved(Box::default())
- );
- let DomainResolution::Resolved(acme) = resolver.resolve("acme.com").await else {
- panic!("expected acme.com to resolve");
- };
- assert_eq!(acme.name.as_deref(), Some("Acme Inc."));
- assert_eq!(
- resolver.resolve("busy.com").await,
- DomainResolution::Unavailable
- );
- assert_eq!(
- ApolloCompanyMetadataResolver::with_base_url(String::new(), server.uri())
- .resolve("acme.com")
- .await,
- DomainResolution::Unavailable
- );
-}
-
-/// Live smoke test against the real Apollo API. Ignored by default — it
-/// makes a network call and consumes an Apollo credit. Run with:
-/// APOLLO_API_KEY=<key> cargo test -p crm apollo_live -- --ignored --nocapture
-/// Optionally set APOLLO_TEST_DOMAIN (defaults to "macro.com").
-#[tokio::test]
-#[ignore = "hits the live Apollo API; requires APOLLO_API_KEY"]
-async fn apollo_live_enrich() {
- let api_key = std::env::var("APOLLO_API_KEY").expect("set APOLLO_API_KEY to run this test");
- let domain = std::env::var("APOLLO_TEST_DOMAIN").unwrap_or_else(|_| "macro.com".to_string());
-
- let resolver = ApolloCompanyMetadataResolver::new(api_key);
- let DomainResolution::Resolved(md) = resolver.resolve(&domain).await else {
- panic!("Apollo was unavailable for {domain} — check API key and credits");
- };
-
- println!("DomainMetadata for {domain}:\n{md:#?}");
- assert!(
- md.name.is_some() || md.raw.is_some(),
- "expected Apollo data for {domain}; got empty — check API key, credits, or domain",
- );
-}
diff --git a/crates/crm/src/outbound/no_op_resolver.rs b/crates/crm/src/outbound/no_op_resolver.rs
index d7a418fe6f..2644684923 100644
--- a/crates/crm/src/outbound/no_op_resolver.rs
+++ b/crates/crm/src/outbound/no_op_resolver.rs
@@ -2,8 +2,8 @@
//!
//! Used by binaries that construct a [`CrmService`] without carrying a
//! metadata provider — HTTP API servers, MCP, and agent tools. Wiring a
-//! real resolver into each of them would force them to carry the reqwest
-//! + scraper deps and provider credentials. Companies they create get a
+//! real resolver into each of them would add website-fetching dependencies
+//! and network work to their request paths. Companies they create get a
//! pending `crm_domain_directory` entry, which the email pubsub workers'
//! enrichment worker resolves with the real resolver.
//!
diff --git a/crates/crm/src/outbound/unfurl_resolver.rs b/crates/crm/src/outbound/unfurl_resolver.rs
index 60b54e71d0..9dcaece190 100644
--- a/crates/crm/src/outbound/unfurl_resolver.rs
+++ b/crates/crm/src/outbound/unfurl_resolver.rs
@@ -28,6 +28,9 @@ use crate::domain::{
};
use unfurl::domain::ports::UnfurlService;
+#[cfg(test)]
+mod test;
+
/// Adapter that resolves [`DomainMetadata`] by unfurling
/// `https://{domain}`. Holds the underlying [`UnfurlService`] behind an
/// [`Arc`] so the resolver itself is cheap to [`Clone`].
diff --git a/crates/crm/src/outbound/unfurl_resolver/test.rs b/crates/crm/src/outbound/unfurl_resolver/test.rs
new file mode 100644
index 0000000000..5e6a0d0667
--- /dev/null
+++ b/crates/crm/src/outbound/unfurl_resolver/test.rs
@@ -0,0 +1,67 @@
+use super::*;
+use unfurl::domain::models::{GetUnfurlResponse, UnfurlErr};
+
+struct WebsiteMetadata(GetUnfurlResponse);
+
+impl UnfurlService for WebsiteMetadata {
+ async fn unfurl(&self, url: &str) -> Result<GetUnfurlResponse, UnfurlErr> {
+ assert_eq!(url, "https://example.com");
+ Ok(self.0.clone())
+ }
+}
+
+#[tokio::test]
+async fn homepage_metadata_supplies_company_name_description_and_icon() {
+ let resolver =
+ UnfurlCompanyMetadataResolver::new(Arc::new(WebsiteMetadata(GetUnfurlResponse {
+ title: " Example ".into(),
+ description: Some("Example description".into()),
+ favicon_url: Some("https://example.com/icon.png".into()),
+ ..Default::default()
+ })));
+
+ let DomainResolution::Resolved(metadata) = resolver.resolve("example.com").await else {
+ panic!("website metadata should resolve the company");
+ };
+ assert_eq!(
+ *metadata,
+ DomainMetadata {
+ name: Some("Example".into()),
+ description: Some("Example description".into()),
+ icon_url: Some("https://example.com/icon.png".into()),
+ ..Default::default()
+ }
+ );
+}
+
+#[tokio::test]
+async fn successful_empty_pages_do_not_turn_fallback_urls_into_company_names() {
+ for title in ["https://example.com", " https://example.com ", " "] {
+ let resolver =
+ UnfurlCompanyMetadataResolver::new(Arc::new(WebsiteMetadata(GetUnfurlResponse {
+ title: title.into(),
+ ..Default::default()
+ })));
+ let DomainResolution::Resolved(metadata) = resolver.resolve("example.com").await else {
+ panic!("a successful page without metadata is a resolved negative-cache entry");
+ };
+ assert_eq!(*metadata, DomainMetadata::default());
+ }
+}
+
+struct UnavailableWebsite;
+
+impl UnfurlService for UnavailableWebsite {
+ async fn unfurl(&self, _url: &str) -> Result<GetUnfurlResponse, UnfurlErr> {
+ Err(UnfurlErr::Fetch(anyhow::anyhow!("website unavailable")))
+ }
+}
+
+#[tokio::test]
+async fn failed_fetch_stays_retryable_instead_of_resolving_an_empty_company() {
+ let resolver = UnfurlCompanyMetadataResolver::new(Arc::new(UnavailableWebsite));
+ assert!(matches!(
+ resolver.resolve("example.com").await,
+ DomainResolution::Unavailable
+ ));
+}
diff --git a/deny.toml b/deny.toml
index a66e92eae3..c55c95a74f 100644
--- a/deny.toml
+++ b/deny.toml
@@ -229,6 +229,7 @@ deny = [
"document_storage_service", # GraphQL telemetry span and log tests only (dev-dependency)
"documents", # Dev logging only
"email", # Dev logging only
+ "email_api_client", # Outlook transport span tests only
"email_db_client", # Purge phase span tests only
"forms", # Anonymous-response span and log leak tests only (dev-dependency)
"genai_telemetry", # GenAI span tests
diff --git a/infra/stacks/email-service/Pulumi.dev.yaml b/infra/stacks/email-service/Pulumi.dev.yaml
index 99af3dce80..b43ec6104b 100644
--- a/infra/stacks/email-service/Pulumi.dev.yaml
+++ b/infra/stacks/email-service/Pulumi.dev.yaml
@@ -12,8 +12,6 @@ config:
email-service:internal_auth_key: document-storage-service-auth-key-dev
email-service:cf_private_key: email-attachments-private-key-dev
email-service:notifications_enabled: true
- email-service:use_apollo_crm_enrichment: true
- email-service:apollo_api_key_secret_name: apollo-io-api-key-master
email-service:sent_undo_delay_secs: 5
email-service:redis_rate_limit_reqs: 14000
email-service:redis_rate_limit_reqs_backfill: 13000
diff --git a/infra/stacks/email-service/Pulumi.prod.yaml b/infra/stacks/email-service/Pulumi.prod.yaml
index 476cabac0d..c6bf387b3c 100644
--- a/infra/stacks/email-service/Pulumi.prod.yaml
+++ b/infra/stacks/email-service/Pulumi.prod.yaml
@@ -12,8 +12,6 @@ config:
email-service:internal_auth_key: document-storage-service-auth-key-prod
email-service:cf_private_key: email-attachments-private-key-prod
email-service:notifications_enabled: true
- email-service:use_apollo_crm_enrichment: false
- email-service:apollo_api_key_secret_name: apollo-io-api-key-master
email-service:sent_undo_delay_secs: 5
email-service:redis_rate_limit_reqs: 14000
email-service:redis_rate_limit_reqs_backfill: 13000
diff --git a/infra/stacks/email-service/index.ts b/infra/stacks/email-service/index.ts
index 9cbfc26ebc..bc822b05ba 100644
--- a/infra/stacks/email-service/index.ts
+++ b/infra/stacks/email-service/index.ts
@@ -37,7 +37,6 @@ export const coparse_api_vpc = get_coparse_api_vpc();
const JWT_SECRET_KEY = config.require(`jwt_secret_key`);
-const APOLLO_API_KEY_SECRET_NAME = config.require(`apollo_api_key_secret_name`);
const jwtSecretKeyArn: pulumi.Output<string> = aws.secretsmanager
.getSecretVersionOutput({ secretId: JWT_SECRET_KEY })
.apply((secret) => secret.arn);
@@ -111,10 +110,6 @@ const macroDbUrlArn: pulumi.Output<string> = aws.secretsmanager
.getSecretVersionOutput({ secretId: MACRO_DB_URL_SECRET_NAME })
.apply((secret) => secret.arn);
-const apolloApiKeySecretArn: pulumi.Output<string> = aws.secretsmanager
- .getSecretVersionOutput({ secretId: APOLLO_API_KEY_SECRET_NAME })
- .apply((secret) => secret.arn);
-
const inbox_sync_queue = new Queue('email-service-gmail-webhook', {
tags,
maxReceiveCount: 3,
@@ -283,7 +278,6 @@ const secretKeyArns = [
macroDbUrlArn,
MACRO_API_TOKENS.macroApiTokenPublicKeyArn,
cloudfrontSecretKey.arn,
- apolloApiKeySecretArn,
];
const queueArns = [
diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.md
index 481e9fa1bf..27b6cebf38 100644
--- a/services/email_service/AGENTS.md
+++ b/services/email_service/AGENTS.md
@@ -18,7 +18,9 @@ the gateway `/email` prefix.
- `src/backfill_outbox.rs`, `src/backfill_init_service.rs`,
`src/backfill_completion_service.rs`: Gmail history backfill orchestration.
- `src/crm_enrichment_worker.rs`: periodic pass that resolves pending CRM
- company directory entries (names/logos) left by binaries without a resolver.
+ company directory entries (names/logos) left by binaries without a resolver,
+ using the public-homepage `UnfurlCompanyMetadataResolver` wired in
+ `pubsub_workers`. It requires no enrichment-provider key or selection flag.
- `src/mailbox_sync_worker.rs`: the Outlook mailbox loops (sync, projections,
contacts, drafts, commands, settings, watches) and `run_health`, the
per-minute Outlook health snapshot.
diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
index 5aeb62eb77..75b5a02912 100644
--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs
@@ -385,38 +385,12 @@ async fn main() -> anyhow::Result<()> {
// The CRM crate's company-metadata resolver is consulted by
// `crm_service.populate_contact` on `crm_domain_directory` misses and by
// the enrichment worker for pending entries other binaries leave behind.
- // `USE_APOLLO_CRM_ENRICHMENT` selects Apollo.io vs. the unfurl-backed
- // resolver; we also fall back to unfurl when the Apollo key can't be
- // loaded. The resolver is cheap to clone.
- let build_unfurl = || -> anyhow::Result<CrmMetadataResolver> {
- // Wrap the SSRF-safe reqwest fetcher in an `UnfurlServiceImpl`,
- // then the `UnfurlCompanyMetadataResolver`.
- let unfurl_service = Arc::new(unfurl::domain::service::UnfurlServiceImpl::new(
- unfurl::outbound::ReqwestUnfurlFetcher::new()
- .context("failed to build ReqwestUnfurlFetcher")?,
- ));
- Ok(CrmMetadataResolver::Unfurl(
- crm::outbound::unfurl_resolver::UnfurlCompanyMetadataResolver::new(unfurl_service),
- ))
- };
-
- let metadata_resolver = if config.use_apollo_crm_enrichment {
- // No usable key (missing/unreadable secret, or unset locally): fall
- // back to unfurl rather than running Apollo with an empty key, which
- // would no-op and pollute the directory with negative-cache rows.
- if config.apollo_api_key.as_ref().is_empty() {
- tracing::warn!("apollo api key unavailable; falling back to unfurl CRM enrichment");
- build_unfurl()?
- } else {
- CrmMetadataResolver::Apollo(
- crm::outbound::apollo_resolver::ApolloCompanyMetadataResolver::new(
- config.apollo_api_key.as_ref().to_string(),
- ),
- )
- }
- } else {
- build_unfurl()?
- };
+ // Resolve public homepage metadata with the SSRF-safe unfurl fetcher.
+ let unfurl_service = Arc::new(unfurl::domain::service::UnfurlServiceImpl::new(
+ unfurl::outbound::ReqwestUnfurlFetcher::new()
+ .context("failed to build ReqwestUnfurlFetcher")?,
+ ));
+ let metadata_resolver = CrmMetadataResolver::new(unfurl_service);
let crm_service = crm::domain::service::CrmServiceImpl::new(
crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
diff --git a/services/email_service/src/config.rs b/services/email_service/src/config.rs
index 0bb1541ca3..cc7f0f5c7d 100644
--- a/services/email_service/src/config.rs
+++ b/services/email_service/src/config.rs
@@ -16,7 +16,6 @@ env_vars! {
pub struct AuthenticationServiceSecretKey;
pub struct EmailServiceCloudfrontDistributionUrl;
pub struct EmailServiceCloudfrontSignerPublicKeyId;
- pub struct ApolloApiKey;
}
#[derive(macro_config::MacroConfig)]
@@ -65,16 +64,6 @@ pub struct Config {
#[macro_config_default(true)]
pub notifications_enabled: bool,
- /// Use Apollo.io for CRM company enrichment. When `false`, fall back
- /// to the unfurl-based resolver.
- #[macro_config_default(false)]
- pub use_apollo_crm_enrichment: bool,
-
- /// Apollo.io API key for CRM enrichment. Locally this is the key
- /// itself; in deployed envs it's the name of the Secrets Manager
- /// secret holding it (resolved at startup). Empty disables enrichment.
- pub apollo_api_key: LocalOrRemoteSecret<ApolloApiKey>,
-
/// The queue max messages per poll
#[macro_config_default(10)]
pub queue_max_messages: i32,
diff --git a/services/email_service/src/crm_enrichment_worker.rs b/services/email_service/src/crm_enrichment_worker.rs
index 467ab269fb..6bac9ff533 100644
--- a/services/email_service/src/crm_enrichment_worker.rs
+++ b/services/email_service/src/crm_enrichment_worker.rs
@@ -6,7 +6,7 @@ use crate::pubsub::context::CrmServiceType;
use std::time::Duration;
use tokio_util::sync::CancellationToken;
-/// One enrichment pass per interval keeps the resolver (Apollo) call rate at
+/// One enrichment pass per interval keeps the resolver call rate at
/// the service's batch size per minute per worker.
const ENRICHMENT_INTERVAL: Duration = Duration::from_secs(60);
diff --git a/services/email_service/src/pubsub/context.rs b/services/email_service/src/pubsub/context.rs
index 2308cf61ad..ca3efa9fe1 100644
--- a/services/email_service/src/pubsub/context.rs
+++ b/services/email_service/src/pubsub/context.rs
@@ -3,9 +3,7 @@ use crate::util::redis::RedisClient;
use connection_gateway_client::client::ConnectionGatewayClient;
use contacts::domain::service::SqsContactsIngress;
use contacts::outbound::ingress::SqsContactsQueue;
-use crm::domain::company_metadata_resolver::{CompanyMetadataResolver, DomainResolution};
use crm::domain::service::CrmServiceImpl;
-use crm::outbound::apollo_resolver::ApolloCompanyMetadataResolver;
use crm::outbound::companies_repo::CompaniesRepositoryImpl;
use crm::outbound::unfurl_resolver::UnfurlCompanyMetadataResolver;
use document_storage_service_client::DocumentStorageServiceClient;
@@ -24,35 +22,13 @@ pub type PubSubEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTr
/// The concrete notification ingress service type.
pub type NotificationIngressType = SqsNotificationIngress<SqsQueue>;
-/// The unfurl-backed resolver used when Apollo enrichment is disabled.
-type UnfurlResolver = UnfurlCompanyMetadataResolver<
+/// CRM company metadata resolved from the company's public homepage.
+pub type CrmMetadataResolver = UnfurlCompanyMetadataResolver<
unfurl::domain::service::UnfurlServiceImpl<unfurl::outbound::ReqwestUnfurlFetcher>,
>;
-/// CRM company-metadata resolver, chosen at startup from the
-/// `USE_APOLLO_CRM_ENRICHMENT` flag: Apollo.io when enabled, the
-/// unfurl-backed resolver otherwise. A single concrete type so it slots
-/// into [`CrmServiceType`] — `CompanyMetadataResolver` is RPITIT and thus
-/// not dyn-compatible, so we dispatch via an enum rather than `dyn`.
-#[derive(Clone)]
-pub enum CrmMetadataResolver {
- /// Apollo.io organization enrichment.
- Apollo(ApolloCompanyMetadataResolver),
- /// Unfurl-backed homepage metadata.
- Unfurl(UnfurlResolver),
-}
-
-impl CompanyMetadataResolver for CrmMetadataResolver {
- async fn resolve(&self, domain: &str) -> DomainResolution {
- match self {
- CrmMetadataResolver::Apollo(r) => r.resolve(domain).await,
- CrmMetadataResolver::Unfurl(r) => r.resolve(domain).await,
- }
- }
-}
-
/// The concrete CRM service type, backed by Postgres and the
-/// flag-selected [`CrmMetadataResolver`]. The resolver is consulted only
+/// homepage-backed [`CrmMetadataResolver`]. The resolver is consulted only
/// on `crm_domain_directory` misses and for pending entries, so it isn't
/// surfaced separately on [`PubSubContext`].
pub type CrmServiceType = CrmServiceImpl<CompaniesRepositoryImpl, CrmMetadataResolver>;
diff --git a/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs b/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs
index 20f585376f..6a65b6c3ac 100644
--- a/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs
+++ b/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs
@@ -13,6 +13,9 @@ use serde_json::{Value, json};
use std::sync::{Arc, Mutex};
use tokio::sync::oneshot;
+#[cfg(test)]
+mod test;
+
#[derive(Clone, Debug)]
pub struct Sent {
pub queue_url: String,
@@ -54,7 +57,8 @@ impl FakeSqs {
.expect("bind fake sqs");
let url = format!("http://{}", listener.local_addr().unwrap());
let fake = FakeSqs { state, url };
- let app = Router::new().fallback(handle).with_state(fake.clone());
+ let app =
+ Router::new().fallback_service(axum::routing::post(handle).with_state(fake.clone()));
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
fake
}
diff --git a/services/email_service/src/pubsub/label_write_sim/fake_sqs/test.rs b/services/email_service/src/pubsub/label_write_sim/fake_sqs/test.rs
new file mode 100644
index 0000000000..0564dea53e
--- /dev/null
+++ b/services/email_service/src/pubsub/label_write_sim/fake_sqs/test.rs
@@ -0,0 +1,20 @@
+use super::*;
+
+#[tokio::test]
+async fn http_probe_does_not_poison_subsequent_sqs_requests() {
+ let sqs = FakeSqs::start().await;
+ let response = reqwest::get(&sqs.url).await.expect("probe response");
+ assert_eq!(response.status().as_u16(), 405);
+
+ client(&sqs.url, false)
+ .send_message()
+ .queue_url(sqs.queue_url("test"))
+ .message_body("after probe")
+ .send()
+ .await
+ .expect("SQS remains usable after a non-SQS request");
+ let (sent, acks) = sqs.take();
+ assert_eq!(sent.len(), 1);
+ assert_eq!(sent[0].body, "after probe");
+ assert!(acks.is_empty());
+}
diff --git a/services/email_service/src/pubsub/label_write_sim/world.rs b/services/email_service/src/pubsub/label_write_sim/world.rs
index 7809eaea02..cd5f603eab 100644
--- a/services/email_service/src/pubsub/label_write_sim/world.rs
+++ b/services/email_service/src/pubsub/label_write_sim/world.rs
@@ -251,13 +251,9 @@ impl World {
);
let crm_service = crm::domain::service::CrmServiceImpl::new(
crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
- CrmMetadataResolver::Unfurl(
- crm::outbound::unfurl_resolver::UnfurlCompanyMetadataResolver::new(Arc::new(
- unfurl::domain::service::UnfurlServiceImpl::new(
- unfurl::outbound::ReqwestUnfurlFetcher::new().expect("unfurl fetcher"),
- ),
- )),
- ),
+ CrmMetadataResolver::new(Arc::new(unfurl::domain::service::UnfurlServiceImpl::new(
+ unfurl::outbound::ReqwestUnfurlFetcher::new().expect("unfurl fetcher"),
+ ))),
);
let sync_context = |retry_worker: bool, queue: &str| PubSubContext {
gmail_history: email::domain::mailbox::gmail_history::GmailHistoryService(
diff --git a/services/email_service/src/pubsub/mod.rs b/services/email_service/src/pubsub/mod.rs
index 7eaa5a3d9c..6c8dd20efa 100644
--- a/services/email_service/src/pubsub/mod.rs
+++ b/services/email_service/src/pubsub/mod.rs
@@ -1,7 +1,7 @@
pub mod backfill;
pub mod context;
pub mod invitation_extraction;
-/// The flag-selected CRM metadata resolver, re-exported for the
+/// The homepage-backed CRM metadata resolver, re-exported for the
/// pubsub_workers binary to construct.
pub use context::CrmMetadataResolver;
pub mod crm_cleanup;
diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env.rs b/tooling/xtask/crates/xtask_local/src/local/local_env.rs
index 989bd4d0e9..2f8367c841 100644
--- a/tooling/xtask/crates/xtask_local/src/local/local_env.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/local_env.rs
@@ -799,7 +799,6 @@ impl BootStubEnv {
// email_service's GCP pubsub queue (gmail watch notifications) and
// its own CloudFront signer for attachment presigned URLs.
env.insert("GMAIL_GCP_QUEUE".into(), "gmail-gcp-queue-local".into());
- env.insert("APOLLO_API_KEY".into(), "local".into());
env.insert(
"EMAIL_SERVICE_CLOUDFRONT_DISTRIBUTION_URL".into(),
"http://localhost:8100".into(),
diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs
index b3c40248ff..e1ee9a99b5 100644
--- a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs
@@ -124,7 +124,6 @@ fn emits_required_keys() {
"MACRO_API_TOKEN_PRIVATE_SECRET_KEY",
"MACRO_API_TOKEN_EXPIRY_SECONDS",
"GMAIL_GCP_QUEUE",
- "APOLLO_API_KEY",
"EMAIL_SERVICE_CLOUDFRONT_DISTRIBUTION_URL",
"EMAIL_SERVICE_CLOUDFRONT_SIGNER_PRIVATE_KEY",
"EMAIL_SERVICE_CLOUDFRONT_SIGNER_PUBLIC_KEY_ID",