Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
refactor(crm): remove Apollo enrichment and legacy access (#9386)
gbirmanf7008ff2c136
diff --git a/Cargo.lock b/Cargo.lockindex 758a67047a..d5e460a464 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5409,7 +5409,6 @@ dependencies = ["models_properties","properties","rdkafka",- "reqwest 0.13.4","rootcause","schemars 1.2.1","serde",@@ -5423,7 +5422,6 @@ dependencies = ["unfurl","utoipa","uuid",- "wiremock","workspace-hack",]diff --git a/crates/crm/AGENTS.md b/crates/crm/AGENTS.mdindex e6a98b8649..6cc27f4ec2 100644--- a/crates/crm/AGENTS.md+++ b/crates/crm/AGENTS.md@@ -31,7 +31,7 @@ tools/HTTP/GraphQL surfaces over them.- Cargo features gate what's pulled in: `lookup` and `search` are read-only,sqlx-only slices for other domains to depend on without pulling in- `reqwest`/`axum`; pick the narrowest feature for a new caller.+ `unfurl`/`axum`; pick the narrowest feature for a new caller.- Depends on: `entity_access` (ports-and-adapters base), `databases` (ports),`filter_ast`, `item_filters`, `macro_authorization`, `properties`.- Used by: `email`, `memory`, `ai_tools`, `soup`/`models_soup`, `graphql_crm`,@@ -87,11 +87,15 @@ tools/HTTP/GraphQL surfaces over them.them; continue valid rows, but propagate lookup storage and authorization failures.- Company names/logos live in the global `crm_domain_directory`. Only the- email pubsub workers carry a real `CompanyMetadataResolver`; every other+ email pubsub workers carry a real `CompanyMetadataResolver`, using public+ homepage metadata through `unfurl`; every otherbinary uses `NoOpCompanyMetadataResolver`, which leaves a pending entry(NULL `enriched_at`) that `CrmServiceImpl::enrich_pending_domains` resolveslater with bounded retries. Never store an empty entry for a domain noresolver answered for: resolved entries are permanent.+- Apollo enrichment has been removed. Historical provider fields in the+ directory remain readable; do not clear existing metadata or drop columns+ as part of provider removal.- Contacts carry their own `team_id`; `company_id` and `email` are optional.Scope contact reads by `ct.team_id` and LEFT JOIN the company for its`hidden` flag and name. Email sync files personal-provider addressesdiff --git a/crates/crm/Cargo.toml b/crates/crm/Cargo.tomlindex 23bf0031ff..ae63e1d53d 100644--- a/crates/crm/Cargo.toml+++ b/crates/crm/Cargo.toml@@ -49,14 +49,13 @@ outbound = ["dep:macro_user_id","dep:models_properties","dep:properties",- "dep:reqwest","dep:sqlx","dep:unfurl","ports",]ports = ["dep:bot_id", "dep:csv", "dep:databases", "dep:futures", "dep:models_databases", "dep:models_permissions", "dep:models_properties", "dep:macro_user_id", "dep:macro_uuid", "dep:rootcause", "dep:utoipa", "dep:properties", "entity_access/ports"]# Read-only CRM search (name/domain). Pulls only sqlx + ports — no-# reqwest/unfurl/axum — so search consumers stay lightweight.+# unfurl/axum — so search consumers stay lightweight.search = ["dep:sqlx", "ports"][dependencies]@@ -106,7 +105,6 @@ model-error-response = { path = "../model-error-response", optional = true }utoipa = { workspace = true, optional = true }# Outbound dependencies-reqwest = { workspace = true, optional = true }sqlx = { workspace = true, optional = true }unfurl = { path = "../unfurl", features = ["ports"], optional = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }@@ -126,4 +124,3 @@ properties = { path = "../properties", default-features = false, features = [rootcause = { workspace = true }tokio = { workspace = true, features = ["macros"] }tower = { workspace = true }-wiremock = { workspace = true }diff --git a/crates/crm/src/domain/model.rs b/crates/crm/src/domain/model.rsindex 40c5cf33cf..67668aaa92 100644--- a/crates/crm/src/domain/model.rs+++ b/crates/crm/src/domain/model.rs@@ -81,25 +81,24 @@ pub struct CrmCompany {}/// Cached metadata about a company keyed on its email domain. Resolved-/// by [`CompanyMetadataResolver`] (the Apollo.io organization-enrichment-/// adapter) and stored in `crm_domain_directory`.+/// by [`CompanyMetadataResolver`] and stored in `crm_domain_directory`.////// Every scalar is `Option` and every list defaults empty: the resolver-/// may succeed with little data or fail entirely, and both collapse to an-/// all-empty row so the directory doubles as a negative cache.+/// may succeed with little data, including an all-empty negative-cache row.+/// Unavailable results stay pending for a bounded retry instead.////// [`CompanyMetadataResolver`]: crate::domain::company_metadata_resolver::CompanyMetadataResolver#[derive(Debug, Clone, Default, PartialEq)]pub struct DomainMetadata {- /// Company display name (Apollo `name`).+ /// Company display name.pub name: Option<String>,- /// Short company description (Apollo `short_description`).+ /// Short company description.pub description: Option<String>,- /// Logo / icon URL (Apollo `logo_url`).+ /// Logo / icon URL.pub icon_url: Option<String>,- /// Apollo's organization id, for re-enrichment / linking.+ /// Historical Apollo organization id retained for stored-data compatibility.pub apollo_organization_id: Option<String>,- /// Canonical company website (Apollo `website_url`).+ /// Canonical company website.pub website_url: Option<String>,/// LinkedIn company URL.pub linkedin_url: Option<String>,@@ -107,11 +106,11 @@ pub struct DomainMetadata {pub twitter_url: Option<String>,/// Facebook URL.pub facebook_url: Option<String>,- /// Primary industry (Apollo `industry`).+ /// Primary industry.pub industry: Option<String>,- /// Free-text keywords / tags (Apollo `keywords`).+ /// Free-text keywords / tags.pub keywords: Vec<String>,- /// Detected technologies (Apollo `technology_names`).+ /// Detected technologies.pub technologies: Vec<String>,/// Estimated headcount.pub estimated_num_employees: Option<i32>,@@ -147,8 +146,7 @@ pub struct DomainMetadata {pub postal_code: Option<String>,/// HQ country.pub country: Option<String>,- /// Full Apollo `organization` payload (minus our workspace `account`),- /// kept so fields we don't model yet aren't lost.+ /// Historical provider payload retained alongside the modeled metadata.pub raw: Option<Value>,}diff --git a/crates/crm/src/domain/service.rs b/crates/crm/src/domain/service.rsindex 6618cfb756..66aa404e1d 100644--- a/crates/crm/src/domain/service.rs+++ b/crates/crm/src/domain/service.rs@@ -671,7 +671,7 @@ where/// Pending directory entries one [`CrmServiceImpl::enrich_pending_domains`]/// pass claims. Passes run on a fixed interval, so this caps the resolver-/// (Apollo) call rate.+/// call rate.const ENRICHMENT_BATCH_SIZE: i64 = 10;/// Enrichment attempts after which a pending directory entry is left alone.diff --git a/crates/crm/src/outbound.rs b/crates/crm/src/outbound.rsindex 0146528440..2da2b3d379 100644--- a/crates/crm/src/outbound.rs+++ b/crates/crm/src/outbound.rs@@ -1,6 +1,3 @@-/// Apollo.io-backed implementation of the company metadata resolver-#[cfg(feature = "outbound")]-pub mod apollo_resolver;/// Postgres store linking archived call records to CRM records#[cfg(feature = "call_link")]pub mod call_link;diff --git a/crates/crm/src/outbound/apollo_resolver.rs b/crates/crm/src/outbound/apollo_resolver.rsdeleted file mode 100644index b4b6dec456..0000000000--- a/crates/crm/src/outbound/apollo_resolver.rs+++ /dev/null@@ -1,204 +0,0 @@-//! [`CompanyMetadataResolver`] adapter backed by the Apollo.io-//! organization-enrichment API.-//!-//! Calls `GET https://api.apollo.io/api/v1/organizations/enrich?domain={domain}`-//! with an `x-api-key` header and maps the returned `organization` object-//! onto [`DomainMetadata`]. The full payload — minus our own Apollo-//! workspace `account` record — is preserved in [`DomainMetadata::raw`].-//!-//! Best-effort: a 2xx response without an `organization` (Apollo knows-//! nothing about the domain) resolves to [`DomainMetadata::default`], the-//! negative cache. A missing API key, network error, non-2xx status-//! (including rate limits), or malformed body is logged and surfaced as-//! [`DomainResolution::Unavailable`], so the domain is retried instead of-//! cached empty.--#[cfg(test)]-mod test;--use std::time::Duration;--use chrono::{DateTime, Utc};-use serde_json::Value;--use crate::domain::{- company_metadata_resolver::{CompanyMetadataResolver, DomainResolution},- model::DomainMetadata,-};--/// Default Apollo API base URL.-const DEFAULT_BASE_URL: &str = "https://api.apollo.io";--/// Request timeout for Apollo calls — a stalled connection must not block-/// the populate worker indefinitely.-const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);--/// Build the reqwest client with the request timeout applied.-fn build_client() -> reqwest::Client {- reqwest::Client::builder()- .timeout(REQUEST_TIMEOUT)- .build()- .expect("apollo reqwest client should build")-}--/// Adapter that resolves [`DomainMetadata`] via Apollo.io organization-/// enrichment. Cheap to [`Clone`] — the inner [`reqwest::Client`] is-/// reference-counted.-#[derive(Clone)]-pub struct ApolloCompanyMetadataResolver {- client: reqwest::Client,- api_key: String,- base_url: String,-}--impl ApolloCompanyMetadataResolver {- /// Build a resolver with the given Apollo API key and a fresh HTTP client.- pub fn new(api_key: String) -> Self {- Self {- client: build_client(),- api_key,- base_url: DEFAULT_BASE_URL.to_string(),- }- }-- /// Build a resolver pointed at a custom base URL (used in tests).- pub fn with_base_url(api_key: String, base_url: String) -> Self {- Self {- client: build_client(),- api_key,- base_url,- }- }-}--impl CompanyMetadataResolver for ApolloCompanyMetadataResolver {- #[tracing::instrument(skip(self))]- async fn resolve(&self, domain: &str) -> DomainResolution {- // No key configured (e.g. local without APOLLO_API_KEY): skip the- // call entirely rather than burning a 401.- if self.api_key.is_empty() {- tracing::debug!(domain, "apollo enrich: no api key configured; skipping");- return DomainResolution::Unavailable;- }-- let url = format!("{}/api/v1/organizations/enrich", self.base_url);- let response = self- .client- .get(&url)- .query(&[("domain", domain)])- .header("x-api-key", self.api_key.as_str())- .header(reqwest::header::ACCEPT, "application/json")- .send()- .await;-- let body = match response {- Ok(resp) if resp.status().is_success() => match resp.json::<Value>().await {- Ok(body) => body,- Err(e) => {- tracing::warn!(error=?e, domain, "apollo enrich: failed to parse body");- return DomainResolution::Unavailable;- }- },- Ok(resp) => {- tracing::warn!(status = %resp.status(), domain, "apollo enrich: non-success status");- return DomainResolution::Unavailable;- }- Err(e) => {- tracing::warn!(error=?e, domain, "apollo enrich: request failed");- return DomainResolution::Unavailable;- }- };-- let Some(organization) = body.get("organization") else {- tracing::warn!(domain, "apollo enrich: response had no organization");- return DomainResolution::Resolved(Box::default());- };-- DomainResolution::Resolved(Box::new(map_organization(organization)))- }-}--/// Map an Apollo `organization` JSON object onto [`DomainMetadata`].-/// Tolerant of missing / mistyped fields — anything unreadable stays-/// `None` / empty. The `account` and `account_id` keys (our own Apollo-/// workspace record, not company data) are stripped from `raw`.-fn map_organization(org: &Value) -> DomainMetadata {- let mut raw = org.clone();- if let Some(obj) = raw.as_object_mut() {- obj.remove("account");- obj.remove("account_id");- }-- DomainMetadata {- name: str_field(org, "name"),- description: str_field(org, "short_description"),- icon_url: str_field(org, "logo_url"),- apollo_organization_id: str_field(org, "id"),- website_url: str_field(org, "website_url"),- linkedin_url: str_field(org, "linkedin_url"),- twitter_url: str_field(org, "twitter_url"),- facebook_url: str_field(org, "facebook_url"),- industry: str_field(org, "industry"),- keywords: str_array(org, "keywords"),- technologies: str_array(org, "technology_names"),- estimated_num_employees: i32_field(org, "estimated_num_employees"),- annual_revenue: i64_field(org, "annual_revenue"),- annual_revenue_printed: str_field(org, "annual_revenue_printed"),- total_funding: i64_field(org, "total_funding"),- total_funding_printed: str_field(org, "total_funding_printed"),- latest_funding_stage: str_field(org, "latest_funding_stage"),- latest_funding_round_date: date_field(org, "latest_funding_round_date"),- founded_year: i32_field(org, "founded_year"),- publicly_traded_symbol: str_field(org, "publicly_traded_symbol"),- publicly_traded_exchange: str_field(org, "publicly_traded_exchange"),- phone: str_field(org, "phone"),- raw_address: str_field(org, "raw_address"),- street_address: str_field(org, "street_address"),- city: str_field(org, "city"),- state: str_field(org, "state"),- postal_code: str_field(org, "postal_code"),- country: str_field(org, "country"),- raw: Some(raw),- }-}--/// Read a string field; `None` when absent, null, non-string, or-/// blank after trimming.-fn str_field(org: &Value, key: &str) -> Option<String> {- let s = org.get(key)?.as_str()?.trim();- (!s.is_empty()).then(|| s.to_string())-}--/// Read an integer field as `i64`. Floats and strings yield `None`.-fn i64_field(org: &Value, key: &str) -> Option<i64> {- org.get(key)?.as_i64()-}--/// Read an integer field as `i32`, dropping out-of-range values.-fn i32_field(org: &Value, key: &str) -> Option<i32> {- i32::try_from(org.get(key)?.as_i64()?).ok()-}--/// Read an array-of-strings field, skipping non-string / blank elements.-fn str_array(org: &Value, key: &str) -> Vec<String> {- org.get(key)- .and_then(Value::as_array)- .map(|items| {- items- .iter()- .filter_map(Value::as_str)- .map(str::trim)- .filter(|s| !s.is_empty())- .map(String::from)- .collect()- })- .unwrap_or_default()-}--/// Read an RFC 3339 timestamp field; `None` when absent or unparseable.-fn date_field(org: &Value, key: &str) -> Option<DateTime<Utc>> {- let s = org.get(key)?.as_str()?;- DateTime::parse_from_rfc3339(s)- .ok()- .map(|dt| dt.with_timezone(&Utc))-}diff --git a/crates/crm/src/outbound/apollo_resolver/test.rs b/crates/crm/src/outbound/apollo_resolver/test.rsdeleted file mode 100644index f1758dddbb..0000000000--- a/crates/crm/src/outbound/apollo_resolver/test.rs+++ /dev/null@@ -1,188 +0,0 @@-use super::*;-use crate::domain::company_metadata_resolver::CompanyMetadataResolver;-use serde_json::json;-use wiremock::matchers::{method, path, query_param};-use wiremock::{Mock, MockServer, ResponseTemplate};--/// The documented Apollo sample payload maps onto every field, and the-/// `account` / `account_id` keys are stripped from `raw`.-#[test]-fn maps_full_apollo_organization() {- let org = json!({- "id": "5e66b6381e05b4008c8331b8",- "name": "Apollo.io",- "website_url": "http://www.apollo.io",- "linkedin_url": "http://www.linkedin.com/company/apolloio",- "twitter_url": "https://twitter.com/meetapollo/",- "facebook_url": "https://www.facebook.com/MeetApollo",- "logo_url": "https://example.com/logo.png",- "founded_year": 2015,- "industry": "information technology & services",- "keywords": ["sales engagement", "lead generation"],- "technology_names": ["AI", "Android"],- "estimated_num_employees": 1600,- "annual_revenue": 100000000_i64,- "annual_revenue_printed": "100M",- "total_funding": 251200000_i64,- "total_funding_printed": "251.2M",- "latest_funding_stage": "Series D",- "latest_funding_round_date": "2023-08-01T00:00:00.000+00:00",- "short_description": "Apollo.io combines a buyer database.",- "raw_address": "415 Mission St, Floor 37, San Francisco, California 94105, US",- "street_address": "415 Mission St",- "city": "San Francisco",- "state": "California",- "postal_code": "94105-2301",- "country": "United States",- "account_id": "63f53afe4ceeca00016bdd37",- "account": { "id": "63f53afe4ceeca00016bdd37", "team_id": "ours" }- });-- let md = map_organization(&org);-- assert_eq!(md.name.as_deref(), Some("Apollo.io"));- assert_eq!(- md.description.as_deref(),- Some("Apollo.io combines a buyer database.")- );- assert_eq!(md.icon_url.as_deref(), Some("https://example.com/logo.png"));- assert_eq!(- md.apollo_organization_id.as_deref(),- Some("5e66b6381e05b4008c8331b8")- );- assert_eq!(md.website_url.as_deref(), Some("http://www.apollo.io"));- assert_eq!(- md.linkedin_url.as_deref(),- Some("http://www.linkedin.com/company/apolloio")- );- assert_eq!(- md.industry.as_deref(),- Some("information technology & services")- );- assert_eq!(md.keywords, vec!["sales engagement", "lead generation"]);- assert_eq!(md.technologies, vec!["AI", "Android"]);- assert_eq!(md.estimated_num_employees, Some(1600));- assert_eq!(md.annual_revenue, Some(100_000_000));- assert_eq!(md.annual_revenue_printed.as_deref(), Some("100M"));- assert_eq!(md.total_funding, Some(251_200_000));- assert_eq!(md.latest_funding_stage.as_deref(), Some("Series D"));- assert_eq!(md.founded_year, Some(2015));- assert_eq!(md.city.as_deref(), Some("San Francisco"));- assert_eq!(md.country.as_deref(), Some("United States"));- assert!(md.latest_funding_round_date.is_some());-- let raw = md.raw.expect("raw present");- assert!(raw.get("account").is_none(), "account stripped from raw");- assert!(- raw.get("account_id").is_none(),- "account_id stripped from raw"- );- assert_eq!(raw.get("name").and_then(Value::as_str), Some("Apollo.io"));-}--/// A sparse / empty organization collapses to an all-empty metadata-/// (negative-cache row), with `raw` still present.-#[test]-fn missing_fields_collapse_to_empty() {- let md = map_organization(&json!({}));-- assert_eq!(md.name, None);- assert_eq!(md.annual_revenue, None);- assert_eq!(md.latest_funding_round_date, None);- assert!(md.keywords.is_empty());- assert!(md.technologies.is_empty());- assert!(md.raw.is_some());-}--/// Mistyped fields are tolerated: a string revenue / blank name don't-/// blow up the whole mapping.-#[test]-fn tolerates_mistyped_and_blank_fields() {- let org = json!({- "name": " ",- "annual_revenue": "not a number",- "estimated_num_employees": 42,- "keywords": ["ok", "", 123, " spaced "],- });-- let md = map_organization(&org);-- assert_eq!(md.name, None, "blank name -> None");- assert_eq!(md.annual_revenue, None, "string revenue -> None");- assert_eq!(md.estimated_num_employees, Some(42));- assert_eq!(- md.keywords,- vec!["ok", "spaced"],- "blank / non-string elements dropped"- );-}--/// Failures that say nothing about the domain stay retryable; only an-/// answer from Apollo becomes a (possibly empty) directory entry.-#[tokio::test]-async fn only_apollo_answers_resolve() {- let server = MockServer::start().await;- Mock::given(method("GET"))- .and(path("/api/v1/organizations/enrich"))- .and(query_param("domain", "unknown.com"))- .respond_with(ResponseTemplate::new(200).set_body_json(json!({})))- .mount(&server)- .await;- Mock::given(method("GET"))- .and(path("/api/v1/organizations/enrich"))- .and(query_param("domain", "acme.com"))- .respond_with(- ResponseTemplate::new(200)- .set_body_json(json!({ "organization": { "name": "Acme Inc." } })),- )- .mount(&server)- .await;- Mock::given(method("GET"))- .and(path("/api/v1/organizations/enrich"))- .and(query_param("domain", "busy.com"))- .respond_with(ResponseTemplate::new(429))- .mount(&server)- .await;- let resolver = ApolloCompanyMetadataResolver::with_base_url("key".to_string(), server.uri());-- assert_eq!(- resolver.resolve("unknown.com").await,- DomainResolution::Resolved(Box::default())- );- let DomainResolution::Resolved(acme) = resolver.resolve("acme.com").await else {- panic!("expected acme.com to resolve");- };- assert_eq!(acme.name.as_deref(), Some("Acme Inc."));- assert_eq!(- resolver.resolve("busy.com").await,- DomainResolution::Unavailable- );- assert_eq!(- ApolloCompanyMetadataResolver::with_base_url(String::new(), server.uri())- .resolve("acme.com")- .await,- DomainResolution::Unavailable- );-}--/// Live smoke test against the real Apollo API. Ignored by default — it-/// makes a network call and consumes an Apollo credit. Run with:-/// APOLLO_API_KEY=<key> cargo test -p crm apollo_live -- --ignored --nocapture-/// Optionally set APOLLO_TEST_DOMAIN (defaults to "macro.com").-#[tokio::test]-#[ignore = "hits the live Apollo API; requires APOLLO_API_KEY"]-async fn apollo_live_enrich() {- let api_key = std::env::var("APOLLO_API_KEY").expect("set APOLLO_API_KEY to run this test");- let domain = std::env::var("APOLLO_TEST_DOMAIN").unwrap_or_else(|_| "macro.com".to_string());-- let resolver = ApolloCompanyMetadataResolver::new(api_key);- let DomainResolution::Resolved(md) = resolver.resolve(&domain).await else {- panic!("Apollo was unavailable for {domain} — check API key and credits");- };-- println!("DomainMetadata for {domain}:\n{md:#?}");- assert!(- md.name.is_some() || md.raw.is_some(),- "expected Apollo data for {domain}; got empty — check API key, credits, or domain",- );-}diff --git a/crates/crm/src/outbound/no_op_resolver.rs b/crates/crm/src/outbound/no_op_resolver.rsindex d7a418fe6f..2644684923 100644--- a/crates/crm/src/outbound/no_op_resolver.rs+++ b/crates/crm/src/outbound/no_op_resolver.rs@@ -2,8 +2,8 @@//!//! Used by binaries that construct a [`CrmService`] without carrying a//! metadata provider — HTTP API servers, MCP, and agent tools. Wiring a-//! real resolver into each of them would force them to carry the reqwest-//! + scraper deps and provider credentials. Companies they create get a+//! real resolver into each of them would add website-fetching dependencies+//! and network work to their request paths. Companies they create get a//! pending `crm_domain_directory` entry, which the email pubsub workers'//! enrichment worker resolves with the real resolver.//!diff --git a/crates/crm/src/outbound/unfurl_resolver.rs b/crates/crm/src/outbound/unfurl_resolver.rsindex 60b54e71d0..9dcaece190 100644--- a/crates/crm/src/outbound/unfurl_resolver.rs+++ b/crates/crm/src/outbound/unfurl_resolver.rs@@ -28,6 +28,9 @@ use crate::domain::{};use unfurl::domain::ports::UnfurlService;+#[cfg(test)]+mod test;+/// Adapter that resolves [`DomainMetadata`] by unfurling/// `https://{domain}`. Holds the underlying [`UnfurlService`] behind an/// [`Arc`] so the resolver itself is cheap to [`Clone`].diff --git a/crates/crm/src/outbound/unfurl_resolver/test.rs b/crates/crm/src/outbound/unfurl_resolver/test.rsnew file mode 100644index 0000000000..5e6a0d0667--- /dev/null+++ b/crates/crm/src/outbound/unfurl_resolver/test.rs@@ -0,0 +1,67 @@+use super::*;+use unfurl::domain::models::{GetUnfurlResponse, UnfurlErr};++struct WebsiteMetadata(GetUnfurlResponse);++impl UnfurlService for WebsiteMetadata {+ async fn unfurl(&self, url: &str) -> Result<GetUnfurlResponse, UnfurlErr> {+ assert_eq!(url, "https://example.com");+ Ok(self.0.clone())+ }+}++#[tokio::test]+async fn homepage_metadata_supplies_company_name_description_and_icon() {+ let resolver =+ UnfurlCompanyMetadataResolver::new(Arc::new(WebsiteMetadata(GetUnfurlResponse {+ title: " Example ".into(),+ description: Some("Example description".into()),+ favicon_url: Some("https://example.com/icon.png".into()),+ ..Default::default()+ })));++ let DomainResolution::Resolved(metadata) = resolver.resolve("example.com").await else {+ panic!("website metadata should resolve the company");+ };+ assert_eq!(+ *metadata,+ DomainMetadata {+ name: Some("Example".into()),+ description: Some("Example description".into()),+ icon_url: Some("https://example.com/icon.png".into()),+ ..Default::default()+ }+ );+}++#[tokio::test]+async fn successful_empty_pages_do_not_turn_fallback_urls_into_company_names() {+ for title in ["https://example.com", " https://example.com ", " "] {+ let resolver =+ UnfurlCompanyMetadataResolver::new(Arc::new(WebsiteMetadata(GetUnfurlResponse {+ title: title.into(),+ ..Default::default()+ })));+ let DomainResolution::Resolved(metadata) = resolver.resolve("example.com").await else {+ panic!("a successful page without metadata is a resolved negative-cache entry");+ };+ assert_eq!(*metadata, DomainMetadata::default());+ }+}++struct UnavailableWebsite;++impl UnfurlService for UnavailableWebsite {+ async fn unfurl(&self, _url: &str) -> Result<GetUnfurlResponse, UnfurlErr> {+ Err(UnfurlErr::Fetch(anyhow::anyhow!("website unavailable")))+ }+}++#[tokio::test]+async fn failed_fetch_stays_retryable_instead_of_resolving_an_empty_company() {+ let resolver = UnfurlCompanyMetadataResolver::new(Arc::new(UnavailableWebsite));+ assert!(matches!(+ resolver.resolve("example.com").await,+ DomainResolution::Unavailable+ ));+}diff --git a/deny.toml b/deny.tomlindex a66e92eae3..c55c95a74f 100644--- a/deny.toml+++ b/deny.toml@@ -229,6 +229,7 @@ deny = ["document_storage_service", # GraphQL telemetry span and log tests only (dev-dependency)"documents", # Dev logging only"email", # Dev logging only+ "email_api_client", # Outlook transport span tests only"email_db_client", # Purge phase span tests only"forms", # Anonymous-response span and log leak tests only (dev-dependency)"genai_telemetry", # GenAI span testsdiff --git a/infra/stacks/email-service/Pulumi.dev.yaml b/infra/stacks/email-service/Pulumi.dev.yamlindex 99af3dce80..b43ec6104b 100644--- a/infra/stacks/email-service/Pulumi.dev.yaml+++ b/infra/stacks/email-service/Pulumi.dev.yaml@@ -12,8 +12,6 @@ config:email-service:internal_auth_key: document-storage-service-auth-key-devemail-service:cf_private_key: email-attachments-private-key-devemail-service:notifications_enabled: true- email-service:use_apollo_crm_enrichment: true- email-service:apollo_api_key_secret_name: apollo-io-api-key-masteremail-service:sent_undo_delay_secs: 5email-service:redis_rate_limit_reqs: 14000email-service:redis_rate_limit_reqs_backfill: 13000diff --git a/infra/stacks/email-service/Pulumi.prod.yaml b/infra/stacks/email-service/Pulumi.prod.yamlindex 476cabac0d..c6bf387b3c 100644--- a/infra/stacks/email-service/Pulumi.prod.yaml+++ b/infra/stacks/email-service/Pulumi.prod.yaml@@ -12,8 +12,6 @@ config:email-service:internal_auth_key: document-storage-service-auth-key-prodemail-service:cf_private_key: email-attachments-private-key-prodemail-service:notifications_enabled: true- email-service:use_apollo_crm_enrichment: false- email-service:apollo_api_key_secret_name: apollo-io-api-key-masteremail-service:sent_undo_delay_secs: 5email-service:redis_rate_limit_reqs: 14000email-service:redis_rate_limit_reqs_backfill: 13000diff --git a/infra/stacks/email-service/index.ts b/infra/stacks/email-service/index.tsindex 9cbfc26ebc..bc822b05ba 100644--- a/infra/stacks/email-service/index.ts+++ b/infra/stacks/email-service/index.ts@@ -37,7 +37,6 @@ export const coparse_api_vpc = get_coparse_api_vpc();const JWT_SECRET_KEY = config.require(`jwt_secret_key`);-const APOLLO_API_KEY_SECRET_NAME = config.require(`apollo_api_key_secret_name`);const jwtSecretKeyArn: pulumi.Output<string> = aws.secretsmanager.getSecretVersionOutput({ secretId: JWT_SECRET_KEY }).apply((secret) => secret.arn);@@ -111,10 +110,6 @@ const macroDbUrlArn: pulumi.Output<string> = aws.secretsmanager.getSecretVersionOutput({ secretId: MACRO_DB_URL_SECRET_NAME }).apply((secret) => secret.arn);-const apolloApiKeySecretArn: pulumi.Output<string> = aws.secretsmanager- .getSecretVersionOutput({ secretId: APOLLO_API_KEY_SECRET_NAME })- .apply((secret) => secret.arn);-const inbox_sync_queue = new Queue('email-service-gmail-webhook', {tags,maxReceiveCount: 3,@@ -283,7 +278,6 @@ const secretKeyArns = [macroDbUrlArn,MACRO_API_TOKENS.macroApiTokenPublicKeyArn,cloudfrontSecretKey.arn,- apolloApiKeySecretArn,];const queueArns = [diff --git a/services/email_service/AGENTS.md b/services/email_service/AGENTS.mdindex 481e9fa1bf..27b6cebf38 100644--- a/services/email_service/AGENTS.md+++ b/services/email_service/AGENTS.md@@ -18,7 +18,9 @@ the gateway `/email` prefix.- `src/backfill_outbox.rs`, `src/backfill_init_service.rs`,`src/backfill_completion_service.rs`: Gmail history backfill orchestration.- `src/crm_enrichment_worker.rs`: periodic pass that resolves pending CRM- company directory entries (names/logos) left by binaries without a resolver.+ company directory entries (names/logos) left by binaries without a resolver,+ using the public-homepage `UnfurlCompanyMetadataResolver` wired in+ `pubsub_workers`. It requires no enrichment-provider key or selection flag.- `src/mailbox_sync_worker.rs`: the Outlook mailbox loops (sync, projections,contacts, drafts, commands, settings, watches) and `run_health`, theper-minute Outlook health snapshot.diff --git a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rsindex 5aeb62eb77..75b5a02912 100644--- a/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs+++ b/services/email_service/src/bin/pubsub_workers/pubsub_workers.rs@@ -385,38 +385,12 @@ async fn main() -> anyhow::Result<()> {// The CRM crate's company-metadata resolver is consulted by// `crm_service.populate_contact` on `crm_domain_directory` misses and by// the enrichment worker for pending entries other binaries leave behind.- // `USE_APOLLO_CRM_ENRICHMENT` selects Apollo.io vs. the unfurl-backed- // resolver; we also fall back to unfurl when the Apollo key can't be- // loaded. The resolver is cheap to clone.- let build_unfurl = || -> anyhow::Result<CrmMetadataResolver> {- // Wrap the SSRF-safe reqwest fetcher in an `UnfurlServiceImpl`,- // then the `UnfurlCompanyMetadataResolver`.- let unfurl_service = Arc::new(unfurl::domain::service::UnfurlServiceImpl::new(- unfurl::outbound::ReqwestUnfurlFetcher::new()- .context("failed to build ReqwestUnfurlFetcher")?,- ));- Ok(CrmMetadataResolver::Unfurl(- crm::outbound::unfurl_resolver::UnfurlCompanyMetadataResolver::new(unfurl_service),- ))- };-- let metadata_resolver = if config.use_apollo_crm_enrichment {- // No usable key (missing/unreadable secret, or unset locally): fall- // back to unfurl rather than running Apollo with an empty key, which- // would no-op and pollute the directory with negative-cache rows.- if config.apollo_api_key.as_ref().is_empty() {- tracing::warn!("apollo api key unavailable; falling back to unfurl CRM enrichment");- build_unfurl()?- } else {- CrmMetadataResolver::Apollo(- crm::outbound::apollo_resolver::ApolloCompanyMetadataResolver::new(- config.apollo_api_key.as_ref().to_string(),- ),- )- }- } else {- build_unfurl()?- };+ // Resolve public homepage metadata with the SSRF-safe unfurl fetcher.+ let unfurl_service = Arc::new(unfurl::domain::service::UnfurlServiceImpl::new(+ unfurl::outbound::ReqwestUnfurlFetcher::new()+ .context("failed to build ReqwestUnfurlFetcher")?,+ ));+ let metadata_resolver = CrmMetadataResolver::new(unfurl_service);let crm_service = crm::domain::service::CrmServiceImpl::new(crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),diff --git a/services/email_service/src/config.rs b/services/email_service/src/config.rsindex 0bb1541ca3..cc7f0f5c7d 100644--- a/services/email_service/src/config.rs+++ b/services/email_service/src/config.rs@@ -16,7 +16,6 @@ env_vars! {pub struct AuthenticationServiceSecretKey;pub struct EmailServiceCloudfrontDistributionUrl;pub struct EmailServiceCloudfrontSignerPublicKeyId;- pub struct ApolloApiKey;}#[derive(macro_config::MacroConfig)]@@ -65,16 +64,6 @@ pub struct Config {#[macro_config_default(true)]pub notifications_enabled: bool,- /// Use Apollo.io for CRM company enrichment. When `false`, fall back- /// to the unfurl-based resolver.- #[macro_config_default(false)]- pub use_apollo_crm_enrichment: bool,-- /// Apollo.io API key for CRM enrichment. Locally this is the key- /// itself; in deployed envs it's the name of the Secrets Manager- /// secret holding it (resolved at startup). Empty disables enrichment.- pub apollo_api_key: LocalOrRemoteSecret<ApolloApiKey>,-/// The queue max messages per poll#[macro_config_default(10)]pub queue_max_messages: i32,diff --git a/services/email_service/src/crm_enrichment_worker.rs b/services/email_service/src/crm_enrichment_worker.rsindex 467ab269fb..6bac9ff533 100644--- a/services/email_service/src/crm_enrichment_worker.rs+++ b/services/email_service/src/crm_enrichment_worker.rs@@ -6,7 +6,7 @@ use crate::pubsub::context::CrmServiceType;use std::time::Duration;use tokio_util::sync::CancellationToken;-/// One enrichment pass per interval keeps the resolver (Apollo) call rate at+/// One enrichment pass per interval keeps the resolver call rate at/// the service's batch size per minute per worker.const ENRICHMENT_INTERVAL: Duration = Duration::from_secs(60);diff --git a/services/email_service/src/pubsub/context.rs b/services/email_service/src/pubsub/context.rsindex 2308cf61ad..ca3efa9fe1 100644--- a/services/email_service/src/pubsub/context.rs+++ b/services/email_service/src/pubsub/context.rs@@ -3,9 +3,7 @@ use crate::util::redis::RedisClient;use connection_gateway_client::client::ConnectionGatewayClient;use contacts::domain::service::SqsContactsIngress;use contacts::outbound::ingress::SqsContactsQueue;-use crm::domain::company_metadata_resolver::{CompanyMetadataResolver, DomainResolution};use crm::domain::service::CrmServiceImpl;-use crm::outbound::apollo_resolver::ApolloCompanyMetadataResolver;use crm::outbound::companies_repo::CompaniesRepositoryImpl;use crm::outbound::unfurl_resolver::UnfurlCompanyMetadataResolver;use document_storage_service_client::DocumentStorageServiceClient;@@ -24,35 +22,13 @@ pub type PubSubEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTr/// The concrete notification ingress service type.pub type NotificationIngressType = SqsNotificationIngress<SqsQueue>;-/// The unfurl-backed resolver used when Apollo enrichment is disabled.-type UnfurlResolver = UnfurlCompanyMetadataResolver<+/// CRM company metadata resolved from the company's public homepage.+pub type CrmMetadataResolver = UnfurlCompanyMetadataResolver<unfurl::domain::service::UnfurlServiceImpl<unfurl::outbound::ReqwestUnfurlFetcher>,>;-/// CRM company-metadata resolver, chosen at startup from the-/// `USE_APOLLO_CRM_ENRICHMENT` flag: Apollo.io when enabled, the-/// unfurl-backed resolver otherwise. A single concrete type so it slots-/// into [`CrmServiceType`] — `CompanyMetadataResolver` is RPITIT and thus-/// not dyn-compatible, so we dispatch via an enum rather than `dyn`.-#[derive(Clone)]-pub enum CrmMetadataResolver {- /// Apollo.io organization enrichment.- Apollo(ApolloCompanyMetadataResolver),- /// Unfurl-backed homepage metadata.- Unfurl(UnfurlResolver),-}--impl CompanyMetadataResolver for CrmMetadataResolver {- async fn resolve(&self, domain: &str) -> DomainResolution {- match self {- CrmMetadataResolver::Apollo(r) => r.resolve(domain).await,- CrmMetadataResolver::Unfurl(r) => r.resolve(domain).await,- }- }-}-/// The concrete CRM service type, backed by Postgres and the-/// flag-selected [`CrmMetadataResolver`]. The resolver is consulted only+/// homepage-backed [`CrmMetadataResolver`]. The resolver is consulted only/// on `crm_domain_directory` misses and for pending entries, so it isn't/// surfaced separately on [`PubSubContext`].pub type CrmServiceType = CrmServiceImpl<CompaniesRepositoryImpl, CrmMetadataResolver>;diff --git a/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs b/services/email_service/src/pubsub/label_write_sim/fake_sqs.rsindex 20f585376f..6a65b6c3ac 100644--- a/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs+++ b/services/email_service/src/pubsub/label_write_sim/fake_sqs.rs@@ -13,6 +13,9 @@ use serde_json::{Value, json};use std::sync::{Arc, Mutex};use tokio::sync::oneshot;+#[cfg(test)]+mod test;+#[derive(Clone, Debug)]pub struct Sent {pub queue_url: String,@@ -54,7 +57,8 @@ impl FakeSqs {.expect("bind fake sqs");let url = format!("http://{}", listener.local_addr().unwrap());let fake = FakeSqs { state, url };- let app = Router::new().fallback(handle).with_state(fake.clone());+ let app =+ Router::new().fallback_service(axum::routing::post(handle).with_state(fake.clone()));tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });fake}diff --git a/services/email_service/src/pubsub/label_write_sim/fake_sqs/test.rs b/services/email_service/src/pubsub/label_write_sim/fake_sqs/test.rsnew file mode 100644index 0000000000..0564dea53e--- /dev/null+++ b/services/email_service/src/pubsub/label_write_sim/fake_sqs/test.rs@@ -0,0 +1,20 @@+use super::*;++#[tokio::test]+async fn http_probe_does_not_poison_subsequent_sqs_requests() {+ let sqs = FakeSqs::start().await;+ let response = reqwest::get(&sqs.url).await.expect("probe response");+ assert_eq!(response.status().as_u16(), 405);++ client(&sqs.url, false)+ .send_message()+ .queue_url(sqs.queue_url("test"))+ .message_body("after probe")+ .send()+ .await+ .expect("SQS remains usable after a non-SQS request");+ let (sent, acks) = sqs.take();+ assert_eq!(sent.len(), 1);+ assert_eq!(sent[0].body, "after probe");+ assert!(acks.is_empty());+}diff --git a/services/email_service/src/pubsub/label_write_sim/world.rs b/services/email_service/src/pubsub/label_write_sim/world.rsindex 7809eaea02..cd5f603eab 100644--- a/services/email_service/src/pubsub/label_write_sim/world.rs+++ b/services/email_service/src/pubsub/label_write_sim/world.rs@@ -251,13 +251,9 @@ impl World {);let crm_service = crm::domain::service::CrmServiceImpl::new(crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),- CrmMetadataResolver::Unfurl(- crm::outbound::unfurl_resolver::UnfurlCompanyMetadataResolver::new(Arc::new(- unfurl::domain::service::UnfurlServiceImpl::new(- unfurl::outbound::ReqwestUnfurlFetcher::new().expect("unfurl fetcher"),- ),- )),- ),+ CrmMetadataResolver::new(Arc::new(unfurl::domain::service::UnfurlServiceImpl::new(+ unfurl::outbound::ReqwestUnfurlFetcher::new().expect("unfurl fetcher"),+ ))),);let sync_context = |retry_worker: bool, queue: &str| PubSubContext {gmail_history: email::domain::mailbox::gmail_history::GmailHistoryService(diff --git a/services/email_service/src/pubsub/mod.rs b/services/email_service/src/pubsub/mod.rsindex 7eaa5a3d9c..6c8dd20efa 100644--- a/services/email_service/src/pubsub/mod.rs+++ b/services/email_service/src/pubsub/mod.rs@@ -1,7 +1,7 @@pub mod backfill;pub mod context;pub mod invitation_extraction;-/// The flag-selected CRM metadata resolver, re-exported for the+/// The homepage-backed CRM metadata resolver, re-exported for the/// pubsub_workers binary to construct.pub use context::CrmMetadataResolver;pub mod crm_cleanup;diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env.rs b/tooling/xtask/crates/xtask_local/src/local/local_env.rsindex 989bd4d0e9..2f8367c841 100644--- a/tooling/xtask/crates/xtask_local/src/local/local_env.rs+++ b/tooling/xtask/crates/xtask_local/src/local/local_env.rs@@ -799,7 +799,6 @@ impl BootStubEnv {// email_service's GCP pubsub queue (gmail watch notifications) and// its own CloudFront signer for attachment presigned URLs.env.insert("GMAIL_GCP_QUEUE".into(), "gmail-gcp-queue-local".into());- env.insert("APOLLO_API_KEY".into(), "local".into());env.insert("EMAIL_SERVICE_CLOUDFRONT_DISTRIBUTION_URL".into(),"http://localhost:8100".into(),diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rsindex b3c40248ff..e1ee9a99b5 100644--- a/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs+++ b/tooling/xtask/crates/xtask_local/src/local/local_env/test.rs@@ -124,7 +124,6 @@ fn emits_required_keys() {"MACRO_API_TOKEN_PRIVATE_SECRET_KEY","MACRO_API_TOKEN_EXPIRY_SECONDS","GMAIL_GCP_QUEUE",- "APOLLO_API_KEY","EMAIL_SERVICE_CLOUDFRONT_DISTRIBUTION_URL","EMAIL_SERVICE_CLOUDFRONT_SIGNER_PRIVATE_KEY","EMAIL_SERVICE_CLOUDFRONT_SIGNER_PUBLIC_KEY_ID",