Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
//! The service's composition root: reads config and wires every adapter and
//! domain service into the HTTP server.
use crate::api::{
self,
context::{ApiContext, AuthorizationService},
};
use crate::calendar_refresh::ConnectionGatewayCalendarRefresh;
use crate::calendar_request_gate::RedisCalendarRequestGate;
use crate::calendar_tokens::CalendarTokenProviderAdapter;
use crate::outbound::email_api::{
EmailServiceTokenSource, GmailApi, RateBudget, RedisGmailRequestGate,
};
use anyhow::Context;
use calendar_events::{
domain::{
invitations::CalendarInvitationResolver, mutations::CalendarMutationServiceImpl,
service::CalendarService,
},
outbound::{google::GoogleCalendarClient, pg::PgCalendarRepository},
};
use document_storage_service_client::DocumentStorageServiceClient;
use email::{
domain::service::EmailServiceImpl,
inbound::axum::{
axum_impls::GmailTokenState, get_thread_router::EmailThreadRouterState,
previews_router::EmailRouterState,
},
outbound::{EmailPgRepo, GmailTokenProviderImpl},
};
use email_api_client::GmailApiClientRepository;
use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
use frecency::{domain::services::FrecencyQueryServiceImpl, outbound::postgres::FrecencyPgStorage};
use macro_auth::middleware::decode_jwt::JwtValidationArgs;
use macro_authorization::{
InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationState,
PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
};
use macro_env::Environment;
use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use macro_service_urls::{
AuthServiceUrl, ConnectionGatewayUrl, DocumentStorageServiceUrl, StaticFileServiceUrl,
};
use sqlx::postgres::PgPoolOptions;
use static_file_service_client::StaticFileServiceClient;
use std::{sync::Arc, time::Duration};
use system_properties::{PgSystemPropertiesRepository, SystemPropertiesServiceImpl};
use tokio_util::task::TaskTracker;
/// Build the service from the environment and serve until shutdown.
// Not a `pub async fn`: see CS-73.
pub fn run() -> impl Future<Output = anyhow::Result<()>> {
run_server()
}
async fn run_server() -> anyhow::Result<()> {
let env = Environment::new_or_prod();
let aws_config = macro_aws_config::get_macro_aws_config().await;
let s3_client = s3_client::S3::new(macro_aws_config::s3_client().await);
let secretsmanager_client = secretsmanager_client::SecretsManager::new(
aws_sdk_secretsmanager::Client::new(&aws_config),
);
// Parse our configuration from the environment, then resolve any secret-manager backed values.
let config = crate::config::Config::from_env()
.context("expected to be able to generate config")?
.resolve_remote_secrets(env, &secretsmanager_client)
.await
.context("expected to be able to resolve config secrets")?;
// limiting to max of 200 connections (12.5% of macrodb total) in prod.
let (min_connections, max_connections): (u32, u32) = match config.environment {
Environment::Production => (3, 20),
Environment::Develop => (1, 10),
Environment::Local => (1, 10),
};
let db = PgPoolOptions::new()
.min_connections(min_connections)
.max_connections(max_connections)
.connect(&config.macro_db_url)
.await
.context("could not connect to db")?;
let gmail_inbox_sync_queue = macro_queues::GmailInboxSyncQueue::new();
let gmail_inbox_sync_retry_queue = macro_queues::GmailInboxSyncRetryQueue::new();
let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
let backfill_queue = macro_queues::EmailBackfillQueue::new();
let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
let sfs_uploader_queue = macro_queues::SfsUploaderQueue::new();
let link_manager_queue = macro_queues::LinkManagerQueue::new();
let sqs_client = sqs_client::SQS::new(macro_aws_config::sqs_client().await)
.gmail_inbox_sync_queue(&gmail_inbox_sync_queue)
.gmail_inbox_sync_retry_queue(&gmail_inbox_sync_retry_queue)
.gmail_ops_queue(&gmail_ops_queue)
.gmail_label_write_batches(config.gmail_label_write_batches_enabled)
.email_backfill_queue(&backfill_queue)
.email_scheduled_queue(&email_scheduled_queue)
.sfs_uploader_queue(&sfs_uploader_queue)
.email_link_manager_queue(&link_manager_queue);
let auth_service_client = authentication_service_client::AuthServiceClient::new(
config
.authentication_service_secret_key
.as_ref()
.to_string(),
AuthServiceUrl::new()?.to_string(),
);
let gmail_client = gmail_client::GmailClient::new(config.gmail_gcp_queue.as_ref().to_string());
let gmail_api_repository = GmailApiClientRepository::new(gmail_client.clone());
let redis_inner_client = redis::Client::open(config.redis_uri.as_ref())
.inspect(|client| {
client
.get_connection()
.map(|_| tracing::info!("initialized redis connection"))
.inspect_err(|e| {
tracing::error!(error=?e, "failed to connect to redis");
})
.ok();
})
.context("failed to connect to redis")?;
let redis_client = crate::util::redis::RedisClient::new(
redis_inner_client,
config.redis_rate_limit_window_secs,
);
let sfs_client = StaticFileServiceClient::new(
config.internal_api_key.to_string(),
StaticFileServiceUrl::new()?.to_string(),
);
let dss_client = DocumentStorageServiceClient::new(
config.internal_api_key.to_string(),
DocumentStorageServiceUrl::new()?.to_string(),
);
let system_properties_service = Arc::new(SystemPropertiesServiceImpl::new(
PgSystemPropertiesRepository::new(db.clone()),
));
let jwt_args =
JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
.await?;
let authorization_state = MacroAuthorizationState::new(Arc::new(AuthorizationService::new(
MacroAuthJwtValidator::new(jwt_args.clone()),
InternalAuthConfig {
api_key: config.internal_api_key.to_string(),
default_user_id: Some("macro|[email protected]".to_string()),
},
macro_authorization::NoBotAuthorizer,
PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
)));
let sqs_client = Arc::new(sqs_client);
let gmail_client = Arc::new(gmail_client);
// HTTP API only reads CRM rows — populate runs in the pubsub
// worker. The no-op resolver makes the unused branch explicit and
// keeps reqwest/scraper out of this binary.
let crm_service = crm::domain::service::CrmServiceImpl::new(
crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
);
let event_broker_tracker = TaskTracker::new();
let macro_event_broker = MacroEventBrokerService::new(
KafkaEventPublisher::new(config.kafka_brokers.as_ref())
.context("failed to create kafka event publisher")?,
event_broker_tracker.clone(),
);
let email_service = EmailRouterState::new(
EmailServiceImpl::new(
EmailPgRepo::new(db.clone()),
FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),
(*sqs_client).clone(),
crm_service,
entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
entity_access_management::outbound::PgRepository::new(db.clone()),
),
config.sent_undo_delay_secs,
)
.with_macro_event_broker(macro_event_broker.clone()),
);
let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
db.clone(),
)));
let email_thread_state = EmailThreadRouterState {
service: email_service.service(),
access_service: entity_access_service.clone(),
authorization_state: authorization_state.clone(),
};
let auth_service_client = Arc::new(auth_service_client);
let redis_conn = redis_client
.inner
.get_multiplexed_async_connection()
.await
.context("failed to get multiplexed redis connection for gmail token provider")?;
let email_api = GmailApi::new(
gmail_api_repository,
EmailServiceTokenSource::new(
db.clone(),
redis_conn.clone(),
auth_service_client.as_ref().clone(),
sqs_client.as_ref().clone(),
),
RedisGmailRequestGate::new(
redis_client.inner.get_connection_manager().await.context(
"failed to get reconnecting redis connection for the gmail request gate",
)?,
RateBudget::Live,
),
);
let redis_client = Arc::new(redis_client);
let gmail_token_state = GmailTokenState::new(GmailTokenProviderImpl::new(
redis_conn.clone(),
auth_service_client.clone(),
));
let calendar_service = Arc::new(CalendarService::new(PgCalendarRepository::new(db.clone())));
let connection_gateway_client = connection_gateway_client::client::ConnectionGatewayClient::new(
config.internal_api_key.to_string(),
ConnectionGatewayUrl::new()?.to_string(),
);
let calendar_mutation_service = Arc::new(
CalendarMutationServiceImpl::new(
PgCalendarRepository::new(db.clone()),
calendar_events::domain::providers::CalendarProviders {
outlook: email_api_client::OutlookApiClientRepository::with_gate(Arc::new(
email_api_client::outbound::microsoft_gate::RedisMicrosoftRequestGate(
redis_conn.clone(),
),
))?
.with_rejected_token_refresh(Arc::new(
CalendarTokenProviderAdapter::new(
redis_conn.clone(),
auth_service_client.clone(),
),
)),
google: GoogleCalendarClient::with_gate(
reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30))
.build()
.context("failed to build the google calendar mutation http client")?,
RedisCalendarRequestGate::new((*redis_client).clone()),
),
},
CalendarTokenProviderAdapter::new(redis_conn.clone(), auth_service_client.clone()),
macro_event_broker.clone(),
ConnectionGatewayCalendarRefresh::new(connection_gateway_client.clone(), db.clone()),
)
.with_outlook_writes_enabled(config.outlook_writes_enabled),
);
let outlook_api: crate::api::context::OutlookApi =
email_api_client::domain::service::mailbox::MailboxApiService::new(
email_api_client::OutlookApiClientRepository::with_gate(Arc::new(
crate::outbound::email_api::RedisMicrosoftRequestGate(redis_conn.clone()),
))?,
crate::api::context::MicrosoftTokens::new(
email::outbound::mailbox_pg::PgMailboxSync::new(db.clone()),
crate::outbound::email_api::MicrosoftCredentialsClient(
auth_service_client.as_ref().clone(),
),
),
email_api_client::domain::ports::AlwaysAllowRateLimiter,
);
let transfer_bytes = Arc::new(
email::domain::mailbox::drafts::content::DraftContentService {
repository: EmailPgRepo::new(db.clone()),
bytes: crate::outbound::mailbox_drafts::MailboxDraftContent {
provider: crate::outbound::attachment_access::ProviderAttachmentBytes {
s3: s3_client.clone(),
bucket: config.attachment_bucket.to_string(),
gmail: email_api.clone(),
outlook: crate::outbound::email_api::ProviderMailboxGateway(
outlook_api.clone(),
),
},
s3: s3_client.clone(),
bucket: config.attachment_bucket.to_string(),
},
access: entity_access_service.as_ref().clone(),
},
);
let api_result = api::setup_and_serve(ApiContext {
draft_transfers: Arc::new(email::domain::draft_transfer::DraftTransferService {
repository: EmailPgRepo::new(db.clone()),
bytes: transfer_bytes,
storage: crate::outbound::draft_attachment_storage::DraftAttachmentS3 {
s3: s3_client.clone(),
bucket: config.attachment_bucket.to_string(),
},
}),
inbox_lifecycle: Arc::new(crate::composition::inbox_lifecycle(
email_api_client::OutlookApiClientRepository::with_gate(Arc::new(
crate::outbound::email_api::RedisMicrosoftRequestGate(redis_conn.clone()),
))?,
db.clone(),
auth_service_client.as_ref().clone(),
sqs_client.as_ref().clone(),
connection_gateway_client.clone(),
)),
mailbox_settings: Arc::new(
email::domain::mailbox::settings::MailboxSettingsService::new(
email::outbound::mailbox_pg::PgMailboxSync::new(db.clone()),
crate::outbound::mailbox_settings::ProviderMailboxSettings {
gmail: email_api.clone(),
outlook: outlook_api.clone(),
},
),
),
mailbox_notifications: Arc::new(
email::domain::mailbox::watches::MailboxWatchNotifications(
email::outbound::mailbox_pg::PgMailboxSync::new(db.clone()),
),
),
inbox_catalog: Arc::new(email::domain::mailbox::catalog::InboxCatalogService(
crate::outbound::inbox_catalog::PgInboxCatalog(db.clone()),
)),
inbox_health: Arc::new(crate::composition::inbox_health(
db.clone(),
email_api.clone(),
auth_service_client.as_ref().clone(),
redis_client.as_ref().clone(),
sqs_client.as_ref().clone(),
macro_event_broker.clone(),
std::sync::Arc::new(contacts::domain::service::SqsContactsIngress {
queue: contacts::outbound::ingress::SqsContactsQueue::new(
aws_sdk_sqs::Client::new(&aws_config),
macro_queues::ContactsQueue::new().to_string(),
),
}),
)),
message_sources: Arc::new(email::domain::message_source::MessageSourceService {
repository: EmailPgRepo::new(db.clone()),
provider: crate::outbound::message_source::ProviderMessageSource {
gmail: email_api.clone(),
outlook: outlook_api.clone(),
},
access: entity_access_service.as_ref().clone(),
}),
attachment_reads: Arc::new(email::domain::attachment_access::AttachmentReadService {
repository: EmailPgRepo::new(db.clone()),
provider: crate::outbound::attachment_access::ProviderAttachmentBytes {
s3: s3_client.clone(),
bucket: config.attachment_bucket.to_string(),
gmail: email_api.clone(),
outlook: crate::outbound::email_api::ProviderMailboxGateway(outlook_api),
},
files: crate::outbound::attachment_access::MailAttachmentFiles {
db: db.clone(),
s3: s3_client.clone(),
bucket: config.attachment_bucket.to_string(),
distribution_url: config.email_service_cloudfront_distribution_url.to_string(),
public_key_id: config
.email_service_cloudfront_signer_public_key_id
.to_string(),
private_key: config
.email_service_cloudfront_signer_private_key
.as_ref()
.to_owned(),
url_ttl_seconds: config.email_service_presigned_url_ttl_secs,
dss: dss_client.clone(),
sfs: sfs_client.clone(),
properties: system_properties_service.clone(),
},
access: entity_access_service.as_ref().clone(),
}),
event_forward_sender: Arc::new(
email::domain::service::attachments::StagedAttachmentSender::new(
email_service.service(),
EmailPgRepo::new(db.clone()),
crate::outbound::draft_attachment_storage::DraftAttachmentS3 {
s3: s3_client.clone(),
bucket: config.attachment_bucket.to_string(),
},
),
),
draft_attachments: Arc::new(email::domain::draft_attachments::DraftAttachmentService {
repository: email::outbound::EmailPgRepo::new(db.clone()),
storage: crate::outbound::draft_attachment_storage::DraftAttachmentS3 {
s3: s3_client.clone(),
bucket: config.attachment_bucket.to_string(),
},
access: entity_access_service.as_ref().clone(),
}),
mailbox_initializer: Arc::new(
email::domain::mailbox::initialization::MailboxInitializationService::new(
crate::outbound::mailbox_init::PgMailboxInitialization(db.clone()),
crate::outbound::mailbox_init::MicrosoftGrantSource(
auth_service_client.as_ref().clone(),
),
)
.with_connections_enabled(config.outlook_connections_enabled),
),
read_receipts: crate::composition::read_receipts(db.clone(), env)?,
invitation_snapshots: email::outbound::invitation_pg::InvitationPgRepository(db.clone()),
// calendar_service's sync kill switch rejects RSVP writes itself.
invitation_resolver: Arc::new(CalendarInvitationResolver::new(PgCalendarRepository::new(
db.clone(),
))),
db,
internal_api_key: config.internal_api_key.clone(),
config: Arc::new(config),
auth_service_client,
redis_client,
sqs_client,
sfs_client: Arc::new(sfs_client),
gmail_client: gmail_client.clone(),
email_api,
s3_client: Arc::new(s3_client),
dss_client: Arc::new(dss_client),
system_properties_service,
authorization_state: authorization_state.clone(),
jwt_args,
email_service,
entity_access_service,
email_thread_state,
gmail_token_state,
macro_event_broker: Arc::new(macro_event_broker),
calendar_service,
calendar_mutation_service,
})
.await;
tracing::info!("waiting for event broker publishes to drain");
event_broker_tracker.close();
match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
Ok(()) => tracing::info!("event broker publishes drained"),
Err(error) => {
tracing::warn!(
error=?error,
timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
"timed out waiting for event broker publishes to drain"
);
}
}
api_result
}
const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);