Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
//! The service's composition root: reads config and wires every adapter and
//! domain service into the HTTP server.
use crate::cognition::{
api::{self, context::ApiContext},
config::{Config, Environment},
service::{
self,
integration_analytics::{IntegrationAnalytics, IntegrationChange, IntegrationEvent},
},
};
use ai_tools::{NoOpCallRtcClient, NoOpConnectionService, NoOpNotificationIngress};
use anyhow::Context;
use bots::outbound::pg_bots_repo::PgBotsRepo;
use call::domain::service::{CallRecordQueryServiceImpl, CallServiceImpl};
use call::inbound::toolset::CallToolContext;
use call::outbound::pg_call_repo::PgCallRepo;
use call::outbound::s3_recording_storage::S3RecordingStorage;
use channels::{
domain::list_service::ChannelListServiceImpl, outbound::pg_channels_repo::PgChannelsRepo,
};
use document_storage_service_client::DocumentStorageServiceClient;
use documents::{
domain::{models::CloudFrontConfig, service::DocumentServiceImpl},
inbound::toolset::DocumentToolContext,
outbound::{pg_document_repo::PgDocumentRepo, s3_upload_url::S3UploadUrlAdapter},
};
use email::domain::ports::ReadonlyEmailPreviewAdapter;
use email::domain::service::EmailServiceImpl;
use email::outbound::EmailPgRepo;
use email_service_client::{EmailServiceClient, EmailServiceClientExternal};
use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
use entity_registry::OwnerGrantPolicy;
use entity_registry_db_utils::OwnedEntityRegistrar;
use foreign_entity::{
domain::service::ForeignEntityServiceImpl,
outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
};
use frecency::domain::services::FrecencyQueryServiceImpl;
use frecency::outbound::postgres::FrecencyPgStorage;
use github_pull_requests::{
domain::service::GithubPullRequestServiceImpl,
outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,
};
use macro_auth::middleware::decode_jwt::JwtValidationArgs;
use macro_authorization::{
InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,
MacroAuthorizationState, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
};
use macro_service_urls::{
AppServiceUrl, AuthServiceUrl, CalendarServiceUrl, ConnectionGatewayUrl,
DocumentStorageServiceUrl, EmailServiceUrl, LexicalServiceUrl, StaticFileServiceUrl,
SyncServiceUrl,
};
use notification::domain::service::{
NotificationReaderService, PlatformArnConfig, SqsNotificationIngress,
};
use notification::outbound::queue::SqsQueue;
use notification::outbound::repository::DbNotificationRepository;
use notification::outbound::websocket::ConnectionGatewayClient;
use readonly_pool::ReadOnlyPool;
use search_service_client::SearchServiceClient;
use std::sync::Arc;
use stream::outbound::redis_pg::RedisPostgresStreamRepo;
use sync_service_client::SyncServiceClient;
use tokio_util::task::TaskTracker;
/// The cognition routes and workers owned by the harness process.
pub struct CognitionRuntime {
/// Routes mounted on the harness's public listener.
pub router: axum::Router,
workers: Vec<tokio_util::task::AbortOnDropHandle<()>>,
cancellation: tokio_util::sync::CancellationToken,
}
impl CognitionRuntime {
/// Stop intake before the shared HTTP listener and sessions drain.
pub fn stop(&self) {
self.cancellation.cancel();
for worker in &self.workers {
worker.abort();
}
}
/// Stop workers before the harness drains pending event publishes.
pub async fn shutdown(self) {
self.stop();
for worker in self.workers {
worker.abort();
let _ = worker.await;
}
}
}
/// Compose chat, completions, integrations, and workers in the harness process.
///
/// `import_jobs` are the harness's import jobs: this runtime's Notion imports
/// and the stalled-import sweeper run on them, and the harness hands them
/// over on shutdown.
pub async fn build(
config: Config,
db: sqlx::PgPool,
event_broker_tracker: TaskTracker,
import_jobs: ai_tools::ImportJobs,
) -> anyhow::Result<CognitionRuntime> {
let public_url = macro_service_urls::CognitionPublicUrl::new()?;
let aws_config = macro_aws_config::get_macro_aws_config().await;
let secretsmanager_client = secretsmanager_client::SecretsManager::new(
aws_sdk_secretsmanager::Client::new(&aws_config),
);
let config = config
.resolve_remote_secrets(Environment::new_or_prod(), &secretsmanager_client)
.await
.context("failed to resolve cognition secrets")?;
let non_user_owners = config.non_user_owners()?;
let dynamodb_client = aws_sdk_dynamodb::Client::new(&aws_config);
let queue_aws_client = aws_sdk_sqs::Client::new(&aws_config);
let document_text_extractor_queue = macro_queues::DocumentTextExtractorQueue::new();
let chat_delete_queue = macro_queues::ChatDeleteQueue::new();
let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
let ai_projection_queue = macro_queues::AiProjectionQueue::new();
let notification_queue = macro_queues::NotificationIngressQueue::new();
let sqs_client = sqs_client::SQS::new(queue_aws_client)
.document_text_extractor_queue(&document_text_extractor_queue)
.chat_delete_queue(&chat_delete_queue)
.email_scheduled_queue(&email_scheduled_queue)
.gmail_ops_queue(&gmail_ops_queue)
.ai_projection_queue(&ai_projection_queue);
let internal_api_key = config.internal_api_key.to_string();
let document_storage_client = DocumentStorageServiceClient::new(
config
.document_storage_service_auth_key
.as_ref()
.to_string(),
DocumentStorageServiceUrl::new()?.to_string(),
);
tracing::info!("initialized dss client");
let sync_service_client = SyncServiceClient::new(
config.sync_service_auth_key.as_ref().to_string(),
SyncServiceUrl::new()?.to_string(),
);
tracing::info!("initialized sync service client");
let search_service_client = SearchServiceClient::new(
config
.document_storage_service_auth_key
.as_ref()
.to_string(),
DocumentStorageServiceUrl::new()?.to_string(),
);
tracing::info!("initialized search service client");
let jwt_args =
JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
.await
.context("failed to create jwt validation args")?;
let authorization_state =
MacroAuthorizationState::new(Arc::new(MacroAuthorizationServiceImpl::new(
MacroAuthJwtValidator::new(jwt_args),
InternalAuthConfig {
api_key: internal_api_key.clone(),
default_user_id: None,
},
macro_authorization::NoBotAuthorizer,
PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
)));
let lexical_client = Arc::new(lexical_client::LexicalClient::new(
internal_api_key.clone(),
LexicalServiceUrl::new()?.to_string(),
));
let email_service_client = Arc::new(EmailServiceClient::new(
internal_api_key.clone(),
EmailServiceUrl::new()?.to_string(),
));
tracing::info!("initialized static file service client");
// Initialize Redis client for stream service
let redis_client = Arc::new(
redis::Client::open(config.redis_host.as_ref())
.inspect(|client| {
client
.get_connection()
.map(|_| tracing::trace!("initialized redis connection"))
.inspect_err(|e| {
tracing::error!(error=?e, "failed to connect to redis");
})
.expect("redis connetion required");
})
.context("failed to connect to redis")?,
);
let stream_repo = RedisPostgresStreamRepo::new((*redis_client).clone(), db.clone()).obj();
tracing::info!("initialized stream repo");
let connection_manager =
connection_gateway::service::dynamodb::create_dynamo_db_connection_manager(dynamodb_client)
.await
.context("failed to create connection manager")?;
tracing::info!("initialized connection repo");
let connection_gateway_client = Arc::new(ConnectionGatewayClient::new(
internal_api_key.clone(),
ConnectionGatewayUrl::new()?.to_string(),
));
let ingress_queue = SqsQueue::new(
aws_sdk_sqs::Client::new(&aws_config),
notification_queue.to_string(),
);
let notification_ingress_service = Arc::new(SqsNotificationIngress {
queue: ingress_queue,
});
let notification_reader_queue = SqsQueue::new(
aws_sdk_sqs::Client::new(&aws_config),
notification_queue.to_string(),
);
let notification_reader_service = NotificationReaderService {
repository: DbNotificationRepository::new(db.clone()),
queue: ai_tools::ToolNotificationQueue::Sqs(notification_reader_queue),
sns_endpoint: ai_tools::NoOpSnsEndpointManager,
platform_config: PlatformArnConfig {
apns_platform_arn: String::new(),
fcm_platform_arn: String::new(),
apns_voip_platform_arn: String::new(),
},
realtime: notification::domain::ports::NoopNotificationRealtimePublisher,
};
let notification_tool_context =
notification::inbound::ai_tool::NotificationToolContext::new(notification_reader_service);
tracing::info!("initialized notification ingress service");
let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
db.clone(),
)));
let frecency_storage = FrecencyPgStorage::new(db.clone());
let frecency_service = FrecencyQueryServiceImpl::new(frecency_storage.clone());
let crm_service = crm::domain::service::CrmServiceImpl::new(
crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
);
let email_service = EmailServiceImpl::new(
EmailPgRepo::new(db.clone()),
frecency_service.clone(),
email::domain::ports::NoOpEnqueuer,
crm_service.clone(),
entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
entity_access_management::outbound::PgRepository::new(db.clone()),
),
0,
);
let channels_service = ChannelListServiceImpl::new(
PgChannelsRepo::new(db.clone()),
PgChannelsRepo::new(db.clone()),
frecency_storage,
);
let email_service_for_tools: Arc<ai_tools::ToolEmailService> = Arc::new(email_service.clone());
let github_pull_request_service = GithubPullRequestServiceImpl::new(
ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
PgGithubPullRequestRepo::new(db.clone()),
);
let favorites_service = ai_tools::build_favorites_service(db.clone());
let soup_service = Arc::new(
soup::domain::service::SoupImpl::new(
soup::outbound::pg_soup_repo::PgSoupRepo::new(ReadOnlyPool(db.clone())),
frecency_service,
ReadonlyEmailPreviewAdapter(email_service),
channels_service,
CallRecordQueryServiceImpl::new(PgCallRepo::new(db.clone())),
crm::domain::service::NoOpCrmService,
github_pull_request_service,
)
.with_favorites(favorites_service.clone()),
);
tracing::info!("initialized soup service");
let s3_client = macro_aws_config::s3_client().await;
let document_files = documents::outbound::s3_document_files::S3DocumentFiles::new(
db.clone(),
s3_client.clone(),
config.document_storage_bucket.to_string(),
);
let s3_upload_adapter = S3UploadUrlAdapter::new(
s3_client,
config.document_storage_bucket.to_string(),
config.docx_document_upload_bucket.to_string(),
);
let owned_entity_registrar =
OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));
let document_repo = PgDocumentRepo::new(db.clone(), owned_entity_registrar.clone());
let cloudfront_config = CloudFrontConfig {
distribution_url: config
.document_storage_service_cloudfront_distribution_url
.to_string(),
signer_public_key_id: config
.document_storage_service_cloudfront_signer_public_key_id
.to_string(),
signer_private_key: config
.document_storage_service_cloudfront_signer_private_key
.as_ref()
.to_string(),
presigned_url_expiry_seconds: 3600,
browser_cache_expiry_seconds: 86400,
};
let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())
.context("failed to create kafka event publisher")?,
event_broker_tracker.clone(),
);
let properties_service = ai_tools::build_properties_service_with_broker(
db.clone(),
entity_access_service.clone(),
macro_event_broker.clone(),
);
let task_properties_service = ai_tools::build_task_properties_adapter(
db.clone(),
properties_service.clone(),
entity_access_service.clone(),
);
// The import pipeline sets the same task system properties on imported
// Linear issues (status, priority, due date, assignee).
let task_properties_for_import = task_properties_service.clone();
let document_properties_for_import =
import::outbound::document_properties::DocumentPropertiesApplicator::new(
properties_service.clone(),
);
let document_service = DocumentServiceImpl::new(
document_repo,
cloudfront_config,
sync_service_client.clone(),
s3_upload_adapter,
task_properties_service,
NoOpConnectionService,
entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
entity_access_management::outbound::PgRepository::new(db.clone()),
),
ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
macro_event_broker.clone(),
);
// Messages sent by AI tools (chat, agents) dispatch the same side effects
// as the document-storage message API, so mentions, replies and document
// comments notify recipients and stream to connected clients.
let channels_connection_gateway =
Arc::new(connection_gateway_client::ConnectionGatewayClient::new(
internal_api_key.clone(),
ConnectionGatewayUrl::new()?.to_string(),
));
let side_effect_clients = ai_tools::ChannelSideEffectClients {
connection_gateway: channels_connection_gateway.clone(),
sqs: aws_sdk_sqs::Client::new(&aws_config),
macro_event_broker: macro_event_broker.clone(),
};
let lexical_client_for_tools = (*lexical_client).clone();
let document_tool_context = DocumentToolContext::new(
document_service,
(*entity_access_service).clone(),
lexical_client_for_tools,
sync_service_client.clone(),
documents::outbound::editing_worker_client::ReqwestEditingWorkerClient::new(
config.ai_editing_worker_url.clone(),
std::sync::Arc::new(reqwest::Client::new()),
),
config.document_permission_jwt.as_ref().to_string(),
ai_tools::build_message_service_with_side_effects(
db.clone(),
lexical_client.clone(),
&side_effect_clients,
),
)
.with_presentation_files(Arc::new(document_files.clone()))
.with_word_document_files(Arc::new(document_files.clone()))
.with_design_files(Arc::new(document_files.clone()))
.with_photoshop_files(Arc::new(document_files.clone()))
.with_illustrator_files(Arc::new(document_files));
let document_tool_context = match ai_tools::canvas_artifacts_rollout(
config.posthog_api_key.value(),
config.posthog_host.value(),
)? {
Some(rollout) => document_tool_context.with_canvas_artifacts(rollout),
None => document_tool_context,
};
tracing::info!("initialized document tool context");
let attachment_provider = attachment::provider::AttachmentProvider {
document: documents::inbound::attachment::DocumentAttachmentService::new(
document_tool_context.service.clone(),
document_tool_context.entity_access_service.clone(),
document_tool_context.lexical_client.clone(),
),
email_thread: email::inbound::attachment::EmailAttachmentService::new(
email_service_for_tools.clone(),
entity_access_service.clone(),
),
chat: chat::inbound::attachment::ChatAttachmentService::new(
Arc::new(chat::outbound::postgres::PgChatRepo::new(
db.clone(),
owned_entity_registrar.clone(),
)),
entity_access_service.clone(),
),
channel: channels::inbound::attachment::ChannelAttachmentService::new(
Arc::new(PgChannelsRepo::new(db.clone())),
entity_access_service.clone(),
),
static_file: static_file::inbound::attachment::StaticFileAttachmentService::new(Arc::new(
static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),
)),
};
let message_service = Arc::new(
chat::domain::service::MessageServiceImpl::new(
chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),
attachment_provider,
)
.with_event_broker(macro_event_broker.clone()),
);
tracing::info!("initialized attachment provider");
let email_service_client_external = Arc::new(EmailServiceClientExternal::new(
email_service_client.url().to_owned(),
));
let search_service_client = Arc::new(search_service_client);
let properties_tool_context = ai_tools::build_properties_tool_context(
properties_service.clone(),
entity_access_service.clone(),
);
tracing::info!("initialized properties tool context");
let user_email_service = Arc::new(
EmailServiceImpl::new(
EmailPgRepo::new(db.clone()),
FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),
sqs_client.clone(),
crm_service.clone(),
entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
entity_access_management::outbound::PgRepository::new(db.clone()),
),
0,
)
.with_macro_event_broker(macro_event_broker.clone()),
);
let email_tool_context = email::inbound::toolset::EmailToolContext::new(
user_email_service.clone(),
Arc::new(email::domain::ports::NoOpGmailTokenProvider),
Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
db.clone(),
))),
lexical_client.clone(),
)
.with_attachment_sender(
ai_tools::email_attachments::build_email_attachment_sender(
user_email_service.clone(),
EmailPgRepo::new(db.clone()),
document_tool_context.service.clone(),
)
.await,
);
tracing::info!("initialized email tool context");
let call_service = CallServiceImpl::new(
PgCallRepo::new(db.clone()),
NoOpCallRtcClient,
NoOpConnectionService,
(*entity_access_service).clone(),
NoOpNotificationIngress,
None::<S3RecordingStorage>,
String::new(),
);
let call_tool_context = CallToolContext::new(call_service, (*entity_access_service).clone());
tracing::info!("initialized call tool context");
let chat_tool_context = chat::inbound::toolset::ChatToolContext::new(
chat::domain::service::ChatServiceImpl::new(
chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),
Arc::new(ai_toolset::AsyncToolCollection::new()),
(),
entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
entity_access_management::outbound::PgRepository::new(db.clone()),
),
),
(*entity_access_service).clone(),
);
tracing::info!("initialized chat tool context");
let channel_tool_context = ai_tools::build_channel_tool_context_with_side_effects(
db.clone(),
lexical_client.clone(),
&side_effect_clients,
);
let user_permissions_service = Arc::new(
roles_and_permissions::domain::service::UserRolesAndPermissionsServiceImpl::new(
roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),
roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),
),
);
// The AI billing gate reads allowances, credits, and overage state here;
// collection (Stripe) lives in the authentication service, which the
// recorder below asks to settle once a payer runs past their allowance
// and ENABLE_AI_USAGE_BILLING is enabled. This instance never settles.
let auth_service_client = Arc::new(authentication_service_client::AuthServiceClient::new(
config
.authentication_service_secret_key
.as_ref()
.to_string(),
AuthServiceUrl::new()?.to_string(),
));
let ai_pricing = config.ai_pricing();
let ai_billing = Arc::new(
ai_billing::domain::BillingServiceImpl::new(
ai_billing::outbound::RolesTeamsEntitlementSource::new(
(*user_permissions_service).clone(),
teams::outbound::team_repo::TeamRepositoryImpl::new(db.clone()),
),
ai_billing::outbound::PgUsageReader::new(db.clone()),
ai_billing::outbound::PgBillingRepo::new(db.clone(), ai_pricing),
ai_billing::outbound::HttpPaymentGateway::new(auth_service_client.clone()),
ai_pricing,
)
.with_enforcement(config.enable_ai_usage_enforcement),
);
let admission: Arc<dyn ai_billing::AiAdmissionService> =
Arc::new(ai_billing::BillingAdmissionService::new(
ai_billing.clone(),
config.enable_ai_usage_enforcement,
));
let recorder: Arc<dyn ai_usage::UsageRecorder> =
Arc::new(ai_billing::outbound::SettlingUsageRecorder::new(
Arc::new(
ai_usage::domain::service::UsageServiceImpl::new(
ai_usage::outbound::PgUsageRepo::new(db.clone()),
)
.with_enforcement(config.enable_ai_usage_enforcement),
),
ai_billing.clone(),
ai_billing::outbound::HttpSettlementTrigger::new(auth_service_client.clone()),
config.enable_ai_usage_billing,
));
// Per-attempt observations use a separate journal and never feed the
// legacy settlement trigger, which remains counted-usage-only and gated by
// ENABLE_AI_USAGE_BILLING.
let recorder = ai_usage::with_tracking(recorder, ai_usage::pg_tracking(db.clone()));
let decision_classifier = config
.typesafe_api_key
.value()
.map(jev::outbound::SystemOneJev::typesafe)
.transpose()
.context("failed to configure the decision classifier")?
.map(|provider| jev::domain::JevClassifier::new(provider, recorder.clone()));
let decisions = Arc::new(jev::domain::decisions::DecisionServiceImpl::new(
decision_classifier,
admission.clone(),
));
let tool_free_completions = Arc::new(
crate::cognition::service::structured_completion::ToolFreeCompletionService::new(
recorder.clone(),
),
);
// The import pipeline: staged/imported external items, Slack discovery,
// and the Notion page import job. Built before the tool service context
// so the chat toolset gets a wired import context.
let mcp_encryption_key = mcp_client::domain::models::AesKey::try_from(
config.mcp_credentials_key_secret_name.as_ref(),
)
.context("invalid MCP credentials encryption key")?;
let granola_signing_key = *mcp_encryption_key.as_bytes();
let mcp_server_repo = mcp_client::outbound::NativeServerStore::new(
mcp_client::outbound::pg_server_repo::PgServerRepo::new(db.clone(), mcp_encryption_key),
Some(Arc::new(
mcp_client::outbound::auth_service_github_tokens::AuthServiceGithubTokens::new(
(*auth_service_client).clone(),
),
)),
);
// The Pipedream MCP stack, fully separate from the native one above
// (own endpoints, own table, own toolset). Without credentials its
// endpoints answer 501 and its toolsets come up empty. Connect-flow
// webhooks need both URI and secret; a half-set pair is ignored so we
// never mint a callback we cannot serve.
let pipedream_client: Option<Arc<pipedream_mcp::outbound::api::PipedreamClient>> = match (
config.pipedream_client_id.value(),
config.pipedream_client_secret.value(),
config.pipedream_project_id.value(),
) {
(Some(client_id), Some(client_secret), Some(project_id)) => {
let webhook_uri = match (
config.pipedream_webhook_uri.value(),
config.pipedream_webhook_secret.value(),
) {
(Some(uri), Some(_)) => Some(uri.to_owned()),
(None, None) => None,
_ => {
tracing::warn!(
"ignoring incomplete Pipedream webhook config; set both PIPEDREAM_WEBHOOK_URI and PIPEDREAM_WEBHOOK_SECRET"
);
None
}
};
let client = pipedream_mcp::outbound::api::PipedreamClient::new(
pipedream_mcp::outbound::api::PipedreamConfig {
client_id: client_id.to_owned(),
client_secret: client_secret.to_owned(),
project_id: project_id.to_owned(),
environment: config
.pipedream_environment
.value()
.unwrap_or(match config.environment {
Environment::Production => "production",
_ => "development",
})
.to_owned(),
api_url: config
.pipedream_api_url
.value()
.unwrap_or(pipedream_mcp::outbound::api::DEFAULT_API_URL)
.to_owned(),
mcp_url: config
.pipedream_mcp_url
.value()
.unwrap_or(pipedream_mcp::outbound::api::DEFAULT_MCP_URL)
.to_owned(),
allowed_origins: match config.pipedream_allowed_origins.value() {
Some(origins) => origins
.split(',')
.map(|origin| origin.trim().to_owned())
.filter(|origin| !origin.is_empty())
.collect(),
None => match config.environment {
Environment::Production => vec!["https://macro.com".to_owned()],
Environment::Develop => vec![
"https://dev.macro.com".to_owned(),
"http://localhost:3000".to_owned(),
],
Environment::Local => vec!["http://localhost:3000".to_owned()],
},
},
},
)
.context("failed to build Pipedream client")?;
Some(Arc::new(match webhook_uri {
Some(uri) => client.with_webhook_uri(uri),
None => client,
}))
}
_ => {
tracing::info!("Pipedream credentials not set; Pipedream MCP connectors disabled");
None
}
};
// Macro Connect beside Pipedream: a user the `macro-connect` rollout is
// on for is on Connect alone; everyone else stays on Pipedream.
let connect_adapter = ai_tools::connect_adapter(config.connect_customer_key.value())?;
let rollout = ai_tools::macro_connect_rollout(
config.posthog_api_key.value(),
config.posthog_host.value(),
)?;
let connectors: ai_tools::ToolPipedreamConnection = pipedream_client.as_ref().map(|client| {
let backends = pipedream_mcp::outbound::dispatch::ConnectorBackends::new(
client.clone(),
connect_adapter.clone(),
);
Arc::new(match &rollout {
Some(rollout) => backends.with_rollout(rollout.clone()),
None => backends,
})
});
let pipedream_repo =
pipedream_mcp::outbound::pg_connection_repo::PgConnectionRepo::new(db.clone());
let granola_sync: Option<Arc<dyn granola_sync::domain::service::SyncService>> =
pipedream_client.as_ref().map(|client| {
Arc::new(granola_sync::domain::service::Service {
repo: granola_sync::outbound::pg_repo::PgSyncRepository::new(
db.clone(),
&granola_signing_key,
),
granola: granola_sync::outbound::api::GranolaApi(client.clone()),
accounts: granola_sync::outbound::accounts::ConnectedAccounts(
pipedream_repo.clone(),
),
verifier: granola_sync::outbound::signature::StandardWebhooks,
calls: Arc::new(call::domain::imports::CallImportServiceImpl(
call::outbound::pg_imported_call_repo::PgImportedCallRepository::new(
db.clone(),
),
macro_event_broker.clone(),
)),
public_url: public_url.to_string(),
}) as Arc<dyn granola_sync::domain::service::SyncService>
});
let granola_worker = granola_sync
.as_ref()
.map(|service| tokio::spawn(granola_sync::inbound::run_worker(service.clone())));
// The one sanctioned meeting point of the two MCP stacks: agents load
// tools through this selector, which prefers a user's Pipedream
// connectors and falls back to the native ones (see `mcp_select`).
// The import service's live Slack source shares this selector.
let mcp_selector: Arc<ai_tools::ToolMcpSelector> = Arc::new(mcp_select::McpToolSelector::new(
Arc::new(mcp_server_repo.clone()),
Arc::new(pipedream_repo.clone()),
Arc::new(connectors.clone()),
));
// Nudges the user's connected clients when import rows flip, so setup
// sections and chat surfaces update immediately instead of on the next
// poll (see import::outbound::gateway_notifier).
let import_notify = import::outbound::gateway_notifier::gateway_import_notify(
channels_connection_gateway.clone(),
);
let initiative_tool_context = ai_tools::build_initiative_tool_context(
db.clone(),
&document_tool_context,
properties_service.clone(),
entity_access_service.clone(),
macro_event_broker.clone(),
);
let databases_tool_context = ai_tools::build_databases_tool_context(
db.clone(),
entity_access_service.clone(),
ai_tools::ToolTableEventPublisher::Gateway(
databases::outbound::gateway_event_publisher::GatewayTableEventPublisher::new(
channels_connection_gateway.as_ref().clone(),
),
),
ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),
);
let project_tool_context = ai_tools::build_project_tool_context(
db.clone(),
macro_event_broker.clone(),
entity_access_service.clone(),
document_tool_context.service.clone(),
user_email_service.clone(),
databases_tool_context.service.clone(),
);
let entity_creator = ai_tools::ToolEntityCreator {
document_creator: document_tool_context.creator.clone(),
entity_access_service: entity_access_service.clone(),
channel_service: channel_tool_context.service.clone(),
task_properties: task_properties_for_import,
document_properties: document_properties_for_import,
team_repository: ai_tools::build_team_repository(db.clone()),
initiative_service: initiative_tool_context.service.clone(),
project_service: project_tool_context.service.clone(),
databases_service: databases_tool_context.service.clone(),
document_service: document_tool_context.service.clone(),
};
let import_service = Arc::new(
import::domain::service::ImportServiceImpl::new(
ai_tools::build_import_repo(db.clone()),
mcp_selector.clone(),
Arc::new(entity_creator),
recorder.clone(),
)
.with_slack_source(Arc::new(
import::outbound::mcp_slack_source::McpSlackSource::new(mcp_selector.clone()),
))
.with_tracker_connections(
Arc::new(
import_linear::outbound::client::PipedreamLinearConnections::new(
Arc::new(pipedream_repo.clone()),
connectors.clone(),
),
),
Arc::new(
import_jira::outbound::client::PipedreamJiraConnections::new(
Arc::new(pipedream_repo.clone()),
connectors.clone(),
),
),
)
.with_notion_connections(Arc::new(
import_notion::outbound::pipedream::PipedreamNotionConnections::new(
Arc::new(pipedream_repo.clone()),
connectors.clone(),
),
))
.with_admission(admission.clone())
.with_notifier(import_notify)
.with_job_shutdown(import_jobs.shutdown.clone(), import_jobs.tracker.clone()),
);
// Notion workspace imports outlive the process running them: on shutdown
// a running job hands its run over, and the sweeper adopts runs whose
// process stopped holding them (other replicas' live runs are leased, so
// it leaves them alone). Slack and single-page jobs heal on read. This is
// the process's only sweeper.
import_jobs.tracker.spawn({
let import_service = import_service.clone();
let shutdown = import_jobs.shutdown.clone();
async move { import_service.sweep_stalled_notion_imports(shutdown).await }
});
tracing::info!("initialized import service");
let databases_sql_tool_context = ai_tools::build_databases_sql_tool_context(
&databases_tool_context,
soup_service.clone(),
db.clone(),
);
let crm_tool_context = ai_tools::build_crm_tool_context(db.clone());
let pipeline_tool_context = ai_tools::build_pipeline_tool_context(
db.clone(),
&databases_tool_context,
&crm_tool_context,
);
let crm_transfer_tool_context = ai_tools::build_crm_transfer_tool_context(
&pipeline_tool_context,
&crm_tool_context,
&document_tool_context,
);
let calendar_tool_context = ai_tools::build_calendar_tool_context(
db.clone(),
CalendarServiceUrl::new()?,
internal_api_key.clone(),
call_tool_context.service.clone(),
AppServiceUrl::new()?,
macro_env_var::maybe_read_env("ENABLE_CALENDAR_ADD_SHARED_EVENT")
.and_then(|value| value.parse::<bool>().ok())
.unwrap_or(false),
);
let forms_tool_context = ai_tools::build_forms_tool_context(
db.clone(),
&document_tool_context,
&databases_tool_context,
&calendar_tool_context,
Some(channels_connection_gateway.as_ref().clone()),
ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),
ai_tools::FormsToolConfig {
app_origin: macro_service_urls::AppServiceUrl::new()?.to_string(),
editing_worker_url: config.ai_editing_worker_url.clone(),
static_files: static_file_service_client::StaticFileServiceClient::new(
internal_api_key.clone(),
macro_service_urls::StaticFileServiceUrl::new()?.to_string(),
),
},
);
let tool_service_context = ai_tools::ToolServiceContext {
memory_tool_context: ai_tools::memory::build_memory_tool_context(
db.clone(),
recorder.clone(),
),
cognition_tool_context: ai_tools::build_cognition_tool_context()?,
connector_tool_context: ai_tools::build_connector_tool_context(db.clone(), None, None),
search_service_client: search_service_client.clone(),
email_service_client: email_service_client_external.clone(),
soup_service: soup_service.clone(),
email_service: email_service_for_tools.clone(),
activity_tool_context: ai_tools::build_activity_tool_context(
db.clone(),
properties_service,
entity_access_service.clone(),
),
image_generation_tool_context: ai_tools::build_image_generation_tool_context(
&document_tool_context,
ai_tools::build_image_generator_from_env(),
recorder.clone(),
)?,
message_tool_context: ai_tools::build_message_tool_context(&document_tool_context),
document_tool_context: document_tool_context.clone(),
properties_tool_context: properties_tool_context.clone(),
email_tool_context: email_tool_context.clone(),
call_tool_context: call_tool_context.clone(),
booking_management_tool_context: ai_tools::build_booking_management_tool_context(
CalendarServiceUrl::new()?,
internal_api_key.clone(),
config.environment,
),
booking_link_tool_context: ai_tools::build_booking_link_tool_context(
db.clone(),
config.environment,
),
calendar_tool_context,
team_calendar_tool_context: ai_tools::build_team_calendar_tool_context(
db.clone(),
macro_env_var::maybe_read_env("CALENDAR_TEAM_SHARING_ENABLED")
.and_then(|value| value.parse::<bool>().ok())
.unwrap_or(false),
),
notification_tool_context: notification_tool_context.clone(),
item_notification_preference_tool_context:
ai_tools::build_item_notification_preference_tool_context(db.clone()),
databases_tool_context,
forms_tool_context,
databases_sql_tool_context,
import_tool_context: import::inbound::toolset::ImportToolContext::wired(
import_service.clone(),
),
mcp_server_tool_context: ai_tools::ToolMcpServerToolContext::wired(
mcp_client::domain::service::ServerDirectory::new(Arc::new(mcp_server_repo.clone())),
),
chat_tool_context,
channel_tool_context,
channel_media_upload_tool_context: ai_tools::build_channel_media_upload_tool_context()?,
bot_tool_context: ai_tools::build_bot_tool_context(
db.clone(),
ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),
entity_access_service.clone(),
DocumentStorageServiceUrl::new()?.to_string(),
pipedream_client.clone(),
),
read_agent_session_context: ai_tools::build_read_agent_session_context(
db.clone(),
entity_access_service.clone(),
),
coding_agent_tool_context: ai_tools::build_coding_agent_tool_context(
macro_service_urls::AgentHarnessServiceUrl::new()?,
internal_api_key.clone(),
)?,
project_tool_context,
favorites_tool_context: ai_tools::build_favorites_tool_context(
favorites_service,
entity_access_service.clone(),
),
initiative_tool_context,
team_tool_context: ai_tools::build_team_tool_context(db.clone()),
crm_tool_context,
pipeline_tool_context,
crm_transfer_tool_context,
skill_tool_context: ai_tools::build_skill_tool_context(
search_service_client.clone(),
soup_service.clone(),
&document_tool_context,
),
schedule_tool_context: ai_tools::build_routine_tool_context()?,
anthropic_tool_context: ai_tools::build_anthropic_tool_context(),
web_search_provider: ai_tools::build_web_search_provider_from_env(),
admission,
recorder,
usage_context: ai_usage::UsageContext::system(ai_usage::AiFeature::Chat),
};
let all_tools = ai_tools::tools_for(ai_tools::AiHost::Chat);
let all_tools_toolset = all_tools.toolset.clone();
let all_tools_prompt: Arc<dyn std::fmt::Display + Send + Sync> =
Arc::new(all_tools.prompt.to_string());
// Build memory service
let memory_service = Arc::new(ai_tools::memory::build_memory_service(
db.clone(),
tool_service_context.clone(),
));
tracing::info!("initialized memory service");
// Build the AI cost service. It backs both the admin query/pricing router
// and the usage recorder threaded through the tool service context.
let usage_service = Arc::new(
ai_usage::domain::service::UsageServiceImpl::new(ai_usage::outbound::PgUsageRepo::new(
db.clone(),
))
.with_enforcement(config.enable_ai_usage_enforcement),
);
tracing::info!("initialized ai cost service");
// Generator that materializes projections by running their prompt through
// the shared AI toolset, attributed to the requesting user.
let projection_generator =
ai_projections::outbound::agent_generator::AgentProjectionGenerator::new(
tool_service_context.clone(),
ai_tools::tools_for(ai_tools::AiHost::Chat),
);
// Notifier that pushes finished materializations to the target's connected
// clients through the connection gateway.
let projection_notifier =
ai_projections::outbound::gateway_notifier::GatewayProjectionNotifier::new(Arc::new(
connection_gateway_client::ConnectionGatewayClient::new(
internal_api_key.clone(),
ConnectionGatewayUrl::new()?.to_string(),
),
));
let ai_projections_service_impl =
ai_projections::domain::ai_projection_service::AiProjectionServiceImpl::new(
ai_projections::outbound::ai_projection_repo::AiProjectionRepositoryImpl::new(
db.clone(),
),
sqs_client.clone(),
projection_generator,
projection_notifier,
);
// Spawn the inbound worker that polls ai_projection_queue and materializes
// projection instances. It owns its own clone of the service.
let ai_projection_worker = ai_projections::worker::AiProjectionWorker::new(
sqs_worker::SQSWorker::new(
aws_sdk_sqs::Client::new(&aws_config),
ai_projection_queue.to_string(),
10,
10,
),
ai_projections_service_impl.clone(),
);
let ai_projection_worker_task = tokio::spawn(async move {
ai_projection_worker.poll().await;
});
let ai_projections_service = Arc::new(ai_projections_service_impl);
tracing::info!("initialized ai projections service");
let onboarding_service = Arc::new(onboarding::domain::service::OnboardingServiceImpl::new(
onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),
Arc::new(mcp_server_repo.clone()),
));
tracing::info!("initialized onboarding service");
let mcp_client_public_url = &public_url;
let mcp_client_metadata = mcp_client::domain::models::OAuthClientMetadata::new(
format!("{mcp_client_public_url}/mcp/servers/auth/client-metadata"),
format!("{mcp_client_public_url}/mcp/servers/auth/callback"),
);
let mcp_oauth_state_store =
mcp_client::outbound::redis_state_store::RedisOAuthStateStore::new(redis_client.clone());
let mcp_pre_registered =
mcp_client::domain::provider_registry::PreRegisteredProviders::from_env()?;
let mcp_oauth = mcp_client::outbound::oauth::OAuthService::new(
mcp_server_repo.clone(),
mcp_oauth_state_store,
mcp_client_metadata.clone(),
mcp_pre_registered,
);
// Record connector connects and disconnects for product analytics.
let integration_analytics = IntegrationAnalytics::from_config(&config, &db);
let analytics_for_auth_hook = integration_analytics.clone();
let mcp_auth_hook: mcp_client::inbound::axum_router::McpAuthCompletedHook =
Arc::new(move |record: mcp_client::domain::models::McpServerRecord| {
analytics_for_auth_hook.record(
record.user_id,
IntegrationChange::Connected,
IntegrationEvent::mcp_server(&record.url),
);
Box::pin(async {})
});
let analytics_for_mcp_removed_hook = integration_analytics.clone();
let mcp_removed_hook: mcp_client::inbound::axum_router::McpServerRemovedHook =
Arc::new(move |record: mcp_client::domain::models::McpServerRecord| {
if let Some(event) = IntegrationEvent::removed_mcp_server(&record) {
analytics_for_mcp_removed_hook.record(
record.user_id,
IntegrationChange::Disconnected,
event,
);
}
Box::pin(async {})
});
let mcp_state = mcp_client::inbound::McpRouterState::new(
mcp_server_repo,
mcp_oauth,
authorization_state.clone(),
mcp_client_metadata,
)
.with_auth_completed_hook(mcp_auth_hook)
.with_server_removed_hook(mcp_removed_hook);
let analytics_for_pipedream_hook = integration_analytics.clone();
let pipedream_auth_hook: pipedream_mcp::inbound::axum_router::PipedreamAuthCompletedHook =
Arc::new(
move |connection: pipedream_mcp::domain::models::PipedreamConnection| {
analytics_for_pipedream_hook.record(
connection.user_id,
IntegrationChange::Connected,
IntegrationEvent::pipedream(&connection.app_slug),
);
Box::pin(async {})
},
);
let pipedream_disconnected_hook: pipedream_mcp::inbound::axum_router::PipedreamDisconnectedHook =
Arc::new(
move |connection: pipedream_mcp::domain::models::PipedreamConnection| {
integration_analytics.record(
connection.user_id,
IntegrationChange::Disconnected,
IntegrationEvent::pipedream(&connection.app_slug),
);
Box::pin(async {})
},
);
let pipedream_state = pipedream_mcp::inbound::PipedreamRouterState::new(
pipedream_repo,
connectors,
authorization_state.clone(),
)
.with_auth_completed_hook(pipedream_auth_hook)
.with_disconnected_hook(pipedream_disconnected_hook);
let cancellation = tokio_util::sync::CancellationToken::new();
super::refresh::start(
db.clone(),
Arc::new(sqs_client.clone()),
cancellation.clone(),
event_broker_tracker.clone(),
);
if !matches!(config.environment, Environment::Local) {
memory::schedule::start(
memory_service.clone(),
&event_broker_tracker,
cancellation.clone(),
)?;
}
let router = api::build_router(ApiContext {
db: db.clone(),
email_service_client_external,
sqs_client: Arc::new(sqs_client),
document_storage_client: Arc::new(document_storage_client),
search_service_client,
authorization_state,
user_permissions_service,
non_user_owners,
ai_admission: tool_service_context.admission.clone(),
decisions,
tool_free_completions,
ai_billing,
internal_api_key: config.internal_api_key.clone(),
config: Arc::new(config),
notification_ingress_service,
connection_repo: connection_manager.persistence,
connection_gateway_client,
soup_service,
email_service: email_service_for_tools,
stream_repo,
document_tool_context,
memory_service,
usage_service,
ai_projections_service,
properties_tool_context,
email_tool_context: email_tool_context.clone(),
call_tool_context,
tool_service_context,
all_tools: all_tools_toolset,
all_tools_prompt,
entity_access_service,
message_service,
ai_stream_registry: service::ai_stream_registry::AiStreamRegistry::new(
redis_client.clone(),
),
web_rate_limiter: rate_limit::RateLimitServiceImpl {
repo: rate_limit::RedisRateLimitAdapter {
redis: (*redis_client).clone(),
},
},
mcp_state,
pipedream_state,
granola_sync,
mcp_selector,
import_service,
onboarding_service,
macro_event_broker: macro_event_broker.clone(),
});
let mut workers = vec![tokio_util::task::AbortOnDropHandle::new(
ai_projection_worker_task,
)];
if let Some(worker) = granola_worker {
workers.push(tokio_util::task::AbortOnDropHandle::new(worker));
}
Ok(CognitionRuntime {
router,
workers,
cancellation,
})
}