Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

49.1 KB
Raw Blame
//! The service's composition root: reads config and wires every adapter and
//! domain service into the HTTP server.

use crate::cognition::{
    api::{self, context::ApiContext},
    config::{Config, Environment},
    service::{
        self,
        integration_analytics::{IntegrationAnalytics, IntegrationChange, IntegrationEvent},
    },
};
use ai_tools::{NoOpCallRtcClient, NoOpConnectionService, NoOpNotificationIngress};
use anyhow::Context;
use bots::outbound::pg_bots_repo::PgBotsRepo;
use call::domain::service::{CallRecordQueryServiceImpl, CallServiceImpl};
use call::inbound::toolset::CallToolContext;
use call::outbound::pg_call_repo::PgCallRepo;
use call::outbound::s3_recording_storage::S3RecordingStorage;
use channels::{
    domain::list_service::ChannelListServiceImpl, outbound::pg_channels_repo::PgChannelsRepo,
};
use document_storage_service_client::DocumentStorageServiceClient;
use documents::{
    domain::{models::CloudFrontConfig, service::DocumentServiceImpl},
    inbound::toolset::DocumentToolContext,
    outbound::{pg_document_repo::PgDocumentRepo, s3_upload_url::S3UploadUrlAdapter},
};
use email::domain::ports::ReadonlyEmailPreviewAdapter;
use email::domain::service::EmailServiceImpl;
use email::outbound::EmailPgRepo;
use email_service_client::{EmailServiceClient, EmailServiceClientExternal};
use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
use entity_registry::OwnerGrantPolicy;
use entity_registry_db_utils::OwnedEntityRegistrar;
use foreign_entity::{
    domain::service::ForeignEntityServiceImpl,
    outbound::pg_foreign_entity_repo::PgForeignEntityRepo,
};
use frecency::domain::services::FrecencyQueryServiceImpl;
use frecency::outbound::postgres::FrecencyPgStorage;
use github_pull_requests::{
    domain::service::GithubPullRequestServiceImpl,
    outbound::pg_github_pull_request_repo::PgGithubPullRequestRepo,
};
use macro_auth::middleware::decode_jwt::JwtValidationArgs;
use macro_authorization::{
    InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,
    MacroAuthorizationState, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
};
use macro_service_urls::{
    AppServiceUrl, AuthServiceUrl, CalendarServiceUrl, ConnectionGatewayUrl,
    DocumentStorageServiceUrl, EmailServiceUrl, LexicalServiceUrl, StaticFileServiceUrl,
    SyncServiceUrl,
};
use notification::domain::service::{
    NotificationReaderService, PlatformArnConfig, SqsNotificationIngress,
};
use notification::outbound::queue::SqsQueue;
use notification::outbound::repository::DbNotificationRepository;
use notification::outbound::websocket::ConnectionGatewayClient;
use readonly_pool::ReadOnlyPool;
use search_service_client::SearchServiceClient;

use std::sync::Arc;
use stream::outbound::redis_pg::RedisPostgresStreamRepo;
use sync_service_client::SyncServiceClient;
use tokio_util::task::TaskTracker;

/// The cognition routes and workers owned by the harness process.
pub struct CognitionRuntime {
    /// Routes mounted on the harness's public listener.
    pub router: axum::Router,
    workers: Vec<tokio_util::task::AbortOnDropHandle<()>>,
    cancellation: tokio_util::sync::CancellationToken,
}

impl CognitionRuntime {
    /// Stop intake before the shared HTTP listener and sessions drain.
    pub fn stop(&self) {
        self.cancellation.cancel();
        for worker in &self.workers {
            worker.abort();
        }
    }
    /// Stop workers before the harness drains pending event publishes.
    pub async fn shutdown(self) {
        self.stop();
        for worker in self.workers {
            worker.abort();
            let _ = worker.await;
        }
    }
}

/// Compose chat, completions, integrations, and workers in the harness process.
///
/// `import_jobs` are the harness's import jobs: this runtime's Notion imports
/// and the stalled-import sweeper run on them, and the harness hands them
/// over on shutdown.
pub async fn build(
    config: Config,
    db: sqlx::PgPool,
    event_broker_tracker: TaskTracker,
    import_jobs: ai_tools::ImportJobs,
) -> anyhow::Result<CognitionRuntime> {
    let public_url = macro_service_urls::CognitionPublicUrl::new()?;
    let aws_config = macro_aws_config::get_macro_aws_config().await;
    let secretsmanager_client = secretsmanager_client::SecretsManager::new(
        aws_sdk_secretsmanager::Client::new(&aws_config),
    );
    let config = config
        .resolve_remote_secrets(Environment::new_or_prod(), &secretsmanager_client)
        .await
        .context("failed to resolve cognition secrets")?;
    let non_user_owners = config.non_user_owners()?;
    let dynamodb_client = aws_sdk_dynamodb::Client::new(&aws_config);
    let queue_aws_client = aws_sdk_sqs::Client::new(&aws_config);

    let document_text_extractor_queue = macro_queues::DocumentTextExtractorQueue::new();
    let chat_delete_queue = macro_queues::ChatDeleteQueue::new();
    let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
    let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
    let ai_projection_queue = macro_queues::AiProjectionQueue::new();
    let notification_queue = macro_queues::NotificationIngressQueue::new();
    let sqs_client = sqs_client::SQS::new(queue_aws_client)
        .document_text_extractor_queue(&document_text_extractor_queue)
        .chat_delete_queue(&chat_delete_queue)
        .email_scheduled_queue(&email_scheduled_queue)
        .gmail_ops_queue(&gmail_ops_queue)
        .ai_projection_queue(&ai_projection_queue);

    let internal_api_key = config.internal_api_key.to_string();

    let document_storage_client = DocumentStorageServiceClient::new(
        config
            .document_storage_service_auth_key
            .as_ref()
            .to_string(),
        DocumentStorageServiceUrl::new()?.to_string(),
    );

    tracing::info!("initialized dss client");

    let sync_service_client = SyncServiceClient::new(
        config.sync_service_auth_key.as_ref().to_string(),
        SyncServiceUrl::new()?.to_string(),
    );
    tracing::info!("initialized sync service client");

    let search_service_client = SearchServiceClient::new(
        config
            .document_storage_service_auth_key
            .as_ref()
            .to_string(),
        DocumentStorageServiceUrl::new()?.to_string(),
    );
    tracing::info!("initialized search service client");

    let jwt_args =
        JwtValidationArgs::new_with_secret_manager(config.environment, &secretsmanager_client)
            .await
            .context("failed to create jwt validation args")?;

    let authorization_state =
        MacroAuthorizationState::new(Arc::new(MacroAuthorizationServiceImpl::new(
            MacroAuthJwtValidator::new(jwt_args),
            InternalAuthConfig {
                api_key: internal_api_key.clone(),
                default_user_id: None,
            },
            macro_authorization::NoBotAuthorizer,
            PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
        )));

    let lexical_client = Arc::new(lexical_client::LexicalClient::new(
        internal_api_key.clone(),
        LexicalServiceUrl::new()?.to_string(),
    ));

    let email_service_client = Arc::new(EmailServiceClient::new(
        internal_api_key.clone(),
        EmailServiceUrl::new()?.to_string(),
    ));

    tracing::info!("initialized static file service client");

    // Initialize Redis client for stream service
    let redis_client = Arc::new(
        redis::Client::open(config.redis_host.as_ref())
            .inspect(|client| {
                client
                    .get_connection()
                    .map(|_| tracing::trace!("initialized redis connection"))
                    .inspect_err(|e| {
                        tracing::error!(error=?e, "failed to connect to redis");
                    })
                    .expect("redis connetion required");
            })
            .context("failed to connect to redis")?,
    );
    let stream_repo = RedisPostgresStreamRepo::new((*redis_client).clone(), db.clone()).obj();

    tracing::info!("initialized stream repo");

    let connection_manager =
        connection_gateway::service::dynamodb::create_dynamo_db_connection_manager(dynamodb_client)
            .await
            .context("failed to create connection manager")?;

    tracing::info!("initialized connection repo");
    let connection_gateway_client = Arc::new(ConnectionGatewayClient::new(
        internal_api_key.clone(),
        ConnectionGatewayUrl::new()?.to_string(),
    ));

    let ingress_queue = SqsQueue::new(
        aws_sdk_sqs::Client::new(&aws_config),
        notification_queue.to_string(),
    );
    let notification_ingress_service = Arc::new(SqsNotificationIngress {
        queue: ingress_queue,
    });

    let notification_reader_queue = SqsQueue::new(
        aws_sdk_sqs::Client::new(&aws_config),
        notification_queue.to_string(),
    );
    let notification_reader_service = NotificationReaderService {
        repository: DbNotificationRepository::new(db.clone()),
        queue: ai_tools::ToolNotificationQueue::Sqs(notification_reader_queue),
        sns_endpoint: ai_tools::NoOpSnsEndpointManager,
        platform_config: PlatformArnConfig {
            apns_platform_arn: String::new(),
            fcm_platform_arn: String::new(),
            apns_voip_platform_arn: String::new(),
        },
        realtime: notification::domain::ports::NoopNotificationRealtimePublisher,
    };
    let notification_tool_context =
        notification::inbound::ai_tool::NotificationToolContext::new(notification_reader_service);

    tracing::info!("initialized notification ingress service");
    let entity_access_service = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
        db.clone(),
    )));

    let frecency_storage = FrecencyPgStorage::new(db.clone());
    let frecency_service = FrecencyQueryServiceImpl::new(frecency_storage.clone());
    let crm_service = crm::domain::service::CrmServiceImpl::new(
        crm::outbound::companies_repo::CompaniesRepositoryImpl::new(db.clone()),
        crm::outbound::no_op_resolver::NoOpCompanyMetadataResolver,
    );
    let email_service = EmailServiceImpl::new(
        EmailPgRepo::new(db.clone()),
        frecency_service.clone(),
        email::domain::ports::NoOpEnqueuer,
        crm_service.clone(),
        entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
            entity_access_management::outbound::PgRepository::new(db.clone()),
        ),
        0,
    );
    let channels_service = ChannelListServiceImpl::new(
        PgChannelsRepo::new(db.clone()),
        PgChannelsRepo::new(db.clone()),
        frecency_storage,
    );
    let email_service_for_tools: Arc<ai_tools::ToolEmailService> = Arc::new(email_service.clone());
    let github_pull_request_service = GithubPullRequestServiceImpl::new(
        ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
        PgGithubPullRequestRepo::new(db.clone()),
    );
    let favorites_service = ai_tools::build_favorites_service(db.clone());
    let soup_service = Arc::new(
        soup::domain::service::SoupImpl::new(
            soup::outbound::pg_soup_repo::PgSoupRepo::new(ReadOnlyPool(db.clone())),
            frecency_service,
            ReadonlyEmailPreviewAdapter(email_service),
            channels_service,
            CallRecordQueryServiceImpl::new(PgCallRepo::new(db.clone())),
            crm::domain::service::NoOpCrmService,
            github_pull_request_service,
        )
        .with_favorites(favorites_service.clone()),
    );

    tracing::info!("initialized soup service");

    let s3_client = macro_aws_config::s3_client().await;
    let document_files = documents::outbound::s3_document_files::S3DocumentFiles::new(
        db.clone(),
        s3_client.clone(),
        config.document_storage_bucket.to_string(),
    );
    let s3_upload_adapter = S3UploadUrlAdapter::new(
        s3_client,
        config.document_storage_bucket.to_string(),
        config.docx_document_upload_bucket.to_string(),
    );
    let owned_entity_registrar =
        OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));
    let document_repo = PgDocumentRepo::new(db.clone(), owned_entity_registrar.clone());

    let cloudfront_config = CloudFrontConfig {
        distribution_url: config
            .document_storage_service_cloudfront_distribution_url
            .to_string(),
        signer_public_key_id: config
            .document_storage_service_cloudfront_signer_public_key_id
            .to_string(),
        signer_private_key: config
            .document_storage_service_cloudfront_signer_private_key
            .as_ref()
            .to_string(),
        presigned_url_expiry_seconds: 3600,
        browser_cache_expiry_seconds: 86400,
    };
    let macro_event_broker = macro_event_broker::MacroEventBrokerService::new(
        macro_event_broker::KafkaEventPublisher::new(config.kafka_brokers.as_ref())
            .context("failed to create kafka event publisher")?,
        event_broker_tracker.clone(),
    );
    let properties_service = ai_tools::build_properties_service_with_broker(
        db.clone(),
        entity_access_service.clone(),
        macro_event_broker.clone(),
    );
    let task_properties_service = ai_tools::build_task_properties_adapter(
        db.clone(),
        properties_service.clone(),
        entity_access_service.clone(),
    );
    // The import pipeline sets the same task system properties on imported
    // Linear issues (status, priority, due date, assignee).
    let task_properties_for_import = task_properties_service.clone();
    let document_properties_for_import =
        import::outbound::document_properties::DocumentPropertiesApplicator::new(
            properties_service.clone(),
        );
    let document_service = DocumentServiceImpl::new(
        document_repo,
        cloudfront_config,
        sync_service_client.clone(),
        s3_upload_adapter,
        task_properties_service,
        NoOpConnectionService,
        entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
            entity_access_management::outbound::PgRepository::new(db.clone()),
        ),
        ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
        macro_event_broker.clone(),
    );
    // Messages sent by AI tools (chat, agents) dispatch the same side effects
    // as the document-storage message API, so mentions, replies and document
    // comments notify recipients and stream to connected clients.
    let channels_connection_gateway =
        Arc::new(connection_gateway_client::ConnectionGatewayClient::new(
            internal_api_key.clone(),
            ConnectionGatewayUrl::new()?.to_string(),
        ));
    let side_effect_clients = ai_tools::ChannelSideEffectClients {
        connection_gateway: channels_connection_gateway.clone(),
        sqs: aws_sdk_sqs::Client::new(&aws_config),
        macro_event_broker: macro_event_broker.clone(),
    };
    let lexical_client_for_tools = (*lexical_client).clone();
    let document_tool_context = DocumentToolContext::new(
        document_service,
        (*entity_access_service).clone(),
        lexical_client_for_tools,
        sync_service_client.clone(),
        documents::outbound::editing_worker_client::ReqwestEditingWorkerClient::new(
            config.ai_editing_worker_url.clone(),
            std::sync::Arc::new(reqwest::Client::new()),
        ),
        config.document_permission_jwt.as_ref().to_string(),
        ai_tools::build_message_service_with_side_effects(
            db.clone(),
            lexical_client.clone(),
            &side_effect_clients,
        ),
    )
    .with_presentation_files(Arc::new(document_files.clone()))
    .with_word_document_files(Arc::new(document_files.clone()))
    .with_design_files(Arc::new(document_files.clone()))
    .with_photoshop_files(Arc::new(document_files.clone()))
    .with_illustrator_files(Arc::new(document_files));
    let document_tool_context = match ai_tools::canvas_artifacts_rollout(
        config.posthog_api_key.value(),
        config.posthog_host.value(),
    )? {
        Some(rollout) => document_tool_context.with_canvas_artifacts(rollout),
        None => document_tool_context,
    };

    tracing::info!("initialized document tool context");

    let attachment_provider = attachment::provider::AttachmentProvider {
        document: documents::inbound::attachment::DocumentAttachmentService::new(
            document_tool_context.service.clone(),
            document_tool_context.entity_access_service.clone(),
            document_tool_context.lexical_client.clone(),
        ),
        email_thread: email::inbound::attachment::EmailAttachmentService::new(
            email_service_for_tools.clone(),
            entity_access_service.clone(),
        ),
        chat: chat::inbound::attachment::ChatAttachmentService::new(
            Arc::new(chat::outbound::postgres::PgChatRepo::new(
                db.clone(),
                owned_entity_registrar.clone(),
            )),
            entity_access_service.clone(),
        ),
        channel: channels::inbound::attachment::ChannelAttachmentService::new(
            Arc::new(PgChannelsRepo::new(db.clone())),
            entity_access_service.clone(),
        ),
        static_file: static_file::inbound::attachment::StaticFileAttachmentService::new(Arc::new(
            static_file::outbound::CdnStaticFileRepo::new(StaticFileServiceUrl::new()?.to_string()),
        )),
    };
    let message_service = Arc::new(
        chat::domain::service::MessageServiceImpl::new(
            chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),
            attachment_provider,
        )
        .with_event_broker(macro_event_broker.clone()),
    );

    tracing::info!("initialized attachment provider");

    let email_service_client_external = Arc::new(EmailServiceClientExternal::new(
        email_service_client.url().to_owned(),
    ));

    let search_service_client = Arc::new(search_service_client);

    let properties_tool_context = ai_tools::build_properties_tool_context(
        properties_service.clone(),
        entity_access_service.clone(),
    );

    tracing::info!("initialized properties tool context");

    let user_email_service = Arc::new(
        EmailServiceImpl::new(
            EmailPgRepo::new(db.clone()),
            FrecencyQueryServiceImpl::new(FrecencyPgStorage::new(db.clone())),
            sqs_client.clone(),
            crm_service.clone(),
            entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
                entity_access_management::outbound::PgRepository::new(db.clone()),
            ),
            0,
        )
        .with_macro_event_broker(macro_event_broker.clone()),
    );
    let email_tool_context = email::inbound::toolset::EmailToolContext::new(
        user_email_service.clone(),
        Arc::new(email::domain::ports::NoOpGmailTokenProvider),
        Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
            db.clone(),
        ))),
        lexical_client.clone(),
    )
    .with_attachment_sender(
        ai_tools::email_attachments::build_email_attachment_sender(
            user_email_service.clone(),
            EmailPgRepo::new(db.clone()),
            document_tool_context.service.clone(),
        )
        .await,
    );

    tracing::info!("initialized email tool context");

    let call_service = CallServiceImpl::new(
        PgCallRepo::new(db.clone()),
        NoOpCallRtcClient,
        NoOpConnectionService,
        (*entity_access_service).clone(),
        NoOpNotificationIngress,
        None::<S3RecordingStorage>,
        String::new(),
    );
    let call_tool_context = CallToolContext::new(call_service, (*entity_access_service).clone());

    tracing::info!("initialized call tool context");

    let chat_tool_context = chat::inbound::toolset::ChatToolContext::new(
        chat::domain::service::ChatServiceImpl::new(
            chat::outbound::postgres::PgChatRepo::new(db.clone(), owned_entity_registrar.clone()),
            Arc::new(ai_toolset::AsyncToolCollection::new()),
            (),
            entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
                entity_access_management::outbound::PgRepository::new(db.clone()),
            ),
        ),
        (*entity_access_service).clone(),
    );

    tracing::info!("initialized chat tool context");

    let channel_tool_context = ai_tools::build_channel_tool_context_with_side_effects(
        db.clone(),
        lexical_client.clone(),
        &side_effect_clients,
    );
    let user_permissions_service = Arc::new(
        roles_and_permissions::domain::service::UserRolesAndPermissionsServiceImpl::new(
            roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),
            roles_and_permissions::outbound::pgpool::MacroDB::new(db.clone()),
        ),
    );

    // The AI billing gate reads allowances, credits, and overage state here;
    // collection (Stripe) lives in the authentication service, which the
    // recorder below asks to settle once a payer runs past their allowance
    // and ENABLE_AI_USAGE_BILLING is enabled. This instance never settles.
    let auth_service_client = Arc::new(authentication_service_client::AuthServiceClient::new(
        config
            .authentication_service_secret_key
            .as_ref()
            .to_string(),
        AuthServiceUrl::new()?.to_string(),
    ));
    let ai_pricing = config.ai_pricing();
    let ai_billing = Arc::new(
        ai_billing::domain::BillingServiceImpl::new(
            ai_billing::outbound::RolesTeamsEntitlementSource::new(
                (*user_permissions_service).clone(),
                teams::outbound::team_repo::TeamRepositoryImpl::new(db.clone()),
            ),
            ai_billing::outbound::PgUsageReader::new(db.clone()),
            ai_billing::outbound::PgBillingRepo::new(db.clone(), ai_pricing),
            ai_billing::outbound::HttpPaymentGateway::new(auth_service_client.clone()),
            ai_pricing,
        )
        .with_enforcement(config.enable_ai_usage_enforcement),
    );
    let admission: Arc<dyn ai_billing::AiAdmissionService> =
        Arc::new(ai_billing::BillingAdmissionService::new(
            ai_billing.clone(),
            config.enable_ai_usage_enforcement,
        ));
    let recorder: Arc<dyn ai_usage::UsageRecorder> =
        Arc::new(ai_billing::outbound::SettlingUsageRecorder::new(
            Arc::new(
                ai_usage::domain::service::UsageServiceImpl::new(
                    ai_usage::outbound::PgUsageRepo::new(db.clone()),
                )
                .with_enforcement(config.enable_ai_usage_enforcement),
            ),
            ai_billing.clone(),
            ai_billing::outbound::HttpSettlementTrigger::new(auth_service_client.clone()),
            config.enable_ai_usage_billing,
        ));

    // Per-attempt observations use a separate journal and never feed the
    // legacy settlement trigger, which remains counted-usage-only and gated by
    // ENABLE_AI_USAGE_BILLING.
    let recorder = ai_usage::with_tracking(recorder, ai_usage::pg_tracking(db.clone()));

    let decision_classifier = config
        .typesafe_api_key
        .value()
        .map(jev::outbound::SystemOneJev::typesafe)
        .transpose()
        .context("failed to configure the decision classifier")?
        .map(|provider| jev::domain::JevClassifier::new(provider, recorder.clone()));
    let decisions = Arc::new(jev::domain::decisions::DecisionServiceImpl::new(
        decision_classifier,
        admission.clone(),
    ));
    let tool_free_completions = Arc::new(
        crate::cognition::service::structured_completion::ToolFreeCompletionService::new(
            recorder.clone(),
        ),
    );

    // The import pipeline: staged/imported external items, Slack discovery,
    // and the Notion page import job. Built before the tool service context
    // so the chat toolset gets a wired import context.
    let mcp_encryption_key = mcp_client::domain::models::AesKey::try_from(
        config.mcp_credentials_key_secret_name.as_ref(),
    )
    .context("invalid MCP credentials encryption key")?;
    let granola_signing_key = *mcp_encryption_key.as_bytes();
    let mcp_server_repo = mcp_client::outbound::NativeServerStore::new(
        mcp_client::outbound::pg_server_repo::PgServerRepo::new(db.clone(), mcp_encryption_key),
        Some(Arc::new(
            mcp_client::outbound::auth_service_github_tokens::AuthServiceGithubTokens::new(
                (*auth_service_client).clone(),
            ),
        )),
    );

    // The Pipedream MCP stack, fully separate from the native one above
    // (own endpoints, own table, own toolset). Without credentials its
    // endpoints answer 501 and its toolsets come up empty. Connect-flow
    // webhooks need both URI and secret; a half-set pair is ignored so we
    // never mint a callback we cannot serve.
    let pipedream_client: Option<Arc<pipedream_mcp::outbound::api::PipedreamClient>> = match (
        config.pipedream_client_id.value(),
        config.pipedream_client_secret.value(),
        config.pipedream_project_id.value(),
    ) {
        (Some(client_id), Some(client_secret), Some(project_id)) => {
            let webhook_uri = match (
                config.pipedream_webhook_uri.value(),
                config.pipedream_webhook_secret.value(),
            ) {
                (Some(uri), Some(_)) => Some(uri.to_owned()),
                (None, None) => None,
                _ => {
                    tracing::warn!(
                        "ignoring incomplete Pipedream webhook config; set both PIPEDREAM_WEBHOOK_URI and PIPEDREAM_WEBHOOK_SECRET"
                    );
                    None
                }
            };
            let client = pipedream_mcp::outbound::api::PipedreamClient::new(
                pipedream_mcp::outbound::api::PipedreamConfig {
                    client_id: client_id.to_owned(),
                    client_secret: client_secret.to_owned(),
                    project_id: project_id.to_owned(),
                    environment: config
                        .pipedream_environment
                        .value()
                        .unwrap_or(match config.environment {
                            Environment::Production => "production",
                            _ => "development",
                        })
                        .to_owned(),
                    api_url: config
                        .pipedream_api_url
                        .value()
                        .unwrap_or(pipedream_mcp::outbound::api::DEFAULT_API_URL)
                        .to_owned(),
                    mcp_url: config
                        .pipedream_mcp_url
                        .value()
                        .unwrap_or(pipedream_mcp::outbound::api::DEFAULT_MCP_URL)
                        .to_owned(),
                    allowed_origins: match config.pipedream_allowed_origins.value() {
                        Some(origins) => origins
                            .split(',')
                            .map(|origin| origin.trim().to_owned())
                            .filter(|origin| !origin.is_empty())
                            .collect(),
                        None => match config.environment {
                            Environment::Production => vec!["https://macro.com".to_owned()],
                            Environment::Develop => vec![
                                "https://dev.macro.com".to_owned(),
                                "http://localhost:3000".to_owned(),
                            ],
                            Environment::Local => vec!["http://localhost:3000".to_owned()],
                        },
                    },
                },
            )
            .context("failed to build Pipedream client")?;
            Some(Arc::new(match webhook_uri {
                Some(uri) => client.with_webhook_uri(uri),
                None => client,
            }))
        }
        _ => {
            tracing::info!("Pipedream credentials not set; Pipedream MCP connectors disabled");
            None
        }
    };
    // Macro Connect beside Pipedream: a user the `macro-connect` rollout is
    // on for is on Connect alone; everyone else stays on Pipedream.
    let connect_adapter = ai_tools::connect_adapter(config.connect_customer_key.value())?;
    let rollout = ai_tools::macro_connect_rollout(
        config.posthog_api_key.value(),
        config.posthog_host.value(),
    )?;
    let connectors: ai_tools::ToolPipedreamConnection = pipedream_client.as_ref().map(|client| {
        let backends = pipedream_mcp::outbound::dispatch::ConnectorBackends::new(
            client.clone(),
            connect_adapter.clone(),
        );
        Arc::new(match &rollout {
            Some(rollout) => backends.with_rollout(rollout.clone()),
            None => backends,
        })
    });
    let pipedream_repo =
        pipedream_mcp::outbound::pg_connection_repo::PgConnectionRepo::new(db.clone());
    let granola_sync: Option<Arc<dyn granola_sync::domain::service::SyncService>> =
        pipedream_client.as_ref().map(|client| {
            Arc::new(granola_sync::domain::service::Service {
                repo: granola_sync::outbound::pg_repo::PgSyncRepository::new(
                    db.clone(),
                    &granola_signing_key,
                ),
                granola: granola_sync::outbound::api::GranolaApi(client.clone()),
                accounts: granola_sync::outbound::accounts::ConnectedAccounts(
                    pipedream_repo.clone(),
                ),
                verifier: granola_sync::outbound::signature::StandardWebhooks,
                calls: Arc::new(call::domain::imports::CallImportServiceImpl(
                    call::outbound::pg_imported_call_repo::PgImportedCallRepository::new(
                        db.clone(),
                    ),
                    macro_event_broker.clone(),
                )),
                public_url: public_url.to_string(),
            }) as Arc<dyn granola_sync::domain::service::SyncService>
        });
    let granola_worker = granola_sync
        .as_ref()
        .map(|service| tokio::spawn(granola_sync::inbound::run_worker(service.clone())));

    // The one sanctioned meeting point of the two MCP stacks: agents load
    // tools through this selector, which prefers a user's Pipedream
    // connectors and falls back to the native ones (see `mcp_select`).
    // The import service's live Slack source shares this selector.
    let mcp_selector: Arc<ai_tools::ToolMcpSelector> = Arc::new(mcp_select::McpToolSelector::new(
        Arc::new(mcp_server_repo.clone()),
        Arc::new(pipedream_repo.clone()),
        Arc::new(connectors.clone()),
    ));

    // Nudges the user's connected clients when import rows flip, so setup
    // sections and chat surfaces update immediately instead of on the next
    // poll (see import::outbound::gateway_notifier).
    let import_notify = import::outbound::gateway_notifier::gateway_import_notify(
        channels_connection_gateway.clone(),
    );

    let initiative_tool_context = ai_tools::build_initiative_tool_context(
        db.clone(),
        &document_tool_context,
        properties_service.clone(),
        entity_access_service.clone(),
        macro_event_broker.clone(),
    );

    let databases_tool_context = ai_tools::build_databases_tool_context(
        db.clone(),
        entity_access_service.clone(),
        ai_tools::ToolTableEventPublisher::Gateway(
            databases::outbound::gateway_event_publisher::GatewayTableEventPublisher::new(
                channels_connection_gateway.as_ref().clone(),
            ),
        ),
        ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),
    );
    let project_tool_context = ai_tools::build_project_tool_context(
        db.clone(),
        macro_event_broker.clone(),
        entity_access_service.clone(),
        document_tool_context.service.clone(),
        user_email_service.clone(),
        databases_tool_context.service.clone(),
    );

    let entity_creator = ai_tools::ToolEntityCreator {
        document_creator: document_tool_context.creator.clone(),
        entity_access_service: entity_access_service.clone(),
        channel_service: channel_tool_context.service.clone(),
        task_properties: task_properties_for_import,
        document_properties: document_properties_for_import,
        team_repository: ai_tools::build_team_repository(db.clone()),
        initiative_service: initiative_tool_context.service.clone(),
        project_service: project_tool_context.service.clone(),
        databases_service: databases_tool_context.service.clone(),
        document_service: document_tool_context.service.clone(),
    };
    let import_service = Arc::new(
        import::domain::service::ImportServiceImpl::new(
            ai_tools::build_import_repo(db.clone()),
            mcp_selector.clone(),
            Arc::new(entity_creator),
            recorder.clone(),
        )
        .with_slack_source(Arc::new(
            import::outbound::mcp_slack_source::McpSlackSource::new(mcp_selector.clone()),
        ))
        .with_tracker_connections(
            Arc::new(
                import_linear::outbound::client::PipedreamLinearConnections::new(
                    Arc::new(pipedream_repo.clone()),
                    connectors.clone(),
                ),
            ),
            Arc::new(
                import_jira::outbound::client::PipedreamJiraConnections::new(
                    Arc::new(pipedream_repo.clone()),
                    connectors.clone(),
                ),
            ),
        )
        .with_notion_connections(Arc::new(
            import_notion::outbound::pipedream::PipedreamNotionConnections::new(
                Arc::new(pipedream_repo.clone()),
                connectors.clone(),
            ),
        ))
        .with_admission(admission.clone())
        .with_notifier(import_notify)
        .with_job_shutdown(import_jobs.shutdown.clone(), import_jobs.tracker.clone()),
    );

    // Notion workspace imports outlive the process running them: on shutdown
    // a running job hands its run over, and the sweeper adopts runs whose
    // process stopped holding them (other replicas' live runs are leased, so
    // it leaves them alone). Slack and single-page jobs heal on read. This is
    // the process's only sweeper.
    import_jobs.tracker.spawn({
        let import_service = import_service.clone();
        let shutdown = import_jobs.shutdown.clone();
        async move { import_service.sweep_stalled_notion_imports(shutdown).await }
    });
    tracing::info!("initialized import service");

    let databases_sql_tool_context = ai_tools::build_databases_sql_tool_context(
        &databases_tool_context,
        soup_service.clone(),
        db.clone(),
    );
    let crm_tool_context = ai_tools::build_crm_tool_context(db.clone());
    let pipeline_tool_context = ai_tools::build_pipeline_tool_context(
        db.clone(),
        &databases_tool_context,
        &crm_tool_context,
    );
    let crm_transfer_tool_context = ai_tools::build_crm_transfer_tool_context(
        &pipeline_tool_context,
        &crm_tool_context,
        &document_tool_context,
    );

    let calendar_tool_context = ai_tools::build_calendar_tool_context(
        db.clone(),
        CalendarServiceUrl::new()?,
        internal_api_key.clone(),
        call_tool_context.service.clone(),
        AppServiceUrl::new()?,
        macro_env_var::maybe_read_env("ENABLE_CALENDAR_ADD_SHARED_EVENT")
            .and_then(|value| value.parse::<bool>().ok())
            .unwrap_or(false),
    );
    let forms_tool_context = ai_tools::build_forms_tool_context(
        db.clone(),
        &document_tool_context,
        &databases_tool_context,
        &calendar_tool_context,
        Some(channels_connection_gateway.as_ref().clone()),
        ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),
        ai_tools::FormsToolConfig {
            app_origin: macro_service_urls::AppServiceUrl::new()?.to_string(),
            editing_worker_url: config.ai_editing_worker_url.clone(),
            static_files: static_file_service_client::StaticFileServiceClient::new(
                internal_api_key.clone(),
                macro_service_urls::StaticFileServiceUrl::new()?.to_string(),
            ),
        },
    );

    let tool_service_context = ai_tools::ToolServiceContext {
        memory_tool_context: ai_tools::memory::build_memory_tool_context(
            db.clone(),
            recorder.clone(),
        ),
        cognition_tool_context: ai_tools::build_cognition_tool_context()?,
        connector_tool_context: ai_tools::build_connector_tool_context(db.clone(), None, None),
        search_service_client: search_service_client.clone(),
        email_service_client: email_service_client_external.clone(),
        soup_service: soup_service.clone(),
        email_service: email_service_for_tools.clone(),
        activity_tool_context: ai_tools::build_activity_tool_context(
            db.clone(),
            properties_service,
            entity_access_service.clone(),
        ),
        image_generation_tool_context: ai_tools::build_image_generation_tool_context(
            &document_tool_context,
            ai_tools::build_image_generator_from_env(),
            recorder.clone(),
        )?,
        message_tool_context: ai_tools::build_message_tool_context(&document_tool_context),
        document_tool_context: document_tool_context.clone(),
        properties_tool_context: properties_tool_context.clone(),
        email_tool_context: email_tool_context.clone(),
        call_tool_context: call_tool_context.clone(),
        booking_management_tool_context: ai_tools::build_booking_management_tool_context(
            CalendarServiceUrl::new()?,
            internal_api_key.clone(),
            config.environment,
        ),
        booking_link_tool_context: ai_tools::build_booking_link_tool_context(
            db.clone(),
            config.environment,
        ),
        calendar_tool_context,
        team_calendar_tool_context: ai_tools::build_team_calendar_tool_context(
            db.clone(),
            macro_env_var::maybe_read_env("CALENDAR_TEAM_SHARING_ENABLED")
                .and_then(|value| value.parse::<bool>().ok())
                .unwrap_or(false),
        ),
        notification_tool_context: notification_tool_context.clone(),
        item_notification_preference_tool_context:
            ai_tools::build_item_notification_preference_tool_context(db.clone()),
        databases_tool_context,
        forms_tool_context,
        databases_sql_tool_context,
        import_tool_context: import::inbound::toolset::ImportToolContext::wired(
            import_service.clone(),
        ),
        mcp_server_tool_context: ai_tools::ToolMcpServerToolContext::wired(
            mcp_client::domain::service::ServerDirectory::new(Arc::new(mcp_server_repo.clone())),
        ),
        chat_tool_context,
        channel_tool_context,
        channel_media_upload_tool_context: ai_tools::build_channel_media_upload_tool_context()?,
        bot_tool_context: ai_tools::build_bot_tool_context(
            db.clone(),
            ai_tools::MaybeToolEventBroker::Real(macro_event_broker.clone()),
            entity_access_service.clone(),
            DocumentStorageServiceUrl::new()?.to_string(),
            pipedream_client.clone(),
        ),
        read_agent_session_context: ai_tools::build_read_agent_session_context(
            db.clone(),
            entity_access_service.clone(),
        ),
        coding_agent_tool_context: ai_tools::build_coding_agent_tool_context(
            macro_service_urls::AgentHarnessServiceUrl::new()?,
            internal_api_key.clone(),
        )?,
        project_tool_context,
        favorites_tool_context: ai_tools::build_favorites_tool_context(
            favorites_service,
            entity_access_service.clone(),
        ),
        initiative_tool_context,
        team_tool_context: ai_tools::build_team_tool_context(db.clone()),
        crm_tool_context,
        pipeline_tool_context,
        crm_transfer_tool_context,
        skill_tool_context: ai_tools::build_skill_tool_context(
            search_service_client.clone(),
            soup_service.clone(),
            &document_tool_context,
        ),
        schedule_tool_context: ai_tools::build_routine_tool_context()?,
        anthropic_tool_context: ai_tools::build_anthropic_tool_context(),
        web_search_provider: ai_tools::build_web_search_provider_from_env(),
        admission,
        recorder,
        usage_context: ai_usage::UsageContext::system(ai_usage::AiFeature::Chat),
    };
    let all_tools = ai_tools::tools_for(ai_tools::AiHost::Chat);
    let all_tools_toolset = all_tools.toolset.clone();
    let all_tools_prompt: Arc<dyn std::fmt::Display + Send + Sync> =
        Arc::new(all_tools.prompt.to_string());

    // Build memory service
    let memory_service = Arc::new(ai_tools::memory::build_memory_service(
        db.clone(),
        tool_service_context.clone(),
    ));

    tracing::info!("initialized memory service");

    // Build the AI cost service. It backs both the admin query/pricing router
    // and the usage recorder threaded through the tool service context.
    let usage_service = Arc::new(
        ai_usage::domain::service::UsageServiceImpl::new(ai_usage::outbound::PgUsageRepo::new(
            db.clone(),
        ))
        .with_enforcement(config.enable_ai_usage_enforcement),
    );

    tracing::info!("initialized ai cost service");

    // Generator that materializes projections by running their prompt through
    // the shared AI toolset, attributed to the requesting user.
    let projection_generator =
        ai_projections::outbound::agent_generator::AgentProjectionGenerator::new(
            tool_service_context.clone(),
            ai_tools::tools_for(ai_tools::AiHost::Chat),
        );
    // Notifier that pushes finished materializations to the target's connected
    // clients through the connection gateway.
    let projection_notifier =
        ai_projections::outbound::gateway_notifier::GatewayProjectionNotifier::new(Arc::new(
            connection_gateway_client::ConnectionGatewayClient::new(
                internal_api_key.clone(),
                ConnectionGatewayUrl::new()?.to_string(),
            ),
        ));
    let ai_projections_service_impl =
        ai_projections::domain::ai_projection_service::AiProjectionServiceImpl::new(
            ai_projections::outbound::ai_projection_repo::AiProjectionRepositoryImpl::new(
                db.clone(),
            ),
            sqs_client.clone(),
            projection_generator,
            projection_notifier,
        );

    // Spawn the inbound worker that polls ai_projection_queue and materializes
    // projection instances. It owns its own clone of the service.
    let ai_projection_worker = ai_projections::worker::AiProjectionWorker::new(
        sqs_worker::SQSWorker::new(
            aws_sdk_sqs::Client::new(&aws_config),
            ai_projection_queue.to_string(),
            10,
            10,
        ),
        ai_projections_service_impl.clone(),
    );
    let ai_projection_worker_task = tokio::spawn(async move {
        ai_projection_worker.poll().await;
    });

    let ai_projections_service = Arc::new(ai_projections_service_impl);

    tracing::info!("initialized ai projections service");

    let onboarding_service = Arc::new(onboarding::domain::service::OnboardingServiceImpl::new(
        onboarding::outbound::pg_onboarding_repo::PgOnboardingRepo::new(db.clone()),
        Arc::new(mcp_server_repo.clone()),
    ));

    tracing::info!("initialized onboarding service");

    let mcp_client_public_url = &public_url;
    let mcp_client_metadata = mcp_client::domain::models::OAuthClientMetadata::new(
        format!("{mcp_client_public_url}/mcp/servers/auth/client-metadata"),
        format!("{mcp_client_public_url}/mcp/servers/auth/callback"),
    );
    let mcp_oauth_state_store =
        mcp_client::outbound::redis_state_store::RedisOAuthStateStore::new(redis_client.clone());
    let mcp_pre_registered =
        mcp_client::domain::provider_registry::PreRegisteredProviders::from_env()?;
    let mcp_oauth = mcp_client::outbound::oauth::OAuthService::new(
        mcp_server_repo.clone(),
        mcp_oauth_state_store,
        mcp_client_metadata.clone(),
        mcp_pre_registered,
    );
    // Record connector connects and disconnects for product analytics.
    let integration_analytics = IntegrationAnalytics::from_config(&config, &db);
    let analytics_for_auth_hook = integration_analytics.clone();
    let mcp_auth_hook: mcp_client::inbound::axum_router::McpAuthCompletedHook =
        Arc::new(move |record: mcp_client::domain::models::McpServerRecord| {
            analytics_for_auth_hook.record(
                record.user_id,
                IntegrationChange::Connected,
                IntegrationEvent::mcp_server(&record.url),
            );
            Box::pin(async {})
        });
    let analytics_for_mcp_removed_hook = integration_analytics.clone();
    let mcp_removed_hook: mcp_client::inbound::axum_router::McpServerRemovedHook =
        Arc::new(move |record: mcp_client::domain::models::McpServerRecord| {
            if let Some(event) = IntegrationEvent::removed_mcp_server(&record) {
                analytics_for_mcp_removed_hook.record(
                    record.user_id,
                    IntegrationChange::Disconnected,
                    event,
                );
            }
            Box::pin(async {})
        });
    let mcp_state = mcp_client::inbound::McpRouterState::new(
        mcp_server_repo,
        mcp_oauth,
        authorization_state.clone(),
        mcp_client_metadata,
    )
    .with_auth_completed_hook(mcp_auth_hook)
    .with_server_removed_hook(mcp_removed_hook);

    let analytics_for_pipedream_hook = integration_analytics.clone();
    let pipedream_auth_hook: pipedream_mcp::inbound::axum_router::PipedreamAuthCompletedHook =
        Arc::new(
            move |connection: pipedream_mcp::domain::models::PipedreamConnection| {
                analytics_for_pipedream_hook.record(
                    connection.user_id,
                    IntegrationChange::Connected,
                    IntegrationEvent::pipedream(&connection.app_slug),
                );
                Box::pin(async {})
            },
        );
    let pipedream_disconnected_hook: pipedream_mcp::inbound::axum_router::PipedreamDisconnectedHook =
        Arc::new(
            move |connection: pipedream_mcp::domain::models::PipedreamConnection| {
                integration_analytics.record(
                    connection.user_id,
                    IntegrationChange::Disconnected,
                    IntegrationEvent::pipedream(&connection.app_slug),
                );
                Box::pin(async {})
            },
        );
    let pipedream_state = pipedream_mcp::inbound::PipedreamRouterState::new(
        pipedream_repo,
        connectors,
        authorization_state.clone(),
    )
    .with_auth_completed_hook(pipedream_auth_hook)
    .with_disconnected_hook(pipedream_disconnected_hook);

    let cancellation = tokio_util::sync::CancellationToken::new();
    super::refresh::start(
        db.clone(),
        Arc::new(sqs_client.clone()),
        cancellation.clone(),
        event_broker_tracker.clone(),
    );
    if !matches!(config.environment, Environment::Local) {
        memory::schedule::start(
            memory_service.clone(),
            &event_broker_tracker,
            cancellation.clone(),
        )?;
    }
    let router = api::build_router(ApiContext {
        db: db.clone(),
        email_service_client_external,
        sqs_client: Arc::new(sqs_client),
        document_storage_client: Arc::new(document_storage_client),
        search_service_client,
        authorization_state,
        user_permissions_service,
        non_user_owners,
        ai_admission: tool_service_context.admission.clone(),
        decisions,
        tool_free_completions,
        ai_billing,
        internal_api_key: config.internal_api_key.clone(),
        config: Arc::new(config),
        notification_ingress_service,
        connection_repo: connection_manager.persistence,
        connection_gateway_client,
        soup_service,
        email_service: email_service_for_tools,
        stream_repo,
        document_tool_context,
        memory_service,
        usage_service,
        ai_projections_service,
        properties_tool_context,
        email_tool_context: email_tool_context.clone(),
        call_tool_context,
        tool_service_context,
        all_tools: all_tools_toolset,
        all_tools_prompt,
        entity_access_service,
        message_service,
        ai_stream_registry: service::ai_stream_registry::AiStreamRegistry::new(
            redis_client.clone(),
        ),
        web_rate_limiter: rate_limit::RateLimitServiceImpl {
            repo: rate_limit::RedisRateLimitAdapter {
                redis: (*redis_client).clone(),
            },
        },
        mcp_state,
        pipedream_state,
        granola_sync,
        mcp_selector,
        import_service,
        onboarding_service,
        macro_event_broker: macro_event_broker.clone(),
    });
    let mut workers = vec![tokio_util::task::AbortOnDropHandle::new(
        ai_projection_worker_task,
    )];
    if let Some(worker) = granola_worker {
        workers.push(tokio_util::task::AbortOnDropHandle::new(worker));
    }
    Ok(CognitionRuntime {
        router,
        workers,
        cancellation,
    })
}