Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

19.4 KB
Raw Blame
use std::{future::Future, sync::Arc, time::Duration};

use anyhow::{Context, Result};
use axum::Router;
use bots::outbound::pg_bots_repo::PgBotsRepo;
use connection_gateway_client::client::ConnectionGatewayClient;
use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
use entity_registry::OwnerGrantPolicy;
use entity_registry_db_utils::OwnedEntityRegistrar;
use github::outbound::pg_installation_claims::PgGithubInstallationClaims;
use jev::{domain::JevClassifier, outbound::SystemOneJev};
use macro_auth::middleware::decode_jwt::JwtValidationArgs;
use macro_authorization::{
    InternalAuthConfig, MacroAuthJwtValidator, MacroAuthorizationServiceImpl,
    MacroAuthorizationState, PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
};
use macro_service_urls::{ConnectionGatewayUrl, LexicalServiceUrl};
pub mod config;
mod routine_analytics;
mod routine_run_notifications;
use config::Config;
use scheduled_action::domain::code_upgrade;
use scheduled_action::domain::drafts::RoutineDrafts;
use scheduled_action::domain::event_runs::{
    PageSize, admission::EventAdmissionService, condition::ConditionGate,
    dispatch::EventDispatchService,
};
use scheduled_action::domain::ports::ScheduledActionDispatcher;
use scheduled_action::domain::program_runner::ProgramRunner;
use scheduled_action::domain::service::ScheduledActionServiceImpl;
use scheduled_action::domain::target_runner::TargetRunner;
use scheduled_action::domain::target_validation::TargetValidation;
use scheduled_action::domain::templates::RoutineTemplateServiceImpl;
use scheduled_action::inbound::axum_router::{ScheduledActionRouterState, scheduled_action_router};
use scheduled_action::inbound::draft_router::{RoutineDraftRouterState, routine_draft_router};
use scheduled_action::inbound::event_run_worker::run_event_worker;
use scheduled_action::inbound::kafka_consumer::run_scheduled_action_event_consumer;
use scheduled_action::inbound::template_router::{
    RoutineTemplateRouterState, routine_template_router,
};
use scheduled_action::outbound::agent_session_client::AgentSessionClient;
use scheduled_action::outbound::conn_gateway_live_updates::ConnGatewayLiveUpdates;
use scheduled_action::outbound::event_access::EventAccessAdapter;
use scheduled_action::outbound::event_content::EventContentAdapter;
use scheduled_action::outbound::inprocess_executor::InProcessExecutor;
use scheduled_action::outbound::pg_code_upgrades::PgCodeUpgrades;
use scheduled_action::outbound::pg_event_run_repo::PgEventRunRepo;
use scheduled_action::outbound::pg_polling_dispatcher::{
    PgPollingDispatcher, PgPollingDispatcherLifecycle,
};
use scheduled_action::outbound::pg_routine_template_repo::PgRoutineTemplateRepo;
use scheduled_action::outbound::pg_scheduled_action_repo::PgScheduledActionRepo;
use scheduled_action::swagger::ApiDoc;
use tokio_util::{sync::CancellationToken, task::TaskTracker};
use utoipa::OpenApi;
use utoipa_swagger_ui::SwaggerUi;

#[cfg(test)]
mod test;

// ECS stopTimeout is ten seconds. One shared budget includes HTTP, all
// dispatch/execution bookkeeping, leaving two seconds.
const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(8);
const CONSUMER_RESTART_DELAY: Duration = Duration::from_secs(5);
// Keep event agent work well below the shared pool's connection limit, leaving
// headroom for HTTP/cron and tool calls (not a per-tool connection reservation).
const EVENT_CONCURRENCY: u16 = 2;
const ADMISSION_PAGE_SIZE: u16 = 100;
const GATEWAY_PATH_PREFIX: &str = "/scheduled-action";

/// Scheduling routes and workers owned by the harness process.
pub struct SchedulingRuntime {
    pub router: Router,
    lifecycle: ServiceLifecycle,
}

impl SchedulingRuntime {
    pub fn stop(&self) {
        self.lifecycle.stop();
    }
    pub async fn shutdown(self) {
        self.lifecycle.stop();
        if tokio::time::timeout(SHUTDOWN_TIMEOUT, self.lifecycle.drain())
            .await
            .is_err()
        {
            tracing::warn!("scheduler shutdown deadline reached");
        }
    }
}

/// Compose routine scheduling and event workers alongside the harness API.
pub async fn build<Ingress: notification::domain::service::NotificationIngress>(
    config: Config,
    db: sqlx::PgPool,
    harness_address: std::net::SocketAddr,
    code: Option<crate::routine_code::SdkRoutineCode>,
    drafter: crate::routine_drafts::ModelDrafts,
    ready: tokio::sync::watch::Receiver<bool>,
    notifications: Arc<Ingress>,
) -> Result<SchedulingRuntime> {
    let environment = config.environment;
    let lifecycle = ServiceLifecycle::default();

    let secretsmanager_client = secretsmanager_client::SecretsManager::new(
        aws_sdk_secretsmanager::Client::new(&macro_aws_config::get_macro_aws_config().await),
    );
    let conn_gateway_client = Arc::new(ConnectionGatewayClient::new(
        config.internal_api_key.to_string(),
        ConnectionGatewayUrl::new()?.to_string(),
    ));
    let live_updates = Arc::new(ConnGatewayLiveUpdates::new(Arc::clone(
        &conn_gateway_client,
    )));

    let registrar = OwnedEntityRegistrar::new(OwnerGrantPolicy::new(PgBotsRepo::new(db.clone())));
    let repo = Arc::new(PgScheduledActionRepo::new(db.clone(), registrar.clone()));

    let event_repo = Arc::new(PgEventRunRepo::new(db.clone()));
    let access = Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
        db.clone(),
    )));
    let event_access = Arc::new(EventAccessAdapter::new(
        access.as_ref().clone(),
        PgGithubInstallationClaims::new(db.clone()),
    ));
    let ai_admission = ai_billing::composition::pg_admission_service(
        db.clone(),
        config.enable_ai_usage_enforcement,
        config.ai_pricing(),
    );
    let sessions = Arc::new(AgentSessionClient::new(
        &format!("http://{harness_address}"),
        &config.internal_api_key,
    )?);
    // Conditions and decisions share one classifier, the only model work this
    // process meters (agent targets are funded by the harness); both are off
    // without a TypeSafe key. Counted usage asks the authentication service to
    // settle the payer, as every other host does.
    let typesafe = config
        .typesafe_api_key
        .value()
        .map(SystemOneJev::typesafe)
        .transpose()
        .context("invalid TYPESAFE_API_KEY")?;
    let conditions_enabled = typesafe.is_some();
    let classifier = typesafe
        .map(|provider| -> Result<_> {
            Ok(Arc::new(JevClassifier::new(
                provider,
                ai_billing::composition::pg_settling_recorder(
                    db.clone(),
                    config.enable_ai_usage_enforcement,
                    config.ai_pricing(),
                    config.settlement_route()?,
                ),
            )))
        })
        .transpose()?;
    // Trigger content is read for templates and the agent's account of the
    // event whether or not there is a classifier.
    let event_content = Arc::new(event_content(
        &db,
        &registrar,
        Arc::clone(&access),
        &config,
    )?);
    // Tool and code steps run in this process, as the owner; agent steps
    // still go through the routine-session operations.
    let code = Arc::new(code);
    let upgrades = Arc::new(PgCodeUpgrades::new(db.clone()));
    // After a release, bring routines' code to the new SDK before their runs
    // need it; a run still upgrades its own code if this hasn't reached it.
    {
        let (code, upgrades, repo) = (Arc::clone(&code), Arc::clone(&upgrades), Arc::clone(&repo));
        let stop = lifecycle.stop_workers.clone();
        let mut ready = ready.clone();
        lifecycle.workers.spawn(async move {
            tokio::select! {
                biased;
                _ = stop.cancelled() => return,
                result = ready.wait_for(|ready| *ready) => if result.is_err() { return; },
            }
            if let Err(error) =
                code_upgrade::sweep(code.as_ref(), upgrades.as_ref(), repo.as_ref(), &stop).await
            {
                tracing::warn!(error = ?error, "routine code sweep stopped");
            }
        });
    }
    let runner = Arc::new(
        ProgramRunner::new(
            Arc::new(TargetRunner::new(Arc::clone(&sessions))),
            Arc::clone(&event_content),
        )
        .with_code(Arc::clone(&code), Arc::clone(&upgrades)),
    );
    let dispatcher_executor = InProcessExecutor::new(
        Arc::clone(&repo),
        runner,
        live_updates,
        lifecycle.executions.clone(),
        lifecycle.stop_executions.clone(),
    )
    .with_admission(Arc::clone(&ai_admission))
    .with_observer(Arc::new((
        routine_analytics::RoutineAnalytics(
            agent_harness_service::product_analytics::ProductAnalytics::new(
                config.posthog_api_key.value(),
                config.posthog_host.value(),
                environment,
                &db,
            ),
        ),
        routine_run_notifications::RoutineRunNotifications::new(
            notifications,
            lifecycle.executions.clone(),
        ),
    )));
    let service_executor = Arc::new(dispatcher_executor.clone());

    let jwt_args = JwtValidationArgs::new_with_secret_manager(environment, &secretsmanager_client)
        .await
        .context("failed to build jwt validation args")?;

    let authorization_service = MacroAuthorizationServiceImpl::new(
        MacroAuthJwtValidator::new(jwt_args),
        InternalAuthConfig {
            api_key: config.internal_api_key.to_string(),
            default_user_id: None,
        },
        macro_authorization::NoBotAuthorizer,
        PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
    );
    let authorization_state = MacroAuthorizationState::new(Arc::new(authorization_service));

    // Finish fallible startup before launching intake or claiming any work.

    let dispatcher = PgPollingDispatcher::new(Arc::clone(&repo), dispatcher_executor)
        .with_lifecycle(
            PgPollingDispatcherLifecycle::new(
                lifecycle.stop_workers.clone(),
                lifecycle.workers.clone(),
            )
            .with_readiness(ready.clone()),
        );
    let (dispatcher_tx, execution_rx) = dispatcher.begin_dispatch_loop();
    // No subscriber consumes these notifications; don't let a full channel
    // eventually block cron dispatch or its shutdown.
    drop(execution_rx);

    let conditions = classifier.map(|classifier| ConditionGate::new(event_content, classifier));
    let event_dispatch = EventDispatchService::new(
        Arc::clone(&event_repo),
        Arc::clone(&event_access),
        Arc::clone(&service_executor),
    )
    .with_conditions(Arc::new(conditions));
    let brokers = config.kafka_brokers.to_string();
    let intake_shutdown = lifecycle.stop_consumers.clone();
    let mut worker_ready = ready.clone();
    let worker_shutdown = lifecycle.stop_workers.clone();
    let executions = lifecycle.executions.clone();
    start_event_tasks(
        config.event_routines_enabled,
        &lifecycle,
        move || {
            let mut ready = ready.clone();
            let admission = EventAdmissionService::new(
                Arc::clone(&event_repo),
                Arc::clone(&event_access),
                PageSize::try_from(ADMISSION_PAGE_SIZE).expect("valid admission bound"),
            );
            let brokers = brokers.clone();
            let shutdown = intake_shutdown.clone();
            async move {
                tokio::select! {
                    biased;
                    _ = shutdown.cancelled() => return Ok(()),
                    result = ready.wait_for(|ready| *ready) => if result.is_err() { return Ok(()); },
                }
                run_scheduled_action_event_consumer(&brokers, admission, shutdown.cancelled()).await
            }
        },
        async move {
            tokio::select! {
                biased;
                _ = worker_shutdown.cancelled() => return,
                result = worker_ready.wait_for(|ready| *ready) => if result.is_err() { return; },
            }
            run_event_worker(
                event_dispatch,
                worker_shutdown,
                executions,
                PageSize::try_from(EVENT_CONCURRENCY).expect("valid worker bound"),
            )
            .await
        },
    );

    // AI drafts routines as their owner would save them, checking code
    // with the same checker.
    let drafts = RoutineDrafts::new(Arc::clone(&code), Arc::new(drafter), ai_admission)
        .with_conditions_enabled(conditions_enabled);
    let service = Arc::new(
        ScheduledActionServiceImpl::new(
            Arc::clone(&repo),
            service_executor,
            dispatcher_tx,
            access.clone(),
        )
        .with_event_management_enabled(config.event_routines_enabled)
        .with_conditions_enabled(conditions_enabled)
        .with_target_validation(TargetValidation::new(sessions))
        .with_github_links(PgGithubInstallationClaims::new(db.clone()))
        .with_code(code, upgrades),
    );
    let template_flags = config
        .posthog_api_key
        .value()
        .map(|key| {
            feature_flags::outbound::posthog::PostHogFeatureFlags::new(
                config
                    .posthog_host
                    .value()
                    .unwrap_or("https://us.i.posthog.com"),
                key.to_owned(),
                Duration::from_secs(3),
            )
        })
        .transpose()
        .map_err(|error| anyhow::anyhow!("invalid routine feature flag configuration: {error}"))?;
    let template_state = RoutineTemplateRouterState {
        service: Arc::new(RoutineTemplateServiceImpl::new(
            Arc::clone(&service),
            Arc::new(PgRoutineTemplateRepo::new(db.clone())),
            template_flags,
        )),
        access_service: Arc::clone(&access),
        authorization_state: authorization_state.clone(),
    };
    let draft_state = RoutineDraftRouterState {
        service: Arc::new(drafts),
        authorization_state: authorization_state.clone(),
    };
    let state = ScheduledActionRouterState {
        service,
        access_service: access,
        authorization_state,
    };
    let authed_routes = scheduled_action_router::<_, _, _, ()>(state)
        .merge(routine_template_router::<_, _, _, ()>(template_state))
        .merge(routine_draft_router::<_, _, ()>(draft_state));

    let router = mount_router(authed_routes)
        .layer(macro_cors::cors_layer())
        .layer(
            macro_tower_layers::MacroRequestIdAndTracingLayer::new(Duration::from_millis(200))
                .into_inner(),
        );
    Ok(SchedulingRuntime { router, lifecycle })
}

fn mount_router(routes: Router) -> Router {
    Router::new().nest(GATEWAY_PATH_PREFIX, routes).merge(
        SwaggerUi::new("/scheduled-action/docs")
            .url("/scheduled-action/api-doc/openapi.json", ApiDoc::openapi()),
    )
}

/// Read-only views of the domains whose events can trigger a routine, for
/// trigger conditions, code and the agent's account of the event. None of
/// these reads notify, publish or enqueue; routine writes go through code
/// steps' SDK calls or the agent.
fn event_content(
    db: &sqlx::PgPool,
    registrar: &OwnedEntityRegistrar<PgBotsRepo>,
    access: Arc<EntityAccessServiceImpl<PgAccessRepository>>,
    config: &Config,
) -> Result<
    impl scheduled_action::domain::event_runs::condition::EventContentReader
    + scheduled_action::domain::ports::RoutineEventReader
    + use<>,
> {
    Ok(EventContentAdapter::new(
        email::domain::service::EmailServiceImpl::new(
            email::outbound::EmailPgRepo::new(db.clone()),
            frecency::domain::services::FrecencyQueryServiceImpl::new(
                frecency::outbound::postgres::FrecencyPgStorage::new(db.clone()),
            ),
            email::domain::ports::NoOpEnqueuer,
            crm::domain::service::NoOpCrmService,
            entity_access_management::domain::service::EntityAccessManagementServiceImpl::new(
                entity_access_management::outbound::PgRepository::new(db.clone()),
            ),
            0,
        ),
        messages::domain::service::MessageService::new(
            messages::outbound::pg_message_repo::PgMessageRepository::new(db.clone()),
            messages::domain::ports::NoMessageEventPublisher,
        ),
        channels::domain::service::ChannelServiceImpl::new(
            channels::outbound::pg_channels_repo::PgChannelsRepo::new(db.clone()),
        ),
        documents::outbound::pg_document_repo::PgDocumentRepo::new(db.clone(), registrar.clone()),
        properties::PropertiesServiceImpl::new(
            properties::PropertiesPgRepo::new(db.clone()),
            Some(properties::PermissionServiceImpl::new(db.clone(), access)),
            None::<NoTaskNotifications>,
        ),
        call::domain::service::CallRecordQueryServiceImpl::new(
            call::outbound::pg_call_repo::PgCallRepo::new(db.clone()),
        ),
        lexical_client::LexicalClient::new(
            config.internal_api_key.to_string(),
            LexicalServiceUrl::new()?.to_string(),
        ),
    ))
}

/// Event content only reads properties; it never assigns tasks.
struct NoTaskNotifications;

impl properties::NotificationService for NoTaskNotifications {
    type Err = anyhow::Error;

    async fn send_task_assigned<'a>(
        &self,
        _: properties::domain::model::TaskAssignedNotification<'a>,
    ) -> Result<(), Self::Err> {
        Ok(())
    }
}

#[derive(Default)]
struct ServiceLifecycle {
    stop_consumers: CancellationToken,
    stop_workers: CancellationToken,
    stop_executions: CancellationToken,
    consumers: TaskTracker,
    workers: TaskTracker,
    executions: TaskTracker,
}

impl ServiceLifecycle {
    fn stop(&self) {
        self.stop_consumers.cancel();
        self.stop_workers.cancel();
        self.stop_executions.cancel();
        self.consumers.close();
        self.workers.close();
        self.executions.close();
    }

    async fn drain(&self) {
        // Workers may still be completing a committed claim. Wait for them
        // before considering the execution tracker permanently empty.
        tokio::join!(self.consumers.wait(), self.workers.wait());
        self.executions.wait().await;
    }
}

fn start_event_tasks<C, F>(
    enabled: bool,
    lifecycle: &ServiceLifecycle,
    consumer: C,
    worker: impl Future<Output = ()> + Send + 'static,
) where
    C: FnMut() -> F + Send + 'static,
    F: Future<Output = Result<(), rootcause::Report>> + Send + 'static,
{
    if !enabled {
        return;
    }
    lifecycle.consumers.spawn(supervise_consumer(
        consumer,
        lifecycle.stop_consumers.clone(),
        CONSUMER_RESTART_DELAY,
    ));
    lifecycle.workers.spawn(worker);
}

async fn supervise_consumer<F: Future<Output = Result<(), rootcause::Report>>>(
    mut consume: impl FnMut() -> F,
    shutdown: CancellationToken,
    restart_delay: Duration,
) {
    loop {
        tokio::select! {
            biased;
            _ = shutdown.cancelled() => return,
            result = consume() => {
                // Do not log admission reports, which can contain event data.
                tracing::warn!(failed = result.is_err(), "event consumer exited; restarting with a fresh consumer");
            }
        }
        // Even an unexpected successful exit must not silently disable intake.
        // A new consumer restores committed offsets; never reuse its position.
        tokio::select! {
            biased;
            _ = shutdown.cancelled() => return,
            _ = tokio::time::sleep(restart_delay) => {},
        }
    }
}