Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

13.3 KB
Raw Blame
#![recursion_limit = "256"]
use std::{sync::Arc, time::Duration};

#[cfg(feature = "processing")]
use crate::inbound::kafka_consumer::{KafkaProcessingContext, run_event_consumer};
use crate::{
    api::context::{ApiContext, AuthorizationService},
    config::DatabaseUrlReadonly,
    domain::{jobs::BackfillJobs, service::BackfillOrchestrator},
    outbound::{
        property_search_indexer::DirectPropertyBackfillIndexer, publisher::SqsSearchEventPublisher,
        source::PgBackfillSource,
    },
    process::{context::SearchProcessingContext, worker::run_search_processing_workers},
};
use anyhow::Context;
use config::{Config, Environment};
use lexical_client::LexicalClient;
use macro_authorization::{
    InternalAuthConfig, MacroAuthorizationState, NoopMacroAuthJwtValidator,
    PgUserApiKeyAuthorizationRepo, PgUserApiKeyAuthorizer,
};
use macro_entrypoint::MacroEntrypoint;
use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use opensearch_client::OpensearchClient;
#[cfg(feature = "pdf")]
use rust_embed::RustEmbed;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
#[cfg(feature = "processing")]
use tokio_retry::{Retry, strategy::FixedInterval};
use tokio_util::{sync::CancellationToken, task::TaskTracker};

mod api;
mod config;
mod domain;
#[cfg(feature = "processing")]
#[allow(dead_code)] // Wired into service startup with the Kafka configuration.
mod inbound;
mod outbound;
mod parsers;
mod process;

/// Concrete [`BackfillOrchestrator`] wired to the production Postgres source,
/// SQS publisher, and direct property indexer. Lives in the wiring module so
/// the domain stays agnostic of which adapters back it.
pub type BackfillServiceImpl =
    BackfillOrchestrator<PgBackfillSource, SqsSearchEventPublisher, DirectPropertyBackfillIndexer>;

/// Production wiring for the authoritative agent-session search projection.
pub type AgentSessionIndexer = domain::agent_session_index::AgentSessionIndexService<
    agent_session::domain::search::indexing::SearchSnapshotServiceImpl<
        agent_session::outbound::postgres::search::PgSearchIndexingRepo,
        agent_fold::domain::service::FoldedMessageService<
            agent_session::outbound::postgres::PgAgentSessionRepo<
                bots::outbound::pg_bots_repo::PgBotsRepo,
            >,
        >,
    >,
    outbound::agent_session_search::OpenSearchAgentSessionIndex,
>;

/// Resolve a read-replica macrodb URL and
/// connect a small pool. Returns `None` when the replica URL is missing,
/// blank. Failures
/// are intentionally warning-level rather than fatal: the readonly pool is a
/// contention optimisation, not a correctness requirement (e.g. local laptop
/// dev cannot reach the VPC-gated read replica).
async fn resolve_readonly_pool(read_only_db_url: DatabaseUrlReadonly) -> Option<PgPool> {
    if let Some(url) = read_only_db_url.value() {
        match PgPoolOptions::new()
            .min_connections(1)
            .max_connections(10)
            .connect(url)
            .await
        {
            Ok(pool) => Some(pool),
            Err(e) => {
                tracing::warn!(error=?e, "could not connect to readonly macrodb; backfills will use primary");
                None
            }
        }
    } else {
        None
    }
}

#[cfg(feature = "pdf")]
#[allow(dead_code)]
#[derive(RustEmbed)]
#[folder = "pdfium-lib/linux/"]
struct PdfiumLib;

#[cfg(feature = "processing")]
const CONSUMER_RESTART_DELAY: Duration = Duration::from_secs(5);
const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);

#[cfg(feature = "processing")]
async fn supervise_event_consumer(
    brokers: String,
    context: KafkaProcessingContext,
    shutdown_token: CancellationToken,
) {
    let _ = Retry::start(FixedInterval::new(CONSUMER_RESTART_DELAY), || {
        let consumer_brokers = brokers.clone();
        let consumer_context = context.clone();
        let consumer_shutdown_token = shutdown_token.clone();

        async move {
            if consumer_shutdown_token.is_cancelled() {
                return Ok(());
            }

            let task_shutdown_token = consumer_shutdown_token.clone();
            let consumer_result = tokio::spawn(async move {
                run_event_consumer(
                    &consumer_brokers,
                    consumer_context,
                    task_shutdown_token.cancelled_owned(),
                )
                .await
            })
            .await;

            if consumer_shutdown_token.is_cancelled() {
                tracing::info!("search processing event consumer stopped after shutdown");
                return Ok(());
            }

            match consumer_result {
                Ok(Ok(())) => {
                    tracing::error!("search processing event consumer exited unexpectedly");
                }
                Ok(Err(error)) => {
                    tracing::error!(error = ?error, "search processing event consumer failed");
                }
                Err(error) if error.is_panic() => {
                    tracing::error!(error = ?error, "search processing event consumer panicked");
                }
                Err(error) => {
                    tracing::error!(error = ?error, "search processing event consumer task was cancelled");
                }
            }

            tracing::warn!(
                restart_delay_seconds = CONSUMER_RESTART_DELAY.as_secs(),
                "search processing event consumer stopped; waiting before restart"
            );
            Err(())
        }
    })
    .await;

    tracing::info!("search processing event consumer supervisor stopped");
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    MacroEntrypoint::default().init();

    // Parse our configuration from the environment.
    let config = Config::from_env().context("expected to be able to generate config")?;
    tracing::trace!("initialized config");

    let (min_connections, max_connections): (u32, u32) = match config.environment {
        Environment::Production => (5, 50),
        Environment::Develop => (1, 25),
        Environment::Local => (1, 10),
    };

    let db = PgPoolOptions::new()
        .min_connections(min_connections)
        .max_connections(max_connections)
        .connect(config.database_url.as_ref())
        .await
        .context("could not connect to db")?;

    tracing::trace!(
        min_connections,
        max_connections,
        "initialized db connection"
    );

    let authorization_state = MacroAuthorizationState::new(Arc::new(AuthorizationService::new(
        NoopMacroAuthJwtValidator, // we only have internal calls in this service.
        InternalAuthConfig {
            api_key: config.internal_api_key.to_string(),
            default_user_id: None,
        },
        macro_authorization::NoBotAuthorizer,
        PgUserApiKeyAuthorizer::new(PgUserApiKeyAuthorizationRepo::new(db.clone())),
    )));

    let aws_config = macro_aws_config::get_macro_aws_config().await;
    let search_event_queue = macro_queues::SearchEventQueue::new();
    let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))
        .search_event_queue(&search_event_queue);

    let s3_client = Arc::new(s3_client::S3::new(macro_aws_config::s3_client().await));

    let opensearch_client = Arc::new(
        OpensearchClient::new(
            config.opensearch_url.to_string(),
            config.opensearch_username.to_string(),
            config.opensearch_password.as_ref().to_string(),
        )
        .context("unable to create opensearch client")?,
    );

    if let Err(e) = opensearch_client.health().await {
        tracing::error!(error=?e, "error connecting to opensearch");
        return Err(e);
    }

    // Global backfills use the replica when available. Explicit channel scopes
    // and queue workers use the primary so replica lag cannot hide imports.
    let backfill_db = match resolve_readonly_pool(config.database_url_readonly.clone()).await {
        Some(pool) => {
            tracing::info!("using read-replica pool for backfill reads");
            pool
        }
        None => {
            tracing::info!("backfills will read from the primary pool");
            db.clone()
        }
    };

    let sqs_client = Arc::new(sqs_client);

    let session_repo = agent_session::outbound::postgres::PgAgentSessionRepo::new(
        db.clone(),
        entity_registry_db_utils::OwnedEntityRegistrar::new(
            entity_registry::OwnerGrantPolicy::new(bots::outbound::pg_bots_repo::PgBotsRepo::new(
                db.clone(),
            )),
        ),
    );
    // Waiting advisory locks must not exhaust the pool used to read ACP logs.
    let session_lock_pool = PgPoolOptions::new()
        .max_connections(10)
        .connect_lazy(config.database_url.as_ref())?;
    let agent_session_indexer = Arc::new(AgentSessionIndexer::new(
        agent_session::domain::search::indexing::SearchSnapshotServiceImpl::new(
            agent_session::outbound::postgres::search::PgSearchIndexingRepo::new(
                db.clone(),
                session_lock_pool,
            ),
            agent_fold::domain::service::FoldedMessageService::new(session_repo),
        ),
        outbound::agent_session_search::OpenSearchAgentSessionIndex(opensearch_client.clone()),
    ));

    let backfill_service = Arc::new(BackfillOrchestrator::new(
        PgBackfillSource::new(backfill_db, db.clone(), config.backfill_page_sizes()?),
        SqsSearchEventPublisher::new(sqs_client.clone()),
        DirectPropertyBackfillIndexer::new(db.clone(), opensearch_client.clone()),
    ));

    let shutdown_token = CancellationToken::new();
    let event_broker_tracker = TaskTracker::new();
    let macro_event_broker = MacroEventBrokerService::new(
        KafkaEventPublisher::new(config.kafka_brokers.as_ref())
            .context("failed to create kafka event publisher")?,
        event_broker_tracker.clone(),
    );

    #[cfg(feature = "processing")]
    let consumer_supervisor = {
        // Ensures that pdfium binary exists so we can kill the container early on failure
        #[cfg(feature = "pdf")]
        if !std::fs::exists("./pdfium-lib/linux/libpdfium.so").expect("able to find file") {
            anyhow::bail!("libpdfium.so is missing");
        } else {
            tracing::trace!("libpdfium is present");
        }

        let lexical_client = Arc::new(LexicalClient::new(
            config.internal_api_key.to_string(),
            config.lexical_service_url.clone(),
        ));

        let worker = sqs_worker::SQSWorker::new(
            aws_sdk_sqs::Client::new(&aws_config),
            search_event_queue.to_string(),
            config.queue_max_messages,
            config.queue_wait_time_seconds,
        );
        let search_processing_context = SearchProcessingContext {
            db: db.clone(),
            worker: Arc::new(worker.clone()),
            document_storage_bucket: config.document_storage_bucket.to_string(),
            s3_client: s3_client.clone(),
            opensearch_client: opensearch_client.clone(),
            lexical_client: lexical_client.clone(),
            calendar_search_enabled: config.calendar_search_enabled,
        };
        run_search_processing_workers(search_processing_context, config.worker_count);

        let kafka_processing_context = KafkaProcessingContext {
            agent_session_indexer: agent_session_indexer.clone(),
            db: db.clone(),
            opensearch_client: opensearch_client.clone(),
            s3_client: s3_client.clone(),
            document_storage_bucket: config.document_storage_bucket.to_string(),
            lexical_client,
            calendar_search_enabled: config.calendar_search_enabled,
        };
        tokio::spawn(supervise_event_consumer(
            config.kafka_brokers.as_ref().to_owned(),
            kafka_processing_context,
            shutdown_token.clone(),
        ))
    };

    let dynamodb_client = aws_sdk_dynamodb::Client::new(&aws_config);
    let backfill_jobs = BackfillJobs::new(
        dynamodb_client,
        config.backfill_jobs_table.to_string(),
        std::time::Duration::from_secs(config.backfill_job_ttl_seconds()?),
    );
    if matches!(config.environment, Environment::Local) {
        backfill_jobs
            .ensure_table()
            .await
            .context("failed to ensure backfill jobs table exists")?;
    }

    let api_result = api::setup_and_serve(
        ApiContext {
            agent_session_indexer,
            db,
            authorization_state,
            opensearch_client,
            config: Arc::new(config),
            backfill_service,
            backfill_jobs,
            macro_event_broker,
        },
        shutdown_token.clone(),
    )
    .await;

    shutdown_token.cancel();
    #[cfg(feature = "processing")]
    let consumer_result = consumer_supervisor
        .await
        .context("search processing event consumer supervisor failed");

    tracing::info!("waiting for event broker publishes to drain");
    event_broker_tracker.close();
    match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
        Ok(()) => tracing::info!("event broker publishes drained"),
        Err(error) => {
            tracing::warn!(
                error=?error,
                timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
                "timed out waiting for event broker publishes to drain"
            );
        }
    }

    #[cfg(feature = "processing")]
    consumer_result?;
    api_result
}