Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

7.2 KB
Raw Blame
use std::sync::Arc;
use std::time::Duration;

use anyhow::{Context, Result};
use common_cohort_team_access::TeamAccess;
use common_database::get_pool_with_config;
use common_kafka::kafka_consumer::SingleTopicConsumer;
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
use tokio::net::TcpListener;
use tracing::{info, warn};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{fmt, EnvFilter, Layer};

use cohort_event_shuffler::config::Config;
use cohort_event_shuffler::consumer::EventShuffler;
use cohort_event_shuffler::filter_team_index::{run_refresh_loop, TeamGate, TeamIndex};
use cohort_event_shuffler::observability;
use cohort_event_shuffler::producer::CohortStreamProducer;

common_alloc::used!();

const SERVICE_NAME: &str = "cohort-event-shuffler";

fn main() -> Result<()> {
    let config = Config::init_from_env()
        .context("Failed to load configuration from environment variables")?;

    let runtime = tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .build()
        .context("Failed to build tokio runtime")?;

    runtime.block_on(async_main(config))
}

async fn async_main(config: Config) -> Result<()> {
    init_tracing();
    log_startup(&config);
    let settings = config.shuffler_settings()?;

    let mut manager = Manager::builder(SERVICE_NAME)
        .with_global_shutdown_timeout(Duration::from_secs(30))
        .build();

    let metrics_handle =
        manager.register("metrics", ComponentOptions::new().is_observability(true));
    // The consumer owns the liveness deadline: a wedged pipeline iteration — or a stalled
    // committer with committable work — stops heartbeats, and deadline × stall_threshold trips
    // coordinated shutdown.
    let consumer_handle = manager.register(
        "consumer",
        ComponentOptions::new()
            .with_graceful_shutdown(Duration::from_secs(15))
            .with_liveness_deadline(Duration::from_secs(60))
            .with_stall_threshold(3),
    );
    // No liveness deadline on either refresh: a refresh outage must not kill the service
    // (staleness is safe). An unexpected exit still signals the manager via the process-scope guard.
    let team_index_handle = manager.register(
        "team-index",
        ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let team_access_handle = manager.register(
        "team-access",
        ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );

    let readiness = manager.readiness_handler();
    let liveness = manager.liveness_handler();

    let recorder_handle = if config.export_prometheus {
        Some(observability::metrics::install_recorder())
    } else {
        None
    };

    // Build infrastructure before starting the monitor: a failure here returns before the monitor
    // runs, so dropped handles are harmless no-ops and the process exits non-zero.
    let pool = get_pool_with_config(&config.database_url, config.pool_config())
        .context("creating posthog_cohort database pool")?;

    let team_index = Arc::new(TeamIndex::new());
    match team_index.refresh(&pool).await {
        Ok(count) => info!(realtime_teams = count, "initial team index loaded"),
        Err(err) => warn!(
            error = %err,
            "initial team index load failed; consumer forwards nothing until the refresh task succeeds",
        ),
    }

    let (team_access, mut team_access_poller) = TeamAccess::new(config.team_allowlist.clone());
    match team_access_poller.poll(&pool).await {
        Ok(granted) => info!(granted_teams = granted, "initial team access loaded"),
        Err(err) => warn!(
            error = %err,
            "initial team access load failed; consumer forwards nothing until a poll succeeds",
        ),
    }

    let kafka_config = config.build_kafka_config();
    let producer = CohortStreamProducer::new(&kafka_config, config.output_topic.clone())
        .await
        .context("creating cohort_stream_events producer")?;
    let consumer = SingleTopicConsumer::new(kafka_config, config.build_consumer_config())
        .context("creating clickhouse_events_json consumer")?;

    let guard = manager.monitor_background();

    let refresh_index = team_index.clone();
    let refresh_pool = pool.clone();
    let refresh_interval = config.team_index_refresh_interval();
    let refresh_jitter = config.team_index_refresh_jitter();
    tokio::spawn(async move {
        run_refresh_loop(
            refresh_index,
            refresh_pool,
            refresh_interval,
            refresh_jitter,
            team_index_handle,
        )
        .await;
    });

    tokio::spawn(team_access_poller.run(pool, team_access_handle));

    let team_gate = TeamGate::new(team_index, team_access);
    let shuffler = EventShuffler::new(consumer, producer, team_gate, consumer_handle, settings);
    tokio::spawn(async move {
        shuffler.process().await;
    });

    let app = observability::health::router(SERVICE_NAME, readiness, liveness, recorder_handle);
    let bind = config.bind_address();
    info!(address = %bind, "observability server starting");

    let listener = TcpListener::bind(&bind)
        .await
        .with_context(|| format!("failed to bind observability server to {bind}"))?;
    axum::serve(listener, app)
        .with_graceful_shutdown(metrics_handle.shutdown_signal())
        .await
        .context("observability server error")?;
    metrics_handle.work_completed();

    guard.wait().await?;

    info!(service = SERVICE_NAME, "service stopped");
    Ok(())
}

/// Deliberately omits `database_url`, which carries credentials.
fn log_startup(config: &Config) {
    info!(
        service = SERVICE_NAME,
        input_topic = %config.input_topic,
        output_topic = %config.output_topic,
        consumer_group = %config.kafka_consumer_group,
        partitioner = %config.kafka_producer_partitioner,
        offset_reset = %config.kafka_consumer_offset_reset,
        max_inflight_forwards = config.max_inflight_forwards,
        commit_interval_ms = config.commit_interval_ms,
        team_index_refresh_secs = config.team_index_refresh_secs,
        team_allowlist = ?config.team_allowlist,
        "starting cohort-event-shuffler",
    );
}

/// JSON structured logging by default; human-readable when `RUST_LOG` contains `debug`.
fn init_tracing() {
    let is_debug = std::env::var("RUST_LOG")
        .map(|v| v.contains("debug"))
        .unwrap_or(false);

    let log_layer = if is_debug {
        fmt::layer()
            .with_target(true)
            .with_level(true)
            .with_ansi(true)
            .with_filter(
                EnvFilter::builder()
                    .with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
                    .from_env_lossy(),
            )
            .boxed()
    } else {
        fmt::layer()
            .json()
            .flatten_event(true)
            .with_current_span(true)
            .with_filter(
                EnvFilter::builder()
                    .with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
                    .from_env_lossy(),
            )
            .boxed()
    };

    tracing_subscriber::registry().with(log_layer).init();
}