Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.4 KB
Raw Blame
use std::sync::Arc;
use std::time::Duration;

use assignment_coordination::store::{EtcdStore, StoreConfig};
use axum::{routing::get, Router};
use common_database::{get_pool_with_config, PoolConfig};
use common_kafka::config::KafkaConfig;
use common_metrics::{setup_metrics_routes_with_overrides, Matcher};
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
use metrics::counter;
use personhog_coordination::changelog::{load_changelog_config, CHANGELOG_BOOT_POLL_INTERVAL};
use personhog_coordination::follow::{follow_changelog, FollowTarget, Follower, Reporter};
use personhog_coordination::store::PersonhogStore;
use personhog_coordination::types::ClusterEndpoint;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::fmt;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::EnvFilter;

use personhog_writer::config::Config;
use personhog_writer::consumer::ConsumerTask;
use personhog_writer::kafka::{base_client_config, PersonConsumer};
use personhog_writer::pg::PgStore;
use personhog_writer::store::PersonWriteStore;
use personhog_writer::writer::WriterTask;

common_alloc::used!();

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let config = Config::init_from_env().expect("Invalid configuration");

    let log_layer = fmt::layer()
        .with_target(true)
        .with_thread_ids(true)
        .with_level(true);

    tracing_subscriber::registry()
        .with(log_layer)
        .with(
            EnvFilter::builder()
                .with_default_directive(LevelFilter::INFO.into())
                .from_env_lossy(),
        )
        .init();

    tracing::info!("Starting personhog-writer");
    tracing::info!("Consumer group: {}", config.kafka_consumer_group);
    tracing::info!("Flush interval: {}ms", config.flush_interval_ms);
    tracing::info!("Flush buffer size: {}", config.flush_buffer_size);
    tracing::info!("Buffer capacity: {}", config.buffer_capacity);
    tracing::info!("Writer lanes: {}", config.writer_lanes);
    tracing::info!("Upsert concurrency: {}", config.upsert_concurrency);
    tracing::info!("Metrics port: {}", config.metrics_port);

    let store = EtcdStore::connect_with_member_failover(StoreConfig {
        endpoints: config.etcd_endpoint_list(),
        prefix: config.etcd_prefix.clone(),
    })
    .await?;
    let store = Arc::new(PersonhogStore::new(store));
    // No signal handler is installed until the lifecycle monitor starts,
    // so SIGTERM ends the process during this wait.
    let Some(loaded) = load_changelog_config(
        &store,
        CHANGELOG_BOOT_POLL_INTERVAL,
        &CancellationToken::new(),
    )
    .await?
    else {
        return Ok(());
    };
    if loaded.writer_group != config.kafka_consumer_group {
        return Err(format!(
            "config/changelog names writer group {} but KAFKA_CONSUMER_GROUP is {}",
            loaded.writer_group, config.kafka_consumer_group
        )
        .into());
    }
    tracing::info!(
        topic = %loaded.topic,
        epoch = loaded.epoch,
        "loaded changelog config"
    );
    // Oldest undrained generation first, so a restart mid-drain resumes it.
    let mut follower = Follower::default();
    let FollowTarget { topic, cluster } = follower
        .select_topic(&loaded, |target| drained(&config, target))
        .await?;

    let mut manager = Manager::builder("personhog-writer")
        .with_global_shutdown_timeout(Duration::from_secs(30))
        .build();

    let consumer_handle = manager.register(
        "consumer",
        ComponentOptions::new()
            .with_graceful_shutdown(Duration::from_secs(15))
            .with_liveness_deadline(Duration::from_secs(30)),
    );
    // All components must be registered before monitor_background() starts,
    // so writer lane handles are created up front.
    let lanes = config.writer_lanes.max(1);
    let mut writer_handles = Vec::with_capacity(lanes);
    for lane in 0..lanes {
        writer_handles.push(
            manager.register(
                &format!("writer-{lane}"),
                ComponentOptions::new()
                    .with_graceful_shutdown(Duration::from_secs(15))
                    .with_liveness_deadline(Duration::from_secs(30)),
            ),
        );
    }
    let metrics_handle = manager.register(
        "metrics-server",
        ComponentOptions::new().is_observability(true),
    );
    let changelog_handle = manager.register("changelog-config", ComponentOptions::new());

    let readiness = manager.readiness_handler();
    let liveness = manager.liveness_handler();

    let monitor_guard = manager.monitor_background();

    // Metrics/health HTTP server
    let metrics_port = config.metrics_port;
    tokio::spawn(async move {
        let _guard = metrics_handle.process_scope();

        let health_router = Router::new()
            .route(
                "/_readiness",
                get(move || {
                    let r = readiness.clone();
                    async move { r.check().await }
                }),
            )
            .route("/_liveness", get(move || async move { liveness.check() }));
        // E2E latency is a lag detector: healthy produce-to-commit lag is
        // seconds (the flush cadence), and the interesting tail is minutes
        // of writer lag. The default buckets cap at 10s, which would
        // collapse every lag excursion into +Inf.
        let metrics_router = setup_metrics_routes_with_overrides(
            health_router,
            &[(
                Matcher::Full("personhog_writer_e2e_latency_ms".into()),
                &[
                    250.0, 1000.0, 2500.0, 5000.0, 10000.0, 30000.0, 60000.0, 120000.0, 300000.0,
                    600000.0,
                ],
            )],
        );

        let bind = format!("0.0.0.0:{metrics_port}");
        let listener = tokio::net::TcpListener::bind(&bind)
            .await
            .expect("Failed to bind metrics port");
        tracing::info!("Metrics server listening on {}", bind);
        axum::serve(listener, metrics_router)
            .with_graceful_shutdown(metrics_handle.shutdown_signal())
            .await
            .expect("Metrics server error");
    });

    // Postgres pool
    let pool_config = PoolConfig {
        max_connections: config.pg_max_connections,
        pool_name: Some("personhog-writer".to_string()),
        statement_timeout_ms: Some(30_000),
        ..Default::default()
    };
    let pool = get_pool_with_config(&config.database_url, pool_config)?;

    // Sample pool state into Prometheus gauges every 5s. Useful for tuning
    // PG_MAX_CONNECTIONS against observed utilization during fallback.
    {
        let pool = pool.clone();
        tokio::spawn(async move {
            let mut ticker = tokio::time::interval(std::time::Duration::from_secs(5));
            loop {
                ticker.tick().await;
                metrics::gauge!("personhog_writer_pg_pool_size").set(pool.size() as f64);
                metrics::gauge!("personhog_writer_pg_pool_idle").set(pool.num_idle() as f64);
            }
        });
    }

    let kafka_consumer = Arc::new(
        PersonConsumer::from_config(
            &changelog_kafka(&config.kafka, cluster.as_ref()),
            &config.kafka_consumer_group,
            topic.clone(),
        )?
        .on_cluster(cluster.clone()),
    );
    kafka_consumer.set_draining(topic != loaded.topic);
    tracing::info!(topic = %topic, draining = kafka_consumer.is_draining(), "Subscribed to Kafka topic");

    // One statement-concurrency budget for the whole pod: every lane's
    // chunk and per-row statements draw from it, so lanes can overlap
    // writes without collectively oversubscribing the pool.
    let upsert_concurrency = config.upsert_concurrency.max(1);
    if upsert_concurrency > config.pg_max_connections as usize {
        tracing::warn!(
            upsert_concurrency,
            pg_max_connections = config.pg_max_connections,
            "UPSERT_CONCURRENCY exceeds PG_MAX_CONNECTIONS; excess statements \
             will queue on pool acquire instead of the semaphore"
        );
    }
    let upsert_permits = Arc::new(tokio::sync::Semaphore::new(upsert_concurrency));

    // Writer lanes: each lane gets its own channel, store, and task, and
    // commits offsets only for the partitions routed to it.
    let mut lane_txs = Vec::with_capacity(lanes);
    for writer_handle in writer_handles {
        let (flush_tx, flush_rx) = mpsc::channel(config.flush_channel_capacity);
        let store = PersonWriteStore::new(
            PgStore::new(pool.clone(), config.pg_target_table.clone()),
            personhog_writer::store::StoreConfig {
                chunk_size: config.upsert_batch_size,
                row_fallback_concurrency: config.row_fallback_concurrency,
            },
            Arc::clone(&upsert_permits),
        );
        let writer_task =
            WriterTask::new(Arc::clone(&kafka_consumer), store, flush_rx, writer_handle);
        tokio::spawn(async move {
            writer_task.run().await;
        });
        lane_txs.push(flush_tx);
    }

    // Consumer task
    let consumer_task = ConsumerTask::new(
        Arc::clone(&kafka_consumer),
        lane_txs,
        (config.buffer_capacity / lanes).max(1),
        config.flush_interval(),
        config.flush_buffer_size,
        consumer_handle,
    );

    tokio::spawn(async move {
        consumer_task.run().await;
    });

    // The subscription follows the config; the drain is read from the group's commits.
    {
        let consumer = Arc::clone(&kafka_consumer);
        let config = config.clone();
        let handle = changelog_handle;
        tokio::spawn(async move {
            let _guard = handle.process_scope();
            let cancel = CancellationToken::new();
            let follow = follow_changelog(
                store,
                loaded,
                follower,
                consumer,
                |target: FollowTarget| drained(&config, target),
                Reporter::for_group(&config.kafka_consumer_group),
                |_, _| counter!("personhog_writer_resubscriptions_total").increment(1),
                cancel.clone(),
            );
            tokio::select! {
                _ = handle.shutdown_signal() => cancel.cancel(),
                rebind = follow => {
                    // The pod's restart subscribes on the next generation's cluster.
                    if rebind.is_some() {
                        handle.request_shutdown();
                        handle.shutdown_signal().await;
                    }
                }
            }
        });
    }

    monitor_guard.wait().await?;
    Ok(())
}

fn changelog_kafka(base: &KafkaConfig, cluster: Option<&ClusterEndpoint>) -> KafkaConfig {
    match cluster {
        Some(cluster) => base.with_endpoint(&cluster.hosts, cluster.tls),
        None => base.clone(),
    }
}

async fn drained(config: &Config, target: FollowTarget) -> Result<bool, String> {
    let base = base_client_config(&changelog_kafka(&config.kafka, target.cluster.as_ref()));
    let topic = target.topic;
    let group = config.kafka_consumer_group.clone();
    tokio::task::spawn_blocking(move || {
        common_kafka::probe::group_drained(&base, &group, &topic, Duration::from_secs(10))
    })
    .await
    .map_err(|e| format!("drain probe join: {e}"))?
    .map_err(|e| format!("drain probe: {e}"))
}