Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

22.0 KB
Raw Blame
use std::sync::Arc;
use std::time::Duration;

use assignment_coordination::store::{EtcdStore, StoreConfig};
use axum::{routing::get, Router};
use common_database::get_pool_with_config;
use common_metrics::{ETCD_OP_LATENCY_BUCKETS_MS, ETCD_PAYLOAD_SIZE_BUCKETS_BYTES};
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
use personhog_common::grpc::{
    tracked_tcp_incoming_until, until_incoming_ends, GrpcLoadShedLayer, GrpcMetricsLayer,
};
use personhog_common::{spawn_pool_monitor, MonitoredPool};
use personhog_proto::personhog::identity::v1::person_hog_identity_server::PersonHogIdentityServer;
use personhog_proto::personhog::lifecycle::v1::person_hog_lifecycle_server::PersonHogLifecycleServer;
use sqlx::postgres::PgPool;
use tonic::codec::CompressionEncoding;
use tonic::transport::Server;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::fmt;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::EnvFilter;

use personhog_common::client::RouterClient;
use personhog_common::query_tag;
use personhog_coordination::changelog::{
    keep_changelog_live, load_changelog_config, LiveChangelog, CHANGELOG_BOOT_POLL_INTERVAL,
};
use personhog_coordination::store::PersonhogStore;
use personhog_identity::config::Config;
use personhog_identity::leader::LifecycleLeader;
use personhog_identity::lifecycle::delete::DeleteDriver;
use personhog_identity::lifecycle::detach::DetachedDrives;
use personhog_identity::lifecycle::engine::Engine;
use personhog_identity::lifecycle::merge::{MergeDriver, MergeOpExecutor};
use personhog_identity::lifecycle::PersonHogLifecycleService;
use personhog_identity::pools::{IdentityPools, Lane};
use personhog_identity::service::merge::MergeEntrance;
use personhog_identity::service::PersonHogIdentityService;
use personhog_identity::storage::postgres::PostgresIdentityStorage;
use personhog_identity::warnings;
use tokio_util::sync::CancellationToken;

common_alloc::used!();

fn create_pools(config: &Config) -> IdentityPools {
    let fast = get_pool_with_config(&config.primary_database_url, config.fast_pool_config())
        .expect("Failed to create fast database pool");
    let heavy = get_pool_with_config(&config.primary_database_url, config.heavy_pool_config())
        .expect("Failed to create heavy database pool");
    tracing::info!("Created fast and heavy database pools");
    IdentityPools::new(fast, heavy)
}

/// Hold `min_connections` connections and run a query on `server_warmup`
/// of them. acquire() only establishes app → PgBouncer; in transaction
/// pooling mode PgBouncer doesn't open a server connection until a query
/// runs.
async fn warm_pool(pool: &PgPool, lane: Lane, min_connections: u32, server_warmup: u32) {
    if min_connections == 0 {
        return;
    }
    let warmup_count = min_connections as usize;
    let server_warmup_count = (server_warmup as usize).min(warmup_count);
    tracing::info!(
        pool = lane.label(),
        count = warmup_count,
        server_warmup = server_warmup_count,
        "Warming database connection pool before accepting traffic"
    );
    let pool_start = std::time::Instant::now();
    let mut conns = Vec::with_capacity(warmup_count);
    for _ in 0..warmup_count {
        match pool.acquire().await {
            Ok(conn) => conns.push(conn),
            Err(e) => {
                tracing::warn!(pool = lane.label(), error = %e, "Failed to warm connection");
                break;
            }
        }
    }
    let mut server_warmed = 0u32;
    for conn in conns.iter_mut().take(server_warmup_count) {
        match sqlx::query(&query_tag!("warm_pool", "SELECT 1"))
            .execute(&mut **conn)
            .await
        {
            Ok(_) => server_warmed += 1,
            Err(e) => {
                tracing::warn!(pool = lane.label(), error = %e, "Failed to warm server-side connection");
            }
        }
    }
    tracing::info!(
        pool = lane.label(),
        client_conns = conns.len(),
        server_conns = server_warmed,
        elapsed_ms = pool_start.elapsed().as_millis() as u64,
        "Pool warmup complete"
    );
}

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let config = Config::init_from_env().expect("Invalid configuration");

    let log_layer = fmt::layer()
        .with_target(true)
        .with_thread_ids(true)
        .with_level(true);

    tracing_subscriber::registry()
        .with(log_layer)
        .with(
            EnvFilter::builder()
                .with_default_directive(LevelFilter::INFO.into())
                .from_env_lossy(),
        )
        .init();

    let tables = config.tables();
    tables.validate().expect("Invalid identity table set");

    tracing::info!("Starting personhog-identity service");
    tracing::info!("gRPC address: {}", config.grpc_address);
    tracing::info!("Metrics port: {}", config.metrics_port);
    tracing::info!("Router URL: {}", config.router_url);
    tracing::info!(
        property_write_concurrency = config.property_write_concurrency,
        leader_call_concurrency = config.lifecycle_leader_call_concurrency,
        router_channels = config.router_channels,
        "Leader fan-out concurrency"
    );
    tracing::info!("Tables: {:?}", tables);
    let outbox_writes = config.outbox_writes();
    if config.lifecycle_outbox_enabled {
        tracing::info!(
            outbox = %tables.person_publish_outbox,
            "Lifecycle outbox enabled; a personhog-publisher on the same table set must drain it"
        );
        if tables.is_validation() {
            tracing::warn!(
                outbox = %tables.person_publish_outbox,
                "Lifecycle outbox is on for the validation table set; the publisher draining it must point at capture topics, never the production names"
            );
        }
    }

    // The delete saga groups its fence calls by the leaders' partition
    // count, which lives in etcd; a service that cannot read it would
    // batch wrong, so refuse to start instead.
    let etcd_store = EtcdStore::connect_with_member_failover(StoreConfig {
        endpoints: config.etcd_endpoint_list(),
        prefix: config.etcd_prefix.clone(),
    })
    .await
    .expect("Failed to connect to etcd");
    let store = Arc::new(PersonhogStore::new(etcd_store));
    // No signal handler is installed until the lifecycle monitor starts,
    // so SIGTERM ends the process during this wait.
    let Some(loaded) = load_changelog_config(
        &store,
        CHANGELOG_BOOT_POLL_INTERVAL,
        &CancellationToken::new(),
    )
    .await
    .expect("Failed to load the changelog config from etcd") else {
        return Ok(());
    };
    tracing::info!(
        num_partitions = loaded.partitions,
        epoch = loaded.epoch,
        "loaded changelog config"
    );
    let changelog = LiveChangelog::new(loaded);

    // Build lifecycle manager and register components
    let mut manager = Manager::builder("personhog-identity").build();

    let grpc_handle = manager.register(
        "grpc_server",
        ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(30)),
    );
    let metrics_handle = manager.register(
        "metrics_server",
        ComponentOptions::new().is_observability(true),
    );
    // Registered here because components must exist before monitoring
    // starts; the sweeper task itself is spawned later, once the engine
    // exists. Supervision matters: with one sweeper per fleet, a silently
    // dead loop means abandoned ops are never resumed.
    let sweeper_handle = config
        .lifecycle_sweeper_enabled
        .then(|| manager.register("lifecycle_sweeper", ComponentOptions::new()));
    let warnings_handle = config.ingestion_warnings_enabled.then(|| {
        manager.register(
            "ingestion_warnings",
            ComponentOptions::new()
                .with_liveness_deadline(Duration::from_secs(30))
                .is_advisory(true),
        )
    });
    let changelog_handle = manager.register("changelog_config", ComponentOptions::new());

    let readiness = manager.readiness_handler();
    let liveness = manager.liveness_handler();

    let monitor = manager.monitor_background();

    // Dense from 250ms to 5s, where merge tails sit.
    const BUCKETS: &[f64] = &[
        1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 750.0, 1000.0, 1500.0, 2000.0, 3000.0,
        5000.0, 10000.0, 30000.0,
    ];
    // Lifecycle ops span "settled in one drive" (tens of ms) to
    // "abandoned, parked, or leader-blocked and resumed by the sweeper"
    // (minutes to an hour); the default latency ladder tops out at 10s
    // and would collapse every resumed op into +Inf.
    const OP_DURATION_BUCKETS: &[f64] = &[
        10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0, 10000.0, 30000.0, 60000.0,
        300000.0, 1800000.0, 3600000.0,
    ];
    // Batch sizes, not latency; the request caps are 250.
    const PER_CALL_BUCKETS: &[f64] = &[1.0, 2.0, 3.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0];
    let recorder_handle = PrometheusBuilder::new()
        .add_global_label("service", "personhog-identity")
        .set_buckets(BUCKETS)
        .unwrap()
        .set_buckets_for_metric(
            Matcher::Full("personhog_lifecycle_op_duration_ms".into()),
            OP_DURATION_BUCKETS,
        )
        .unwrap()
        .set_buckets_for_metric(Matcher::Suffix("_per_call".into()), PER_CALL_BUCKETS)
        .unwrap()
        .set_buckets_for_metric(
            Matcher::Full("assignment_coordination_etcd_op_ms".into()),
            ETCD_OP_LATENCY_BUCKETS_MS,
        )
        .unwrap()
        .set_buckets_for_metric(
            Matcher::Full("assignment_coordination_etcd_payload_bytes".into()),
            ETCD_PAYLOAD_SIZE_BUCKETS_BYTES,
        )
        .unwrap()
        .install_recorder()
        .expect("Failed to install metrics recorder");
    {
        let store = Arc::clone(&store);
        let changelog = changelog.clone();
        tokio::spawn(async move {
            let _guard = changelog_handle.process_scope();
            let cancel = CancellationToken::new();
            tokio::select! {
                _ = changelog_handle.shutdown_signal() => cancel.cancel(),
                _ = keep_changelog_live(store, changelog, cancel.clone()) => {}
            }
        });
    }

    // Metrics/health HTTP server (observability handle — stays alive during standard drain)
    let metrics_port = config.metrics_port;
    tokio::spawn(async move {
        let _guard = metrics_handle.process_scope();

        let health_router = Router::new()
            .route(
                "/_readiness",
                get(move || {
                    let r = readiness.clone();
                    async move { r.check().await }
                }),
            )
            .route("/_liveness", get(move || async move { liveness.check() }));
        let metrics_router = health_router.route(
            "/metrics",
            get(move || std::future::ready(recorder_handle.render())),
        );

        let bind = format!("0.0.0.0:{metrics_port}");
        let listener = tokio::net::TcpListener::bind(&bind)
            .await
            .expect("Failed to bind metrics port");
        tracing::info!("Metrics server listening on {}", bind);
        axum::serve(listener, metrics_router)
            .with_graceful_shutdown(metrics_handle.shutdown_signal())
            .await
            .expect("Metrics server error");
    });

    let pools = create_pools(&config);
    if config.lifecycle_outbox_enabled {
        // With the outbox on, every create, attach and op completion names
        // the table; refuse to start without it instead of failing each
        // request once the pod is ready. Any other error is left to the
        // request path, as warm_pool leaves a connection failure.
        let probe = format!("SELECT 1 FROM {} LIMIT 0", tables.person_publish_outbox);
        if let Err(e) = sqlx::query(&query_tag!("startup_outbox_probe", probe))
            .execute(pools.fast())
            .await
        {
            let undefined_table =
                e.as_database_error().and_then(|db| db.code()).as_deref() == Some("42P01");
            if undefined_table {
                panic!(
                    "LIFECYCLE_OUTBOX_ENABLED is set but {} does not exist; apply the persons migration first",
                    tables.person_publish_outbox
                );
            }
            tracing::warn!(
                error = %e,
                table = %tables.person_publish_outbox,
                "Could not probe the publish outbox table at startup"
            );
        }
    }
    let storage = Arc::new(PostgresIdentityStorage::new(
        pools.clone(),
        config.tables(),
        outbox_writes,
    ));
    let merge_warnings = warnings::install(&config.ingestion_warnings_settings(), warnings_handle);
    let warnings_to_flush = merge_warnings.clone();

    // Pre-warm the DB connection pools before accepting traffic.
    // connect_lazy() starts with zero connections; without this, the first
    // burst of requests after K8s routes traffic all pay the cold-start cost.
    // Warming here is safe because the gRPC server hasn't bound its port yet.
    warm_pool(
        pools.fast(),
        Lane::Fast,
        config.min_pg_connections,
        config.warmup_server_connections,
    )
    .await;
    warm_pool(
        pools.heavy(),
        Lane::Heavy,
        config.heavy_min_pg_connections,
        config.warmup_server_connections,
    )
    .await;

    spawn_pool_monitor(
        vec![
            MonitoredPool {
                pool: pools.fast().clone(),
                label: Lane::Fast.label().to_string(),
                max_connections: config.max_pg_connections,
            },
            MonitoredPool {
                pool: pools.heavy().clone(),
                label: Lane::Heavy.label().to_string(),
                max_connections: config.heavy_max_pg_connections,
            },
        ],
        Duration::from_secs(config.pool_monitor_interval_secs),
    );

    let property_writer = Arc::new(
        RouterClient::with_channels_and_windows(
            &config.router_url,
            config.leader_request_timeout(),
            config.router_channels,
            config.router_http2_windows(),
        )
        .expect("Invalid router URL")
        .with_client_name("personhog-identity")
        .with_attempts(config.leader_request_attempts),
    );
    // Both sagas' leader surface, reached through the router like the
    // property writes.
    let lifecycle_leader: Arc<dyn LifecycleLeader> = property_writer.clone();
    let engine = Arc::new(Engine::new(
        pools.clone(),
        config.lifecycle_engine_config(),
        config.tables(),
    ));
    if let Some(sweeper_handle) = sweeper_handle {
        let sweeper_merge_driver = MergeDriver::new(
            property_writer.clone(),
            config.tables(),
            config.lifecycle_leader_call_concurrency,
            outbox_writes,
        )
        .with_warnings(merge_warnings.clone());
        let sweeper_delete_driver = DeleteDriver::new(
            lifecycle_leader.clone(),
            config.tables(),
            config.lifecycle_leader_call_concurrency,
            changelog.clone(),
            outbox_writes,
        );
        let sweeper_engine = engine.clone();
        let sweep_interval = config.lifecycle_sweep_interval();
        let retention = config.lifecycle_op_retention();
        tracing::info!(
            interval_secs = sweep_interval.as_secs(),
            retention_hours = retention.as_secs() / 3600,
            "Lifecycle sweeper enabled"
        );
        tokio::spawn(async move {
            // The scope guard tells the manager if this task dies (e.g. a
            // panic in a sweep pass), so the pod restarts instead of running
            // on with no resumer.
            let _guard = sweeper_handle.process_scope();
            let mut ticker = tokio::time::interval(sweep_interval);
            ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
            loop {
                tokio::select! {
                    _ = sweeper_handle.shutdown_recv() => break,
                    _ = ticker.tick() => {}
                }
                match sweeper_engine
                    .sweep(&[&sweeper_delete_driver, &sweeper_merge_driver])
                    .await
                {
                    Ok(resumed) if resumed > 0 => {
                        tracing::info!(resumed, "Lifecycle sweeper resumed abandoned ops")
                    }
                    Ok(_) => {}
                    Err(e) => tracing::warn!(error = %e, "Lifecycle sweep pass failed"),
                }
                if let Err(e) = sweeper_engine.gc(retention).await {
                    tracing::warn!(error = %e, "Lifecycle GC pass failed");
                }
            }
        });
    }

    let detached_drives = DetachedDrives::new(config.lifecycle_detached_drive_limit);
    // Separate proto service co-served on the same server so lifecycle
    // callers are insulated from any future split.
    let merge_entrance = MergeEntrance::new(
        storage.clone(),
        property_writer.clone(),
        property_writer.clone(),
        MergeOpExecutor::new(
            engine.clone(),
            MergeDriver::new(
                property_writer.clone(),
                config.tables(),
                config.lifecycle_leader_call_concurrency,
                outbox_writes,
            )
            .with_warnings(merge_warnings.clone()),
            detached_drives.clone(),
        ),
    )
    .with_warnings(merge_warnings);
    let lifecycle_service = PersonHogLifecycleService::new(
        engine,
        lifecycle_leader,
        config.tables(),
        config.lifecycle_leader_call_concurrency,
        changelog.clone(),
        outbox_writes,
    );
    let service = PersonHogIdentityService::new(
        storage,
        property_writer,
        config.request_limits(),
        merge_entrance,
        config.property_write_concurrency,
    );

    let grpc_addr = config.grpc_address;
    let keepalive_interval = config.grpc_keepalive_interval();
    let keepalive_timeout = config.grpc_keepalive_timeout();
    let http2_windows = config.grpc_http2_windows();
    let max_connection_age = config.grpc_max_connection_age();
    let max_send = config.grpc_max_send_message_size;
    let max_recv = config.grpc_max_recv_message_size;
    let max_concurrent_requests = config.max_concurrent_requests;
    let load_shed_wait = config.load_shed_wait();
    let load_shed_max_waiters = if config.load_shed_max_waiters == 0 {
        max_concurrent_requests
    } else {
        config.load_shed_max_waiters
    };
    if max_concurrent_requests > 0 {
        tracing::info!(
            limit = max_concurrent_requests,
            wait_ms = load_shed_wait.as_millis() as u64,
            max_waiters = load_shed_max_waiters,
            "gRPC load shedding enabled"
        );
    }
    tracing::info!("Starting gRPC server on {}", grpc_addr);

    tokio::spawn(async move {
        let _guard = grpc_handle.process_scope();
        let listener = match tokio::net::TcpListener::bind(grpc_addr).await {
            Ok(l) => l,
            Err(e) => {
                grpc_handle.signal_failure(format!("Failed to bind gRPC port: {e}"));
                return;
            }
        };
        let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());
        let mut server = http2_windows.apply_to_server(
            Server::builder()
                .http2_keepalive_interval(keepalive_interval)
                .http2_keepalive_timeout(keepalive_timeout),
        );
        if let Some(age) = max_connection_age {
            server = server.max_connection_age(age);
        }
        // accept_compressed only decodes gzip request frames from opted-in
        // clients (never send_compressed — see the tonic entry in Cargo.toml).
        // Note max_decoding_message_size bounds the wire (compressed) size, so
        // a compressed request can decode larger than the limit; tonic 0.12
        // does not cap the decompressed size (0.14 does — bounding this means
        // upgrading the workspace tonic). Until then this trusts internal
        // callers not to send pathological frames, same as the replica.
        if let Err(e) = server
            .layer(GrpcMetricsLayer::default().with_processing_time_header())
            // Identity pods share one fleet-wide load, so a shed pod means the
            // fleet is saturated. RESOURCE_EXHAUSTED stops the router from
            // retrying the shed on another saturated pod.
            .layer(
                GrpcLoadShedLayer::new(max_concurrent_requests)
                    .with_code(tonic::Code::ResourceExhausted)
                    .with_wait(load_shed_wait, load_shed_max_waiters),
            )
            .add_service(
                PersonHogIdentityServer::new(service)
                    .accept_compressed(CompressionEncoding::Gzip)
                    .max_encoding_message_size(max_send)
                    .max_decoding_message_size(max_recv),
            )
            .add_service(
                PersonHogLifecycleServer::new(lifecycle_service)
                    .accept_compressed(CompressionEncoding::Gzip)
                    .max_encoding_message_size(max_send)
                    .max_decoding_message_size(max_recv),
            )
            .serve_with_incoming_shutdown(incoming, until_incoming_ends())
            .await
        {
            grpc_handle.signal_failure(format!("gRPC server error: {e}"));
        }
        tracing::info!(
            in_flight = detached_drives.in_flight(),
            "waiting for detached merge drives before exit"
        );
        detached_drives.shutdown().await;
    });

    let waited = monitor.wait().await;

    // Flush before the result propagates: a failed component or a timed-out
    // drain is exactly the shutdown that leaves rows queued.
    if let Err(e) = tokio::task::spawn_blocking(move || warnings_to_flush.flush()).await {
        tracing::warn!(error = %e, "ingestion warnings flush task panicked");
    }

    waited?;
    Ok(())
}