Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.8 KB
Raw Blame
use std::sync::Arc;
use std::time::Duration;

use axum::{routing::get, Router};
use common_database::{get_pool_with_config, PoolConfig};
use common_grpc::GrpcMetricsLayer;
use common_kafka::kafka_producer::create_kafka_producer;
use envconfig::Envconfig;
use health::HealthRegistry;
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
use tonic::transport::Server;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, Layer};
use usage_ingestion::config::Config;
use usage_ingestion::counters::{spawn_flush_task, CounterAccumulator};
use usage_ingestion::resolver::PostgresOrganizationResolver;
use usage_ingestion::service::UsageIngestionService;
use usage_ingestion_proto::usage_ingestion::v1::usage_ingestion_server::UsageIngestionServer;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    // Without this, the first TLS handshake to Valkey panics the task that made it, which
    // takes the counter flush loop down before it reports anything.
    rustls::crypto::aws_lc_rs::default_provider()
        .install_default()
        .expect("failed to install rustls CryptoProvider");

    let log_layer = {
        let base = tracing_subscriber::fmt::layer()
            .with_target(true)
            .with_thread_ids(true)
            .with_level(true);

        if std::env::var_os("DEBUG").is_some() {
            base.with_span_events(
                FmtSpan::NEW | FmtSpan::CLOSE | FmtSpan::ENTER | FmtSpan::EXIT | FmtSpan::ACTIVE,
            )
            .with_ansi(true)
            .boxed()
        } else {
            base.json()
                .flatten_event(true)
                .with_span_list(true)
                .with_current_span(true)
                .boxed()
        }
    };

    tracing_subscriber::registry()
        .with(log_layer)
        .with(
            EnvFilter::builder()
                .with_default_directive(LevelFilter::INFO.into())
                .from_env_lossy(),
        )
        .init();

    let config = Config::init_from_env()?;
    config.validate()?;

    let database = get_pool_with_config(
        &config.database_url,
        PoolConfig {
            max_connections: 10,
            pool_name: Some("usage-ingestion".to_string()),
            ..Default::default()
        },
    )?;
    let resolver = Arc::new(PostgresOrganizationResolver::new(database));

    let kafka_config = config.kafka_config();
    let health = Arc::new(HealthRegistry::new("usage-ingestion"));
    let producer_liveness = health
        .register("kafka_producer".to_string(), Duration::from_secs(30))
        .await;
    let producer = create_kafka_producer(&kafka_config, producer_liveness).await?;
    let grpc_max_connection_age = config.grpc_max_connection_age();
    let redis_counter_config = config.redis_counter_config();
    let counters = (!config.redis_url.is_empty()).then(|| Arc::new(CounterAccumulator::default()));
    let service = UsageIngestionService::new(
        producer,
        resolver,
        config.max_batch_size,
        config.topic.clone(),
        counters.as_ref().map(Arc::clone),
    );

    // Buckets only for the shared gRPC histogram, so it renders the same way personhog's does
    // and quantiles aggregate across pods. Left global, these millisecond bounds would also
    // apply to kafka_delivery_seconds, which is in seconds and would land in one bucket.
    const GRPC_DURATION_BUCKETS_MS: &[f64] = &[
        1.0, 5.0, 10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2000.0, 5000.0, 10000.0,
    ];
    let metrics_handle = PrometheusBuilder::new()
        .set_buckets_for_metric(
            Matcher::Full("grpc_server_request_duration_ms".to_string()),
            GRPC_DURATION_BUCKETS_MS,
        )?
        .install_recorder()?;
    if let Some(accumulator) = counters {
        spawn_flush_task(
            accumulator,
            config.redis_url,
            Duration::from_secs(config.redis_flush_interval_seconds),
            redis_counter_config,
        );
    }
    let metrics_address = config.metrics_address.clone();
    let health_for_routes = health.clone();
    tokio::spawn(async move {
        let router = Router::new()
            .route(
                "/_readiness",
                get(move || {
                    let health = health_for_routes.clone();
                    async move { health.get_status() }
                }),
            )
            .route("/_liveness", get(|| async { "ok" }))
            .route(
                "/metrics",
                get(move || std::future::ready(metrics_handle.render())),
            );
        let listener = tokio::net::TcpListener::bind(metrics_address)
            .await
            .expect("failed to bind usage-ingestion metrics listener");
        axum::serve(listener, router)
            .await
            .expect("usage-ingestion metrics server failed");
    });

    tracing::info!(address = %config.grpc_address, "Starting usage-ingestion gRPC service");
    // This listener is limited to trusted in-cluster callers. Add authenticated caller identity
    // before exposing it beyond that boundary because records affect tenant billing.
    // Producers pin one HTTP/2 connection for the life of the process, so a scale-up takes no
    // traffic until connections churn. A periodic GOAWAY makes them re-resolve the service.
    // ponytail: tonic 0.12 adds no jitter here. Move to client-side round-robin if the
    // synchronized reconnect shows up as a latency sawtooth.
    let mut builder = Server::builder();
    if let Some(age) = grpc_max_connection_age {
        builder = builder.max_connection_age(age);
    }
    builder
        .layer(GrpcMetricsLayer)
        .add_service(UsageIngestionServer::new(service))
        .serve(config.grpc_address.parse()?)
        .await?;
    Ok(())
}