Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.8 KB
Raw Blame
//! OTLP and Prometheus remote-write metrics capture. This binary serves only
//! the metrics routes of `capture-logs`, so metrics traffic can scale and
//! deploy on its own. All handlers, config, and the Kafka sink come from the
//! `capture-logs` library.

use std::sync::Arc;
use std::time::Duration;

use axum::{extract::DefaultBodyLimit, http::Method, routing::get, routing::post, Router};
use capture::metrics_middleware::track_metrics;
use capture_apm_metrics::config::Config;
use capture_apm_metrics::k8s_resource_attributes::K8sResourceAttributes;
use capture_apm_metrics::management;
use capture_apm_metrics::prometheus;
use capture_apm_metrics::series_label_gate::{spawn_redis_writer, CacheLimits, SeriesLabelGate};
use capture_apm_metrics::service::{export_metrics_http, MetricsService};
use capture_logs::authorizer::Authorizer;
use capture_logs::kafka::KafkaSink;
use capture_logs::middleware::translate_compression_query_param;
use capture_logs::service::options_handler;
use common_metrics::setup_metrics_routes;
use common_redis::{Client, CompressionConfig, RedisClient, RedisValueFormat};
use std::future::ready;
use std::net::SocketAddr;

use health::HealthRegistry;
use tokio::signal;
use tower_http::cors::{AllowHeaders, AllowOrigin, CorsLayer};
use tower_http::decompression::RequestDecompressionLayer;
use tracing::level_filters::LevelFilter;
use tracing::{error, info, warn};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};

use limiters::token_dropper::TokenDropper;

common_alloc::used!();

async fn shutdown() {
    let mut term = signal::unix::signal(signal::unix::SignalKind::terminate())
        .expect("failed to register SIGTERM handler");

    let mut interrupt = signal::unix::signal(signal::unix::SignalKind::interrupt())
        .expect("failed to register SIGINT handler");

    tokio::select! {
        _ = term.recv() => {},
        _ = interrupt.recv() => {},
    };

    tracing::info!("Shutting down gracefully...");
}

fn setup_tracing() {
    let log_layer = tracing_subscriber::fmt::layer()
        .json()
        .with_span_list(false)
        .with_filter(
            EnvFilter::builder()
                .with_default_directive(LevelFilter::INFO.into())
                .from_env_lossy()
                .add_directive("pyroscope=warn".parse().unwrap()),
        );
    tracing_subscriber::registry().with(log_layer).init();
}

pub async fn index() -> &'static str {
    "metric hog hogs metrics

.|||||||||.
|||||||||||||  gimme ur metrics 📈
|||||||||||' .\\
`||||||||||_,__o
"
}

/// Build the series label gate and, when Redis is configured, its background
/// tasks. The Redis client is optional on purpose: without it the gate still
/// dedupes labels within this pod.
async fn start_series_label_gate(config: &Config) -> Arc<SeriesLabelGate> {
    let window = Duration::from_secs(config.metrics_series_label_interval_secs);
    let enabled = config.metrics_series_label_gate_enabled;
    let limits = CacheLimits {
        max_entries: config.metrics_series_cache_max_entries,
        max_entries_per_token: config.metrics_series_cache_max_entries_per_token,
    };
    info!(
        "Series label gate {} (window {}s)",
        if enabled {
            "enabled"
        } else {
            "in dry-run mode"
        },
        window.as_secs()
    );

    let Some(redis_url) = config.redis_url.clone() else {
        info!("REDIS_URL unset, series label gate runs with the local cache only");
        let gate = SeriesLabelGate::local_only(window, enabled, limits);
        gate.spawn_pruner();
        return gate;
    };

    let redis_timeout = Duration::from_millis(config.metrics_series_redis_timeout_ms);
    let seed_budget = Duration::from_millis(config.metrics_series_redis_seed_timeout_ms);
    let pull_timeout = Duration::from_millis(config.metrics_series_redis_pull_timeout_ms);
    let pull_interval = Duration::from_secs(config.metrics_series_redis_pull_interval_secs);
    let client: Arc<dyn Client> = match RedisClient::with_config(
        redis_url.clone(),
        CompressionConfig::disabled(),
        RedisValueFormat::Utf8,
        Some(pull_timeout),
        Some(pull_timeout),
    )
    .await
    {
        Ok(client) => Arc::new(client),
        Err(e) => {
            error!(
                "Could not connect to Redis, series label gate runs with the local cache only: {e}"
            );
            let gate = SeriesLabelGate::local_only(window, enabled, limits);
            gate.spawn_pruner();
            return gate;
        }
    };

    let writer_client: Arc<dyn Client> = match RedisClient::with_config(
        redis_url,
        CompressionConfig::disabled(),
        RedisValueFormat::Utf8,
        Some(redis_timeout),
        Some(redis_timeout),
    )
    .await
    {
        Ok(client) => Arc::new(client),
        Err(e) => {
            warn!("Could not open a second Redis connection, series label writer shares the pull connection: {e}");
            Arc::clone(&client)
        }
    };

    let (gate, rx) = SeriesLabelGate::new(window, enabled, limits);
    gate.seed_from_redis(client.as_ref(), seed_budget).await;
    gate.spawn_redis_puller(client, pull_interval, pull_timeout);
    gate.spawn_pruner();
    spawn_redis_writer(writer_client, rx, redis_timeout, window);
    gate
}

fn main() {
    // Without this, the first TLS handshake to Valkey panics the task that made it.
    rustls::crypto::aws_lc_rs::default_provider()
        .install_default()
        .expect("failed to install rustls CryptoProvider");

    setup_tracing();
    info!("Starting up...");

    let config = Config::init_with_defaults().unwrap();

    let management_runtime = management::build_runtime();

    // The registry spawns its status task on the current runtime. The task
    // must live on the management runtime, so the probe result stays current
    // when the data plane is busy.
    let health_registry = {
        let _guard = management_runtime.enter();
        HealthRegistry::new("liveness")
    };

    // Bind here so a port conflict still stops the process at startup.
    let management_bind = format!(
        "{}:{}",
        config.base.management_host, config.base.management_port
    );
    let management_listener = management_runtime
        .block_on(tokio::net::TcpListener::bind(&management_bind))
        .expect("could not bind management port");

    let data_runtime = tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .build()
        .expect("failed to build data-plane runtime");

    data_runtime.block_on(run(
        config,
        health_registry,
        management_runtime.handle().clone(),
        management_listener,
        management_bind,
    ));
}

async fn run(
    config: Config,
    health_registry: HealthRegistry,
    management_runtime: tokio::runtime::Handle,
    management_listener: tokio::net::TcpListener,
    management_bind: String,
) {
    // Start continuous profiling if enabled (keep _agent alive for the duration of the program)
    let _profiling_agent = match config.base.continuous_profiling.start_agent() {
        Ok(agent) => agent,
        Err(e) => {
            tracing::warn!("Failed to start continuous profiling agent: {e}");
            None
        }
    };

    // The shared sink owns one producer per signal. This binary only sends
    // metrics, but the sink constructor is unchanged, so the other producers
    // still connect and report liveness.
    let logs_sink_liveness = health_registry
        .register("rdkafka".to_string(), Duration::from_secs(30))
        .await;
    let traces_sink_liveness = health_registry
        .register("rdkafka_traces".to_string(), Duration::from_secs(30))
        .await;
    let metrics_sink_liveness = health_registry
        .register("rdkafka_metrics".to_string(), Duration::from_secs(30))
        .await;

    let kafka_sink = KafkaSink::new(
        config.base.kafka.clone(),
        logs_sink_liveness,
        traces_sink_liveness,
        metrics_sink_liveness,
    )
    .await
    .expect("failed to start Kafka sink");

    let management_router = Router::new()
        .route("/", get(index))
        .route("/_readiness", get(index))
        .route(
            "/_liveness",
            get(move || ready(health_registry.get_status())),
        );
    let management_router = setup_metrics_routes(management_router);

    let series_label_gate = start_series_label_gate(&config).await;
    let token_dropper =
        TokenDropper::new(&config.base.drop_events_by_token.clone().unwrap_or_default());
    let authorizer = Authorizer::new(Arc::new(token_dropper));
    let metrics_service = MetricsService::new(
        kafka_sink,
        authorizer,
        config.base.max_request_body_size_bytes,
        config.remote_write_max_expanded_bytes,
        series_label_gate,
        Arc::new(K8sResourceAttributes::from_token_list(
            &config.remote_write_k8s_resource_attribute_tokens,
        )),
    );
    let http_bind = format!("{}:{}", config.base.host, config.base.port);
    info!("Listening on {}", http_bind);
    let http_listener = tokio::net::TcpListener::bind(http_bind)
        .await
        .expect("could not bind http port");

    // Very permissive CORS policy
    let cors = CorsLayer::new()
        .allow_methods([Method::GET, Method::POST, Method::OPTIONS])
        .allow_headers(AllowHeaders::mirror_request())
        .allow_credentials(true)
        .allow_origin(AllowOrigin::mirror_request());

    let http_router = Router::new()
        .route(
            "/v1/metrics",
            post(export_metrics_http).options(options_handler),
        )
        .route(
            "/i/v1/metrics",
            post(export_metrics_http).options(options_handler),
        )
        .with_state(metrics_service.clone())
        .layer(DefaultBodyLimit::max(
            config.base.max_request_body_size_bytes,
        ))
        .layer(axum::middleware::from_fn(track_metrics))
        .layer(RequestDecompressionLayer::new())
        .layer(axum::middleware::from_fn(translate_compression_query_param));

    // Prometheus remote-write sends `Content-Encoding: snappy` and vmagent
    // sends `Content-Encoding: zstd`, which RequestDecompressionLayer rejects
    // with 415 before the handler runs. This route is deliberately kept off
    // that layer (and the gzip query-param shim); the handler decodes the body
    // itself.
    let prometheus_router = Router::new()
        .route(
            "/i/v1/prometheus/write",
            post(prometheus::export_prometheus_remote_write_http).options(options_handler),
        )
        .route(
            "/i/v1/prometheus/write/:token",
            post(prometheus::export_prometheus_remote_write_http).options(options_handler),
        )
        .with_state(metrics_service)
        .layer(DefaultBodyLimit::max(
            config.base.max_request_body_size_bytes,
        ))
        .layer(axum::middleware::from_fn(track_metrics));

    let http_router = http_router.merge(prometheus_router).layer(cors);

    let http_server = tokio::spawn(async move {
        if let Err(e) = axum::serve(
            http_listener,
            http_router.into_make_service_with_connect_info::<SocketAddr>(),
        )
        .with_graceful_shutdown(shutdown())
        .await
        {
            error!("HTTP server failed: {}", e);
        }
    });

    info!("Healthcheck and metrics listening on {}", management_bind);

    // Serve the probes only once the data plane listens. `/_readiness` answers
    // 200 unconditionally, so an earlier start would put the pod in service
    // while its ingestion port still refuses connections.
    let mgmt_server =
        management::serve(&management_runtime, management_listener, management_router);

    // Wait for any server to finish
    tokio::select! {
        _ = http_server => {
            error!("HTTP server stopped");
        }
        _ = mgmt_server => {
            error!("Management server stopped");
        }
    }
}