Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

8.0 KB
Raw Blame
use std::time::Duration;

use envconfig::Envconfig;
use opentelemetry::{KeyValue, Value};
use opentelemetry_otlp::WithExportConfig;
use opentelemetry_sdk::trace::{BatchConfig, RandomIdGenerator, Sampler, Tracer};
use opentelemetry_sdk::{runtime, Resource};
use tracing::level_filters::LevelFilter;
use tracing_opentelemetry::OpenTelemetryLayer;
use tracing_subscriber::fmt;
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, Layer};

use feature_flags::config::{Config, ThreadCounts};
use feature_flags::handler::body_logger::BODY_LOG_TARGET;
use feature_flags::rayon_dispatcher::RayonDispatcher;
use feature_flags::server::{register_components, serve};

common_alloc::used!();

fn init_tracer(
    sink_url: &str,
    sampling_rate: f64,
    service_name: &str,
    export_timeout_secs: u64,
) -> Tracer {
    opentelemetry_otlp::new_pipeline()
        .tracing()
        .with_trace_config(
            opentelemetry_sdk::trace::Config::default()
                .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
                    sampling_rate,
                ))))
                .with_id_generator(RandomIdGenerator::default())
                .with_resource(Resource::new(vec![KeyValue::new(
                    "service.name",
                    Value::from(service_name.to_string()),
                )])),
        )
        .with_batch_config(BatchConfig::default())
        .with_exporter(
            opentelemetry_otlp::new_exporter()
                .tonic()
                .with_endpoint(sink_url)
                .with_timeout(Duration::from_secs(export_timeout_secs)),
        )
        .install_batch(runtime::Tokio)
        .expect("Failed to initialize OpenTelemetry tracer")
}

async fn async_main(mut config: Config, rayon_dispatcher: RayonDispatcher) {
    // Instantiate tracing outputs following Django's DEBUG-based approach:
    //   - stdout with a level configured by the RUST_LOG envvar
    //   - OpenTelemetry if enabled, for levels INFO and higher
    // Read DEBUG environment variable (same as Django)
    let debug: bool = *config.debug;

    let log_layer = {
        let base_layer = fmt::layer()
            .with_target(true)
            .with_thread_ids(true)
            .with_level(true);

        if debug {
            // Development: Pretty colored output (like Django's ConsoleRenderer(colors=DEBUG))
            base_layer
                .with_span_events(
                    FmtSpan::NEW
                        | FmtSpan::CLOSE
                        | FmtSpan::ENTER
                        | FmtSpan::EXIT
                        | FmtSpan::ACTIVE,
                )
                .with_ansi(true)
                .with_filter(
                    EnvFilter::builder()
                        .with_default_directive(LevelFilter::INFO.into())
                        .from_env_lossy()
                        .add_directive("pyroscope=warn".parse().unwrap()),
                )
                .boxed()
        } else {
            // Production: JSON format (like Django's JSONRenderer())
            base_layer
                .json()
                .with_span_list(false)
                .with_filter(
                    EnvFilter::builder()
                        .with_default_directive(LevelFilter::INFO.into())
                        .from_env_lossy()
                        .add_directive("pyroscope=warn".parse().unwrap()),
                )
                .boxed()
        }
    };

    let otel_layer = if let Some(ref otel_url) = config.otel_url {
        // Build the OTEL filter explicitly: keep the configured level for
        // every other target, but drop the body-log target. Body logs go to
        // stdout (and from there to Loki on a short retention) by design;
        // routing them through OTEL too would re-sink PII (`distinct_id`,
        // `person_properties`, `ip_address`) into a tier the body-log
        // retention budget was not sized for. The directive is parsed at
        // runtime from `BODY_LOG_TARGET` so the constant stays the single
        // source of truth.
        let otel_filter = EnvFilter::builder()
            .with_default_directive(LevelFilter::from_level(config.otel_log_level).into())
            .parse_lossy("")
            .add_directive(
                format!("{BODY_LOG_TARGET}=off")
                    .parse()
                    .expect("BODY_LOG_TARGET must form a valid tracing directive"),
            );
        Some(
            OpenTelemetryLayer::new(init_tracer(
                otel_url,
                config.otel_sampling_rate,
                &config.otel_service_name,
                config.otel_export_timeout_secs,
            ))
            .with_filter(otel_filter),
        )
    } else {
        None
    };

    tracing_subscriber::registry()
        .with(log_layer)
        .with(otel_layer)
        .init();

    // This call must follow tracing init. Before init, tracing has no subscriber and drops the
    // warnings.
    config.validate_and_fix_timeouts();

    // Start continuous profiling if enabled (keep _agent alive for the duration of the program)
    // NOTE: Must be after tracing is initialized so logs are visible
    let _profiling_agent = match config.continuous_profiling.start_agent() {
        Ok(agent) => agent,
        Err(e) => {
            tracing::warn!("Failed to start continuous profiling agent: {e}");
            None
        }
    };

    // Drain for 20 s after the preStop file appears, then shut down. Kubelet sends SIGTERM
    // only after the preStop sleep, so keep that sleep at 20 s: a shorter sleep can end the
    // drain early, and a longer one can outlive the process and fail the preStop hook.
    // The termination grace period must cover the 20 s drain plus the 45 s shutdown timeout.
    let mut manager = lifecycle::Manager::builder("feature-flags")
        .with_global_shutdown_timeout(Duration::from_secs(45))
        .with_prestop_action(lifecycle::PrestopAction::Drain {
            max: Duration::from_secs(20),
        })
        .build();

    let handles = register_components(&mut manager);
    let monitor_guard = manager.monitor_background();

    // Open the TCP port and start the server
    let listener = tokio::net::TcpListener::bind(config.address)
        .await
        .expect("could not bind port");
    serve(config, listener, rayon_dispatcher, handles, None).await;

    // Exit non-zero on dirty shutdown so `restart: on-failure` policies
    // (hobby docker-compose, default Docker) restart the container.
    // Clean shutdown exits 0.
    if let Err(e) = monitor_guard.wait().await {
        tracing::error!("Lifecycle monitor reported: {e}");
        std::process::exit(1);
    }
}

fn main() {
    let config = Config::init_from_env().expect("Invalid configuration:");

    let threads = ThreadCounts::new(config.thread_pool_cores);

    rayon::ThreadPoolBuilder::new()
        .num_threads(threads.rayon_threads)
        .build_global()
        .expect("failed to create rayon thread pool");

    let tokio_runtime = tokio::runtime::Builder::new_multi_thread()
        .worker_threads(threads.tokio_workers)
        .enable_all()
        .build()
        .expect("failed to create tokio thread pool");

    let max_concurrent_batch_evals = if config.max_concurrent_batch_evals == 0 {
        threads.default_max_concurrent_batch_evals()
    } else {
        config.max_concurrent_batch_evals
    };
    let semaphore_timeout = if config.rayon_semaphore_timeout_ms == 0 {
        None
    } else {
        Some(std::time::Duration::from_millis(
            config.rayon_semaphore_timeout_ms,
        ))
    };
    let rayon_dispatcher = RayonDispatcher::new(max_concurrent_batch_evals, semaphore_timeout);

    eprintln!(
        "Initialized thread pools: tokio_workers={}, rayon_threads={}, max_concurrent_batch_evals={}, semaphore_timeout_ms={}",
        threads.tokio_workers, threads.rayon_threads, max_concurrent_batch_evals, config.rayon_semaphore_timeout_ms,
    );

    tokio_runtime.block_on(async_main(config, rayon_dispatcher))
}