Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

10.8 KB
Raw Blame
use std::time::Duration;

use anyhow::{Context, Result};
use axum::{routing::get, Router};
use futures::future::ready;
use health::HealthRegistry;
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};
use opentelemetry::{KeyValue, Value};
use opentelemetry_otlp::WithExportConfig;
use opentelemetry_sdk::trace::{BatchConfig, RandomIdGenerator, Sampler, Tracer};
use opentelemetry_sdk::{runtime, Resource};
use serve_metrics::serve;
use tokio::task::JoinHandle;
use tracing::level_filters::LevelFilter;
use tracing::{error, info};
use tracing_opentelemetry::OpenTelemetryLayer;
use tracing_subscriber::fmt;
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, Layer};

/// Install a panic hook that logs panics through the structured tracing logger.
///
/// Without this, panics write to stderr using the default formatter, which
/// bypasses the JSON log layer and makes them invisible in Loki/Grafana.
fn install_tracing_panic_hook() {
    let default_hook = std::panic::take_hook();
    std::panic::set_hook(Box::new(move |panic_info| {
        let thread = std::thread::current();
        let thread_name = thread.name().unwrap_or("<unnamed>");
        let location = panic_info
            .location()
            .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
            .unwrap_or_else(|| "unknown".to_string());
        let payload = if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
            s.to_string()
        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
            s.clone()
        } else {
            "unknown panic payload".to_string()
        };

        error!(
            thread = %thread_name,
            location = %location,
            payload = %payload,
            "Thread panicked"
        );

        // Still call the default hook so the process aborts/unwinds as expected
        default_hook(panic_info);
    }));
}

use kafka_deduplicator::{config::Config, service::KafkaDeduplicatorService};

common_alloc::used!();

fn init_tracer(sink_url: &str, sampling_rate: f64, service_name: &str) -> Tracer {
    opentelemetry_otlp::new_pipeline()
        .tracing()
        .with_trace_config(
            opentelemetry_sdk::trace::Config::default()
                .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
                    sampling_rate,
                ))))
                .with_id_generator(RandomIdGenerator::default())
                .with_resource(Resource::new(vec![KeyValue::new(
                    "service.name",
                    Value::from(service_name.to_string()),
                )])),
        )
        .with_batch_config(BatchConfig::default())
        .with_exporter(
            opentelemetry_otlp::new_exporter()
                .tonic()
                .with_endpoint(sink_url)
                .with_timeout(Duration::from_secs(3)),
        )
        .install_batch(runtime::Tokio)
        .expect("Failed to initialize OpenTelemetry tracer")
}

pub async fn index() -> &'static str {
    "kafka deduplicator service"
}

/// Setup metrics recorder with optimized histogram buckets for kafka-deduplicator
/// Using fewer buckets to reduce cardinality
fn setup_kafka_deduplicator_metrics() -> PrometheusHandle {
    const BUCKETS: &[f64] = &[
        0.001, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
    ];
    // similarity scores are all in the range [0.0, 1.0] so we want
    // granular bucket ranges for higher fidelity metrics
    const SIMILARITY_BUCKETS: &[f64] = &[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0];

    const CHECKPOINT_FILE_COUNT_BUCKETS: &[f64] = &[
        1.0, 10.0, 50.0, 100.0, 200.0, 400.0, 600.0, 800.0, 1000.0, 1500.0,
    ];
    const CHECKPOINT_SIZE_BYTES_BUCKETS: &[f64] = &[
        1.0,
        10.0,
        100.0,
        1024.0,
        10.0 * 1024.0,
        100.0 * 1024.0,
        1024.0 * 1024.0,
        10.0 * 1024.0 * 1024.0,
        100.0 * 1024.0 * 1024.0,
        1024.0 * 1024.0 * 1024.0,
        10.0 * 1024.0 * 1024.0 * 1024.0,
        100.0 * 1024.0 * 1024.0 * 1024.0,
    ];

    // Buckets for counting unique items (UUIDs, timestamps)
    const COUNT_BUCKETS: &[f64] = &[
        1.0, 2.0, 5.0, 10.0, 50.0, 100.0, 500.0, 1000.0, 5000.0, 10000.0,
    ];

    PrometheusBuilder::new()
        .set_buckets(BUCKETS)
        .unwrap()
        .set_buckets_for_metric(
            Matcher::Suffix("similarity_score".to_string()),
            SIMILARITY_BUCKETS,
        )
        .unwrap()
        .set_buckets_for_metric(
            Matcher::Suffix("checkpoint_file_count".to_string()),
            CHECKPOINT_FILE_COUNT_BUCKETS,
        )
        .unwrap()
        .set_buckets_for_metric(
            Matcher::Suffix("checkpoint_size_bytes".to_string()),
            CHECKPOINT_SIZE_BYTES_BUCKETS,
        )
        .unwrap()
        .set_buckets_for_metric(Matcher::Suffix("unique_uuids".to_string()), COUNT_BUCKETS)
        .unwrap()
        .set_buckets_for_metric(
            Matcher::Suffix("unique_timestamps".to_string()),
            COUNT_BUCKETS,
        )
        .unwrap()
        .install_recorder()
        .unwrap()
}

fn start_server(config: &Config, liveness: HealthRegistry) -> JoinHandle<()> {
    let router = Router::new()
        .route("/", get(index))
        .route("/_readiness", get(index))
        .route(
            "/_liveness",
            get(move || async move {
                let status = liveness.get_status();
                if !status.healthy {
                    let unhealthy_components: Vec<String> = status
                        .components
                        .iter()
                        .filter(|(_, component_status)| !component_status.is_healthy())
                        .map(|(name, component_status)| format!("{name}: {component_status:?}"))
                        .collect();
                    error!(
                        "Health check FAILED - unhealthy components: [{}]",
                        unhealthy_components.join(", ")
                    );
                }
                status
            }),
        );

    // Don't install metrics unless asked to
    // Installing a global recorder when capture is used as a library (during tests etc)
    // does not work well.
    let router = if config.export_prometheus {
        let recorder_handle = setup_kafka_deduplicator_metrics();
        router.route("/metrics", get(move || ready(recorder_handle.render())))
    } else {
        router
    };

    let bind = config.bind_address();

    tokio::task::spawn(async move {
        serve(router, &bind)
            .await
            .expect("failed to start serving metrics");
    })
}

fn main() -> Result<()> {
    // Load configuration first (sync) so we can use worker_threads to size the runtime
    let config = Config::init_with_defaults()
        .context("Failed to load configuration from environment variables. Please check your environment setup.")?;

    let runtime = tokio::runtime::Builder::new_multi_thread()
        .worker_threads(config.worker_threads)
        .enable_all()
        .build()
        .context("Failed to build tokio runtime")?;

    runtime.block_on(async_main(config))
}

async fn async_main(config: Config) -> Result<()> {
    // Initialize tracing with structured output similar to feature-flags
    let log_layer = {
        let base = fmt::layer()
            .with_target(true)
            .with_thread_ids(true)
            .with_level(true);

        if config.otel_log_level == tracing::Level::DEBUG {
            // local dev: pretty-print output to console
            base.with_span_events(
                FmtSpan::NEW | FmtSpan::CLOSE | FmtSpan::ENTER | FmtSpan::EXIT | FmtSpan::ACTIVE,
            )
            .with_ansi(true)
            .with_filter(
                EnvFilter::builder()
                    .with_default_directive(LevelFilter::INFO.into())
                    .from_env_lossy()
                    .add_directive("pyroscope=warn".parse().unwrap()),
            )
            .boxed()
        } else {
            // production: use JSON format Loki/Grafana can extract useful filter tags from
            base.json()
                .flatten_event(true)
                .with_span_list(true)
                .with_current_span(true)
                .with_filter(
                    EnvFilter::builder()
                        .with_default_directive(LevelFilter::INFO.into())
                        .from_env_lossy()
                        .add_directive("pyroscope=warn".parse().unwrap()),
                )
                .boxed()
        }
    };

    // OpenTelemetry layer if configured
    let otel_layer = if let Some(ref otel_url) = config.otel_url {
        Some(
            OpenTelemetryLayer::new(init_tracer(
                otel_url,
                config.otel_sampling_rate,
                &config.otel_service_name,
            ))
            .with_filter(LevelFilter::from_level(config.otel_log_level)),
        )
    } else {
        None
    };

    tracing_subscriber::registry()
        .with(log_layer)
        .with(otel_layer)
        .init();

    // Install panic hook after tracing is initialized so panics are logged
    // through the structured JSON logger, making them visible in Loki/Grafana.
    install_tracing_panic_hook();

    // Create a root span with pod hostname for all logs
    // This adds "pod" field to every log line for Loki/Grafana filtering
    let pod = config
        .pod_hostname
        .clone()
        .unwrap_or_else(|| "unknown".to_string());
    let _root_span = tracing::info_span!("service", pod = %pod).entered();

    // Start continuous profiling if enabled (keep _agent alive for the duration of the program)
    // NOTE: Must be after tracing is initialized so logs are visible
    let _profiling_agent = match config.continuous_profiling.start_agent() {
        Ok(agent) => agent,
        Err(e) => {
            tracing::warn!("Failed to start continuous profiling agent: {e:#}");
            None
        }
    };

    info!("Starting Kafka Deduplicator service");

    info!("Configuration loaded: {:?}", config);

    // Create health registry for liveness checks
    let liveness = HealthRegistry::new("liveness");

    // Start HTTP server with metrics endpoint
    let server_handle = start_server(&config, liveness.clone());
    info!("Started metrics server on {}", config.bind_address());

    // Create and run the service
    let service = KafkaDeduplicatorService::new(config, liveness)
        .await
        .with_context(|| "Failed to create Kafka Deduplicator service. Check your Kafka connection and RocksDB configuration.".to_string())?;

    // Run the service (this blocks until shutdown)
    service.run().await?;

    // Clean up metrics server
    server_handle.abort();

    Ok(())
}