:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::time::Duration;
use anyhow::{Context, Result};
use axum::{routing::get, Router};
use futures::future::ready;
use health::HealthRegistry;
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};
use opentelemetry::{KeyValue, Value};
use opentelemetry_otlp::WithExportConfig;
use opentelemetry_sdk::trace::{BatchConfig, RandomIdGenerator, Sampler, Tracer};
use opentelemetry_sdk::{runtime, Resource};
use serve_metrics::serve;
use tokio::task::JoinHandle;
use tracing::level_filters::LevelFilter;
use tracing::{error, info};
use tracing_opentelemetry::OpenTelemetryLayer;
use tracing_subscriber::fmt;
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, Layer};
/// Install a panic hook that logs panics through the structured tracing logger.
///
/// Without this, panics write to stderr using the default formatter, which
/// bypasses the JSON log layer and makes them invisible in Loki/Grafana.
fn install_tracing_panic_hook() {
let default_hook = std::panic::take_hook();
std::panic::set_hook(Box::new(move |panic_info| {
let thread = std::thread::current();
let thread_name = thread.name().unwrap_or("<unnamed>");
let location = panic_info
.location()
.map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
.unwrap_or_else(|| "unknown".to_string());
let payload = if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
s.to_string()
} else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
s.clone()
} else {
"unknown panic payload".to_string()
};
error!(
thread = %thread_name,
location = %location,
payload = %payload,
"Thread panicked"
);
// Still call the default hook so the process aborts/unwinds as expected
default_hook(panic_info);
}));
}
use kafka_deduplicator::{config::Config, service::KafkaDeduplicatorService};
common_alloc::used!();
fn init_tracer(sink_url: &str, sampling_rate: f64, service_name: &str) -> Tracer {
opentelemetry_otlp::new_pipeline()
.tracing()
.with_trace_config(
opentelemetry_sdk::trace::Config::default()
.with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
sampling_rate,
))))
.with_id_generator(RandomIdGenerator::default())
.with_resource(Resource::new(vec![KeyValue::new(
"service.name",
Value::from(service_name.to_string()),
)])),
)
.with_batch_config(BatchConfig::default())
.with_exporter(
opentelemetry_otlp::new_exporter()
.tonic()
.with_endpoint(sink_url)
.with_timeout(Duration::from_secs(3)),
)
.install_batch(runtime::Tokio)
.expect("Failed to initialize OpenTelemetry tracer")
}
pub async fn index() -> &'static str {
"kafka deduplicator service"
}
/// Setup metrics recorder with optimized histogram buckets for kafka-deduplicator
/// Using fewer buckets to reduce cardinality
fn setup_kafka_deduplicator_metrics() -> PrometheusHandle {
const BUCKETS: &[f64] = &[
0.001, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
];
// similarity scores are all in the range [0.0, 1.0] so we want
// granular bucket ranges for higher fidelity metrics
const SIMILARITY_BUCKETS: &[f64] = &[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0];
const CHECKPOINT_FILE_COUNT_BUCKETS: &[f64] = &[
1.0, 10.0, 50.0, 100.0, 200.0, 400.0, 600.0, 800.0, 1000.0, 1500.0,
];
const CHECKPOINT_SIZE_BYTES_BUCKETS: &[f64] = &[
1.0,
10.0,
100.0,
1024.0,
10.0 * 1024.0,
100.0 * 1024.0,
1024.0 * 1024.0,
10.0 * 1024.0 * 1024.0,
100.0 * 1024.0 * 1024.0,
1024.0 * 1024.0 * 1024.0,
10.0 * 1024.0 * 1024.0 * 1024.0,
100.0 * 1024.0 * 1024.0 * 1024.0,
];
// Buckets for counting unique items (UUIDs, timestamps)
const COUNT_BUCKETS: &[f64] = &[
1.0, 2.0, 5.0, 10.0, 50.0, 100.0, 500.0, 1000.0, 5000.0, 10000.0,
];
PrometheusBuilder::new()
.set_buckets(BUCKETS)
.unwrap()
.set_buckets_for_metric(
Matcher::Suffix("similarity_score".to_string()),
SIMILARITY_BUCKETS,
)
.unwrap()
.set_buckets_for_metric(
Matcher::Suffix("checkpoint_file_count".to_string()),
CHECKPOINT_FILE_COUNT_BUCKETS,
)
.unwrap()
.set_buckets_for_metric(
Matcher::Suffix("checkpoint_size_bytes".to_string()),
CHECKPOINT_SIZE_BYTES_BUCKETS,
)
.unwrap()
.set_buckets_for_metric(Matcher::Suffix("unique_uuids".to_string()), COUNT_BUCKETS)
.unwrap()
.set_buckets_for_metric(
Matcher::Suffix("unique_timestamps".to_string()),
COUNT_BUCKETS,
)
.unwrap()
.install_recorder()
.unwrap()
}
fn start_server(config: &Config, liveness: HealthRegistry) -> JoinHandle<()> {
let router = Router::new()
.route("/", get(index))
.route("/_readiness", get(index))
.route(
"/_liveness",
get(move || async move {
let status = liveness.get_status();
if !status.healthy {
let unhealthy_components: Vec<String> = status
.components
.iter()
.filter(|(_, component_status)| !component_status.is_healthy())
.map(|(name, component_status)| format!("{name}: {component_status:?}"))
.collect();
error!(
"Health check FAILED - unhealthy components: [{}]",
unhealthy_components.join(", ")
);
}
status
}),
);
// Don't install metrics unless asked to
// Installing a global recorder when capture is used as a library (during tests etc)
// does not work well.
let router = if config.export_prometheus {
let recorder_handle = setup_kafka_deduplicator_metrics();
router.route("/metrics", get(move || ready(recorder_handle.render())))
} else {
router
};
let bind = config.bind_address();
tokio::task::spawn(async move {
serve(router, &bind)
.await
.expect("failed to start serving metrics");
})
}
fn main() -> Result<()> {
// Load configuration first (sync) so we can use worker_threads to size the runtime
let config = Config::init_with_defaults()
.context("Failed to load configuration from environment variables. Please check your environment setup.")?;
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(config.worker_threads)
.enable_all()
.build()
.context("Failed to build tokio runtime")?;
runtime.block_on(async_main(config))
}
async fn async_main(config: Config) -> Result<()> {
// Initialize tracing with structured output similar to feature-flags
let log_layer = {
let base = fmt::layer()
.with_target(true)
.with_thread_ids(true)
.with_level(true);
if config.otel_log_level == tracing::Level::DEBUG {
// local dev: pretty-print output to console
base.with_span_events(
FmtSpan::NEW | FmtSpan::CLOSE | FmtSpan::ENTER | FmtSpan::EXIT | FmtSpan::ACTIVE,
)
.with_ansi(true)
.with_filter(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy()
.add_directive("pyroscope=warn".parse().unwrap()),
)
.boxed()
} else {
// production: use JSON format Loki/Grafana can extract useful filter tags from
base.json()
.flatten_event(true)
.with_span_list(true)
.with_current_span(true)
.with_filter(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy()
.add_directive("pyroscope=warn".parse().unwrap()),
)
.boxed()
}
};
// OpenTelemetry layer if configured
let otel_layer = if let Some(ref otel_url) = config.otel_url {
Some(
OpenTelemetryLayer::new(init_tracer(
otel_url,
config.otel_sampling_rate,
&config.otel_service_name,
))
.with_filter(LevelFilter::from_level(config.otel_log_level)),
)
} else {
None
};
tracing_subscriber::registry()
.with(log_layer)
.with(otel_layer)
.init();
// Install panic hook after tracing is initialized so panics are logged
// through the structured JSON logger, making them visible in Loki/Grafana.
install_tracing_panic_hook();
// Create a root span with pod hostname for all logs
// This adds "pod" field to every log line for Loki/Grafana filtering
let pod = config
.pod_hostname
.clone()
.unwrap_or_else(|| "unknown".to_string());
let _root_span = tracing::info_span!("service", pod = %pod).entered();
// Start continuous profiling if enabled (keep _agent alive for the duration of the program)
// NOTE: Must be after tracing is initialized so logs are visible
let _profiling_agent = match config.continuous_profiling.start_agent() {
Ok(agent) => agent,
Err(e) => {
tracing::warn!("Failed to start continuous profiling agent: {e:#}");
None
}
};
info!("Starting Kafka Deduplicator service");
info!("Configuration loaded: {:?}", config);
// Create health registry for liveness checks
let liveness = HealthRegistry::new("liveness");
// Start HTTP server with metrics endpoint
let server_handle = start_server(&config, liveness.clone());
info!("Started metrics server on {}", config.bind_address());
// Create and run the service
let service = KafkaDeduplicatorService::new(config, liveness)
.await
.with_context(|| "Failed to create Kafka Deduplicator service. Check your Kafka connection and RocksDB configuration.".to_string())?;
// Run the service (this blocks until shutdown)
service.run().await?;
// Clean up metrics server
server_handle.abort();
Ok(())
}