:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result};
use axum::response::IntoResponse;
use axum::routing::get;
use axum::Router;
use common_kafka_consumer::AssignmentEpoch;
use envconfig::Envconfig;
use futures::future::ready;
use futures::StreamExt;
use lifecycle::{ComponentOptions, Manager};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
use tokio::net::TcpListener;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn};
use tracing_subscriber::fmt;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{EnvFilter, Layer};
use ingestion_consumer::aperture;
use ingestion_consumer::batcher::worker_pool::WorkerPoolSource;
use ingestion_consumer::batcher::{Batcher, BatcherObserver};
use ingestion_consumer::config::{Config, GRACEFUL_SHUTDOWN, LIVENESS_DEADLINE};
use ingestion_consumer::consumer::IngestionConsumer;
use ingestion_consumer::debug_recorder::{
BatcherLoad, DebugLoad, DebugRecorder, DebugState, GovernorLoad, RoutingDebug, WorkerStatus,
};
use ingestion_consumer::discovery::{
DiscoveryMode, EndpointSliceDiscovery, StaticDiscovery, WorkerDiscovery,
};
use ingestion_consumer::grpc_transport::{GrpcPort, GrpcTransport};
use ingestion_consumer::routing::RoutingStrategy;
use ingestion_consumer::worker_registry::{WorkerId, WorkerRegistry, WorkerRegistryConfig};
common_alloc::used!();
fn main() -> Result<()> {
// Install a process-wide rustls CryptoProvider before any TLS use. kube's
// HTTPS client (EndpointSlice discovery) uses rustls 0.23, which can't
// auto-pick a provider with both aws-lc-rs and ring compiled in — it panics.
// Matches personhog-router / cymbal / kafka-assigner.
rustls::crypto::ring::default_provider()
.install_default()
.expect("failed to install rustls ring CryptoProvider");
let config = Config::init_from_env()
.context("Failed to load configuration from environment variables")?;
config
.request_budget()
.map_err(|err| anyhow::anyhow!("invalid request budget: {err}"))?;
config
.partition_budget()
.map_err(|err| anyhow::anyhow!("invalid partition budget: {err}"))?;
config
.in_flight_limit()
.map_err(|err| anyhow::anyhow!("invalid in-flight limit: {err}"))?;
config
.revoke_drain_timeout()
.map_err(|err| anyhow::anyhow!("invalid revoke drain timeout: {err}"))?;
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.context("Failed to build tokio runtime")?;
let result = runtime.block_on(async_main(config));
// A blocking commit that outlived its timeout must not hold the exit.
runtime.shutdown_timeout(Duration::from_secs(5));
result
}
async fn async_main(config: Config) -> Result<()> {
let is_debug = std::env::var("RUST_LOG")
.map(|v| v.contains("debug"))
.unwrap_or(false);
let log_layer = if is_debug {
fmt::layer()
.with_target(true)
.with_level(true)
.with_ansi(true)
.with_filter(
EnvFilter::builder()
.with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
.from_env_lossy(),
)
.boxed()
} else {
fmt::layer()
.json()
.flatten_event(true)
.with_current_span(true)
.with_filter(
EnvFilter::builder()
.with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
.from_env_lossy(),
)
.boxed()
};
tracing_subscriber::registry().with(log_layer).init();
info!("Starting ingestion consumer");
// Keep the returned agents alive for the duration of the program.
// Fails gracefully - logs error but doesn't prevent service from starting.
let _profiling_agents = match config.continuous_profiling.start_agent() {
Ok(agents) => agents,
Err(e) => {
error!("Failed to start continuous profiling agent: {e:#}");
None
}
};
// Lifecycle manager handles signals, health, and shutdown coordination
let mut manager = Manager::builder("ingestion-consumer")
.with_global_shutdown_timeout(Duration::from_secs(90))
.build();
let consumer_handle = manager.register(
"consumer",
ComponentOptions::new()
.with_graceful_shutdown(GRACEFUL_SHUTDOWN)
.with_liveness_deadline(LIVENESS_DEADLINE)
.with_stall_threshold(3),
);
let metrics_handle =
manager.register("metrics", ComponentOptions::new().is_observability(true));
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
// Install Prometheus recorder with custom histogram buckets matching the
// Node.js ingestion consumer, so existing dashboards and alerts work during switchover.
let recorder_handle = if config.export_prometheus {
let mut builder = PrometheusBuilder::new()
.set_buckets_for_metric(
Matcher::Full("ingestion_lag_ms_histogram".into()),
&[
1_000.0, 2_000.0, 5_000.0, 10_000.0, 30_000.0, 60_000.0, 120_000.0, 300_000.0,
600_000.0, 900_000.0,
],
)
.expect("ingestion_lag_ms_histogram buckets")
.set_buckets_for_metric(
Matcher::Full("consumer_batch_size".into()),
&[
0.0, 50.0, 100.0, 250.0, 500.0, 750.0, 1_000.0, 1_500.0, 2_000.0, 3_000.0,
],
)
.expect("consumer_batch_size buckets")
.set_buckets_for_metric(
Matcher::Full("consumer_batch_size_kb".into()),
&[
0.0, 128.0, 512.0, 1_024.0, 5_120.0, 10_240.0, 20_480.0, 51_200.0, 102_400.0,
204_800.0,
],
)
.expect("consumer_batch_size_kb buckets")
.set_buckets_for_metric(
Matcher::Full("ingestion_consumer_request_queue_wait_seconds".into()),
&[
0.005, 0.025, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0,
],
)
.expect("request_queue_wait buckets")
.set_buckets_for_metric(
Matcher::Full("ingestion_consumer_request_events".into()),
&[
1.0, 10.0, 50.0, 100.0, 250.0, 500.0, 1_000.0, 2_500.0, 5_000.0,
],
)
.expect("request_events buckets")
.set_buckets_for_metric(
Matcher::Full("ingestion_consumer_request_bytes".into()),
&[
1_024.0,
16_384.0,
65_536.0,
262_144.0,
1_048_576.0,
4_194_304.0,
16_777_216.0,
],
)
.expect("request_bytes buckets");
// Global default labels (match the Node.js `initializePrometheusLabels`
// defaults): every metric carries ingestion_pipeline / ingestion_lane so
// dashboards, alerts, and the lag-based KEDA autoscaler select this
// consumer's series — notably `ingestion_lag_ms` for the scaling triggers.
if let Some(pipeline) = config.ingestion_pipeline.as_deref() {
builder = builder.add_global_label("ingestion_pipeline", pipeline);
}
if let Some(lane) = config.ingestion_lane.as_deref() {
builder = builder.add_global_label("ingestion_lane", lane);
}
Some(
builder
.install_recorder()
.expect("failed to install Prometheus recorder"),
)
} else {
None
};
let guard = manager.monitor_background();
// Debug event recorder: a bounded in-memory event buffer plus live
// broadcast, injected into every component below and served by the /debug
// API (consumed by the ingestion control plane UI). `None` (the default)
// records nothing. Fail closed: enabling without the dedicated secret
// mounts nothing rather than exposing the API unauthenticated.
let debug_recorder = if config.debug_api_enabled && !config.debug_api_secret.is_empty() {
Some(DebugRecorder::new(5_000, Duration::from_secs(900)))
} else {
if config.debug_api_enabled {
error!("DEBUG_API_ENABLED is set but DEBUG_API_SECRET is empty; debug API disabled");
}
None
};
// Build the worker health registry empty; the discovery provider below
// populates it (statically from config, or dynamically from EndpointSlices).
let registry_config = WorkerRegistryConfig::from(&config);
let mut registry = WorkerRegistry::new(&[], registry_config);
if let Some(recorder) = &debug_recorder {
registry.set_debug_recorder(Arc::clone(recorder));
}
let registry = Arc::new(registry);
// Probe tasks run until shutdown.
let probe_token = CancellationToken::new();
Arc::clone(®istry).start_probing(probe_token.clone());
// Peer awareness: track this consumer's own ready replicas via its
// Service's EndpointSlices, giving every pod a stable agreed index among
// its peers (the basis for deterministic aperture routing). Fails open:
// misconfiguration disables peer awareness rather than blocking startup.
let discovery_token = CancellationToken::new();
let peer_tracker: Option<Arc<k8s_awareness::PeerTracker>> =
if config.peer_service_name.is_empty() {
None
} else if let Some(self_ip) = config.pod_ip.as_deref().and_then(|s| s.parse().ok()) {
let client = kube::Client::try_default()
.await
.context("Failed to create Kubernetes client for peer discovery")?;
let tracker = k8s_awareness::PeerTracker::new(self_ip);
tracker.watch(
client,
config.worker_namespace.clone(),
config.peer_service_name.clone(),
discovery_token.clone(),
);
Some(tracker)
} else {
error!(
pod_ip = ?config.pod_ip,
"PEER_SERVICE_NAME is set but POD_IP is missing or invalid; peer awareness disabled"
);
None
};
let mut pool_source = WorkerPoolSource::new(Arc::clone(®istry), config.routing_strategy);
match &peer_tracker {
Some(tracker) => pool_source.set_aperture(Arc::clone(tracker), config.min_aperture),
None if config.routing_strategy == RoutingStrategy::Aperture => {
warn!(
"INGESTION_ROUTING_STRATEGY=aperture without peer awareness (PEER_SERVICE_NAME); \
routing falls back to p2c over the full pool"
);
}
None => {}
}
let grpc_port = if config.ingestion_worker_grpc_port_offset > 0 {
GrpcPort::OffsetFromHttp(config.ingestion_worker_grpc_port_offset)
} else {
GrpcPort::Fixed(config.ingestion_worker_grpc_port)
};
// The process-wide assignment epoch: bumped by the consumer's rebalance
// context, stamped by the transport on wire sub-batches and by the
// batcher on group completions. Created here so one counter serves all
// three.
let assignment_epoch = AssignmentEpoch::new();
let mut transport = GrpcTransport::new(
grpc_port,
Duration::from_millis(config.ingestion_worker_stream_ack_timeout_ms),
);
transport.set_max_body_bytes(config.transport_max_body_bytes);
transport.set_idle_close(Duration::from_millis(
config.ingestion_worker_stream_idle_close_ms,
));
transport.set_assignment_epoch(assignment_epoch);
let transport = Arc::new(transport);
// The batcher owns the dispatch orchestration; the consumer loop only
// submits polls to it and reads group completions back.
let state_machine = config
.batcher_state_machine()
.map_err(|err| anyhow::anyhow!("invalid batcher configuration: {err}"))?;
let (batcher, batcher_outputs) = Batcher::new(
state_machine,
pool_source.clone(),
Arc::clone(&transport),
debug_recorder.clone(),
);
// Select the worker discovery provider and start it (static applies the
// configured list immediately; endpointslice watches and keeps in sync).
let discovery: Box<dyn WorkerDiscovery> = match config.worker_discovery_mode {
DiscoveryMode::Static => Box::new(StaticDiscovery::new(config.worker_urls())),
DiscoveryMode::EndpointSlice => {
if config.worker_service_name.is_empty() {
anyhow::bail!("WORKER_SERVICE_NAME is required for endpointslice discovery");
}
let client = kube::Client::try_default()
.await
.context("Failed to create Kubernetes client for EndpointSlice discovery")?;
Box::new(EndpointSliceDiscovery::new(
client,
config.worker_namespace.clone(),
config.worker_service_name.clone(),
config.worker_port,
))
}
};
let _discovery_handle = discovery.start(Arc::clone(®istry), discovery_token.clone());
// Build and serve the health/metrics HTTP server BEFORE gating on worker
// discovery. Otherwise /_readiness and /_liveness stay unbound while we wait
// for the first workers, so kubelet sees "connection refused" instead of a
// proper 503 — and since /_liveness is meant to always answer 200, an unbound
// port lets the liveness probe kill the pod mid-startup.
let mut app = Router::new()
.route("/", get(|| async { "ingestion consumer" }))
.route(
"/_readiness",
get({
let r = readiness.clone();
move || {
let r = r.clone();
async move { r.check().await }
}
}),
)
.route(
"/_liveness",
get({
let l = liveness.clone();
move || {
let l = l.clone();
async move { l.check().into_response() }
}
}),
);
if let Some(handle) = recorder_handle {
app = app.route("/metrics", get(move || ready(handle.render())));
}
// Debug API: fast load snapshots, a full state snapshot, and the live SSE
// event feed, consumed by the ingestion control plane UI. Only mounted
// when DEBUG_API_ENABLED, and every request must present the dedicated
// DEBUG_API_SECRET (the health server binds broadly by default, so the
// routes must not be open to anything that can reach the port).
if let Some(recorder) = &debug_recorder {
let secret: Arc<str> = Arc::from(config.debug_api_secret.as_str());
let group_id = config.ingestion_consumer_group_id.clone();
{
let registry = Arc::clone(®istry);
let pool_source = pool_source.clone();
let observer = batcher.observer();
let peer_tracker = peer_tracker.clone();
let group_id = group_id.clone();
let secret = Arc::clone(&secret);
app = app.route(
"/debug/load",
get(move |headers: axum::http::HeaderMap| {
let registry = Arc::clone(®istry);
let pool_source = pool_source.clone();
let observer = observer.clone();
let peer_tracker = peer_tracker.clone();
let group_id = group_id.clone();
let secret = Arc::clone(&secret);
async move {
if !debug_authorized(&headers, &secret) {
return Err(axum::http::StatusCode::UNAUTHORIZED);
}
let load = build_debug_load(
&group_id,
®istry,
&pool_source,
&observer,
&peer_tracker,
)
.await;
Ok(axum::Json(load))
}
}),
);
}
{
let recorder = Arc::clone(recorder);
let registry = Arc::clone(®istry);
let pool_source = pool_source.clone();
let observer = batcher.observer();
let peer_tracker = peer_tracker.clone();
let group_id = group_id.clone();
let secret = Arc::clone(&secret);
app = app.route(
"/debug/state",
get(move |headers: axum::http::HeaderMap| {
let recorder = Arc::clone(&recorder);
let registry = Arc::clone(®istry);
let pool_source = pool_source.clone();
let observer = observer.clone();
let peer_tracker = peer_tracker.clone();
let group_id = group_id.clone();
let secret = Arc::clone(&secret);
async move {
if !debug_authorized(&headers, &secret) {
return Err(axum::http::StatusCode::UNAUTHORIZED);
}
let load = build_debug_load(
&group_id,
®istry,
&pool_source,
&observer,
&peer_tracker,
)
.await;
Ok(axum::Json(DebugState {
group_id: load.group_id,
workers: load.workers,
batcher: load.batcher,
peer_index: load.peer_index,
peer_count: load.peer_count,
routing: load.routing,
events: recorder.backlog(),
}))
}
}),
);
}
{
let recorder = Arc::clone(recorder);
let secret = Arc::clone(&secret);
// Cap concurrent SSE subscribers: each replays the retained backlog
// and then holds a connection open, so an unbounded count could
// pressure the shared health server.
let active_subscribers = Arc::new(std::sync::atomic::AtomicUsize::new(0));
app = app.route(
"/debug/events",
get(move |headers: axum::http::HeaderMap| {
let response = if !debug_authorized(&headers, &secret) {
Err(axum::http::StatusCode::UNAUTHORIZED)
} else if active_subscribers.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
>= MAX_SSE_SUBSCRIBERS
{
active_subscribers.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
Err(axum::http::StatusCode::TOO_MANY_REQUESTS)
} else {
// Decrements when the stream (and the closure owning the
// guard) is dropped, covering client disconnects.
let guard = SseSlotGuard(Arc::clone(&active_subscribers));
// Subscribe BEFORE snapshotting the backlog so an event
// recorded in between lands in the channel instead of
// being missed; the seq filter below drops the overlap
// (events present in both the backlog and the channel).
let rx = recorder.subscribe();
let backlog = recorder.backlog();
let last_backlog_seq = backlog.last().map(|e| e.seq);
let live = futures::stream::unfold(rx, |mut rx| async {
loop {
// A lagged subscriber skips dropped events
// rather than dying.
match rx.recv().await {
Ok(event) => return Some((event, rx)),
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
continue
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
return None
}
}
}
})
.filter(move |event| {
ready(last_backlog_seq.is_none_or(|seq| event.seq > seq))
});
let stream = futures::stream::iter(backlog)
.chain(live)
.map(move |event| {
let _held = &guard;
axum::response::sse::Event::default().json_data(&event)
});
Ok(axum::response::Sse::new(stream)
.keep_alive(axum::response::sse::KeepAlive::default()))
};
ready(response)
}),
);
}
}
let bind = config.bind_address();
info!(address = %bind, "Health/metrics server starting");
let listener = TcpListener::bind(&bind).await?;
tokio::spawn(async move {
if let Err(err) = axum::serve(listener, app)
.with_graceful_shutdown(metrics_handle.shutdown_signal())
.await
{
error!(error = %err, "Health/metrics server error");
}
metrics_handle.work_completed();
});
// For dynamic discovery, wait for the first workers before consuming so the
// first batch has somewhere to route. Readiness stays 503 until the consumer
// starts; the server above keeps the probes answering during the wait.
if config.worker_discovery_mode == DiscoveryMode::EndpointSlice {
info!("Waiting for the first workers from EndpointSlice discovery");
while registry.worker_count() == 0 {
tokio::select! {
_ = consumer_handle.shutdown_recv() => {
info!("Shutdown received while waiting for worker discovery");
break;
}
_ = tokio::time::sleep(Duration::from_millis(250)) => {}
}
}
}
// Reap drained workers: once a departed worker has finished its in-flight
// batches (or hit the drain timeout), remove it from the registry and prune
// its transport semaphore. No-op in static mode (workers never drain).
{
let registry = Arc::clone(®istry);
let transport = Arc::clone(&transport);
let observer = batcher.observer();
let token = probe_token.clone();
tokio::spawn(async move {
loop {
tokio::select! {
_ = token.cancelled() => break,
_ = tokio::time::sleep(Duration::from_secs(1)) => {}
}
// A worker that left the pool while idle has no in-flight to
// resolve, so `on_sub_batch_resolved` never completes its drain.
// Complete it here so it's reaped now rather than at the timeout.
for worker in registry.draining_workers() {
if !observer.has_in_flight(&worker) {
registry.complete_drain(&worker);
}
}
for worker in registry.reapable_workers() {
registry.remove_worker(&worker);
transport.remove_worker(&worker);
}
}
});
}
let consumer = IngestionConsumer::new(
&config,
batcher,
batcher_outputs,
transport,
consumer_handle,
debug_recorder,
)
.context("Failed to create Kafka consumer")?;
tokio::spawn(async move {
consumer.process().await;
// Cancel background tasks once the consumer loop exits.
probe_token.cancel();
discovery_token.cancel();
});
guard.wait().await?;
info!("Ingestion consumer stopped");
Ok(())
}
/// Maximum concurrent `/debug/events` SSE subscribers.
const MAX_SSE_SUBSCRIBERS: usize = 8;
/// Releases an SSE subscriber slot when the stream is dropped.
struct SseSlotGuard(Arc<std::sync::atomic::AtomicUsize>);
impl Drop for SseSlotGuard {
fn drop(&mut self) {
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
}
}
/// Whether the request carries the dedicated debug API secret. The secret is
/// never empty here (an empty secret disables the API entirely).
fn debug_authorized(headers: &axum::http::HeaderMap, secret: &str) -> bool {
headers
.get("x-debug-api-secret")
.and_then(|value| value.to_str().ok())
.is_some_and(|presented| constant_time_eq(presented, secret))
}
/// Compare without short-circuiting on the first mismatched byte, so response
/// timing doesn't leak how much of the secret matched.
fn constant_time_eq(a: &str, b: &str) -> bool {
let (a, b) = (a.as_bytes(), b.as_bytes());
if a.len() != b.len() {
return false;
}
a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0
}
/// Merge the registry's health snapshots with the batcher's load into the
/// `/debug/load` payload.
async fn build_debug_load(
group_id: &str,
registry: &WorkerRegistry,
pool_source: &WorkerPoolSource,
observer: &BatcherObserver,
peer_tracker: &Option<Arc<k8s_awareness::PeerTracker>>,
) -> DebugLoad {
let (peer_index, peer_count) = match peer_tracker {
Some(tracker) => {
let peers = tracker.snapshot();
(peers.self_index, Some(peers.peer_count()))
}
None => (None, None),
};
let batcher = observer.snapshot().await;
// One registry pass feeds the worker rows AND the routing ring/slice, so a
// worker joining or changing health mid-request can't make the payload
// show slice badges that contradict the worker table.
let snapshots = registry.health_snapshots();
let ring = aperture::sorted_ring(
snapshots
.iter()
.filter(|snap| !snap.draining)
.map(|snap| WorkerId::from(snap.url.as_str()))
.collect(),
);
let healthy: Vec<WorkerId> = snapshots
.iter()
.filter(|snap| snap.routable)
.map(|snap| WorkerId::from(snap.url.as_str()))
.collect();
let slice = pool_source.slice(&ring, &healthy);
let routing = RoutingDebug {
strategy: pool_source.strategy().as_str().to_string(),
min_aperture: pool_source.min_aperture(),
ring: ring.iter().map(|w| w.to_string()).collect(),
slice: slice.map(|s| s.iter().map(|c| c.worker.to_string()).collect()),
};
let workers = snapshots
.into_iter()
.map(|snap| {
let (in_flight_requests, in_flight_messages) = batcher
.per_worker
.iter()
.find(|load| load.worker.as_ref() == snap.url)
.map_or((0, 0), |load| (load.requests, load.messages));
WorkerStatus {
url: snap.url,
state: snap.state,
draining: snap.draining,
consecutive_probe_failures: snap.consecutive_probe_failures,
passive_error_rate: snap.passive_error_rate,
passive_samples: snap.passive_samples,
in_flight_requests,
in_flight_messages,
}
})
.collect();
DebugLoad {
group_id: group_id.to_string(),
workers,
batcher: BatcherLoad {
total_in_flight: batcher.in_flight_messages,
pending_messages: batcher.pending_messages,
in_flight_requests: batcher.in_flight_requests,
keys: batcher.keys,
queued_bytes: batcher.queued_bytes,
waiting_keys: batcher.waiting_keys,
claimed_keys: batcher.claimed_keys,
governor: GovernorLoad {
adaptation: batcher.governor.adaptation.as_str(),
permits: batcher.governor.permits,
ceiling: batcher.governor.ceiling,
free_permits: batcher.governor.free_permits,
starved_messages: batcher.governor.starved_messages,
rtt_seconds: batcher.governor.rtt.map(|rtt| rtt.as_secs_f64()),
},
},
peer_index,
peer_count,
routing,
}
}