:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use assignment_coordination::store::{EtcdStore, StoreConfig};
use axum::{routing::get, Router};
use common_kafka::kafka_producer::{create_kafka_producer, spawn_bounded_flush_on_shutdown};
use common_metrics::{setup_metrics_routes_with_overrides, Matcher};
use dashmap::DashMap;
use envconfig::Envconfig;
use k8s_awareness::{K8sAwareness, PodInfo};
use kube::Client;
use lifecycle::{ComponentOptions, Manager};
use personhog_common::async_gzip::{AsyncGzipConfig, AsyncGzipLayer};
use personhog_common::grpc::{
tracked_tcp_incoming_until, until_incoming_ends, GrpcLoadShedLayer, GrpcMetricsLayer,
};
use personhog_common::metrics::{WARM_LATENCY_BUCKETS_MS, WRITE_PATH_LATENCY_BUCKETS_MS};
use personhog_coordination::authority::AuthorityClock;
use personhog_coordination::changelog::{
keep_changelog_live, load_changelog_config, LiveChangelog, CHANGELOG_BOOT_POLL_INTERVAL,
};
use personhog_coordination::pod::{PodConfig, PodHandle};
use personhog_coordination::store::PersonhogStore;
use personhog_proto::personhog::leader::v1::person_hog_leader_server::PersonHogLeaderServer;
use tokio::sync::Notify;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use tonic::codec::CompressionEncoding;
use tonic::transport::Server;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::fmt;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::EnvFilter;
use metrics::{counter, gauge};
use personhog_leader::cache::{DirtyIndex, PartitionedCache, PersonCacheKey};
use personhog_leader::config::Config;
use personhog_leader::coordination::{ChangelogCaptures, LeaderHandoffHandler};
use personhog_leader::fencing::{
preregister_fencing_metrics, FencedChangelogProducers, FencedProducerConfig,
};
use personhog_leader::inflight::InflightTracker;
use personhog_leader::migration::SealDuty;
use personhog_leader::pg::{validate_table_name, LifecycleTables, PgFallback};
use personhog_leader::recovery::{ChangelogRecovery, RecoveryConfig};
use personhog_leader::service::{sweep_idle_locks, PersonHogLeaderService, PropertySizeLimits};
use personhog_leader::settle::{prune_and_settle_tick, retired_topic};
use personhog_leader::warming::{
changelog_kafka, changelog_partition_count, WarmClientPools, WarmingConfig, WarmingRetryPolicy,
};
use personhog_leader::warnings;
common_alloc::used!();
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Install a process-wide rustls CryptoProvider before any TLS use. kube's
// HTTPS client (controller discovery) uses rustls 0.23, which can't
// auto-pick a provider with both aws-lc-rs and ring compiled in — it
// panics. Matches personhog-router / cymbal / ingestion-consumer.
rustls::crypto::ring::default_provider()
.install_default()
.expect("failed to install rustls ring CryptoProvider");
let config = Config::init_from_env().expect("Invalid configuration");
config
.validate_lease_timescales()
.expect("Invalid lease configuration");
config
.validate_fencing_timescales()
.expect("Invalid fencing configuration");
config
.validate_shutdown_budgets()
.expect("Invalid shutdown configuration");
validate_table_name(&config.fallback_table).expect("Invalid FALLBACK_TABLE");
validate_table_name(&config.lifecycle_op_table).expect("Invalid LIFECYCLE_OP_TABLE");
validate_table_name(&config.lifecycle_op_person_table)
.expect("Invalid LIFECYCLE_OP_PERSON_TABLE");
// Initialize tracing
let log_layer = fmt::layer()
.with_target(true)
.with_thread_ids(true)
.with_level(true);
tracing_subscriber::registry()
.with(log_layer)
.with(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy(),
)
.init();
tracing::info!("Starting personhog-leader service");
tracing::info!("gRPC address: {}", config.grpc_address);
tracing::info!(
"Cache capacity: {} bytes per partition, properties compression: {}",
config.cache_memory_capacity_bytes,
config.cache_properties_compression_enabled
);
tracing::info!("Metrics port: {}", config.metrics_port);
tracing::info!("etcd endpoints: {}", config.etcd_endpoints);
tracing::info!("etcd prefix: {}", config.etcd_prefix);
tracing::info!("Pod name: {}", config.pod_name);
// Shutdown order matters: the coordination drain (phase 0) hands this
// pod's partitions off, which requires the gRPC server to keep serving
// reads and completing in-flight writes and the producer to keep
// delivering their changelog records. Server and producer therefore
// stop in phase 1, only after the drain finishes — signalling them
// together with coordination black-holed every partition for the whole
// drain (dead server, still the registered owner). The coordination
// window must fit the pod's whole teardown — drain, fence, keepalive
// join, revoke — and the global timeout both phases;
// `validate_lease_timescales` refuses a configuration that breaks
// the first relation at startup.
let mut manager = Manager::builder("personhog-leader")
.with_global_shutdown_timeout(config.global_shutdown_timeout())
.build();
let grpc_handle = manager.register(
"grpc-server",
ComponentOptions::new()
.with_graceful_shutdown(config.phase1_graceful_shutdown())
.with_shutdown_phase(1),
);
let metrics_handle = manager.register(
"metrics-server",
ComponentOptions::new().is_observability(true),
);
let coordination_handle = manager.register(
"coordination",
ComponentOptions::new().with_graceful_shutdown(config.coordination_graceful_shutdown()),
);
let kafka_handle = manager.register(
"kafka-producer",
// The graceful window is the ceiling; the bounded flush task
// spawned after the producer is built normally completes well
// inside it.
ComponentOptions::new()
.with_graceful_shutdown(config.phase1_graceful_shutdown())
.with_shutdown_phase(1),
);
let authority_metrics_handle = manager.register(
"authority-metrics",
ComponentOptions::new().is_observability(true),
);
let changelog_handle = manager.register("changelog-config", ComponentOptions::new());
let migration_handle = manager.register("migration-duty", ComponentOptions::new());
let warnings_settings = config.ingestion_warnings_settings();
let warnings_handle = warnings_settings.disabled_reason().is_none().then(|| {
manager.register(
"ingestion-warnings",
ComponentOptions::new()
.with_liveness_deadline(Duration::from_secs(30))
.is_advisory(true),
)
});
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
let monitor_guard = manager.monitor_background();
// Metrics/health HTTP server. The router is built here rather than
// inside the task because `setup_metrics_routes_with_overrides` is
// what installs the process-wide recorder: anything preregistered
// before it lands in a no-op recorder and never becomes a series.
let metrics_port = config.metrics_port;
let health_router = Router::new()
.route(
"/_readiness",
get(move || {
let r = readiness.clone();
async move { r.check().await }
}),
)
.route("/_liveness", get(move || async move { liveness.check() }));
// Changelog payload sizes: dense through the small-person range,
// with the top boundaries straddling the broker's message.max.bytes
// (typically 1 MiB) so p99 creeping toward the produce limit is
// visible before messages start getting rejected.
const CHANGELOG_PRODUCE_SIZE_BUCKETS_BYTES: &[f64] = &[
256.0, 1024.0, 4096.0, 16384.0, 65536.0, 262144.0, 524288.0, 1048576.0, 2097152.0,
];
// The write path and warms are tuned in single-digit milliseconds;
// the default ladder's 10 → 50 ms step blurs exactly the spans the
// fencing and warm work steers by, and pins interpolated quantiles
// to bucket edges. Overridden per metric so seconds-scale histograms
// keep the cheap default ladder.
let metrics_router = setup_metrics_routes_with_overrides(
health_router,
&[
(
Matcher::Full("personhog_leader_kafka_produce_duration_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_fence_send_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_fence_ack_wait_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_fence_window_wait_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_fence_commit_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_fence_init_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_person_lock_wait_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_release_phase_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_fallback_pool_acquire_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_fenced_producer_window_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("grpc_server_request_duration_ms".into()),
WRITE_PATH_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_warm_duration_ms".into()),
WARM_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_coordination_partition_warm_ms".into()),
WARM_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_warm_span_ms".into()),
WARM_LATENCY_BUCKETS_MS,
),
(
Matcher::Full("personhog_leader_kafka_produce_bytes".into()),
CHANGELOG_PRODUCE_SIZE_BUCKETS_BYTES,
),
],
);
preregister_metrics();
// The refusals the gate can emit: rare by design, and a burst is
// exactly what a deploy-window scrape would otherwise miss.
for reason in ["surrendered", "stale"] {
counter!(
"personhog_leader_authority_lapsed_rejections_total",
"reason" => reason
)
.increment(0);
for phase in ["warm", "resume"] {
counter!(
"personhog_leader_authority_lapsed_acquires_total",
"phase" => phase,
"reason" => reason
)
.increment(0);
}
}
for phase in ["warm", "resume"] {
counter!(
"personhog_leader_authority_lapsed_mid_acquire_total",
"phase" => phase
)
.increment(0);
}
counter!("personhog_leader_unresolved_versions_total").increment(0);
counter!("personhog_leader_unresolved_versions_spilled_total").increment(0);
gauge!("personhog_leader_unresolved_versions").set(0.0);
tokio::spawn(async move {
let _guard = metrics_handle.process_scope();
let bind = format!("0.0.0.0:{metrics_port}");
let listener = tokio::net::TcpListener::bind(&bind)
.await
.expect("Failed to bind metrics port");
tracing::info!("Metrics server listening on {}", bind);
axum::serve(listener, metrics_router)
.with_graceful_shutdown(metrics_handle.shutdown_signal())
.await
.expect("Metrics server error");
});
// Initialize partitioned cache and Kafka producer
let cache = Arc::new(
PartitionedCache::new(config.cache_memory_capacity_bytes)
.with_properties_compression(config.cache_properties_compression_enabled),
);
let kafka_producer = match create_kafka_producer(&config.kafka, kafka_handle.clone()).await {
Ok(producer) => producer,
Err(e) => {
tracing::error!(error = %e, "failed to create Kafka producer");
return Err(e.into());
}
};
// Runs at phase 1, after the coordination drain, so the drain's last
// records are in the queue it flushes. The changelog path awaits each
// record's delivery before acking, so a timed-out flush drops only
// records nobody acked.
spawn_bounded_flush_on_shutdown(
kafka_producer.clone(),
kafka_handle,
Duration::from_secs(10),
);
// PG fallback pool for cache misses (optional, disabled if URL is empty)
let fallback = if config.fallback_database_url.is_empty() {
tracing::info!("PG fallback disabled (no FALLBACK_DATABASE_URL)");
None
} else {
tracing::info!("PG fallback enabled");
let pool_config = common_database::PoolConfig {
max_connections: config.fallback_pg_max_connections,
min_connections: config.fallback_pg_min_connections,
pool_name: Some("personhog-leader-fallback".to_string()),
statement_timeout_ms: Some(5_000),
..Default::default()
};
let fallback = PgFallback {
pool: common_database::get_pool_with_config(
&config.fallback_database_url,
pool_config,
)?,
table: config.fallback_table.clone(),
lifecycle: LifecycleTables::new(
&config.lifecycle_op_table,
&config.lifecycle_op_person_table,
),
};
personhog_common::spawn_pool_monitor(
vec![personhog_common::MonitoredPool {
pool: fallback.pool.clone(),
label: "fallback".to_string(),
max_connections: config.fallback_pg_max_connections,
}],
Duration::from_secs(10),
);
Some(fallback)
};
// Connect to etcd for coordination and the partition count
let etcd_config = StoreConfig {
endpoints: config.etcd_endpoint_list(),
prefix: config.etcd_prefix.clone(),
};
let etcd_store = EtcdStore::connect_with_member_failover(etcd_config)
.await
.expect("Failed to connect to etcd");
let store = Arc::new(PersonhogStore::new(etcd_store));
// The same source the router hashes against.
let Some(loaded) = load_changelog_config(
&store,
CHANGELOG_BOOT_POLL_INTERVAL,
&changelog_handle.shutdown_token(),
)
.await
.expect("Failed to load the changelog config from etcd") else {
return Ok(());
};
let changelog_topic = loaded.topic.clone();
let num_partitions = loaded.partitions;
let live_cluster = loaded.cluster.clone();
let topic_partitions = changelog_partition_count(
&changelog_kafka(&config.kafka, live_cluster.as_ref()),
&changelog_topic,
Duration::from_secs(10),
)
.await
.expect("Failed to read the changelog topic's metadata");
assert_eq!(
topic_partitions, num_partitions,
"changelog topic {changelog_topic} has {topic_partitions} partitions but the config says {num_partitions}"
);
// Retirement settles marks on the writer group the config names, so it must be ours.
assert_eq!(
loaded.writer_group, config.writer_consumer_group,
"config/changelog names writer group {} but WRITER_CONSUMER_GROUP is {}",
loaded.writer_group, config.writer_consumer_group
);
tracing::info!(
num_partitions,
topic = %changelog_topic,
epoch = loaded.epoch,
"loaded changelog config"
);
let changelog = LiveChangelog::new(loaded);
{
let store = Arc::clone(&store);
let changelog = changelog.clone();
tokio::spawn(async move {
let _guard = changelog_handle.process_scope();
let cancel = CancellationToken::new();
tokio::select! {
_ = changelog_handle.shutdown_signal() => cancel.cancel(),
_ = keep_changelog_live(store, changelog, cancel.clone()) => {}
}
});
}
let locks = Arc::new(DashMap::new());
let captures: ChangelogCaptures = Arc::new(DashMap::new());
let fences: personhog_leader::fence::FenceMap = Arc::new(DashMap::new());
let inflight = Arc::new(InflightTracker::new());
let dirty_index = Arc::new(DirtyIndex::new(config.dirty_index_max_entries));
// One per pod, shared by the service that raises floors and the
// handler that drops them with the partition. The same bound the
// dirty index uses: both hold one entry per person written but not
// yet settled, and both are attackable the same way.
let emitted_versions = Arc::new(personhog_leader::emitted::EmittedVersions::new(
config.emitted_versions_max_entries,
));
let recovery = Arc::new(
ChangelogRecovery::new(RecoveryConfig {
kafka: config.kafka.clone(),
pod_name: config.pod_name.clone(),
recv_timeout: Duration::from_secs(config.recovery_recv_timeout_secs),
pool_size: config.recovery_pool_size,
})
.expect("Failed to build changelog recovery consumer pool"),
);
let warnings = warnings::install(&warnings_settings, warnings_handle);
let warnings_to_flush = warnings.clone();
let fence_scan = fallback.clone();
// All derived from LEASE_TTL, so an operator debugging a timeout
// cannot recover them without redoing the derivation.
tracing::info!(
window_ms = config.fencing_window_ms,
message_timeout_ms = config.fencing_message_timeout().as_millis(),
txn_timeout_ms = config.fencing_txn_timeout().as_millis(),
broker_txn_timeout_ms = config.fencing_broker_txn_timeout().as_millis(),
lease_runway_ms = config.lease_fence_runway().as_millis(),
"producing the changelog through per-partition transactional producers"
);
preregister_fencing_metrics();
// A condemned producer's repair otherwise waits for the next
// reconcile tick; this nudge lets the condemnation itself
// trigger the repair pass that heals it.
let fence_repair_nudge = Arc::new(Notify::new());
// The fenced producer runs on a tighter message timeout than the
// shared one: its writes must resolve inside the lease runway.
let fencing_kafka = common_kafka::config::KafkaConfig {
kafka_message_timeout_ms: config.fencing_message_timeout().as_millis() as u32,
..config.kafka.clone()
};
let fenced = Arc::new(
FencedChangelogProducers::new(FencedProducerConfig {
kafka: fencing_kafka,
init_timeout: config.fencing_init_timeout(),
commit_timeout: config.fencing_txn_timeout(),
broker_txn_timeout: config.fencing_broker_txn_timeout(),
window: Duration::from_millis(config.fencing_window_ms),
window_max_writes: config.fencing_window_max_writes,
settle_budget: config.fencing_settle_budget(),
lanes: config.fencing_lanes,
})
.with_repair_nudge(Arc::clone(&fence_repair_nudge)),
);
{
let duty = SealDuty {
store: Arc::clone(&store),
pod_name: config.pod_name.clone(),
fenced: Arc::clone(&fenced),
};
tokio::spawn(async move {
let _guard = migration_handle.process_scope();
let cancel = CancellationToken::new();
tokio::select! {
_ = migration_handle.shutdown_signal() => cancel.cancel(),
_ = duty.run(cancel.clone()) => {}
}
});
}
// One clock for the process: the coordination session claims and
// surrenders it, the data plane reads it per request.
let authority = Arc::new(AuthorityClock::unclaimed());
// Publish the live headroom whether or not the gate is armed: the
// question before enabling it is how close this fleet routinely runs
// to the margin, and that has to be answerable from a deployment
// that is not yet enforcing anything.
//
// This says nothing about a process-wide stall — a task that cannot
// run cannot report that it cannot run, which is the same limitation
// the clock exists to route around, and why enforcement reads the
// stamp inline on the request path instead of trusting a publisher.
// What it does show is a keepalive falling behind while the rest of
// the process is healthy, and the steady-state distance from the
// margin.
{
let authority = Arc::clone(&authority);
let handle = authority_metrics_handle;
tokio::spawn(async move {
let _guard = handle.process_scope();
let mut shutdown = std::pin::pin!(handle.shutdown_signal());
let mut tick = tokio::time::interval(Duration::from_secs(5));
loop {
tokio::select! {
_ = &mut shutdown => break,
_ = tick.tick() => {}
}
// Before the first grant there is no claim to measure
// against: age would read as process uptime and margin as
// zero, which any threshold would treat as a permanent
// emergency.
if !authority.is_claimed() {
continue;
}
gauge!("personhog_leader_authority_valid").set(if authority.is_valid() {
1.0
} else {
0.0
});
gauge!("personhog_leader_authority_age_ms")
.set(authority.since_confirmed().as_secs_f64() * 1000.0);
gauge!("personhog_leader_authority_margin_ms")
.set(authority.margin().as_secs_f64() * 1000.0);
}
});
}
let service = PersonHogLeaderService::new(
Arc::clone(&cache),
fallback,
Arc::clone(&locks),
Arc::clone(&inflight),
Arc::clone(&captures),
Arc::clone(&dirty_index),
Arc::clone(&recovery),
PropertySizeLimits::new(
config.properties_size_threshold,
config.properties_trim_target,
),
warnings.clone(),
Arc::clone(&fences),
Arc::clone(&fenced),
Arc::clone(&authority),
Arc::clone(&emitted_versions),
)
.with_fence_capacity(config.fence_map_max_entries);
let warm_pools = Arc::new(WarmClientPools::new(
&config.kafka,
&config.pod_name,
&config.writer_consumer_group,
));
let handler = LeaderHandoffHandler::new(
Arc::clone(&cache),
Arc::clone(&inflight),
Arc::clone(&dirty_index),
WarmingConfig {
kafka: config.kafka.clone(),
pod_name: config.pod_name.clone(),
writer_consumer_group: config.writer_consumer_group.clone(),
committed_offsets_timeout: Duration::from_secs(
config.warm_committed_offsets_timeout_secs,
),
fetch_watermarks_timeout: Duration::from_secs(
config.warm_fetch_watermarks_timeout_secs,
),
recv_timeout: Duration::from_secs(config.warm_recv_timeout_secs),
retry: WarmingRetryPolicy {
max_attempts: config.warm_retry_max_attempts,
initial_backoff: Duration::from_millis(config.warm_retry_initial_backoff_ms),
max_backoff: Duration::from_millis(config.warm_retry_max_backoff_ms),
},
},
Arc::clone(&fences),
fence_scan,
changelog.clone(),
Arc::clone(&captures),
Arc::clone(&warm_pools),
Arc::clone(&fenced),
Arc::clone(&authority),
Arc::clone(&emitted_versions),
);
let advertise_address =
personhog_leader::config::derive_advertise_address(&config.grpc_address, &config.pod_ip)
.expect("Invalid advertise address configuration");
tracing::info!(%advertise_address, "advertising gRPC address for routing");
// Discover this pod's owning controller and generation so the
// coordinator can steer placement away from old-generation pods
// during rollouts. Fail-open, and bounded: this runs before the pod
// handle and gRPC server exist, so an unresponsive API server must
// cost a few seconds of startup at worst — never availability.
let (controller, generation, k8s_awareness) = if config.k8s_awareness_enabled {
let discovery = discover_own_controller(&config, coordination_handle.shutdown_token());
match timeout(K8S_DISCOVERY_TIMEOUT, discovery).await {
Ok(Ok((awareness, info))) => {
tracing::info!(
controller = %info.controller,
generation = %info.generation,
"K8s awareness enabled; controller discovered"
);
(Some(info.controller), info.generation, Some(awareness))
}
Ok(Err(e)) => {
tracing::warn!(
error = %e,
"K8s awareness enabled but controller discovery failed; \
registering without rollout awareness"
);
(None, String::new(), None)
}
Err(_) => {
tracing::warn!(
timeout = ?K8S_DISCOVERY_TIMEOUT,
"K8s awareness enabled but controller discovery timed out; \
registering without rollout awareness"
);
(None, String::new(), None)
}
}
} else {
(None, String::new(), None)
};
// Timescale and concurrency knobs come from `base_pod_config`, the
// same values `validate_lease_timescales` summed at startup; only
// the identity fields, which no validation reads, are filled here.
let pod_config = PodConfig {
pod_name: config.pod_name.clone(),
generation,
controller,
advertise_address: Some(advertise_address),
..config.base_pod_config()
};
// Open connections up front: warms cluster in deploy bursts, and a
// cold pool would make every burst's first operations pay the client
// setup the pool exists to amortize. Sized from the pod's actual
// configured warm concurrency — the bound the semaphore enforces —
// so the pool and the concurrency limit cannot drift apart.
// Committed-offset queries run one per concurrent warm, so the
// offsets pool needs the same depth; it also serves the dirty-index
// prune tick.
//
// Awaited before the pod registers for partitions: a deploy burst
// hands this pod partitions immediately, and a warm that finds the
// pool empty builds its clients cold inside the handoff — seconds of
// connection setup on the path the pool exists to keep off. The
// deadline keeps a broker outage from wedging boot; past it,
// registration proceeds and the remaining slots are built cold, so
// this stays best-effort.
{
let warm_slots = pod_config.warm_concurrency;
let live_pools = warm_pools.on(live_cluster.as_ref());
let warm_up = async {
tokio::join!(
live_pools.offsets.warm_up(warm_slots),
live_pools.warming.warm_up(warm_slots),
);
};
if timeout(POOL_WARM_UP_DEADLINE, warm_up).await.is_err() {
tracing::warn!(
deadline_secs = POOL_WARM_UP_DEADLINE.as_secs(),
"consumer pool warm-up hit its deadline; continuing with a partially cold pool"
);
}
}
let mut pod = PodHandle::new(
store,
pod_config,
Arc::new(handler),
k8s_awareness,
Arc::clone(&authority),
);
pod = pod.with_repair_nudge(fence_repair_nudge);
tokio::spawn(async move {
let _guard = coordination_handle.process_scope();
if let Err(e) = pod.run(coordination_handle.shutdown_token()).await {
coordination_handle.signal_failure(format!("Coordination error: {e}"));
}
});
// Periodic sweep of idle per-key locks and parked fence connections
// nothing consumed (a cancelled inbound handoff leaves no convergence
// behind to discard them).
let sweep_locks = Arc::clone(&locks);
let sweep_fenced = Arc::clone(&fenced);
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60));
loop {
interval.tick().await;
sweep_idle_locks(&sweep_locks);
sweep_fenced.sweep_prepared();
}
});
tokio::spawn(run_dirty_index_prune_loop(
Arc::clone(&dirty_index),
Arc::clone(&cache),
Arc::clone(&locks),
Arc::clone(&warm_pools),
changelog.clone(),
config.writer_consumer_group.clone(),
Duration::from_secs(config.warm_committed_offsets_timeout_secs),
Duration::from_secs(config.dirty_index_prune_interval_secs.max(1)),
));
// gRPC server. Mirrors the replica's middleware stack so the router's
// per-backend metrics (processing time, transport/network overhead) and
// response compression behave identically on both backends. No tonic
// codec compression: requests always arrive uncompressed (the router
// rejects compressed leader requests before forwarding — it scans the
// request bytes for the routing key), and response compression is
// exclusively the gzip layer.
let grpc_addr = config.grpc_address;
let keepalive_interval = config.grpc_keepalive_interval();
let keepalive_timeout = config.grpc_keepalive_timeout();
let http2_windows = config.grpc_http2_windows();
let max_connection_age = config.grpc_max_connection_age();
let max_send = config.grpc_max_send_message_size;
let max_recv = config.grpc_max_recv_message_size;
let max_concurrent_requests = config.max_concurrent_requests;
let max_response_size = if config.gzip_max_response_size > 0 {
Some(config.gzip_max_response_size)
} else {
None
};
let gzip_config = AsyncGzipConfig::new(
config.gzip_response_compression,
config.gzip_compression_level,
config.gzip_min_payload_size,
)
.with_max_response_size(max_response_size, config.gzip_max_response_size_enforce);
if gzip_config.enabled {
tracing::info!(
level = gzip_config.compression_level,
min_payload_size = gzip_config.min_payload_size,
"Async gzip response compression enabled"
);
}
if max_concurrent_requests > 0 {
tracing::info!(
limit = max_concurrent_requests,
"gRPC load shedding enabled"
);
}
tracing::info!("Starting gRPC server on {}", grpc_addr);
tokio::spawn(async move {
let _guard = grpc_handle.process_scope();
let listener = match tokio::net::TcpListener::bind(grpc_addr).await {
Ok(l) => l,
Err(e) => {
grpc_handle.signal_failure(format!("Failed to bind gRPC port: {e}"));
return;
}
};
let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());
let mut server = http2_windows.apply_to_server(
Server::builder()
.http2_keepalive_interval(keepalive_interval)
.http2_keepalive_timeout(keepalive_timeout),
);
if let Some(age) = max_connection_age {
server = server.max_connection_age(age);
}
if let Err(e) = server
.layer(AsyncGzipLayer::new(gzip_config))
.layer(GrpcMetricsLayer::default().with_processing_time_header())
.layer(GrpcLoadShedLayer::new(max_concurrent_requests))
.add_service(
// accept_compressed only decodes gzip request frames from
// opted-in clients; responses stay with the AsyncGzipLayer
// (never send_compressed — see the tonic entry in Cargo.toml).
PersonHogLeaderServer::new(service)
.accept_compressed(CompressionEncoding::Gzip)
.max_encoding_message_size(max_send)
.max_decoding_message_size(max_recv),
)
.serve_with_incoming_shutdown(incoming, until_incoming_ends())
.await
{
grpc_handle.signal_failure(format!("gRPC server error: {e}"));
}
});
let waited = monitor_guard.wait().await;
// Flush before the result propagates: a failed component or a timed-out
// drain is exactly the shutdown that leaves warnings queued.
if let Err(e) = tokio::task::spawn_blocking(move || warnings_to_flush.flush()).await {
tracing::warn!(error = %e, "ingestion warnings flush task panicked");
}
waited?;
Ok(())
}
/// Periodically drop dirty-index marks the writer has applied to PG, and
/// export the dirty-count and writer-lag gauges as a side effect. One
/// batched OffsetFetch covers every partition with marks, on a pooled
/// client — each tick reuses the connection instead of rebuilding one.
#[allow(clippy::too_many_arguments)]
async fn run_dirty_index_prune_loop(
dirty_index: Arc<DirtyIndex>,
cache: Arc<PartitionedCache>,
locks: Arc<DashMap<PersonCacheKey, Arc<tokio::sync::Mutex<()>>>>,
pools: Arc<WarmClientPools>,
changelog: LiveChangelog,
writer_group: String,
offsets_timeout: Duration,
prune_interval: Duration,
) {
let mut interval = tokio::time::interval(prune_interval);
loop {
interval.tick().await;
gauge!("personhog_leader_dirty_index_size").set(dirty_index.len() as f64);
gauge!("personhog_leader_dirty_index_max_entries").set(dirty_index.max_entries() as f64);
gauge!("personhog_leader_cache_weight_bytes").set(cache.usage_bytes() as f64);
// Read when judged, not when the tick starts: a capture adopts before its marks
// exist, so a stream seen with marks is never judged on an older config.
let lags = prune_and_settle_tick(
&dirty_index,
&cache,
&locks,
&pools,
offsets_timeout,
|topic| retired_topic(&changelog.current(), &writer_group, topic),
)
.await;
// A partition's lag sums the streams feeding it; a stream the writer never
// committed on lags by its whole marked backlog, and a reclaimed one by zero.
let mut per_partition: HashMap<u32, i64> = HashMap::new();
for lag in &lags {
let outstanding = match dirty_index.max_offset(lag.partition, &lag.stream) {
Some(max_offset) => (max_offset + 1 - lag.committed.unwrap_or(0)).max(0),
None => 0,
};
*per_partition.entry(lag.partition).or_default() += outstanding;
}
for (partition, lag) in per_partition {
gauge!(
"personhog_leader_writer_uncommitted_offsets",
"partition" => partition.to_string()
)
.set(lag as f64);
}
}
}
/// How long startup may spend on controller discovery before falling
/// open. Generous against a healthy API server (three small reads);
/// tight against an unresponsive one, which must not delay serving.
const K8S_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(5);
/// Upper bound on the pre-registration consumer-pool warm-up. Generous
/// against MSK connection setup (~1–2s per client, run concurrently),
/// tight enough that a broker outage delays boot rather than blocking it.
const POOL_WARM_UP_DEADLINE: Duration = Duration::from_secs(10);
/// Build a K8s awareness client and discover this pod's owning controller
/// and generation. The awareness handle is returned alongside so the pod
/// can also classify its own departure at drain time.
async fn discover_own_controller(
config: &Config,
cancel: CancellationToken,
) -> Result<(Arc<K8sAwareness>, PodInfo), String> {
let namespace = config.resolve_k8s_namespace()?;
let client = Client::try_default()
.await
.map_err(|e| format!("failed to create K8s client: {e}"))?;
let awareness = Arc::new(K8sAwareness::new(client, namespace, cancel));
let info = awareness
.discover_controller(&config.pod_name)
.await
.map_err(|e| format!("controller discovery failed: {e}"))?;
Ok((awareness, info))
}
/// Touch the leader's deploy-burst counters so their series exist with
/// zero samples before any burst. metrics registration is lazy: a counter
/// that first fires between two scrapes materializes with the burst
/// already inside it, and no rate function can recover a delta that
/// precedes a series' first sample.
fn preregister_metrics() {
for fenced in ["true", "false"] {
counter!("personhog_leader_indeterminate_outcomes_total", "fenced" => fenced).increment(0);
}
counter!("personhog_leader_unresolved_versions_total").increment(0);
counter!("personhog_leader_death_documents_settled_total").increment(0);
gauge!("personhog_leader_unresolved_versions").set(0.0);
counter!("personhog_leader_warmed_messages_total").increment(0);
counter!("personhog_leader_warm_retries_exhausted_total", "stage" => "committed_offset")
.increment(0);
counter!("personhog_leader_warm_retries_exhausted_total", "stage" => "fetch_watermarks")
.increment(0);
for stage in ["committed_offset", "fetch_watermarks"] {
counter!("personhog_leader_warm_retries_total", "stage" => stage).increment(0);
}
// A broker blip is short enough to fall entirely between two
// scrapes, and these fire only during one. The codes seen in dev are
// preregistered so that burst is not swallowed; anything else stays
// lazy, as elsewhere for dynamic labels.
for code in ["BrokerTransportFailure", "AllBrokersDown"] {
counter!("personhog_leader_warm_transient_errors_total", "code" => code).increment(0);
}
}