:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::sync::Arc;
use std::time::Duration;
use axum::{routing::get, Router};
use common_database::{get_pool_with_config, PoolConfig};
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
use metrics_exporter_prometheus::PrometheusBuilder;
use personhog_common::async_gzip::{AsyncGzipConfig, AsyncGzipLayer};
use personhog_common::grpc::{
tracked_tcp_incoming_until, until_incoming_ends, GrpcLoadShedLayer, GrpcMetricsLayer,
};
use personhog_proto::personhog::replica::v1::person_hog_replica_server::PersonHogReplicaServer;
use tonic::codec::CompressionEncoding;
use tonic::transport::Server;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::fmt;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::EnvFilter;
use personhog_common::{spawn_pool_monitor, MonitoredPool};
use personhog_replica::config::Config;
use personhog_replica::service::PersonHogReplicaService;
use personhog_replica::storage::postgres::PostgresStorage;
common_alloc::used!();
async fn create_storage(config: &Config) -> Arc<PostgresStorage> {
match config.storage_backend.as_str() {
"postgres" => {
let primary_pool_config = PoolConfig {
min_connections: config.min_pg_connections,
max_connections: config.max_pg_connections,
acquire_timeout: config.acquire_timeout(),
idle_timeout: config.idle_timeout(),
test_before_acquire: false,
statement_timeout_ms: config.statement_timeout(),
pool_name: Some("primary".to_string()),
};
let replica_pool_config = PoolConfig {
pool_name: Some("replica".to_string()),
..primary_pool_config.clone()
};
let bulk_primary_pool_config = PoolConfig {
min_connections: config
.min_pg_connections
.min(config.bulk_max_pg_connections),
max_connections: config.bulk_max_pg_connections,
acquire_timeout: config.bulk_acquire_timeout(),
test_before_acquire: true,
statement_timeout_ms: config.bulk_statement_timeout(),
pool_name: Some("bulk_primary".to_string()),
..primary_pool_config.clone()
};
let bulk_replica_pool_config = PoolConfig {
pool_name: Some("bulk_replica".to_string()),
..bulk_primary_pool_config.clone()
};
// Create primary pool
let primary_pool =
get_pool_with_config(&config.primary_database_url, primary_pool_config)
.expect("Failed to create primary database pool");
tracing::info!("Created primary database pool");
// Create replica pool (uses primary URL if replica URL not configured)
let replica_url = config.replica_database_url();
let replica_pool = if replica_url == config.primary_database_url {
tracing::info!("Replica URL not configured, using primary pool for both");
primary_pool.clone()
} else {
let pool = get_pool_with_config(replica_url, replica_pool_config)
.expect("Failed to create replica database pool");
tracing::info!("Created separate replica database pool");
pool
};
// Create bulk pools (same URLs, smaller pool with longer timeouts)
let bulk_primary_pool =
get_pool_with_config(&config.primary_database_url, bulk_primary_pool_config)
.expect("Failed to create bulk primary database pool");
let bulk_replica_pool = if replica_url == config.primary_database_url {
bulk_primary_pool.clone()
} else {
get_pool_with_config(replica_url, bulk_replica_pool_config)
.expect("Failed to create bulk replica database pool")
};
tracing::info!(
max_connections = config.bulk_max_pg_connections,
statement_timeout_ms = config.bulk_statement_timeout_ms,
"Created bulk database pools"
);
assert!(
config.bulk_chunk_size >= 1,
"BULK_CHUNK_SIZE must be at least 1"
);
assert!(
config.tombstoned_delete_max_rows >= 1,
"TOMBSTONED_DELETE_MAX_ROWS must be at least 1"
);
assert!(
(1..=i32::MAX as u64).contains(&config.hash_key_override_statement_timeout_ms),
"HASH_KEY_OVERRIDE_STATEMENT_TIMEOUT_MS must be between 1 and {}",
i32::MAX
);
assert!(
config.bulk_max_concurrent_chunks >= 1,
"BULK_MAX_CONCURRENT_CHUNKS must be at least 1"
);
assert!(
config.bulk_max_concurrent_chunks <= config.bulk_max_pg_connections as usize,
"BULK_MAX_CONCURRENT_CHUNKS ({}) must not exceed BULK_MAX_PG_CONNECTIONS ({})",
config.bulk_max_concurrent_chunks,
config.bulk_max_pg_connections
);
Arc::new(PostgresStorage::new(
primary_pool,
replica_pool,
bulk_primary_pool,
bulk_replica_pool,
config.bulk_chunk_size,
config.bulk_max_concurrent_chunks,
config.tombstoned_delete_max_rows,
config.hash_key_override_statement_timeout_ms,
))
}
other => {
panic!("Unknown storage backend: {other}. Supported: postgres");
}
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let config = Config::init_from_env().expect("Invalid configuration");
// Initialize tracing
let log_layer = fmt::layer()
.with_target(true)
.with_thread_ids(true)
.with_level(true);
tracing_subscriber::registry()
.with(log_layer)
.with(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy(),
)
.init();
tracing::info!("Starting personhog-replica service");
tracing::info!("gRPC address: {}", config.grpc_address);
tracing::info!("Metrics port: {}", config.metrics_port);
tracing::info!("Storage backend: {}", config.storage_backend);
// Build lifecycle manager and register components
let mut manager = Manager::builder("personhog-replica").build();
let grpc_handle = manager.register(
"grpc_server",
ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(30)),
);
let metrics_handle = manager.register(
"metrics_server",
ComponentOptions::new().is_observability(true),
);
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
let monitor = manager.monitor_background();
// Metrics/health HTTP server (observability handle — stays alive during standard drain)
let metrics_port = config.metrics_port;
tokio::spawn(async move {
let _guard = metrics_handle.process_scope();
let health_router = Router::new()
.route(
"/_readiness",
get(move || {
let r = readiness.clone();
async move { r.check().await }
}),
)
.route("/_liveness", get(move || async move { liveness.check() }));
const BUCKETS: &[f64] = &[
1.0, 5.0, 10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2000.0, 5000.0, 10000.0,
];
let recorder_handle = PrometheusBuilder::new()
.add_global_label("service", "personhog-replica")
.set_buckets(BUCKETS)
.unwrap()
.install_recorder()
.expect("Failed to install metrics recorder");
let router = health_router.route(
"/metrics",
get(move || std::future::ready(recorder_handle.render())),
);
let bind = format!("0.0.0.0:{metrics_port}");
let listener = tokio::net::TcpListener::bind(&bind)
.await
.expect("Failed to bind metrics port");
tracing::info!("Metrics server listening on {}", bind);
axum::serve(listener, router)
.with_graceful_shutdown(metrics_handle.shutdown_signal())
.await
.expect("Metrics server error");
});
// gRPC server
let storage = create_storage(&config).await;
// Pre-warm DB connection pools before accepting traffic.
// connect_lazy() starts with zero connections; without this, the first
// burst of requests after K8s routes traffic all pay the cold-start cost
// (TCP + TLS + PgBouncer + SET statement_timeout). Warming here is safe
// because the gRPC server hasn't bound its port yet — the startup probe
// (TCP on 50051) can't pass until after this completes.
if config.min_pg_connections > 0 {
let warmup_count = config.min_pg_connections as usize;
let server_warmup_count = (config.warmup_server_connections as usize).min(warmup_count);
tracing::info!(
count = warmup_count,
server_warmup = server_warmup_count,
"Warming database connection pools before accepting traffic"
);
let separate_replica = config.replica_database_url() != config.primary_database_url;
let pools: Vec<(&sqlx::PgPool, &str)> = if separate_replica {
vec![
(&storage.primary_pool, "primary"),
(&storage.replica_pool, "replica"),
]
} else {
vec![(&storage.primary_pool, "primary")]
};
for (pool, label) in &pools {
let pool_start = std::time::Instant::now();
let mut conns = Vec::with_capacity(warmup_count);
for _ in 0..warmup_count {
match pool.acquire().await {
Ok(conn) => conns.push(conn),
Err(e) => {
tracing::warn!(pool = label, error = %e, "Failed to warm connection");
break;
}
}
}
// Run a query on a subset of held connections to warm PgBouncer → PG.
// acquire() only establishes app → PgBouncer; in transaction pooling
// mode PgBouncer doesn't open a server connection until a query runs.
// We cap this to warmup_server_connections to avoid pinning too many
// PgBouncer backend connections — the remaining client connections are
// still warm (app → PgBouncer) and will get a server connection on
// first real query.
let mut server_warmed = 0u32;
for conn in conns.iter_mut().take(server_warmup_count) {
match sqlx::query("SELECT 1").execute(&mut **conn).await {
Ok(_) => server_warmed += 1,
Err(e) => {
tracing::warn!(pool = label, error = %e, "Failed to warm server-side connection");
}
}
}
tracing::info!(
pool = label,
client_conns = conns.len(),
server_conns = server_warmed,
elapsed_ms = pool_start.elapsed().as_millis() as u64,
"Pool warmup complete"
);
// Connections drop here, returning to the pool as idle
}
}
// Spawn background pool health monitor
let mut pools = vec![MonitoredPool {
pool: storage.primary_pool.clone(),
label: "primary".to_string(),
max_connections: config.max_pg_connections,
}];
// Always monitor the replica pool so query metrics (pool="replica") can be
// correlated with pool health gauges. When no separate replica URL is
// configured, replica_pool is a clone of primary_pool — monitoring it under
// both labels ensures the metrics align.
pools.push(MonitoredPool {
pool: storage.replica_pool.clone(),
label: "replica".to_string(),
max_connections: config.max_pg_connections,
});
pools.push(MonitoredPool {
pool: storage.bulk_primary_pool.clone(),
label: "bulk_primary".to_string(),
max_connections: config.bulk_max_pg_connections,
});
pools.push(MonitoredPool {
pool: storage.bulk_replica_pool.clone(),
label: "bulk_replica".to_string(),
max_connections: config.bulk_max_pg_connections,
});
spawn_pool_monitor(
pools,
Duration::from_secs(config.pool_monitor_interval_secs),
);
let service = PersonHogReplicaService::new(storage);
let grpc_addr = config.grpc_address;
let keepalive_interval = config.grpc_keepalive_interval();
let keepalive_timeout = config.grpc_keepalive_timeout();
let http2_windows = config.grpc_http2_windows();
let max_connection_age = config.grpc_max_connection_age();
let max_send = config.grpc_max_send_message_size;
let max_recv = config.grpc_max_recv_message_size;
let max_concurrent_requests = config.max_concurrent_requests;
let max_response_size = if config.gzip_max_response_size > 0 {
Some(config.gzip_max_response_size)
} else {
None
};
let gzip_config = AsyncGzipConfig::new(
config.gzip_response_compression,
config.gzip_compression_level,
config.gzip_min_payload_size,
)
.with_max_response_size(max_response_size, config.gzip_max_response_size_enforce);
if gzip_config.enabled {
tracing::info!(
level = gzip_config.compression_level,
min_payload_size = gzip_config.min_payload_size,
"Async gzip response compression enabled"
);
}
if let Some(limit) = gzip_config.max_response_size {
tracing::info!(
limit_bytes = limit,
enforce = gzip_config.max_response_size_enforce,
"Response size limit active"
);
}
if max_concurrent_requests > 0 {
tracing::info!(
limit = max_concurrent_requests,
"gRPC load shedding enabled"
);
}
tracing::info!("Starting gRPC server on {}", grpc_addr);
tokio::spawn(async move {
let _guard = grpc_handle.process_scope();
let listener = match tokio::net::TcpListener::bind(grpc_addr).await {
Ok(l) => l,
Err(e) => {
grpc_handle.signal_failure(format!("Failed to bind gRPC port: {e}"));
return;
}
};
let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());
let mut server = http2_windows.apply_to_server(
Server::builder()
.http2_keepalive_interval(keepalive_interval)
.http2_keepalive_timeout(keepalive_timeout),
);
if let Some(age) = max_connection_age {
server = server.max_connection_age(age);
}
// accept_compressed only decodes gzip request frames from opted-in
// clients; response compression is exclusively the gzip layer above
// (never send_compressed — see the tonic entry in Cargo.toml). Note
// max_decoding_message_size bounds the wire (compressed) size, so a
// compressed request can decode larger than the limit.
if let Err(e) = server
.layer(AsyncGzipLayer::new(gzip_config))
.layer(GrpcMetricsLayer::default().with_processing_time_header())
.layer(GrpcLoadShedLayer::new(max_concurrent_requests))
.add_service(
PersonHogReplicaServer::new(service)
.accept_compressed(CompressionEncoding::Gzip)
.max_encoding_message_size(max_send)
.max_decoding_message_size(max_recv),
)
.serve_with_incoming_shutdown(incoming, until_incoming_ends())
.await
{
grpc_handle.signal_failure(format!("gRPC server error: {e}"));
}
});
monitor.wait().await?;
Ok(())
}