:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(personhog): refuse new connections while a gRPC server drains (#124412)
Nick Best70d5974987a0
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 9c85c2c77a09..aacf94d2872f 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -9257,6 +9257,7 @@ dependencies = ["sqlx","thiserror 2.0.18","tokio",+ "tokio-util","tonic 0.12.3","tower 0.4.13","tracing",diff --git a/rust/personhog-common/Cargo.toml b/rust/personhog-common/Cargo.tomlindex f034e7d6b7cc..1bacfc3383bd 100644--- a/rust/personhog-common/Cargo.toml+++ b/rust/personhog-common/Cargo.toml@@ -20,6 +20,7 @@ serde_json = { workspace = true }pin-project = "1.1"sqlx = { workspace = true }tokio = { workspace = true }+tokio-util = { workspace = true }tonic = { workspace = true }tower = { workspace = true }thiserror = { workspace = true }diff --git a/rust/personhog-common/src/grpc.rs b/rust/personhog-common/src/grpc.rsindex 2cb2bfad1abf..83c9d109317b 100644--- a/rust/personhog-common/src/grpc.rs+++ b/rust/personhog-common/src/grpc.rs@@ -15,6 +15,7 @@ use tokio::net::{TcpListener, TcpStream};use tokio::sync::{AcquireError, OwnedSemaphorePermit, Semaphore};use tokio::task::futures::TaskLocalFuture;use tokio::time::error::Elapsed;+use tokio_util::sync::CancellationToken;use tonic::metadata::MetadataValue;use tonic::transport::server::Connected;use tower::util::Oneshot;@@ -314,15 +315,38 @@ impl Connected for TrackedTcpStream {pub fn tracked_tcp_incoming(listener: TcpListener,) -> impl futures::Stream<Item = Result<TrackedTcpStream, io::Error>> {- unfold(listener, |listener| async move {- match listener.accept().await {- Ok((stream, _addr)) => Some((Ok(TrackedTcpStream::new(stream)), listener)),- Err(e) if is_fatal_accept_error(&e) => None,- Err(e) => Some((Err(e), listener)),+ tracked_tcp_incoming_until(listener, CancellationToken::new())+}++/// [`tracked_tcp_incoming`] that drops the listener when `shutdown` is cancelled.+/// A bound listener queues a late connection that nothing serves, so the client+/// waits out its deadline. Serve this stream with [`until_incoming_ends`].+pub fn tracked_tcp_incoming_until(+ listener: TcpListener,+ shutdown: CancellationToken,+) -> impl futures::Stream<Item = Result<TrackedTcpStream, io::Error>> {+ unfold((listener, shutdown), |(listener, shutdown)| async move {+ tokio::select! {+ biased;+ _ = shutdown.cancelled() => None,+ accepted = listener.accept() => match accepted {+ Ok((stream, _addr)) => {+ Some((Ok(TrackedTcpStream::new(stream)), (listener, shutdown)))+ }+ Err(e) if is_fatal_accept_error(&e) => None,+ Err(e) => Some((Err(e), (listener, shutdown))),+ },}})}+/// A shutdown signal that never fires, so the end of the stream stops tonic's+/// accept loop. Tonic selects at random between its signal and the stream, and+/// a signal win keeps the listener bound for the whole drain.+pub fn until_incoming_ends() -> impl Future<Output = ()> + Send + 'static {+ std::future::pending()+}+/// Drop guard that decrements a client-side in-flight gauge on drop,/// ensuring cancellation-safety for outbound request tracking.pub struct ClientInFlightGuard {@@ -844,6 +868,9 @@ mod tests {use futures::poll;use std::convert::Infallible;use std::task::{Context, Poll};+ use tonic::body::BoxBody;+ use tonic::server::NamedService;+ use tonic::transport::{Endpoint, Server};use tower::Layer;fn grpc_request(path: &str) -> Request<()> {@@ -911,6 +938,85 @@ mod tests {assert!(resp.headers().get("grpc-status").is_none());}+ #[derive(Clone)]+ struct HeldRpcService {+ entered: Arc<tokio::sync::Notify>,+ release: CancellationToken,+ }++ impl NamedService for HeldRpcService {+ const NAME: &'static str = "test.Held";+ }++ impl Service<Request<BoxBody>> for HeldRpcService {+ type Response = Response<BoxBody>;+ type Error = Infallible;+ type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Infallible>> + Send>>;++ fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {+ Poll::Ready(Ok(()))+ }++ fn call(&mut self, _req: Request<BoxBody>) -> Self::Future {+ let service = self.clone();+ Box::pin(async move {+ service.entered.notify_one();+ service.release.cancelled().await;+ Ok(Response::new(tonic::body::empty_body()))+ })+ }+ }++ #[tokio::test]+ async fn a_draining_server_refuses_connections_and_finishes_accepted_rpcs() {+ // Tonic picks its select branch at random, so one round can pass by luck.+ for _ in 0..16 {+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let shutdown = CancellationToken::new();+ let service = HeldRpcService {+ entered: Arc::new(tokio::sync::Notify::new()),+ release: CancellationToken::new(),+ };++ let server = tokio::spawn(+ Server::builder()+ .add_service(service.clone())+ .serve_with_incoming_shutdown(+ tracked_tcp_incoming_until(listener, shutdown.clone()),+ until_incoming_ends(),+ ),+ );++ let mut channel = Endpoint::from_shared(format!("http://{addr}"))+ .unwrap()+ .connect()+ .await+ .unwrap();+ let request = Request::builder()+ .method("POST")+ .uri(format!("http://{addr}/test.Held/Call"))+ .body(tonic::body::empty_body())+ .unwrap();+ let held_rpc = tokio::spawn(channel.ready().await.unwrap().call(request));+ service.entered.notified().await;++ shutdown.cancel();+ tokio::time::timeout(Duration::from_secs(5), async {+ while TcpStream::connect(addr).await.is_ok() {+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("the port must refuse connections during the drain");+ assert!(!server.is_finished());++ service.release.cancel();+ assert!(held_rpc.await.unwrap().is_ok());+ server.await.unwrap().unwrap();+ }+ }+#[tokio::test]async fn passthrough_when_disabled() {let layer = GrpcLoadShedLayer::new(0);diff --git a/rust/personhog-identity/src/main.rs b/rust/personhog-identity/src/main.rsindex 8cbe5c7fcec9..7406c012443a 100644--- a/rust/personhog-identity/src/main.rs+++ b/rust/personhog-identity/src/main.rs@@ -8,7 +8,9 @@ use common_metrics::{ETCD_OP_LATENCY_BUCKETS_MS, ETCD_PAYLOAD_SIZE_BUCKETS_BYTESuse envconfig::Envconfig;use lifecycle::{ComponentOptions, Manager};use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};-use personhog_common::grpc::{tracked_tcp_incoming, GrpcLoadShedLayer, GrpcMetricsLayer};+use personhog_common::grpc::{+ tracked_tcp_incoming_until, until_incoming_ends, GrpcLoadShedLayer, GrpcMetricsLayer,+};use personhog_common::{spawn_pool_monitor, MonitoredPool};use personhog_proto::personhog::identity::v1::person_hog_identity_server::PersonHogIdentityServer;use personhog_proto::personhog::lifecycle::v1::person_hog_lifecycle_server::PersonHogLifecycleServer;@@ -497,7 +499,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {return;}};- let incoming = tracked_tcp_incoming(listener);+ let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());let mut server = http2_windows.apply_to_server(Server::builder().http2_keepalive_interval(keepalive_interval)@@ -535,7 +537,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.max_encoding_message_size(max_send).max_decoding_message_size(max_recv),)- .serve_with_incoming_shutdown(incoming, grpc_handle.shutdown_signal())+ .serve_with_incoming_shutdown(incoming, until_incoming_ends()).await{grpc_handle.signal_failure(format!("gRPC server error: {e}"));diff --git a/rust/personhog-leader/src/main.rs b/rust/personhog-leader/src/main.rsindex f470fa555eda..3ee80987c3c0 100644--- a/rust/personhog-leader/src/main.rs+++ b/rust/personhog-leader/src/main.rs@@ -12,7 +12,9 @@ use k8s_awareness::{K8sAwareness, PodInfo};use kube::Client;use lifecycle::{ComponentOptions, Manager};use personhog_common::async_gzip::{AsyncGzipConfig, AsyncGzipLayer};-use personhog_common::grpc::{tracked_tcp_incoming, GrpcLoadShedLayer, GrpcMetricsLayer};+use personhog_common::grpc::{+ tracked_tcp_incoming_until, until_incoming_ends, GrpcLoadShedLayer, GrpcMetricsLayer,+};use personhog_common::metrics::{WARM_LATENCY_BUCKETS_MS, WRITE_PATH_LATENCY_BUCKETS_MS};use personhog_coordination::authority::AuthorityClock;use personhog_coordination::changelog::{@@ -766,7 +768,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {return;}};- let incoming = tracked_tcp_incoming(listener);+ let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());let mut server = http2_windows.apply_to_server(Server::builder().http2_keepalive_interval(keepalive_interval)@@ -788,7 +790,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.max_encoding_message_size(max_send).max_decoding_message_size(max_recv),)- .serve_with_incoming_shutdown(incoming, grpc_handle.shutdown_signal())+ .serve_with_incoming_shutdown(incoming, until_incoming_ends()).await{grpc_handle.signal_failure(format!("gRPC server error: {e}"));diff --git a/rust/personhog-replica/src/main.rs b/rust/personhog-replica/src/main.rsindex c4fdf8e2e41b..a35f75925013 100644--- a/rust/personhog-replica/src/main.rs+++ b/rust/personhog-replica/src/main.rs@@ -7,7 +7,9 @@ use envconfig::Envconfig;use lifecycle::{ComponentOptions, Manager};use metrics_exporter_prometheus::PrometheusBuilder;use personhog_common::async_gzip::{AsyncGzipConfig, AsyncGzipLayer};-use personhog_common::grpc::{tracked_tcp_incoming, GrpcLoadShedLayer, GrpcMetricsLayer};+use personhog_common::grpc::{+ tracked_tcp_incoming_until, until_incoming_ends, GrpcLoadShedLayer, GrpcMetricsLayer,+};use personhog_proto::personhog::replica::v1::person_hog_replica_server::PersonHogReplicaServer;use tonic::codec::CompressionEncoding;use tonic::transport::Server;@@ -364,7 +366,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {return;}};- let incoming = tracked_tcp_incoming(listener);+ let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());let mut server = http2_windows.apply_to_server(Server::builder().http2_keepalive_interval(keepalive_interval)@@ -388,7 +390,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.max_encoding_message_size(max_send).max_decoding_message_size(max_recv),)- .serve_with_incoming_shutdown(incoming, grpc_handle.shutdown_signal())+ .serve_with_incoming_shutdown(incoming, until_incoming_ends()).await{grpc_handle.signal_failure(format!("gRPC server error: {e}"));diff --git a/rust/personhog-router/src/main.rs b/rust/personhog-router/src/main.rsindex 08751f27c71c..02fceba8e69d 100644--- a/rust/personhog-router/src/main.rs+++ b/rust/personhog-router/src/main.rs@@ -7,7 +7,7 @@ use envconfig::Envconfig;use k8s_awareness::K8sAwareness;use lifecycle::{ComponentOptions, Handle, Manager};use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};-use personhog_common::grpc::{tracked_tcp_incoming, GrpcMetricsLayer};+use personhog_common::grpc::{tracked_tcp_incoming_until, until_incoming_ends, GrpcMetricsLayer};use personhog_common::metrics::WRITE_PATH_LATENCY_BUCKETS_MS;use personhog_coordination::changelog::{keep_changelog_live, load_changelog_config, LiveChangelog, CHANGELOG_BOOT_POLL_INTERVAL,@@ -446,7 +446,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {return;}};- let incoming = tracked_tcp_incoming(listener);+ let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());let proxy = RawProxyService::new(replica_backend,@@ -469,7 +469,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.add_service(proxy.clone()).add_service(IdentityProxyService(proxy.clone())).add_service(LifecycleProxyService(proxy))- .serve_with_incoming_shutdown(incoming, grpc_handle.shutdown_signal())+ .serve_with_incoming_shutdown(incoming, until_incoming_ends()).await;if let Err(e) = result {