Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(personhog): refuse new connections while a gRPC server drains (#124412)

Nick Best70d5974987a0
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 9c85c2c77a09..aacf94d2872f 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -9257,6 +9257,7 @@ dependencies = [
"sqlx",
"thiserror 2.0.18",
"tokio",
+ "tokio-util",
"tonic 0.12.3",
"tower 0.4.13",
"tracing",
diff --git a/rust/personhog-common/Cargo.toml b/rust/personhog-common/Cargo.toml
index f034e7d6b7cc..1bacfc3383bd 100644
--- a/rust/personhog-common/Cargo.toml
+++ b/rust/personhog-common/Cargo.toml
@@ -20,6 +20,7 @@ serde_json = { workspace = true }
pin-project = "1.1"
sqlx = { workspace = true }
tokio = { workspace = true }
+tokio-util = { workspace = true }
tonic = { workspace = true }
tower = { workspace = true }
thiserror = { workspace = true }
diff --git a/rust/personhog-common/src/grpc.rs b/rust/personhog-common/src/grpc.rs
index 2cb2bfad1abf..83c9d109317b 100644
--- a/rust/personhog-common/src/grpc.rs
+++ b/rust/personhog-common/src/grpc.rs
@@ -15,6 +15,7 @@ use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{AcquireError, OwnedSemaphorePermit, Semaphore};
use tokio::task::futures::TaskLocalFuture;
use tokio::time::error::Elapsed;
+use tokio_util::sync::CancellationToken;
use tonic::metadata::MetadataValue;
use tonic::transport::server::Connected;
use tower::util::Oneshot;
@@ -314,15 +315,38 @@ impl Connected for TrackedTcpStream {
pub fn tracked_tcp_incoming(
listener: TcpListener,
) -> impl futures::Stream<Item = Result<TrackedTcpStream, io::Error>> {
- unfold(listener, |listener| async move {
- match listener.accept().await {
- Ok((stream, _addr)) => Some((Ok(TrackedTcpStream::new(stream)), listener)),
- Err(e) if is_fatal_accept_error(&e) => None,
- Err(e) => Some((Err(e), listener)),
+ tracked_tcp_incoming_until(listener, CancellationToken::new())
+}
+
+/// [`tracked_tcp_incoming`] that drops the listener when `shutdown` is cancelled.
+/// A bound listener queues a late connection that nothing serves, so the client
+/// waits out its deadline. Serve this stream with [`until_incoming_ends`].
+pub fn tracked_tcp_incoming_until(
+ listener: TcpListener,
+ shutdown: CancellationToken,
+) -> impl futures::Stream<Item = Result<TrackedTcpStream, io::Error>> {
+ unfold((listener, shutdown), |(listener, shutdown)| async move {
+ tokio::select! {
+ biased;
+ _ = shutdown.cancelled() => None,
+ accepted = listener.accept() => match accepted {
+ Ok((stream, _addr)) => {
+ Some((Ok(TrackedTcpStream::new(stream)), (listener, shutdown)))
+ }
+ Err(e) if is_fatal_accept_error(&e) => None,
+ Err(e) => Some((Err(e), (listener, shutdown))),
+ },
}
})
}
+/// A shutdown signal that never fires, so the end of the stream stops tonic's
+/// accept loop. Tonic selects at random between its signal and the stream, and
+/// a signal win keeps the listener bound for the whole drain.
+pub fn until_incoming_ends() -> impl Future<Output = ()> + Send + 'static {
+ std::future::pending()
+}
+
/// Drop guard that decrements a client-side in-flight gauge on drop,
/// ensuring cancellation-safety for outbound request tracking.
pub struct ClientInFlightGuard {
@@ -844,6 +868,9 @@ mod tests {
use futures::poll;
use std::convert::Infallible;
use std::task::{Context, Poll};
+ use tonic::body::BoxBody;
+ use tonic::server::NamedService;
+ use tonic::transport::{Endpoint, Server};
use tower::Layer;
fn grpc_request(path: &str) -> Request<()> {
@@ -911,6 +938,85 @@ mod tests {
assert!(resp.headers().get("grpc-status").is_none());
}
+ #[derive(Clone)]
+ struct HeldRpcService {
+ entered: Arc<tokio::sync::Notify>,
+ release: CancellationToken,
+ }
+
+ impl NamedService for HeldRpcService {
+ const NAME: &'static str = "test.Held";
+ }
+
+ impl Service<Request<BoxBody>> for HeldRpcService {
+ type Response = Response<BoxBody>;
+ type Error = Infallible;
+ type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Infallible>> + Send>>;
+
+ fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+ Poll::Ready(Ok(()))
+ }
+
+ fn call(&mut self, _req: Request<BoxBody>) -> Self::Future {
+ let service = self.clone();
+ Box::pin(async move {
+ service.entered.notify_one();
+ service.release.cancelled().await;
+ Ok(Response::new(tonic::body::empty_body()))
+ })
+ }
+ }
+
+ #[tokio::test]
+ async fn a_draining_server_refuses_connections_and_finishes_accepted_rpcs() {
+ // Tonic picks its select branch at random, so one round can pass by luck.
+ for _ in 0..16 {
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let shutdown = CancellationToken::new();
+ let service = HeldRpcService {
+ entered: Arc::new(tokio::sync::Notify::new()),
+ release: CancellationToken::new(),
+ };
+
+ let server = tokio::spawn(
+ Server::builder()
+ .add_service(service.clone())
+ .serve_with_incoming_shutdown(
+ tracked_tcp_incoming_until(listener, shutdown.clone()),
+ until_incoming_ends(),
+ ),
+ );
+
+ let mut channel = Endpoint::from_shared(format!("http://{addr}"))
+ .unwrap()
+ .connect()
+ .await
+ .unwrap();
+ let request = Request::builder()
+ .method("POST")
+ .uri(format!("http://{addr}/test.Held/Call"))
+ .body(tonic::body::empty_body())
+ .unwrap();
+ let held_rpc = tokio::spawn(channel.ready().await.unwrap().call(request));
+ service.entered.notified().await;
+
+ shutdown.cancel();
+ tokio::time::timeout(Duration::from_secs(5), async {
+ while TcpStream::connect(addr).await.is_ok() {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("the port must refuse connections during the drain");
+ assert!(!server.is_finished());
+
+ service.release.cancel();
+ assert!(held_rpc.await.unwrap().is_ok());
+ server.await.unwrap().unwrap();
+ }
+ }
+
#[tokio::test]
async fn passthrough_when_disabled() {
let layer = GrpcLoadShedLayer::new(0);
diff --git a/rust/personhog-identity/src/main.rs b/rust/personhog-identity/src/main.rs
index 8cbe5c7fcec9..7406c012443a 100644
--- a/rust/personhog-identity/src/main.rs
+++ b/rust/personhog-identity/src/main.rs
@@ -8,7 +8,9 @@ use common_metrics::{ETCD_OP_LATENCY_BUCKETS_MS, ETCD_PAYLOAD_SIZE_BUCKETS_BYTES
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
-use personhog_common::grpc::{tracked_tcp_incoming, GrpcLoadShedLayer, GrpcMetricsLayer};
+use personhog_common::grpc::{
+ tracked_tcp_incoming_until, until_incoming_ends, GrpcLoadShedLayer, GrpcMetricsLayer,
+};
use personhog_common::{spawn_pool_monitor, MonitoredPool};
use personhog_proto::personhog::identity::v1::person_hog_identity_server::PersonHogIdentityServer;
use personhog_proto::personhog::lifecycle::v1::person_hog_lifecycle_server::PersonHogLifecycleServer;
@@ -497,7 +499,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
return;
}
};
- let incoming = tracked_tcp_incoming(listener);
+ let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());
let mut server = http2_windows.apply_to_server(
Server::builder()
.http2_keepalive_interval(keepalive_interval)
@@ -535,7 +537,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.max_encoding_message_size(max_send)
.max_decoding_message_size(max_recv),
)
- .serve_with_incoming_shutdown(incoming, grpc_handle.shutdown_signal())
+ .serve_with_incoming_shutdown(incoming, until_incoming_ends())
.await
{
grpc_handle.signal_failure(format!("gRPC server error: {e}"));
diff --git a/rust/personhog-leader/src/main.rs b/rust/personhog-leader/src/main.rs
index f470fa555eda..3ee80987c3c0 100644
--- a/rust/personhog-leader/src/main.rs
+++ b/rust/personhog-leader/src/main.rs
@@ -12,7 +12,9 @@ use k8s_awareness::{K8sAwareness, PodInfo};
use kube::Client;
use lifecycle::{ComponentOptions, Manager};
use personhog_common::async_gzip::{AsyncGzipConfig, AsyncGzipLayer};
-use personhog_common::grpc::{tracked_tcp_incoming, GrpcLoadShedLayer, GrpcMetricsLayer};
+use personhog_common::grpc::{
+ tracked_tcp_incoming_until, until_incoming_ends, GrpcLoadShedLayer, GrpcMetricsLayer,
+};
use personhog_common::metrics::{WARM_LATENCY_BUCKETS_MS, WRITE_PATH_LATENCY_BUCKETS_MS};
use personhog_coordination::authority::AuthorityClock;
use personhog_coordination::changelog::{
@@ -766,7 +768,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
return;
}
};
- let incoming = tracked_tcp_incoming(listener);
+ let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());
let mut server = http2_windows.apply_to_server(
Server::builder()
.http2_keepalive_interval(keepalive_interval)
@@ -788,7 +790,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.max_encoding_message_size(max_send)
.max_decoding_message_size(max_recv),
)
- .serve_with_incoming_shutdown(incoming, grpc_handle.shutdown_signal())
+ .serve_with_incoming_shutdown(incoming, until_incoming_ends())
.await
{
grpc_handle.signal_failure(format!("gRPC server error: {e}"));
diff --git a/rust/personhog-replica/src/main.rs b/rust/personhog-replica/src/main.rs
index c4fdf8e2e41b..a35f75925013 100644
--- a/rust/personhog-replica/src/main.rs
+++ b/rust/personhog-replica/src/main.rs
@@ -7,7 +7,9 @@ use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
use metrics_exporter_prometheus::PrometheusBuilder;
use personhog_common::async_gzip::{AsyncGzipConfig, AsyncGzipLayer};
-use personhog_common::grpc::{tracked_tcp_incoming, GrpcLoadShedLayer, GrpcMetricsLayer};
+use personhog_common::grpc::{
+ tracked_tcp_incoming_until, until_incoming_ends, GrpcLoadShedLayer, GrpcMetricsLayer,
+};
use personhog_proto::personhog::replica::v1::person_hog_replica_server::PersonHogReplicaServer;
use tonic::codec::CompressionEncoding;
use tonic::transport::Server;
@@ -364,7 +366,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
return;
}
};
- let incoming = tracked_tcp_incoming(listener);
+ let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());
let mut server = http2_windows.apply_to_server(
Server::builder()
.http2_keepalive_interval(keepalive_interval)
@@ -388,7 +390,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.max_encoding_message_size(max_send)
.max_decoding_message_size(max_recv),
)
- .serve_with_incoming_shutdown(incoming, grpc_handle.shutdown_signal())
+ .serve_with_incoming_shutdown(incoming, until_incoming_ends())
.await
{
grpc_handle.signal_failure(format!("gRPC server error: {e}"));
diff --git a/rust/personhog-router/src/main.rs b/rust/personhog-router/src/main.rs
index 08751f27c71c..02fceba8e69d 100644
--- a/rust/personhog-router/src/main.rs
+++ b/rust/personhog-router/src/main.rs
@@ -7,7 +7,7 @@ use envconfig::Envconfig;
use k8s_awareness::K8sAwareness;
use lifecycle::{ComponentOptions, Handle, Manager};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};
-use personhog_common::grpc::{tracked_tcp_incoming, GrpcMetricsLayer};
+use personhog_common::grpc::{tracked_tcp_incoming_until, until_incoming_ends, GrpcMetricsLayer};
use personhog_common::metrics::WRITE_PATH_LATENCY_BUCKETS_MS;
use personhog_coordination::changelog::{
keep_changelog_live, load_changelog_config, LiveChangelog, CHANGELOG_BOOT_POLL_INTERVAL,
@@ -446,7 +446,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
return;
}
};
- let incoming = tracked_tcp_incoming(listener);
+ let incoming = tracked_tcp_incoming_until(listener, grpc_handle.shutdown_token());
let proxy = RawProxyService::new(
replica_backend,
@@ -469,7 +469,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.add_service(proxy.clone())
.add_service(IdentityProxyService(proxy.clone()))
.add_service(LifecycleProxyService(proxy))
- .serve_with_incoming_shutdown(incoming, grpc_handle.shutdown_signal())
+ .serve_with_incoming_shutdown(incoming, until_incoming_ends())
.await;
if let Err(e) = result {