:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): wait a bounded time for an admission slot before shedding (#123771)
Nick Beste7edb9822e04
diff --git a/rust/personhog-common/Cargo.toml b/rust/personhog-common/Cargo.tomlindex 66911b7666d9..f034e7d6b7cc 100644--- a/rust/personhog-common/Cargo.toml+++ b/rust/personhog-common/Cargo.toml@@ -28,7 +28,7 @@ uuid = { workspace = true }[dev-dependencies]rstest = "0.26"-tokio = { workspace = true, features = ["macros", "rt"] }+tokio = { workspace = true, features = ["macros", "rt", "test-util"] }tower = { workspace = true, features = ["util"] }[lints]diff --git a/rust/personhog-common/src/grpc.rs b/rust/personhog-common/src/grpc.rsindex cd731366a385..2cb2bfad1abf 100644--- a/rust/personhog-common/src/grpc.rs+++ b/rust/personhog-common/src/grpc.rs@@ -4,7 +4,7 @@ use std::pin::Pin;use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::Arc;use std::task::{Context, Poll};-use std::time::Instant;+use std::time::{Duration, Instant};use futures::stream::unfold;use http::{HeaderValue, Request, Response};@@ -12,10 +12,13 @@ use metrics::{counter, gauge, histogram};use pin_project::{pin_project, pinned_drop};use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};use tokio::net::{TcpListener, TcpStream};+use tokio::sync::{AcquireError, OwnedSemaphorePermit, Semaphore};use tokio::task::futures::TaskLocalFuture;+use tokio::time::error::Elapsed;use tonic::metadata::MetadataValue;use tonic::transport::server::Connected;-use tower::{Layer, Service};+use tower::util::Oneshot;+use tower::{Layer, Service, ServiceExt};use crate::storage_error::TimeoutKind;@@ -426,7 +429,7 @@ where}fn call(&mut self, request: Request<ReqBody>) -> Self::Future {- let method: Arc<str> = Arc::from(extract_grpc_method(request.uri().path()));+ let method = extract_grpc_method(request.uri().path());let client = extract_client_name(&request);let caller_tag = extract_caller_tag(&request);gauge!("grpc_server_requests_in_flight", "method" => method.clone(), "client" => client.clone())@@ -526,14 +529,17 @@ impl<F> PinnedDrop for GrpcMetricsFuture<F> {/// Tower layer that sheds gRPC requests when the server is at capacity.///-/// Tracks in-flight requests with an atomic counter shared across all-/// connections on the pod. When the count exceeds `max_requests`,-/// immediately returns gRPC `UNAVAILABLE` so the router retries on-/// another pod. When `max_requests` is 0, the layer is a pass-through.+/// A request that finds no permit is refused with `x-not-applied`, so the+/// caller knows nothing ran. [`with_wait`](Self::with_wait) lets it wait a+/// bounded time first, so a sub-second burst shows up as latency rather+/// than refusals. When `max_requests` is 0, the layer is a pass-through.#[derive(Clone)]pub struct GrpcLoadShedLayer {max_requests: usize,- in_flight: Arc<AtomicUsize>,+ permits: Arc<Semaphore>,+ wait_for_slot: Duration,+ max_waiters: usize,+ waiters: Arc<AtomicUsize>,grpc_status: HeaderValue,}@@ -541,7 +547,10 @@ impl GrpcLoadShedLayer {pub fn new(max_requests: usize) -> Self {Self {max_requests,- in_flight: Arc::new(AtomicUsize::new(0)),+ permits: Arc::new(Semaphore::new(max_requests)),+ wait_for_slot: Duration::ZERO,+ max_waiters: max_requests,+ waiters: Arc::new(AtomicUsize::new(0)),grpc_status: grpc_status_header(tonic::Code::Unavailable),}}@@ -550,6 +559,20 @@ impl GrpcLoadShedLayer {self.grpc_status = grpc_status_header(code);self}++ /// Past `max_waiters` queued requests the pod sheds on arrival again:+ /// a deep queue is the signal that the overload is not a burst, and+ /// shedding late only holds the stream and body longer for the same+ /// answer. `max_waiters` of 0 means `max_requests`.+ pub fn with_wait(mut self, wait_for_slot: Duration, max_waiters: usize) -> Self {+ self.wait_for_slot = wait_for_slot;+ self.max_waiters = if max_waiters == 0 {+ self.max_requests+ } else {+ max_waiters+ };+ self+ }}fn grpc_status_header(code: tonic::Code) -> HeaderValue {@@ -563,7 +586,10 @@ impl<S> Layer<S> for GrpcLoadShedLayer {GrpcLoadShedService {inner: service,max_requests: self.max_requests,- in_flight: self.in_flight.clone(),+ permits: self.permits.clone(),+ wait_for_slot: self.wait_for_slot,+ max_waiters: self.max_waiters,+ waiters: self.waiters.clone(),grpc_status: self.grpc_status.clone(),}}@@ -573,18 +599,75 @@ impl<S> Layer<S> for GrpcLoadShedLayer {pub struct GrpcLoadShedService<S> {inner: S,max_requests: usize,- in_flight: Arc<AtomicUsize>,+ permits: Arc<Semaphore>,+ wait_for_slot: Duration,+ max_waiters: usize,+ waiters: Arc<AtomicUsize>,grpc_status: HeaderValue,}+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum ShedReason {+ NoWait,+ QueueFull,+ WaitTimeout,+}++impl ShedReason {+ fn as_str(self) -> &'static str {+ match self {+ ShedReason::NoWait => "no_wait",+ ShedReason::QueueFull => "queue_full",+ ShedReason::WaitTimeout => "wait_timeout",+ }+ }+}++fn record_shed(method: Arc<str>, client: Arc<str>, reason: ShedReason) {+ counter!("grpc_server_load_shed_total",+ "method" => method,+ "client" => client,+ "reason" => reason.as_str(),+ )+ .increment(1);+}++fn shed_response<ResBody: Default>(grpc_status: &HeaderValue) -> Response<ResBody> {+ Response::builder()+ .status(200)+ .header("content-type", "application/grpc")+ .header("grpc-status", grpc_status.clone())+ .header("grpc-message", "Server at capacity")+ .header(NOT_APPLIED_HEADER, "load_shed")+ .body(ResBody::default())+ .unwrap()+}++/// Holds one waiter slot; dropping it releases the slot, so a client that+/// disconnects mid-wait cannot leak one.+pub struct WaiterSlot {+ waiters: Arc<AtomicUsize>,+}++impl Drop for WaiterSlot {+ fn drop(&mut self) {+ self.waiters.fetch_sub(1, Ordering::Relaxed);+ gauge!("grpc_server_admission_waiters").decrement(1.0);+ }+}++type AcquireFuture = Pin<+ Box<dyn Future<Output = Result<Result<OwnedSemaphorePermit, AcquireError>, Elapsed>> + Send>,+>;+impl<S, ReqBody, ResBody> Service<Request<ReqBody>> for GrpcLoadShedService<S>where- S: Service<Request<ReqBody>, Response = Response<ResBody>>,+ S: Service<Request<ReqBody>, Response = Response<ResBody>> + Clone,ResBody: Default,{type Response = S::Response;type Error = S::Error;- type Future = GrpcLoadShedFuture<S::Future, ResBody>;+ type Future = GrpcLoadShedFuture<S, ReqBody, ResBody>;fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {self.inner.poll_ready(cx)@@ -594,79 +677,149 @@ whereif self.max_requests == 0 {return GrpcLoadShedFuture::Inner {inner: self.inner.call(request),- in_flight: None,+ permit: None,};}- let current = self.in_flight.fetch_add(1, Ordering::Relaxed);- if current >= self.max_requests {- self.in_flight.fetch_sub(1, Ordering::Relaxed);-- let client = extract_client_name(&request);- let method = extract_grpc_method(request.uri().path());- counter!("grpc_server_load_shed_total",- "method" => method,- "client" => client,- )- .increment(1);-- let response = Response::builder()- .status(200)- .header("content-type", "application/grpc")- .header("grpc-status", self.grpc_status.clone())- .header("grpc-message", "Server at capacity")- .header(NOT_APPLIED_HEADER, "load_shed")- .body(ResBody::default())- .unwrap();+ if let Ok(permit) = self.permits.clone().try_acquire_owned() {+ return GrpcLoadShedFuture::Inner {+ inner: self.inner.call(request),+ permit: Some(permit),+ };+ }++ let client = extract_client_name(&request);+ let method = extract_grpc_method(request.uri().path());+ if self.wait_for_slot.is_zero() {+ record_shed(method, client, ShedReason::NoWait);return GrpcLoadShedFuture::Shed {- response: Some(response),+ response: Some(shed_response(&self.grpc_status)),};}- GrpcLoadShedFuture::Inner {- inner: self.inner.call(request),- in_flight: Some(self.in_flight.clone()),+ if self.waiters.fetch_add(1, Ordering::Relaxed) >= self.max_waiters {+ self.waiters.fetch_sub(1, Ordering::Relaxed);+ record_shed(method, client, ShedReason::QueueFull);+ return GrpcLoadShedFuture::Shed {+ response: Some(shed_response(&self.grpc_status)),+ };+ }+ gauge!("grpc_server_admission_waiters").increment(1.0);++ GrpcLoadShedFuture::Waiting {+ acquire: Box::pin(tokio::time::timeout(+ self.wait_for_slot,+ self.permits.clone().acquire_owned(),+ )),+ slot: Some(WaiterSlot {+ waiters: self.waiters.clone(),+ }),+ request: Some(request),+ service: Some(self.inner.clone()),+ method,+ client,+ started: Instant::now(),+ grpc_status: self.grpc_status.clone(),}}}-#[pin_project(project = GrpcLoadShedFutureProj, PinnedDrop)]-pub enum GrpcLoadShedFuture<F, ResBody> {+/// The permit and the `WaiterSlot` release on drop, on completion and on+/// cancellation alike, which keeps the counts correct under client+/// disconnects.+#[pin_project(project = GrpcLoadShedFutureProj)]+pub enum GrpcLoadShedFuture<S, ReqBody, ResBody>+where+ S: Service<Request<ReqBody>>,+{Shed {response: Option<Response<ResBody>>,},Inner {#[pin]- inner: F,- in_flight: Option<Arc<AtomicUsize>>,+ inner: S::Future,+ permit: Option<OwnedSemaphorePermit>,+ },+ Waiting {+ acquire: AcquireFuture,+ slot: Option<WaiterSlot>,+ request: Option<Request<ReqBody>>,+ service: Option<S>,+ method: Arc<str>,+ client: Arc<str>,+ started: Instant,+ grpc_status: HeaderValue,+ },+ /// `Oneshot` polls the clone's readiness, which `poll_ready` never+ /// covered.+ Admitted {+ #[pin]+ inner: Oneshot<S, Request<ReqBody>>,+ permit: OwnedSemaphorePermit,},}-impl<F, ResBody, E> Future for GrpcLoadShedFuture<F, ResBody>+impl<S, ReqBody, ResBody> Future for GrpcLoadShedFuture<S, ReqBody, ResBody>where- F: Future<Output = Result<Response<ResBody>, E>>,+ S: Service<Request<ReqBody>, Response = Response<ResBody>>,+ ResBody: Default,{- type Output = F::Output;+ type Output = Result<Response<ResBody>, S::Error>;- fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {- match self.project() {- GrpcLoadShedFutureProj::Shed { response } => Poll::Ready(Ok(response.take().unwrap())),- GrpcLoadShedFutureProj::Inner { inner, .. } => inner.poll(cx),- }- }-}--#[pinned_drop]-impl<F, ResBody> PinnedDrop for GrpcLoadShedFuture<F, ResBody> {- fn drop(self: Pin<&mut Self>) {- if let GrpcLoadShedFutureProj::Inner {- in_flight: Some(counter),- ..- } = self.project()- {- counter.fetch_sub(1, Ordering::Relaxed);- }+ fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {+ let admitted = match self.as_mut().project() {+ GrpcLoadShedFutureProj::Shed { response } => {+ return Poll::Ready(Ok(response.take().unwrap()))+ }+ GrpcLoadShedFutureProj::Inner { inner, .. } => return inner.poll(cx),+ GrpcLoadShedFutureProj::Admitted { inner, .. } => return inner.poll(cx),+ GrpcLoadShedFutureProj::Waiting {+ acquire,+ slot,+ request,+ service,+ method,+ client,+ started,+ grpc_status,+ } => {+ let outcome = match acquire.as_mut().poll(cx) {+ Poll::Pending => return Poll::Pending,+ Poll::Ready(outcome) => outcome,+ };+ // Release the queue slot at the decision, not when hyper+ // drops the future.+ drop(slot.take());+ let waited_ms = started.elapsed().as_secs_f64() * 1000.0;+ match outcome {+ Ok(Ok(permit)) => {+ histogram!("grpc_server_admission_wait_ms",+ "method" => method.clone(),+ "client" => client.clone(),+ "outcome" => "admitted",+ )+ .record(waited_ms);+ let inner = service.take().unwrap().oneshot(request.take().unwrap());+ GrpcLoadShedFuture::Admitted { inner, permit }+ }+ // The semaphore is never closed, so `Ok(Err)` cannot+ // happen; shed rather than panic in a server path.+ Ok(Err(_)) | Err(_) => {+ histogram!("grpc_server_admission_wait_ms",+ "method" => method.clone(),+ "client" => client.clone(),+ "outcome" => "shed",+ )+ .record(waited_ms);+ record_shed(method.clone(), client.clone(), ShedReason::WaitTimeout);+ return Poll::Ready(Ok(shed_response(grpc_status)));+ }+ }+ }+ };+ self.set(admitted);+ self.poll(cx)}}@@ -677,19 +830,19 @@ impl<F, ResBody> PinnedDrop for GrpcLoadShedFuture<F, ResBody> {////// Cardinality is bounded because tonic rejects requests to unknown methods/// before they reach this middleware.-fn extract_grpc_method(path: &str) -> String {+fn extract_grpc_method(path: &str) -> Arc<str> {path.rsplit_once('/').map(|(_, method)| method).filter(|m| !m.is_empty()).unwrap_or("unknown")- .to_string()+ .into()}#[cfg(test)]mod tests {use super::*;+ use futures::poll;use std::convert::Infallible;- use std::sync::atomic::AtomicUsize;use std::task::{Context, Poll};use tower::Layer;@@ -733,14 +886,38 @@ mod tests {}}+ fn hold_pod_full(layer: &GrpcLoadShedLayer) -> Vec<GrpcLoadShedFuture<PendingService, (), ()>> {+ let mut svc = layer.layer(PendingService);+ (0..layer.max_requests)+ .map(|_| svc.call(grpc_request("/pkg.Svc/Held")))+ .collect()+ }++ fn waiting_layer(max_waiters: usize) -> GrpcLoadShedLayer {+ GrpcLoadShedLayer::new(1).with_wait(Duration::from_millis(500), max_waiters)+ }++ fn assert_shed(resp: &Response<()>, expected_status: &str) {+ assert_eq!(resp.headers().get("grpc-status").unwrap(), expected_status);+ assert_eq!(+ resp.headers().get("grpc-message").unwrap(),+ "Server at capacity"+ );+ assert_eq!(resp.headers().get(NOT_APPLIED_HEADER).unwrap(), "load_shed");+ }++ fn assert_admitted(resp: &Response<()>) {+ assert_eq!(resp.status(), 200);+ assert!(resp.headers().get("grpc-status").is_none());+ }+#[tokio::test]async fn passthrough_when_disabled() {let layer = GrpcLoadShedLayer::new(0);let mut svc = layer.layer(OkService);let resp = svc.call(grpc_request("/pkg.Svc/Method")).await.unwrap();- assert_eq!(resp.status(), 200);- assert!(resp.headers().get("grpc-status").is_none());+ assert_admitted(&resp);}#[tokio::test]@@ -749,8 +926,7 @@ mod tests {let mut svc = layer.layer(OkService);let resp = svc.call(grpc_request("/pkg.Svc/Method")).await.unwrap();- assert_eq!(resp.status(), 200);- assert!(resp.headers().get("grpc-status").is_none());+ assert_admitted(&resp);}#[tokio::test]@@ -762,86 +938,145 @@ mod tests {"8",),] {- let in_flight = Arc::new(AtomicUsize::new(0));- let layer = GrpcLoadShedLayer {- in_flight: in_flight.clone(),- ..layer- };- in_flight.store(2, Ordering::Relaxed);-+ let held = hold_pod_full(&layer);let mut svc = layer.layer(OkService);let resp = svc.call(grpc_request("/pkg.Svc/GetPerson")).await.unwrap();- assert_eq!(resp.headers().get("grpc-status").unwrap(), expected_status);- assert_eq!(- resp.headers().get("grpc-message").unwrap(),- "Server at capacity"- );- assert_eq!(resp.headers().get(NOT_APPLIED_HEADER).unwrap(), "load_shed");- // Shed path: increment then immediate decrement, net zero change- assert_eq!(in_flight.load(Ordering::Relaxed), 2);+ assert_shed(&resp, expected_status);+ assert_eq!(layer.permits.available_permits(), 0);+ drop(held);+ assert_eq!(layer.permits.available_permits(), 2);}}#[tokio::test]- async fn counter_decrements_on_completion() {- let in_flight = Arc::new(AtomicUsize::new(0));- let layer = GrpcLoadShedLayer {- in_flight: in_flight.clone(),- ..GrpcLoadShedLayer::new(10)- };+ async fn permit_returns_on_completion() {+ let layer = GrpcLoadShedLayer::new(10);let mut svc = layer.layer(OkService);- assert_eq!(in_flight.load(Ordering::Relaxed), 0);-let resp = svc.call(grpc_request("/pkg.Svc/Method")).await.unwrap();- assert!(resp.headers().get("grpc-status").is_none());+ assert_admitted(&resp);- // Future completed and dropped — counter back to 0- assert_eq!(in_flight.load(Ordering::Relaxed), 0);+ assert_eq!(layer.permits.available_permits(), 10);}#[tokio::test]- async fn counter_decrements_on_drop() {- let in_flight = Arc::new(AtomicUsize::new(0));- let layer = GrpcLoadShedLayer {- in_flight: in_flight.clone(),- ..GrpcLoadShedLayer::new(10)- };+ async fn permit_returns_on_drop() {+ let layer = GrpcLoadShedLayer::new(10);let mut svc = layer.layer(PendingService);let future = svc.call(grpc_request("/pkg.Svc/Method"));- assert_eq!(in_flight.load(Ordering::Relaxed), 1);+ assert_eq!(layer.permits.available_permits(), 9);// Drop without polling to completion — simulates request cancellationdrop(future);- assert_eq!(in_flight.load(Ordering::Relaxed), 0);+ assert_eq!(layer.permits.available_permits(), 10);}- #[tokio::test]- async fn shed_response_does_not_decrement_existing() {- let in_flight = Arc::new(AtomicUsize::new(5));- let layer = GrpcLoadShedLayer {- in_flight: in_flight.clone(),- ..GrpcLoadShedLayer::new(5)- };+ #[tokio::test(start_paused = true)]+ async fn waits_for_a_permit_that_frees_inside_the_window() {+ let layer = waiting_layer(4);+ let mut held = hold_pod_full(&layer);let mut svc = layer.layer(OkService);- let _resp = svc.call(grpc_request("/pkg.Svc/Method")).await.unwrap();+ let waiter = svc.call(grpc_request("/pkg.Svc/GetPerson"));+ tokio::pin!(waiter);+ assert!(poll!(waiter.as_mut()).is_pending());+ assert_eq!(layer.waiters.load(Ordering::Relaxed), 1);- // Existing in-flight count unchanged after shed- assert_eq!(in_flight.load(Ordering::Relaxed), 5);+ held.pop();+ let resp = waiter.await.unwrap();+ assert_admitted(&resp);+ assert_eq!(layer.waiters.load(Ordering::Relaxed), 0);+ }++ #[tokio::test(start_paused = true)]+ async fn sheds_after_the_wait_window() {+ for (layer, expected_status) in [+ (waiting_layer(4), "14"),+ (+ waiting_layer(4).with_code(tonic::Code::ResourceExhausted),+ "8",+ ),+ ] {+ let _held = hold_pod_full(&layer);+ let mut svc = layer.layer(OkService);++ let start = tokio::time::Instant::now();+ let resp = svc.call(grpc_request("/pkg.Svc/GetPerson")).await.unwrap();+ assert!(start.elapsed() >= Duration::from_millis(500));+ assert_shed(&resp, expected_status);+ assert_eq!(layer.waiters.load(Ordering::Relaxed), 0);+ }+ }++ #[tokio::test(start_paused = true)]+ async fn sheds_immediately_when_the_queue_is_full() {+ let layer = waiting_layer(1);+ let _held = hold_pod_full(&layer);+ let mut svc = layer.layer(OkService);++ let first = svc.call(grpc_request("/pkg.Svc/GetPerson"));+ tokio::pin!(first);+ assert!(poll!(first.as_mut()).is_pending());++ let start = tokio::time::Instant::now();+ let resp = svc.call(grpc_request("/pkg.Svc/GetPerson")).await.unwrap();+ assert_eq!(start.elapsed(), Duration::ZERO);+ assert_shed(&resp, "14");+ assert_eq!(layer.waiters.load(Ordering::Relaxed), 1);+ }++ #[tokio::test(start_paused = true)]+ async fn dropped_waiter_frees_its_queue_slot() {+ let layer = waiting_layer(1);+ let _held = hold_pod_full(&layer);+ let mut svc = layer.layer(OkService);++ {+ let first = svc.call(grpc_request("/pkg.Svc/GetPerson"));+ tokio::pin!(first);+ assert!(poll!(first.as_mut()).is_pending());+ assert_eq!(layer.waiters.load(Ordering::Relaxed), 1);+ }+ assert_eq!(layer.waiters.load(Ordering::Relaxed), 0);++ let second = svc.call(grpc_request("/pkg.Svc/GetPerson"));+ tokio::pin!(second);+ assert!(poll!(second.as_mut()).is_pending());+ assert_eq!(layer.waiters.load(Ordering::Relaxed), 1);+ }++ #[tokio::test(start_paused = true)]+ async fn earliest_waiter_takes_the_freed_permit() {+ let layer = waiting_layer(4);+ let mut held = hold_pod_full(&layer);+ let mut svc = layer.layer(OkService);++ let first = svc.call(grpc_request("/pkg.Svc/GetPerson"));+ tokio::pin!(first);+ let second = svc.call(grpc_request("/pkg.Svc/GetPerson"));+ tokio::pin!(second);+ assert!(poll!(first.as_mut()).is_pending());+ assert!(poll!(second.as_mut()).is_pending());++ held.pop();+ assert!(poll!(second.as_mut()).is_pending());+ match poll!(first.as_mut()) {+ Poll::Ready(Ok(resp)) => assert_admitted(&resp),+ other => panic!("first waiter not admitted: {other:?}"),+ }}#[test]fn extract_method_from_grpc_path() {assert_eq!(- extract_grpc_method("/package.Service/GetPerson"),+ &*extract_grpc_method("/package.Service/GetPerson"),"GetPerson");- assert_eq!(extract_grpc_method("/a.b.c/Method"), "Method");- assert_eq!(extract_grpc_method("/"), "unknown");- assert_eq!(extract_grpc_method(""), "unknown");+ assert_eq!(&*extract_grpc_method("/a.b.c/Method"), "Method");+ assert_eq!(&*extract_grpc_method("/"), "unknown");+ assert_eq!(&*extract_grpc_method(""), "unknown");}#[test]diff --git a/rust/personhog-identity/src/config.rs b/rust/personhog-identity/src/config.rsindex 0098b4499d1e..ed9957c45224 100644--- a/rust/personhog-identity/src/config.rs+++ b/rust/personhog-identity/src/config.rs@@ -148,6 +148,16 @@ pub struct Config {#[envconfig(default = "0")]pub max_concurrent_requests: usize,+ /// How long a request at a full pod waits for an admission slot before+ /// it is shed (ms). 0 = shed immediately.+ #[envconfig(default = "0")]+ pub load_shed_wait_ms: u64,++ /// Requests allowed to wait for a slot at once; beyond this the pod+ /// sheds immediately. 0 = max_concurrent_requests.+ #[envconfig(default = "0")]+ pub load_shed_max_waiters: usize,+/// Leader property writes in flight per get-or-create batch (0 acts as 1).#[envconfig(default = "8")]pub property_write_concurrency: usize,@@ -492,6 +502,10 @@ impl Config {)}+ pub fn load_shed_wait(&self) -> Duration {+ Duration::from_millis(self.load_shed_wait_ms)+ }+pub fn grpc_max_connection_age(&self) -> Option<Duration> {if self.grpc_max_connection_age_secs == 0 {Nonediff --git a/rust/personhog-identity/src/main.rs b/rust/personhog-identity/src/main.rsindex 30db8d126c81..23555f3d4bba 100644--- a/rust/personhog-identity/src/main.rs+++ b/rust/personhog-identity/src/main.rs@@ -461,9 +461,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {let max_send = config.grpc_max_send_message_size;let max_recv = config.grpc_max_recv_message_size;let max_concurrent_requests = config.max_concurrent_requests;+ let load_shed_wait = config.load_shed_wait();+ let load_shed_max_waiters = if config.load_shed_max_waiters == 0 {+ max_concurrent_requests+ } else {+ config.load_shed_max_waiters+ };if max_concurrent_requests > 0 {tracing::info!(limit = max_concurrent_requests,+ wait_ms = load_shed_wait.as_millis() as u64,+ max_waiters = load_shed_max_waiters,"gRPC load shedding enabled");}@@ -501,7 +509,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {// retrying the shed on another saturated pod..layer(GrpcLoadShedLayer::new(max_concurrent_requests)- .with_code(tonic::Code::ResourceExhausted),+ .with_code(tonic::Code::ResourceExhausted)+ .with_wait(load_shed_wait, load_shed_max_waiters),).add_service(PersonHogIdentityServer::new(service)