Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

13.8 KB
Raw Blame
//! Serving middleware for PostHog gRPC servers.
//!
//! `personhog-common` carries its own copy of this layer alongside personhog's routing
//! contract. This crate holds the part that is not about personhog, so a service can take
//! request metrics without taking `personhog-proto`, sqlx, and the persons modules with it.

use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Instant;

use http::{Request, Response};
use metrics::{counter, gauge, histogram};
use pin_project::{pin_project, pinned_drop};
use tower::{Layer, Service};

/// Header a caller sets to name itself. Requests without it are labelled `unknown`.
const CLIENT_NAME_HEADER: &str = "x-client-name";

/// Label for a failure carrying no gRPC status at all. Part of the same vocabulary as
/// [`code_as_str`], and deliberately distinct from its `unknown`, which is a real code a
/// server can return.
pub const NON_STATUS: &str = "non_status";

/// Stable snake_case name for a gRPC status code, for use as a metric label. Codes are a
/// fixed vocabulary, which is what makes them safe to label with; status *messages* never
/// are.
pub fn code_as_str(code: tonic::Code) -> &'static str {
    use tonic::Code;
    match code {
        Code::Ok => "ok",
        Code::Cancelled => "cancelled",
        Code::Unknown => "unknown",
        Code::InvalidArgument => "invalid_argument",
        Code::DeadlineExceeded => "deadline_exceeded",
        Code::NotFound => "not_found",
        Code::AlreadyExists => "already_exists",
        Code::PermissionDenied => "permission_denied",
        Code::ResourceExhausted => "resource_exhausted",
        Code::FailedPrecondition => "failed_precondition",
        Code::Aborted => "aborted",
        Code::OutOfRange => "out_of_range",
        Code::Unimplemented => "unimplemented",
        Code::Internal => "internal",
        Code::Unavailable => "unavailable",
        Code::DataLoss => "data_loss",
        Code::Unauthenticated => "unauthenticated",
    }
}

const MAX_HEADER_TAG_LEN: usize = 128;

fn is_safe_tag_char(c: char) -> bool {
    c.is_ascii_alphanumeric() || matches!(c, '/' | '_' | '-' | ':' | '.')
}

/// A caller-controlled value becomes a metric label, so an unbounded or hostile one becomes
/// unbounded cardinality. Anything outside the safe set collapses to `unknown`.
fn sanitize_header_tag(raw: &str) -> &str {
    let value = if raw.len() > MAX_HEADER_TAG_LEN {
        &raw[..MAX_HEADER_TAG_LEN]
    } else {
        raw
    };
    if value.chars().all(is_safe_tag_char) {
        value
    } else {
        "unknown"
    }
}

fn extract_client_name<B>(request: &Request<B>) -> Arc<str> {
    request
        .headers()
        .get(CLIENT_NAME_HEADER)
        .and_then(|value| value.to_str().ok())
        .filter(|value| !value.is_empty())
        .map(sanitize_header_tag)
        .unwrap_or("unknown")
        .into()
}

/// Extract the method name from a gRPC URI path, which looks like
/// `/package.Service/MethodName`.
///
/// `Server::layer` wraps tonic's router, so this runs before routing and sees the path of a
/// request no method will serve. The value is therefore caller-controlled and goes through
/// the same sanitizer as a header, or one client calling a wrong path mints a series per
/// path it tries.
fn extract_grpc_method(path: &str) -> &str {
    let method = path
        .rsplit_once('/')
        .map(|(_, method)| method)
        .filter(|method| !method.is_empty())
        .unwrap_or("unknown");
    sanitize_header_tag(method)
}

/// The `code` label for a response, in the [`code_as_str`] vocabulary.
///
/// A failed gRPC call is a trailers-only response: an empty body with `grpc-status` in the
/// headers. tonic gives a handler error that shape, so the layer reads the status at the
/// head and never polls the body. An absent header therefore means the handler returned Ok,
/// whose `grpc-status: 0` rides the body trailers instead.
///
/// This holds while every method is unary. A server-streaming method that failed part way
/// through its stream would read as `ok` here, and would need the trailers read from the
/// response body.
fn response_code(headers: &http::HeaderMap) -> &'static str {
    let Some(raw) = headers.get("grpc-status") else {
        return "ok";
    };
    raw.to_str()
        .ok()
        .and_then(|value| value.parse::<i32>().ok())
        .map_or(NON_STATUS, |code| code_as_str(tonic::Code::from_i32(code)))
}

/// Tower layer that instruments gRPC requests with timing and concurrency metrics.
///
/// Records:
/// - `grpc_server_requests_total` — counter with method, client and code labels
/// - `grpc_server_request_duration_ms` — histogram with method and client labels
/// - `grpc_server_requests_in_flight` — gauge with method and client labels
#[derive(Clone, Default)]
pub struct GrpcMetricsLayer;

impl<S> Layer<S> for GrpcMetricsLayer {
    type Service = GrpcMetricsService<S>;

    fn layer(&self, service: S) -> Self::Service {
        GrpcMetricsService { inner: service }
    }
}

#[derive(Clone)]
pub struct GrpcMetricsService<S> {
    inner: S,
}

impl<S, ReqBody, ResBody> Service<Request<ReqBody>> for GrpcMetricsService<S>
where
    S: Service<Request<ReqBody>, Response = Response<ResBody>>,
{
    type Response = S::Response;
    type Error = S::Error;
    type Future = GrpcMetricsFuture<S::Future>;

    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
        self.inner.poll_ready(cx)
    }

    fn call(&mut self, request: Request<ReqBody>) -> Self::Future {
        let method: Arc<str> = Arc::from(extract_grpc_method(request.uri().path()));
        let client = extract_client_name(&request);
        gauge!("grpc_server_requests_in_flight", "method" => method.clone(), "client" => client.clone())
            .increment(1.0);

        GrpcMetricsFuture {
            inner: self.inner.call(request),
            method,
            client,
            start: Instant::now(),
        }
    }
}

/// Future returned by [`GrpcMetricsService`]. Lives inline in the caller's async state
/// machine, so instrumenting a request costs no heap allocation.
#[pin_project(PinnedDrop)]
pub struct GrpcMetricsFuture<F> {
    #[pin]
    inner: F,
    method: Arc<str>,
    client: Arc<str>,
    start: Instant,
}

impl<F, ResBody, E> Future for GrpcMetricsFuture<F>
where
    F: Future<Output = Result<Response<ResBody>, E>>,
{
    type Output = F::Output;

    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
        let this = self.project();
        let Poll::Ready(result) = this.inner.poll(cx) else {
            return Poll::Pending;
        };

        // A transport-level error never reached the handler, so it carries no gRPC status.
        let code = match result {
            Ok(ref response) => response_code(response.headers()),
            Err(_) => NON_STATUS,
        };
        counter!("grpc_server_requests_total",
            "method" => this.method.clone(),
            "client" => this.client.clone(),
            "code" => code)
        .increment(1);
        histogram!("grpc_server_request_duration_ms",
            "method" => this.method.clone(),
            "client" => this.client.clone())
        .record(this.start.elapsed().as_secs_f64() * 1000.0);

        Poll::Ready(result)
    }
}

/// Decrements the in-flight gauge on both normal completion and cancellation.
#[pinned_drop]
impl<F> PinnedDrop for GrpcMetricsFuture<F> {
    fn drop(self: Pin<&mut Self>) {
        let this = self.project();
        gauge!("grpc_server_requests_in_flight",
            "method" => this.method.clone(),
            "client" => this.client.clone())
        .decrement(1.0);
    }
}

#[cfg(test)]
mod tests {
    use std::convert::Infallible;

    use tower::ServiceExt;

    use super::*;

    /// Serves whatever response it is built with, so a test can drive the layer with the
    /// exact shape tonic produces.
    #[derive(Clone)]
    struct FixedService(http::StatusCode, Option<&'static str>);

    impl Service<Request<()>> for FixedService {
        type Response = Response<()>;
        type Error = Infallible;
        type Future = std::future::Ready<Result<Response<()>, Infallible>>;

        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
            Poll::Ready(Ok(()))
        }

        fn call(&mut self, _request: Request<()>) -> Self::Future {
            let mut builder = Response::builder().status(self.0);
            if let Some(status) = self.1 {
                builder = builder.header("grpc-status", status);
            }
            std::future::ready(Ok(builder.body(()).unwrap()))
        }
    }

    fn grpc_request(path: &str) -> Request<()> {
        Request::builder().uri(path).body(()).unwrap()
    }

    #[tokio::test]
    async fn the_layer_passes_the_response_through_untouched() {
        let service = GrpcMetricsLayer.layer(FixedService(http::StatusCode::OK, None));

        let response = service
            .oneshot(grpc_request("/pkg.Svc/Method"))
            .await
            .unwrap();

        assert_eq!(response.status(), 200);
        assert!(response.headers().get("grpc-status").is_none());
    }

    /// Serves the response tonic builds for a failed handler, which is the shape the layer
    /// has to read the status out of.
    #[derive(Clone)]
    struct StatusService(tonic::Code);

    impl Service<Request<()>> for StatusService {
        type Response = Response<tonic::body::BoxBody>;
        type Error = Infallible;
        type Future = std::future::Ready<Result<Self::Response, Infallible>>;

        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
            Poll::Ready(Ok(()))
        }

        fn call(&mut self, _request: Request<()>) -> Self::Future {
            std::future::ready(Ok(tonic::Status::new(self.0, "test").into_http()))
        }
    }

    #[test]
    fn the_counter_carries_the_method_client_and_code_of_a_real_response() {
        let recorder = metrics_util::debugging::DebuggingRecorder::new();
        let snapshotter = recorder.snapshotter();
        let runtime = tokio::runtime::Builder::new_current_thread()
            .build()
            .unwrap();

        metrics::with_local_recorder(&recorder, || {
            let service = GrpcMetricsLayer.layer(StatusService(tonic::Code::NotFound));
            let request = Request::builder()
                .uri("/usage_ingestion.v1.UsageIngestion/IngestBillingUsage")
                .header("x-client-name", "feature-flags")
                .body(())
                .unwrap();
            runtime.block_on(service.oneshot(request)).unwrap();
        });

        let labels = snapshotter
            .snapshot()
            .into_vec()
            .into_iter()
            .find(|(key, _, _, _)| key.key().name() == "grpc_server_requests_total")
            .map(|(key, _, _, _)| {
                let mut labels = key
                    .key()
                    .labels()
                    .map(|l| format!("{}={}", l.key(), l.value()))
                    .collect::<Vec<_>>();
                labels.sort();
                labels
            })
            .expect("the layer must count the request");

        assert_eq!(
            labels,
            vec![
                "client=feature-flags",
                "code=not_found",
                "method=IngestBillingUsage"
            ]
        );
    }

    #[test]
    fn a_handler_error_reads_as_its_own_code() {
        // Pinned against tonic rather than a hand-built response: the whole design rests on
        // tonic putting the status in the headers of a trailers-only response.
        let response = tonic::Status::not_found("nope").into_http();

        assert_eq!(response_code(response.headers()), "not_found");
    }

    #[test]
    fn response_code_vocabulary() {
        let code_for = |header: Option<&str>| {
            let mut headers = http::HeaderMap::new();
            if let Some(value) = header {
                headers.insert("grpc-status", value.parse().unwrap());
            }
            response_code(&headers)
        };

        // Absent means the handler returned Ok and grpc-status rides the body trailers.
        assert_eq!(code_for(None), "ok");
        assert_eq!(code_for(Some("0")), "ok");
        assert_eq!(code_for(Some("14")), "unavailable");
        // An out-of-range code is still a code; only an unparseable one is non_status.
        assert_eq!(code_for(Some("99")), "unknown");
        assert_eq!(code_for(Some("not-a-number")), NON_STATUS);
    }

    #[test]
    fn extract_method_from_grpc_path() {
        assert_eq!(
            extract_grpc_method("/package.Service/IngestBillingUsage"),
            "IngestBillingUsage"
        );
        assert_eq!(extract_grpc_method("/"), "unknown");
        assert_eq!(extract_grpc_method(""), "unknown");
    }

    #[test]
    fn a_method_that_would_blow_up_cardinality_becomes_unknown() {
        // This layer sits above tonic's router, so it labels paths no method will serve.
        assert_eq!(extract_grpc_method("/pkg.Svc/not a method"), "unknown");
        assert_eq!(
            extract_grpc_method(&format!("/pkg.Svc/{}", "a".repeat(8192))).len(),
            MAX_HEADER_TAG_LEN
        );
    }

    #[test]
    fn a_client_name_that_would_blow_up_cardinality_becomes_unknown() {
        let named = |value: &str| {
            let request = Request::builder()
                .uri("/pkg.Svc/Method")
                .header("x-client-name", value)
                .body(())
                .unwrap();
            extract_client_name(&request)
        };

        assert_eq!(&*named("feature-flags"), "feature-flags");
        assert_eq!(&*named(""), "unknown");
        assert_eq!(&*named("bad name!"), "unknown");
        assert_eq!(named(&"b".repeat(200)).len(), MAX_HEADER_TAG_LEN);
        assert_eq!(
            &*extract_client_name(&grpc_request("/pkg.Svc/Method")),
            "unknown"
        );
    }
}