Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

20.7 KB
Raw Blame
//! Shared client for the router's public PersonHogService.
//!
//! The router routes leader-bound calls by hashing the person routing
//! headers — it never decodes request bodies — so every caller must stamp
//! `x-team-id`/`x-person-id` on property writes and strong reads. This
//! client owns that contract so callers cannot get it wrong.

use std::future::Future;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};

use metrics::{counter, histogram};
use tonic::metadata::MetadataValue;
use tonic::transport::{Channel, Endpoint};
use tonic::{Request, Response, Status};

use crate::grpc::{code_as_str, CLIENT_NAME_HEADER};
use crate::h2_window::Http2Windows;

use personhog_proto::personhog::service::v1::person_hog_service_client::PersonHogServiceClient;
use personhog_proto::personhog::types::v1::{
    ConsistencyLevel, FencePersonRequest, FencePersonResponse, FencePersonsRequest,
    FencePersonsResponse, FoldPersonDocumentRequest, FoldPersonDocumentResponse, GetPersonRequest,
    Person, ReadOptions, ReleaseFenceRequest, ReleaseFenceResponse, ReleaseFencesRequest,
    ReleaseFencesResponse, UpdatePersonPropertiesRequest, UpdatePersonPropertiesResponse,
};

/// Routing headers for leader-bound calls through the router.
pub const TEAM_ID_HEADER: &str = "x-team-id";
pub const PERSON_ID_HEADER: &str = "x-person-id";
pub const READ_CONSISTENCY_HEADER: &str = "x-read-consistency";

const CONNECT_TIMEOUT: Duration = Duration::from_secs(5);

const RETRY_BACKOFF: Duration = Duration::from_millis(100);

pub const ROUTER_CLIENT_RETRIES_TOTAL: &str = "personhog_router_client_retries_total";

/// The channel's pod went away, the request timed out, or the router ran out of routes.
fn is_transport_failure(status: &Status) -> bool {
    matches!(
        status.code(),
        tonic::Code::Unavailable | tonic::Code::Unknown | tonic::Code::Cancelled
    )
}

pub const ROUTER_CLIENT_CALLS_TOTAL: &str = "personhog_router_client_calls_total";
pub const ROUTER_CLIENT_CALL_DURATION_MS: &str = "personhog_router_client_call_duration_ms";

/// Channels opened per router URL when the caller does not choose. One is
/// correct only for low-rate callers; see `with_channels`.
pub const DEFAULT_ROUTER_CHANNELS: usize = 1;

#[derive(Clone)]
pub struct RouterClient {
    /// One tonic channel is one TCP connection, and a Kubernetes Service
    /// pins a connection to a single backend pod for its lifetime. A
    /// single-channel client therefore sends everything this process
    /// produces to one router pod, however many pods exist. Holding
    /// several channels spreads the load across as many pods, and gives
    /// each connection its own HTTP/2 stream budget and flow-control
    /// windows.
    clients: Vec<PersonHogServiceClient<Channel>>,
    /// Shared across clones so round-robin continues across them rather
    /// than every clone restarting at the first channel.
    next: Arc<AtomicUsize>,
    request_timeout: Duration,
    attempts: u32,
    client_name: Option<&'static str>,
}

impl RouterClient {
    /// Connect lazily to the router over a single channel; the first RPC
    /// establishes the connection.
    pub fn new(
        router_url: &str,
        request_timeout: Duration,
    ) -> Result<Self, tonic::transport::Error> {
        Self::with_channels(router_url, request_timeout, DEFAULT_ROUTER_CHANNELS)
    }

    /// Connect lazily over `channels` connections, selected round-robin
    /// per request. Callers driving meaningful request rates want more
    /// than one: a single connection caps them at one router pod's
    /// capacity no matter how many pods are running. A count below one is
    /// treated as one.
    pub fn with_channels(
        router_url: &str,
        request_timeout: Duration,
        channels: usize,
    ) -> Result<Self, tonic::transport::Error> {
        Self::with_channels_and_windows(
            router_url,
            request_timeout,
            channels,
            Http2Windows::default(),
        )
    }

    pub fn with_channels_and_windows(
        router_url: &str,
        request_timeout: Duration,
        channels: usize,
        http2_windows: Http2Windows,
    ) -> Result<Self, tonic::transport::Error> {
        let clients = (0..channels.max(1))
            .map(|_| {
                let endpoint = Endpoint::from_shared(router_url.to_string())?
                    .connect_timeout(CONNECT_TIMEOUT)
                    .tcp_nodelay(true);
                let channel = http2_windows.apply_to_endpoint(endpoint).connect_lazy();
                Ok(PersonHogServiceClient::new(channel))
            })
            .collect::<Result<Vec<_>, tonic::transport::Error>>()?;
        Ok(Self {
            clients,
            next: Arc::new(AtomicUsize::new(0)),
            request_timeout,
            attempts: 1,
            client_name: None,
        })
    }

    /// Stamp every request with `x-client-name`, so the router and leader
    /// attribute their server-side metrics to this caller instead of
    /// `unknown`.
    pub fn with_client_name(mut self, name: &'static str) -> Self {
        self.client_name = Some(name);
        self
    }

    /// Retries a strong read or property write on a transport failure, each attempt on the next
    /// channel. Both are safe to repeat: the leader answers an applied write with no change.
    pub fn with_attempts(mut self, attempts: u32) -> Self {
        self.attempts = attempts.max(1);
        self
    }

    fn request<T>(&self, message: T) -> Request<T> {
        let mut request = Request::new(message);
        request.set_timeout(self.request_timeout);
        if let Some(name) = self.client_name {
            request
                .metadata_mut()
                .insert(CLIENT_NAME_HEADER, MetadataValue::from_static(name));
        }
        request
    }

    async fn timed<T, F>(method: &'static str, call: F) -> Result<T, Status>
    where
        F: Future<Output = Result<Response<T>, Status>>,
    {
        let start = Instant::now();
        let result = call.await;
        let outcome: &'static str = match &result {
            Ok(_) => "ok",
            Err(status) => code_as_str(status.code()),
        };
        let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
        counter!(ROUTER_CLIENT_CALLS_TOTAL, "method" => method, "outcome" => outcome).increment(1);
        histogram!(ROUTER_CLIENT_CALL_DURATION_MS, "method" => method, "outcome" => outcome)
            .record(duration_ms);
        result.map(Response::into_inner)
    }

    /// The next channel in round-robin order. Selection is load-oblivious:
    /// a stalled connection still takes its share, which is acceptable
    /// because every channel targets the same interchangeable pod set.
    fn client(&self) -> PersonHogServiceClient<Channel> {
        let idx = self.next.fetch_add(1, Ordering::Relaxed) % self.clients.len();
        self.clients[idx].clone()
    }

    /// Up to `attempts` calls, each timed, stopping at the first answer that is not a transport failure.
    async fn retried<T, F, Fut>(
        attempts: u32,
        method: &'static str,
        mut call: F,
    ) -> Result<T, Status>
    where
        F: FnMut() -> Fut,
        Fut: Future<Output = Result<Response<T>, Status>>,
    {
        let mut attempt = 1;
        loop {
            match Self::timed(method, call()).await {
                Err(status) if attempt < attempts && is_transport_failure(&status) => {
                    counter!(ROUTER_CLIENT_RETRIES_TOTAL, "method" => method, "outcome" => code_as_str(status.code()))
                        .increment(1);
                    tokio::time::sleep(RETRY_BACKOFF * attempt).await;
                    attempt += 1;
                }
                result => return result,
            }
        }
    }

    /// Leader-routed property write. The routing headers are stamped from
    /// the request's own team_id/person_id.
    pub async fn update_person_properties(
        &self,
        request: UpdatePersonPropertiesRequest,
    ) -> Result<UpdatePersonPropertiesResponse, Status> {
        Self::retried(self.attempts, "UpdatePersonProperties", || {
            let request = self.build_update_request(request.clone());
            let mut client = self.client();
            async move { client.update_person_properties(request).await }
        })
        .await
    }

    /// Person read. Strong reads route to the owning leader and therefore
    /// carry the routing headers plus the strong-consistency opt-in;
    /// eventual reads go to the replica with no headers.
    pub async fn get_person(
        &self,
        team_id: i64,
        person_id: i64,
        consistency: ConsistencyLevel,
    ) -> Result<Option<Person>, Status> {
        Self::retried(self.attempts, "GetPerson", || {
            let request = self.build_get_person_request(team_id, person_id, consistency);
            let mut client = self.client();
            async move { client.get_person(request).await }
        })
        .await
        .map(|response| response.person)
    }

    /// Leader-routed lifecycle fence (saga runner only): freeze the person
    /// and return its sealed state.
    pub async fn fence_person(
        &self,
        request: FencePersonRequest,
    ) -> Result<FencePersonResponse, Status> {
        let (team_id, person_id) = (request.team_id, request.person_id);
        let mut request = self.request(request);
        stamp_person_routing_headers(&mut request, team_id, person_id);
        Self::timed("FencePerson", self.client().fence_person(request)).await
    }

    /// Leader-routed batch fence (saga runner only) for persons that share
    /// one partition: the routing headers name the first person, and the
    /// leader refuses the batch if any person hashes elsewhere.
    pub async fn fence_persons(
        &self,
        request: FencePersonsRequest,
    ) -> Result<FencePersonsResponse, Status> {
        let Some(first) = request.person_ids.first() else {
            return Err(Status::invalid_argument(
                "FencePersons needs at least one person",
            ));
        };
        let (team_id, person_id) = (request.team_id, *first);
        let mut request = self.request(request);
        stamp_person_routing_headers(&mut request, team_id, person_id);
        Self::timed("FencePersons", self.client().fence_persons(request)).await
    }

    /// Leader-routed fence release (saga runner only): committed produces
    /// the death document, aborted resumes the person's normal life.
    pub async fn release_fence(
        &self,
        request: ReleaseFenceRequest,
    ) -> Result<ReleaseFenceResponse, Status> {
        let (team_id, person_id) = (request.team_id, request.person_id);
        let mut request = self.request(request);
        stamp_person_routing_headers(&mut request, team_id, person_id);
        Self::timed("ReleaseFence", self.client().release_fence(request)).await
    }

    /// Leader-routed batch release (saga runner only) for persons that
    /// share one partition: the routing headers name the first person,
    /// and the leader refuses the batch if any person hashes elsewhere.
    pub async fn release_fences(
        &self,
        request: ReleaseFencesRequest,
    ) -> Result<ReleaseFencesResponse, Status> {
        let Some(first) = request.persons.first() else {
            return Err(Status::invalid_argument(
                "ReleaseFences needs at least one person",
            ));
        };
        let (team_id, person_id) = (request.team_id, first.person_id);
        let mut request = self.request(request);
        stamp_person_routing_headers(&mut request, team_id, person_id);
        Self::timed("ReleaseFences", self.client().release_fences(request)).await
    }

    /// Leader-routed merge fold (saga runner only): fold sealed source
    /// snapshots into the target's document and return the folded result.
    pub async fn fold_person_document(
        &self,
        request: FoldPersonDocumentRequest,
    ) -> Result<FoldPersonDocumentResponse, Status> {
        let (team_id, person_id) = (request.team_id, request.person_id);
        let mut request = self.request(request);
        stamp_person_routing_headers(&mut request, team_id, person_id);
        Self::timed(
            "FoldPersonDocument",
            self.client().fold_person_document(request),
        )
        .await
    }

    fn build_update_request(
        &self,
        request: UpdatePersonPropertiesRequest,
    ) -> Request<UpdatePersonPropertiesRequest> {
        let (team_id, person_id) = (request.team_id, request.person_id);
        let mut request = self.request(request);
        stamp_person_routing_headers(&mut request, team_id, person_id);
        request
    }

    fn build_get_person_request(
        &self,
        team_id: i64,
        person_id: i64,
        consistency: ConsistencyLevel,
    ) -> Request<GetPersonRequest> {
        let mut request = self.request(GetPersonRequest {
            team_id,
            person_id,
            read_options: Some(ReadOptions {
                consistency: consistency.into(),
                ..Default::default()
            }),
        });
        if consistency == ConsistencyLevel::Strong {
            stamp_person_routing_headers(&mut request, team_id, person_id);
            request.metadata_mut().insert(
                READ_CONSISTENCY_HEADER,
                tonic::metadata::MetadataValue::from_static("strong"),
            );
        }
        request
    }
}

/// Stamp the person routing headers the router hashes for leader routing.
pub fn stamp_person_routing_headers<T>(request: &mut Request<T>, team_id: i64, person_id: i64) {
    let metadata = request.metadata_mut();
    metadata.insert(
        TEAM_ID_HEADER,
        team_id
            .to_string()
            .parse()
            .expect("numeric header is valid metadata"),
    );
    metadata.insert(
        PERSON_ID_HEADER,
        person_id
            .to_string()
            .parse()
            .expect("numeric header is valid metadata"),
    );
}

/// The router fails leader-bound calls closed on missing headers and would
/// misroute on wrong ones, so the header decisions per routing branch are
/// pinned here: leader-bound requests (writes, strong reads) must carry the
/// person routing headers, replica-bound reads must not.
#[cfg(test)]
mod tests {
    use rstest::rstest;

    use super::*;

    fn client() -> RouterClient {
        RouterClient::new("http://127.0.0.1:1", Duration::from_secs(1))
            .expect("lazy client construction cannot fail on a valid url")
    }

    /// A single connection is pinned to one router pod by the Service, so
    /// a pooled client must actually hand out different channels — and
    /// keep doing so across clones, which is how callers hold it.
    #[tokio::test]
    async fn requests_round_robin_across_the_pool_including_clones() {
        let client = RouterClient::with_channels("http://127.0.0.1:1", Duration::from_secs(1), 4)
            .expect("lazy client construction cannot fail on a valid url");

        let first = client.next.load(Ordering::Relaxed);
        for _ in 0..4 {
            drop(client.clone().client());
        }
        assert_eq!(client.next.load(Ordering::Relaxed), first + 4);
    }

    /// Zero would panic on the modulo; callers reading it from config get
    /// the single-channel behavior instead.
    #[tokio::test]
    async fn a_channel_count_below_one_still_yields_a_usable_client() {
        let client = RouterClient::with_channels("http://127.0.0.1:1", Duration::from_secs(1), 0)
            .expect("lazy client construction cannot fail on a valid url");
        assert_eq!(client.clients.len(), 1);
        drop(client.client());
    }

    // Channel construction (even lazy) needs a Tokio reactor, hence async tests.
    #[tokio::test]
    async fn update_request_carries_the_routing_headers() {
        let request = client().build_update_request(UpdatePersonPropertiesRequest {
            force_update: false,
            team_id: 7,
            person_id: 42,
            ..Default::default()
        });
        let metadata = request.metadata();
        assert_eq!(metadata.get(TEAM_ID_HEADER).unwrap(), "7");
        assert_eq!(metadata.get(PERSON_ID_HEADER).unwrap(), "42");
        assert!(metadata.get(READ_CONSISTENCY_HEADER).is_none());
    }

    /// The eventual-read branch carries no routing headers, so it is the
    /// branch most likely to lose the client name in a request-building
    /// refactor; an unnamed client must stay anonymous so the server label
    /// falls back to `unknown` rather than an empty string.
    #[tokio::test]
    async fn client_name_is_stamped_only_when_configured() {
        let anonymous = client().build_update_request(UpdatePersonPropertiesRequest::default());
        assert!(anonymous.metadata().get(CLIENT_NAME_HEADER).is_none());

        let named = client()
            .with_client_name("personhog-identity")
            .build_get_person_request(7, 42, ConsistencyLevel::Eventual);
        assert_eq!(
            named.metadata().get(CLIENT_NAME_HEADER).unwrap(),
            "personhog-identity"
        );
    }

    #[rstest]
    #[case::strong(ConsistencyLevel::Strong, true)]
    #[case::eventual(ConsistencyLevel::Eventual, false)]
    #[case::unspecified(ConsistencyLevel::Unspecified, false)]
    #[tokio::test]
    async fn get_person_request_headers_follow_the_routing_branch(
        #[case] consistency: ConsistencyLevel,
        #[case] leader_bound: bool,
    ) {
        let request = client().build_get_person_request(7, 42, consistency);
        let metadata = request.metadata();
        if leader_bound {
            assert_eq!(metadata.get(TEAM_ID_HEADER).unwrap(), "7");
            assert_eq!(metadata.get(PERSON_ID_HEADER).unwrap(), "42");
            assert_eq!(metadata.get(READ_CONSISTENCY_HEADER).unwrap(), "strong");
        } else {
            assert!(metadata.get(TEAM_ID_HEADER).is_none());
            assert!(metadata.get(PERSON_ID_HEADER).is_none());
            assert!(metadata.get(READ_CONSISTENCY_HEADER).is_none());
        }
    }

    /// Answers the scripted statuses in order, then succeeds; counts calls.
    fn scripted(
        failures: Vec<tonic::Code>,
        calls: Arc<AtomicUsize>,
    ) -> impl FnMut() -> std::future::Ready<Result<Response<u8>, Status>> {
        let mut failures = failures.into_iter();
        move || {
            calls.fetch_add(1, Ordering::Relaxed);
            std::future::ready(match failures.next() {
                Some(code) => Err(Status::new(code, "scripted")),
                None => Ok(Response::new(1)),
            })
        }
    }

    #[rstest]
    #[case::unavailable(2, vec![tonic::Code::Unavailable])]
    #[case::unknown(2, vec![tonic::Code::Unknown])]
    #[case::cancelled(2, vec![tonic::Code::Cancelled])]
    #[case::two_failures_under_three_attempts(3, vec![tonic::Code::Unavailable, tonic::Code::Cancelled])]
    #[tokio::test]
    async fn transport_failures_are_retried_up_to_the_attempt_count(
        #[case] attempts: u32,
        #[case] failures: Vec<tonic::Code>,
    ) {
        let calls = Arc::new(AtomicUsize::new(0));
        let expected_calls = failures.len() + 1;

        let result =
            RouterClient::retried(attempts, "test", scripted(failures, calls.clone())).await;

        assert_eq!(result.unwrap(), 1);
        assert_eq!(calls.load(Ordering::Relaxed), expected_calls);
    }

    #[rstest]
    #[case::not_found(tonic::Code::NotFound)]
    #[case::failed_precondition(tonic::Code::FailedPrecondition)]
    #[case::invalid_argument(tonic::Code::InvalidArgument)]
    #[tokio::test]
    async fn an_answer_about_the_request_is_not_retried(#[case] code: tonic::Code) {
        let calls = Arc::new(AtomicUsize::new(0));

        let result = RouterClient::retried(2, "test", scripted(vec![code], calls.clone())).await;

        assert_eq!(result.unwrap_err().code(), code);
        assert_eq!(calls.load(Ordering::Relaxed), 1);
    }

    #[tokio::test]
    async fn a_single_attempt_fails_on_the_first_transport_failure() {
        let calls = Arc::new(AtomicUsize::new(0));

        let result = RouterClient::retried(
            client().attempts,
            "test",
            scripted(vec![tonic::Code::Unavailable], calls.clone()),
        )
        .await;

        assert_eq!(result.unwrap_err().code(), tonic::Code::Unavailable);
        assert_eq!(calls.load(Ordering::Relaxed), 1);
    }

    #[tokio::test]
    async fn exhausted_attempts_return_the_last_failure() {
        let calls = Arc::new(AtomicUsize::new(0));

        let result = RouterClient::retried(
            2,
            "test",
            scripted(
                vec![tonic::Code::Unavailable, tonic::Code::Cancelled],
                calls.clone(),
            ),
        )
        .await;

        assert_eq!(result.unwrap_err().code(), tonic::Code::Cancelled);
        assert_eq!(calls.load(Ordering::Relaxed), 2);
    }
}