Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

23.2 KB
Raw Blame
//! Mirrors flushed billing counts into the usage-ingestion service. Records are emitted only for a
//! chunk already credited to Redis, so a Redis retry cannot bill twice.
//!
//! `record_id` is a fresh UUID per emission, not derived from the aggregation key: a Redis outage
//! rebuckets and merges requeued entries, so one key legitimately carries different quantities
//! across flushes. A retry reuses the record it already built, which is what makes retrying safe:
//! the service deduplicates on `record_id`, including when Kafka took only part of the batch.

use std::collections::HashSet;
use std::sync::Arc;
use std::sync::Mutex;

use common_metrics::inc;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use tonic::transport::{Channel, Endpoint};
use tonic::Code;
use usage_ingestion_proto::usage_ingestion::v1::{
    usage_ingestion_client::UsageIngestionClient, BillingUsageRecord, IngestBillingUsageRequest,
};
use uuid::Uuid;

use crate::config::TeamIdCollection;
use crate::flags::flag_request::FlagRequestType;
use crate::metrics::consts::{
    FLAGS_USAGE_RECORDS_FAILED, FLAGS_USAGE_RECORDS_SENT, FLAGS_USAGE_RETRIES,
};

use super::AggregationKey;

const PRODUCER_ID: &str = "feature-flags";
const UNIT: &str = "requests";
const MAX_BATCH_SIZE: usize = 500;
/// Queued sends waiting on the sender task. A full queue drops rather than growing
/// unboundedly; the drop is counted, and Redis still holds the authoritative count.
const QUEUE_CAPACITY: usize = 64;
/// Attempts per chunk, including the first. The sender is one task, so a chunk that
/// keeps failing holds up the queue behind it — hence a small number and a short wait.
const SEND_ATTEMPTS: u32 = 3;
const RETRY_BACKOFF: std::time::Duration = std::time::Duration::from_millis(100);

struct SenderState {
    queue: mpsc::Sender<Vec<BillingUsageRecord>>,
    task: JoinHandle<()>,
}

pub struct UsageReporter {
    teams: TeamIdCollection,
    state: Mutex<Option<SenderState>>,
}

impl UsageReporter {
    /// `None` when no URL is configured or no team is enabled, so the aggregator
    /// skips the work entirely.
    pub fn new(
        addr: &str,
        use_tls: bool,
        teams: TeamIdCollection,
        timeout_ms: u64,
    ) -> Result<Option<Arc<Self>>, String> {
        if addr.is_empty() || matches!(teams, TeamIdCollection::None) {
            return Ok(None);
        }
        // tonic is built here without its TLS feature, so an https endpoint fails at connect
        // time with nothing naming the cause. Refuse at startup instead.
        if use_tls {
            return Err("USAGE_INGESTION_TLS is not supported by the flags reporter".to_string());
        }
        let endpoint = Endpoint::from_shared(format!("http://{addr}"))
            .map_err(|e| format!("invalid USAGE_INGESTION_ADDR: {e}"))?
            .timeout(std::time::Duration::from_millis(timeout_ms));
        let client = UsageIngestionClient::new(endpoint.connect_lazy());

        // One owned task drains the queue, so concurrency is bounded by construction and
        // `shutdown` has something to await. Spawning per flush chunk left thousands of
        // untracked tasks that a shutdown could not drain.
        let (queue, receiver) = mpsc::channel(QUEUE_CAPACITY);
        let task = tokio::spawn(run_sender(client, receiver));

        Ok(Some(Arc::new(Self {
            teams,
            state: Mutex::new(Some(SenderState { queue, task })),
        })))
    }

    /// `timestamp_ms` must be the reporter's own clock, never anything derived from a request.
    /// toDate of it is part of the storage sorting key, so a caller-controlled value would
    /// decide whether these records deduplicate.
    pub fn report(&self, entries: &[(AggregationKey, u64)], timestamp_ms: i64) {
        let records = build_records(entries, &self.teams, timestamp_ms);
        if records.is_empty() {
            return;
        }
        let record_count = records.len() as u64;
        let queued = match self.state.lock().unwrap().as_ref() {
            Some(state) => state.queue.try_send(records).is_ok(),
            None => false,
        };
        if !queued {
            inc(
                FLAGS_USAGE_RECORDS_FAILED,
                &error_code("queue_full"),
                record_count,
            );
            tracing::warn!(
                records = record_count,
                "usage-ingestion send queue is full or closed; dropped usage records"
            );
        }
    }

    /// Closes the queue and awaits the sender so a graceful shutdown does not lose records
    /// the Redis flush already credited. Bounded by `timeout`.
    pub async fn shutdown(&self, timeout: std::time::Duration) {
        let state = self.state.lock().unwrap().take();
        let Some(SenderState { queue, task }) = state else {
            return;
        };
        // Dropping the only sender closes the channel, so the task drains its queue and returns.
        drop(queue);
        let abort = task.abort_handle();
        match tokio::time::timeout(timeout, task).await {
            Ok(_) => tracing::info!("UsageReporter: drained pending usage records"),
            Err(_) => {
                abort.abort();
                tracing::warn!(
                    timeout_ms = timeout.as_millis() as u64,
                    "UsageReporter: timed out draining usage records"
                );
            }
        }
    }
}

/// A code the service returns for a condition that clears on its own: an unreachable
/// or draining pod, a timeout, a full queue. The rest, `invalid_argument` above all,
/// describe the records themselves and would fail the same way forever.
///
/// The last three are how a connection that went away underneath the call reaches us
/// rather than anything the service decided. The service ends every connection with
/// GOAWAY at max_connection_age, and tonic maps that h2 NO_ERROR to `internal`, an
/// RST_STREAM CANCEL to `cancelled`, and a transport error it does not recognise to
/// `unknown`. Retrying is safe because the service deduplicates on `record_id`.
fn is_retryable(code: Code) -> bool {
    matches!(
        code,
        Code::Unavailable
            | Code::DeadlineExceeded
            | Code::ResourceExhausted
            | Code::Aborted
            | Code::Internal
            | Code::Cancelled
            | Code::Unknown
    )
}

/// A stable label value for `code`. `Display` gives a human sentence, which makes a
/// useless metric label.
fn code_label(code: Code) -> &'static str {
    match code {
        Code::Cancelled => "cancelled",
        Code::Unknown => "unknown",
        Code::InvalidArgument => "invalid_argument",
        Code::DeadlineExceeded => "deadline_exceeded",
        Code::NotFound => "not_found",
        Code::AlreadyExists => "already_exists",
        Code::PermissionDenied => "permission_denied",
        Code::ResourceExhausted => "resource_exhausted",
        Code::FailedPrecondition => "failed_precondition",
        Code::Aborted => "aborted",
        Code::OutOfRange => "out_of_range",
        Code::Unimplemented => "unimplemented",
        Code::Internal => "internal",
        Code::Unavailable => "unavailable",
        Code::DataLoss => "data_loss",
        Code::Unauthenticated => "unauthenticated",
        Code::Ok => "ok",
    }
}

fn error_code(value: &'static str) -> [(String, String); 1] {
    [("error_code".to_string(), value.to_string())]
}

async fn run_sender(
    client: UsageIngestionClient<Channel>,
    mut receiver: mpsc::Receiver<Vec<BillingUsageRecord>>,
) {
    while let Some(records) = receiver.recv().await {
        for chunk in records.chunks(MAX_BATCH_SIZE) {
            send_chunk(&client, chunk).await;
        }
    }
}

/// Retries the same records, never rebuilt ones, so the service's `record_id` dedup holds.
/// A bounded retry is the ceiling here: a crash still loses what the queue holds, and Redis
/// plus the nightly report stay authoritative for what a team owes.
async fn send_chunk(client: &UsageIngestionClient<Channel>, chunk: &[BillingUsageRecord]) {
    let count = chunk.len() as u64;
    let message = IngestBillingUsageRequest {
        records: chunk.to_vec(),
    };
    for attempt in 1..=SEND_ATTEMPTS {
        let mut client = client.clone();
        match client.ingest_billing_usage(tagged(message.clone())).await {
            Ok(response) => {
                let accepted = response.into_inner().accepted_record_ids;
                inc(FLAGS_USAGE_RECORDS_SENT, &[], accepted.len() as u64);
                // The service drops records it cannot attribute rather than failing the
                // batch, so a success can still leave some behind. Retrying them would fail
                // the same way, and the teams are the only lead worth logging.
                if accepted.len() < chunk.len() {
                    inc(
                        FLAGS_USAGE_RECORDS_FAILED,
                        &error_code("rejected"),
                        count - accepted.len() as u64,
                    );
                    tracing::warn!(
                        records = count - accepted.len() as u64,
                        teams = ?rejected_teams(chunk, &accepted),
                        "usage-ingestion rejected feature flag usage records"
                    );
                }
                return;
            }
            Err(status) => {
                let label = code_label(status.code());
                let retryable = is_retryable(status.code());
                if !retryable || attempt == SEND_ATTEMPTS {
                    inc(FLAGS_USAGE_RECORDS_FAILED, &error_code(label), count);
                    tracing::warn!(
                        records = count,
                        attempts = attempt,
                        code = %status.code(),
                        teams = ?distinct_teams(chunk),
                        "failed to report feature flag usage records"
                    );
                    return;
                }
                // A retry that lands drops nothing, so it reads on its own counter.
                inc(FLAGS_USAGE_RETRIES, &error_code(label), 1);
                tokio::time::sleep(RETRY_BACKOFF * attempt).await;
            }
        }
    }
}

/// Names this producer in the service's own request metrics, which otherwise aggregate every
/// caller into one series.
fn tagged(message: IngestBillingUsageRequest) -> tonic::Request<IngestBillingUsageRequest> {
    let mut request = tonic::Request::new(message);
    request.metadata_mut().insert(
        "x-client-name",
        tonic::metadata::MetadataValue::from_static(PRODUCER_ID),
    );
    request
}

/// The teams behind the records the service did not accept, deduplicated so a chunk that
/// repeats a team reads as one lead rather than many.
fn rejected_teams(chunk: &[BillingUsageRecord], accepted: &[String]) -> Vec<i64> {
    let accepted: HashSet<&str> = accepted.iter().map(String::as_str).collect();
    dedup(
        chunk
            .iter()
            .filter(|record| !accepted.contains(record.record_id.as_str()))
            .map(|record| record.team_id),
    )
}

fn distinct_teams(chunk: &[BillingUsageRecord]) -> Vec<i64> {
    dedup(chunk.iter().map(|record| record.team_id))
}

fn dedup(teams: impl Iterator<Item = i64>) -> Vec<i64> {
    let mut teams = teams.collect::<Vec<_>>();
    teams.sort_unstable();
    teams.dedup();
    teams
}

/// Each billable request type has its own key because the nightly report prices them apart:
/// one local evaluation costs ten decide requests, and one 304 costs one. A 304 does not use
/// the decide key even at the same price, because each key feeds its own usage counter and a
/// shared key would count the 304 twice. The weighting itself stays out of here: a producer
/// reports what happened, and pricing is applied downstream.
fn usage_key(request_type: FlagRequestType) -> &'static str {
    match request_type {
        FlagRequestType::Decide => "feature_flag_requests",
        FlagRequestType::FlagDefinitions => "feature_flag_local_evaluation_requests",
        FlagRequestType::FlagDefinitionsNotModified => {
            "feature_flag_local_evaluation_not_modified_requests"
        }
    }
}

fn build_records(
    entries: &[(AggregationKey, u64)],
    teams: &TeamIdCollection,
    timestamp_ms: i64,
) -> Vec<BillingUsageRecord> {
    entries
        .iter()
        .filter(|(key, count)| *count > 0 && teams.includes_team(key.team_id))
        .map(|(key, count)| BillingUsageRecord {
            record_id: Uuid::now_v7().to_string(),
            producer_id: PRODUCER_ID.to_string(),
            team_id: i64::from(key.team_id),
            usage_key: usage_key(key.request_type).to_string(),
            unit: UNIT.to_string(),
            quantity: i64::try_from(*count).unwrap_or(i64::MAX),
            timestamp_ms,
        })
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::billing::usage_test_support::{serve, RecordingIngestion};
    use crate::handler::types::Library;
    use rstest::rstest;

    fn key(team_id: i32, library: Option<Library>) -> AggregationKey {
        AggregationKey {
            team_id,
            request_type: FlagRequestType::Decide,
            library,
            bucket: 7,
        }
    }

    #[test]
    fn build_records_keys_the_two_billable_request_types_apart() {
        // The report weighs one local evaluation as ten decide requests, so a shared key
        // would price them the same.
        let entries = vec![
            (key(1, None), 3),
            (
                AggregationKey {
                    request_type: FlagRequestType::FlagDefinitions,
                    ..key(1, None)
                },
                4,
            ),
            (
                AggregationKey {
                    request_type: FlagRequestType::FlagDefinitionsNotModified,
                    ..key(1, None)
                },
                5,
            ),
        ];
        let records = build_records(&entries, &TeamIdCollection::All, 1);

        assert_eq!(
            records
                .iter()
                .map(|record| (record.usage_key.as_str(), record.quantity))
                .collect::<Vec<_>>(),
            vec![
                ("feature_flag_requests", 3),
                ("feature_flag_local_evaluation_requests", 4),
                ("feature_flag_local_evaluation_not_modified_requests", 5)
            ]
        );
    }

    #[test]
    fn build_records_applies_the_team_filter() {
        let entries = vec![(key(1, None), 5), (key(2, None), 9)];
        let records = build_records(
            &entries,
            &TeamIdCollection::TeamIds(vec![2]),
            1_700_000_000_000,
        );

        assert_eq!(records.len(), 1);
        assert_eq!(records[0].team_id, 2);
        assert_eq!(records[0].quantity, 9);
    }

    #[test]
    fn build_records_skips_zero_counts() {
        let entries = vec![(key(1, None), 0)];
        assert!(build_records(&entries, &TeamIdCollection::All, 1).is_empty());
    }

    #[test]
    fn build_records_gives_every_emission_its_own_id() {
        let entries = vec![(key(1, None), 1), (key(2, None), 1)];
        let records = build_records(&entries, &TeamIdCollection::All, 1);

        assert_ne!(records[0].record_id, records[1].record_id);
    }

    #[tokio::test]
    async fn shutdown_is_idempotent_and_closes_the_queue() {
        let reporter = UsageReporter::new("127.0.0.1:1", false, TeamIdCollection::All, 50)
            .unwrap()
            .unwrap();

        reporter
            .shutdown(std::time::Duration::from_millis(200))
            .await;
        // A second shutdown must not panic on the already-taken sender, and a late report must be
        // counted as dropped rather than queued onto a closed channel.
        reporter
            .shutdown(std::time::Duration::from_millis(200))
            .await;
        reporter.report(&[(key(1, None), 1)], 1);
    }

    #[tokio::test]
    async fn new_is_disabled_without_an_address_or_teams() {
        assert!(UsageReporter::new("", false, TeamIdCollection::All, 100)
            .unwrap()
            .is_none());
        assert!(
            UsageReporter::new("localhost:7143", false, TeamIdCollection::None, 100)
                .unwrap()
                .is_none()
        );
        assert!(
            UsageReporter::new("localhost:7143", false, TeamIdCollection::All, 100)
                .unwrap()
                .is_some()
        );
    }

    #[tokio::test]
    async fn new_refuses_tls_it_cannot_honor() {
        assert!(UsageReporter::new("localhost:7143", true, TeamIdCollection::All, 100).is_err());
    }

    async fn reporter_for(addr: std::net::SocketAddr) -> Arc<UsageReporter> {
        UsageReporter::new(&addr.to_string(), false, TeamIdCollection::All, 1_000)
            .unwrap()
            .unwrap()
    }

    /// `shutdown` is the drain, so it also stands in for a wait: it returns once the sender
    /// task has emptied the queue, which is what the aggregator's ordering promises.
    #[tokio::test]
    async fn a_queued_record_reaches_the_service_by_shutdown() {
        let service = RecordingIngestion::default();
        let reporter = reporter_for(serve(service.clone()).await).await;

        reporter.report(&[(key(7, None), 4)], 1_700_000_000_000);
        reporter.shutdown(std::time::Duration::from_secs(5)).await;

        let requests = service.requests();
        assert_eq!(requests.len(), 1);
        assert_eq!(requests[0].len(), 1);
        assert_eq!(requests[0][0].team_id, 7);
        assert_eq!(requests[0][0].quantity, 4);
        assert_eq!(requests[0][0].producer_id, PRODUCER_ID);
        assert_eq!(requests[0][0].usage_key, usage_key(FlagRequestType::Decide));
    }

    #[tokio::test]
    async fn every_request_names_this_producer_to_the_service() {
        // Without the header the service's request metrics read client=unknown for everyone.
        let service = RecordingIngestion::default();
        let reporter = reporter_for(serve(service.clone()).await).await;

        reporter.report(&[(key(7, None), 1)], 1_700_000_000_000);
        reporter.shutdown(std::time::Duration::from_secs(5)).await;

        assert_eq!(service.client_names(), vec![Some(PRODUCER_ID.to_string())]);
    }

    // The last three are how a connection that went away underneath the call reaches us,
    // not something the service decided. Dropping them loses usage the service never saw.
    // Each case fails twice, so the whole budget runs rather than one attempt of it.
    #[rstest]
    #[case::draining_pod(Code::Unavailable)]
    #[case::goaway_at_max_connection_age(Code::Internal)]
    #[case::reset_stream(Code::Cancelled)]
    #[case::unrecognised_transport_error(Code::Unknown)]
    #[tokio::test]
    async fn retries_a_transient_failure_with_the_same_record_id(#[case] code: Code) {
        let service = RecordingIngestion::default();
        service.fail_next(code);
        service.fail_next(code);
        let reporter = reporter_for(serve(service.clone()).await).await;

        reporter.report(&[(key(7, None), 1)], 1_700_000_000_000);
        reporter.shutdown(std::time::Duration::from_secs(5)).await;

        let requests = service.requests();
        assert_eq!(requests.len(), SEND_ATTEMPTS as usize);
        // Reusing the ID is what makes the retry safe: the service deduplicates on it, so a
        // batch Kafka took only part of does not bill twice.
        assert_eq!(requests[0][0].record_id, requests[2][0].record_id);
    }

    /// Counter totals by name, captured while the reporter ran. The recorder is
    /// thread-scoped, so the sender task has to be driven on this thread.
    fn counted(run: impl FnOnce(&tokio::runtime::Runtime)) -> Vec<(String, u64)> {
        let recorder = metrics_util::debugging::DebuggingRecorder::new();
        let snapshotter = recorder.snapshotter();
        let runtime = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()
            .unwrap();

        metrics::with_local_recorder(&recorder, || run(&runtime));

        let mut totals = snapshotter
            .snapshot()
            .into_vec()
            .into_iter()
            .filter_map(|(key, _, _, value)| match value {
                metrics_util::debugging::DebugValue::Counter(count) => {
                    Some((key.key().name().to_string(), count))
                }
                _ => None,
            })
            .collect::<Vec<_>>();
        totals.sort();
        totals
    }

    #[test]
    fn a_partially_accepted_batch_counts_only_what_the_service_took() {
        let totals = counted(|runtime| {
            runtime.block_on(async {
                let service = RecordingIngestion::default();
                service.reject_team(8);
                let reporter = reporter_for(serve(service.clone()).await).await;

                reporter.report(
                    &[(key(7, None), 1), (key(8, None), 1), (key(9, None), 1)],
                    1_700_000_000_000,
                );
                reporter.shutdown(std::time::Duration::from_secs(5)).await;

                // The service already decided; re-sending would drop the same record again.
                assert_eq!(service.requests().len(), 1);
            });
        });

        assert_eq!(
            totals,
            vec![
                (FLAGS_USAGE_RECORDS_FAILED.to_string(), 1),
                (FLAGS_USAGE_RECORDS_SENT.to_string(), 2),
            ]
        );
    }

    #[test]
    fn a_fully_accepted_batch_counts_nothing_as_failed() {
        let totals = counted(|runtime| {
            runtime.block_on(async {
                let reporter = reporter_for(serve(RecordingIngestion::default()).await).await;

                reporter.report(&[(key(7, None), 1), (key(8, None), 1)], 1_700_000_000_000);
                reporter.shutdown(std::time::Duration::from_secs(5)).await;
            });
        });

        assert_eq!(totals, vec![(FLAGS_USAGE_RECORDS_SENT.to_string(), 2)]);
    }

    // A chunk the budget could not save is lost, so it has to read as one drop on the failed
    // counter and not once per attempt.
    #[test]
    fn a_chunk_that_exhausts_its_attempts_counts_as_failed_once() {
        let totals = counted(|runtime| {
            runtime.block_on(async {
                let service = RecordingIngestion::default();
                for _ in 0..SEND_ATTEMPTS {
                    service.fail_next(Code::Internal);
                }
                let reporter = reporter_for(serve(service.clone()).await).await;

                reporter.report(&[(key(7, None), 1)], 1_700_000_000_000);
                reporter.shutdown(std::time::Duration::from_secs(5)).await;

                assert_eq!(service.requests().len(), SEND_ATTEMPTS as usize);
            });
        });

        assert_eq!(
            totals,
            vec![
                (FLAGS_USAGE_RECORDS_FAILED.to_string(), 1),
                (FLAGS_USAGE_RETRIES.to_string(), 2),
            ]
        );
    }

    #[tokio::test]
    async fn does_not_retry_a_rejected_batch() {
        let service = RecordingIngestion::default();
        // The records themselves are wrong, so every attempt fails the same way.
        service.fail_next(Code::InvalidArgument);
        let reporter = reporter_for(serve(service.clone()).await).await;

        reporter.report(&[(key(7, None), 1)], 1_700_000_000_000);
        reporter.shutdown(std::time::Duration::from_secs(5)).await;

        assert_eq!(service.requests().len(), 1);
    }
}