:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Records dedupe by (team, usage key, record id), retry on transient codes, and never fail the batch.
use std::collections::HashSet;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use async_trait::async_trait;
use tonic::transport::Channel;
use tonic::Code;
use usage_ingestion_proto::usage_ingestion::v1::usage_ingestion_client::UsageIngestionClient;
use usage_ingestion_proto::usage_ingestion::v1::{BillingUsageRecord, IngestBillingUsageRequest};
use crate::pipeline::UsageRecordBatch;
use crate::property_filter::coerce::js_parse_int;
use crate::rate_limiter::Clock;
use super::config::UsageConfig;
const MAX_ATTEMPTS: u32 = 3;
const RETRY_BACKOFF_MS: u64 = 100;
#[derive(Debug, Clone)]
struct Pending {
team_id: i64,
usage_key: String,
record_id: String,
quantity: f64,
unit: String,
}
#[derive(Default)]
struct Records {
keys: HashSet<String>,
list: Vec<Pending>,
}
pub struct GrpcUsageRecordBatch {
client: Option<UsageIngestionClient<Channel>>,
producer_id: Arc<str>,
report_teams: String,
max_batch_size: usize,
timeout: Duration,
records: Mutex<Records>,
clock: Clock,
}
/// Only an exact `*` matches all, and each item is `parseInt(item)` with no radix.
fn report_teams_match(raw: &str, team_id: i64) -> bool {
if raw.trim().is_empty() {
return false;
}
if raw == "*" {
return true;
}
raw.split(',')
.any(|item| js_parse_int(item, None) == team_id as f64)
}
impl GrpcUsageRecordBatch {
/// No address or no report teams means no client.
pub fn new(config: &UsageConfig, producer_id: &str, clock: Clock) -> anyhow::Result<Self> {
let client = if config.addr.is_empty() || config.report_teams.is_empty() {
None
} else {
if config.tls {
anyhow::bail!(
"USAGE_INGESTION_TLS=true is not supported by the Rust logs consumer"
);
}
let channel = Channel::from_shared(format!("http://{}", config.addr))?
.timeout(config.timeout)
.connect_lazy();
Some(UsageIngestionClient::new(channel))
};
Ok(Self {
client,
producer_id: Arc::from(producer_id),
report_teams: config.report_teams.clone(),
max_batch_size: config.max_batch_size.max(1),
timeout: config.timeout,
records: Mutex::new(Records::default()),
clock,
})
}
fn accepts(&self, team_id: i64) -> bool {
self.client.is_some() && report_teams_match(&self.report_teams, team_id)
}
async fn ingest_chunk(
&self,
client: &UsageIngestionClient<Channel>,
chunk: &[BillingUsageRecord],
) {
let mut attempt = 0;
loop {
let mut request = tonic::Request::new(IngestBillingUsageRequest {
records: chunk.to_vec(),
});
if let Ok(v) = self.producer_id.parse() {
request.metadata_mut().insert("x-client-name", v);
}
// connect-rpc's `defaultTimeoutMs` bounds the whole call, connecting included.
let mut client = client.clone();
let call = tokio::time::timeout(self.timeout, client.ingest_billing_usage(request));
let reply = match call.await {
Ok(reply) => reply,
Err(_) => Err(tonic::Status::deadline_exceeded(
"usage-ingestion call timed out",
)),
};
match reply {
Ok(response) => {
let accepted: HashSet<String> = response
.into_inner()
.accepted_record_ids
.into_iter()
.collect();
let mut rejected = 0;
for r in chunk {
let key: Arc<str> = Arc::from(r.usage_key.as_str());
if accepted.contains(&r.record_id) {
metrics::counter!("usage_ingestion_records_sent_total", "producer_id" => self.producer_id.clone(), "usage_key" => key).increment(1);
} else {
rejected += 1;
metrics::counter!("usage_ingestion_records_failed_total", "producer_id" => self.producer_id.clone(), "usage_key" => key, "error_code" => "rejected").increment(1);
}
}
if rejected > 0 {
tracing::warn!(producer_id = %self.producer_id, records = rejected, "usage-ingestion rejected usage records");
}
return;
}
Err(status) => {
let code = status.code();
let retryable = matches!(
code,
Code::Unavailable
| Code::DeadlineExceeded
| Code::ResourceExhausted
| Code::Aborted
| Code::Cancelled
| Code::Internal
);
let code_label: Arc<str> = Arc::from(format!("{code:?}"));
if attempt + 1 < MAX_ATTEMPTS && retryable {
metrics::counter!("usage_ingestion_retries_total", "producer_id" => self.producer_id.clone(), "error_code" => code_label).increment(1);
tokio::time::sleep(Duration::from_millis(
RETRY_BACKOFF_MS * 2u64.pow(attempt),
))
.await;
attempt += 1;
continue;
}
for r in chunk {
let key: Arc<str> = Arc::from(r.usage_key.as_str());
metrics::counter!("usage_ingestion_records_failed_total", "producer_id" => self.producer_id.clone(), "usage_key" => key, "error_code" => code_label.clone()).increment(1);
}
tracing::warn!(producer_id = %self.producer_id, records = chunk.len(), error = %status, "failed to ingest usage records");
return;
}
}
}
}
}
#[async_trait]
impl UsageRecordBatch for GrpcUsageRecordBatch {
fn add(&self, team_id: i64, usage_key: &str, record_id: &str, quantity: f64, unit: &str) {
if quantity <= 0.0 || !self.accepts(team_id) {
return;
}
let key = format!("{team_id}:{usage_key}:{record_id}");
let mut records = self.records.lock().unwrap();
if records.keys.insert(key) {
records.list.push(Pending {
team_id,
usage_key: usage_key.to_string(),
record_id: record_id.to_string(),
quantity,
unit: unit.to_string(),
});
}
}
async fn flush(&self) -> anyhow::Result<()> {
let pending = std::mem::take(&mut *self.records.lock().unwrap()).list;
let Some(client) = &self.client else {
return Ok(());
};
if pending.is_empty() {
return Ok(());
}
let flushed_at = (self.clock)() as i64;
let mut records = Vec::with_capacity(pending.len());
for p in pending {
// `BigInt(quantity)` throws on a fraction, which fails the whole flush.
if p.quantity.fract() != 0.0 || !p.quantity.is_finite() {
anyhow::bail!(
"The number {} cannot be converted to a BigInt because it is not an integer",
p.quantity
);
}
records.push(BillingUsageRecord {
team_id: p.team_id,
timestamp_ms: flushed_at,
producer_id: self.producer_id.to_string(),
usage_key: p.usage_key,
record_id: p.record_id,
quantity: p.quantity as i64,
unit: p.unit,
});
}
futures::future::join_all(
records
.chunks(self.max_batch_size)
.map(|chunk| self.ingest_chunk(client, chunk)),
)
.await;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn report_teams_match_like_build_integer_matcher() {
for (raw, expected) in [
("*", true),
(" * ", false),
("*,5", true),
("*,6", false),
("3, 5", true),
("0x5", true),
("5abc", true),
("", false),
(" ", false),
("6", false),
] {
assert_eq!(report_teams_match(raw, 5), expected, "{raw:?}");
}
}
#[tokio::test]
async fn a_usage_call_to_a_silent_server_ends_at_the_timeout() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
let mut held = Vec::new();
while let Ok((socket, _)) = listener.accept().await {
held.push(socket);
}
});
let batch = GrpcUsageRecordBatch::new(
&UsageConfig {
addr: addr.to_string(),
tls: false,
timeout: Duration::from_millis(50),
max_batch_size: 10,
report_teams: "*".to_string(),
},
"logs",
crate::rate_limiter::system_clock(),
)
.unwrap();
batch.add(1, "logs_bytes", "record-1", 5.0, "bytes");
tokio::time::timeout(Duration::from_secs(5), batch.flush())
.await
.expect("each attempt must end at its timeout")
.unwrap();
}
}