:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::HashSet;
use std::convert::Infallible;
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::{num::ParseIntError, str::FromStr};
use common_continuous_profiling::ContinuousProfilingConfig;
use common_kafka::config::{ConsumerConfig, KafkaConfig};
use common_kafka::kafka_producer::EnvelopeEncoding;
use crate::producer::WireFormat;
use envconfig::Envconfig;
use siphasher::sip::SipHasher13;
#[derive(Envconfig, Clone)]
pub struct Config {
#[envconfig(nested = true)]
pub continuous_profiling: ContinuousProfilingConfig,
#[envconfig(nested = true)]
pub kafka: KafkaConfig,
#[envconfig(nested = true)]
pub consumer: ConsumerConfig,
#[envconfig(default = "clickhouse_property_values")]
pub output_topic: String,
#[envconfig(default = "property_vals_intermediate")]
pub intermediate_topic: String,
// Envelope encoding for payloads produced to the intermediate topic. The
// merger consumer decodes transparently, so `lz4` can roll out
// incrementally. `none` by default.
#[envconfig(default = "none")]
pub intermediate_topic_encoding: EnvelopeEncoding,
#[envconfig(default = "json")]
pub intermediate_topic_format: WireFormat,
#[envconfig(default = "clickhouse-property-vals-rs-merger")]
pub merger_consumer_group: String,
#[envconfig(default = "30")]
pub flush_interval_secs: u64,
#[envconfig(default = "500000")]
pub max_buffered_tuples: usize,
#[envconfig(default = "0")]
pub max_values_per_key: usize,
#[envconfig(nested = true)]
pub length_caps: LengthCaps,
#[envconfig(default = "0")]
pub events_seen_cache_capacity: usize,
#[envconfig(default = "0")]
pub merger_seen_cache_capacity: usize,
#[envconfig(default = "60")]
pub kafka_produce_timeout_secs: u64,
#[envconfig(default = "clickhouse_groups")]
pub groups_kafka_consumer_topic: String,
#[envconfig(default = "clickhouse-property-vals-rs-groups")]
pub groups_kafka_consumer_group: String,
#[envconfig(default = "")]
pub allowed_teams: TeamList,
#[envconfig(default = "")]
pub blocked_teams: TeamList,
#[envconfig(default = "100")]
pub rollout_percentage: u8,
#[envconfig(default = "")]
pub excluded_property_keys: ExcludedPropertyKeys,
#[envconfig(from = "BIND_HOST", default = "::")]
pub host: String,
#[envconfig(from = "BIND_PORT", default = "3302")]
pub port: u16,
}
/// Length caps applied at fan-out; tuples exceeding any cap are dropped.
/// Nested envconfig: each field reads its own UPPER_SNAKE_CASE env var.
#[derive(Envconfig, Clone, Copy)]
pub struct LengthCaps {
#[envconfig(default = "400")]
pub max_property_key_len: usize,
#[envconfig(default = "10000")]
pub max_property_value_len: usize,
}
#[derive(Clone)]
pub struct TeamList {
pub teams: Vec<i64>,
}
impl FromStr for TeamList {
type Err = ParseIntError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let mut teams = Vec::new();
for team in s.trim().split(',') {
if team.is_empty() {
continue;
}
teams.push(team.parse()?);
}
Ok(TeamList { teams })
}
}
#[derive(Clone, Default)]
pub struct ExcludedPropertyKeys {
pub keys: Arc<HashSet<String>>,
pub prefixes: Arc<Vec<String>>,
}
impl ExcludedPropertyKeys {
pub fn contains(&self, key: &str) -> bool {
self.keys.contains(key)
|| self
.prefixes
.iter()
.any(|prefix| key.starts_with(prefix.as_str()))
}
}
impl FromStr for ExcludedPropertyKeys {
type Err = Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let mut keys = HashSet::new();
let mut prefixes = Vec::new();
for entry in s.split(',').map(str::trim).filter(|k| !k.is_empty()) {
match entry.strip_suffix('*') {
Some(prefix) => prefixes.push(prefix.to_string()),
None => {
keys.insert(entry.to_string());
}
}
}
Ok(ExcludedPropertyKeys {
keys: Arc::new(keys),
prefixes: Arc::new(prefixes),
})
}
}
impl Config {
pub fn init_with_defaults() -> Result<Self, envconfig::Error> {
// auto_commit=true: background timer ships stored offsets to the
// broker every ~5s. We manually advance the stored offset only after
// a successful produce (see worker.rs::flush), so the broker never
// sees an offset for input we haven't actually written out.
ConsumerConfig::set_defaults(
"clickhouse-property-vals-rs",
"clickhouse_events_json",
true,
);
Config::init_from_env()
}
pub fn should_process(&self, team_id: i64) -> bool {
team_filter_allows(
team_id,
&self.allowed_teams.teams,
&self.blocked_teams.teams,
self.rollout_percentage,
)
}
}
pub fn team_filter_allows(
team_id: i64,
allowed: &[i64],
blocked: &[i64],
rollout_percentage: u8,
) -> bool {
if blocked.contains(&team_id) {
return false;
}
if allowed.contains(&team_id) {
return true;
}
team_in_rollout(team_id, rollout_percentage)
}
fn team_in_rollout(team_id: i64, rollout_percentage: u8) -> bool {
if rollout_percentage >= 100 {
return true;
}
if rollout_percentage == 0 {
return false;
}
let mut hasher = SipHasher13::new();
team_id.hash(&mut hasher);
(hasher.finish() % 100) < rollout_percentage as u64
}
#[cfg(test)]
mod tests {
use super::*;
use proptest::prelude::*;
#[test]
fn team_list_parses() {
let list: TeamList = "1,2,3".parse().unwrap();
assert_eq!(list.teams, vec![1, 2, 3]);
let empty: TeamList = "".parse().unwrap();
assert!(empty.teams.is_empty());
}
#[test]
fn length_caps_default_when_env_unset() {
let caps = LengthCaps::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
assert_eq!(caps.max_property_key_len, 400);
assert_eq!(caps.max_property_value_len, 10000);
}
#[test]
fn length_caps_read_from_env_vars() {
let env = std::collections::HashMap::from([
("MAX_PROPERTY_KEY_LEN".to_string(), "500".to_string()),
("MAX_PROPERTY_VALUE_LEN".to_string(), "1000".to_string()),
]);
let caps = LengthCaps::init_from_hashmap(&env).unwrap();
assert_eq!(caps.max_property_key_len, 500);
assert_eq!(caps.max_property_value_len, 1000);
}
fn arb_team_list() -> impl Strategy<Value = Vec<i64>> {
prop::collection::vec(-1_000i64..=1_000, 0..10)
}
proptest! {
#[test]
fn blocked_teams_never_process(
team_id: i64,
rollout in 0u8..=100,
allowed in arb_team_list(),
mut blocked in arb_team_list(),
) {
blocked.push(team_id);
prop_assert!(!team_filter_allows(team_id, &allowed, &blocked, rollout));
}
#[test]
fn allowed_teams_process_when_not_blocked(
team_id: i64,
rollout in 0u8..=100,
blocked in arb_team_list(),
mut allowed in arb_team_list(),
) {
let blocked: Vec<i64> = blocked.into_iter().filter(|t| *t != team_id).collect();
allowed.push(team_id);
prop_assert!(team_filter_allows(team_id, &allowed, &blocked, rollout));
}
#[test]
fn full_rollout_processes_any_non_blocked_team(
team_id: i64,
allowed in arb_team_list(),
blocked in arb_team_list(),
) {
let blocked: Vec<i64> = blocked.into_iter().filter(|t| *t != team_id).collect();
prop_assert!(team_filter_allows(team_id, &allowed, &blocked, 100));
}
#[test]
fn zero_rollout_drops_any_non_allowed_team(
team_id: i64,
allowed in arb_team_list(),
blocked in arb_team_list(),
) {
let allowed: Vec<i64> = allowed.into_iter().filter(|t| *t != team_id).collect();
let blocked: Vec<i64> = blocked.into_iter().filter(|t| *t != team_id).collect();
prop_assert!(!team_filter_allows(team_id, &allowed, &blocked, 0));
}
#[test]
fn should_process_is_deterministic(
team_id: i64,
rollout in 0u8..=100,
allowed in arb_team_list(),
blocked in arb_team_list(),
) {
let first = team_filter_allows(team_id, &allowed, &blocked, rollout);
for _ in 0..5 {
prop_assert_eq!(team_filter_allows(team_id, &allowed, &blocked, rollout), first);
}
}
}
#[test]
fn rollout_percentage_approximates_target_share() {
let included = (1..=10_000)
.filter(|t| team_filter_allows(*t, &[], &[], 10))
.count();
assert!(
(900..=1100).contains(&included),
"expected ~1000 of 10000 at 10%, got {included}"
);
}
}