Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

22.6 KB
Raw Blame
//! Service configuration and infrastructure client builders — the `envconfig` mirror plus the pool
//! and Kafka builders. A leaf: it names no other seeder module, so dependency arrows point at it,
//! never away.

use std::fmt;
use std::num::{NonZeroU32, NonZeroU64};
use std::str::FromStr;
use std::time::Duration;

use common_database::PoolConfig;
use common_kafka::config::KafkaConfig;
use common_types::cohort::TeamAllowlist;
use envconfig::Envconfig;

const POOL_NAME: &str = "posthog_cohort_seeder";

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum KafkaProducerPartitioner {
    Murmur2Random,
}

impl KafkaProducerPartitioner {
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Murmur2Random => "murmur2_random",
        }
    }
}

impl fmt::Display for KafkaProducerPartitioner {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str(self.as_str())
    }
}

impl FromStr for KafkaProducerPartitioner {
    type Err = InvalidKafkaProducerPartitioner;

    fn from_str(value: &str) -> Result<Self, Self::Err> {
        match value {
            "murmur2_random" => Ok(Self::Murmur2Random),
            other => Err(InvalidKafkaProducerPartitioner(other.to_string())),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[error("Kafka producer partitioner must be murmur2_random, got {0:?}")]
pub struct InvalidKafkaProducerPartitioner(String);

#[derive(Clone, Debug, Envconfig)]
pub struct Config {
    #[envconfig(default = "0.0.0.0")]
    pub bind_host: String,

    #[envconfig(default = "3324")]
    pub bind_port: u16,

    #[envconfig(default = "true")]
    pub export_prometheus: bool,

    #[envconfig(default = "localhost:9092")]
    pub kafka_hosts: String,

    #[envconfig(default = "false")]
    pub kafka_tls: bool,

    #[envconfig(default = "")]
    pub kafka_client_rack: String,

    #[envconfig(default = "")]
    pub kafka_client_id: String,

    #[envconfig(
        from = "COHORT_STREAM_SEED_EVENTS_TOPIC",
        default = "cohort_stream_seed_events"
    )]
    pub seed_events_topic: String,

    #[envconfig(default = "murmur2_random")]
    pub kafka_producer_partitioner: KafkaProducerPartitioner,

    /// The partition count every co-partitioned cohort topic must have — the consumer owns a
    /// person by `partition_for(key, COHORT_PARTITION_COUNT)`, so a seed topic provisioned with a
    /// different count would route a person's seed tiles to a worker that does not own them.
    /// Mirrors the processor's `COHORT_PARTITION_COUNT`; startup verifies the seed topic against it.
    #[envconfig(from = "COHORT_PARTITION_COUNT", default = "64")]
    pub cohort_partition_count: u32,

    #[envconfig(default = "none")]
    pub kafka_compression_codec: String,

    #[envconfig(default = "100")]
    pub kafka_producer_linger_ms: u32,

    #[envconfig(default = "64")]
    pub kafka_producer_queue_mib: u32,

    #[envconfig(default = "100000")]
    pub kafka_producer_queue_messages: u32,

    #[envconfig(default = "postgres://posthog:posthog@localhost:5432/posthog")]
    pub database_url: String,

    #[envconfig(default = "1")]
    pub min_pg_connections: u32,

    #[envconfig(default = "5")]
    pub max_pg_connections: u32,

    #[envconfig(default = "10")]
    pub pg_acquire_timeout_secs: u64,

    #[envconfig(default = "5000")]
    pub pg_statement_timeout_ms: u64,

    #[envconfig(default = "")]
    pub clickhouse_url: String,

    #[envconfig(default = "")]
    pub clickhouse_offline_cluster_host: String,

    #[envconfig(default = "localhost:8123")]
    pub clickhouse_host: String,

    #[envconfig(default = "default")]
    pub clickhouse_user: String,

    #[envconfig(default = "")]
    pub clickhouse_password: String,

    #[envconfig(default = "")]
    pub clickhouse_password_file: String,

    #[envconfig(default = "default")]
    pub clickhouse_database: String,

    #[envconfig(default = "false")]
    pub clickhouse_secure: bool,

    /// Validate the server certificate against the public root CAs. Defaults on so an unconfigured
    /// deployment fails closed; the chart sets it to `false` for internal ClickHouse.
    #[envconfig(default = "true")]
    pub clickhouse_verify: bool,

    /// PEM CA bundle to validate the ClickHouse server certificate against, as Django's
    /// `CLICKHOUSE_CA`. Naming a CA is an explicit request to authenticate the server, so it wins
    /// over `clickhouse_verify` rather than being downgraded by an inherited `CLICKHOUSE_VERIFY=false`.
    #[envconfig(default = "")]
    pub clickhouse_ca: String,

    #[envconfig(from = "REALTIME_COHORT_TEAM_ALLOWLIST", default = "2")]
    pub team_allowlist: TeamAllowlist,

    #[envconfig(default = "15")]
    pub seeder_run_poll_secs: u64,

    #[envconfig(default = "1")]
    pub seeder_max_concurrent_chunks: usize,

    #[envconfig(default = "900")]
    pub seeder_chunk_lease_secs: u64,

    #[envconfig(default = "5")]
    pub seeder_max_chunk_attempts: u32,

    /// The first retry's wait ceiling after a chunk fails; it doubles per attempt up to
    /// [`Config::seeder_retry_backoff_cap_secs`]. Without a wait, the poll loop re-claims a chunk
    /// that failed for a durable reason within seconds and spends its whole attempt budget on the
    /// same failure, which fails the run.
    #[envconfig(default = "30")]
    pub seeder_retry_backoff_base_secs: u64,

    /// The longest a failed chunk waits before it is claimable again.
    ///
    /// This bounds how long a durably-failing chunk holds its run open, and the bound is a sum over
    /// attempts rather than the cap alone: the ceilings double from
    /// [`Config::seeder_retry_backoff_base_secs`] until they reach the cap, and the jitter draws
    /// uniformly from `[0, ceiling]`, so the expected drain is half the worst case. At the defaults
    /// (base 30s, cap 1800s) the doubling reaches the cap on the 7th attempt, so the sum is about
    /// 15 minutes at 5 attempts and 3.5 hours at 12 — the cap times the attempt count is a real
    /// upper bound but a loose one, and reading it as the answer overstates a small budget by an
    /// order of magnitude.
    ///
    /// A run cannot complete until every chunk is `confirmed`, so that whole window is time the run
    /// holds its cohort's uniqueness slot against every future backfill for that cohort.
    /// `charts/apps/cohort-seeder/values.prod-us.yaml` is where the deployed
    /// `SEEDER_MAX_CHUNK_ATTEMPTS` lives; the two must be sized together.
    ///
    /// The chunk lease budget is not the comparison to make. A `failed` chunk holds no lease and
    /// cannot heartbeat, so the two quantities never meet in any predicate.
    #[envconfig(default = "1800")]
    pub seeder_retry_backoff_cap_secs: u64,

    /// ClickHouse resource errors in a row on one run before the seeder stops claiming its chunks.
    #[envconfig(default = "3")]
    pub seeder_ch_breaker_threshold: u32,

    /// The first opening's length. Each later opening doubles it, up to the cap.
    #[envconfig(default = "300")]
    pub seeder_ch_breaker_cooldown_base_secs: u64,

    #[envconfig(default = "1800")]
    pub seeder_ch_breaker_cooldown_cap_secs: u64,

    /// The opening, counted since the last confirmed chunk, that fails the run instead.
    #[envconfig(default = "4")]
    pub seeder_ch_breaker_max_trips: u32,

    #[envconfig(default = "3000")]
    pub seeder_tiles_per_sec: u32,

    #[envconfig(default = "4000")]
    pub seeder_max_inflight_tiles: usize,

    /// Entries all newest-day indexes may hold together. A run skips a person's tile for a day
    /// older than one it already scanned, on conditions where every leaf keeps only the newest day,
    /// and its index holds one entry per (person, condition) pair, about 30-55 bytes, so the default
    /// takes up to about 550 MB. Runs take entries first come, first served, and free them when they
    /// end. Past the cap new pairs are not recorded, so their tiles are sent for every day.
    #[envconfig(default = "10000000")]
    pub seeder_newest_day_index_max_entries: usize,

    #[envconfig(default = "400")]
    pub seeder_max_lookback_days: u32,

    /// Person-hash bands each planned day is split into, bounding one chunk's in-memory aggregate
    /// to roughly `uniq(person, condition) / bands`.
    ///
    /// Never change it while a behavioral run is seeding. A claim takes its divisor from the day's
    /// band count at claim time and confirmed bands are not rescanned, so some persons fall in no
    /// scanned band while the run still stamps readiness.
    #[envconfig(default = "1")]
    pub seeder_bands_per_day: u16,

    /// How long after a run's boundary the stream processor may take to start counting a new cohort
    /// leaf. Keep it at least the processor's `FILTER_CATALOG_REFRESH_SECS` plus
    /// `FILTER_CATALOG_REFRESH_JITTER_SECS`. The seeder plans every day this reaches as a trailing
    /// day, so a boundary set just before midnight also seeds the minutes the live path missed of
    /// the next day.
    #[envconfig(default = "420")]
    pub seeder_live_tracking_lag_secs: u64,

    /// How long the seeder waits after a trailing day ends, in the run's timezone, before it scans
    /// that day. An event ingested for the day after the scan is missing from its tile.
    #[envconfig(default = "1800")]
    pub seeder_trailing_day_grace_secs: u64,

    /// Enable the person-property seed path: discovery widens to `person_property` runs and the
    /// planning/scan/emission pipeline arms. Default off — the processor's decode arm and
    /// `COHORT_SEED_PERSON_APPLY_ENABLED` must be deployed everywhere first, or an old processor
    /// skip-and-commits the seeds.
    #[envconfig(default = "false")]
    pub seeder_person_seeds_enabled: bool,

    /// Enable the person-property *reconcile* path: completion discovery widens to `person_property`
    /// runs, so a fully-seeded one transitions `seeding -> reconciling` and produces
    /// `reconcile_person` tiles. The orchestrator also needs `SEEDER_PERSON_SEEDS_ENABLED` before
    /// this does anything, since there is no person seed path to reconcile without it; the CLI
    /// reads this flag on its own and will dispatch a person run's tiles with seeds off.
    ///
    /// Deploy order, in this order, no overlap:
    ///   1. Roll the processor fleet-wide with a build that decodes `reconcile_person` tiles and
    ///      loads the person shape-hash map. An older processor routes the person kind to
    ///      `UnknownKind` and skip-commits it without a marker, stranding the run as a shortfall.
    ///   2. Flip `SEEDER_PERSON_SEEDS_ENABLED` and let person runs seed.
    ///   3. Flip this once step 1 is confirmed everywhere.
    ///   4. Set `REALTIME_COHORT_MEMBERSHIP_STAMP_POLICY=events_or_calculation_stamp` on every
    ///      region's flags service, then flip Django's
    ///      `BEHAVIORAL_BACKFILL_PERSON_READINESS_ENABLED`. Until that gate opens the finalizer
    ///      skips person runs, so they stay `reconciling` after step 3 and
    ///      `seeder_runs_reconciling{kind="person_property"}` climbs. That is expected, not a
    ///      stalled dispatch.
    ///
    /// Flipping this back off does not undo a bad rollout: a run already in `reconciling` stops
    /// being discovered, so it never reaches `reconcile_observed_at` and never finalizes. Recovery
    /// is an operator re-dispatch through the CLI after the fleet is upgraded.
    #[envconfig(default = "false")]
    pub seeder_person_reconcile_dispatch_enabled: bool,

    /// Person-seed produce rate, shared across concurrent person chunks and separate from
    /// `seeder_tiles_per_sec` so the two throughputs tune independently.
    #[envconfig(default = "2000")]
    pub seeder_person_seeds_per_sec: u32,

    /// Target persons per planned UUID-range chunk; the planning scan keeps every Nth id as a
    /// range boundary. Sized so one chunk's paced emission completes in single-digit minutes —
    /// settings validation refuses a value the ClickHouse execution-time budget cannot cover.
    #[envconfig(default = "1000000")]
    pub seeder_persons_per_chunk: u64,

    /// The person path's own chunk-slot budget, so a person scan never occupies a behavioral slot.
    #[envconfig(default = "1")]
    pub seeder_person_max_concurrent_chunks: usize,

    /// Which scanned persons a person chunk emits.
    ///
    /// On, this is the healer cadence: every scanned person is seeded, including empty-`matched`
    /// ones, which is what retracts stale TRUE state. It has to see everyone, so it prunes nothing
    /// and asks ClickHouse for no key filter.
    ///
    /// Off, a chunk emits only a person whose leaf truths can move a participating cohort's verdict
    /// against an absent prior. Conditions whose keys the person's blob lacks are decided from a
    /// verdict cached at validation instead of through the VM, and when every condition is decidable
    /// that way the scan drops key-less rows in ClickHouse rather than transferring them.
    #[envconfig(default = "true")]
    pub seeder_person_emit_nonmatchers: bool,

    /// Run the legacy wide scan alongside the projected scan and diff the resulting tiles.
    ///
    /// On by default, because taking this measurement is the only reason the layer exists and a
    /// run that silently skipped it reads exactly like a clean one. Nothing downstream depends on
    /// it either way: the projected arm's tiles are what a chunk emits regardless.
    ///
    /// Turn it off in charts to finish a long reseed at full speed once the measurement is in
    /// hand, then delete the layer. While it is on a chunk pays its projected scan plus a full
    /// wide one.
    #[envconfig(default = "true")]
    pub seeder_scan_shadow_compare: bool,

    #[envconfig(default = "14400")]
    pub seeder_ch_max_execution_time_secs: u64,

    #[envconfig(default = "20000000000")]
    pub seeder_ch_max_bytes_before_external_group_by: u64,

    #[envconfig(default = "20000000000")]
    pub seeder_ch_max_bytes_before_external_sort: u64,

    /// Runaway guard on sets built from `IN (SELECT …)` subqueries. Exceeding it throws a set-size
    /// error rather than pushing the server toward an OOM that takes unrelated queries down.
    #[envconfig(default = "20000000000")]
    pub seeder_ch_max_bytes_in_set: u64,

    #[envconfig(default = "grace_hash")]
    pub seeder_ch_join_algorithm: String,

    /// This and the next two settings are sent only when set. The `cohort_seeder` profile constrains
    /// them, and ClickHouse rejects a query that sends a value above a constraint, so a default could
    /// fail every scan. Not 0, which ClickHouse reads as one thread per core.
    pub seeder_ch_max_threads: Option<NonZeroU64>,

    /// Not 0, which ClickHouse reads as unlimited.
    pub seeder_ch_max_memory_usage: Option<NonZeroU64>,

    /// ClickHouse ranks a shard's replicas by recent error count before `<priority>`. A high value
    /// treats a replica with up to that many recent errors as healthy, so a shard's read stays on its
    /// offline replica instead of moving to an online one.
    pub seeder_ch_distributed_replica_max_ignored_errors: Option<u64>,

    #[envconfig(default = "100")]
    pub seeder_queue_full_backoff_ms: u64,

    /// Enable the dark-by-default automatic reconcile dispatch driver. Enabling it without
    /// [`Config::seeder_confirm_register_backfilled`] is a startup error.
    #[envconfig(default = "false")]
    pub seeder_reconcile_auto_dispatch_enabled: bool,

    /// Attest that every run's data tiles were seeded after membership-register writers deployed —
    /// the automatic equivalent of the CLI's `--confirm-register-backfilled`. Required to arm
    /// automatic dispatch.
    #[envconfig(default = "false")]
    pub seeder_confirm_register_backfilled: bool,

    /// How many reconcile dispatches this replica may run at once — the bound on how hard a backlog
    /// of completed runs can press the producer queue the chunk pipeline shares. Separate from
    /// [`Config::seeder_max_inflight_tiles`], which bounds a single dispatch.
    #[envconfig(default = "4")]
    pub seeder_reconcile_max_concurrent_dispatches: usize,

    /// The reconcile-marker topic whose high watermarks anchor the marker watcher's start positions,
    /// captured at dispatch time. The observer reads markers from the same topic. Its partition count
    /// must not change while runs are in flight: a run's watch covers the partitions that existed at
    /// its dispatch, so one added later is never read — holding the run open if it appeared before
    /// the observation ends were captured, and settling the run short if it appeared after them.
    #[envconfig(default = "cohort_reconcile_markers")]
    pub cohort_reconcile_markers_topic: String,

    /// Enable the dark-by-default reconcile observer: the marker-watch task and the driver's
    /// observation pass. A separate gate from auto-dispatch — observation can run against
    /// CLI-dispatched runs without auto-dispatch, and vice versa.
    #[envconfig(default = "false")]
    pub seeder_reconcile_observer_enabled: bool,

    /// The seed processor's consumer group id. The observer queries this group's committed offsets on
    /// the seed topic as the reconcile-liveness signal; it never commits to it.
    #[envconfig(default = "cohort-stream-seeds")]
    pub kafka_seed_consumer_group: String,

    /// Timeout for the seed-group OffsetFetch and marker-topic watermark metadata calls the
    /// observer makes.
    #[envconfig(default = "10000")]
    pub seeder_reconcile_offsets_timeout_ms: u64,

    /// Flush the marker watcher's accumulated bits and positions at least this often.
    #[envconfig(default = "5000")]
    pub seeder_reconcile_persist_interval_ms: u64,

    /// Flush the marker watcher after this many consumed messages even if the interval has not
    /// elapsed, bounding how many observations a crash can lose.
    #[envconfig(default = "5000")]
    pub seeder_reconcile_persist_max_batch: u64,
}

impl Config {
    pub fn bind_address(&self) -> String {
        format!("{}:{}", self.bind_host, self.bind_port)
    }

    pub fn pool_config(&self) -> PoolConfig {
        PoolConfig {
            min_connections: self.min_pg_connections,
            max_connections: self.max_pg_connections,
            acquire_timeout: Duration::from_secs(self.pg_acquire_timeout_secs),
            idle_timeout: Some(Duration::from_secs(300)),
            test_before_acquire: true,
            statement_timeout_ms: (self.pg_statement_timeout_ms != 0)
                .then_some(self.pg_statement_timeout_ms),
            pool_name: Some(POOL_NAME.to_string()),
        }
    }

    pub fn build_kafka_config(&self) -> KafkaConfig {
        KafkaConfig {
            kafka_hosts: self.kafka_hosts.clone(),
            kafka_tls: self.kafka_tls,
            kafka_client_rack: self.kafka_client_rack.clone(),
            kafka_client_id: self.kafka_client_id.clone(),
            kafka_compression_codec: self.kafka_compression_codec.clone(),
            kafka_producer_partitioner: Some(self.kafka_producer_partitioner.as_str().to_string()),
            kafka_producer_linger_ms: self.kafka_producer_linger_ms,
            kafka_producer_queue_mib: self.kafka_producer_queue_mib,
            kafka_producer_queue_messages: self.kafka_producer_queue_messages,
            kafka_message_timeout_ms: 20_000,
            kafka_producer_batch_size: None,
            kafka_producer_batch_num_messages: None,
            kafka_producer_enable_idempotence: None,
            kafka_producer_max_in_flight_requests_per_connection: None,
            kafka_producer_topic_metadata_refresh_interval_ms: None,
            kafka_producer_message_max_bytes: None,
            kafka_producer_sticky_partitioning_linger_ms: None,
            kafka_producer_acks: None,
            kafka_producer_retries: None,
        }
    }

    pub fn tiles_per_second(&self) -> Result<NonZeroU32, ConfigValidationError> {
        NonZeroU32::new(self.seeder_tiles_per_sec).ok_or(ConfigValidationError::ZeroTileRate)
    }
}

#[derive(Debug, thiserror::Error)]
pub enum ConfigValidationError {
    #[error("seed tiles per second must be greater than zero")]
    ZeroTileRate,
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use super::*;

    fn default_config() -> Config {
        Config::init_from_hashmap(&HashMap::new()).unwrap()
    }

    #[test]
    fn producer_partitioner_is_pinned_to_the_cohort_affinity_contract() {
        let config = Config::init_from_hashmap(&HashMap::new()).unwrap();
        assert_eq!(
            config.kafka_producer_partitioner,
            KafkaProducerPartitioner::Murmur2Random
        );
        assert_eq!(
            config
                .build_kafka_config()
                .kafka_producer_partitioner
                .as_deref(),
            Some("murmur2_random"),
        );
    }

    #[test]
    fn producer_partitioner_rejects_unsafe_environment_overrides() {
        for value in ["consistent_random", "random", ""] {
            let env =
                HashMap::from([("KAFKA_PRODUCER_PARTITIONER".to_string(), value.to_string())]);
            assert!(
                Config::init_from_hashmap(&env).is_err(),
                "accepted unsafe partitioner {value:?}"
            );
        }
    }

    #[test]
    fn partition_count_defaults_to_the_shared_cohort_contract() {
        assert_eq!(
            default_config().cohort_partition_count,
            cohort_core::partitioner::COHORT_PARTITION_COUNT,
        );
        let env = HashMap::from([("COHORT_PARTITION_COUNT".to_string(), "8".to_string())]);
        assert_eq!(
            Config::init_from_hashmap(&env)
                .unwrap()
                .cohort_partition_count,
            8
        );
    }

    /// The fail-closed TLS posture lives entirely in these defaults, so pin them.
    #[test]
    fn clickhouse_tls_defaults_are_fail_closed() {
        let config = default_config();
        assert!(config.clickhouse_verify);
        assert!(config.clickhouse_ca.is_empty());
    }

    /// A default of off would make the validation run a silent no-op: the compare emits nothing,
    /// so its counters are absent rather than zero, and an unmeasured run is indistinguishable
    /// from a clean one. The measurement is the whole point of the layer, so it is what a pod does
    /// unless an operator says otherwise.
    #[test]
    fn the_shadow_compare_runs_unless_an_operator_turns_it_off() {
        assert!(default_config().seeder_scan_shadow_compare);
    }

    #[test]
    fn service_limits_reject_disabled_tile_rate() {
        let mut config = default_config();
        config.seeder_tiles_per_sec = 0;
        assert!(matches!(
            config.tiles_per_second(),
            Err(ConfigValidationError::ZeroTileRate)
        ));
    }
}