Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

117 KB
Raw Blame
//! Service configuration, loaded from environment variables via `envconfig`.

use std::num::{NonZeroU16, NonZeroUsize};
use std::path::{Component, Path, PathBuf};
use std::time::Duration;

use anyhow::{ensure, Context};
use common_database::PoolConfig;
use common_kafka::config::KafkaConfig;
use common_types::cohort::TeamAllowlist;
use envconfig::Envconfig;
use rdkafka::ClientConfig;
use tracing::warn;

use crate::partitions::pacing::{AgeMs, Hysteresis, SeedPacingConfig, UsedPct};
use crate::partitions::{PartitionOwnership, SplitClaim};
use crate::store::durability::stage::staging_path;
use crate::store::durability::{CheckpointLineage, DurabilityConfig, NotAnOrdinalPod, PodOrdinal};
use crate::store::{OffloadConfig, OffloadMode, StoreConfig};
use crate::workers::seed_run::RunBudget;
use crate::workers::{CascadeConfig, EventNameGating, TransferRetryPolicy};

const POOL_NAME: &str = "posthog_cohort";

/// The smallest person-record TTL that stays safely beyond the replay horizon of the merge and
/// `clickhouse_events_json` topics (~7 days): a record reclaimed while its events can still replay
/// loses its replay dedup, so a non-zero TTL below this floor warns at startup.
const MIN_SAFE_PERSON_RECORD_TTL_DAYS: u32 = 30;

/// `message.max.bytes` on every consumer, although the key reads as a producer setting. librdkafka
/// also uses it to cap the first buffer it allocates for decompressing an LZ4 record batch, whose
/// frame usually carries no content size, and then grows that buffer in steps until the batch fits
/// (`rd_kafka_lz4_decompress` in librdkafka's `rdkafka_lz4.c`). The 1 MB default starts below a
/// large decompressed batch. The key also floors `fetch.max.bytes`: an explicit value below it
/// fails client creation, and a smaller `queued.max.messages.kbytes` does not bound the fetch
/// size.
const CONSUMER_MESSAGE_MAX_BYTES: u32 = 8 * 1024 * 1024;

#[derive(Envconfig, Clone, Debug)]
pub struct Config {
    /// Host for the observability HTTP server (`/_health`, `/_ready`, `/metrics`).
    #[envconfig(default = "0.0.0.0")]
    pub bind_host: String,

    /// Port for the observability HTTP server.
    #[envconfig(default = "3323")]
    pub bind_port: u16,

    /// Install the Prometheus recorder and expose `/metrics`.
    #[envconfig(default = "true")]
    pub export_prometheus: bool,

    /// DSN for the main PostHog database that owns `posthog_cohort`.
    #[envconfig(default = "postgres://posthog:posthog@localhost:5432/posthog")]
    pub database_url: String,

    #[envconfig(default = "1")]
    pub min_pg_connections: u32,

    #[envconfig(default = "5")]
    pub max_pg_connections: u32,

    #[envconfig(default = "10")]
    pub pg_acquire_timeout_secs: u64,

    /// Statement timeout for the catalog SELECT (ms). `0` → database default.
    #[envconfig(default = "5000")]
    pub pg_statement_timeout_ms: u64,

    #[envconfig(default = "300")]
    pub filter_catalog_refresh_secs: u64,

    #[envconfig(default = "60")]
    pub filter_catalog_refresh_jitter_secs: u64,

    /// Teams the filter catalog covers, on top of the teams that a row of
    /// `cohort_realtime_team_access` grants. Set `all` to disable the gate. See [`TeamAllowlist`].
    #[envconfig(from = "REALTIME_COHORT_TEAM_ALLOWLIST", default = "2")]
    pub team_allowlist: TeamAllowlist,

    /// Evaluate only the behavioral conditions whose event name matches the incoming event. Default on.
    #[envconfig(from = "COHORT_EVENT_NAME_GATING_ENABLED", default = "true")]
    pub cohort_event_name_gating_enabled: bool,

    /// Bounded buffer (in sub-batches) per per-partition worker channel.
    /// Routing to a partition this far behind blocks rather than growing memory unbounded.
    #[envconfig(default = "128")]
    pub partition_channel_buffer: usize,

    /// Per-partition ceiling on un-drained events in a worker's **live** lane — the binding intake
    /// bound (the 128-slot buffer counts sub-batches, not the events inside them). Seeds have their
    /// own lane and their own cap, [`PARTITION_INTAKE_MAX_SEEDS`](Self::partition_intake_max_seeds).
    /// Worst case in channels ≈ `cap × owned_partitions × avg_event_bytes`. Tune down if soak RSS
    /// runs hot; too low churns pause/resume.
    #[envconfig(from = "PARTITION_INTAKE_MAX_EVENTS", default = "1024")]
    pub partition_intake_max_events: usize,

    /// The seed lane's capacity, in seeds, for one partition worker. The lane holds one seed per
    /// slot, so this value *is* the seed intake cap — there is no separate counter.
    ///
    /// The resident bound per partition is `cap + COHORT_SEED_APPLY_BATCH_MAX`: `recv_many` frees
    /// the slots it takes at the start of a seed turn and the dispatcher refills them while the
    /// turn runs, which is wanted — the next turn is already full when this one ends.
    ///
    /// Additive with the event cap, so a shutdown drain covers up to this many seeds
    /// **plus** `PARTITION_INTAKE_MAX_EVENTS` events per partition. At the default that is about
    /// four quanta of seeds, seconds of drain at the rates the apply path sustains.
    #[envconfig(from = "PARTITION_INTAKE_MAX_SEEDS", default = "1024")]
    pub partition_intake_max_seeds: usize,

    #[envconfig(default = "localhost:9092")]
    pub kafka_hosts: String,

    #[envconfig(default = "false")]
    pub kafka_tls: bool,

    #[envconfig(default = "")]
    pub kafka_client_id: String,

    #[envconfig(default = "")]
    pub kafka_client_rack: String,

    /// The hot-path input topic.
    #[envconfig(default = "cohort_stream_events")]
    pub cohort_stream_events_topic: String,

    #[envconfig(default = "cohort-stream-processor")]
    pub kafka_consumer_group: String,

    /// `queued.max.messages.kbytes` (KB). For the events consumer this is an *aggregate* cap
    /// across all its partitions — the ceiling on librdkafka's fetch buffer. 128 MB.
    #[envconfig(from = "COHORT_KAFKA_QUEUED_MAX_MESSAGES_KBYTES", default = "131072")]
    pub kafka_queued_max_messages_kbytes: u32,

    /// `queued.min.messages` — per-partition prefetch floor. Low so a paused partition hoards fewer
    /// stragglers; too low risks `fetch.queue.backoff.ms` inter-fetch gaps.
    #[envconfig(from = "COHORT_KAFKA_QUEUED_MIN_MESSAGES", default = "2000")]
    pub kafka_queued_min_messages: u32,

    /// The person-merge trigger topic, keyed by `hash(team_id, old_person_uuid)` so a merge lands on
    /// P_old's worker.
    #[envconfig(default = "person_merge_events")]
    pub person_merge_events_topic: String,

    /// The internal state-transfer topic, keyed by `hash(team_id, new_person_uuid)` so a packaged
    /// drain lands on P_new's worker. Must be co-partitioned with `cohort_stream_events`.
    #[envconfig(default = "cohort_merge_state_transfer")]
    pub cohort_merge_state_transfer_topic: String,

    /// Group for the `person_merge_events` follower — separate so drain-path lag is observable.
    #[envconfig(default = "cohort-stream-merges")]
    pub kafka_merge_consumer_group: String,

    /// Group for the `cohort_merge_state_transfer` follower.
    #[envconfig(default = "cohort-stream-merge-apply")]
    pub kafka_merge_apply_consumer_group: String,

    /// Inline retries after the first transfer-produce attempt. The default budget
    /// (5 x 0.5/1/2/4/8 s) keeps a blocked worker inside the liveness and graceful-shutdown
    /// windows; exhaustion leaves the transfer staged for the periodic redrive.
    #[envconfig(default = "5")]
    pub merge_transfer_max_retries: u32,

    /// First retry backoff (ms); doubles per retry.
    #[envconfig(default = "500")]
    pub merge_transfer_retry_base_ms: u64,

    /// Backoff ceiling (ms).
    #[envconfig(default = "8000")]
    pub merge_transfer_retry_cap_ms: u64,

    /// `message.timeout.ms` for the transfer producer alone — much shorter than the shared 20 s so a
    /// single produce attempt fast-fails against a black-hole broker instead of stalling a worker.
    /// The inline retry loop wraps `merge_transfer_max_retries + 1` attempts; the budget only holds
    /// inside the 30 s graceful-shutdown window if each attempt itself fast-fails. See
    /// `build_transfer_kafka_config` and the budget arithmetic in `workers::merge_path`.
    #[envconfig(default = "2000")]
    pub merge_transfer_message_timeout_ms: u32,

    /// How often the redrive scans `cf_pending_transfers` and re-produces staged transfers whose
    /// inline retry budget was exhausted.
    #[envconfig(default = "60000")]
    pub merge_redrive_interval_ms: u64,

    // STANDING COUPLING: these floors are derived from the Kafka topic `retention.ms` values
    // (`person_merge_events`/`cohort_merge_state_transfer` = 7d, `cohort_stream_events` = 24h,
    // `clickhouse_events_json` = 7d). If any of those `retention.ms` values change, these knobs
    // MUST follow — a marker evicted before its source topic's retention lapses re-opens the
    // replay-dedup window it exists to close. `cf_pending_transfers` is NEVER GC'd here.
    /// Retention floor for both merge marker CFs (`cf_merge_drains_applied`, `cf_merge_applied`). A
    /// marker is GC'd once its `drained_at_ms` / `applied_at_ms` is older than `now − this`. Default
    /// 8d = 7d merge-topic retention + 1d safety: a marker must outlive every replay of the
    /// 7d-retention `person_merge_events` / `cohort_merge_state_transfer` topics.
    #[envconfig(default = "691200000")]
    pub merge_marker_retention_ms: u64,

    /// Retention floor for `cf_merge_tombstones`. A tombstone is GC'd once its `merged_at_ms` is
    /// older than `now − this`. Default 9d = `clickhouse_events_json` 7d + `cohort_stream_events`
    /// 24h + 1d safety: a tombstone must outlive every straggler event the redirect closes.
    #[envconfig(default = "777600000")]
    pub merge_tombstone_retention_ms: u64,

    /// How often the merge-CF GC sweep fires. Default 1h.
    #[envconfig(default = "3600000")]
    pub merge_gc_interval_ms: u64,

    /// Max keys scanned (and at most deleted) per CF, per partition, per GC tick. Bounds the GC's
    /// per-tick work; the per-CF resume cursor continues where the last tick stopped. Default 10k —
    /// at the prod merge rate (~16k markers/day/partition) that is ~15× headroom over the 1h cadence.
    #[envconfig(default = "10000")]
    pub merge_gc_scan_limit: usize,

    /// Whether the `cf_stage2` orphan GC pass runs on each merge-GC tick — reclaims composed-membership
    /// rows whose cohort has left the composable set. Default-on kill-switch; reuses `merge_gc_scan_limit`
    /// as the page cap.
    #[envconfig(from = "STAGE2_ORPHAN_GC_ENABLED", default = "true")]
    pub stage2_orphan_gc_enabled: bool,

    /// Max cascade hops before an outgoing cascade is dropped (`incoming.depth >= this`). Default 8.
    #[envconfig(default = "8")]
    pub cohort_cascade_depth_cap: u8,

    /// Max referrer re-evaluations per upstream flip. Default 1000.
    #[envconfig(default = "1000")]
    pub cohort_cascade_fanout_cap: usize,

    /// Promote resolvable, cycle-free cohort-of-cohort cohorts to composition. Default off:
    /// ref-bearing cohorts stay excluded.
    #[envconfig(from = "COHORT_CASCADE_ENABLED", default = "false")]
    pub cohort_cascade_enabled: bool,

    /// The internal cascade topic, keyed by `hash(team_id, person_id)` so a flip lands on the worker
    /// owning that person's `cf_stage2`. Must be co-partitioned with `cohort_stream_events`.
    #[envconfig(default = "cohort_cascade_events")]
    pub cohort_cascade_events_topic: String,

    /// Group for the `cohort_cascade_events` follower — separate so cascade-path lag is observable.
    #[envconfig(default = "cohort-stream-cascade")]
    pub kafka_cascade_consumer_group: String,

    /// Consume `cohort_stream_seed_events` and apply backfill day-tiles. Default off.
    #[envconfig(from = "COHORT_SEED_CONSUMER_ENABLED", default = "false")]
    pub cohort_seed_consumer_enabled: bool,

    /// The backfill seed-tile topic; must be co-partitioned with `cohort_stream_events`.
    #[envconfig(default = "cohort_stream_seed_events")]
    pub cohort_stream_seed_events_topic: String,

    /// Group for the seed follower — separate so backfill lag age is observable on its own.
    #[envconfig(default = "cohort-stream-seeds")]
    pub kafka_seed_consumer_group: String,

    /// Apply-fence margin (ms) over `s_chunk`: covers shuffler clock skew, lag spread, and
    /// producer linger.
    #[envconfig(from = "COHORT_SEED_FENCE_MARGIN_MS", default = "600000")]
    pub cohort_seed_fence_margin_ms: i64,

    /// How often the seed consumer probes idle live partitions so a quiet partition's fence can
    /// still open.
    #[envconfig(
        from = "COHORT_SEED_WATERMARK_IDLE_PROBE_INTERVAL_MS",
        default = "30000"
    )]
    pub cohort_seed_watermark_idle_probe_interval_ms: u64,

    /// Permit cross-partition membership-register transfer. Local register writers and transfer
    /// receivers are always active; only drain-side emission is gated. Enable only after every
    /// processor pod can apply the additive transfer payload. While off, a cross-partition merge ships
    /// leaves only and never holds, so it cannot wedge live merge consumption.
    #[envconfig(from = "COHORT_REGISTER_TRANSFER_ENABLED", default = "false")]
    pub cohort_register_transfer_enabled: bool,

    /// Admit and drain reconcile controls. Default off; enable only once `COHORT_RECONCILE_MARKERS_TOPIC`
    /// exists in the environment, which startup asserts. Register transfer is gated separately by
    /// `COHORT_REGISTER_TRANSFER_ENABLED`.
    #[envconfig(from = "COHORT_SEED_RECONCILE_ENABLED", default = "false")]
    pub cohort_seed_reconcile_enabled: bool,

    /// Maximum Stage 2 rows one partition worker evaluates per reconcile tick.
    #[envconfig(from = "COHORT_SEED_RECONCILE_SCAN_PAGE", default = "256")]
    pub cohort_seed_reconcile_scan_page: usize,

    /// Cadence for routing one bounded reconcile-drain tick to each active partition worker.
    #[envconfig(from = "COHORT_SEED_RECONCILE_TICK_INTERVAL_MS", default = "2000")]
    pub cohort_seed_reconcile_tick_interval_ms: u64,

    /// Apply `person_property` seeds. Default off; while off they skip and commit, so a run
    /// produced against a gate-off fleet has to be re-produced after enabling.
    #[envconfig(from = "COHORT_SEED_PERSON_APPLY_ENABLED", default = "false")]
    pub cohort_seed_person_apply_enabled: bool,

    /// How far a person seed's scan instant must beat the stored record's stamp before it may
    /// overwrite live-evaluated state (ms). Covers ClickHouse replication lag plus modest client
    /// clock skew; ties go to live.
    #[envconfig(from = "COHORT_SEED_PERSON_LIVE_MARGIN_MS", default = "900000")]
    pub cohort_seed_person_live_margin_ms: i64,

    /// Seeds one partition worker applies as a single run. `1` limits each run to one seed through
    /// the same apply pipeline; it does not restore a different implementation.
    #[envconfig(from = "COHORT_SEED_APPLY_BATCH_MAX", default = "256")]
    pub cohort_seed_apply_batch_max: usize,

    /// Store rows one run may touch: the stage-1 rows its seeds fold plus one stage-2 register per
    /// cohort those leaves back. This bounds entry counts in the overlay, register read, recompute
    /// set, and outputs. It is not a byte limit: behavioral row sizes grow with their windows.
    /// Reads inside each composed evaluation scale with that cohort's tree and use the maintenance
    /// lane's permits. A run closes before the seed that would exceed the budget; a seed heavier
    /// than the whole budget still runs alone.
    #[envconfig(from = "COHORT_SEED_APPLY_BATCH_MAX_ROWS", default = "4096")]
    pub cohort_seed_apply_batch_max_rows: usize,

    /// Live-priority gate: pause a seed partition once its live watermark age reaches this (ms).
    /// `0` disables the trigger.
    #[envconfig(from = "COHORT_SEED_LIVE_LAG_PAUSE_MS", default = "120000")]
    pub cohort_seed_live_lag_pause_ms: i64,

    /// Resume a live-lag-paused seed partition once its watermark age drops below this (ms).
    /// Must be positive and below the pause threshold when the trigger is enabled.
    #[envconfig(from = "COHORT_SEED_LIVE_LAG_RESUME_MS", default = "60000")]
    pub cohort_seed_live_lag_resume_ms: i64,

    /// Disk gate: pause all seed partitions once the store filesystem's used share reaches
    /// this (%). `0` (the default) disables the trigger: pausing seeds cannot shrink a store
    /// grown by the live path, so a threshold below the steady-state footprint would engage and
    /// never release. Opt in per deployment once the utilization gauge has baseline history.
    #[envconfig(from = "COHORT_SEED_DISK_PAUSE_PCT", default = "0")]
    pub cohort_seed_disk_pause_pct: f64,

    /// Resume disk-paused seed partitions once the used share drops below this (%). Must be
    /// positive and below the pause threshold when the trigger is enabled.
    #[envconfig(from = "COHORT_SEED_DISK_RESUME_PCT", default = "55")]
    pub cohort_seed_disk_resume_pct: f64,

    /// The StatefulSet pod name. Its ordinal suffix picks the partitions this pod owns and its
    /// checkpoint lineage, and it names the Kafka clients. Absent outside Kubernetes.
    #[envconfig(from = "POD_NAME")]
    pub pod_name: Option<String>,

    /// How many processor pods split the partitions. The pod with ordinal `i` owns every partition
    /// `p` with `p mod COHORT_POD_COUNT = i`, on every input. It must divide
    /// `COHORT_PARTITION_COUNT`. Set it with the StatefulSet's replica count.
    #[envconfig(from = "COHORT_POD_COUNT", default = "1")]
    pub cohort_pod_count: NonZeroU16,

    /// The pod count before a reshard, which must divide `COHORT_POD_COUNT`. While it is set, each
    /// pod whose ordinal is below it keeps its store, and every other pod takes its partitions over
    /// from the pod that owned them. Remove it once the reshard is done.
    #[envconfig(from = "COHORT_RESHARD_FROM_POD_COUNT")]
    pub cohort_reshard_from_pod_count: Option<NonZeroU16>,

    #[envconfig(from = "HOSTNAME")]
    pub pod_hostname: Option<String>,

    /// The output topic for membership changes. Defaulting to the shadow topic keeps the cut-over
    /// a config-only change, so no code deploy can redirect production output.
    #[envconfig(default = "cohort_membership_changed_shadow")]
    pub cohort_membership_changed_topic: String,

    /// Reconcile completion markers ride their own topic: the membership topic's consumers reject
    /// any record without `person_id`/`status`, and the seeder's watcher tails this one end to end.
    #[envconfig(default = "cohort_reconcile_markers")]
    pub cohort_reconcile_markers_topic: String,

    /// `message.timeout.ms` for the marker producer alone — much shorter than the shared 20 s. The
    /// marker produce runs inline on a partition worker, so a broker that black-holes it stalls live
    /// evaluation on that partition for the whole timeout, and the drain sweeper re-queues the job a
    /// tick later and stalls it again. Half of `COHORT_SEED_RECONCILE_TICK_INTERVAL_MS`, so the
    /// worker is free for events between attempts rather than held end to end, and still above the
    /// ingestion cluster's p99 produce ack (~950 ms on prod-us) — below that, healthy tail produces
    /// turn into spurious marker failures and bury the signal in
    /// `cohort_reconcile_marker_produce_errors_total`.
    #[envconfig(default = "1000")]
    pub reconcile_marker_message_timeout_ms: u32,

    /// `murmur2_random` co-partitions a `person_id` key identically to the Node/Python producers.
    #[envconfig(default = "murmur2_random")]
    pub kafka_producer_partitioner: String,

    /// Partition count of the co-partitioned cohort topics. Production is 64; test lanes lower it
    /// (e.g. 8) to fit many merge lanes on a single-node broker. The merge protocol's partition
    /// arithmetic must use this value, not a literal.
    #[envconfig(default = "64")]
    pub cohort_partition_count: u32,

    #[envconfig(default = "none")]
    pub kafka_compression_codec: String,

    /// Max events pulled per consume-route cycle.
    #[envconfig(default = "1000")]
    pub recv_batch_size: usize,

    /// Max wait before a partial batch is routed.
    #[envconfig(default = "500")]
    pub recv_batch_timeout_ms: u64,

    /// How often processed offsets are committed back to Kafka.
    #[envconfig(default = "5000")]
    pub offset_commit_interval_ms: u64,

    /// Tokio runtime worker threads. `0` (default) lets Tokio size the pool from
    /// `available_parallelism()`, which accounts for a CFS CPU *limit* (`cpu.max`) but not CPU
    /// *requests*/shares — so on a requests-only pod (no `limits.cpu`), or when the cgroup fs isn't
    /// readable, it returns the *node's* core count and over-subscribes the runtime. Set this to the
    /// pod's CPU budget to cap the pool regardless of how the limit is expressed.
    #[envconfig(default = "0")]
    pub tokio_worker_threads: usize,

    /// How often the sweep fires to evict state whose eviction deadline has passed.
    #[envconfig(default = "30000")]
    pub sweep_interval_ms: u64,

    /// Startup delay before the *first* eviction sweep, so its overdue-eviction read burst doesn't
    /// compete with backlog catch-up on a cold, idle store at boot. Counts from sweep-task spawn
    /// (≈ process boot), not "backlog drained", so raise it if the burst still lands on catch-up.
    #[envconfig(from = "COHORT_FIRST_EVICTION_SWEEP_DELAY_MS", default = "120000")]
    pub first_eviction_sweep_delay_ms: u64,

    /// Grace period added to every eviction deadline before the sweep acts. The sweep evicts a key
    /// only once `deadline + safety_margin < now`, absorbing consumer-lag spikes.
    #[envconfig(default = "300000")]
    pub sweep_safety_margin_ms: u64,

    /// How often the store-stats publisher and Tokio runtime monitor sample and emit their gauges
    /// (seconds).
    #[envconfig(from = "COHORT_STATS_PUBLISH_INTERVAL_SECS", default = "15")]
    pub stats_publish_interval_secs: u64,

    /// On-disk path for the per-process RocksDB state store.
    #[envconfig(default = "cohort-store")]
    pub store_path: String,

    /// Destroy any existing store at `store_path` before opening, so re-acquiring a partition never
    /// serves stale state left by a previous owner.
    #[envconfig(default = "true")]
    pub wipe_store_on_start: bool,

    /// On a store-schema version mismatch, destroy and recreate the store instead of failing fast.
    /// Off by default: a mismatch is an operator decision (wipe or roll back), not something to eat
    /// silently.
    #[envconfig(from = "COHORT_WIPE_ON_SCHEMA_MISMATCH", default = "false")]
    pub cohort_wipe_on_schema_mismatch: bool,

    /// Enable RocksDB statistics so the store-stats publisher can report cache tickers and per-CF
    /// sizes. See [`StoreConfig::statistics_enabled`].
    #[envconfig(from = "COHORT_STORE_STATISTICS_ENABLED", default = "true")]
    pub store_statistics_enabled: bool,

    /// Sample 1-in-N reads into the read-latency histogram; the read counter stays exact. See
    /// [`StoreConfig::read_sample_ratio`]. `1` records every read; `0` floors to `1`.
    #[envconfig(from = "COHORT_STORE_READ_SAMPLE_RATIO", default = "64")]
    pub store_read_sample_ratio: u32,

    /// RocksDB block-cache size in bytes, shared across all column families.
    #[envconfig(from = "COHORT_BLOCK_CACHE_BYTES", default = "134217728")]
    pub cohort_block_cache_bytes: usize,

    /// Cache and partition RocksDB index/filter blocks for faster point lookups.
    #[envconfig(from = "COHORT_TUNED_BLOCK_OPTIONS_ENABLED", default = "true")]
    pub cohort_tuned_block_options_enabled: bool,

    /// Mark tombstone-heavy SSTs for compaction so deletions reclaim disk.
    #[envconfig(from = "COHORT_COMPACT_ON_DELETION_ENABLED", default = "true")]
    pub cohort_compact_on_deletion_enabled: bool,

    /// Compact SSTs older than this many seconds; opt-in, `0` (the default) disables it so a
    /// persisted store isn't mass-rewritten on reopen.
    #[envconfig(from = "COHORT_PERIODIC_COMPACTION_SECONDS", default = "0")]
    pub cohort_periodic_compaction_seconds: u64,

    /// Cap on RocksDB background compaction/flush jobs. Non-positive leaves RocksDB's default.
    #[envconfig(from = "COHORT_MAX_BACKGROUND_JOBS", default = "0")]
    pub cohort_max_background_jobs: i32,

    /// Size at which RocksDB rolls its info `LOG` file.
    #[envconfig(from = "COHORT_MAX_LOG_FILE_SIZE_BYTES", default = "268435456")]
    pub cohort_max_log_file_size_bytes: NonZeroUsize,

    /// Number of info `LOG` files RocksDB keeps.
    #[envconfig(from = "COHORT_KEEP_LOG_FILE_NUM", default = "10")]
    pub cohort_keep_log_file_num: NonZeroUsize,

    /// TTL in days for `cf_person_records`: a compaction filter drops a person record whose
    /// `last_seen_ms` is older than this. `0` (the default) disables it — no filter is installed, so a
    /// persisted record never ages out. Attached to `cf_person_records` only; `cf_behavioral` eviction
    /// stays the sweep's contract.
    ///
    /// In production this MUST be well beyond the replay horizon of the merge / `clickhouse_events_json`
    /// topics, so a live-again person's replayed events cannot arrive after their record was reclaimed.
    /// A non-zero value below [`MIN_SAFE_PERSON_RECORD_TTL_DAYS`] days warns at startup.
    /// A TTL-dropped dormant person re-derives on their next event and re-emits `Entered` for
    /// single-leaf person cohorts (at-least-once across dormancy); composable cohorts are suppressed by
    /// the surviving `cf_stage2` bit.
    #[envconfig(from = "COHORT_PERSON_RECORD_TTL_DAYS", default = "0")]
    pub cohort_person_record_ttl_days: u32,

    /// Where store I/O runs relative to the runtime worker threads: `off` (inline on the caller — the
    /// operator kill switch), `maintenance` (only maintenance-lane reads, the WAL fsync, and sections
    /// offload; the event path stays inline), or `all` (every op offloads to the blocking pool).
    /// Default `all`.
    #[envconfig(from = "COHORT_STORE_OFFLOAD_MODE", default = "all")]
    pub cohort_store_offload_mode: OffloadMode,

    /// Bound on event-path store reads executing concurrently on the blocking pool. Keeps a burst of
    /// event reads from saturating the disk queue; `0` disables the bound (unbounded lane).
    #[envconfig(from = "COHORT_STORE_EVENT_READ_PERMITS", default = "16")]
    pub cohort_store_event_read_permits: usize,

    /// Bound on maintenance-lane store reads and sections executing concurrently on the blocking
    /// pool (sweep prefetch, boot rebuild scans, GC). Held lower than the event lane so a
    /// maintenance storm leaves disk headroom for the event path; `0` disables the bound.
    #[envconfig(from = "COHORT_STORE_MAINTENANCE_PERMITS", default = "6")]
    pub cohort_store_maintenance_permits: usize,

    /// When on, reopen the existing store on restart instead of wiping it: recent Stage 1 state is
    /// restored and only the gap since the last committed offset is replayed (idempotent via per-key
    /// `AppliedOffsets`). See [`Self::effective_wipe_on_start`].
    #[envconfig(from = "DURABLE_RESTORE_ENABLED", default = "false")]
    pub durable_restore_enabled: bool,

    /// Master gate for the whole-DB checkpoint + S3 backup/restore layer. Default off.
    #[envconfig(from = "CHECKPOINT_ENABLED", default = "false")]
    pub checkpoint_enabled: bool,

    /// How often a local PVC checkpoint is taken (ms).
    #[envconfig(default = "300000")]
    pub checkpoint_interval_ms: u64,

    /// How often the local checkpoint is also uploaded to S3 (ms). The upload rides every Nth local
    /// tick (`N = max(1, this / checkpoint_interval_ms)`), so the checkpoint is taken once and
    /// conditionally uploaded — never two racing `create_checkpoint`s.
    #[envconfig(default = "900000")]
    pub checkpoint_s3_upload_interval_ms: u64,

    /// Base directory for local checkpoints. Must be an absolute path outside `store_path` and its
    /// `<store_path>.restore` staging directory, on the same filesystem (RocksDB refuses to
    /// checkpoint into its own directory and hard-links SSTs, and every boot deletes the stage).
    #[envconfig(default = "cohort-checkpoints")]
    pub checkpoint_local_dir: String,

    /// S3 bucket for checkpoint uploads. Empty (default) makes the uploader unavailable, so uploads
    /// no-op.
    #[envconfig(default = "")]
    pub checkpoint_s3_bucket: String,

    /// S3 key prefix (bucket namespace) under which checkpoint attempts are stored.
    #[envconfig(default = "cohort-stream-checkpoints")]
    pub checkpoint_s3_prefix: String,

    /// AWS region for S3 (None → SDK default / IRSA-provided region).
    #[envconfig(from = "CHECKPOINT_S3_REGION")]
    pub checkpoint_s3_region: Option<String>,

    /// S3 endpoint override for S3-compatible stores (MinIO/SeaweedFS in dev). None → real AWS S3.
    #[envconfig(from = "CHECKPOINT_S3_ENDPOINT")]
    pub checkpoint_s3_endpoint: Option<String>,

    /// S3 access key id for local dev without an IAM role. None → IRSA → env → default chain.
    #[envconfig(from = "CHECKPOINT_S3_ACCESS_KEY_ID")]
    pub checkpoint_s3_access_key_id: Option<String>,

    /// S3 secret access key for local dev without an IAM role.
    #[envconfig(from = "CHECKPOINT_S3_SECRET_ACCESS_KEY")]
    pub checkpoint_s3_secret_access_key: Option<String>,

    /// Force path-style S3 URLs (required for MinIO/SeaweedFS).
    #[envconfig(default = "false")]
    pub checkpoint_s3_force_path_style: bool,

    /// Max concurrent S3 file uploads during a checkpoint export.
    #[envconfig(default = "40")]
    pub checkpoint_max_concurrent_uploads: usize,

    /// Max concurrent S3 file downloads during a checkpoint import.
    #[envconfig(default = "40")]
    pub checkpoint_max_concurrent_downloads: usize,

    /// Max upload futures actively polled per export (bounds memory: each holds read + write
    /// buffers).
    #[envconfig(default = "40")]
    pub checkpoint_max_upload_buffers: usize,

    /// Retries per S3 operation before giving up.
    #[envconfig(default = "10")]
    pub checkpoint_s3_max_retries: usize,

    /// Total timeout for one S3 operation including retries (secs).
    #[envconfig(default = "120")]
    pub checkpoint_s3_operation_timeout_secs: u64,

    /// Timeout for a single S3 operation attempt (secs).
    #[envconfig(default = "20")]
    pub checkpoint_s3_attempt_timeout_secs: u64,

    /// Max age of a local PVC checkpoint before it is distrusted on restore and the service falls
    /// back to S3 (secs). Tighter than the S3 listing window: if a pod was down longer than this,
    /// another pod likely consumed the partitions and the local copy is behind.
    #[envconfig(default = "7200")]
    pub checkpoint_local_max_staleness_secs: u64,

    /// Hours prior to now the S3 importer searches for checkpoint attempts in a PVC-lost recovery.
    /// Keep it below the `cohort_stream_events` retention by more than a restore's catch-up time: a
    /// restore resumes about this far back, and a consumer that falls below the low watermark while
    /// it catches up resets to `earliest` without a trace, so the events that retention deleted are
    /// lost.
    #[envconfig(default = "20")]
    pub checkpoint_import_window_hours: u32,

    /// Historical S3 checkpoint attempts to try (newest first) before giving up on import.
    #[envconfig(default = "10")]
    pub checkpoint_import_attempt_depth: usize,

    /// Max time to download one S3 checkpoint candidate's files (secs). A slower candidate fails and
    /// ends the search, because older candidates are about as large. It bounds the size of a store
    /// that can restore, so keep it above a full store volume at the volume's write throughput.
    #[envconfig(default = "7200")]
    pub checkpoint_import_timeout_secs: u64,

    /// Age of an upload chain past which the next upload is full (secs). Bounds how old the oldest
    /// object a restore needs can be.
    #[envconfig(default = "86400")]
    pub checkpoint_full_upload_interval_secs: u64,

    /// Average rate one upload may read from the store's disk (bytes per second).
    #[envconfig(default = "67108864")]
    pub checkpoint_upload_max_bytes_per_sec: u64,
}

/// librdkafka consumer fetch-queue bounds: an aggregate byte cap across all partitions and a
/// per-partition prefetch floor.
#[derive(Clone, Copy, Debug)]
pub struct FetchQueueConfig {
    pub queued_max_messages_kbytes: u32,
    pub queued_min_messages: u32,
}

impl Config {
    pub fn bind_address(&self) -> String {
        format!("{}:{}", self.bind_host, self.bind_port)
    }

    pub fn filter_catalog_refresh_interval(&self) -> Duration {
        Duration::from_secs(self.filter_catalog_refresh_secs)
    }

    pub fn filter_catalog_refresh_jitter(&self) -> Duration {
        Duration::from_secs(self.filter_catalog_refresh_jitter_secs)
    }

    fn pg_statement_timeout(&self) -> Option<u64> {
        (self.pg_statement_timeout_ms != 0).then_some(self.pg_statement_timeout_ms)
    }

    pub fn pool_config(&self) -> PoolConfig {
        PoolConfig {
            min_connections: self.min_pg_connections,
            max_connections: self.max_pg_connections,
            acquire_timeout: Duration::from_secs(self.pg_acquire_timeout_secs),
            idle_timeout: Some(Duration::from_secs(300)),
            test_before_acquire: true,
            statement_timeout_ms: self.pg_statement_timeout(),
            pool_name: Some(POOL_NAME.to_string()),
        }
    }

    pub fn recv_batch_timeout(&self) -> Duration {
        Duration::from_millis(self.recv_batch_timeout_ms)
    }

    pub fn offset_commit_interval(&self) -> Duration {
        Duration::from_millis(self.offset_commit_interval_ms)
    }

    pub fn sweep_interval(&self) -> Duration {
        Duration::from_millis(self.sweep_interval_ms)
    }

    pub fn first_eviction_sweep_delay(&self) -> Duration {
        Duration::from_millis(self.first_eviction_sweep_delay_ms)
    }

    pub fn fetch_queue_config(&self) -> FetchQueueConfig {
        FetchQueueConfig {
            queued_max_messages_kbytes: self.kafka_queued_max_messages_kbytes,
            queued_min_messages: self.kafka_queued_min_messages,
        }
    }

    pub fn sweep_safety_margin(&self) -> Duration {
        Duration::from_millis(self.sweep_safety_margin_ms)
    }

    pub fn stats_publish_interval(&self) -> Duration {
        // Floor at 1s: `tokio::time::interval` panics on a zero period.
        Duration::from_secs(self.stats_publish_interval_secs).max(Duration::from_secs(1))
    }

    pub fn transfer_retry_policy(&self) -> TransferRetryPolicy {
        TransferRetryPolicy {
            max_retries: self.merge_transfer_max_retries,
            base: Duration::from_millis(self.merge_transfer_retry_base_ms),
            cap: Duration::from_millis(self.merge_transfer_retry_cap_ms),
        }
    }

    pub fn merge_redrive_interval(&self) -> Duration {
        Duration::from_millis(self.merge_redrive_interval_ms)
    }

    pub fn cascade_config(&self) -> CascadeConfig {
        CascadeConfig {
            enabled: self.cohort_cascade_enabled,
            depth_cap: self.cohort_cascade_depth_cap,
            fanout_cap: self.cohort_cascade_fanout_cap,
        }
    }

    pub fn event_name_gating(&self) -> EventNameGating {
        EventNameGating::from_enabled(self.cohort_event_name_gating_enabled)
    }

    pub fn merge_gc_interval(&self) -> Duration {
        Duration::from_millis(self.merge_gc_interval_ms)
    }

    pub fn seed_idle_probe_interval(&self) -> Duration {
        // Floor at 1s: `tokio::time::interval` panics on a zero period.
        Duration::from_millis(self.cohort_seed_watermark_idle_probe_interval_ms)
            .max(Duration::from_secs(1))
    }

    /// The run ceilings the partition workers group seeds under. Validated at startup, so a
    /// zero here is a bug rather than a config error.
    pub fn seed_run_budget(&self) -> RunBudget {
        RunBudget {
            seeds: NonZeroUsize::new(self.cohort_seed_apply_batch_max).unwrap_or(NonZeroUsize::MIN),
            rows: NonZeroUsize::new(self.cohort_seed_apply_batch_max_rows)
                .unwrap_or(NonZeroUsize::MIN),
        }
    }

    /// The seed lane's capacity for one partition worker. Validated at startup, so a zero here is
    /// a bug rather than a config error.
    pub fn seed_lane_cap(&self) -> NonZeroUsize {
        NonZeroUsize::new(self.partition_intake_max_seeds).unwrap_or(NonZeroUsize::MIN)
    }

    pub fn reconcile_tick_interval(&self) -> Duration {
        Duration::from_millis(self.cohort_seed_reconcile_tick_interval_ms)
    }

    /// The seed consumer's pacing gates. `0` on a pause threshold disables that trigger; an
    /// enabled trigger requires `0 < resume < pause` (and `pause <= 100` for the disk share) so a
    /// flapping or never-releasing pair is refused at startup.
    pub fn seed_pacing_config(&self) -> anyhow::Result<SeedPacingConfig> {
        let live_lag = if self.cohort_seed_live_lag_pause_ms == 0 {
            None
        } else {
            ensure!(
                self.cohort_seed_live_lag_pause_ms > 0,
                "COHORT_SEED_LIVE_LAG_PAUSE_MS must be positive (0 disables the trigger).",
            );
            ensure!(
                self.cohort_seed_live_lag_resume_ms > 0,
                "COHORT_SEED_LIVE_LAG_RESUME_MS must be positive: a non-positive resume \
                 threshold could never release the pause.",
            );
            Some(
                Hysteresis::new(
                    AgeMs(self.cohort_seed_live_lag_pause_ms),
                    AgeMs(self.cohort_seed_live_lag_resume_ms),
                )
                .context(
                    "COHORT_SEED_LIVE_LAG_RESUME_MS must be below COHORT_SEED_LIVE_LAG_PAUSE_MS",
                )?,
            )
        };
        let disk = if self.cohort_seed_disk_pause_pct == 0.0 {
            None
        } else {
            ensure!(
                self.cohort_seed_disk_pause_pct > 0.0 && self.cohort_seed_disk_pause_pct <= 100.0,
                "COHORT_SEED_DISK_PAUSE_PCT must be within (0, 100] (0 disables the trigger).",
            );
            ensure!(
                self.cohort_seed_disk_resume_pct > 0.0,
                "COHORT_SEED_DISK_RESUME_PCT must be positive: a non-positive resume threshold \
                 could never release the pause.",
            );
            Some(
                Hysteresis::new(
                    UsedPct(self.cohort_seed_disk_pause_pct),
                    UsedPct(self.cohort_seed_disk_resume_pct),
                )
                .context("COHORT_SEED_DISK_RESUME_PCT must be below COHORT_SEED_DISK_PAUSE_PCT")?,
            )
        };
        Ok(SeedPacingConfig { live_lag, disk })
    }

    pub fn checkpoint_interval(&self) -> Duration {
        Duration::from_millis(self.checkpoint_interval_ms)
    }

    pub fn checkpoint_s3_upload_interval(&self) -> Duration {
        Duration::from_millis(self.checkpoint_s3_upload_interval_ms)
    }

    pub fn checkpoint_local_max_staleness(&self) -> Duration {
        Duration::from_secs(self.checkpoint_local_max_staleness_secs)
    }

    pub fn durability_config(&self) -> DurabilityConfig {
        DurabilityConfig {
            local_checkpoint_dir: self.checkpoint_local_dir.clone(),
            s3_bucket: self.checkpoint_s3_bucket.clone(),
            s3_key_prefix: self.checkpoint_s3_prefix.clone(),
            aws_region: self.checkpoint_s3_region.clone(),
            s3_endpoint: self.checkpoint_s3_endpoint.clone(),
            s3_access_key_id: self.checkpoint_s3_access_key_id.clone(),
            s3_secret_access_key: self.checkpoint_s3_secret_access_key.clone(),
            s3_force_path_style: self.checkpoint_s3_force_path_style,
            checkpoint_import_window_hours: self.checkpoint_import_window_hours,
            s3_operation_timeout: Duration::from_secs(self.checkpoint_s3_operation_timeout_secs),
            s3_attempt_timeout: Duration::from_secs(self.checkpoint_s3_attempt_timeout_secs),
            s3_max_retries: self.checkpoint_s3_max_retries,
            checkpoint_import_attempt_depth: self.checkpoint_import_attempt_depth,
            max_concurrent_checkpoint_file_downloads: self.checkpoint_max_concurrent_downloads,
            max_concurrent_checkpoint_file_uploads: self.checkpoint_max_concurrent_uploads,
            max_upload_buffers: self.checkpoint_max_upload_buffers,
            checkpoint_import_timeout: Duration::from_secs(self.checkpoint_import_timeout_secs),
            local_checkpoint_max_staleness: self.checkpoint_local_max_staleness(),
            full_upload_interval: Duration::from_secs(self.checkpoint_full_upload_interval_secs),
            upload_max_bytes_per_sec: self.checkpoint_upload_max_bytes_per_sec,
        }
    }

    /// Refuse unsafe startup combinations (durability, reconcile, and pacing knobs). Pure
    /// (no I/O), so unit-testable without a broker.
    ///
    /// Guards:
    /// - Reconcile scan and tick limits must be non-zero; a zero page would falsely certify an
    ///   unscanned snapshot, while Tokio rejects a zero timer interval.
    /// - `checkpoint_enabled` requires `durable_restore_enabled`: without it the restored DB is wiped
    ///   on open, silently discarding the restore. It also requires a bucket, non-zero knobs, an
    ///   absolute checkpoint directory apart from the store, and a pod name with an ordinal, so a
    ///   misconfigured pod fails at start instead of running without checkpoints.
    pub fn validate_startup(&self) -> anyhow::Result<()> {
        ensure!(
            self.cohort_seed_reconcile_scan_page > 0,
            "COHORT_SEED_RECONCILE_SCAN_PAGE must be greater than zero.",
        );
        ensure!(
            self.cohort_seed_reconcile_tick_interval_ms > 0,
            "COHORT_SEED_RECONCILE_TICK_INTERVAL_MS must be greater than zero.",
        );
        // A zero run ceiling would form no run at all, so every seed would be neither marked nor
        // held and the partition would wedge behind the first one.
        ensure!(
            self.cohort_seed_apply_batch_max > 0,
            "COHORT_SEED_APPLY_BATCH_MAX must be greater than zero (1 = one seed per run).",
        );
        ensure!(
            self.cohort_seed_apply_batch_max_rows > 0,
            "COHORT_SEED_APPLY_BATCH_MAX_ROWS must be greater than zero.",
        );
        // A zero-capacity mpsc channel panics on construction, and so does one above tokio's
        // permit bound, so either would take the pod down at the first worker spawn.
        ensure!(
            self.partition_intake_max_seeds > 0,
            "PARTITION_INTAKE_MAX_SEEDS must be greater than zero (it is the seed lane's capacity in seeds).",
        );
        ensure!(
            self.partition_intake_max_seeds <= tokio::sync::Semaphore::MAX_PERMITS,
            "PARTITION_INTAKE_MAX_SEEDS must not exceed {} (tokio's channel capacity bound).",
            tokio::sync::Semaphore::MAX_PERMITS,
        );

        let pacing = self.seed_pacing_config()?;
        // Idle partitions' watermarks advance only once per probe, so a pause threshold inside
        // two probe intervals would flap on quiet partitions.
        let idle_probe_ms =
            i64::try_from(self.cohort_seed_watermark_idle_probe_interval_ms).unwrap_or(i64::MAX);
        if pacing.live_lag.is_some()
            && self.cohort_seed_live_lag_pause_ms <= idle_probe_ms.saturating_mul(2)
        {
            warn!(
                cohort_seed_live_lag_pause_ms = self.cohort_seed_live_lag_pause_ms,
                idle_probe_interval_ms = self.cohort_seed_watermark_idle_probe_interval_ms,
                "COHORT_SEED_LIVE_LAG_PAUSE_MS within 2× the idle-probe interval: idle \
                 partitions advance only per probe, so the live-lag gate may flap on quiet \
                 partitions.",
            );
        }

        if self.cohort_seed_reconcile_enabled && !self.cohort_seed_consumer_enabled {
            warn!(
                "COHORT_SEED_RECONCILE_ENABLED without COHORT_SEED_CONSUMER_ENABLED: no reconcile \
                 controls can be consumed; enable the seed consumer or turn reconcile off.",
            );
        }

        // A negative margin biases the person-seed verdict toward the seed, letting a stale scan
        // overwrite fresher live state.
        ensure!(
            self.cohort_seed_person_live_margin_ms >= 0,
            "COHORT_SEED_PERSON_LIVE_MARGIN_MS must not be negative.",
        );

        if self.cohort_seed_person_apply_enabled && !self.cohort_seed_consumer_enabled {
            warn!(
                "COHORT_SEED_PERSON_APPLY_ENABLED without COHORT_SEED_CONSUMER_ENABLED: no person \
                 seeds can be consumed; enable the seed consumer or turn person apply off.",
            );
        }

        // Seed redelivery repairs failed produces only when the register preserves the retired
        // value. Legacy seed writers stored truth before producing, so a held seed replayed across
        // that writer change can read an agreeing row and emit nothing. Reconcile must cover those
        // deliveries, cohort edits, and live/merge produces whose state already committed.
        if self.cohort_seed_person_apply_enabled && !self.cohort_seed_reconcile_enabled {
            warn!(
                "COHORT_SEED_PERSON_APPLY_ENABLED without COHORT_SEED_RECONCILE_ENABLED: a live \
                 event that fails its produce, a cohort edited mid-run, and a merge apply that \
                 fails its produce have no repair path. Failed deliveries from seed writers \
                 that stored truth before producing also require reconcile.",
            );
        }

        ensure!(
            !self.checkpoint_enabled || self.durable_restore_enabled,
            "CHECKPOINT_ENABLED requires DURABLE_RESTORE_ENABLED: restoring a checkpoint without \
             reopen-live is meaningless.",
        );
        if self.checkpoint_enabled {
            self.validate_checkpoints()?;
        }

        // Not a refusal (dev runs with durability off), but a crash then loses applied tiles
        // that never replay.
        if self.cohort_seed_consumer_enabled && !self.durable_restore_enabled {
            warn!(
                "COHORT_SEED_CONSUMER_ENABLED without DURABLE_RESTORE_ENABLED: a committed seed \
                 offset does not imply a durably applied tile across restarts; do not stamp \
                 backfill readiness against this deployment.",
            );
        }

        Ok(())
    }

    fn validate_checkpoints(&self) -> anyhow::Result<()> {
        ensure!(
            !self.checkpoint_s3_bucket.is_empty(),
            "CHECKPOINT_ENABLED requires CHECKPOINT_S3_BUCKET.",
        );
        let knobs: [(&str, u64); 13] = [
            ("CHECKPOINT_INTERVAL_MS", self.checkpoint_interval_ms),
            (
                "CHECKPOINT_S3_UPLOAD_INTERVAL_MS",
                self.checkpoint_s3_upload_interval_ms,
            ),
            (
                "CHECKPOINT_FULL_UPLOAD_INTERVAL_SECS",
                self.checkpoint_full_upload_interval_secs,
            ),
            (
                "CHECKPOINT_MAX_CONCURRENT_UPLOADS",
                self.checkpoint_max_concurrent_uploads as u64,
            ),
            (
                "CHECKPOINT_MAX_CONCURRENT_DOWNLOADS",
                self.checkpoint_max_concurrent_downloads as u64,
            ),
            (
                "CHECKPOINT_MAX_UPLOAD_BUFFERS",
                self.checkpoint_max_upload_buffers as u64,
            ),
            (
                "CHECKPOINT_IMPORT_WINDOW_HOURS",
                u64::from(self.checkpoint_import_window_hours),
            ),
            (
                "CHECKPOINT_LOCAL_MAX_STALENESS_SECS",
                self.checkpoint_local_max_staleness_secs,
            ),
            (
                "CHECKPOINT_IMPORT_ATTEMPT_DEPTH",
                self.checkpoint_import_attempt_depth as u64,
            ),
            (
                "CHECKPOINT_S3_OPERATION_TIMEOUT_SECS",
                self.checkpoint_s3_operation_timeout_secs,
            ),
            (
                "CHECKPOINT_S3_ATTEMPT_TIMEOUT_SECS",
                self.checkpoint_s3_attempt_timeout_secs,
            ),
            (
                "CHECKPOINT_IMPORT_TIMEOUT_SECS",
                self.checkpoint_import_timeout_secs,
            ),
            (
                "CHECKPOINT_UPLOAD_MAX_BYTES_PER_SEC",
                self.checkpoint_upload_max_bytes_per_sec,
            ),
        ];
        for (name, value) in knobs {
            ensure!(value > 0, "{name} must be greater than zero.");
        }
        let checkpoints = Path::new(&self.checkpoint_local_dir);
        let store = Path::new(&self.store_path);
        // `starts_with` compares path components, so a `..` could name the store or its staging
        // directory under a spelling the checks below do not match.
        let plain = |path: &Path| {
            path.is_absolute() && !path.components().any(|part| part == Component::ParentDir)
        };
        ensure!(
            plain(checkpoints) && plain(store),
            "CHECKPOINT_LOCAL_DIR {checkpoints:?} and STORE_PATH {store:?} must be absolute paths \
             without `..`.",
        );
        ensure!(
            !checkpoints.starts_with(store) && !store.starts_with(checkpoints),
            "CHECKPOINT_LOCAL_DIR {checkpoints:?} and STORE_PATH {store:?} must not contain one \
             another: RocksDB refuses a checkpoint inside its own directory.",
        );
        let staging = staging_path(store);
        ensure!(
            !checkpoints.starts_with(&staging),
            "CHECKPOINT_LOCAL_DIR {checkpoints:?} must not be inside {staging:?}, where a restore \
             stages a checkpoint: every boot deletes that directory.",
        );
        self.checkpoint_lineage().context(
            "POD_NAME must end in a StatefulSet ordinal, or be unset outside Kubernetes",
        )?;
        Ok(())
    }

    /// The ordinal at the end of `POD_NAME`, or `None` for a process without `POD_NAME` (a local
    /// run or a test). `HOSTNAME` is not consulted: outside Kubernetes it names the machine, not a
    /// StatefulSet pod.
    fn pod_ordinal(&self) -> Result<Option<PodOrdinal>, NotAnOrdinalPod> {
        self.pod_name
            .as_deref()
            .filter(|name| !name.is_empty())
            .map(PodOrdinal::from_pod_name)
            .transpose()
    }

    /// The checkpoint lineage of this pod, from its ordinal. A process without `POD_NAME` uses the
    /// single pod's lineage.
    pub fn checkpoint_lineage(&self) -> Result<CheckpointLineage, NotAnOrdinalPod> {
        Ok(CheckpointLineage::new(
            self.pod_ordinal()?.unwrap_or(PodOrdinal::STANDALONE),
        ))
    }

    /// This pod's split and its place in a reshard: pod `ordinal` of `COHORT_POD_COUNT`, over
    /// `COHORT_PARTITION_COUNT` partitions, resharded from `COHORT_RESHARD_FROM_POD_COUNT` pods
    /// when that is set.
    pub fn split_claim(&self) -> anyhow::Result<SplitClaim> {
        ensure!(
            self.cohort_reshard_from_pod_count.is_none()
                || self.checkpoint_enabled
                || !self.durable_restore_enabled,
            "COHORT_RESHARD_FROM_POD_COUNT with DURABLE_RESTORE_ENABLED requires CHECKPOINT_ENABLED: \
             without checkpoints each child starts empty while its parent deletes the partitions it \
             gave away",
        );
        SplitClaim::new(
            self.partition_ownership()?,
            self.cohort_reshard_from_pod_count,
        )
        .context("COHORT_RESHARD_FROM_POD_COUNT must divide COHORT_POD_COUNT and be below it")
    }

    /// The partitions this pod owns. Only a single pod may run without `POD_NAME`: every pod
    /// without it would take ordinal 0, and they would all own the same partitions.
    fn partition_ownership(&self) -> anyhow::Result<PartitionOwnership> {
        let ordinal = match self
            .pod_ordinal()
            .context("POD_NAME must end in a StatefulSet ordinal, or be unset outside Kubernetes")?
        {
            Some(ordinal) => ordinal,
            None => {
                ensure!(
                    self.cohort_pod_count == NonZeroU16::MIN,
                    "COHORT_POD_COUNT={} requires POD_NAME, so that each pod owns its own partitions",
                    self.cohort_pod_count,
                );
                PodOrdinal::STANDALONE
            }
        };
        PartitionOwnership::new(self.cohort_pod_count, ordinal, self.partition_count())
            .with_context(|| {
                format!(
                    "COHORT_POD_COUNT={} must divide COHORT_PARTITION_COUNT={}, and the ordinal of \
                     POD_NAME must be below it",
                    self.cohort_pod_count, self.cohort_partition_count,
                )
            })
    }

    /// `COHORT_PARTITION_COUNT` as a slice id range. A count past `u16::MAX` fails the topic checks
    /// at startup.
    pub fn partition_count(&self) -> u16 {
        u16::try_from(self.cohort_partition_count).unwrap_or(u16::MAX)
    }

    /// Whether to wipe the store on start, folding in the durable-restore gate: keep the live store
    /// only when restore is on and one already exists on disk. Touches the filesystem (`.exists()`) —
    /// a one-shot decision evaluated once at store open.
    pub fn effective_wipe_on_start(&self) -> bool {
        let db_dir_exists = PathBuf::from(&self.store_path).exists();
        self.wipe_store_on_start && !(self.durable_restore_enabled && db_dir_exists)
    }

    pub fn store_config(&self, ownership: PartitionOwnership) -> StoreConfig {
        if self.person_record_ttl_below_safe_floor() {
            warn!(
                cohort_person_record_ttl_days = self.cohort_person_record_ttl_days,
                safe_floor_days = MIN_SAFE_PERSON_RECORD_TTL_DAYS,
                "COHORT_PERSON_RECORD_TTL_DAYS is inside the replay horizon: a reclaimed record \
                 loses its replay dedup, so replayed events for a live-again person re-fold and \
                 single-leaf person cohorts re-emit Entered; raise it to the safe floor or above",
            );
        }
        StoreConfig {
            path: PathBuf::from(&self.store_path),
            wipe_on_start: self.effective_wipe_on_start(),
            wipe_on_schema_mismatch: self.cohort_wipe_on_schema_mismatch,
            statistics_enabled: self.store_statistics_enabled,
            read_sample_ratio: self.store_read_sample_ratio,
            block_cache_bytes: self.cohort_block_cache_bytes,
            tuned_block_options: self.cohort_tuned_block_options_enabled,
            compact_on_deletion: self.cohort_compact_on_deletion_enabled,
            periodic_compaction_seconds: self.cohort_periodic_compaction_seconds,
            max_background_jobs: self.cohort_max_background_jobs,
            max_log_file_size_bytes: self.cohort_max_log_file_size_bytes,
            keep_log_file_num: self.cohort_keep_log_file_num,
            person_record_ttl_days: self.cohort_person_record_ttl_days,
            ownership,
            ..StoreConfig::default()
        }
    }

    /// Whether a configured (non-zero) person-record TTL sits inside
    /// [`MIN_SAFE_PERSON_RECORD_TTL_DAYS`] — a misconfiguration worth a startup warning, not a
    /// refusal: the knob is operator-owned and `0` (off) is always safe.
    fn person_record_ttl_below_safe_floor(&self) -> bool {
        (1..MIN_SAFE_PERSON_RECORD_TTL_DAYS).contains(&self.cohort_person_record_ttl_days)
    }

    /// Resolve the store-offload strategy and per-lane concurrency bounds handed to the
    /// [`StoreHandle`](crate::store::StoreHandle).
    pub fn offload_config(&self) -> OffloadConfig {
        OffloadConfig {
            mode: self.cohort_store_offload_mode,
            event_read_permits: self.cohort_store_event_read_permits,
            maintenance_permits: self.cohort_store_maintenance_permits,
        }
    }

    /// Stable per-pod name for the Kafka `client.id`, `POD_NAME` preferred over `HOSTNAME`.
    pub fn pod_identity(&self) -> Option<&str> {
        [self.pod_name.as_deref(), self.pod_hostname.as_deref()]
            .into_iter()
            .flatten()
            .find(|id| !id.is_empty())
    }

    /// Build the `rdkafka` client config for the `cohort_stream_events` consumer: the follower
    /// config on the events group, with a bounded fetch buffer.
    pub fn consumer_client_config(&self) -> ClientConfig {
        let mut config = self.follower_client_config(&self.kafka_consumer_group);
        // Bound librdkafka's fetch buffer: an aggregate byte ceiling plus a per-partition prefetch floor.
        let fetch = self.fetch_queue_config();
        config
            .set(
                "queued.max.messages.kbytes",
                fetch.queued_max_messages_kbytes.to_string(),
            )
            .set("queued.min.messages", fetch.queued_min_messages.to_string());
        config
    }

    /// Build the `rdkafka` client config for a consumer of `group`: the events consumer or a
    /// follower.
    ///
    /// Auto-commit and auto-offset-store are off: a consume loop marks offsets only after its
    /// messages are processed. Every consumer assigns itself the partitions this pod owns and never
    /// joins its group, so `group.id` only names where its commits land. `auto.offset.reset` is
    /// hard-coded to `earliest`: a group without members is `Empty` to the broker, which can expire
    /// its committed offsets, and a tail reset would then silently skip the messages in between. It
    /// sets no `group.instance.id`: librdkafka sends that id with every offset commit, and a broker
    /// can refuse a commit that names a static member from a client outside the group.
    pub fn follower_client_config(&self, group: &str) -> ClientConfig {
        let mut config = ClientConfig::new();
        config
            .set("bootstrap.servers", &self.kafka_hosts)
            .set("group.id", group)
            .set("enable.auto.commit", "false")
            .set("enable.auto.offset.store", "false")
            .set("auto.offset.reset", "earliest")
            .set("socket.timeout.ms", "10000")
            .set("message.max.bytes", CONSUMER_MESSAGE_MAX_BYTES.to_string());

        if let Some(id) = self.pod_identity() {
            config.set("client.id", id);
        }
        if !self.kafka_client_id.is_empty() {
            config.set("client.id", &self.kafka_client_id);
        }
        if !self.kafka_client_rack.is_empty() {
            config.set("client.rack", &self.kafka_client_rack);
        }
        if self.kafka_tls {
            config
                .set("security.protocol", "ssl")
                .set("enable.ssl.certificate.verification", "false");
        }
        config
    }

    pub fn build_kafka_config(&self) -> KafkaConfig {
        KafkaConfig {
            kafka_hosts: self.kafka_hosts.clone(),
            kafka_tls: self.kafka_tls,
            kafka_client_rack: self.kafka_client_rack.clone(),
            kafka_client_id: self.kafka_client_id.clone(),
            kafka_compression_codec: self.kafka_compression_codec.clone(),
            kafka_producer_partitioner: Some(self.kafka_producer_partitioner.clone()),
            kafka_producer_linger_ms: 20,
            kafka_producer_queue_mib: 400,
            kafka_producer_queue_messages: 10_000_000,
            kafka_message_timeout_ms: 20_000,
            kafka_producer_batch_size: None,
            kafka_producer_batch_num_messages: None,
            kafka_producer_enable_idempotence: None,
            kafka_producer_max_in_flight_requests_per_connection: None,
            kafka_producer_topic_metadata_refresh_interval_ms: None,
            kafka_producer_message_max_bytes: None,
            kafka_producer_sticky_partitioning_linger_ms: None,
            kafka_producer_acks: None,
            kafka_producer_retries: None,
        }
    }

    /// Producer config for the merge state-transfer sink: the shared config with a shorter
    /// `message.timeout.ms`. The transfer produce runs inline on a partition worker under a bounded
    /// retry loop, so a long per-attempt timeout multiplies into a worst-case worker hold that blows
    /// the 30 s graceful-shutdown window (see the budget arithmetic in `workers::merge_path`).
    pub fn build_transfer_kafka_config(&self) -> KafkaConfig {
        KafkaConfig {
            kafka_message_timeout_ms: self.merge_transfer_message_timeout_ms,
            ..self.build_kafka_config()
        }
    }

    /// Producer config for the reconcile-marker sink: the shared config with a shorter
    /// `message.timeout.ms`. Same inline-on-a-worker reason as the transfer sink, except the retry
    /// is the drain sweeper's rather than an inline loop, so an unreachable marker topic would
    /// otherwise hold the worker for the full timeout on every tick.
    pub fn build_marker_kafka_config(&self) -> KafkaConfig {
        KafkaConfig {
            kafka_message_timeout_ms: self.reconcile_marker_message_timeout_ms,
            ..self.build_kafka_config()
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn test_config() -> Config {
        Config {
            bind_host: "0.0.0.0".to_string(),
            bind_port: 3323,
            export_prometheus: true,
            database_url: "postgres://posthog:posthog@localhost:5432/posthog".to_string(),
            min_pg_connections: 1,
            max_pg_connections: 5,
            pg_acquire_timeout_secs: 10,
            pg_statement_timeout_ms: 5000,
            filter_catalog_refresh_secs: 300,
            filter_catalog_refresh_jitter_secs: 60,
            team_allowlist: TeamAllowlist::All,
            cohort_event_name_gating_enabled: true,
            partition_channel_buffer: 128,
            partition_intake_max_events: 1024,
            partition_intake_max_seeds: 1024,
            kafka_hosts: "localhost:9092".to_string(),
            kafka_tls: false,
            kafka_client_id: String::new(),
            kafka_client_rack: String::new(),
            cohort_stream_events_topic: "cohort_stream_events".to_string(),
            kafka_consumer_group: "cohort-stream-processor".to_string(),
            kafka_queued_max_messages_kbytes: 131_072,
            kafka_queued_min_messages: 2000,
            person_merge_events_topic: "person_merge_events".to_string(),
            cohort_merge_state_transfer_topic: "cohort_merge_state_transfer".to_string(),
            kafka_merge_consumer_group: "cohort-stream-merges".to_string(),
            kafka_merge_apply_consumer_group: "cohort-stream-merge-apply".to_string(),
            merge_transfer_max_retries: 5,
            merge_transfer_retry_base_ms: 500,
            merge_transfer_retry_cap_ms: 8000,
            merge_transfer_message_timeout_ms: 2000,
            merge_redrive_interval_ms: 60_000,
            merge_marker_retention_ms: 691_200_000,
            merge_tombstone_retention_ms: 777_600_000,
            merge_gc_interval_ms: 3_600_000,
            merge_gc_scan_limit: 10_000,
            stage2_orphan_gc_enabled: true,
            cohort_cascade_depth_cap: 8,
            cohort_cascade_fanout_cap: 1000,
            cohort_cascade_enabled: false,
            cohort_cascade_events_topic: "cohort_cascade_events".to_string(),
            kafka_cascade_consumer_group: "cohort-stream-cascade".to_string(),
            pod_name: None,
            cohort_pod_count: NonZeroU16::MIN,
            cohort_reshard_from_pod_count: None,
            pod_hostname: None,
            cohort_membership_changed_topic: "cohort_membership_changed_shadow".to_string(),
            cohort_reconcile_markers_topic: "cohort_reconcile_markers".to_string(),
            reconcile_marker_message_timeout_ms: 1000,
            kafka_producer_partitioner: "murmur2_random".to_string(),
            cohort_partition_count: 64,
            kafka_compression_codec: "none".to_string(),
            recv_batch_size: 1000,
            recv_batch_timeout_ms: 500,
            offset_commit_interval_ms: 5000,
            tokio_worker_threads: 0,
            sweep_interval_ms: 30000,
            first_eviction_sweep_delay_ms: 120_000,
            sweep_safety_margin_ms: 300000,
            stats_publish_interval_secs: 15,
            store_path: "cohort-store".to_string(),
            wipe_store_on_start: true,
            cohort_wipe_on_schema_mismatch: false,
            store_statistics_enabled: true,
            store_read_sample_ratio: 64,
            cohort_block_cache_bytes: 134_217_728,
            cohort_tuned_block_options_enabled: true,
            cohort_compact_on_deletion_enabled: true,
            cohort_periodic_compaction_seconds: 0,
            cohort_max_background_jobs: 0,
            cohort_max_log_file_size_bytes: NonZeroUsize::new(268_435_456).unwrap(),
            cohort_keep_log_file_num: NonZeroUsize::new(10).unwrap(),
            cohort_person_record_ttl_days: 0,
            cohort_store_offload_mode: OffloadMode::All,
            cohort_store_event_read_permits: 16,
            cohort_store_maintenance_permits: 6,
            durable_restore_enabled: false,
            checkpoint_enabled: false,
            checkpoint_interval_ms: 300_000,
            checkpoint_s3_upload_interval_ms: 900_000,
            checkpoint_local_dir: "cohort-checkpoints".to_string(),
            checkpoint_s3_bucket: String::new(),
            checkpoint_s3_prefix: "cohort-stream-checkpoints".to_string(),
            checkpoint_s3_region: None,
            checkpoint_s3_endpoint: None,
            checkpoint_s3_access_key_id: None,
            checkpoint_s3_secret_access_key: None,
            checkpoint_s3_force_path_style: false,
            checkpoint_max_concurrent_uploads: 40,
            checkpoint_max_concurrent_downloads: 40,
            checkpoint_max_upload_buffers: 40,
            checkpoint_s3_max_retries: 10,
            checkpoint_s3_operation_timeout_secs: 120,
            checkpoint_s3_attempt_timeout_secs: 20,
            checkpoint_local_max_staleness_secs: 7200,
            checkpoint_import_window_hours: 20,
            checkpoint_import_attempt_depth: 10,
            checkpoint_import_timeout_secs: 7200,
            checkpoint_full_upload_interval_secs: 86_400,
            checkpoint_upload_max_bytes_per_sec: 64 * 1024 * 1024,
            cohort_seed_consumer_enabled: false,
            cohort_stream_seed_events_topic: "cohort_stream_seed_events".to_string(),
            kafka_seed_consumer_group: "cohort-stream-seeds".to_string(),
            cohort_seed_fence_margin_ms: 600_000,
            cohort_seed_watermark_idle_probe_interval_ms: 30_000,
            cohort_register_transfer_enabled: false,
            cohort_seed_reconcile_enabled: false,
            cohort_seed_reconcile_scan_page: 256,
            cohort_seed_reconcile_tick_interval_ms: 2_000,
            cohort_seed_person_apply_enabled: false,
            cohort_seed_person_live_margin_ms: 900_000,
            cohort_seed_apply_batch_max: 256,
            cohort_seed_apply_batch_max_rows: 4096,
            cohort_seed_live_lag_pause_ms: 120_000,
            cohort_seed_live_lag_resume_ms: 60_000,
            cohort_seed_disk_pause_pct: 60.0,
            cohort_seed_disk_resume_pct: 55.0,
        }
    }

    #[test]
    fn refresh_interval_and_jitter_map_from_seconds() {
        let config = test_config();
        assert_eq!(
            config.filter_catalog_refresh_interval(),
            Duration::from_secs(300)
        );
        assert_eq!(
            config.filter_catalog_refresh_jitter(),
            Duration::from_secs(60)
        );
    }

    #[test]
    fn reconcile_knobs_default_dark_and_override_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert!(!defaults.cohort_seed_reconcile_enabled);
        assert_eq!(defaults.cohort_seed_reconcile_scan_page, 256);
        assert_eq!(
            defaults.reconcile_tick_interval(),
            Duration::from_millis(2_000)
        );
        // The marker produce blocks a partition worker, so its timeout has to stay under the tick
        // that re-queues it — at parity a black-holed topic occupies the worker end to end.
        assert_eq!(
            defaults
                .build_marker_kafka_config()
                .kafka_message_timeout_ms,
            1000,
        );
        assert!(
            u64::from(defaults.reconcile_marker_message_timeout_ms)
                < defaults.reconcile_tick_interval().as_millis() as u64
        );

        let env: std::collections::HashMap<String, String> = [
            ("COHORT_SEED_RECONCILE_ENABLED", "true"),
            ("COHORT_SEED_RECONCILE_SCAN_PAGE", "17"),
            ("COHORT_SEED_RECONCILE_TICK_INTERVAL_MS", "345"),
            ("RECONCILE_MARKER_MESSAGE_TIMEOUT_MS", "125"),
        ]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        assert!(config.cohort_seed_reconcile_enabled);
        assert_eq!(config.cohort_seed_reconcile_scan_page, 17);
        assert_eq!(config.reconcile_tick_interval(), Duration::from_millis(345));
        assert_eq!(
            config.build_marker_kafka_config().kafka_message_timeout_ms,
            125,
        );
    }

    #[test]
    fn reconcile_startup_validation_rejects_zero_work_limits() {
        let mut config = test_config();
        config.cohort_seed_reconcile_scan_page = 0;
        assert!(config
            .validate_startup()
            .unwrap_err()
            .to_string()
            .contains("COHORT_SEED_RECONCILE_SCAN_PAGE"),);

        config.cohort_seed_reconcile_scan_page = 1;
        config.cohort_seed_reconcile_tick_interval_ms = 0;
        assert!(config
            .validate_startup()
            .unwrap_err()
            .to_string()
            .contains("COHORT_SEED_RECONCILE_TICK_INTERVAL_MS"),);
    }

    /// A zero ceiling would form no run at all, so every seed would be neither marked nor held
    /// and the partition would wedge behind the first one.
    #[test]
    fn seed_run_budget_rejects_zero_ceilings_and_maps_the_defaults() {
        let defaults = test_config();
        assert_eq!(defaults.seed_run_budget().seeds.get(), 256);
        assert_eq!(defaults.seed_run_budget().rows.get(), 4096);

        let mut config = test_config();
        config.cohort_seed_apply_batch_max = 0;
        assert!(config
            .validate_startup()
            .unwrap_err()
            .to_string()
            .contains("COHORT_SEED_APPLY_BATCH_MAX"),);

        config.cohort_seed_apply_batch_max = 1;
        config.cohort_seed_apply_batch_max_rows = 0;
        assert!(config
            .validate_startup()
            .unwrap_err()
            .to_string()
            .contains("COHORT_SEED_APPLY_BATCH_MAX_ROWS"),);

        // A zero-capacity seed lane would panic the first worker spawn, and so would one above
        // tokio's channel bound, so both must be refused too.
        config.cohort_seed_apply_batch_max_rows = 1;
        for cap in [0, usize::MAX] {
            config.partition_intake_max_seeds = cap;
            assert!(config
                .validate_startup()
                .unwrap_err()
                .to_string()
                .contains("PARTITION_INTAKE_MAX_SEEDS"),);
        }

        // `1` is the documented hatch back to the per-seed apply, so it must start.
        config.partition_intake_max_seeds = 1;
        assert!(config.validate_startup().is_ok());
        assert_eq!(config.seed_run_budget().seeds.get(), 1);
    }

    #[test]
    fn reconcile_without_the_seed_consumer_warns_but_starts() {
        let mut config = test_config();
        config.cohort_seed_reconcile_enabled = true;
        config.cohort_seed_consumer_enabled = false;

        assert!(config.validate_startup().is_ok());
    }

    #[test]
    fn person_seed_knobs_default_dark_and_refuse_a_negative_live_margin() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert!(!defaults.cohort_seed_person_apply_enabled);
        assert_eq!(defaults.cohort_seed_person_live_margin_ms, 900_000);

        let env: std::collections::HashMap<String, String> = [
            ("COHORT_SEED_PERSON_APPLY_ENABLED", "true"),
            ("COHORT_SEED_PERSON_LIVE_MARGIN_MS", "1234"),
        ]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        assert!(config.cohort_seed_person_apply_enabled);
        assert_eq!(config.cohort_seed_person_live_margin_ms, 1234);

        let mut negative = test_config();
        negative.cohort_seed_person_live_margin_ms = -1;
        assert!(negative
            .validate_startup()
            .unwrap_err()
            .to_string()
            .contains("COHORT_SEED_PERSON_LIVE_MARGIN_MS"));

        // Apply-on without the consumer is a warning, not a refusal — same as reconcile.
        let mut orphaned = test_config();
        orphaned.cohort_seed_person_apply_enabled = true;
        orphaned.cohort_seed_consumer_enabled = false;
        assert!(orphaned.validate_startup().is_ok());
    }

    /// A flapping (inverted/equal) threshold pair or a never-releasing resume must be refused at
    /// startup, not discovered as a wedged or flapping gate in production.
    #[test]
    fn seed_pacing_validation_rejects_inverted_equal_and_non_positive_thresholds() {
        let defaults = test_config();
        let pacing = defaults.seed_pacing_config().unwrap();
        assert!(pacing.live_lag.is_some());
        assert!(pacing.disk.is_some());

        type Mutation = fn(&mut Config);
        let cases: [(&str, Mutation); 6] = [
            ("COHORT_SEED_LIVE_LAG_RESUME_MS must be below", |config| {
                config.cohort_seed_live_lag_resume_ms = 120_000; // equal
            }),
            ("COHORT_SEED_LIVE_LAG_RESUME_MS must be below", |config| {
                config.cohort_seed_live_lag_resume_ms = 240_000; // inverted
            }),
            (
                "COHORT_SEED_LIVE_LAG_RESUME_MS must be positive",
                |config| {
                    config.cohort_seed_live_lag_resume_ms = -1;
                },
            ),
            ("COHORT_SEED_LIVE_LAG_PAUSE_MS must be positive", |config| {
                config.cohort_seed_live_lag_pause_ms = -1;
            }),
            ("COHORT_SEED_DISK_RESUME_PCT must be below", |config| {
                config.cohort_seed_disk_resume_pct = 60.0; // equal
            }),
            ("COHORT_SEED_DISK_PAUSE_PCT must be within", |config| {
                config.cohort_seed_disk_pause_pct = 101.0;
            }),
        ];
        for (expected, mutate) in cases {
            let mut config = test_config();
            mutate(&mut config);
            let err = config.validate_startup().unwrap_err().to_string();
            assert!(err.contains(expected), "expected {expected:?} in {err:?}");
        }
    }

    /// The env defaults must ship the disk gate dark: it can engage on deployments that already
    /// run the seed consumer, and a threshold below the store's steady-state footprint would
    /// never release.
    #[test]
    fn seed_pacing_env_defaults_enable_live_lag_and_disable_disk() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        let pacing = defaults.seed_pacing_config().unwrap();
        assert!(pacing.live_lag.is_some());
        assert!(pacing.disk.is_none());
    }

    #[test]
    fn seed_pacing_zero_pause_thresholds_disable_their_triggers() {
        let mut config = test_config();
        config.cohort_seed_live_lag_pause_ms = 0;
        config.cohort_seed_disk_pause_pct = 0.0;
        // Resume thresholds are irrelevant while disabled — even nonsense must not refuse boot.
        config.cohort_seed_live_lag_resume_ms = -5;
        config.cohort_seed_disk_resume_pct = -5.0;

        let pacing = config.seed_pacing_config().unwrap();
        assert!(pacing.live_lag.is_none());
        assert!(pacing.disk.is_none());
        assert!(config.validate_startup().is_ok());
    }

    #[test]
    fn sweep_interval_and_safety_margin_map_from_millis() {
        let config = test_config();
        assert_eq!(config.sweep_interval(), Duration::from_millis(30_000));
        assert_eq!(config.sweep_safety_margin(), Duration::from_millis(300_000));
    }

    #[test]
    fn statement_timeout_zero_means_database_default() {
        let mut config = test_config();
        config.pg_statement_timeout_ms = 0;
        assert!(config.pool_config().statement_timeout_ms.is_none());
    }

    #[test]
    fn pool_config_uses_the_named_pool() {
        let config = test_config();
        assert_eq!(config.pool_config().pool_name.as_deref(), Some(POOL_NAME));
    }

    #[test]
    fn consumer_config_disables_auto_commit_and_offset_store() {
        let config = test_config();
        let client = config.consumer_client_config();
        assert_eq!(client.get("enable.auto.commit"), Some("false"));
        assert_eq!(client.get("enable.auto.offset.store"), Some("false"));
        assert_eq!(client.get("group.id"), Some("cohort-stream-processor"));
        assert_eq!(client.get("auto.offset.reset"), Some("earliest"));
        assert_eq!(client.get("bootstrap.servers"), Some("localhost:9092"));
    }

    #[test]
    fn consumer_config_sets_the_fetch_queue_bounds() {
        let config = test_config();
        let client = config.consumer_client_config();
        assert_eq!(client.get("queued.max.messages.kbytes"), Some("131072"));
        assert_eq!(client.get("queued.min.messages"), Some("2000"));
        assert_eq!(client.get("message.max.bytes"), Some("8388608"));
        assert_eq!(
            config
                .follower_client_config("cohort-stream-seeds")
                .get("message.max.bytes"),
            Some("8388608"),
        );
    }

    #[test]
    fn intake_and_boot_ordering_knobs_default_and_override_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert_eq!(defaults.partition_intake_max_events, 1024);
        assert_eq!(defaults.partition_intake_max_seeds, 1024);
        assert_eq!(defaults.seed_lane_cap().get(), 1024);
        assert_eq!(defaults.kafka_queued_max_messages_kbytes, 131_072);
        assert_eq!(defaults.kafka_queued_min_messages, 2000);
        assert_eq!(
            defaults.first_eviction_sweep_delay(),
            Duration::from_millis(120_000),
        );

        let env: std::collections::HashMap<String, String> = [
            ("PARTITION_INTAKE_MAX_EVENTS", "512"),
            ("PARTITION_INTAKE_MAX_SEEDS", "64"),
            ("COHORT_KAFKA_QUEUED_MAX_MESSAGES_KBYTES", "65536"),
            ("COHORT_KAFKA_QUEUED_MIN_MESSAGES", "1000"),
            ("COHORT_FIRST_EVICTION_SWEEP_DELAY_MS", "30000"),
        ]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        assert_eq!(config.partition_intake_max_events, 512);
        assert_eq!(config.seed_lane_cap().get(), 64);
        assert_eq!(
            config.fetch_queue_config().queued_max_messages_kbytes,
            65536
        );
        assert_eq!(config.fetch_queue_config().queued_min_messages, 1000);
        assert_eq!(
            config.first_eviction_sweep_delay(),
            Duration::from_millis(30_000),
        );
    }

    #[test]
    fn consumer_config_names_the_client_after_the_pod_without_a_static_member_id() {
        let mut config = test_config();
        config.pod_name = Some("cohort-stream-processor-2".to_string());
        let client = config.consumer_client_config();
        assert_eq!(client.get("group.instance.id"), None);
        assert_eq!(client.get("client.id"), Some("cohort-stream-processor-2"));
    }

    #[test]
    fn the_split_claim_comes_from_the_pod_count_the_pod_name_ordinal_and_the_reshard_marker() {
        type Env = &'static [(&'static str, &'static str)];
        let cases: [(Env, Result<&str, &str>); 9] = [
            (&[], Ok("pod 0 of 1 over 64 partitions")),
            (
                &[
                    ("COHORT_POD_COUNT", "2"),
                    ("POD_NAME", "cohort-stream-processor-1"),
                ],
                Ok("pod 1 of 2 over 64 partitions"),
            ),
            (&[("COHORT_POD_COUNT", "4")], Err("requires POD_NAME")),
            (
                &[
                    ("COHORT_POD_COUNT", "3"),
                    ("POD_NAME", "cohort-stream-processor-0"),
                ],
                Err("evenly"),
            ),
            (
                &[
                    ("COHORT_POD_COUNT", "2"),
                    ("POD_NAME", "cohort-stream-processor-2"),
                ],
                Err("not below"),
            ),
            (&[("POD_NAME", "cohort-stream-processor")], Err("POD_NAME")),
            (
                &[
                    ("COHORT_POD_COUNT", "2"),
                    ("COHORT_RESHARD_FROM_POD_COUNT", "1"),
                    ("POD_NAME", "cohort-stream-processor-1"),
                ],
                Ok("pod 1 of 2 over 64 partitions, resharded from pod 0 of 1 over 64 partitions"),
            ),
            (
                &[
                    ("COHORT_POD_COUNT", "2"),
                    ("COHORT_RESHARD_FROM_POD_COUNT", "1"),
                    ("DURABLE_RESTORE_ENABLED", "true"),
                    ("POD_NAME", "cohort-stream-processor-1"),
                ],
                Err("requires CHECKPOINT_ENABLED"),
            ),
            (
                &[
                    ("COHORT_POD_COUNT", "2"),
                    ("COHORT_RESHARD_FROM_POD_COUNT", "4"),
                    ("POD_NAME", "cohort-stream-processor-1"),
                ],
                Err("must raise the pod count"),
            ),
        ];
        for (env, expected) in cases {
            let env = env
                .iter()
                .map(|&(key, value)| (key.to_string(), value.to_string()))
                .collect();
            let claim = Config::init_from_hashmap(&env)
                .unwrap()
                .split_claim()
                .map(|claim| claim.to_string())
                .map_err(|err| format!("{err:#}"));
            match (claim, expected) {
                (Ok(claim), Ok(expected)) => assert_eq!(claim, expected, "{env:?}"),
                (Err(err), Err(expected)) => assert!(err.contains(expected), "{env:?}: {err}"),
                (actual, expected) => panic!("{env:?}: got {actual:?}, expected {expected:?}"),
            }
        }

        let zero = [("COHORT_POD_COUNT".to_string(), "0".to_string())].into();
        assert!(Config::init_from_hashmap(&zero).is_err());
    }

    #[test]
    fn pod_identity_prefers_pod_name_and_ignores_blanks() {
        let mut config = test_config();
        config.pod_name = Some("pod-from-downward-api".to_string());
        config.pod_hostname = Some("hostname".to_string());
        assert_eq!(config.pod_identity(), Some("pod-from-downward-api"));

        config.pod_name = Some(String::new());
        assert_eq!(config.pod_identity(), Some("hostname"));

        config.pod_hostname = None;
        assert_eq!(config.pod_identity(), None);
    }

    #[test]
    fn explicit_client_id_overrides_pod_identity() {
        let mut config = test_config();
        config.pod_hostname = Some("hostname".to_string());
        config.kafka_client_id = "explicit-client".to_string();
        let client = config.consumer_client_config();
        assert_eq!(client.get("client.id"), Some("explicit-client"));
    }

    #[test]
    fn store_config_threads_the_wipe_on_start_flag() {
        let mut config = test_config();
        let ownership = PartitionOwnership::single(64);
        config.wipe_store_on_start = true;
        assert!(config.store_config(ownership).wipe_on_start);
        config.wipe_store_on_start = false;
        assert!(!config.store_config(ownership).wipe_on_start);
    }

    #[test]
    fn register_transfer_gate_defaults_dark_and_overrides_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert!(
            !defaults.cohort_register_transfer_enabled,
            "register transfer must stay off until the whole fleet can apply the payload",
        );

        let enabled = Config::init_from_hashmap(&std::collections::HashMap::from([(
            "COHORT_REGISTER_TRANSFER_ENABLED".to_owned(),
            "true".to_owned(),
        )]))
        .unwrap();
        assert!(enabled.cohort_register_transfer_enabled);
    }

    #[test]
    fn wipe_on_schema_mismatch_defaults_off_and_threads_into_store_config() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert!(
            !defaults.cohort_wipe_on_schema_mismatch,
            "a schema mismatch must fail fast unless the operator opts in",
        );
        assert!(
            !defaults
                .store_config(PartitionOwnership::single(64))
                .wipe_on_schema_mismatch
        );

        let enabled = Config::init_from_hashmap(&std::collections::HashMap::from([(
            "COHORT_WIPE_ON_SCHEMA_MISMATCH".to_owned(),
            "true".to_owned(),
        )]))
        .unwrap();
        assert!(
            enabled
                .store_config(PartitionOwnership::single(64))
                .wipe_on_schema_mismatch
        );
    }

    #[test]
    fn person_record_ttl_defaults_off_and_threads_into_store_config() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert_eq!(
            defaults.cohort_person_record_ttl_days, 0,
            "the person-record TTL defaults off (no compaction filter)",
        );
        assert_eq!(
            defaults
                .store_config(PartitionOwnership::single(64))
                .person_record_ttl_days,
            0
        );

        let enabled = Config::init_from_hashmap(&std::collections::HashMap::from([(
            "COHORT_PERSON_RECORD_TTL_DAYS".to_owned(),
            "30".to_owned(),
        )]))
        .unwrap();
        assert_eq!(enabled.cohort_person_record_ttl_days, 30);
        assert_eq!(
            enabled
                .store_config(PartitionOwnership::single(64))
                .person_record_ttl_days,
            30,
            "the TTL reaches StoreConfig",
        );
    }

    #[test]
    fn person_record_ttl_safe_floor_flags_only_non_zero_values_below_it() {
        let cases = [
            (0u32, false, "off is always safe"),
            (1, true, "inside the replay horizon"),
            (7, true, "the replay horizon itself is unsafe"),
            (29, true, "one below the floor"),
            (30, false, "the floor is safe"),
            (365, false, "well above the floor"),
        ];
        for (days, risky, why) in cases {
            let mut config = test_config();
            config.cohort_person_record_ttl_days = days;
            assert_eq!(
                config.person_record_ttl_below_safe_floor(),
                risky,
                "{days} days: {why}",
            );
        }
    }

    #[test]
    fn stats_knobs_default_on_and_thread_into_store_config() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert!(defaults.store_statistics_enabled);
        assert!(
            defaults
                .store_config(PartitionOwnership::single(64))
                .statistics_enabled
        );
        assert_eq!(defaults.stats_publish_interval(), Duration::from_secs(15));
        assert_eq!(defaults.store_read_sample_ratio, 64);
        assert_eq!(
            defaults
                .store_config(PartitionOwnership::single(64))
                .read_sample_ratio,
            64
        );

        let env: std::collections::HashMap<String, String> = [
            ("COHORT_STORE_STATISTICS_ENABLED", "false"),
            ("COHORT_STATS_PUBLISH_INTERVAL_SECS", "30"),
            ("COHORT_STORE_READ_SAMPLE_RATIO", "8"),
        ]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        assert!(!config.store_statistics_enabled);
        assert!(
            !config
                .store_config(PartitionOwnership::single(64))
                .statistics_enabled,
            "the flag reaches StoreConfig",
        );
        assert_eq!(config.stats_publish_interval(), Duration::from_secs(30));
        assert_eq!(config.store_read_sample_ratio, 8);
        assert_eq!(
            config
                .store_config(PartitionOwnership::single(64))
                .read_sample_ratio,
            8,
            "the sample ratio reaches StoreConfig",
        );
    }

    #[test]
    fn offload_knobs_default_and_override_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        let offload = defaults.offload_config();
        assert_eq!(offload.mode, OffloadMode::All, "offload defaults to All");
        assert_eq!(offload.event_read_permits, 16);
        assert_eq!(offload.maintenance_permits, 6);

        let env: std::collections::HashMap<String, String> = [
            ("COHORT_STORE_OFFLOAD_MODE", "maintenance"),
            ("COHORT_STORE_EVENT_READ_PERMITS", "8"),
            ("COHORT_STORE_MAINTENANCE_PERMITS", "0"),
        ]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        let offload = config.offload_config();
        assert_eq!(offload.mode, OffloadMode::Maintenance);
        assert_eq!(offload.event_read_permits, 8);
        assert_eq!(offload.maintenance_permits, 0, "0 = unbounded lane");

        let off_env: std::collections::HashMap<String, String> =
            [("COHORT_STORE_OFFLOAD_MODE", "off")]
                .into_iter()
                .map(|(key, value)| (key.to_string(), value.to_string()))
                .collect();
        assert_eq!(
            Config::init_from_hashmap(&off_env)
                .unwrap()
                .offload_config()
                .mode,
            OffloadMode::Off,
        );

        let bad_env: std::collections::HashMap<String, String> =
            [("COHORT_STORE_OFFLOAD_MODE", "occasionally")]
                .into_iter()
                .map(|(key, value)| (key.to_string(), value.to_string()))
                .collect();
        assert!(
            Config::init_from_hashmap(&bad_env).is_err(),
            "an invalid offload mode must fail config init",
        );
    }

    #[test]
    fn stats_publish_interval_floors_zero_at_one_second() {
        // Zero would panic `tokio::time::interval`; the accessor clamps to 1s.
        let env: std::collections::HashMap<String, String> =
            [("COHORT_STATS_PUBLISH_INTERVAL_SECS", "0")]
                .into_iter()
                .map(|(key, value)| (key.to_string(), value.to_string()))
                .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        assert_eq!(config.stats_publish_interval(), Duration::from_secs(1));
    }

    #[test]
    fn consumer_config_sets_tls_keys_only_when_enabled() {
        let mut config = test_config();
        assert_eq!(
            config.consumer_client_config().get("security.protocol"),
            None
        );

        config.kafka_tls = true;
        assert_eq!(
            config.consumer_client_config().get("security.protocol"),
            Some("ssl"),
        );
    }

    #[test]
    fn membership_topic_defaults_to_the_shadow_and_overrides_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert_eq!(
            defaults.cohort_membership_changed_topic, "cohort_membership_changed_shadow",
            "a code deploy must not redirect membership output to the production topic",
        );

        let env: std::collections::HashMap<String, String> = [(
            "COHORT_MEMBERSHIP_CHANGED_TOPIC",
            "cohort_membership_changed",
        )]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        assert_eq!(
            config.cohort_membership_changed_topic, "cohort_membership_changed",
            "the cut-over must be reachable as a config-only change",
        );
    }

    #[test]
    fn durable_restore_defaults_off_and_overrides_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert!(
            !defaults.durable_restore_enabled,
            "durable restore defaults off"
        );

        let env: std::collections::HashMap<String, String> = [("DURABLE_RESTORE_ENABLED", "true")]
            .into_iter()
            .map(|(key, value)| (key.to_string(), value.to_string()))
            .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        assert!(config.durable_restore_enabled);
    }

    #[test]
    fn durability_startup_guard_passes_for_the_default_config() {
        assert!(test_config().validate_startup().is_ok());
    }

    #[test]
    fn durability_startup_guard_passes_for_durable_restore_with_cascades() {
        let mut config = test_config();
        config.durable_restore_enabled = true;
        config.cohort_cascade_enabled = true;
        assert!(config.validate_startup().is_ok());
    }

    #[test]
    fn durability_startup_guard_refuses_checkpoint_without_durable_restore() {
        let mut config = test_config();
        config.checkpoint_enabled = true;
        config.durable_restore_enabled = false;
        let err = config
            .validate_startup()
            .expect_err("checkpoint without durable restore must be refused");
        assert!(
            err.to_string().contains("DURABLE_RESTORE_ENABLED"),
            "the refusal names the durable-restore requirement: {err}",
        );
    }

    #[test]
    fn checkpoints_start_only_with_a_bucket_nonzero_knobs_a_separate_absolute_dir_and_an_ordinal() {
        fn enabled() -> Config {
            let mut config = test_config();
            config.checkpoint_enabled = true;
            config.durable_restore_enabled = true;
            config.checkpoint_s3_bucket = "bkt".to_string();
            config.store_path = "/data/store".to_string();
            config.checkpoint_local_dir = "/data/checkpoints".to_string();
            config.pod_name = Some("cohort-stream-processor-2".to_string());
            config
        }
        enabled().validate_startup().unwrap();

        type Misconfigure = fn(&mut Config);
        let cases: [(&str, Misconfigure, &str); 9] = [
            (
                "no bucket",
                |c| c.checkpoint_s3_bucket.clear(),
                "CHECKPOINT_S3_BUCKET",
            ),
            (
                "a zero interval",
                |c| c.checkpoint_interval_ms = 0,
                "CHECKPOINT_INTERVAL_MS",
            ),
            (
                "a relative checkpoint dir",
                |c| c.checkpoint_local_dir = "checkpoints".to_string(),
                "absolute",
            ),
            (
                "a relative store path",
                |c| c.store_path = "cohort-store".to_string(),
                "absolute",
            ),
            (
                "a checkpoint dir inside the store",
                |c| c.checkpoint_local_dir = "/data/store/checkpoints".to_string(),
                "contain",
            ),
            (
                "a store inside the checkpoint dir",
                |c| c.checkpoint_local_dir = "/data".to_string(),
                "contain",
            ),
            (
                "a checkpoint dir inside the restore's staging dir",
                |c| c.checkpoint_local_dir = "/data/store.restore/checkpoints".to_string(),
                "stages",
            ),
            (
                "a checkpoint dir that reaches the staging dir through `..`",
                |c| c.checkpoint_local_dir = "/data/../data/store.restore".to_string(),
                "without `..`",
            ),
            (
                "a pod name without an ordinal",
                |c| c.pod_name = Some("cohort-stream-processor".to_string()),
                "POD_NAME",
            ),
        ];
        for (name, misconfigure, expected) in cases {
            let mut config = enabled();
            misconfigure(&mut config);
            let err = config.validate_startup().expect_err(name);
            assert!(format!("{err:#}").contains(expected), "{name}: {err:#}");
        }
    }

    #[test]
    fn checkpoint_config_defaults_off_and_overrides_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert!(
            !defaults.checkpoint_enabled,
            "the checkpoint master gate defaults off",
        );
        assert_eq!(
            defaults.checkpoint_interval(),
            Duration::from_millis(300_000)
        );
        assert_eq!(
            defaults.checkpoint_s3_upload_interval(),
            Duration::from_millis(900_000),
        );
        assert_eq!(defaults.checkpoint_local_dir, "cohort-checkpoints");
        assert!(
            defaults.checkpoint_s3_bucket.is_empty(),
            "no S3 bucket by default → uploader inert",
        );
        assert_eq!(
            defaults.checkpoint_local_max_staleness(),
            Duration::from_secs(7200),
        );
        assert_eq!(defaults.checkpoint_import_window_hours, 20);
        assert_eq!(defaults.checkpoint_import_attempt_depth, 10);

        let env: std::collections::HashMap<String, String> = [
            ("CHECKPOINT_ENABLED", "true"),
            ("CHECKPOINT_INTERVAL_MS", "60000"),
            ("CHECKPOINT_S3_UPLOAD_INTERVAL_MS", "120000"),
            ("CHECKPOINT_LOCAL_DIR", "/data/ckpt"),
            ("CHECKPOINT_S3_BUCKET", "my-bucket"),
            ("CHECKPOINT_S3_PREFIX", "ckpt-prefix"),
            ("CHECKPOINT_S3_REGION", "us-east-1"),
            ("CHECKPOINT_S3_ENDPOINT", "http://minio:9000"),
            ("CHECKPOINT_S3_ACCESS_KEY_ID", "ak"),
            ("CHECKPOINT_S3_SECRET_ACCESS_KEY", "sk"),
            ("CHECKPOINT_S3_FORCE_PATH_STYLE", "true"),
            ("CHECKPOINT_MAX_CONCURRENT_UPLOADS", "8"),
            ("CHECKPOINT_MAX_CONCURRENT_DOWNLOADS", "9"),
            ("CHECKPOINT_MAX_UPLOAD_BUFFERS", "10"),
            ("CHECKPOINT_S3_MAX_RETRIES", "5"),
            ("CHECKPOINT_S3_OPERATION_TIMEOUT_SECS", "60"),
            ("CHECKPOINT_S3_ATTEMPT_TIMEOUT_SECS", "11"),
            ("CHECKPOINT_LOCAL_MAX_STALENESS_SECS", "3600"),
            ("CHECKPOINT_IMPORT_WINDOW_HOURS", "12"),
            ("CHECKPOINT_IMPORT_ATTEMPT_DEPTH", "4"),
            ("CHECKPOINT_IMPORT_TIMEOUT_SECS", "90"),
        ]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();

        let config = Config::init_from_hashmap(&env).unwrap();
        assert!(config.checkpoint_enabled);
        assert_eq!(config.checkpoint_interval(), Duration::from_millis(60_000));
        assert_eq!(
            config.checkpoint_s3_upload_interval(),
            Duration::from_millis(120_000),
        );
        assert_eq!(config.checkpoint_local_dir, "/data/ckpt");
        assert_eq!(config.checkpoint_s3_bucket, "my-bucket");
        assert_eq!(config.checkpoint_import_window_hours, 12);
        assert_eq!(config.checkpoint_import_attempt_depth, 4);
    }

    #[test]
    fn durability_config_maps_the_checkpoint_fields() {
        let mut config = test_config();
        config.checkpoint_local_dir = "/data/ckpt".to_string();
        config.checkpoint_s3_bucket = "bkt".to_string();
        config.checkpoint_s3_prefix = "pfx".to_string();
        config.checkpoint_s3_region = Some("eu-central-1".to_string());
        config.checkpoint_s3_endpoint = Some("http://seaweedfs:8333".to_string());
        config.checkpoint_s3_access_key_id = Some("ak".to_string());
        config.checkpoint_s3_secret_access_key = Some("sk".to_string());
        config.checkpoint_s3_force_path_style = true;
        config.checkpoint_max_concurrent_uploads = 7;
        config.checkpoint_max_concurrent_downloads = 8;
        config.checkpoint_max_upload_buffers = 9;
        config.checkpoint_s3_max_retries = 2;
        config.checkpoint_s3_operation_timeout_secs = 60;
        config.checkpoint_s3_attempt_timeout_secs = 11;
        config.checkpoint_local_max_staleness_secs = 3600;
        config.checkpoint_import_window_hours = 6;
        config.checkpoint_import_attempt_depth = 4;
        config.checkpoint_import_timeout_secs = 90;
        config.checkpoint_full_upload_interval_secs = 3601;
        config.checkpoint_upload_max_bytes_per_sec = 1024;

        let durability = config.durability_config();
        assert_eq!(durability.local_checkpoint_dir, "/data/ckpt");
        assert_eq!(durability.s3_bucket, "bkt");
        assert_eq!(durability.s3_key_prefix, "pfx");
        assert_eq!(durability.aws_region.as_deref(), Some("eu-central-1"));
        assert_eq!(
            durability.s3_endpoint.as_deref(),
            Some("http://seaweedfs:8333"),
        );
        assert_eq!(durability.s3_access_key_id.as_deref(), Some("ak"));
        assert!(durability.s3_force_path_style);
        assert_eq!(durability.max_concurrent_checkpoint_file_uploads, 7);
        assert_eq!(durability.max_concurrent_checkpoint_file_downloads, 8);
        assert_eq!(durability.max_upload_buffers, 9);
        assert_eq!(durability.s3_max_retries, 2);
        assert_eq!(durability.s3_operation_timeout, Duration::from_secs(60));
        assert_eq!(durability.s3_attempt_timeout, Duration::from_secs(11));
        assert_eq!(
            durability.local_checkpoint_max_staleness,
            Duration::from_secs(3600),
        );
        assert_eq!(durability.checkpoint_import_window_hours, 6);
        assert_eq!(durability.checkpoint_import_attempt_depth, 4);
        assert_eq!(
            durability.checkpoint_import_timeout,
            Duration::from_secs(90)
        );
        assert_eq!(durability.full_upload_interval, Duration::from_secs(3601));
        assert_eq!(durability.upload_max_bytes_per_sec, 1024);
    }

    #[test]
    fn effective_wipe_on_start_truth_table() {
        let dir = tempfile::TempDir::new().unwrap();
        let existing = dir.path().join("db");
        std::fs::create_dir_all(&existing).unwrap();
        let missing = dir.path().join("does-not-exist");

        // (wipe_store_on_start, durable_restore_enabled, store exists) -> effective wipe.
        let cases = [
            (true, false, true, true, "wipe-on + restore-off + present"),
            (true, false, false, true, "wipe-on + restore-off + absent"),
            (
                false,
                false,
                true,
                false,
                "wipe-off + restore-off + present",
            ),
            (
                true,
                true,
                true,
                false,
                "wipe-on + restore-on + present → reopen-live",
            ),
            (
                true,
                true,
                false,
                true,
                "wipe-on + restore-on + absent → fresh wipe (no-op)",
            ),
            (false, true, true, false, "wipe-off + restore-on + present"),
        ];
        for (wipe, durable, exists, expected, why) in cases {
            let mut config = test_config();
            config.wipe_store_on_start = wipe;
            config.durable_restore_enabled = durable;
            config.store_path = if exists {
                existing.to_string_lossy().into_owned()
            } else {
                missing.to_string_lossy().into_owned()
            };
            assert_eq!(config.effective_wipe_on_start(), expected, "{why}");
            assert_eq!(
                config
                    .store_config(PartitionOwnership::single(64))
                    .wipe_on_start,
                expected,
                "store_config must thread the effective wipe: {why}",
            );
        }
    }

    #[test]
    fn store_config_threads_through_the_configured_path() {
        let mut config = test_config();
        config.store_path = "/var/lib/cohort/state".to_string();
        assert_eq!(
            config.store_config(PartitionOwnership::single(64)).path,
            std::path::PathBuf::from("/var/lib/cohort/state"),
        );
    }

    #[test]
    fn merge_envs_default_to_the_expected_topology() {
        let config = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert_eq!(config.person_merge_events_topic, "person_merge_events");
        assert_eq!(
            config.cohort_merge_state_transfer_topic,
            "cohort_merge_state_transfer",
        );
        assert_eq!(config.kafka_merge_consumer_group, "cohort-stream-merges");
        assert_eq!(
            config.kafka_merge_apply_consumer_group,
            "cohort-stream-merge-apply",
        );
        assert_eq!(
            config.transfer_retry_policy(),
            TransferRetryPolicy::default()
        );
        assert_eq!(
            config.merge_redrive_interval(),
            Duration::from_millis(60_000)
        );
        // The transfer sink fast-fails per attempt; the shared sink keeps the 20 s timeout.
        assert_eq!(config.merge_transfer_message_timeout_ms, 2000);
        assert_eq!(
            config
                .build_transfer_kafka_config()
                .kafka_message_timeout_ms,
            2000,
        );
        assert_eq!(config.build_kafka_config().kafka_message_timeout_ms, 20_000);
    }

    #[test]
    fn merge_gc_defaults_match_the_tdd_retention_floors() {
        let config = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        // 8d = 7d merge-topic retention + 1d safety — covers both marker CFs.
        assert_eq!(config.merge_marker_retention_ms, 691_200_000);
        assert_eq!(
            config.merge_marker_retention_ms,
            (7 + 1) * 24 * 60 * 60 * 1000,
        );
        // 9d = events 7d + stream 24h + 1d safety.
        assert_eq!(config.merge_tombstone_retention_ms, 777_600_000);
        assert_eq!(
            config.merge_tombstone_retention_ms,
            (7 * 24 + 24 + 24) * 60 * 60 * 1000,
        );
        assert_eq!(config.merge_gc_interval(), Duration::from_millis(3_600_000));
        assert_eq!(config.merge_gc_scan_limit, 10_000);
        assert!(
            config.stage2_orphan_gc_enabled,
            "the cf_stage2 orphan GC defaults on",
        );
    }

    #[test]
    fn merge_gc_knobs_override_from_env() {
        let env: std::collections::HashMap<String, String> = [
            ("MERGE_MARKER_RETENTION_MS", "123"),
            ("MERGE_TOMBSTONE_RETENTION_MS", "456"),
            ("MERGE_GC_INTERVAL_MS", "789"),
            ("MERGE_GC_SCAN_LIMIT", "5"),
            ("STAGE2_ORPHAN_GC_ENABLED", "false"),
        ]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();

        let config = Config::init_from_hashmap(&env).unwrap();
        assert_eq!(config.merge_marker_retention_ms, 123);
        assert_eq!(config.merge_tombstone_retention_ms, 456);
        assert_eq!(config.merge_gc_interval(), Duration::from_millis(789));
        assert_eq!(config.merge_gc_scan_limit, 5);
        assert!(
            !config.stage2_orphan_gc_enabled,
            "the kill-switch disables the cf_stage2 orphan GC",
        );
    }

    #[test]
    fn event_name_gating_defaults_on_and_overrides_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert!(
            defaults.cohort_event_name_gating_enabled,
            "the event-name gate defaults on",
        );
        assert_eq!(defaults.event_name_gating(), EventNameGating::Enabled);

        let env: std::collections::HashMap<String, String> =
            [("COHORT_EVENT_NAME_GATING_ENABLED", "false")]
                .into_iter()
                .map(|(key, value)| (key.to_string(), value.to_string()))
                .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        assert!(!config.cohort_event_name_gating_enabled);
        assert_eq!(config.event_name_gating(), EventNameGating::Disabled);
    }

    #[test]
    fn cascade_caps_default_and_override_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert_eq!(defaults.cohort_cascade_depth_cap, 8);
        assert_eq!(defaults.cohort_cascade_fanout_cap, 1000);
        assert!(
            !defaults.cohort_cascade_enabled,
            "the cascade gate defaults off",
        );
        assert_eq!(
            defaults.cohort_cascade_events_topic,
            "cohort_cascade_events"
        );
        assert_eq!(
            defaults.kafka_cascade_consumer_group,
            "cohort-stream-cascade"
        );

        let env: std::collections::HashMap<String, String> = [
            ("COHORT_CASCADE_DEPTH_CAP", "3"),
            ("COHORT_CASCADE_FANOUT_CAP", "50"),
            ("COHORT_CASCADE_ENABLED", "true"),
            ("COHORT_CASCADE_EVENTS_TOPIC", "cascade_test"),
            ("KAFKA_CASCADE_CONSUMER_GROUP", "cascade-group-test"),
        ]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();

        let config = Config::init_from_hashmap(&env).unwrap();
        assert_eq!(config.cohort_cascade_depth_cap, 3);
        assert_eq!(config.cohort_cascade_fanout_cap, 50);
        assert!(config.cohort_cascade_enabled);
        assert_eq!(config.cohort_cascade_events_topic, "cascade_test");
        assert_eq!(config.kafka_cascade_consumer_group, "cascade-group-test");
        assert_eq!(
            config.cascade_config(),
            CascadeConfig {
                enabled: true,
                depth_cap: 3,
                fanout_cap: 50,
            },
        );
    }

    #[test]
    fn build_transfer_kafka_config_overrides_only_the_message_timeout() {
        let config = test_config();
        let shared = config.build_kafka_config();
        let transfer = config.build_transfer_kafka_config();
        // Only `message.timeout.ms` differs; every other producer knob is inherited.
        assert_eq!(transfer.kafka_message_timeout_ms, 2000);
        assert_eq!(
            config.build_marker_kafka_config().kafka_message_timeout_ms,
            1000,
        );
        assert_eq!(shared.kafka_message_timeout_ms, 20_000);
        assert_eq!(
            transfer.kafka_producer_partitioner,
            shared.kafka_producer_partitioner,
        );
        assert_eq!(transfer.kafka_hosts, shared.kafka_hosts);
        assert_eq!(
            transfer.kafka_compression_codec,
            shared.kafka_compression_codec
        );
        assert_eq!(
            transfer.kafka_producer_linger_ms,
            shared.kafka_producer_linger_ms
        );
        assert_eq!(
            transfer.kafka_producer_queue_mib,
            shared.kafka_producer_queue_mib
        );
    }

    #[test]
    fn merge_envs_override_the_defaults() {
        let env: std::collections::HashMap<String, String> = [
            ("PERSON_MERGE_EVENTS_TOPIC", "pme_test"),
            ("COHORT_MERGE_STATE_TRANSFER_TOPIC", "transfer_test"),
            ("KAFKA_MERGE_CONSUMER_GROUP", "merges-test"),
            ("KAFKA_MERGE_APPLY_CONSUMER_GROUP", "apply-test"),
            ("MERGE_TRANSFER_MAX_RETRIES", "2"),
            ("MERGE_TRANSFER_RETRY_BASE_MS", "100"),
            ("MERGE_TRANSFER_RETRY_CAP_MS", "400"),
            ("MERGE_TRANSFER_MESSAGE_TIMEOUT_MS", "750"),
            ("MERGE_REDRIVE_INTERVAL_MS", "5000"),
        ]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();

        let config = Config::init_from_hashmap(&env).unwrap();
        assert_eq!(config.person_merge_events_topic, "pme_test");
        assert_eq!(config.cohort_merge_state_transfer_topic, "transfer_test");
        assert_eq!(config.kafka_merge_consumer_group, "merges-test");
        assert_eq!(config.kafka_merge_apply_consumer_group, "apply-test");
        assert_eq!(
            config.transfer_retry_policy(),
            TransferRetryPolicy {
                max_retries: 2,
                base: Duration::from_millis(100),
                cap: Duration::from_millis(400),
            },
        );
        assert_eq!(config.merge_redrive_interval(), Duration::from_millis(5000));
        assert_eq!(config.merge_transfer_message_timeout_ms, 750);
        assert_eq!(
            config
                .build_transfer_kafka_config()
                .kafka_message_timeout_ms,
            750,
        );
    }

    #[test]
    fn follower_config_hard_codes_earliest() {
        let config = test_config();
        let client = config.follower_client_config("cohort-stream-merges");
        assert_eq!(client.get("auto.offset.reset"), Some("earliest"));
        assert_eq!(client.get("group.id"), Some("cohort-stream-merges"));
        assert_eq!(client.get("enable.auto.commit"), Some("false"));
        assert_eq!(client.get("enable.auto.offset.store"), Some("false"));
        assert_eq!(client.get("bootstrap.servers"), Some("localhost:9092"));
    }

    #[test]
    fn follower_config_never_joins_the_group_protocol() {
        let mut config = test_config();
        config.pod_hostname = Some("cohort-stream-processor-2".to_string());
        let client = config.follower_client_config("cohort-stream-merge-apply");
        assert_eq!(client.get("partition.assignment.strategy"), None);
        assert_eq!(client.get("group.instance.id"), None);
        assert_eq!(client.get("client.id"), Some("cohort-stream-processor-2"));
    }

    #[test]
    fn follower_config_sets_tls_keys_only_when_enabled() {
        let mut config = test_config();
        assert_eq!(
            config.follower_client_config("g").get("security.protocol"),
            None,
        );
        config.kafka_tls = true;
        assert_eq!(
            config.follower_client_config("g").get("security.protocol"),
            Some("ssl"),
        );
    }

    #[test]
    fn cohort_partition_count_defaults_to_64_and_overrides_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert_eq!(
            defaults.cohort_partition_count, 64,
            "production default is 64 so behavior stays byte-identical",
        );

        let env: std::collections::HashMap<String, String> = [("COHORT_PARTITION_COUNT", "8")]
            .into_iter()
            .map(|(key, value)| (key.to_string(), value.to_string()))
            .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        assert_eq!(config.cohort_partition_count, 8);
    }

    #[test]
    fn store_tuning_config_defaults_and_overrides_from_env() {
        let defaults = Config::init_from_hashmap(&std::collections::HashMap::new()).unwrap();
        assert_eq!(defaults.cohort_block_cache_bytes, 134_217_728);
        assert!(defaults.cohort_tuned_block_options_enabled);
        assert!(defaults.cohort_compact_on_deletion_enabled);
        assert_eq!(defaults.cohort_periodic_compaction_seconds, 0);
        assert_eq!(defaults.cohort_max_background_jobs, 0);
        assert_eq!(defaults.cohort_max_log_file_size_bytes.get(), 268_435_456);
        assert_eq!(defaults.cohort_keep_log_file_num.get(), 10);
        assert_eq!(defaults.partition_channel_buffer, 128);

        let env: std::collections::HashMap<String, String> = [
            ("COHORT_BLOCK_CACHE_BYTES", "3221225472"),
            ("COHORT_TUNED_BLOCK_OPTIONS_ENABLED", "false"),
            ("COHORT_COMPACT_ON_DELETION_ENABLED", "false"),
            ("COHORT_PERIODIC_COMPACTION_SECONDS", "3600"),
            ("COHORT_MAX_BACKGROUND_JOBS", "2"),
            ("COHORT_MAX_LOG_FILE_SIZE_BYTES", "1048576"),
            ("COHORT_KEEP_LOG_FILE_NUM", "3"),
            ("PARTITION_CHANNEL_BUFFER", "256"),
        ]
        .into_iter()
        .map(|(key, value)| (key.to_string(), value.to_string()))
        .collect();
        let config = Config::init_from_hashmap(&env).unwrap();
        assert_eq!(config.cohort_block_cache_bytes, 3_221_225_472);
        assert!(!config.cohort_tuned_block_options_enabled);
        assert!(!config.cohort_compact_on_deletion_enabled);
        assert_eq!(config.cohort_periodic_compaction_seconds, 3600);
        assert_eq!(config.cohort_max_background_jobs, 2);
        assert_eq!(config.cohort_max_log_file_size_bytes.get(), 1_048_576);
        assert_eq!(config.cohort_keep_log_file_num.get(), 3);
        assert_eq!(config.partition_channel_buffer, 256);
    }

    #[test]
    fn store_config_threads_the_rocksdb_tuning_knobs() {
        let mut config = test_config();
        config.cohort_block_cache_bytes = 3_221_225_472;
        config.cohort_tuned_block_options_enabled = false;
        config.cohort_compact_on_deletion_enabled = false;
        config.cohort_periodic_compaction_seconds = 3600;
        config.cohort_max_background_jobs = 2;
        config.cohort_max_log_file_size_bytes = NonZeroUsize::new(1_048_576).unwrap();
        config.cohort_keep_log_file_num = NonZeroUsize::new(3).unwrap();

        let store = config.store_config(PartitionOwnership::single(64));
        assert_eq!(store.block_cache_bytes, 3_221_225_472);
        assert!(!store.tuned_block_options);
        assert!(!store.compact_on_deletion);
        assert_eq!(store.periodic_compaction_seconds, 3600);
        assert_eq!(store.max_background_jobs, 2);
        assert_eq!(store.max_log_file_size_bytes.get(), 1_048_576);
        assert_eq!(store.keep_log_file_num.get(), 3);
    }

    #[test]
    fn build_kafka_config_pins_the_murmur2_partitioner() {
        let kafka = test_config().build_kafka_config();
        assert_eq!(
            kafka.kafka_producer_partitioner.as_deref(),
            Some("murmur2_random"),
        );
        assert_eq!(kafka.kafka_compression_codec, "none");
        assert_eq!(kafka.kafka_hosts, "localhost:9092");
    }
}