Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

39.3 KB
Raw Blame
use std::{fs, path::PathBuf, time::Duration};

use anyhow::{Context, Result};
use bytesize::ByteSize;
use common_continuous_profiling::ContinuousProfilingConfig;
use envconfig::Envconfig;

use crate::rocksdb::store::{parse_compression_per_level, parse_compression_type, RocksDbConfig};

/// Pipeline type for the deduplicator service.
///
/// Each pipeline type handles a different event format:
/// - `IngestionEvents`: Events from capture (CapturedEvent/RawEvent format)
/// - `ClickhouseEvents`: Events from ingestion pipeline (ClickhouseEvent format)
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, strum_macros::EnumString)]
#[strum(serialize_all = "snake_case")]
pub enum PipelineType {
    #[default]
    IngestionEvents,
    ClickhouseEvents,
}

#[derive(Envconfig, Clone, Debug)]
pub struct Config {
    #[envconfig(nested = true)]
    pub continuous_profiling: ContinuousProfilingConfig,

    /// Pipeline type determines the event format and processing logic.
    /// Valid values: "ingestion_events" (default), "clickhouse_events"
    #[envconfig(default = "ingestion_events")]
    pub pipeline_type: PipelineType,

    // Kafka configuration
    #[envconfig(default = "localhost:9092")]
    pub kafka_hosts: String,

    #[envconfig(default = "kafka-deduplicator")]
    pub kafka_consumer_group: String,

    #[envconfig(default = "10485760")] // 10MB
    pub kafka_consumer_max_partition_fetch_bytes: u32,

    #[envconfig(default = "10000")] // 10 seconds
    pub kafka_topic_metadata_refresh_interval_ms: u32,

    #[envconfig(default = "30000")] // 30 seconds
    pub kafka_metadata_max_age_ms: u32,

    // Session timeout: how long broker waits for heartbeats before declaring consumer dead.
    // With static membership (group.instance.id), broker holds partition assignments for this
    // duration after a consumer disappears. Should be longer than typical pod restart time.
    #[envconfig(default = "60000")] // 60 seconds - covers slow pod restarts
    pub kafka_session_timeout_ms: u32,

    // Heartbeat interval: how often consumer sends heartbeats to broker.
    // With 60s session timeout and 5s heartbeat, 12 heartbeats can miss before timeout.
    #[envconfig(default = "5000")] // 5 seconds
    pub kafka_heartbeat_interval_ms: u32,

    // supplied by k8s deploy env, used as part of kafka
    // consumer client ID for sticky partition mappings
    #[envconfig(from = "HOSTNAME")]
    pub pod_hostname: Option<String>,

    #[envconfig(default = "events")]
    pub kafka_consumer_topic: String,

    #[envconfig(default = "latest")]
    pub kafka_consumer_offset_reset: String,

    // Kafka Producer configuration
    #[envconfig(default = "20")]
    pub kafka_producer_linger_ms: u32,

    #[envconfig(default = "400")]
    pub kafka_producer_queue_mib: u32,

    #[envconfig(default = "10000000")]
    pub kafka_producer_queue_messages: u32,

    #[envconfig(default = "20000")]
    pub kafka_message_timeout_ms: u32,

    #[envconfig(default = "5000")] // 5 seconds
    pub kafka_producer_send_timeout_ms: u32,

    #[envconfig(default = "snappy")]
    pub kafka_compression_codec: String,

    #[envconfig(default = "false")]
    pub kafka_tls: bool,

    // Output topic for deduplicated events (optional - if not set, events are only consumed for metrics)
    pub output_topic: Option<String>,

    // Topic for publishing duplicate detection results (optional)
    pub duplicate_events_topic: Option<String>,

    // RocksDB storage configuration
    #[envconfig(default = "/tmp/deduplication-store")]
    pub store_path: String,

    // RocksDB tuning (optional — omit to use compiled-in defaults from RocksDbConfig)
    pub rocksdb_shared_cache_size_bytes: Option<usize>,
    pub rocksdb_total_write_buffer_size_bytes: Option<usize>,
    pub rocksdb_max_background_jobs: Option<i32>,
    pub rocksdb_write_buffer_size_bytes: Option<usize>,
    pub rocksdb_target_file_size_base_bytes: Option<u64>,
    pub rocksdb_max_open_files: Option<i32>,
    pub rocksdb_l0_compaction_trigger: Option<i32>,
    pub rocksdb_l0_slowdown_writes_trigger: Option<i32>,
    pub rocksdb_l0_stop_writes_trigger: Option<i32>,
    pub rocksdb_write_buffer_manager_allow_stall: Option<bool>,

    // RocksDB compression (optional — omit to keep LZ4 everywhere, matching compiled-in defaults)
    #[envconfig(default = "lz4")]
    pub rocksdb_compression_type: String,
    pub rocksdb_compression_per_level: Option<String>,
    pub rocksdb_bottommost_compression_type: Option<String>,
    pub rocksdb_universal_compression_size_percent: Option<i32>,

    #[envconfig(default = "1073741824")]
    // 1GB default, supports: raw bytes, scientific notation (9.663676416e+09), or units (9Gi, 1GB)
    pub max_store_capacity: String,

    #[envconfig(default = "120")]
    // 2 minutes default - interval for checking and cleaning up old data when capacity is exceeded
    pub cleanup_interval_secs: u64,

    #[envconfig(default = "900")]
    // 15 minutes default - minimum staleness (no recent WAL activity) before orphan directories can be deleted
    pub orphan_cleanup_min_staleness_secs: u64,

    #[envconfig(default = "16")]
    // Max parallel directory deletions during rebalance cleanup (bounded scatter-gather)
    pub rebalance_cleanup_parallelism: usize,

    // Consumer processing configuration
    #[envconfig(default = "100")]
    pub max_in_flight_messages: usize,

    #[envconfig(default = "100")]
    pub max_in_flight_messages_per_partition: usize,

    #[envconfig(default = "67108864")] // 64MB default
    pub max_memory_bytes: usize,

    #[envconfig(default = "4")]
    pub worker_threads: usize,

    #[envconfig(default = "1")] // 1 second
    pub poll_timeout_secs: u64,

    #[envconfig(default = "60")] // 60 seconds
    pub shutdown_timeout_secs: u64,

    #[envconfig(default = "5")] // 5 seconds
    pub commit_interval_secs: u64,

    #[envconfig(default = "5000")] // 5000 messages (increased from 1000 for higher throughput)
    pub kafka_consumer_batch_size: usize,

    #[envconfig(default = "200")] // 200ms (reduced from 500ms for lower latency)
    pub kafka_consumer_batch_timeout_ms: u64,

    // Timeout for consumer.seek_partitions() after checkpoint import (seconds)
    #[envconfig(default = "5")]
    pub kafka_consumer_seek_timeout_secs: u64,

    // Kafka consumer fetch settings for throughput optimization
    #[envconfig(default = "1048576")] // 1MB minimum fetch size
    pub kafka_consumer_fetch_min_bytes: u32,

    #[envconfig(default = "52428800")] // 50MB maximum fetch size
    pub kafka_consumer_fetch_max_bytes: u32,

    #[envconfig(default = "100")] // 100ms wait when min bytes not reached
    pub kafka_consumer_fetch_wait_max_ms: u32,

    #[envconfig(default = "100000")] // 100K messages to queue for prefetching
    pub kafka_consumer_queued_min_messages: u32,

    #[envconfig(default = "102400")] // 100MB max bytes to prefetch (value is in KB)
    pub kafka_consumer_queued_max_messages_kbytes: u32,

    #[envconfig(default = "300000")]
    // 5 minutes - max time between poll() calls before consumer leaves group
    pub kafka_max_poll_interval_ms: u32,

    // Partition worker channel buffer size for pipeline parallelism
    #[envconfig(default = "10")]
    pub partition_worker_channel_buffer_size: usize,

    #[envconfig(default = "120")] // 120 seconds (2 minutes)
    pub flush_interval_secs: u64,

    // HTTP server configuration
    #[envconfig(from = "BIND_HOST", default = "0.0.0.0")]
    pub host: String,

    #[envconfig(from = "BIND_PORT", default = "8000")]
    pub port: u16,

    //// Checkpoint configuration ////

    // Checkpoint S3 remote storage bucket. If set, this also
    // enables local successful checkpoints to be exported to S3
    pub s3_bucket: Option<String>,

    // Checkpoint S3 remote storage key prefix
    #[envconfig(default = "deduplication-checkpoints")]
    pub s3_key_prefix: String,

    pub aws_region: Option<String>,

    #[envconfig(default = "120")] // 2 minutes
    pub s3_operation_timeout_secs: u64,

    #[envconfig(default = "20")] // 20 seconds
    pub s3_attempt_timeout_secs: u64,

    /// Maximum number of retries for S3 operations before giving up.
    /// Works in conjunction with s3_operation_timeout which provides the total retry budget.
    #[envconfig(default = "3")]
    pub s3_max_retries: usize,

    /// S3 endpoint URL (for non-AWS S3-compatible stores like MinIO)
    pub s3_endpoint: Option<String>,

    /// S3 access key (for local dev without IAM role)
    pub s3_access_key_id: Option<String>,

    /// S3 secret key (for local dev without IAM role)
    pub s3_secret_access_key: Option<String>,

    /// Force path-style S3 URLs (required for MinIO)
    #[envconfig(default = "false")]
    pub s3_force_path_style: bool,

    // Checkpoint configuration - integrated from checkpoint::config
    #[envconfig(default = "1800")] // 30 minutes in seconds
    pub checkpoint_interval_secs: u64,

    // max checkpoint attempts to perform on a single pod at once. each
    // concurrent attempt is against a different locally assigned partition
    #[envconfig(default = "8")]
    pub max_concurrent_checkpoints: usize,

    #[envconfig(default = "200")]
    pub checkpoint_gate_interval_millis: u64,

    #[envconfig(default = "10")]
    pub checkpoint_worker_shutdown_timeout_secs: u64,

    #[envconfig(default = "1")]
    pub checkpoints_per_partition: usize,

    // Base directory where local checkpoints are created.
    // cleaned up on success or failure
    #[envconfig(default = "/tmp/local_checkpoints")]
    pub local_checkpoint_dir: String,

    // how often to perform a full checkpoint vs. incremental
    // if 0, then we will always do full uploads
    #[envconfig(default = "0")]
    pub checkpoint_full_upload_interval: u32,

    // Whether to enable checkpoint import from remote storage
    #[envconfig(default = "false")]
    pub checkpoint_import_enabled: bool,

    // Whether to enable export to remote storage
    // on successful local checkpoint attempts
    #[envconfig(default = "false")]
    pub checkpoint_export_enabled: bool,

    // Number of historical recent checkpoints to attempt to import
    // as fallbacks when most recent download fails or files are corrupted
    #[envconfig(default = "10")]
    pub checkpoint_import_attempt_depth: usize,

    // number of hours prior to "now" that the checkpoint import mechanism
    // will search for valid checkpoint attempts in a DR recovery scenario
    #[envconfig(default = "24")]
    pub checkpoint_import_window_hours: u32,

    // Maximum concurrent S3 file downloads during checkpoint import
    // Limits memory usage by bounding the number of in-flight HTTP connections
    // Critical during rebalance when many partitions are assigned simultaneously
    // Higher values speed up rebalance; streaming bounds memory per download to ~8KB
    #[envconfig(default = "40")]
    pub max_concurrent_checkpoint_file_downloads: usize,

    // Maximum concurrent S3 file uploads during checkpoint export
    // Controls the LimitStore semaphore that bounds concurrent S3 HTTP requests
    #[envconfig(default = "40")]
    pub max_concurrent_checkpoint_file_uploads: usize,

    // Maximum upload futures actively polled per partition checkpoint (files open with
    // read buffers + BufWriters). Bounds memory independently from S3 HTTP concurrency.
    // Each active buffer consumes ~18MB (8MB read buffer + ~10MB BufWriter).
    #[envconfig(default = "40")]
    pub max_upload_buffers_per_partition: usize,

    // Maximum time allowed for a complete checkpoint import for a single partition (seconds).
    // This includes listing checkpoints, downloading metadata, and downloading all files.
    // Should be less than kafka max.poll.interval.ms to prevent consumer group kicks.
    #[envconfig(default = "240")]
    pub checkpoint_partition_import_timeout_secs: u64,

    // Maximum age of a local metadata.json before we consider the local store stale
    // and fall back to S3 import. Separate from checkpoint_import_window_hours which
    // controls the S3 listing window.
    #[envconfig(default = "7200")] // 2 hours
    pub local_checkpoint_max_staleness_secs: u64,

    //// End checkpoint configuration ////
    //// Kafka-assigner mode configuration ////
    /// gRPC endpoint for the kafka-assigner service (e.g. "http://kafka-assigner:50051").
    /// When set, the consumer uses externally-driven partition assignment instead of
    /// Kafka's consumer group protocol.
    pub kafka_assigner_endpoint: Option<String>,

    /// Consumer name for registration with the kafka-assigner.
    /// Defaults to HOSTNAME (pod name) if not set.
    pub consumer_name: Option<String>,

    //// End kafka-assigner mode configuration ////
    /// Fail-open mode: bypass all deduplication and forward events directly to output topic.
    /// When enabled, the deduplicator skips store operations, checkpoint import/export,
    /// and treats all events as unique. Use as an emergency kill switch when the
    /// deduplication store is causing issues.
    #[envconfig(default = "false")]
    pub fail_open: bool,

    #[envconfig(default = "true")]
    pub export_prometheus: bool,

    // OpenTelemetry configuration
    #[envconfig(from = "OTEL_EXPORTER_OTLP_ENDPOINT")]
    pub otel_url: Option<String>,

    #[envconfig(from = "OTEL_TRACES_SAMPLER_ARG", default = "0.001")]
    pub otel_sampling_rate: f64,

    #[envconfig(from = "OTEL_SERVICE_NAME", default = "posthog-kafka-deduplicator")]
    pub otel_service_name: String,

    #[envconfig(from = "OTEL_LOG_LEVEL", default = "info")]
    pub otel_log_level: tracing::Level,
}

impl Config {
    /// Initialize from environment variables (for production and tests)
    pub fn init_with_defaults() -> Result<Self, envconfig::Error> {
        Config::init_from_env()
    }

    /// Validate configuration settings
    pub fn validate(&self) -> Result<()> {
        fs::create_dir_all(&self.store_path).with_context(|| {
            format!(
                "Cannot create RocksDB store directory '{}' for consumer group '{}'",
                self.store_path, self.kafka_consumer_group
            )
        })?;

        let test_file = self.store_path_buf().join(".write_test");
        fs::write(&test_file, b"test").with_context(|| {
            format!(
                "RocksDB store path '{}' is not writable for consumer group '{}'",
                self.store_path, self.kafka_consumer_group
            )
        })?;
        fs::remove_file(test_file).ok();

        if let Some(ref bucket) = self.s3_bucket {
            fs::create_dir_all(&self.local_checkpoint_dir).with_context(|| {
                format!(
                    "Cannot create local checkpoint directory '{}' for S3 bucket '{}'",
                    self.local_checkpoint_dir, bucket
                )
            })?;
        }

        Ok(())
    }

    /// Get RocksDB storage path as PathBuf
    pub fn store_path_buf(&self) -> PathBuf {
        PathBuf::from(&self.store_path)
    }

    /// Get server bind address
    pub fn bind_address(&self) -> String {
        format!("{}:{}", self.host, self.port)
    }

    /// Get poll timeout as Duration
    pub fn poll_timeout(&self) -> Duration {
        Duration::from_secs(self.poll_timeout_secs)
    }

    /// Get shutdown timeout as Duration
    pub fn shutdown_timeout(&self) -> Duration {
        Duration::from_secs(self.shutdown_timeout_secs)
    }

    /// Get commit interval as Duration
    pub fn commit_interval(&self) -> Duration {
        Duration::from_secs(self.commit_interval_secs)
    }

    /// Get kafka consumer batch timeout as Duration
    pub fn kafka_consumer_batch_timeout(&self) -> Duration {
        Duration::from_millis(self.kafka_consumer_batch_timeout_ms)
    }

    /// Get kafka consumer seek timeout as Duration (for seek_partitions after checkpoint import)
    pub fn kafka_consumer_seek_timeout(&self) -> Duration {
        Duration::from_secs(self.kafka_consumer_seek_timeout_secs)
    }

    /// Get flush interval as Duration
    pub fn flush_interval(&self) -> Duration {
        Duration::from_secs(self.flush_interval_secs)
    }

    /// Get cleanup interval as Duration
    pub fn cleanup_interval(&self) -> Duration {
        Duration::from_secs(self.cleanup_interval_secs)
    }

    /// Get orphan cleanup minimum staleness as Duration
    pub fn orphan_cleanup_min_staleness(&self) -> Duration {
        Duration::from_secs(self.orphan_cleanup_min_staleness_secs)
    }

    /// Get producer send timeout as Duration
    pub fn producer_send_timeout(&self) -> Duration {
        Duration::from_millis(self.kafka_producer_send_timeout_ms as u64)
    }

    /// Parse storage capacity from various formats:
    /// - Raw bytes: "1073741824"
    /// - Scientific notation: "9.663676416e+09"
    /// - Kubernetes units: "9Gi", "500Mi", "1Ki"
    /// - Decimal units: "1GB", "500MB", "1KB"
    pub fn parse_storage_capacity(&self) -> Result<u64> {
        let s = self.max_store_capacity.trim();

        // First try to parse as ByteSize (handles Gi, Mi, KB, MB, etc.)
        if let Ok(size) = s.parse::<ByteSize>() {
            return Ok(size.as_u64());
        }

        // Handle scientific notation (e.g., "9.663676416e+09")
        if s.contains('e') || s.contains('E') {
            let float_val: f64 = s
                .parse()
                .with_context(|| format!("Failed to parse scientific notation: {s}"))?;
            if float_val < 0.0 {
                return Err(anyhow::anyhow!("Storage capacity cannot be negative: {s}"));
            }
            if float_val > u64::MAX as f64 {
                return Err(anyhow::anyhow!(
                    "Storage capacity exceeds maximum value: {s}"
                ));
            }
            return Ok(float_val as u64);
        }

        // Try as raw integer
        s.parse::<u64>()
            .with_context(|| format!("Failed to parse storage capacity: '{s}'. Expected format: raw bytes, scientific notation, or units (1Gi, 1GB)"))
    }

    /// Build a RocksDbConfig from env-configured overrides, falling back to defaults.
    /// Validates max_background_jobs >= 1, clamping with a warning if invalid.
    pub fn build_rocksdb_config(&self) -> RocksDbConfig {
        let defaults = RocksDbConfig::default();
        let max_background_jobs = match self.rocksdb_max_background_jobs {
            Some(v) if v >= 1 => v,
            Some(v) => {
                tracing::warn!(
                    value = v,
                    default = defaults.max_background_jobs,
                    "ROCKSDB_MAX_BACKGROUND_JOBS must be >= 1, using default"
                );
                defaults.max_background_jobs
            }
            None => defaults.max_background_jobs,
        };
        let config = RocksDbConfig {
            shared_cache_size_bytes: self
                .rocksdb_shared_cache_size_bytes
                .unwrap_or(defaults.shared_cache_size_bytes),
            total_write_buffer_size_bytes: self
                .rocksdb_total_write_buffer_size_bytes
                .unwrap_or(defaults.total_write_buffer_size_bytes),
            max_background_jobs,
            write_buffer_size_bytes: self
                .rocksdb_write_buffer_size_bytes
                .unwrap_or(defaults.write_buffer_size_bytes),
            target_file_size_base_bytes: self
                .rocksdb_target_file_size_base_bytes
                .unwrap_or(defaults.target_file_size_base_bytes),
            max_open_files: self
                .rocksdb_max_open_files
                .unwrap_or(defaults.max_open_files),
            l0_compaction_trigger: self
                .rocksdb_l0_compaction_trigger
                .unwrap_or(defaults.l0_compaction_trigger),
            l0_slowdown_writes_trigger: self
                .rocksdb_l0_slowdown_writes_trigger
                .unwrap_or(defaults.l0_slowdown_writes_trigger),
            l0_stop_writes_trigger: self
                .rocksdb_l0_stop_writes_trigger
                .unwrap_or(defaults.l0_stop_writes_trigger),
            write_buffer_manager_allow_stall: self
                .rocksdb_write_buffer_manager_allow_stall
                .unwrap_or(defaults.write_buffer_manager_allow_stall),
            compression_type: parse_compression_type(&self.rocksdb_compression_type)
                .unwrap_or_else(|e| {
                    tracing::warn!(
                        value = self.rocksdb_compression_type,
                        error = %e,
                        "invalid ROCKSDB_COMPRESSION_TYPE, using default (lz4)"
                    );
                    defaults.compression_type
                }),
            compression_per_level: self
                .rocksdb_compression_per_level
                .as_deref()
                .map(|s| {
                    parse_compression_per_level(s).unwrap_or_else(|e| {
                        tracing::warn!(
                            value = s,
                            error = %e,
                            "invalid ROCKSDB_COMPRESSION_PER_LEVEL, ignoring"
                        );
                        defaults.compression_per_level.clone().unwrap_or_default()
                    })
                })
                .filter(|v| !v.is_empty()),
            bottommost_compression_type: self
                .rocksdb_bottommost_compression_type
                .as_deref()
                .and_then(|s| {
                    parse_compression_type(s)
                        .map_err(|e| {
                            tracing::warn!(
                                value = s,
                                error = %e,
                                "invalid ROCKSDB_BOTTOMMOST_COMPRESSION_TYPE, ignoring"
                            );
                            e
                        })
                        .ok()
                }),
            universal_compression_size_percent: self
                .rocksdb_universal_compression_size_percent
                .unwrap_or(defaults.universal_compression_size_percent),
        };

        if config.compression_per_level.is_some() && config.universal_compression_size_percent < 0 {
            tracing::warn!(
                "ROCKSDB_COMPRESSION_PER_LEVEL is set but ROCKSDB_UNIVERSAL_COMPRESSION_SIZE_PERCENT \
                 is < 0 (compress-all). Per-level settings will be IGNORED by RocksDB in Universal \
                 compaction mode. Set ROCKSDB_UNIVERSAL_COMPRESSION_SIZE_PERCENT=0 to enable \
                 per-level compression."
            );
        }

        config
    }

    // Check multiple conditions for safe checkpoint export enablement
    pub fn checkpoint_export_enabled(&self) -> bool {
        self.checkpoint_export_enabled
            && self.s3_bucket.is_some()
            && (self.s3_endpoint.is_some() || self.aws_region.is_some())
    }

    // Check multiple conditions for safe checkpoint import enablement
    pub fn checkpoint_import_enabled(&self) -> bool {
        self.checkpoint_import_enabled
            && self.s3_bucket.is_some()
            && (self.s3_endpoint.is_some() || self.aws_region.is_some())
    }

    /// Get checkpoint interval as Duration
    pub fn checkpoint_interval(&self) -> Duration {
        Duration::from_secs(self.checkpoint_interval_secs)
    }

    pub fn checkpoint_gate_interval(&self) -> Duration {
        Duration::from_millis(self.checkpoint_gate_interval_millis)
    }

    pub fn checkpoint_worker_shutdown_timeout(&self) -> Duration {
        Duration::from_secs(self.checkpoint_worker_shutdown_timeout_secs)
    }

    /// Get S3 per-operation (including all retries) timeout as Duration
    pub fn s3_operation_timeout(&self) -> Duration {
        Duration::from_secs(self.s3_operation_timeout_secs)
    }

    /// Get S3 per-attempt timeout as Duration
    pub fn s3_attempt_timeout(&self) -> Duration {
        Duration::from_secs(self.s3_attempt_timeout_secs)
    }

    /// Get checkpoint partition import timeout as Duration
    pub fn checkpoint_partition_import_timeout(&self) -> Duration {
        Duration::from_secs(self.checkpoint_partition_import_timeout_secs)
    }

    /// Get max staleness for local checkpoint data as Duration
    pub fn local_checkpoint_max_staleness(&self) -> Duration {
        Duration::from_secs(self.local_checkpoint_max_staleness_secs)
    }

    /// Build Kafka consumer configuration for the group-based batch consumer.
    /// Applies all relevant env-configured settings (connection, TLS, fetch/queued,
    /// group membership, sticky assignment, offset reset).
    pub fn build_batch_consumer_config(&self) -> rdkafka::ClientConfig {
        use crate::kafka::config::ConsumerConfigBuilder;

        ConsumerConfigBuilder::for_batch_consumer(&self.kafka_hosts, &self.kafka_consumer_group)
            .with_tls(self.kafka_tls)
            .with_max_partition_fetch_bytes(self.kafka_consumer_max_partition_fetch_bytes)
            .with_topic_metadata_refresh_interval_ms(self.kafka_topic_metadata_refresh_interval_ms)
            .with_metadata_max_age_ms(self.kafka_metadata_max_age_ms)
            .with_sticky_partition_assignment(self.pod_hostname.as_deref())
            .with_offset_reset(&self.kafka_consumer_offset_reset)
            .with_fetch_min_bytes(self.kafka_consumer_fetch_min_bytes)
            .with_fetch_max_bytes(self.kafka_consumer_fetch_max_bytes)
            .with_fetch_wait_max_ms(self.kafka_consumer_fetch_wait_max_ms)
            .with_queued_min_messages(self.kafka_consumer_queued_min_messages)
            .with_queued_max_messages_kbytes(self.kafka_consumer_queued_max_messages_kbytes)
            .with_max_poll_interval_ms(self.kafka_max_poll_interval_ms)
            .with_session_timeout_ms(self.kafka_session_timeout_ms)
            .with_heartbeat_interval_ms(self.kafka_heartbeat_interval_ms)
            .build()
    }

    /// Build Kafka consumer configuration for the assigner-driven consumer.
    /// Uses manual `assign()` with offset commits via the consumer group, but no
    /// group-coordination settings (session, heartbeat, max.poll, sticky assignment).
    pub fn build_assigner_consumer_config(&self) -> rdkafka::ClientConfig {
        use crate::kafka::config::ConsumerConfigBuilder;

        ConsumerConfigBuilder::for_assigner_consumer(&self.kafka_hosts, &self.kafka_consumer_group)
            .with_tls(self.kafka_tls)
            .with_max_partition_fetch_bytes(self.kafka_consumer_max_partition_fetch_bytes)
            .with_topic_metadata_refresh_interval_ms(self.kafka_topic_metadata_refresh_interval_ms)
            .with_metadata_max_age_ms(self.kafka_metadata_max_age_ms)
            .with_offset_reset(&self.kafka_consumer_offset_reset)
            .with_fetch_min_bytes(self.kafka_consumer_fetch_min_bytes)
            .with_fetch_max_bytes(self.kafka_consumer_fetch_max_bytes)
            .with_fetch_wait_max_ms(self.kafka_consumer_fetch_wait_max_ms)
            .with_queued_min_messages(self.kafka_consumer_queued_min_messages)
            .with_queued_max_messages_kbytes(self.kafka_consumer_queued_max_messages_kbytes)
            .build()
    }

    /// Build Kafka consumer configuration for the assign-only watermark consumer.
    /// Applies only connection, TLS, and fetch/queued settings — no group-coordination
    /// options (session, heartbeat, max.poll, sticky, offset reset).
    pub fn build_watermark_consumer_config(&self, group_id: &str) -> rdkafka::ClientConfig {
        use crate::kafka::config::ConsumerConfigBuilder;

        ConsumerConfigBuilder::for_watermark_consumer(&self.kafka_hosts, group_id)
            .with_tls(self.kafka_tls)
            .with_max_partition_fetch_bytes(self.kafka_consumer_max_partition_fetch_bytes)
            .with_topic_metadata_refresh_interval_ms(self.kafka_topic_metadata_refresh_interval_ms)
            .with_metadata_max_age_ms(self.kafka_metadata_max_age_ms)
            .with_fetch_min_bytes(self.kafka_consumer_fetch_min_bytes)
            .with_fetch_max_bytes(self.kafka_consumer_fetch_max_bytes)
            .with_fetch_wait_max_ms(self.kafka_consumer_fetch_wait_max_ms)
            .with_queued_min_messages(self.kafka_consumer_queued_min_messages)
            .with_queued_max_messages_kbytes(self.kafka_consumer_queued_max_messages_kbytes)
            .build()
    }

    /// Build Kafka producer configuration
    pub fn build_producer_config(&self) -> rdkafka::ClientConfig {
        let mut config = rdkafka::ClientConfig::new();
        config
            .set("bootstrap.servers", &self.kafka_hosts)
            .set(
                "message.timeout.ms",
                self.kafka_message_timeout_ms.to_string(),
            )
            .set(
                "queue.buffering.max.messages",
                self.kafka_producer_queue_messages.to_string(),
            )
            .set(
                "queue.buffering.max.ms",
                self.kafka_producer_linger_ms.to_string(),
            )
            .set("compression.type", &self.kafka_compression_codec);

        if self.kafka_tls {
            config
                .set("security.protocol", "ssl")
                .set("enable.ssl.certificate.verification", "false");
        }

        config
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_parse_storage_capacity_raw_bytes() {
        let mut config = Config::init_with_defaults().unwrap();

        // Test raw bytes
        config.max_store_capacity = "1073741824".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1073741824);

        config.max_store_capacity = "0".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 0);

        config.max_store_capacity = "1234567890".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1234567890);
    }

    #[test]
    fn test_parse_storage_capacity_scientific_notation() {
        let mut config = Config::init_with_defaults().unwrap();

        // Test scientific notation
        config.max_store_capacity = "9.663676416e+09".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 9663676416);

        config.max_store_capacity = "1e9".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1000000000);

        config.max_store_capacity = "1.5e10".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 15000000000);

        config.max_store_capacity = "2.5E9".to_string(); // Capital E
        assert_eq!(config.parse_storage_capacity().unwrap(), 2500000000);
    }

    #[test]
    fn test_parse_storage_capacity_kubernetes_units() {
        let mut config = Config::init_with_defaults().unwrap();

        // Test Kubernetes binary units (base 1024)
        config.max_store_capacity = "1Ki".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1024);

        config.max_store_capacity = "1Mi".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1024 * 1024);

        config.max_store_capacity = "1Gi".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1024 * 1024 * 1024);

        config.max_store_capacity = "9Gi".to_string();
        assert_eq!(
            config.parse_storage_capacity().unwrap(),
            9 * 1024 * 1024 * 1024
        );

        config.max_store_capacity = "500Mi".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 500 * 1024 * 1024);

        // With B suffix
        config.max_store_capacity = "1GiB".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1024 * 1024 * 1024);
    }

    #[test]
    fn test_parse_storage_capacity_decimal_units() {
        let mut config = Config::init_with_defaults().unwrap();

        // Test decimal units (base 1000)
        config.max_store_capacity = "1KB".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1000);

        config.max_store_capacity = "1MB".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1000 * 1000);

        config.max_store_capacity = "1GB".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1000 * 1000 * 1000);

        config.max_store_capacity = "500MB".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 500 * 1000 * 1000);
    }

    #[test]
    fn test_parse_storage_capacity_whitespace() {
        let mut config = Config::init_with_defaults().unwrap();

        // Test with whitespace
        config.max_store_capacity = "  1Gi  ".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 1024 * 1024 * 1024);

        config.max_store_capacity = " 9.663676416e+09 ".to_string();
        assert_eq!(config.parse_storage_capacity().unwrap(), 9663676416);
    }

    #[test]
    fn test_parse_storage_capacity_errors() {
        let mut config = Config::init_with_defaults().unwrap();

        // Test invalid formats
        config.max_store_capacity = "invalid".to_string();
        assert!(config.parse_storage_capacity().is_err());

        config.max_store_capacity = "-1".to_string();
        assert!(config.parse_storage_capacity().is_err());

        config.max_store_capacity = "-1e9".to_string();
        assert!(config.parse_storage_capacity().is_err());

        config.max_store_capacity = "".to_string();
        assert!(config.parse_storage_capacity().is_err());
    }

    #[test]
    fn test_checkpoint_export_enabled() {
        let mut config = Config::init_with_defaults().unwrap();

        // All disabled by default (no bucket, no endpoint, no region)
        config.checkpoint_export_enabled = true;
        assert!(!config.checkpoint_export_enabled());

        // Flag disabled - should be false regardless of other settings
        config.checkpoint_export_enabled = false;
        config.s3_bucket = Some("test-bucket".to_string());
        config.aws_region = Some("us-east-1".to_string());
        assert!(!config.checkpoint_export_enabled());

        // Production AWS: region + bucket (no endpoint)
        config.checkpoint_export_enabled = true;
        config.s3_bucket = Some("test-bucket".to_string());
        config.aws_region = Some("us-east-1".to_string());
        config.s3_endpoint = None;
        assert!(config.checkpoint_export_enabled());

        // Local dev MinIO: endpoint + bucket (no region)
        config.s3_bucket = Some("test-bucket".to_string());
        config.s3_endpoint = Some("http://localhost:9000".to_string());
        config.aws_region = None;
        assert!(config.checkpoint_export_enabled());

        // Local dev MinIO with region: endpoint + bucket + region
        config.s3_bucket = Some("test-bucket".to_string());
        config.s3_endpoint = Some("http://localhost:9000".to_string());
        config.aws_region = Some("us-east-1".to_string());
        assert!(config.checkpoint_export_enabled());

        // Missing bucket - should be false
        config.s3_bucket = None;
        config.s3_endpoint = Some("http://localhost:9000".to_string());
        config.aws_region = Some("us-east-1".to_string());
        assert!(!config.checkpoint_export_enabled());

        // Missing both endpoint and region - should be false
        config.s3_bucket = Some("test-bucket".to_string());
        config.s3_endpoint = None;
        config.aws_region = None;
        assert!(!config.checkpoint_export_enabled());
    }

    #[test]
    fn test_checkpoint_import_enabled() {
        let mut config = Config::init_with_defaults().unwrap();

        // All disabled by default (no bucket, no endpoint, no region)
        config.checkpoint_import_enabled = true;
        assert!(!config.checkpoint_import_enabled());

        // Flag disabled - should be false regardless of other settings
        config.checkpoint_import_enabled = false;
        config.s3_bucket = Some("test-bucket".to_string());
        config.aws_region = Some("us-east-1".to_string());
        assert!(!config.checkpoint_import_enabled());

        // Production AWS: region + bucket (no endpoint)
        config.checkpoint_import_enabled = true;
        config.s3_bucket = Some("test-bucket".to_string());
        config.aws_region = Some("us-east-1".to_string());
        config.s3_endpoint = None;
        assert!(config.checkpoint_import_enabled());

        // Local dev MinIO: endpoint + bucket (no region)
        config.s3_bucket = Some("test-bucket".to_string());
        config.s3_endpoint = Some("http://localhost:9000".to_string());
        config.aws_region = None;
        assert!(config.checkpoint_import_enabled());

        // Local dev MinIO with region: endpoint + bucket + region
        config.s3_bucket = Some("test-bucket".to_string());
        config.s3_endpoint = Some("http://localhost:9000".to_string());
        config.aws_region = Some("us-east-1".to_string());
        assert!(config.checkpoint_import_enabled());

        // Missing bucket - should be false
        config.s3_bucket = None;
        config.s3_endpoint = Some("http://localhost:9000".to_string());
        config.aws_region = Some("us-east-1".to_string());
        assert!(!config.checkpoint_import_enabled());

        // Missing both endpoint and region - should be false
        config.s3_bucket = Some("test-bucket".to_string());
        config.s3_endpoint = None;
        config.aws_region = None;
        assert!(!config.checkpoint_import_enabled());
    }

    #[test]
    fn test_build_rocksdb_config_default_compression() {
        let config = Config::init_with_defaults().unwrap();
        let rocksdb = config.build_rocksdb_config();

        assert_eq!(rocksdb.compression_type, rocksdb::DBCompressionType::Lz4);
        assert!(rocksdb.compression_per_level.is_none());
        assert!(rocksdb.bottommost_compression_type.is_none());
        assert_eq!(rocksdb.universal_compression_size_percent, -1);
    }

    #[test]
    fn test_build_rocksdb_config_compression_per_level() {
        let mut config = Config::init_with_defaults().unwrap();
        config.rocksdb_compression_per_level = Some("none,none,lz4,lz4,lz4".to_string());
        config.rocksdb_universal_compression_size_percent = Some(0);
        let rocksdb = config.build_rocksdb_config();

        let expected = vec![
            rocksdb::DBCompressionType::None,
            rocksdb::DBCompressionType::None,
            rocksdb::DBCompressionType::Lz4,
            rocksdb::DBCompressionType::Lz4,
            rocksdb::DBCompressionType::Lz4,
        ];
        assert_eq!(rocksdb.compression_per_level.unwrap(), expected);
        assert_eq!(rocksdb.universal_compression_size_percent, 0);
    }

    #[test]
    fn test_build_rocksdb_config_bottommost_compression() {
        let mut config = Config::init_with_defaults().unwrap();
        config.rocksdb_bottommost_compression_type = Some("zstd".to_string());
        let rocksdb = config.build_rocksdb_config();

        assert_eq!(
            rocksdb.bottommost_compression_type.unwrap(),
            rocksdb::DBCompressionType::Zstd
        );
    }

    #[test]
    fn test_build_rocksdb_config_invalid_compression_type_falls_back() {
        let mut config = Config::init_with_defaults().unwrap();
        config.rocksdb_compression_type = "brotli".to_string();
        let rocksdb = config.build_rocksdb_config();

        assert_eq!(rocksdb.compression_type, rocksdb::DBCompressionType::Lz4);
    }

    #[test]
    fn test_build_rocksdb_config_invalid_per_level_ignored() {
        let mut config = Config::init_with_defaults().unwrap();
        config.rocksdb_compression_per_level = Some("none,invalid,lz4".to_string());
        let rocksdb = config.build_rocksdb_config();

        assert!(rocksdb.compression_per_level.is_none());
    }

    #[test]
    fn test_build_rocksdb_config_invalid_bottommost_ignored() {
        let mut config = Config::init_with_defaults().unwrap();
        config.rocksdb_bottommost_compression_type = Some("invalid".to_string());
        let rocksdb = config.build_rocksdb_config();

        assert!(rocksdb.bottommost_compression_type.is_none());
    }
}