Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

32.1 KB
Raw Blame
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;

use anyhow::{Context, Result};
use common_kafka::config::KafkaConfig;
use common_kafka::kafka_producer::create_kafka_producer;

use health::{HealthHandle, HealthRegistry};
use tokio::sync::oneshot;
use tokio_util::sync::CancellationToken;
use tracing::{error, info};

use crate::config::PipelineType;
use crate::pipelines::ingestion_events::{DeduplicationConfig, DuplicateEventProducerWrapper};
use crate::pipelines::{PipelineBuilder, PipelineConsumer};
use crate::{
    checkpoint::{
        config::CheckpointConfig, export::CheckpointExporter, import::CheckpointImporter,
        s3_downloader::S3Downloader, s3_uploader::S3Uploader,
    },
    checkpoint_manager::CheckpointManager,
    config::Config,
    kafka::{OffsetTracker, PartitionRouterConfig, PartitionWorkerConfig},
    rebalance_tracker::RebalanceTracker,
    rocksdb::store::init_shared_resources,
    store::DeduplicationStoreConfig,
    store_manager::{CleanupTaskHandle, StoreManager},
};

/// The main Kafka Deduplicator service that encapsulates all components.
///
/// Note: `PipelineConsumer` is intentionally NOT stored on this struct. In assigner
/// mode the consumer contains `tonic::Streaming<T>` which is `!Sync`, and storing it
/// here would make the service `!Sync` — breaking `tokio::spawn` compatibility.
/// Instead, `initialize()` returns the consumer and callers pass it to the run methods.
pub struct KafkaDeduplicatorService {
    config: Config,
    store_manager: Arc<StoreManager>,
    offset_tracker: Arc<OffsetTracker>,
    checkpoint_manager: Option<CheckpointManager>,
    checkpoint_importer: Option<Arc<CheckpointImporter>>,
    cleanup_task_handle: Option<CleanupTaskHandle>,
    shutdown_tx: Option<oneshot::Sender<()>>,
    liveness: HealthRegistry,
    service_health: Option<HealthHandle>,
    health_task_cancellation: CancellationToken,
    health_task_handles: Vec<tokio::task::JoinHandle<()>>,
}

impl KafkaDeduplicatorService {
    /// Reset the local checkpoint directory (clear contents, preserving the directory itself)
    fn reset_checkpoint_directory(cfg: &CheckpointConfig) -> Result<()> {
        let checkpoint_dir = &cfg.local_checkpoint_dir;
        let path = std::path::Path::new(checkpoint_dir);

        // Create directory if it doesn't exist
        if !path.exists() {
            std::fs::create_dir_all(path).with_context(|| {
                format!("Failed to create checkpoint directory: {checkpoint_dir}")
            })?;
        } else {
            // Clear contents but preserve the directory (may be a mount point)
            info!("Clearing local checkpoint directory contents: {checkpoint_dir}");
            for entry in std::fs::read_dir(path)
                .with_context(|| format!("Failed to read checkpoint directory: {checkpoint_dir}"))?
            {
                let entry = entry?;
                let entry_path = entry.path();
                if entry_path.is_dir() {
                    std::fs::remove_dir_all(&entry_path).with_context(|| {
                        format!(
                            "Failed to remove checkpoint subdirectory: {}",
                            entry_path.display()
                        )
                    })?;
                } else {
                    std::fs::remove_file(&entry_path).with_context(|| {
                        format!("Failed to remove checkpoint file: {}", entry_path.display())
                    })?;
                }
            }
        }

        info!("Local checkpoint directory ready: {checkpoint_dir}");
        Ok(())
    }

    /// Create a new service from configuration
    pub async fn new(config: Config, liveness: HealthRegistry) -> Result<Self> {
        // Validate configuration
        config.validate().with_context(|| format!("Configuration validation failed for service with consumer topic '{}' and group '{}'", config.kafka_consumer_topic, config.kafka_consumer_group))?;

        // Build RocksDB config from env overrides and initialize shared resources
        let rocksdb_config = config.build_rocksdb_config();
        init_shared_resources(&rocksdb_config);

        // Create store configuration
        let store_config = DeduplicationStoreConfig {
            path: config.store_path_buf(),
            max_capacity: config
                .parse_storage_capacity()
                .context("Failed to parse max_store_capacity")?,
            rocksdb: rocksdb_config,
        };

        // Create rebalance coordinator first (other components depend on it)
        let rebalance_tracker = Arc::new(RebalanceTracker::new());

        // Create store manager for handling concurrent store creation
        let store_manager = Arc::new(StoreManager::new(
            store_config.clone(),
            rebalance_tracker.clone(),
        ));

        // Create offset tracker — shared between CheckpointManager (for writing offsets
        // into checkpoint metadata) and PipelineBuilder (for tracking committed offsets)
        let offset_tracker = Arc::new(OffsetTracker::new(rebalance_tracker));

        // In fail-open mode, skip store cleanup and checkpoint infrastructure
        let (cleanup_task_handle, checkpoint_manager, importer) = if config.fail_open {
            info!("Fail-open mode enabled — skipping cleanup task, checkpoint export/import");
            let checkpoint_config = CheckpointConfig::default();
            let checkpoint_manager = CheckpointManager::new_with_offset_tracker(
                checkpoint_config,
                store_manager.clone(),
                None,
                offset_tracker.clone(),
            );
            (None, checkpoint_manager, None)
        } else {
            Self::create_store_infrastructure(
                &config,
                &store_config,
                &store_manager,
                &offset_tracker,
            )
            .await?
        };

        Ok(Self {
            config,
            store_manager,
            offset_tracker,
            checkpoint_manager: Some(checkpoint_manager),
            checkpoint_importer: importer,
            cleanup_task_handle,
            shutdown_tx: None,
            liveness,
            service_health: None,
            health_task_cancellation: CancellationToken::new(),
            health_task_handles: Vec::new(),
        })
    }

    /// Create cleanup task, checkpoint exporter, and checkpoint importer.
    /// Skipped entirely when fail-open mode is active.
    async fn create_store_infrastructure(
        config: &Config,
        store_config: &DeduplicationStoreConfig,
        store_manager: &Arc<StoreManager>,
        offset_tracker: &Arc<OffsetTracker>,
    ) -> Result<(
        Option<CleanupTaskHandle>,
        CheckpointManager,
        Option<Arc<CheckpointImporter>>,
    )> {
        // Start periodic cleanup task if max_capacity is configured
        let cleanup_task_handle = if store_config.max_capacity > 0 {
            let cleanup_interval = config.cleanup_interval();
            let orphan_min_staleness = config.orphan_cleanup_min_staleness();
            let handle = store_manager
                .clone()
                .start_periodic_cleanup(cleanup_interval, orphan_min_staleness);
            info!(
                "Started periodic cleanup task with interval: {:?} for max capacity: {} bytes",
                cleanup_interval, store_config.max_capacity
            );
            Some(handle)
        } else {
            info!("Cleanup task not started - max_capacity is unlimited (0)");
            None
        };

        // Create checkpoint manager and inject an exporter to enable uploads
        let checkpoint_config = CheckpointConfig {
            checkpoint_interval: config.checkpoint_interval(),
            checkpoint_full_upload_interval: config.checkpoint_full_upload_interval,
            local_checkpoint_dir: config.local_checkpoint_dir.clone(),
            s3_bucket: config.s3_bucket.clone().unwrap_or_default(),
            s3_key_prefix: config.s3_key_prefix.clone(),
            aws_region: config.aws_region.clone(),
            s3_endpoint: config.s3_endpoint.clone(),
            s3_access_key_id: config.s3_access_key_id.clone(),
            s3_secret_access_key: config.s3_secret_access_key.clone(),
            s3_force_path_style: config.s3_force_path_style,
            max_concurrent_checkpoints: config.max_concurrent_checkpoints,
            checkpoint_gate_interval: config.checkpoint_gate_interval(),
            checkpoint_worker_shutdown_timeout: config.checkpoint_worker_shutdown_timeout(),
            checkpoint_import_window_hours: config.checkpoint_import_window_hours,
            s3_operation_timeout: config.s3_operation_timeout(),
            s3_attempt_timeout: config.s3_attempt_timeout(),
            s3_max_retries: config.s3_max_retries,
            checkpoint_import_attempt_depth: config.checkpoint_import_attempt_depth,
            max_concurrent_checkpoint_file_downloads: config
                .max_concurrent_checkpoint_file_downloads,
            max_concurrent_checkpoint_file_uploads: config.max_concurrent_checkpoint_file_uploads,
            max_upload_buffers_per_partition: config.max_upload_buffers_per_partition,
            checkpoint_partition_import_timeout: config.checkpoint_partition_import_timeout(),
            local_checkpoint_max_staleness: config.local_checkpoint_max_staleness(),
        };

        // Reset local checkpoint directory on startup (it's temporary storage)
        Self::reset_checkpoint_directory(&checkpoint_config)?;

        // create exporter conditionally if S3 config is populated
        let exporter = if config.checkpoint_export_enabled() {
            let uploader = match S3Uploader::new(checkpoint_config.clone()).await {
                Ok(uploader) => Box::new(uploader),
                Err(e) => {
                    error!(
                        error = ?e,
                        bucket = %config.s3_bucket.as_deref().unwrap_or(""),
                        region = %config.aws_region.as_deref().unwrap_or(""),
                        "Failed to initialize S3 client for checkpoint uploads"
                    );
                    return Err(e.context("S3 uploader: client initialization failed"));
                }
            };
            Some(Arc::new(CheckpointExporter::new(uploader)))
        } else {
            None
        };

        // if checkpoint import is enabled, create and configure the importer
        let importer = if config.checkpoint_import_enabled() {
            let downloader = match S3Downloader::new(&checkpoint_config).await {
                Ok(downloader) => Box::new(downloader),
                Err(e) => {
                    error!(
                        error = ?e,
                        bucket = %config.s3_bucket.as_deref().unwrap_or(""),
                        region = %config.aws_region.as_deref().unwrap_or(""),
                        "Failed to initialize S3 client for checkpoint downloads"
                    );
                    return Err(e.context("S3 downloader: client initialization failed"));
                }
            };
            Some(Arc::new(CheckpointImporter::new(
                downloader,
                store_config.path.clone(),
                config.checkpoint_import_attempt_depth,
                config.checkpoint_partition_import_timeout(),
            )))
        } else {
            None
        };

        let checkpoint_manager = CheckpointManager::new_with_offset_tracker(
            checkpoint_config,
            store_manager.clone(),
            exporter,
            offset_tracker.clone(),
        );

        Ok((cleanup_task_handle, checkpoint_manager, importer))
    }

    /// Initialize the Kafka consumer and prepare for running.
    /// Returns the consumer so callers can spawn it without storing a `!Sync` type on `self`.
    pub async fn initialize(&mut self) -> Result<PipelineConsumer> {
        let assigner_mode = self.config.kafka_assigner_endpoint.is_some();
        let consumer_config = if assigner_mode {
            self.config.build_assigner_consumer_config()
        } else {
            self.config.build_batch_consumer_config()
        };

        // Create partition router for parallel processing across partitions
        let router_config = PartitionRouterConfig {
            worker_config: PartitionWorkerConfig {
                channel_buffer_size: self.config.partition_worker_channel_buffer_size,
            },
        };

        // Create shutdown channel
        let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>();
        self.shutdown_tx = Some(shutdown_tx);

        // start checkpoint manager and async work loop threads, register health monitor
        let checkpoint_health_reporter = self.checkpoint_manager.as_mut().unwrap().start();

        // if health reporter is Some, this is the first time initializing
        // the checkpoint manager, and we should start the health monitor thread
        if checkpoint_health_reporter.is_some() {
            let checkpoint_health_handle = self
                .liveness
                .register("checkpoint_manager".to_string(), Duration::from_secs(30))
                .await;
            let cancellation = self.health_task_cancellation.child_token();
            let handle = tokio::spawn(async move {
                let mut interval = tokio::time::interval(Duration::from_secs(10));
                loop {
                    tokio::select! {
                        _ = cancellation.cancelled() => {
                            break;
                        }
                        _ = interval.tick() => {
                            if checkpoint_health_reporter.as_ref().unwrap().load(Ordering::SeqCst) {
                                checkpoint_health_handle.report_healthy().await;
                            } else {
                                // Explicitly report unhealthy when a worker dies
                                checkpoint_health_handle.report_status(health::ComponentStatus::Unhealthy).await;
                                error!("Checkpoint manager is unhealthy - checkpoint and/or cleanup loops died");
                            }
                        }
                    }
                }
            });
            self.health_task_handles.push(handle);
        }

        info!(
            "Started checkpoint manager (export enabled = {:?}, checkpoint interval = {:?})",
            self.checkpoint_manager.as_ref().unwrap().export_enabled(),
            self.config.checkpoint_interval(),
        );

        // Build pipeline consumer using the builder
        let mut builder = PipelineBuilder::new(
            self.config.pipeline_type,
            self.store_manager.clone(),
            consumer_config,
            router_config,
            self.config.kafka_consumer_topic.clone(),
            self.config.kafka_consumer_batch_size,
            self.config.kafka_consumer_batch_timeout(),
            self.config.commit_interval(),
            self.config.kafka_consumer_seek_timeout(),
            self.config.rebalance_cleanup_parallelism,
            self.config.fail_open,
        )
        .with_checkpoint_importer(self.checkpoint_importer.clone())
        .with_local_checkpoint_staleness(self.config.local_checkpoint_max_staleness())
        .with_offset_tracker(self.offset_tracker.clone());

        // Configure pipeline-specific options for ingestion events
        if self.config.pipeline_type == PipelineType::IngestionEvents {
            let (main_producer, duplicate_producer) =
                self.create_producers_for_ingestion_pipeline().await?;

            // Normalize empty strings to None for optional topic configs
            let output_topic = self
                .config
                .output_topic
                .as_ref()
                .filter(|s| !s.is_empty())
                .cloned();
            let duplicate_events_topic = self
                .config
                .duplicate_events_topic
                .as_ref()
                .filter(|s| !s.is_empty())
                .cloned();

            // Create deduplication config (store config already in store_manager)
            let dedup_config = DeduplicationConfig {
                output_topic,
                duplicate_events_topic,
                producer_config: self.config.build_producer_config(),
                store_config: self.store_manager.config().clone(),
                producer_send_timeout: self.config.producer_send_timeout(),
                flush_interval: self.config.flush_interval(),
                fail_open: self.config.fail_open,
            };

            builder =
                builder.with_ingestion_config(dedup_config, main_producer, duplicate_producer);
        }

        let pipeline_consumer =
            if let Some(ref assigner_endpoint) = self.config.kafka_assigner_endpoint {
                let consumer_name = self
                    .config
                    .consumer_name
                    .clone()
                    .or_else(|| self.config.pod_hostname.clone())
                    .unwrap_or_else(|| "unknown".to_string());

                info!(
                    assigner_endpoint,
                    consumer_name, "Building assigner-driven consumer"
                );

                builder
                    .build_assigner(assigner_endpoint, consumer_name, shutdown_rx)
                    .await?
            } else {
                builder.build(shutdown_rx)?
            };

        info!(
            pipeline_type = ?self.config.pipeline_type,
            topic = self.config.kafka_consumer_topic,
            assigner_mode,
            "Initialized pipeline"
        );

        // Register health check for the service
        self.service_health = Some(
            self.liveness
                .register("kafka_deduplicator".to_string(), Duration::from_secs(30))
                .await,
        );

        Ok(pipeline_consumer)
    }

    /// Create Kafka producers for the ingestion events pipeline
    async fn create_producers_for_ingestion_pipeline(
        &self,
    ) -> Result<(
        Option<Arc<rdkafka::producer::FutureProducer<common_kafka::kafka_producer::KafkaContext>>>,
        Option<DuplicateEventProducerWrapper>,
    )> {
        // Create KafkaConfig from our Config (used for both producers)
        let kafka_config = KafkaConfig {
            kafka_hosts: self.config.kafka_hosts.clone(),
            kafka_producer_linger_ms: self.config.kafka_producer_linger_ms,
            kafka_producer_queue_mib: self.config.kafka_producer_queue_mib,
            kafka_producer_queue_messages: self.config.kafka_producer_queue_messages,
            kafka_message_timeout_ms: self.config.kafka_message_timeout_ms,
            kafka_compression_codec: self.config.kafka_compression_codec.clone(),
            kafka_tls: self.config.kafka_tls,
            kafka_client_rack: String::new(),
            kafka_client_id: String::new(),
            kafka_producer_batch_size: None,
            kafka_producer_batch_num_messages: None,
            kafka_producer_enable_idempotence: None,
            kafka_producer_max_in_flight_requests_per_connection: None,
            kafka_producer_topic_metadata_refresh_interval_ms: None,
            kafka_producer_message_max_bytes: None,
            kafka_producer_sticky_partitioning_linger_ms: None,
            kafka_producer_partitioner: None,
            kafka_producer_acks: None,
            kafka_producer_retries: None,
        };

        // Normalize empty strings to None for optional topic configs
        let output_topic = self
            .config
            .output_topic
            .as_ref()
            .filter(|s| !s.is_empty())
            .cloned();
        let duplicate_events_topic = self
            .config
            .duplicate_events_topic
            .as_ref()
            .filter(|s| !s.is_empty())
            .cloned();

        // Create main producer for output topic if configured
        let main_producer = match &output_topic {
            Some(topic) => {
                info!("Creating Kafka producer for output topic: {}", topic);

                // Create a health handle for the main producer
                let main_producer_health = self
                    .liveness
                    .register(format!("main_producer_{topic}"), Duration::from_secs(30))
                    .await;

                // Create the producer using common module's function
                let producer = create_kafka_producer(&kafka_config, main_producer_health)
                    .await
                    .with_context(|| {
                        format!("Failed to create Kafka producer for output topic '{topic}'")
                    })?;

                Some(Arc::new(producer))
            }
            None => {
                info!("Output topic not configured, skipping main producer creation");
                None
            }
        };

        // Create duplicate events producer if configured
        let duplicate_producer = match &duplicate_events_topic {
            Some(topic) => {
                info!(
                    "Creating Kafka producer for duplicate events topic: {}",
                    topic
                );

                // Create a health handle for the duplicate producer
                let duplicate_producer_health = self
                    .liveness
                    .register(
                        format!("duplicate_producer_{topic}"),
                        Duration::from_secs(30),
                    )
                    .await;

                // Create the producer using common module's function
                let producer = create_kafka_producer(&kafka_config, duplicate_producer_health)
                    .await
                    .with_context(|| {
                        format!(
                            "Failed to create Kafka producer for duplicate events topic '{topic}'"
                        )
                    })?;

                // Wrap in DuplicateEventProducerWrapper
                Some(DuplicateEventProducerWrapper::new(
                    topic.clone(),
                    Arc::new(producer),
                )?)
            }
            None => {
                info!(
                    "Duplicate events topic not configured, skipping duplicate producer creation"
                );
                None
            }
        };

        Ok((main_producer, duplicate_producer))
    }

    /// Run the service (blocking until shutdown)
    pub async fn run(mut self) -> Result<()> {
        let consumer = self.initialize().await?;

        info!("Starting Kafka Deduplicator service");

        // Start health reporting task for the main service
        if let Some(health_handle) = self.service_health.clone() {
            let cancellation = self.health_task_cancellation.child_token();
            let handle = tokio::spawn(async move {
                let mut interval = tokio::time::interval(Duration::from_secs(10));
                loop {
                    tokio::select! {
                        _ = cancellation.cancelled() => {
                            break;
                        }
                        _ = interval.tick() => {
                            health_handle.report_healthy().await;
                        }
                    }
                }
            });
            self.health_task_handles.push(handle);
        }

        // Start consumption
        let consumer_handle = tokio::spawn(async move { consumer.start_consumption().await });

        // Wait for SIGTERM signal (Kubernetes graceful shutdown)
        use tokio::signal::unix::{signal, SignalKind};
        let mut sigterm = signal(SignalKind::terminate()).expect("Failed to listen for SIGTERM");

        sigterm.recv().await;
        info!("Received SIGTERM signal, shutting down gracefully...");

        // Send shutdown signal
        if let Some(shutdown_tx) = self.shutdown_tx.take() {
            let _ = shutdown_tx.send(());
        }

        // Cancel health reporting tasks
        self.health_task_cancellation.cancel();
        for handle in self.health_task_handles.drain(..) {
            let _ = handle.await;
        }

        // Stop the checkpoint manager
        if let Some(mut checkpoint_manager) = self.checkpoint_manager.take() {
            checkpoint_manager.stop().await;
        }

        // Wait for consumer to finish with timeout
        match tokio::time::timeout(self.config.shutdown_timeout(), consumer_handle).await {
            Ok(Ok(Ok(_))) => info!("Consumer stopped normally"),
            Ok(Ok(Err(e))) => error!("Consumer stopped with error: {e:#}"),
            Ok(Err(e)) => error!("Consumer task panicked: {e:#}"),
            Err(_) => error!(
                "Consumer shutdown timed out after {:?}",
                self.config.shutdown_timeout()
            ),
        }

        // Shutdown all stores cleanly
        self.store_manager.shutdown().await;

        info!("Kafka Deduplicator service stopped");
        Ok(())
    }

    /// Run the service with a custom shutdown signal (useful for testing)
    pub async fn run_with_shutdown(
        mut self,
        shutdown_signal: impl std::future::Future<Output = ()>,
    ) -> Result<()> {
        let consumer = self.initialize().await?;

        info!("Starting Kafka Deduplicator service");

        // Start health reporting task for the main service
        if let Some(health_handle) = self.service_health.clone() {
            let cancellation = self.health_task_cancellation.child_token();
            let handle = tokio::spawn(async move {
                let mut interval = tokio::time::interval(Duration::from_secs(10));
                loop {
                    tokio::select! {
                        _ = cancellation.cancelled() => {
                            break;
                        }
                        _ = interval.tick() => {
                            health_handle.report_healthy().await;
                        }
                    }
                }
            });
            self.health_task_handles.push(handle);
        }

        // Start consumption
        let consumer_handle = tokio::spawn(async move { consumer.start_consumption().await });

        // Wait for shutdown signal
        shutdown_signal.await;

        info!("Received shutdown signal, shutting down gracefully...");

        // Send shutdown signal
        if let Some(shutdown_tx) = self.shutdown_tx.take() {
            let _ = shutdown_tx.send(());
        }

        // Cancel health reporting tasks
        self.health_task_cancellation.cancel();
        for handle in self.health_task_handles.drain(..) {
            let _ = handle.await;
        }

        // Stop the checkpoint manager
        if let Some(mut checkpoint_manager) = self.checkpoint_manager.take() {
            checkpoint_manager.stop().await;
        }

        // Wait for consumer to finish with timeout
        match tokio::time::timeout(self.config.shutdown_timeout(), consumer_handle).await {
            Ok(Ok(Ok(_))) => info!("Consumer stopped normally"),
            Ok(Ok(Err(e))) => error!("Consumer stopped with error: {e:#}"),
            Ok(Err(e)) => error!("Consumer task panicked: {e:#}"),
            Err(_) => error!(
                "Consumer shutdown timed out after {:?}",
                self.config.shutdown_timeout()
            ),
        }

        // Shutdown all stores cleanly
        self.store_manager.shutdown().await;

        Ok(())
    }

    /// Shutdown the service gracefully
    pub async fn shutdown(&mut self) -> Result<()> {
        info!("Shutting down service...");

        if let Some(shutdown_tx) = self.shutdown_tx.take() {
            let _ = shutdown_tx.send(());
        }

        // Stop cleanup task if running
        if let Some(handle) = self.cleanup_task_handle.take() {
            info!("Stopping cleanup task...");
            handle.stop().await;
        }

        // Give some time for graceful shutdown
        tokio::time::sleep(std::time::Duration::from_millis(500)).await;

        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::checkpoint::config::CheckpointConfig;
    use tempfile::TempDir;

    fn make_config(dir: &std::path::Path) -> CheckpointConfig {
        CheckpointConfig {
            local_checkpoint_dir: dir.to_string_lossy().to_string(),
            ..Default::default()
        }
    }

    #[test]
    fn test_reset_checkpoint_directory_creates_if_not_exists() {
        let temp_dir = TempDir::new().unwrap();
        let checkpoint_dir = temp_dir.path().join("checkpoints");

        // Directory doesn't exist yet
        assert!(!checkpoint_dir.exists());

        let cfg = make_config(&checkpoint_dir);
        KafkaDeduplicatorService::reset_checkpoint_directory(&cfg).unwrap();

        // Directory now exists and is empty
        assert!(checkpoint_dir.exists());
        assert!(checkpoint_dir.is_dir());
        assert_eq!(std::fs::read_dir(&checkpoint_dir).unwrap().count(), 0);
    }

    #[test]
    fn test_reset_checkpoint_directory_clears_existing_files() {
        let temp_dir = TempDir::new().unwrap();
        let checkpoint_dir = temp_dir.path().join("checkpoints");
        std::fs::create_dir_all(&checkpoint_dir).unwrap();

        // Create some files
        std::fs::write(checkpoint_dir.join("file1.txt"), "content1").unwrap();
        std::fs::write(checkpoint_dir.join("file2.txt"), "content2").unwrap();
        assert_eq!(std::fs::read_dir(&checkpoint_dir).unwrap().count(), 2);

        let cfg = make_config(&checkpoint_dir);
        KafkaDeduplicatorService::reset_checkpoint_directory(&cfg).unwrap();

        // Directory still exists but is now empty
        assert!(checkpoint_dir.exists());
        assert_eq!(std::fs::read_dir(&checkpoint_dir).unwrap().count(), 0);
    }

    #[test]
    fn test_reset_checkpoint_directory_clears_nested_subdirs() {
        let temp_dir = TempDir::new().unwrap();
        let checkpoint_dir = temp_dir.path().join("checkpoints");
        std::fs::create_dir_all(&checkpoint_dir).unwrap();

        // Create nested structure like real checkpoint dirs
        let topic_dir = checkpoint_dir.join("topic-name");
        let partition_dir = topic_dir.join("0");
        let attempt_dir = partition_dir.join("20260115_061456");
        std::fs::create_dir_all(&attempt_dir).unwrap();
        std::fs::write(attempt_dir.join("checkpoint.sst"), "sst data").unwrap();
        std::fs::write(attempt_dir.join("MANIFEST"), "manifest").unwrap();

        // Also a file at top level
        std::fs::write(checkpoint_dir.join("lockfile"), "").unwrap();

        assert_eq!(std::fs::read_dir(&checkpoint_dir).unwrap().count(), 2);

        let cfg = make_config(&checkpoint_dir);
        KafkaDeduplicatorService::reset_checkpoint_directory(&cfg).unwrap();

        // Directory still exists but all contents cleared
        assert!(checkpoint_dir.exists());
        assert_eq!(std::fs::read_dir(&checkpoint_dir).unwrap().count(), 0);
        assert!(!topic_dir.exists());
    }

    #[test]
    fn test_reset_checkpoint_directory_preserves_base_directory() {
        let temp_dir = TempDir::new().unwrap();
        // Use the temp_dir itself as the checkpoint dir (simulates mount point)
        let checkpoint_dir = temp_dir.path().to_path_buf();

        // Create some content
        let subdir = checkpoint_dir.join("subdir");
        std::fs::create_dir_all(&subdir).unwrap();
        std::fs::write(subdir.join("file.txt"), "content").unwrap();
        std::fs::write(checkpoint_dir.join("root_file.txt"), "root").unwrap();

        let cfg = make_config(&checkpoint_dir);
        KafkaDeduplicatorService::reset_checkpoint_directory(&cfg).unwrap();

        // Base directory preserved, contents cleared
        assert!(checkpoint_dir.exists());
        assert_eq!(std::fs::read_dir(&checkpoint_dir).unwrap().count(), 0);
    }

    #[test]
    fn test_reset_checkpoint_directory_idempotent_on_empty() {
        let temp_dir = TempDir::new().unwrap();
        let checkpoint_dir = temp_dir.path().join("checkpoints");
        std::fs::create_dir_all(&checkpoint_dir).unwrap();

        let cfg = make_config(&checkpoint_dir);

        // Call multiple times on empty directory
        KafkaDeduplicatorService::reset_checkpoint_directory(&cfg).unwrap();
        KafkaDeduplicatorService::reset_checkpoint_directory(&cfg).unwrap();

        assert!(checkpoint_dir.exists());
        assert_eq!(std::fs::read_dir(&checkpoint_dir).unwrap().count(), 0);
    }
}