Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

91.0 KB
Raw Blame
use anyhow::{Context, Result};
use dashmap::DashMap;
use futures::stream::{self, StreamExt};
use std::collections::{HashMap, HashSet};
use std::fmt;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime};
use thiserror::Error;
use tokio::task::JoinHandle;
use tracing::{debug, error, info, warn};

/// Error type for store lookup operations
#[derive(Debug, Error)]
pub enum StoreError {
    /// Store not found - partition may have been revoked during rebalance
    #[error("No store registered for partition {topic}:{partition}")]
    NotFound { topic: String, partition: i32 },

    /// Other store operation error
    #[error("Store operation failed: {0}")]
    Other(#[from] anyhow::Error),
}

use chrono::Utc;

use crate::checkpoint::metadata::CheckpointMetadata;
use crate::kafka::batch_consumer::OnCommit;
use crate::kafka::offset_tracker::OffsetTracker;
use crate::kafka::types::Partition;
use crate::metrics::MetricsHelper;
use crate::metrics_const::{
    ACTIVE_STORE_COUNT, CLEANUP_BYTES_FREED_HISTOGRAM, CLEANUP_DURATION_HISTOGRAM,
    CLEANUP_OPERATIONS_COUNTER, REBALANCE_DIRECTORY_CLEANUP_DURATION_HISTOGRAM,
    STORE_CREATION_DURATION_MS, STORE_CREATION_EVENTS,
};
use crate::rebalance_tracker::RebalanceTracker;
use crate::rocksdb::metrics_consts::ROCKSDB_OLDEST_DATA_AGE_SECONDS_GAUGE;
use crate::store::{DeduplicationStore, DeduplicationStoreConfig};
use crate::utils::format_store_path;

/// Information about folder sizes on disk
#[derive(Debug, Clone)]
struct FolderInfo {
    name: String,
    size_bytes: u64,
}

impl FolderInfo {
    fn size_mb(&self) -> f64 {
        self.size_bytes as f64 / (1024.0 * 1024.0)
    }
}

impl fmt::Display for FolderInfo {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}: {:.2} MB", self.name, self.size_mb())
    }
}

/// Information about assigned partitions
#[derive(Debug, Clone)]
struct AssignedPartition {
    topic: String,
    partition: i32,
}

impl fmt::Display for AssignedPartition {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}:{}", self.topic, self.partition)
    }
}

/// Combined cleanup status information
#[derive(Debug)]
struct CleanupStatus {
    assigned_partitions: Vec<AssignedPartition>,
    folder_info: Vec<FolderInfo>,
    total_disk_usage_mb: f64,
}

impl fmt::Display for CleanupStatus {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        // Format partitions as a compact list
        let partitions: Vec<String> = self
            .assigned_partitions
            .iter()
            .map(|p| p.to_string())
            .collect();

        // Format folder info as compact list with sizes
        let folders: Vec<String> = self
            .folder_info
            .iter()
            .map(|fi| format!("{}:{:.2}MB", fi.name, fi.size_mb()))
            .collect();

        write!(
            f,
            "CleanupStatus {{ partitions: [{}], folders: [{}], total_disk_mb: {:.2}, partition_count: {}, folder_count: {} }}",
            partitions.join(", "),
            folders.join(", "),
            self.total_disk_usage_mb,
            self.assigned_partitions.len(),
            self.folder_info.len()
        )
    }
}

/// Manages the lifecycle of deduplication stores, handling concurrent access
/// and creation in a thread-safe manner.
///
/// The StoreManager ensures that:
/// - Only one store exists per partition
/// - Store creation is atomic and handles race conditions
/// - Parent directories are created as needed
/// - Failed store creations are properly handled
pub struct StoreManager {
    /// Map of partition to deduplication store
    stores: DashMap<Partition, DeduplicationStore>,

    /// Configuration for creating new stores
    store_config: DeduplicationStoreConfig,

    /// Metrics helper for global metrics
    metrics: MetricsHelper,

    /// Flag to prevent concurrent cleanup operations
    cleanup_running: AtomicBool,

    /// Coordinator for rebalance state (single source of truth)
    rebalance_tracker: Arc<RebalanceTracker>,
}

impl StoreManager {
    /// Create a new store manager with the given configuration and rebalance coordinator.
    ///
    /// The coordinator is used to check rebalance state during cleanup operations.
    pub fn new(
        store_config: DeduplicationStoreConfig,
        rebalance_tracker: Arc<RebalanceTracker>,
    ) -> Self {
        let metrics = MetricsHelper::new();

        Self {
            stores: DashMap::new(),
            store_config,
            metrics,
            cleanup_running: AtomicBool::new(false),
            rebalance_tracker,
        }
    }

    /// Get a reference to the rebalance coordinator.
    pub fn rebalance_tracker(&self) -> &Arc<RebalanceTracker> {
        &self.rebalance_tracker
    }

    /// Get an existing store for a partition, if it exists
    pub fn get(&self, topic: &str, partition: i32) -> Option<DeduplicationStore> {
        let partition_key = Partition::new(topic.to_string(), partition);
        self.stores.get(&partition_key).map(|entry| entry.clone())
    }

    /// Get an existing store for a partition during message processing.
    ///
    /// Returns `StoreError::NotFound` if the store doesn't exist. This is expected
    /// during rebalances due to rdkafka message buffering - the partition worker may
    /// still have buffered messages after the store is unregistered.
    ///
    /// The caller should handle `StoreError::NotFound` gracefully by:
    /// - Logging at warn level (expected during rebalance)
    /// - Recording metrics with topic/partition tags
    /// - Dropping the messages
    ///
    /// Use this method in the batch processor instead of `get_or_create` to avoid
    /// accidentally creating stores for revoked partitions.
    pub fn get_store(&self, topic: &str, partition: i32) -> Result<DeduplicationStore, StoreError> {
        let partition_key = Partition::new(topic.to_string(), partition);

        self.stores
            .get(&partition_key)
            .map(|entry| entry.clone())
            .ok_or_else(|| StoreError::NotFound {
                topic: topic.to_string(),
                partition,
            })
    }

    /// Get or create a deduplication store for a specific partition
    ///
    /// This method handles concurrent access safely:
    /// - If the store exists, it returns it immediately
    /// - If the store doesn't exist, it creates it atomically
    /// - If multiple workers try to create the same store, only one succeeds
    ///
    /// Use `get_or_create_for_rebalance` during rebalancing to pre-create stores.
    /// Use this method during message processing - it will warn if a store needs to be created
    /// (indicating pre-creation didn't complete in time).
    pub async fn get_or_create(&self, topic: &str, partition: i32) -> Result<DeduplicationStore> {
        self.get_or_create_internal(topic, partition, false).await
    }

    /// Get or create a deduplication store during rebalancing (pre-creation)
    ///
    /// This should be called during `async_setup_assigned_partitions` to pre-create stores
    /// before messages start flowing. Unlike `get_or_create`, this won't emit a warning
    /// when creating a new store.
    pub async fn get_or_create_for_rebalance(
        &self,
        topic: &str,
        partition: i32,
    ) -> Result<DeduplicationStore> {
        self.get_or_create_internal(topic, partition, true).await
    }

    /// Internal implementation of get_or_create with rebalance context
    async fn get_or_create_internal(
        &self,
        topic: &str,
        partition: i32,
        is_rebalance: bool,
    ) -> Result<DeduplicationStore> {
        let partition_key = Partition::new(topic.to_string(), partition);

        // Fast path: check if store already exists
        if let Some(store) = self.stores.get(&partition_key) {
            debug!(
                "Using existing deduplication store for partition {}:{}",
                topic, partition
            );
            return Ok(store.clone());
        }

        // Slow path: need to create the store
        // DashMap's entry API ensures only one worker creates the store
        let creation_start = std::time::Instant::now();
        let result = self
            .stores
            .entry(partition_key.clone())
            .or_try_insert_with(|| {
                // Generate store path inside the closure so only the creating thread generates it
                let store_path = self.build_store_path(topic, partition);
                // Ensure parent directory exists
                // Note: This is inside the closure so only the creating thread does this
                self.ensure_directory_exists(&store_path)?;

                // Warn if store is being created during message processing (not during rebalance)
                // This indicates pre-creation didn't complete in time
                if !is_rebalance {
                    warn!(
                        topic = topic,
                        partition = partition,
                        path = %store_path,
                        "Creating store during message processing - pre-creation did not complete in time"
                    );
                } else {
                    info!(
                        topic = topic,
                        partition = partition,
                        path = store_path,
                        "Pre-creating deduplication store during rebalance"
                    );
                }

                let mut partition_config = self.store_config.clone();
                partition_config.path = PathBuf::from(&store_path);

                DeduplicationStore::new(partition_config, topic.to_string(), partition)
                    .with_context(|| {
                        format!(
                            "Failed to create deduplication store for {topic}:{partition} at path {store_path}",
                        )
                    })
            });

        let creation_duration = creation_start.elapsed();
        match result {
            Ok(entry) => {
                metrics::histogram!(STORE_CREATION_DURATION_MS)
                    .record(creation_duration.as_millis() as f64);
                metrics::counter!(STORE_CREATION_EVENTS, "outcome" => "success").increment(1);
                let store = entry.clone();
                // Drop the entry reference before accessing stores.len() to avoid deadlock
                drop(entry);
                let store_count = self.stores.len();
                metrics::gauge!(ACTIVE_STORE_COUNT).set(store_count as f64);

                // Warn on slow store creation (> 5 seconds indicates potential issues)
                if creation_duration.as_secs() > 5 {
                    warn!(
                        topic = topic,
                        partition = partition,
                        duration_ms = creation_duration.as_millis(),
                        active_stores = store_count,
                        "Slow deduplication store creation"
                    );
                } else {
                    info!(
                        topic = topic,
                        partition = partition,
                        duration_ms = creation_duration.as_millis(),
                        active_stores = store_count,
                        "Successfully created deduplication store"
                    );
                }
                Ok(store)
            }
            Err(e) => {
                // This could happen if:
                // 1. Another worker created the store between our check and creation
                // 2. RocksDB failed to open/create the database

                // Check if another worker succeeded
                if let Some(store) = self.stores.get(&partition_key) {
                    warn!(
                        topic = topic,
                        partition = partition,
                        "Store was created by another worker, using existing store"
                    );
                    Ok(store.clone())
                } else {
                    // Real failure - no one succeeded in creating the store
                    metrics::counter!(STORE_CREATION_EVENTS, "outcome" => "failure").increment(1);

                    error!(
                        topic = topic,
                        partition = partition,
                        duration_ms = creation_duration.as_millis(),
                        error = ?e,
                        "Failed to create deduplication store"
                    );

                    Err(e)
                }
            }
        }
    }

    /// Remove a store from management and clean up its files.
    ///
    /// This is a convenience method that calls `unregister_store()` followed by
    /// `cleanup_store_files()`. Use this for simple cleanup outside rebalance scenarios.
    ///
    /// For rebalance scenarios, prefer the two-step process:
    /// 1. Call `unregister_store()` BEFORE shutting down workers (prevents new store creation)
    /// 2. Call `cleanup_store_files()` AFTER workers are fully stopped (safe file deletion)
    ///
    /// See `test_rebalance_removes_stores_before_workers_shutdown` for rationale.
    pub fn remove(&self, topic: &str, partition: i32) -> Result<()> {
        self.unregister_store(topic, partition);
        self.cleanup_store_files(topic, partition)
    }

    // Internally register a restored set of checkpoint files at the given store path
    // and topic/partition coordinates
    pub fn restore_imported_store(&self, topic: &str, partition: i32, path: &Path) -> Result<()> {
        let store_config = DeduplicationStoreConfig {
            path: path.to_path_buf(),
            max_capacity: self.store_config.max_capacity,
            rocksdb: self.store_config.rocksdb.clone(),
        };
        let restored = DeduplicationStore::new(store_config, topic.to_string(), partition)
            .with_context(|| {
                format!(
                    "Failed to restore imported checkpoint for {topic}:{partition} at path {}",
                    path.display(),
                )
            })?;

        // Don't fail here but do report this it's evidence of a race condition
        if let Some(existing_store) = self
            .stores
            .insert(Partition::new(topic.to_string(), partition), restored)
        {
            metrics::counter!(
                STORE_CREATION_EVENTS,
                "outcome" => "duplicate_on_restore",
            )
            .increment(1);
            error!(
                existing_store_path =% existing_store.get_db_path().display(),
                restored_store_path =% path.display(),
                topic = topic,
                partition = partition,
                "Unexpected duplicate store found when registering imported checkpoint"
            );
        }

        Ok(())
    }

    /// Check if a partition has fresh local data and return its metadata if so.
    ///
    /// Reads `metadata.json` from the partition's store directory and validates that its
    /// `updated_at` timestamp is within `max_staleness` of now. Returns `Some(metadata)` if
    /// the local store is fresh, `None` if stale, missing, or corrupt.
    ///
    /// The caller is responsible for opening RocksDB (via `restore_imported_store`) after a
    /// `Some` return — this method only validates freshness.
    pub async fn try_restore_local_store(
        &self,
        topic: &str,
        partition: i32,
        max_staleness: Duration,
    ) -> Option<CheckpointMetadata> {
        let store_path = format_store_path(&self.store_config.path, topic, partition);

        let metadata = match CheckpointMetadata::load_from_dir(&store_path).await {
            Ok(m) => m,
            Err(_) => return None, // missing or corrupt
        };

        let age = (Utc::now() - metadata.updated_at)
            .to_std()
            .unwrap_or(Duration::MAX);

        if age > max_staleness {
            return None; // stale
        }

        Some(metadata)
    }

    /// Update metadata.json for a partition after a successful Kafka offset commit.
    ///
    /// Loads existing metadata (or creates a new skeleton if missing), updates the consumer
    /// and producer offsets, and writes it back. Failures are non-fatal: if the store
    /// directory doesn't exist yet or the write fails, a warning is logged and processing
    /// continues normally.
    pub async fn update_metadata_for_partition(
        &self,
        topic: &str,
        partition: i32,
        consumer_offset: i64,
        producer_offset: i64,
    ) {
        // Skip if the store has been unregistered (e.g. during rebalance).
        // The directory may be deleted concurrently by cleanup_store_files,
        // causing write_to_dir's rename to fail with ENOENT.
        if self.get(topic, partition).is_none() {
            return;
        }

        let store_path = format_store_path(&self.store_config.path, topic, partition);
        if !store_path.exists() {
            return;
        }

        let mut metadata = match CheckpointMetadata::load_from_dir(&store_path).await {
            Ok(m) => m,
            Err(_) => {
                // No existing metadata.json — create a skeleton for local tracking only.
                // sequence=0 and empty files list are fine: this metadata is only used for
                // local recovery (offsets + freshness), never for S3 restoration.
                CheckpointMetadata::new(
                    topic.to_string(),
                    partition,
                    Utc::now(),
                    0,
                    consumer_offset,
                    producer_offset,
                )
            }
        };

        metadata.consumer_offset = consumer_offset;
        metadata.producer_offset = producer_offset;

        if let Err(e) = metadata.write_to_dir(&store_path).await {
            warn!(
                topic = topic,
                partition = partition,
                error = %e,
                "Failed to update metadata.json after commit (non-fatal)"
            );
        }
    }

    /// Unregister a store from the DashMap without deleting files (Step 1 of two-step cleanup).
    ///
    /// Call this BEFORE shutting down partition workers during rebalance. This prevents
    /// workers from creating new stores via `get_or_create()` during their shutdown.
    ///
    /// After workers are fully stopped, call `cleanup_store_files()` to delete the files.
    ///
    /// The two-step process prevents a race condition where:
    /// 1. Worker is processing during shutdown
    /// 2. Worker calls `get_or_create()` which creates a new store
    /// 3. `remove()` deletes the directory
    /// 4. Worker's write fails with "No such file or directory"
    pub fn unregister_store(&self, topic: &str, partition: i32) {
        let partition_key = Partition::new(topic.to_string(), partition);

        if let Some((_, store)) = self.stores.remove(&partition_key) {
            info!(
                topic = topic,
                partition = partition,
                "Unregistering deduplication store"
            );
            // Drop the store explicitly to close RocksDB
            drop(store);
        }
    }

    /// Delete all files for a partition from disk (Step 2 of two-step cleanup).
    ///
    /// Call this AFTER workers are fully shut down to avoid race conditions where
    /// a worker tries to write to a deleted directory.
    ///
    /// Must be called after `unregister_store()` to ensure RocksDB is closed first.
    pub fn cleanup_store_files(&self, topic: &str, partition: i32) -> Result<()> {
        let partition_path = format_store_path(&self.store_config.path, topic, partition);
        let partition_dir_str = partition_path.to_string_lossy().to_string();

        if partition_path.exists() {
            match std::fs::remove_dir_all(&partition_path) {
                Ok(_) => {
                    info!(
                        topic = topic,
                        partition = partition,
                        path = partition_dir_str,
                        "Deleted partition directory"
                    );
                }
                Err(e) => {
                    warn!(
                        topic = topic,
                        partition = partition,
                        path = partition_dir_str,
                        error = %e,
                        "Failed to remove partition directory (usually harmless)"
                    );
                }
            }
        } else {
            debug!(
                topic = topic,
                partition = partition,
                path = partition_dir_str,
                "Partition directory doesn't exist"
            );
        }

        Ok(())
    }

    /// Delete partition directories not in the owned set using bounded parallelism.
    ///
    /// Called at end of rebalance cycle to clean up directories for revoked partitions.
    /// Uses scatter-gather pattern with configurable parallelism for fast cleanup
    /// before resuming consumption.
    ///
    /// This is simpler and more aggressive than the periodic orphan cleaner:
    /// - No staleness check (we know ownership is final at end of cycle)
    /// - Also catches orphans from previous runs
    pub async fn cleanup_unowned_partition_directories(
        &self,
        owned: &[Partition],
        parallelism: usize,
    ) -> Result<()> {
        let start = Instant::now();

        // Build set of owned store paths for O(1) lookup
        let owned_paths: HashSet<PathBuf> = owned
            .iter()
            .map(|p| format_store_path(&self.store_config.path, p.topic(), p.partition_number()))
            .collect();

        // Scan disk: <base>/<topic>/<partition>/ structure
        let base_path = &self.store_config.path;
        let mut unowned_paths = Vec::new();

        let mut topic_dirs = match tokio::fs::read_dir(base_path).await {
            Err(e) => {
                warn!(
                    path = %base_path.display(),
                    error = ?e,
                    "Failed to read store base directory for cleanup"
                );
                return Ok(());
            }
            Ok(rd) => rd,
        };

        while let Ok(Some(topic_entry)) = topic_dirs.next_entry().await {
            let is_dir = topic_entry
                .file_type()
                .await
                .map(|ft| ft.is_dir())
                .unwrap_or(false);
            if !is_dir {
                continue;
            }

            let topic_path = topic_entry.path();
            let mut partition_dirs = match tokio::fs::read_dir(&topic_path).await {
                Ok(rd) => rd,
                Err(_) => continue,
            };

            let mut topic_has_owned = false;
            let mut unowned_partition_paths: Vec<PathBuf> = Vec::new();
            while let Ok(Some(partition_entry)) = partition_dirs.next_entry().await {
                let is_dir = partition_entry
                    .file_type()
                    .await
                    .map(|ft| ft.is_dir())
                    .unwrap_or(false);
                if !is_dir {
                    continue;
                }

                let partition_path = partition_entry.path();
                if owned_paths.contains(&partition_path) {
                    topic_has_owned = true;
                } else {
                    unowned_partition_paths.push(partition_path);
                }
            }

            if !topic_has_owned {
                // No owned partitions remain — delete the whole topic dir in one shot
                unowned_paths.push(topic_path);
            } else {
                // Only delete unowned partitions; keep topic dir for owned children
                unowned_paths.extend(unowned_partition_paths);
            }
        }

        if unowned_paths.is_empty() {
            debug!("No unowned partition directories to clean up");
            return Ok(());
        }

        let count = unowned_paths.len();
        info!(
            count = count,
            parallelism = parallelism,
            "Cleaning up unowned partition directories"
        );

        // Delete in parallel with bounded concurrency
        let results: Vec<std::io::Result<()>> = stream::iter(unowned_paths)
            .map(|path| async move {
                let path_str = path.display().to_string();
                match tokio::fs::remove_dir_all(&path).await {
                    Ok(_) => {
                        debug!(path = %path_str, "Deleted unowned partition directory");
                        Ok(())
                    }
                    Err(e) => {
                        warn!(path = %path_str, error = ?e, "Failed to delete partition directory");
                        Err(e)
                    }
                }
            })
            .buffer_unordered(parallelism)
            .collect()
            .await;

        // Record timing
        let duration = start.elapsed();
        self.metrics
            .histogram(REBALANCE_DIRECTORY_CLEANUP_DURATION_HISTOGRAM)
            .record(duration.as_secs_f64());

        let failed = results.iter().filter(|r| r.is_err()).count();
        let succeeded = count - failed;

        if failed > 0 {
            warn!(
                succeeded = succeeded,
                failed = failed,
                duration_ms = duration.as_millis(),
                "Partition directory cleanup completed with failures (orphan cleaner will retry)"
            );
        } else {
            info!(
                deleted = succeeded,
                duration_ms = duration.as_millis(),
                "Partition directory cleanup completed"
            );
        }

        Ok(())
    }

    /// Get a reference to the underlying stores map
    /// Used by checkpoint manager and rebalance handler
    pub fn stores(&self) -> &DashMap<Partition, DeduplicationStore> {
        &self.stores
    }

    pub fn get_active_store_count(&self) -> usize {
        self.stores.len()
    }

    /// Get the base path where stores are created
    pub fn base_path(&self) -> &Path {
        &self.store_config.path
    }

    /// Get the store configuration
    pub fn config(&self) -> &DeduplicationStoreConfig {
        &self.store_config
    }

    /// Cleanup old entries across all stores to maintain global capacity
    ///
    /// This method checks the total size across all stores and triggers cleanup
    /// on individual stores if the global capacity is exceeded. Cleanup is distributed
    /// across all stores by removing a percentage of each store's time range.
    pub fn cleanup_old_entries_if_needed(&self) -> Result<u64> {
        // Try to acquire cleanup lock - if another cleanup is running, skip this one
        if self
            .cleanup_running
            .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
            .is_err()
        {
            debug!("Cleanup already running, skipping this cycle");
            return Ok(0);
        }

        // Ensure we release the lock when we're done
        let _guard = CleanupGuard {
            flag: &self.cleanup_running,
        };

        // Skip cleanup during rebalance to avoid deleting entries from stores
        // that are being populated with imported checkpoints
        if self.rebalance_tracker.is_rebalancing() {
            debug!("Skipping capacity cleanup - rebalance in progress");
            return Ok(0);
        }

        let start_time = Instant::now();

        // If max_capacity is 0, no cleanup needed (unlimited)
        if self.store_config.max_capacity == 0 {
            return Ok(0);
        }

        // Collect store clones first to release DashMap guards before slow operations.
        // This prevents blocking other DashMap operations during size calculation and cleanup.
        let stores: Vec<DeduplicationStore> = self
            .stores
            .iter()
            .map(|entry| entry.value().clone())
            .collect();

        // Calculate total size across all stores (no longer holding DashMap guards)
        let mut total_size = 0u64;
        for store in &stores {
            if let Ok(size) = store.get_total_size() {
                total_size += size;
            }
        }

        // Start cleanup at 80% capacity to give compaction headroom
        let cleanup_threshold = (self.store_config.max_capacity as f64 * 0.8) as u64;
        if total_size <= cleanup_threshold {
            return Ok(0); // Under threshold, no cleanup needed
        }

        // Determine capacity level for logging and cleanup aggressiveness
        let capacity_ratio = total_size as f64 / self.store_config.max_capacity as f64;
        let capacity_percent = (capacity_ratio * 100.0) as u32;

        info!(
            "Global store size {} ({}% of max capacity {}) exceeds cleanup threshold, triggering cleanup",
            total_size, capacity_percent, self.store_config.max_capacity
        );

        // We need to clean up - target 70% of max capacity to create buffer
        let target_size = (self.store_config.max_capacity as f64 * 0.7) as u64;
        let bytes_to_free = total_size.saturating_sub(target_size);

        // Calculate cleanup percentage based on how much we need to free
        // If we need to free 20% of total size, clean up 20% of time range from each store
        let raw_cleanup_percentage = bytes_to_free as f64 / total_size as f64;

        // When over 90% capacity, be more aggressive - no cap on cleanup percentage
        // Otherwise cap at 30% to avoid removing too much data at once
        let cleanup_percentage = if capacity_ratio > 0.9 {
            info!(
                "Critical capacity ({}%) - using aggressive cleanup without cap",
                capacity_percent
            );
            raw_cleanup_percentage.min(0.5) // Still cap at 50% to avoid removing everything
        } else {
            raw_cleanup_percentage.min(0.3) // Normal cap at 30%
        };

        info!(
            "Cleaning up {:.1}% of time range from each store (need to free {} bytes)",
            cleanup_percentage * 100.0,
            bytes_to_free
        );

        // Cleanup stores with the calculated percentage (no longer holding DashMap guards)
        let mut total_bytes_freed = 0u64;
        for store in &stores {
            // Check if rebalance started mid-cleanup - abort to avoid deleting
            // entries from stores being populated with imported checkpoints
            if self.rebalance_tracker.is_rebalancing() {
                info!(
                    "Aborting capacity cleanup - rebalance started. Freed {} bytes so far",
                    total_bytes_freed
                );
                return Ok(total_bytes_freed);
            }

            match store.cleanup_old_entries_with_percentage(cleanup_percentage) {
                Ok(bytes_freed) => {
                    total_bytes_freed += bytes_freed;
                    if bytes_freed > 0 {
                        info!(
                            "Freed {} bytes from store {}:{}",
                            bytes_freed,
                            store.get_topic(),
                            store.get_partition()
                        );
                    }
                }
                Err(e) => {
                    warn!(
                        "Failed to cleanup store {}:{}: {:#}",
                        store.get_topic(),
                        store.get_partition(),
                        e
                    );
                }
            }
        }

        // Emit cleanup metrics
        let duration = start_time.elapsed();
        self.metrics
            .counter(CLEANUP_OPERATIONS_COUNTER)
            .increment(1);
        self.metrics
            .histogram(CLEANUP_DURATION_HISTOGRAM)
            .record(duration.as_secs_f64());
        self.metrics
            .histogram(CLEANUP_BYTES_FREED_HISTOGRAM)
            .record(total_bytes_freed as f64);

        info!(
            "Global cleanup completed: freed {} bytes in {:?}",
            total_bytes_freed, duration
        );

        Ok(total_bytes_freed)
    }

    /// Check if cleanup is needed based on current global size
    /// Cleanup triggers at 80% capacity to give background compaction time to reclaim space
    pub fn needs_cleanup(&self) -> bool {
        // Log folder sizes and assigned partitions
        self.log_folder_sizes_and_partitions();

        if self.store_config.max_capacity == 0 {
            return false;
        }

        // Collect store clones first to release DashMap guards before slow operations
        let stores: Vec<DeduplicationStore> = self
            .stores
            .iter()
            .map(|entry| entry.value().clone())
            .collect();

        let mut total_size = 0u64;
        let mut max_oldest_data_age: Option<u64> = None;
        for store in &stores {
            if let Ok(size) = store.get_total_size() {
                total_size += size;
            }
            // Emit per-partition oldest data age metric and track max
            if let Ok(Some(age)) = store.get_oldest_data_age_seconds() {
                metrics::gauge!(
                    ROCKSDB_OLDEST_DATA_AGE_SECONDS_GAUGE,
                    "topic" => store.get_topic().to_string(),
                    "partition" => store.get_partition().to_string()
                )
                .set(age as f64);
                max_oldest_data_age =
                    Some(max_oldest_data_age.map_or(age, |current| current.max(age)));
            }
        }

        // Start cleanup at 80% capacity to give compaction headroom
        let cleanup_threshold = (self.store_config.max_capacity as f64 * 0.8) as u64;

        info!(
            "Total size of all stores: {} bytes, cleanup threshold: {} bytes, max capacity: {} bytes, oldest data age: {}s",
            total_size, cleanup_threshold, self.store_config.max_capacity, max_oldest_data_age.unwrap_or(0)
        );
        total_size > cleanup_threshold
    }

    /// Start a periodic cleanup task that runs in the background
    /// Returns a handle that can be used to stop the task
    pub fn start_periodic_cleanup(
        self: Arc<Self>,
        cleanup_interval: Duration,
        orphan_min_staleness: Duration,
    ) -> CleanupTaskHandle {
        let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
        let manager = self;

        let handle = tokio::spawn(async move {
            let mut interval = tokio::time::interval(cleanup_interval);
            interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);

            info!(
                "Started periodic cleanup task with interval of {:?}, orphan staleness {:?}",
                cleanup_interval, orphan_min_staleness
            );

            loop {
                tokio::select! {
                    _ = interval.tick() => {
                        info!("Cleanup task tick - running periodic cleanup check");

                        // First, clean up orphaned directories (unassigned partitions)
                        match manager.cleanup_orphaned_directories(orphan_min_staleness) {
                            Ok(0) => {
                                debug!("No orphaned directories found");
                            }
                            Ok(bytes_freed) => {
                                info!("Cleaned up {} bytes of orphaned directories", bytes_freed);
                            }
                            Err(e) => {
                                warn!("Failed to clean up orphaned directories: {e:#}");
                            }
                        }

                        // Then check if we need capacity-based cleanup
                        if manager.needs_cleanup() {
                            match manager.cleanup_old_entries_if_needed() {
                                Ok(0) => {
                                    debug!("Cleanup skipped (may be already running or no data to clean)");
                                }
                                Ok(bytes_freed) => {
                                    info!("Periodic cleanup freed {} bytes", bytes_freed);
                                }
                                Err(e) => {
                                    error!("Periodic cleanup failed: {e:#}");
                                }
                            }
                        } else {
                            info!("No cleanup needed, stores within capacity");
                        }
                    }
                    _ = &mut shutdown_rx => {
                        info!("Cleanup task received shutdown signal");
                        break;
                    }
                }
            }

            info!("Cleanup task shutting down");
        });

        CleanupTaskHandle {
            handle,
            shutdown_tx: Some(shutdown_tx),
        }
    }

    /// Shutdown all stores cleanly
    ///
    /// This closes all RocksDB instances but does NOT delete the files
    /// (since they may be needed when the service restarts)
    pub async fn shutdown(&self) {
        info!("Shutting down all deduplication stores");

        // Clear the map, which will drop all stores
        self.stores.clear();

        info!("All deduplication stores have been closed");
    }

    /// Build the path for a store based on topic and partition.
    /// Each partition gets a single directory: `<base>/<topic>/<partition>/`
    fn build_store_path(&self, topic: &str, partition: i32) -> String {
        format_store_path(&self.store_config.path, topic, partition)
            .to_string_lossy()
            .to_string()
    }

    /// Ensure the parent directory for a store path exists
    fn ensure_directory_exists(&self, store_path: &str) -> Result<()> {
        let path_buf = PathBuf::from(store_path);

        // RocksDB will create the final directory, we just need the parent
        if let Some(parent) = path_buf.parent() {
            if !parent.exists() {
                std::fs::create_dir_all(parent).with_context(|| {
                    format!("Failed to create parent directory: {}", parent.display())
                })?;
                debug!("Created parent directory: {}", parent.display());
            }
        }

        Ok(())
    }

    /// Log folder sizes and assigned partitions for debugging
    fn log_folder_sizes_and_partitions(&self) {
        let status = self.get_cleanup_status();
        info!("{}", status);
    }

    /// Get cleanup status information
    fn get_cleanup_status(&self) -> CleanupStatus {
        // Get assigned partitions
        let mut assigned_partitions = Vec::new();
        for entry in self.stores.iter() {
            let partition = entry.key();
            assigned_partitions.push(AssignedPartition {
                topic: partition.topic().to_string(),
                partition: partition.partition_number(),
            });
        }

        // Sort partitions for consistent output
        assigned_partitions
            .sort_by(|a, b| a.topic.cmp(&b.topic).then(a.partition.cmp(&b.partition)));

        // Get folder sizes from filesystem (<base>/<topic>/<partition>/ structure)
        let mut folder_info = Vec::new();
        let mut total_disk_usage_bytes = 0u64;

        if let Ok(topic_entries) = std::fs::read_dir(&self.store_config.path) {
            for topic_entry in topic_entries.flatten() {
                if !topic_entry.metadata().map(|m| m.is_dir()).unwrap_or(false) {
                    continue;
                }
                let topic_name = topic_entry.file_name().to_string_lossy().to_string();

                if let Ok(partition_entries) = std::fs::read_dir(topic_entry.path()) {
                    for partition_entry in partition_entries.flatten() {
                        if !partition_entry
                            .metadata()
                            .map(|m| m.is_dir())
                            .unwrap_or(false)
                        {
                            continue;
                        }
                        let partition_name =
                            partition_entry.file_name().to_string_lossy().to_string();
                        let folder_size =
                            StoreManager::get_directory_size(&partition_entry.path()).unwrap_or(0);

                        total_disk_usage_bytes += folder_size;
                        folder_info.push(FolderInfo {
                            name: format!("{topic_name}/{partition_name}"),
                            size_bytes: folder_size,
                        });
                    }
                }
            }

            // Sort by size descending
            folder_info.sort_by(|a, b| b.size_bytes.cmp(&a.size_bytes));
        }

        CleanupStatus {
            assigned_partitions,
            folder_info,
            total_disk_usage_mb: total_disk_usage_bytes as f64 / (1024.0 * 1024.0),
        }
    }

    /// Calculate the size of a directory recursively
    fn get_directory_size(path: &std::path::Path) -> Result<u64> {
        let mut size = 0u64;

        if let Ok(entries) = std::fs::read_dir(path) {
            for entry in entries.flatten() {
                if let Ok(metadata) = entry.metadata() {
                    if metadata.is_file() {
                        size += metadata.len();
                    } else if metadata.is_dir() {
                        size += StoreManager::get_directory_size(&entry.path()).unwrap_or(0);
                    }
                }
            }
        }

        Ok(size)
    }

    /// Get the modification time of a directory.
    fn get_dir_mtime(dir: &Path) -> Option<SystemTime> {
        std::fs::metadata(dir).ok().and_then(|m| m.modified().ok())
    }

    /// Clean up unowned partition directories that are stale.
    ///
    /// With single-directory-per-partition layout (`<base>/<topic>/<partition>/`),
    /// this finds partition directories that don't belong to any active store
    /// and whose directory mtime exceeds the staleness threshold.
    ///
    /// Safety checks:
    /// 1. Skip if stores map is empty (startup race)
    /// 2. Skip if rebalancing is in progress
    /// 3. Only delete directories older than orphan_min_staleness
    pub fn cleanup_orphaned_directories(&self, orphan_min_staleness: Duration) -> Result<u64> {
        if self.stores.is_empty() {
            debug!("Skipping orphan cleanup - no stores registered");
            return Ok(0);
        }

        if self.rebalance_tracker.is_rebalancing() {
            debug!("Skipping orphan cleanup - rebalance in progress");
            return Ok(0);
        }

        // Build set of active store paths
        let active_paths: HashSet<PathBuf> = self
            .stores
            .iter()
            .map(|entry| entry.value().get_db_path().clone())
            .collect();

        let mut total_freed = 0u64;

        // Scan <base>/<topic>/<partition>/ structure
        let Ok(topic_entries) = std::fs::read_dir(&self.store_config.path) else {
            return Ok(0);
        };

        for topic_entry in topic_entries.flatten() {
            if !topic_entry.metadata().map(|m| m.is_dir()).unwrap_or(false) {
                continue;
            }

            if self.rebalance_tracker.is_rebalancing() {
                info!(
                    "Aborting orphan cleanup - rebalance started. Freed {} bytes so far",
                    total_freed
                );
                return Ok(total_freed);
            }

            let topic_path = topic_entry.path();
            let Ok(partition_entries) = std::fs::read_dir(&topic_path) else {
                continue;
            };

            for partition_entry in partition_entries.flatten() {
                if !partition_entry
                    .metadata()
                    .map(|m| m.is_dir())
                    .unwrap_or(false)
                {
                    continue;
                }

                let partition_path = partition_entry.path();

                // Skip active stores
                if active_paths.contains(&partition_path) {
                    continue;
                }

                // Check staleness via directory mtime
                let is_stale = Self::get_dir_mtime(&partition_path)
                    .map(|mtime| mtime.elapsed().unwrap_or(Duration::ZERO) >= orphan_min_staleness)
                    .unwrap_or(true);

                if !is_stale {
                    debug!(
                        path = %partition_path.display(),
                        "Skipping orphan candidate - directory too recent"
                    );
                    continue;
                }

                let dir_size = Self::get_directory_size(&partition_path).unwrap_or(0);

                match std::fs::remove_dir_all(&partition_path) {
                    Ok(_) => {
                        info!(
                            path = %partition_path.display(),
                            size_mb = dir_size as f64 / (1024.0 * 1024.0),
                            "Removed orphaned partition directory"
                        );
                        metrics::counter!(
                            CLEANUP_OPERATIONS_COUNTER,
                            "partition_status" => "unassigned"
                        )
                        .increment(1);
                        total_freed += dir_size;
                    }
                    Err(e) => {
                        warn!(
                            "Failed to remove orphaned partition directory {}: {}",
                            partition_path.display(),
                            e
                        );
                    }
                }
            }

            // Clean up empty topic directories
            if std::fs::read_dir(&topic_path)
                .map(|mut rd| rd.next().is_none())
                .unwrap_or(false)
            {
                let _ = std::fs::remove_dir(&topic_path);
            }
        }

        if total_freed > 0 {
            info!(
                "Cleaned up {:.2} MB of orphaned partition directories",
                total_freed as f64 / (1024.0 * 1024.0)
            );
        }

        Ok(total_freed)
    }
}

/// Guard to ensure cleanup flag is released when dropped
struct CleanupGuard<'a> {
    flag: &'a AtomicBool,
}

impl<'a> Drop for CleanupGuard<'a> {
    fn drop(&mut self) {
        self.flag.store(false, Ordering::Release);
    }
}

/// Handle for the cleanup task that allows graceful shutdown
pub struct CleanupTaskHandle {
    handle: JoinHandle<()>,
    shutdown_tx: Option<tokio::sync::oneshot::Sender<()>>,
}

impl CleanupTaskHandle {
    /// Stop the cleanup task gracefully
    pub async fn stop(mut self) {
        if let Some(tx) = self.shutdown_tx.take() {
            let _ = tx.send(());
        }

        match tokio::time::timeout(Duration::from_secs(5), self.handle).await {
            Ok(Ok(())) => info!("Cleanup task shut down successfully"),
            Ok(Err(e)) => warn!("Cleanup task failed: {e:#}"),
            Err(_) => warn!("Cleanup task shutdown timed out"),
        }
    }
}

/// Bridges `OnCommit` to `StoreManager::update_metadata_for_partition`.
///
/// Constructed by the pipeline builder so `BatchConsumer` stays decoupled from
/// store internals. Each committed partition triggers a metadata update;
/// failures are non-fatal and logged inside `update_metadata_for_partition`.
pub struct MetadataCommitCallback {
    store_manager: Arc<StoreManager>,
    offset_tracker: Arc<OffsetTracker>,
}

impl MetadataCommitCallback {
    pub fn new(store_manager: Arc<StoreManager>, offset_tracker: Arc<OffsetTracker>) -> Self {
        Self {
            store_manager,
            offset_tracker,
        }
    }
}

#[async_trait::async_trait]
impl OnCommit for MetadataCommitCallback {
    async fn on_commit(&self, offsets: &HashMap<Partition, i64>) {
        for (partition, consumer_offset) in offsets {
            let producer_offset = self
                .offset_tracker
                .get_producer_offset(partition)
                .unwrap_or(0);
            self.store_manager
                .update_metadata_for_partition(
                    partition.topic(),
                    partition.partition_number(),
                    *consumer_offset,
                    producer_offset,
                )
                .await;
        }
    }
}

#[cfg(test)]
mod tests {
    use crate::rocksdb::store::RocksDbConfig;
    use crate::store::{TimestampKey, TimestampMetadata};
    use crate::test_utils::create_test_tracker;

    use super::*;
    use common_types::RawEvent;
    use std::sync::Arc;
    use tempfile::TempDir;

    #[tokio::test]
    async fn test_global_capacity_management() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 100, // Very small capacity to test the logic
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Test that needs_cleanup works correctly
        assert!(
            !manager.needs_cleanup(),
            "Should not need cleanup when empty"
        );

        // Create stores
        let store1 = manager.get_or_create("test-topic", 0).await.unwrap();
        let _store2 = manager.get_or_create("test-topic", 1).await.unwrap();

        // Add some events
        for i in 0..10 {
            let event = RawEvent {
                uuid: Some(uuid::Uuid::new_v4()),
                event: format!("test_event_{i}"),
                distinct_id: Some(serde_json::Value::String(format!("user_{i}"))),
                token: Some("test_token".to_string()),
                timestamp: Some("2021-01-01T00:00:00Z".to_string()),
                properties: std::collections::HashMap::new(),
                ..Default::default()
            };
            // Add test data directly to the store
            let key = TimestampKey::from(&event);
            let metadata = TimestampMetadata::new(&event);
            store1.put_timestamp_record(&key, &metadata).unwrap();
        }

        // Test that cleanup can be called without error
        let result = manager.cleanup_old_entries_if_needed();
        assert!(result.is_ok(), "Cleanup should not error");

        // Test with zero capacity
        let zero_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 0,
            rocksdb: RocksDbConfig::default(),
        };
        let zero_manager = Arc::new(StoreManager::new(zero_config, create_test_tracker()));
        assert!(
            !zero_manager.needs_cleanup(),
            "Should never need cleanup with zero capacity"
        );
    }

    #[tokio::test]
    async fn test_periodic_cleanup_task() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 5_000, // Small capacity
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Start periodic cleanup with short interval for testing
        let cleanup_handle = manager.clone().start_periodic_cleanup(
            Duration::from_millis(100), // Very short interval for testing
            Duration::from_secs(0),     // No staleness for testing
        );

        // Create a store and add data
        let store = manager.get_or_create("test-topic", 0).await.unwrap();

        // Add old events that should be cleaned up
        for i in 0..50 {
            let event = RawEvent {
                uuid: Some(uuid::Uuid::new_v4()),
                event: "test_event".to_string(),
                distinct_id: Some(serde_json::Value::String(format!("user_{i}"))),
                token: Some("test_token".to_string()),
                timestamp: Some("2021-01-01T00:00:00Z".to_string()), // Old timestamp
                properties: std::collections::HashMap::from([(
                    "data".to_string(),
                    serde_json::json!(format!("value_{}", i)),
                )]),
                ..Default::default()
            };
            // Add test data directly to the store
            let key = TimestampKey::from(&event);
            let metadata = TimestampMetadata::new(&event);
            store.put_timestamp_record(&key, &metadata).unwrap();
        }

        store.flush().unwrap();

        // Wait for cleanup task to run at least once
        tokio::time::sleep(Duration::from_millis(200)).await;

        // Stop the cleanup task
        cleanup_handle.stop().await;
    }

    #[tokio::test]
    async fn test_cleanup_task_graceful_shutdown() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1_000_000,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Start cleanup task
        let cleanup_handle = manager.clone().start_periodic_cleanup(
            Duration::from_secs(60), // Long interval
            Duration::from_secs(0),  // No staleness for testing
        );

        // Give it time to start
        tokio::time::sleep(Duration::from_millis(10)).await;

        // Stop should complete quickly even with long interval
        let start = std::time::Instant::now();
        cleanup_handle.stop().await;
        let elapsed = start.elapsed();

        // Should shutdown within the timeout (5 seconds)
        assert!(elapsed < Duration::from_secs(6));
    }

    #[tokio::test]
    async fn test_cleanup_with_zero_capacity() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 0, // Unlimited capacity
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Should never need cleanup with unlimited capacity
        assert!(!manager.needs_cleanup());

        // Cleanup should return 0 bytes freed
        let bytes_freed = manager.cleanup_old_entries_if_needed().unwrap();
        assert_eq!(bytes_freed, 0);
    }

    #[tokio::test]
    async fn test_get_or_create_store() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024, // 1GB
            rocksdb: RocksDbConfig::default(),
        };

        let manager = StoreManager::new(config, create_test_tracker());

        // First creation should succeed
        let store1 = manager.get_or_create("test-topic", 0).await.unwrap();

        // Second call should return the same store
        let store2 = manager.get_or_create("test-topic", 0).await.unwrap();

        // Stores should be the same instance - verify by checking they share state
        // Add an event to store1
        let event = RawEvent {
            event: "test_event".to_string(),
            distinct_id: Some(serde_json::Value::String("test_user".to_string())),
            token: Some("test_token".to_string()),
            timestamp: Some("2021-01-01T00:00:00Z".to_string()),
            ..Default::default()
        };

        // Add event through store1
        let key = TimestampKey::from(&event);
        let metadata = TimestampMetadata::new(&event);

        // Store1 shouldn't have it yet
        assert!(store1.get_timestamp_record(&key).unwrap().is_none());

        // Add to store1
        store1.put_timestamp_record(&key, &metadata).unwrap();

        // Now both store1 and store2 should have it (proving they're the same store instance)
        assert!(store1.get_timestamp_record(&key).unwrap().is_some());
        assert!(store2.get_timestamp_record(&key).unwrap().is_some());
    }

    #[tokio::test]
    async fn test_concurrent_store_creation() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Spawn multiple tasks trying to create the same store
        let mut handles = vec![];
        for _ in 0..10 {
            let manager_clone = manager.clone();
            let handle =
                tokio::spawn(
                    async move { manager_clone.get_or_create("concurrent-topic", 42).await },
                );
            handles.push(handle);
        }

        // All should succeed and return the same store
        let mut stores = vec![];
        for handle in handles {
            let store = handle.await.unwrap().unwrap();
            stores.push(store);
        }

        // All stores should be the same instance - verify by checking they share state
        // Add an event through the first store
        let event = RawEvent {
            event: "concurrent_test_event".to_string(),
            distinct_id: Some(serde_json::Value::String("concurrent_user".to_string())),
            token: Some("concurrent_token".to_string()),
            timestamp: Some("2021-01-01T00:00:00Z".to_string()),
            ..Default::default()
        };

        // Add event through first store
        let key = TimestampKey::from(&event);
        let metadata = TimestampMetadata::new(&event);

        // First store shouldn't have it yet
        assert!(stores[0].get_timestamp_record(&key).unwrap().is_none());

        // Add to first store
        stores[0].put_timestamp_record(&key, &metadata).unwrap();

        // All stores should now have it (proving they're all the same store instance)
        for store in &stores {
            assert!(store.get_timestamp_record(&key).unwrap().is_some());
        }
    }

    #[tokio::test]
    async fn test_get_or_create_for_rebalance() {
        // Test that get_or_create_for_rebalance works the same as get_or_create
        // but is intended for pre-creation during rebalancing
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = StoreManager::new(config, create_test_tracker());

        // Pre-create during rebalance
        let store1 = manager
            .get_or_create_for_rebalance("test-topic", 0)
            .await
            .unwrap();

        // Subsequent call (during message processing) should return the same store
        let store2 = manager.get_or_create("test-topic", 0).await.unwrap();

        // Verify they share state
        let event = RawEvent {
            event: "test_event".to_string(),
            distinct_id: Some(serde_json::Value::String("test_user".to_string())),
            token: Some("test_token".to_string()),
            timestamp: Some("2021-01-01T00:00:00Z".to_string()),
            ..Default::default()
        };

        let key = TimestampKey::from(&event);
        let metadata = TimestampMetadata::new(&event);

        store1.put_timestamp_record(&key, &metadata).unwrap();
        assert!(store2.get_timestamp_record(&key).unwrap().is_some());
    }

    #[tokio::test]
    async fn test_store_precreation_before_message_processing() {
        // Simulates the ideal flow: store is pre-created during rebalance,
        // then used during message processing without creating a new one
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = StoreManager::new(config, create_test_tracker());

        // Step 1: Pre-create during rebalance (async_setup_assigned_partitions)
        let _store = manager
            .get_or_create_for_rebalance("test-topic", 0)
            .await
            .unwrap();
        assert_eq!(manager.get_active_store_count(), 1);

        // Step 2: Message processing calls get_or_create
        // This should return the existing store (no new creation)
        let store = manager.get_or_create("test-topic", 0).await.unwrap();

        // Still only 1 store
        assert_eq!(manager.get_active_store_count(), 1);

        // Verify it's the same store by checking it works
        let event = RawEvent {
            event: "test_event".to_string(),
            distinct_id: Some(serde_json::Value::String("test_user".to_string())),
            token: Some("test_token".to_string()),
            timestamp: Some("2021-01-01T00:00:00Z".to_string()),
            ..Default::default()
        };

        let key = TimestampKey::from(&event);
        let metadata = TimestampMetadata::new(&event);
        store.put_timestamp_record(&key, &metadata).unwrap();
        assert!(store.get_timestamp_record(&key).unwrap().is_some());
    }

    #[tokio::test]
    async fn test_rapid_revoke_assign_store_recreation() {
        // Simulates rapid revoke -> assign where:
        // 1. Store exists
        // 2. Revoke removes it from map
        // 3. Assign pre-creates a new one
        // 4. Messages can use the new store
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = StoreManager::new(config, create_test_tracker());

        // Initial assignment - pre-create store
        let store1 = manager
            .get_or_create_for_rebalance("test-topic", 0)
            .await
            .unwrap();
        assert_eq!(manager.get_active_store_count(), 1);

        // Add some data
        let event = RawEvent {
            event: "test_event".to_string(),
            distinct_id: Some(serde_json::Value::String("test_user".to_string())),
            token: Some("test_token".to_string()),
            timestamp: Some("2021-01-01T00:00:00Z".to_string()),
            ..Default::default()
        };
        let key = TimestampKey::from(&event);
        let metadata = TimestampMetadata::new(&event);
        store1.put_timestamp_record(&key, &metadata).unwrap();

        // Revoke - unregister store (but don't delete files yet)
        manager.unregister_store("test-topic", 0);
        assert_eq!(manager.get_active_store_count(), 0);

        // Drop store1 to release the RocksDB lock before re-opening the same path.
        // With flat partition paths, store1 and store2 share the same directory, so
        // the lock must be released before the re-assign can open a new instance.
        drop(store1);

        // Rapid re-assign - pre-create new store for the same partition path
        let store2 = manager
            .get_or_create_for_rebalance("test-topic", 0)
            .await
            .unwrap();
        assert_eq!(manager.get_active_store_count(), 1);

        // The new store reuses the same partition directory and retains previously written data
        let event2 = RawEvent {
            event: "test_event_2".to_string(),
            distinct_id: Some(serde_json::Value::String("test_user_2".to_string())),
            token: Some("test_token".to_string()),
            timestamp: Some("2021-01-01T00:00:00Z".to_string()),
            ..Default::default()
        };
        let key2 = TimestampKey::from(&event2);
        let metadata2 = TimestampMetadata::new(&event2);
        store2.put_timestamp_record(&key2, &metadata2).unwrap();
        assert!(store2.get_timestamp_record(&key2).unwrap().is_some());

        // Message processing should use the new store
        let store3 = manager.get_or_create("test-topic", 0).await.unwrap();
        assert!(store3.get_timestamp_record(&key2).unwrap().is_some());
    }

    #[tokio::test]
    async fn test_message_processing_creates_store_if_precreation_missed() {
        // Simulates the case where messages arrive before pre-creation completes
        // The processor's get_or_create should still work (and emit a warning)
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = StoreManager::new(config, create_test_tracker());

        // No pre-creation - messages arrive first
        // This would emit a warning in production
        let store = manager.get_or_create("test-topic", 0).await.unwrap();
        assert_eq!(manager.get_active_store_count(), 1);

        // Store should work normally
        let event = RawEvent {
            event: "test_event".to_string(),
            distinct_id: Some(serde_json::Value::String("test_user".to_string())),
            token: Some("test_token".to_string()),
            timestamp: Some("2021-01-01T00:00:00Z".to_string()),
            ..Default::default()
        };

        let key = TimestampKey::from(&event);
        let metadata = TimestampMetadata::new(&event);
        store.put_timestamp_record(&key, &metadata).unwrap();
        assert!(store.get_timestamp_record(&key).unwrap().is_some());

        // Late pre-creation should just return the existing store
        let store2 = manager
            .get_or_create_for_rebalance("test-topic", 0)
            .await
            .unwrap();
        assert_eq!(manager.get_active_store_count(), 1);
        assert!(store2.get_timestamp_record(&key).unwrap().is_some());
    }

    #[tokio::test]
    async fn test_concurrent_rebalance_and_message_processing() {
        // Simulates concurrent pre-creation and message processing
        // Only one store should be created
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Spawn concurrent tasks: some for rebalance, some for message processing
        let mut handles = vec![];

        for i in 0..5 {
            let manager_clone = manager.clone();
            let handle = if i % 2 == 0 {
                // Rebalance pre-creation
                tokio::spawn(async move {
                    manager_clone
                        .get_or_create_for_rebalance("test-topic", 0)
                        .await
                })
            } else {
                // Message processing
                tokio::spawn(async move { manager_clone.get_or_create("test-topic", 0).await })
            };
            handles.push(handle);
        }

        // All should succeed
        let mut stores = vec![];
        for handle in handles {
            let store = handle.await.unwrap().unwrap();
            stores.push(store);
        }

        // Only one store should exist
        assert_eq!(manager.get_active_store_count(), 1);

        // All stores should be the same instance
        let event = RawEvent {
            event: "test_event".to_string(),
            distinct_id: Some(serde_json::Value::String("test_user".to_string())),
            token: Some("test_token".to_string()),
            timestamp: Some("2021-01-01T00:00:00Z".to_string()),
            ..Default::default()
        };

        let key = TimestampKey::from(&event);
        let metadata = TimestampMetadata::new(&event);

        stores[0].put_timestamp_record(&key, &metadata).unwrap();
        for store in &stores {
            assert!(store.get_timestamp_record(&key).unwrap().is_some());
        }
    }

    #[tokio::test]
    async fn test_cleanup_orphaned_directories_skips_during_rebalance() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Create a store for partition 0
        manager.get_or_create("test-topic", 0).await.unwrap();
        assert_eq!(manager.get_active_store_count(), 1);

        // Create an "orphaned" directory manually (not in stores map)
        // Must have timestamp subdir structure: topic_partition/timestamp/files
        let orphan_dir = temp_dir.path().join("other-topic_1");
        let timestamp_subdir = orphan_dir.join("1234567890");
        std::fs::create_dir_all(&timestamp_subdir).unwrap();
        std::fs::write(timestamp_subdir.join("dummy.txt"), b"test data").unwrap();
        assert!(timestamp_subdir.exists());

        // When NOT rebalancing, cleanup should remove the orphan timestamp dir
        assert!(!manager.rebalance_tracker().is_rebalancing());
        let freed = manager
            .cleanup_orphaned_directories(Duration::from_secs(0))
            .unwrap();
        assert!(freed > 0);
        assert!(
            !timestamp_subdir.exists(),
            "Orphan timestamp dir should be removed when not rebalancing"
        );

        // Recreate the orphan directory structure
        std::fs::create_dir_all(&timestamp_subdir).unwrap();
        std::fs::write(timestamp_subdir.join("dummy.txt"), b"test data").unwrap();
        assert!(timestamp_subdir.exists());

        // Start rebalancing via coordinator
        manager.rebalance_tracker().start_rebalancing();
        assert!(manager.rebalance_tracker().is_rebalancing());

        // During rebalance, cleanup should skip and not remove the orphan
        let freed = manager
            .cleanup_orphaned_directories(Duration::from_secs(0))
            .unwrap();
        assert_eq!(freed, 0);
        assert!(
            timestamp_subdir.exists(),
            "Orphan should NOT be removed during rebalance"
        );

        // Finish rebalancing via coordinator
        manager.rebalance_tracker().finish_rebalancing();
        assert!(!manager.rebalance_tracker().is_rebalancing());

        // Now cleanup should remove the orphan again
        let freed = manager
            .cleanup_orphaned_directories(Duration::from_secs(0))
            .unwrap();
        assert!(freed > 0);
        assert!(
            !timestamp_subdir.exists(),
            "Orphan timestamp dir should be removed after rebalance ends"
        );
    }

    #[tokio::test]
    async fn test_capacity_cleanup_skips_during_rebalance() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 100, // Very small to trigger cleanup
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Create a store and add data
        let store = manager.get_or_create("test-topic", 0).await.unwrap();
        for i in 0..10 {
            let event = RawEvent {
                uuid: Some(uuid::Uuid::new_v4()),
                event: format!("test_event_{i}"),
                distinct_id: Some(serde_json::Value::String(format!("user_{i}"))),
                token: Some("test_token".to_string()),
                timestamp: Some("2021-01-01T00:00:00Z".to_string()),
                properties: std::collections::HashMap::new(),
                ..Default::default()
            };
            let key = TimestampKey::from(&event);
            let metadata = TimestampMetadata::new(&event);
            store.put_timestamp_record(&key, &metadata).unwrap();
        }

        // Start rebalancing via coordinator
        manager.rebalance_tracker().start_rebalancing();
        assert!(manager.rebalance_tracker().is_rebalancing());

        // During rebalance, capacity cleanup should skip
        let freed = manager.cleanup_old_entries_if_needed().unwrap();
        assert_eq!(freed, 0, "Should skip cleanup during rebalance");

        // Finish rebalancing via coordinator
        manager.rebalance_tracker().finish_rebalancing();
        assert!(!manager.rebalance_tracker().is_rebalancing());

        // Now cleanup should run (may or may not free bytes depending on actual size)
        let result = manager.cleanup_old_entries_if_needed();
        assert!(result.is_ok(), "Cleanup should run after rebalance ends");
    }

    #[tokio::test]
    async fn test_orphan_cleanup_recent_wal_prevents_deletion() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Create a store for partition 0 (so stores map is not empty)
        manager.get_or_create("test-topic", 0).await.unwrap();

        // Create an "orphaned" directory with a recent WAL file
        let orphan_dir = temp_dir.path().join("other-topic_1");
        let timestamp_subdir = orphan_dir.join("1234567890");
        std::fs::create_dir_all(&timestamp_subdir).unwrap();
        std::fs::write(timestamp_subdir.join("000001.log"), b"wal data").unwrap();
        std::fs::write(timestamp_subdir.join("dummy.sst"), b"test data").unwrap();
        assert!(timestamp_subdir.exists());

        // Cleanup should NOT remove the timestamp dir because WAL file is too recent (staleness=15min)
        let freed = manager
            .cleanup_orphaned_directories(Duration::from_secs(900))
            .unwrap();
        assert_eq!(freed, 0, "Should not delete directory with recent WAL");
        assert!(
            timestamp_subdir.exists(),
            "Timestamp dir with recent WAL should NOT be removed"
        );
    }

    #[tokio::test]
    async fn test_orphan_cleanup_old_wal_allows_deletion() {
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Create a store for partition 0 (so stores map is not empty)
        manager.get_or_create("test-topic", 0).await.unwrap();

        // Create an "orphaned" directory with a WAL file (no LOCK)
        let orphan_dir = temp_dir.path().join("other-topic_1");
        let timestamp_subdir = orphan_dir.join("1234567890");
        std::fs::create_dir_all(&timestamp_subdir).unwrap();
        std::fs::write(timestamp_subdir.join("000001.log"), b"wal data").unwrap();
        std::fs::write(timestamp_subdir.join("dummy.sst"), b"test data").unwrap();
        assert!(timestamp_subdir.exists());

        // With zero staleness, cleanup should remove the timestamp dir
        let freed = manager
            .cleanup_orphaned_directories(Duration::from_secs(0))
            .unwrap();
        assert!(freed > 0, "Should delete directory with stale WAL");
        assert!(
            !timestamp_subdir.exists(),
            "Timestamp dir with stale WAL should be removed"
        );
    }

    #[tokio::test]
    async fn test_orphan_safety_checks_combined() {
        // Verify all safety checks work together
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Create a store for partition 0
        manager.get_or_create("test-topic", 0).await.unwrap();

        // Test 1: Directory in stores map should NOT be deleted
        // (the existing test-topic_0 directory)
        let active_dir = temp_dir.path().join("test-topic").join("0");
        assert!(active_dir.exists(), "Active store directory should exist");
        let freed = manager
            .cleanup_orphaned_directories(Duration::from_secs(0))
            .unwrap();
        assert_eq!(freed, 0, "No orphans should be found initially");
        assert!(
            active_dir.exists(),
            "Active store directory should NOT be deleted"
        );

        // Test 2: True orphan (no LOCK, no recent WAL, not in stores) should be deleted
        let orphan_dir = temp_dir.path().join("orphan-topic_99");
        let timestamp_subdir = orphan_dir.join("9999999999");
        std::fs::create_dir_all(&timestamp_subdir).unwrap();
        std::fs::write(timestamp_subdir.join("dummy.sst"), b"test data").unwrap();

        let freed = manager
            .cleanup_orphaned_directories(Duration::from_secs(0))
            .unwrap();
        assert!(freed > 0, "True orphan should be deleted");
        assert!(
            !timestamp_subdir.exists(),
            "Orphan timestamp directory should be removed"
        );
    }

    #[tokio::test]
    async fn test_get_store_returns_not_found_when_not_exists() {
        // Test that get_store() returns StoreError::NotFound when no store exists
        // This is the expected behavior during message processing for revoked partitions
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = StoreManager::new(config, create_test_tracker());

        // get_store() should return StoreError::NotFound when store doesn't exist
        let result = manager.get_store("test-topic", 0);
        match result {
            Err(StoreError::NotFound { topic, partition }) => {
                assert_eq!(topic, "test-topic");
                assert_eq!(partition, 0);
            }
            Ok(_) => panic!("Expected StoreError::NotFound, got Ok"),
            Err(e) => panic!("Expected StoreError::NotFound, got {:?}", e),
        }
    }

    #[tokio::test]
    async fn test_get_store_returns_store_when_exists() {
        // Test that get_store() returns the store when it exists
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = StoreManager::new(config, create_test_tracker());

        // Pre-create store (as would happen during rebalance)
        manager
            .get_or_create_for_rebalance("test-topic", 0)
            .await
            .unwrap();

        // get_store() should now return the store
        let result = manager.get_store("test-topic", 0);
        assert!(
            result.is_ok(),
            "get_store() should return Ok when store exists"
        );
    }

    #[tokio::test]
    async fn test_get_store_after_unregister_returns_not_found() {
        // Test that get_store() returns StoreError::NotFound after store is unregistered
        // This simulates the revocation scenario where buffered messages arrive after revoke
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = StoreManager::new(config, create_test_tracker());

        // Create store
        manager
            .get_or_create_for_rebalance("test-topic", 0)
            .await
            .unwrap();

        // get_store() should work
        assert!(manager.get_store("test-topic", 0).is_ok());

        // Unregister store (as would happen during revocation)
        manager.unregister_store("test-topic", 0);

        // get_store() should now return StoreError::NotFound
        let result = manager.get_store("test-topic", 0);
        match result {
            Err(StoreError::NotFound { topic, partition }) => {
                assert_eq!(topic, "test-topic");
                assert_eq!(partition, 0);
            }
            Ok(_) => panic!("Expected StoreError::NotFound after unregister, got Ok"),
            Err(e) => panic!(
                "Expected StoreError::NotFound after unregister, got {:?}",
                e
            ),
        }
    }

    #[tokio::test]
    async fn test_orphan_cleanup_never_deletes_active_store_path() {
        // Defense-in-depth test: even if collect_orphan_candidates somehow includes
        // the active store path, is_safe_to_delete_timestamp_dir should prevent deletion.
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Create a store for partition 0
        manager.get_or_create("test-topic", 0).await.unwrap();
        assert_eq!(manager.get_active_store_count(), 1);

        // Get the active store's path
        let active_store = manager.get("test-topic", 0).unwrap();
        let active_path = active_store.get_db_path().clone();

        // Verify active store exists
        assert!(active_path.exists(), "Active store path should exist");

        // Run orphan cleanup multiple times - should never touch active store
        for _ in 0..3 {
            let freed = manager
                .cleanup_orphaned_directories(Duration::from_secs(0))
                .unwrap();
            assert_eq!(
                freed, 0,
                "Should not free anything - only active store exists"
            );
            assert!(
                active_path.exists(),
                "Active store path should NEVER be deleted"
            );
        }

        assert_eq!(
            manager.get_active_store_count(),
            1,
            "Store should still be registered"
        );
    }

    #[tokio::test]
    async fn test_orphan_cleanup_mixed_assigned_and_unassigned_partitions() {
        // Test that orphan cleanup correctly handles a mix of:
        // - Assigned partitions (keep active partition dir intact)
        // - Unassigned partitions (delete entire partition dir)
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Create stores for partitions 0 and 1 (assigned)
        manager.get_or_create("test-topic", 0).await.unwrap();
        manager.get_or_create("test-topic", 1).await.unwrap();
        assert_eq!(manager.get_active_store_count(), 2);

        let active_store_0 = manager.get("test-topic", 0).unwrap();
        let active_path_0 = active_store_0.get_db_path().clone();
        let active_store_1 = manager.get("test-topic", 1).unwrap();
        let active_path_1 = active_store_1.get_db_path().clone();

        // Create an unassigned partition (partition 99) with some data files
        let unassigned_dir = temp_dir.path().join("test-topic").join("99");
        std::fs::create_dir_all(&unassigned_dir).unwrap();
        std::fs::write(unassigned_dir.join("dummy1.sst"), b"orphan1").unwrap();
        std::fs::write(unassigned_dir.join("dummy2.sst"), b"orphan2").unwrap();

        // Verify setup
        assert!(unassigned_dir.exists());

        // Run orphan cleanup
        let freed = manager
            .cleanup_orphaned_directories(Duration::from_secs(0))
            .unwrap();

        assert!(
            freed > 0,
            "Should have freed bytes from the unassigned partition"
        );

        // Verify: active stores for assigned partitions are untouched
        assert!(
            active_path_0.exists(),
            "Active store for partition 0 should exist"
        );
        assert!(
            active_path_1.exists(),
            "Active store for partition 1 should exist"
        );

        // Verify: the unassigned partition directory is fully cleaned
        assert!(
            !unassigned_dir.exists(),
            "Unassigned partition dir should be deleted"
        );

        // Stores still registered
        assert_eq!(manager.get_active_store_count(), 2);
    }

    #[tokio::test]
    async fn test_revoke_then_orphan_cleanup_flow() {
        // Test the full flow: assign → create store → revoke → orphan cleanup cleans files
        // This verifies that revoked partition files ARE eventually cleaned by the orphan cleaner.
        let temp_dir = TempDir::new().unwrap();
        let config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1024 * 1024 * 1024,
            rocksdb: RocksDbConfig::default(),
        };

        let manager = Arc::new(StoreManager::new(config, create_test_tracker()));

        // Create stores for partitions 0 and 1
        // We need at least one store to remain so the cleanup doesn't skip due to empty stores
        manager.get_or_create("test-topic", 0).await.unwrap();
        manager.get_or_create("test-topic", 1).await.unwrap();
        assert_eq!(manager.get_active_store_count(), 2);

        let store_0 = manager.get("test-topic", 0).unwrap();
        let store_path_0 = store_0.get_db_path().clone();
        let partition_dir_0 = temp_dir.path().join("test-topic").join("0");

        // Verify store 0 exists on disk
        assert!(store_path_0.exists(), "Store 0 path should exist");
        assert!(partition_dir_0.exists(), "Partition 0 dir should exist");

        // Drop the store reference before unregistering
        drop(store_0);

        // Simulate revoke of partition 0: unregister store (but don't delete files)
        manager.unregister_store("test-topic", 0);
        assert_eq!(
            manager.get_active_store_count(),
            1,
            "Only partition 1 store should remain"
        );

        // Files for partition 0 should still exist after unregister
        assert!(
            partition_dir_0.exists(),
            "Partition 0 dir should still exist after unregister"
        );
        assert!(
            store_path_0.exists(),
            "Store 0 path should still exist after unregister"
        );

        // Now run orphan cleanup - should clean partition 0 files since it's no longer assigned
        // (Partition 1 is still assigned so cleanup won't skip due to empty stores)
        let freed = manager
            .cleanup_orphaned_directories(Duration::from_secs(0))
            .unwrap();

        assert!(freed > 0, "Should have freed bytes from orphaned files");
        assert!(
            !store_path_0.exists(),
            "Store 0 path should be cleaned by orphan cleaner"
        );

        // Partition 1 store should still exist
        let store_1 = manager.get("test-topic", 1).unwrap();
        assert!(
            store_1.get_db_path().exists(),
            "Store 1 should still exist (still assigned)"
        );
    }

    mod try_restore_local_store_tests {
        use super::*;
        use crate::checkpoint::config::DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS;
        use crate::checkpoint::metadata::CheckpointMetadata;
        use crate::test_utils::create_test_store_manager;
        use crate::utils::format_store_path;
        use chrono::Utc;

        #[tokio::test]
        async fn test_returns_metadata_when_fresh() {
            let temp_dir = TempDir::new().unwrap();
            let manager = create_test_store_manager(temp_dir.path().to_path_buf());
            let store_path = format_store_path(temp_dir.path(), "test-topic", 0);
            tokio::fs::create_dir_all(&store_path).await.unwrap();

            let mut metadata =
                CheckpointMetadata::new("test-topic".to_string(), 0, Utc::now(), 1, 100, 50);
            metadata.write_to_dir(&store_path).await.unwrap();

            let result = manager
                .try_restore_local_store(
                    "test-topic",
                    0,
                    Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
                )
                .await;

            assert!(result.is_some(), "Should return metadata for fresh store");
            let loaded = result.unwrap();
            assert_eq!(loaded.consumer_offset, 100);
            assert_eq!(loaded.producer_offset, 50);
        }

        #[tokio::test]
        async fn test_returns_none_when_missing() {
            let temp_dir = TempDir::new().unwrap();
            let manager = create_test_store_manager(temp_dir.path().to_path_buf());

            // No metadata.json written — directory doesn't even exist
            let result = manager
                .try_restore_local_store(
                    "test-topic",
                    0,
                    Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
                )
                .await;

            assert!(
                result.is_none(),
                "Should return None when metadata is missing"
            );
        }

        #[tokio::test]
        async fn test_returns_none_when_stale() {
            let temp_dir = TempDir::new().unwrap();
            let manager = create_test_store_manager(temp_dir.path().to_path_buf());
            let store_path = format_store_path(temp_dir.path(), "test-topic", 0);
            tokio::fs::create_dir_all(&store_path).await.unwrap();

            // Write metadata with an old updated_at by setting attempt_timestamp far in the past
            // then manually writing JSON with an old updated_at
            let old_timestamp = Utc::now() - chrono::Duration::hours(3);
            let json = serde_json::json!({
                "id": CheckpointMetadata::generate_id(old_timestamp),
                "topic": "test-topic",
                "partition": 0,
                "attempt_timestamp": old_timestamp,
                "sequence": 1,
                "consumer_offset": 50,
                "producer_offset": 25,
                "updated_at": old_timestamp,
                "files": []
            });
            let path = store_path.join("metadata.json");
            tokio::fs::write(&path, serde_json::to_string_pretty(&json).unwrap())
                .await
                .unwrap();

            // 2h max staleness — 3h old timestamp should be stale
            let result = manager
                .try_restore_local_store(
                    "test-topic",
                    0,
                    Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
                )
                .await;

            assert!(result.is_none(), "Should return None for stale metadata");
        }

        #[tokio::test]
        async fn test_returns_none_when_corrupt() {
            let temp_dir = TempDir::new().unwrap();
            let manager = create_test_store_manager(temp_dir.path().to_path_buf());
            let store_path = format_store_path(temp_dir.path(), "test-topic", 0);
            tokio::fs::create_dir_all(&store_path).await.unwrap();

            // Write invalid JSON
            let path = store_path.join("metadata.json");
            tokio::fs::write(&path, b"not valid json at all!!!")
                .await
                .unwrap();

            let result = manager
                .try_restore_local_store(
                    "test-topic",
                    0,
                    Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
                )
                .await;

            assert!(result.is_none(), "Should return None for corrupt metadata");
        }
    }
}