Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

83.1 KB
Raw Blame
use std::sync::Arc;
use std::time::Duration;

use anyhow::Result;
use async_trait::async_trait;
use dashmap::mapref::entry::Entry;
use dashmap::DashMap;
use futures::future::{join_all, Shared};
use rdkafka::{Offset, TopicPartitionList};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};

use crate::checkpoint::import::CheckpointImporter;
use crate::checkpoint::metadata::CheckpointMetadata;
use crate::kafka::batch_consumer::BatchConsumerProcessor;
use crate::kafka::batch_context::{ConsumerCommand, ConsumerCommandSender};
use crate::kafka::offset_tracker::OffsetTracker;
use crate::kafka::partition_router::{shutdown_workers, PartitionRouter};
use crate::kafka::rebalance_handler::RebalanceHandler;
use crate::kafka::types::Partition;
use crate::metrics_const::{
    CHECKPOINT_IMPORT_CANCELLED_CLEANUP_COUNTER, LOCAL_STORE_RESTORE_COUNTER,
    PARTITION_STORE_FALLBACK_EMPTY, PARTITION_STORE_SETUP_SKIPPED,
    REBALANCE_CHECKPOINT_IMPORT_COUNTER, REBALANCE_PARTITION_STATE_CHANGE,
    REBALANCE_RESUME_SKIPPED_NO_OWNED,
};
use crate::rebalance_tracker::RebalanceTracker;
use crate::store_manager::StoreManager;
use crate::utils::async_helpers::unwrap_blocking_task;
use crate::utils::format_store_path;

/// Type alias for the shared task handle. The closure maps JoinHandle's Result to ()
/// so it can be Clone (required by Shared).
type SharedTaskHandle =
    Shared<futures::future::Map<JoinHandle<()>, fn(Result<(), tokio::task::JoinError>) -> ()>>;

/// Tracks a partition setup task with its cancellation token.
///
/// The `Shared` future allows multiple async_setup calls to await the same task.
/// The `CancellationToken` allows cancelling ONLY this partition's download on revoke.
struct PartitionSetupTask {
    /// None = claimed but task not yet spawned, Some = task spawned and awaitable
    handle: Option<SharedTaskHandle>,
    cancel_token: CancellationToken,
}

/// Coordinates store cleanup and partition workers across rebalances.
///
/// Ownership lives in RebalanceTracker. This handler updates it and drives resume vs cleanup.
/// Setup tasks are tracked here (not in RebalanceTracker): only this handler spawns/awaits them
/// and needs StoreManager to detect stale entries.
pub struct ProcessorRebalanceHandler<T, P>
where
    T: Send + 'static,
    P: BatchConsumerProcessor<T> + 'static,
{
    store_manager: Arc<StoreManager>,
    rebalance_tracker: Arc<RebalanceTracker>,
    router: Option<Arc<PartitionRouter<T, P>>>,
    offset_tracker: Arc<OffsetTracker>,
    checkpoint_importer: Option<Arc<CheckpointImporter>>,
    /// Max parallel directory deletions during rebalance cleanup
    rebalance_cleanup_parallelism: usize,
    /// Maximum age of local metadata.json before falling back to S3 import
    local_checkpoint_max_staleness: Duration,
    /// In-flight checkpoint import tasks (per partition). Cancel on revoke; await before resume; stale entries allow re-claim.
    partition_setup_tasks: DashMap<Partition, PartitionSetupTask>,
    /// Reason a partition got a fallback empty store (no_importer | import_failed | import_cancelled) for metrics.
    partition_fallback_reasons: Arc<DashMap<Partition, &'static str>>,
    /// Metadata from successfully restored partitions (local or S3). Used in finalize to seed
    /// OffsetTracker and seek consumer without re-reading metadata.json from disk.
    partition_restored_metadata: Arc<DashMap<Partition, CheckpointMetadata>>,
}

impl<T, P> ProcessorRebalanceHandler<T, P>
where
    T: Send + 'static,
    P: BatchConsumerProcessor<T> + 'static,
{
    pub fn new(
        store_manager: Arc<StoreManager>,
        rebalance_tracker: Arc<RebalanceTracker>,
        offset_tracker: Arc<OffsetTracker>,
        checkpoint_importer: Option<Arc<CheckpointImporter>>,
        rebalance_cleanup_parallelism: usize,
        local_checkpoint_max_staleness: Duration,
    ) -> Self {
        Self {
            store_manager,
            rebalance_tracker,
            router: None,
            offset_tracker,
            checkpoint_importer,
            rebalance_cleanup_parallelism,
            local_checkpoint_max_staleness,
            partition_setup_tasks: DashMap::new(),
            partition_fallback_reasons: Arc::new(DashMap::new()),
            partition_restored_metadata: Arc::new(DashMap::new()),
        }
    }

    pub fn with_router(
        store_manager: Arc<StoreManager>,
        rebalance_tracker: Arc<RebalanceTracker>,
        router: Arc<PartitionRouter<T, P>>,
        offset_tracker: Arc<OffsetTracker>,
        checkpoint_importer: Option<Arc<CheckpointImporter>>,
        rebalance_cleanup_parallelism: usize,
        local_checkpoint_max_staleness: Duration,
    ) -> Self {
        Self {
            store_manager,
            rebalance_tracker,
            router: Some(router),
            offset_tracker,
            checkpoint_importer,
            rebalance_cleanup_parallelism,
            local_checkpoint_max_staleness,
            partition_setup_tasks: DashMap::new(),
            partition_fallback_reasons: Arc::new(DashMap::new()),
            partition_restored_metadata: Arc::new(DashMap::new()),
        }
    }

    // ============================================
    // PARTITION SETUP TASK TRACKING
    // ============================================

    /// Claim a partition for setup (fresh or by replacing a stale entry).
    /// Returns false if a store already exists or a task is still running.
    /// Stale = task finished but no store (revoke/fail/cancel); we replace it to retry import.
    fn try_claim_partition_setup(
        &self,
        partition: &Partition,
        cancel_token: CancellationToken,
    ) -> bool {
        match self.partition_setup_tasks.entry(partition.clone()) {
            Entry::Vacant(e) => {
                // Fresh claim - no existing entry
                e.insert(PartitionSetupTask {
                    handle: None,
                    cancel_token,
                });
                true
            }
            Entry::Occupied(mut e) => {
                let task = e.get();
                let store_exists = self
                    .store_manager
                    .get(partition.topic(), partition.partition_number())
                    .is_some();

                // peek() returns Some if resolved, None if still pending
                let task_completed = task
                    .handle
                    .as_ref()
                    .map(|h| h.peek().is_some())
                    .unwrap_or(false); // None = not spawned yet, treat as running

                if store_exists {
                    // Task succeeded, keep entry
                    false
                } else if task_completed {
                    // Task completed but no store → stale entry (task bailed), safe to replace
                    e.get().cancel_token.cancel();
                    e.insert(PartitionSetupTask {
                        handle: None,
                        cancel_token,
                    });
                    debug!(
                        topic = partition.topic(),
                        partition = partition.partition_number(),
                        "Replaced stale setup task entry (task completed, no store)"
                    );
                    true
                } else {
                    // Task still running, let it finish
                    debug!(
                        topic = partition.topic(),
                        partition = partition.partition_number(),
                        "Setup task already in progress, skipping"
                    );
                    false
                }
            }
        }
    }

    /// Attach task handle after spawn (call only when try_claim_partition_setup returned true). Shared so multiple callers can await.
    fn finalize_partition_setup(&self, partition: &Partition, handle: JoinHandle<()>) {
        use futures::future::FutureExt;

        // Helper function to discard JoinHandle result (must be fn, not closure, for type matching)
        fn discard_result(_: Result<(), tokio::task::JoinError>) {}

        if let Some(mut task) = self.partition_setup_tasks.get_mut(partition) {
            // Map the JoinHandle result to () so it's Clone (required for Shared)
            let shared_handle = handle.map(discard_result as fn(_) -> ()).shared();
            task.handle = Some(shared_handle);
        }
    }

    /// Setup task handle for a partition (None if not yet finalized or missing). Clone is awaitable by multiple callers.
    fn get_setup_task(&self, partition: &Partition) -> Option<SharedTaskHandle> {
        self.partition_setup_tasks
            .get(partition)
            .and_then(|t| t.handle.clone())
    }

    /// Cancel and remove setup tasks for revoked partitions (stops in-flight S3 downloads).
    fn cancel_setup_tasks(&self, partitions: &[Partition]) {
        for partition in partitions {
            if let Some((_, task)) = self.partition_setup_tasks.remove(partition) {
                // Cancel the token - S3 download will stop at next chunk
                task.cancel_token.cancel();
                // Handle is dropped, task continues but we won't wait for it
            }
            self.partition_fallback_reasons.remove(partition);
            self.partition_restored_metadata.remove(partition);
        }
    }

    /// Remove setup task entry after awaiting (cleanup).
    fn complete_setup_task(&self, partition: &Partition) {
        self.partition_setup_tasks.remove(partition);
    }

    // ============================================
    // PARTITION SETUP TASK SPAWNING
    // ============================================

    /// Spawn checkpoint import for one partition. Per-partition cancel token stops S3 on revoke.
    /// Fallback empty store is created in finalize_rebalance_cycle. Partial/orphan files: importer
    /// deletes dir at start; orphan cleaner handles cancelled downloads.
    fn spawn_partition_setup_task(
        &self,
        partition: Partition,
        cancel_token: CancellationToken,
    ) -> tokio::task::JoinHandle<()> {
        let store_manager = self.store_manager.clone();
        let coordinator = self.rebalance_tracker.clone();
        let importer = self.checkpoint_importer.clone();
        let fallback_reasons = Arc::clone(&self.partition_fallback_reasons);
        let restored_metadata = Arc::clone(&self.partition_restored_metadata);
        let local_max_staleness = self.local_checkpoint_max_staleness;

        tokio::spawn(async move {
            if should_skip_setup(
                &partition,
                &cancel_token,
                &coordinator,
                &store_manager,
                &fallback_reasons,
            ) {
                return;
            }

            if try_restore_from_local(
                &partition,
                &store_manager,
                &restored_metadata,
                local_max_staleness,
            )
            .await
            {
                return;
            }

            try_import_from_s3(
                &partition,
                &cancel_token,
                &coordinator,
                &store_manager,
                &importer,
                &fallback_reasons,
                &restored_metadata,
            )
            .await;
        })
    }

    /// End of rebalance cycle (counter 0 or we're last): await all import tasks; if no new rebalance,
    /// create fallback stores for owned partitions without stores, delete unowned dirs, resume consumption.
    /// When `we_are_finalizing_last`, we were called before decrement (count still 1); proceed only if count == 1
    /// so is_rebalancing() stays true during finalize and orphan/capacity cleanup skips.
    async fn finalize_rebalance_cycle(
        &self,
        consumer_command_tx: &ConsumerCommandSender,
        we_are_finalizing_last: bool,
    ) -> Result<()> {
        info!("Finalizing rebalance cycle - awaiting import tasks");

        // Step 1: Await all pending import tasks in parallel
        let owned_partitions = self.rebalance_tracker.get_owned_partitions();
        let task_futures: Vec<(Partition, SharedTaskHandle)> = owned_partitions
            .iter()
            .filter_map(|p| {
                let p = p.clone();
                self.get_setup_task(&p).map(|h| (p, h))
            })
            .collect();
        let (partitions, handles): (Vec<_>, Vec<_>) = task_futures.into_iter().unzip();
        let _ = join_all(handles).await; // Ignore panic results - tasks handle their own errors
        for partition in &partitions {
            self.complete_setup_task(partition);
        }

        // Snapshot owned before count check to avoid TOCTOU if a new rebalance starts; steps 3–5 use this snapshot.
        let owned = self.rebalance_tracker.get_owned_partitions();

        // Step 2: If new rebalance started while awaiting, skip. (we_are_finalizing_last: proceed iff count == 1; else iff count == 0.)
        let count = self.rebalance_tracker.rebalancing_count();
        let should_skip = if we_are_finalizing_last {
            count != 1
        } else {
            count != 0
        };
        if should_skip {
            info!("New rebalance started during finalize - skipping fallback stores, cleanup and resume");
            return Ok(());
        }

        // Step 3: Create fallback stores for any owned partitions that don't have a registered store
        // This handles cases where checkpoint import failed, was cancelled, or was disabled
        for partition in &owned {
            if self
                .store_manager
                .get(partition.topic(), partition.partition_number())
                .is_none()
            {
                let reason = self
                    .partition_fallback_reasons
                    .remove(partition)
                    .map(|(_, r)| r)
                    .unwrap_or("unknown");
                match self
                    .store_manager
                    .get_or_create_for_rebalance(partition.topic(), partition.partition_number())
                    .await
                {
                    Ok(_) => {
                        metrics::counter!(PARTITION_STORE_FALLBACK_EMPTY, "reason" => reason, "assignment_mode" => "consumer_group")
                            .increment(1);
                        warn!(
                            topic = partition.topic(),
                            partition = partition.partition_number(),
                            reason = reason,
                            "Created fallback empty store - deduplication quality may be degraded"
                        );
                    }
                    Err(e) => {
                        error!(
                            topic = partition.topic(),
                            partition = partition.partition_number(),
                            error = ?e,
                            "Failed to create fallback store - processor will retry on first message"
                        );
                    }
                }
            }
        }

        // Step 4: Delete unowned partition dirs (scan disk; delete anything not in owned). Catches orphans.
        if let Err(e) = self
            .store_manager
            .cleanup_unowned_partition_directories(&owned, self.rebalance_cleanup_parallelism)
            .await
        {
            warn!(
                error = ?e,
                "Partition directory cleanup failed - orphan cleaner will handle it"
            );
        }

        // Step 5: Seek consumer to restored offsets and seed OffsetTracker.
        //
        // For each owned partition, use metadata cached during partition setup (local or S3
        // restore). Falls back to reading metadata.json from disk if the cache entry is missing.
        // If consumer_offset > 0, the store was restored from a prior checkpoint and we seek
        // the consumer to that position so it resumes exactly where processing left off.
        // We also seed OffsetTracker so the first checkpoint export reflects the correct
        // committed and producer offsets rather than starting from zero.
        //
        // Partitions with no valid metadata (empty fallback stores) are naturally skipped
        // because their consumer_offset is 0 or metadata doesn't exist.
        let mut seek_tpl = TopicPartitionList::new();
        for p in &owned {
            // Use cached metadata from partition setup, fall back to disk read
            let metadata = match self.partition_restored_metadata.remove(p) {
                Some((_, m)) => Some(m),
                None => {
                    let store_path = format_store_path(
                        self.store_manager.base_path(),
                        p.topic(),
                        p.partition_number(),
                    );
                    CheckpointMetadata::load_from_dir(&store_path).await.ok()
                }
            };

            if let Some(metadata) = metadata {
                if metadata.consumer_offset > 0 {
                    self.offset_tracker.init_partition_from_metadata(
                        p,
                        metadata.consumer_offset,
                        metadata.producer_offset,
                    );
                    seek_tpl
                        .add_partition_offset(
                            p.topic(),
                            p.partition_number(),
                            Offset::Offset(metadata.consumer_offset),
                        )
                        .unwrap_or_else(|e| {
                            warn!(
                                topic = p.topic(),
                                partition = p.partition_number(),
                                error = ?e,
                                "Failed to add partition to seek list — will resume from broker offset"
                            );
                        });
                    debug!(
                        topic = p.topic(),
                        partition = p.partition_number(),
                        consumer_offset = metadata.consumer_offset,
                        producer_offset = metadata.producer_offset,
                        "Seeding OffsetTracker and seeking consumer from metadata"
                    );
                }
            }
        }
        if seek_tpl.count() > 0 {
            info!(
                seek_count = seek_tpl.count(),
                "Seeking consumer to restored offsets before resume"
            );
            if let Err(e) = consumer_command_tx.send(ConsumerCommand::SeekPartitions(seek_tpl)) {
                warn!("Failed to send seek command: {e:#} — will resume from broker offsets");
            }
        }

        // Step 6: Resume consumption
        if owned.is_empty() {
            info!("No owned partitions to resume");
            metrics::counter!(REBALANCE_RESUME_SKIPPED_NO_OWNED).increment(1);
        } else {
            let mut resume_tpl = TopicPartitionList::new();
            for p in &owned {
                resume_tpl.add_partition(p.topic(), p.partition_number());
            }
            info!(
                owned_count = owned.len(),
                "Resuming all owned partitions (rebalance cycle complete)"
            );
            if let Err(e) = consumer_command_tx.send(ConsumerCommand::Resume(resume_tpl)) {
                error!("Failed to send resume command after store setup: {e:#}");
                return Err(anyhow::Error::from(e).context("Failed to send resume command"));
            }
        }

        Ok(())
    }
}

// ============================================
// PARTITION SETUP HELPERS
// ============================================

/// Check if partition setup should be skipped.
///
/// Returns `true` (skip) when the partition was cancelled, is no longer owned,
/// or already has a registered store (rapid revoke→assign race).
fn should_skip_setup(
    partition: &Partition,
    cancel_token: &CancellationToken,
    coordinator: &Arc<RebalanceTracker>,
    store_manager: &Arc<StoreManager>,
    fallback_reasons: &Arc<DashMap<Partition, &'static str>>,
) -> bool {
    if cancel_token.is_cancelled() || !coordinator.is_partition_owned(partition) {
        metrics::counter!(
            PARTITION_STORE_SETUP_SKIPPED,
            "reason" => if cancel_token.is_cancelled() { "cancelled" } else { "not_owned" },
            "assignment_mode" => "consumer_group",
        )
        .increment(1);
        fallback_reasons.insert(partition.clone(), "import_cancelled");
        return true;
    }

    if store_manager
        .get(partition.topic(), partition.partition_number())
        .is_some()
    {
        metrics::counter!(
            REBALANCE_CHECKPOINT_IMPORT_COUNTER,
            "result" => "skipped",
            "reason" => "store_exists",
            "assignment_mode" => "consumer_group",
        )
        .increment(1);
        return true;
    }

    false
}

/// Attempt to restore from local RocksDB data.
///
/// Returns `true` if a fresh local store was restored (caller should skip S3 import).
/// Returns `false` if local data is stale, missing, or corrupt (fall through to S3).
async fn try_restore_from_local(
    partition: &Partition,
    store_manager: &Arc<StoreManager>,
    restored_metadata: &Arc<DashMap<Partition, CheckpointMetadata>>,
    local_max_staleness: Duration,
) -> bool {
    // metadata.json is updated on every Kafka offset commit, so its
    // updated_at reflects the last time this pod was actively consuming.
    let local_metadata = store_manager
        .try_restore_local_store(
            partition.topic(),
            partition.partition_number(),
            local_max_staleness,
        )
        .await;

    if let Some(metadata) = local_metadata {
        // Fresh local data: open RocksDB from the existing partition directory.
        // Use spawn_blocking because RocksDB::open is a blocking operation.
        let topic = partition.topic().to_string();
        let partition_number = partition.partition_number();
        let store_path = format_store_path(store_manager.base_path(), &topic, partition_number);
        match unwrap_blocking_task(
            tokio::task::spawn_blocking({
                let store_manager = Arc::clone(store_manager);
                move || store_manager.restore_imported_store(&topic, partition_number, &store_path)
            }),
            "restore_local_store task panicked",
        )
        .await
        {
            Ok(_) => {
                metrics::counter!(
                    LOCAL_STORE_RESTORE_COUNTER,
                    "result" => "fresh",
                )
                .increment(1);
                info!(
                    topic = partition.topic(),
                    partition = partition.partition_number(),
                    consumer_offset = metadata.consumer_offset,
                    producer_offset = metadata.producer_offset,
                    "Restored local store — skipping S3 import"
                );
                restored_metadata.insert(partition.clone(), metadata);
                return true;
            }
            Err(e) => {
                warn!(
                    topic = partition.topic(),
                    partition = partition.partition_number(),
                    error = %e,
                    "Failed to open local store, falling back to S3 import"
                );
            }
        }
    } else {
        // Stale, missing, or corrupt — record which and fall through to S3 import.
        let store_path = format_store_path(
            store_manager.base_path(),
            partition.topic(),
            partition.partition_number(),
        );
        let metadata_path = store_path.join(crate::checkpoint::metadata::METADATA_FILENAME);
        let result_label = if !metadata_path.exists() {
            "missing"
        } else if CheckpointMetadata::load_from_dir(&store_path)
            .await
            .is_err()
        {
            "corrupt"
        } else {
            "stale"
        };
        metrics::counter!(
            LOCAL_STORE_RESTORE_COUNTER,
            "result" => result_label,
        )
        .increment(1);
        debug!(
            topic = partition.topic(),
            partition = partition.partition_number(),
            result = result_label,
            "Local store not usable, falling back to S3 import"
        );
    }

    false
}

/// Attempt to import a checkpoint from S3.
///
/// Uses the per-partition cancellation token to stop S3 downloads on revoke.
/// Records fallback reasons on failure so finalize_rebalance_cycle can create empty stores.
async fn try_import_from_s3(
    partition: &Partition,
    cancel_token: &CancellationToken,
    coordinator: &Arc<RebalanceTracker>,
    store_manager: &Arc<StoreManager>,
    importer: &Option<Arc<CheckpointImporter>>,
    fallback_reasons: &Arc<DashMap<Partition, &'static str>>,
    restored_metadata: &Arc<DashMap<Partition, CheckpointMetadata>>,
) {
    let Some(importer) = importer else {
        metrics::counter!(
            REBALANCE_CHECKPOINT_IMPORT_COUNTER,
            "result" => "skipped",
            "reason" => "disabled",
            "assignment_mode" => "consumer_group",
        )
        .increment(1);
        fallback_reasons.insert(partition.clone(), "no_importer");
        return;
    };

    match importer
        .import_checkpoint_for_topic_partition_cancellable(
            partition.topic(),
            partition.partition_number(),
            Some(cancel_token),
        )
        .await
    {
        Ok(path) => {
            // Check if we should skip registration (token cancelled or ownership lost)
            let is_cancelled = cancel_token.is_cancelled();
            let is_owned = coordinator.is_partition_owned(partition);

            if is_cancelled || !is_owned {
                let reason = if is_cancelled {
                    "cancelled"
                } else {
                    "not_owned"
                };
                metrics::counter!(
                    PARTITION_STORE_SETUP_SKIPPED,
                    "reason" => reason,
                    "assignment_mode" => "consumer_group",
                )
                .increment(1);
                fallback_reasons.insert(partition.clone(), "import_cancelled");

                // Clean up the successfully imported directory since we can't use it.
                if path.exists() {
                    match tokio::fs::remove_dir_all(&path).await {
                        Ok(_) => {
                            metrics::counter!(
                                CHECKPOINT_IMPORT_CANCELLED_CLEANUP_COUNTER,
                                "result" => "success",
                                "assignment_mode" => "consumer_group",
                            )
                            .increment(1);
                            info!(
                                topic = partition.topic(),
                                partition = partition.partition_number(),
                                path = %path.display(),
                                reason = reason,
                                "Cleaned up unused checkpoint import"
                            );
                        }
                        Err(e) => {
                            metrics::counter!(
                                CHECKPOINT_IMPORT_CANCELLED_CLEANUP_COUNTER,
                                "result" => "failed",
                                "assignment_mode" => "consumer_group",
                            )
                            .increment(1);
                            warn!(
                                topic = partition.topic(),
                                partition = partition.partition_number(),
                                path = %path.display(),
                                error = ?e,
                                "Failed to clean up checkpoint import, orphan cleaner will handle it"
                            );
                        }
                    }
                }
                return;
            }

            // Register imported store (sync RocksDB open; run on blocking pool)
            let topic = partition.topic().to_string();
            let partition_number = partition.partition_number();
            let import_path = path.clone();
            match unwrap_blocking_task(
                tokio::task::spawn_blocking({
                    let store_manager = Arc::clone(store_manager);
                    move || {
                        store_manager.restore_imported_store(&topic, partition_number, &import_path)
                    }
                }),
                "restore_imported_store task panicked",
            )
            .await
            {
                Ok(_) => {
                    metrics::counter!(
                        REBALANCE_CHECKPOINT_IMPORT_COUNTER,
                        "result" => "success",
                        "assignment_mode" => "consumer_group",
                    )
                    .increment(1);
                    info!(
                        topic = partition.topic(),
                        partition = partition.partition_number(),
                        path = %path.display(),
                        "Imported checkpoint for partition"
                    );
                    // Cache metadata so finalize can seed OffsetTracker without re-reading disk
                    match CheckpointMetadata::load_from_dir(&path).await {
                        Ok(metadata) => {
                            restored_metadata.insert(partition.clone(), metadata);
                        }
                        Err(e) => {
                            warn!(
                                topic = partition.topic(),
                                partition = partition.partition_number(),
                                error = ?e,
                                "Failed to load metadata after import — finalize will re-read from disk"
                            );
                        }
                    }
                }
                Err(e) => {
                    metrics::counter!(
                        REBALANCE_CHECKPOINT_IMPORT_COUNTER,
                        "result" => "failed",
                        "reason" => "restore",
                        "assignment_mode" => "consumer_group",
                    )
                    .increment(1);
                    error!(
                        topic = partition.topic(),
                        partition = partition.partition_number(),
                        error = %e,
                        "Failed to restore checkpoint"
                    );
                    fallback_reasons.insert(partition.clone(), "import_failed");
                }
            }
        }
        Err(e) => {
            // Only log if not cancelled (expected during revoke)
            if !cancel_token.is_cancelled() {
                metrics::counter!(
                    REBALANCE_CHECKPOINT_IMPORT_COUNTER,
                    "result" => "failed",
                    "reason" => "import",
                    "assignment_mode" => "consumer_group",
                )
                .increment(1);
                warn!(
                    topic = partition.topic(),
                    partition = partition.partition_number(),
                    error = ?e,
                    "Failed to import checkpoint"
                );
                fallback_reasons.insert(partition.clone(), "import_failed");
            } else {
                fallback_reasons.insert(partition.clone(), "import_cancelled");
            }
        }
    }
}

#[async_trait]
impl<T, P> RebalanceHandler for ProcessorRebalanceHandler<T, P>
where
    T: Send + Sync + 'static,
    P: BatchConsumerProcessor<T> + 'static,
{
    // ============================================
    // SETUP METHODS - Called synchronously within librdkafka callbacks
    // These run BEFORE messages can arrive/stop
    // ============================================

    fn setup_assigned_partitions(&self, partitions: &TopicPartitionList) {
        let partition_infos: Vec<Partition> = partitions
            .elements()
            .into_iter()
            .map(Partition::from)
            .collect();

        info!(
            partition_count = partition_infos.len(),
            caller = "assign_callback",
            "Setting up assigned partitions (sync)"
        );

        // Record per-partition assignment metrics for observability
        for partition in &partition_infos {
            metrics::counter!(
                REBALANCE_PARTITION_STATE_CHANGE,
                "topic" => partition.topic().to_string(),
                "partition" => partition.partition_number().to_string(),
                "op" => "assign",
                "assignment_mode" => "consumer_group",
            )
            .increment(1);
        }

        // Add to owned partitions FIRST (coordinator is the source of truth)
        // If partition was revoked then re-assigned, this adds it back
        self.rebalance_tracker
            .add_owned_partitions(&partition_infos);

        // Create partition workers BEFORE messages can arrive
        // This is fast - just spawning tokio tasks and creating channels
        // If worker already exists (rapid re-assignment), it will be reused
        if let Some(ref router) = self.router {
            router.add_partitions(&partition_infos);
            debug!(
                worker_count = router.worker_count(),
                "Created partition workers"
            );
        }

        // Increment rebalancing counter SYNCHRONOUSLY before async work is queued
        // This ensures no gap where orphan cleanup could run
        self.rebalance_tracker.start_rebalancing();

        // NOTE: Per-partition setup tasks are spawned in async_setup_assigned_partitions,
        // not here (sync callback can't do async work like checkpoint import)
    }

    fn setup_revoked_partitions(&self, partitions: &TopicPartitionList) {
        let partition_infos: Vec<Partition> = partitions
            .elements()
            .into_iter()
            .map(Partition::from)
            .collect();

        info!(
            partition_count = partition_infos.len(),
            caller = "revoke_callback",
            "Setting up revoked partitions (sync)"
        );

        // Record per-partition revocation metrics for observability
        for partition in &partition_infos {
            metrics::counter!(
                REBALANCE_PARTITION_STATE_CHANGE,
                "topic" => partition.topic().to_string(),
                "partition" => partition.partition_number().to_string(),
                "op" => "revoke",
                "assignment_mode" => "consumer_group",
            )
            .increment(1);
        }

        // Remove from owned partitions (coordinator is the source of truth)
        // This happens BEFORE async cleanup, so cleanup can check ownership
        self.rebalance_tracker
            .remove_owned_partitions(&partition_infos);

        // CANCEL and remove setup tasks for revoked partitions
        // This immediately cancels any in-flight S3 downloads to save cost/time
        // Tasks will also check is_partition_owned() as a backup
        self.cancel_setup_tasks(&partition_infos);

        // Unregister stores from DashMap BEFORE revocation completes
        // This prevents new store creation during shutdown (Step 1 of two-step cleanup)
        // This is fast - just DashMap removes
        for partition in &partition_infos {
            self.store_manager
                .unregister_store(partition.topic(), partition.partition_number());
        }

        debug!(
            unregistered_count = partition_infos.len(),
            active_stores = self.store_manager.get_active_store_count(),
            "Unregistered stores for revoked partitions"
        );
    }

    // ============================================
    // CLEANUP METHODS - Called asynchronously after callbacks return
    // For slow operations like I/O, draining queues, etc.
    // ============================================

    async fn async_setup_assigned_partitions(
        &self,
        consumer_command_tx: &ConsumerCommandSender,
    ) -> Result<()> {
        // Get ALL owned partitions (not just incremental from this rebalance)
        // This handles retained partitions across overlapping rebalances
        let owned_partitions = self.rebalance_tracker.get_owned_partitions();

        debug!(
            owned_count = owned_partitions.len(),
            "Async setup starting - spawning tasks for owned partitions"
        );

        // Spawn setup tasks for partitions that don't have one yet.
        // Uses atomic two-phase registration to prevent race conditions where overlapping
        // rebalances could both spawn tasks for the same partition.
        // NOTE: We do NOT await these tasks here - that's deferred to finalize_rebalance_cycle
        for partition in &owned_partitions {
            let cancel_token = CancellationToken::new();
            // Atomically claim the partition - if stale entry exists, it will be replaced
            if self.try_claim_partition_setup(partition, cancel_token.clone()) {
                // We claimed it - now spawn the task and finalize registration
                let handle =
                    self.spawn_partition_setup_task(partition.clone(), cancel_token.clone());
                self.finalize_partition_setup(partition, handle);
            }
            // If claim failed, either a store exists or a task is still running
        }

        // If we're the last rebalance (count == 1), run finalize BEFORE decrementing so is_rebalancing()
        // stays true during finalize. That prevents orphan/capacity cleanup from deleting dirs we're setting up.
        let is_last = self.rebalance_tracker.rebalancing_count() == 1;
        if is_last {
            // IMPORTANT: Always call finish_rebalancing even if finalize errors, otherwise the
            // counter leaks and is_rebalancing() returns true permanently — blocking all
            // checkpoint exports and offset commits.
            let result = self
                .finalize_rebalance_cycle(consumer_command_tx, true)
                .await;
            self.rebalance_tracker.finish_rebalancing();
            if let Err(e) = result {
                error!("Finalize failed after decrementing rebalance counter: {e:#}");
                return Err(e);
            }
        } else {
            self.rebalance_tracker.finish_rebalancing();
            if !self.rebalance_tracker.is_rebalancing() {
                self.finalize_rebalance_cycle(consumer_command_tx, false)
                    .await?;
            } else {
                debug!(
                    "Rebalance async setup complete, but other rebalances still in progress - deferring finalize"
                );
            }
        }

        Ok(())
    }

    async fn cleanup_revoked_partitions(&self, partitions: &TopicPartitionList) -> Result<()> {
        let partition_infos: Vec<Partition> = partitions
            .elements()
            .into_iter()
            .map(Partition::from)
            .collect();

        // Only clean up partitions that are NOT currently owned
        // If a partition was re-assigned, it's now owned and shouldn't be cleaned up
        let partitions_to_cleanup = self
            .rebalance_tracker
            .get_unowned_partitions(&partition_infos);

        let skipped_count = partition_infos.len() - partitions_to_cleanup.len();
        if skipped_count > 0 {
            debug!(
                skipped_reassigned = skipped_count,
                "Skipped cleanup for re-assigned partitions"
            );
        }

        if partitions_to_cleanup.is_empty() {
            info!("No partitions to clean up (all were re-assigned)");
            return Ok(());
        }

        info!(
            cleanup_count = partitions_to_cleanup.len(),
            "Cleaning up revoked partitions (async)"
        );

        // Shutdown partition workers - drain their queues
        // Stores are already removed from map (done in setup_revoked_partitions)
        if let Some(ref router) = self.router {
            let workers = router.remove_partitions(&partitions_to_cleanup);
            shutdown_workers(workers).await;
            debug!(
                active_workers = router.worker_count(),
                "Shut down partition workers"
            );
        }

        // Clear offset tracking state for revoked partitions
        // This prevents stale offsets from being committed after rebalance
        for partition in &partitions_to_cleanup {
            self.offset_tracker.clear_partition(partition);
        }

        // File cleanup is handled by finalize_rebalance_cycle at end of the rebalance cycle.
        // This uses disk scan + owned_partitions to delete unowned directories.

        info!(
            cleaned_count = partitions_to_cleanup.len(),
            "Revoked partition cleanup completed (files will be cleaned at end of cycle)"
        );

        Ok(())
    }

    async fn on_pre_rebalance(&self) -> Result<()> {
        // Note: rebalance_tracker.start_rebalancing() is called in setup_assigned_partitions()
        // (sync callback) to ensure no gap before async work is queued.
        // The rebalance_tracker's counter is the single source of truth.
        Ok(())
    }

    async fn on_post_rebalance(&self) -> Result<()> {
        info!("Post-rebalance: Sync callbacks complete, async cleanup may continue");
        // Note: rebalance_tracker counter is decremented via finish_rebalancing()
        // at the end of async_setup_assigned_partitions.
        // The rebalance_tracker's rebalancing counter is the single source of truth.

        // Log current stats
        let store_count = self.store_manager.stores().len();
        info!("Active deduplication stores: {}", store_count);

        if let Some(ref router) = self.router {
            info!("Active partition workers: {}", router.worker_count());
        }

        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::checkpoint::config::DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS;
    use crate::kafka::batch_message::KafkaMessage;
    use crate::kafka::offset_tracker::OffsetTracker;
    use crate::kafka::partition_router::PartitionRouterConfig;
    use crate::rocksdb::store::RocksDbConfig;
    use crate::store::DeduplicationStoreConfig;
    use crate::test_utils::create_test_tracker;
    use rdkafka::Offset;
    use tempfile::TempDir;

    struct TestProcessor;

    #[async_trait]
    impl BatchConsumerProcessor<String> for TestProcessor {
        async fn process_batch(&self, _messages: Vec<KafkaMessage<String>>) -> Result<()> {
            Ok(())
        }
    }

    #[tokio::test]
    async fn test_rebalance_handler_creation() {
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));

        // Test handler without router
        let handler: ProcessorRebalanceHandler<String, TestProcessor> =
            ProcessorRebalanceHandler::new(
                store_manager.clone(),
                coordinator.clone(),
                offset_tracker.clone(),
                None,
                16, // rebalance_cleanup_parallelism
                Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
            );
        assert!(handler.router.is_none());

        // Test handler with router
        let processor = Arc::new(TestProcessor);
        let router = Arc::new(PartitionRouter::new(
            processor,
            offset_tracker.clone(),
            PartitionRouterConfig::default(),
        ));
        let handler_with_router = ProcessorRebalanceHandler::with_router(
            store_manager,
            coordinator,
            router.clone(),
            offset_tracker,
            None,
            16, // rebalance_cleanup_parallelism
            Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
        );
        assert!(handler_with_router.router.is_some());
    }

    #[tokio::test]
    async fn test_rebalance_handler_manages_workers() {
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let processor = Arc::new(TestProcessor);
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));
        let router = Arc::new(PartitionRouter::new(
            processor,
            offset_tracker.clone(),
            PartitionRouterConfig::default(),
        ));

        let handler = ProcessorRebalanceHandler::with_router(
            store_manager,
            coordinator,
            router.clone(),
            offset_tracker,
            None,
            16, // rebalance_cleanup_parallelism
            Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
        );

        // Initially no workers
        assert_eq!(router.worker_count(), 0);

        // Assign partitions (sync setup creates workers)
        let mut partitions = rdkafka::TopicPartitionList::new();
        partitions
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();
        partitions
            .add_partition_offset("test-topic", 1, Offset::Beginning)
            .unwrap();

        handler.setup_assigned_partitions(&partitions);
        assert_eq!(router.worker_count(), 2);

        // Revoke one partition (sync setup + async cleanup)
        let mut revoked = rdkafka::TopicPartitionList::new();
        revoked
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();

        handler.setup_revoked_partitions(&revoked);
        handler.cleanup_revoked_partitions(&revoked).await.unwrap();
        assert_eq!(router.worker_count(), 1);

        // Cleanup
        let workers = router.shutdown_all();
        shutdown_workers(workers).await;
    }

    #[tokio::test]
    async fn test_rebalance_removes_stores_before_workers_shutdown() {
        // This test verifies that stores are removed from the map (in setup_revoked_partitions)
        // BEFORE workers are shut down (in cleanup_revoked_partitions).
        //
        // Note: File cleanup is handled by finalize_rebalance_cycle at end of cycle,
        // NOT during cleanup_revoked_partitions. This avoids race conditions with rapid revoke→assign.

        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let processor = Arc::new(TestProcessor);
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));
        let router = Arc::new(PartitionRouter::new(
            processor,
            offset_tracker.clone(),
            PartitionRouterConfig::default(),
        ));

        let handler = ProcessorRebalanceHandler::with_router(
            store_manager.clone(),
            coordinator,
            router.clone(),
            offset_tracker,
            None,
            16, // rebalance_cleanup_parallelism
            Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
        );

        // Assign partition and create a store
        let mut partitions = rdkafka::TopicPartitionList::new();
        partitions
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();

        handler.setup_assigned_partitions(&partitions);
        assert_eq!(router.worker_count(), 1);

        // Create a store for the partition (simulating what happens during processing)
        store_manager.get_or_create("test-topic", 0).await.unwrap();
        assert_eq!(store_manager.get_active_store_count(), 1);

        // Revoke the partition (sync setup removes store from map)
        handler.setup_revoked_partitions(&partitions);
        assert_eq!(store_manager.get_active_store_count(), 0);
        // Worker still exists at this point
        assert_eq!(router.worker_count(), 1);

        // Async cleanup shuts down workers (files are deleted in finalize_rebalance_cycle)
        handler
            .cleanup_revoked_partitions(&partitions)
            .await
            .unwrap();

        // After cleanup:
        // - Worker should be shut down
        // - Store should be unregistered from map
        // - Files still exist (will be cleaned in finalize_rebalance_cycle)
        assert_eq!(router.worker_count(), 0);

        // Files are NOT immediately deleted - finalize_rebalance_cycle handles this
        let partition_dir = temp_dir.path().join("test-topic").join("0");
        assert!(
            partition_dir.exists(),
            "Partition directory still exists (orphan cleaner will handle cleanup)"
        );
    }

    #[tokio::test]
    async fn test_rebalance_store_not_found_during_shutdown() {
        // This test verifies that after stores are removed from the map,
        // any attempt to get_or_create will return an error rather than
        // creating a new store that would be immediately deleted.

        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator));

        // Create a store
        store_manager.get_or_create("test-topic", 0).await.unwrap();
        assert_eq!(store_manager.get_active_store_count(), 1);

        // Unregister store (Step 1 of two-step cleanup)
        store_manager.unregister_store("test-topic", 0);
        assert_eq!(store_manager.get_active_store_count(), 0);

        // Verify we can still create a new store if needed
        // (this would happen if partition is re-assigned)
        store_manager.get_or_create("test-topic", 0).await.unwrap();
        assert_eq!(store_manager.get_active_store_count(), 1);

        // Cleanup
        store_manager.remove("test-topic", 0).unwrap();
    }

    #[tokio::test]
    async fn test_rapid_revoke_assign_does_not_remove_new_worker() {
        // This test verifies that when a partition is rapidly revoked and re-assigned,
        // the cleanup for the revocation does NOT remove the newly created worker.
        //
        // Scenario:
        // 1. Partition 0 is assigned (worker created)
        // 2. Partition 0 is revoked (store removed from map)
        // 3. Partition 0 is immediately re-assigned (NEW worker created)
        // 4. Async cleanup for step 2 runs - should NOT remove the new worker

        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let processor = Arc::new(TestProcessor);
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));
        let router = Arc::new(PartitionRouter::new(
            processor,
            offset_tracker.clone(),
            PartitionRouterConfig::default(),
        ));

        let handler = ProcessorRebalanceHandler::with_router(
            store_manager.clone(),
            coordinator,
            router.clone(),
            offset_tracker,
            None,
            16, // rebalance_cleanup_parallelism
            Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
        );

        // Step 1: Initial assignment
        let mut partitions = rdkafka::TopicPartitionList::new();
        partitions
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();

        handler.setup_assigned_partitions(&partitions);
        assert_eq!(router.worker_count(), 1);

        // Step 2: Revoke (sync - removes store from map)
        handler.setup_revoked_partitions(&partitions);

        // Step 3: Immediate re-assign (sync - creates NEW worker)
        handler.setup_assigned_partitions(&partitions);
        assert_eq!(router.worker_count(), 1); // Still have 1 worker (the new one)

        // Step 4: Async cleanup for the revoke runs
        // This should detect that partition 0 is now assigned and skip cleanup
        handler
            .cleanup_revoked_partitions(&partitions)
            .await
            .unwrap();

        // The new worker should still exist!
        assert_eq!(
            router.worker_count(),
            1,
            "New worker should NOT be removed by stale revocation cleanup"
        );

        // Cleanup
        let workers = router.shutdown_all();
        shutdown_workers(workers).await;
    }

    #[tokio::test]
    async fn test_cleanup_store_files_after_unregister_store() {
        // Test the two-step cleanup process:
        // Step 1: unregister_store() - closes RocksDB, removes from map
        // Step 2: cleanup_store_files() - deletes files from disk

        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator));

        // Create a store (this creates the directory)
        store_manager.get_or_create("test-topic", 0).await.unwrap();

        let partition_dir = temp_dir.path().join("test-topic").join("0");
        assert!(partition_dir.exists(), "Partition directory should exist");

        // Step 1: Unregister store (store is dropped, RocksDB is closed)
        store_manager.unregister_store("test-topic", 0);
        assert_eq!(store_manager.get_active_store_count(), 0);

        // Directory should still exist (files not deleted yet)
        assert!(
            partition_dir.exists(),
            "Partition directory should still exist after unregister_store"
        );

        // Step 2: Cleanup the files
        store_manager.cleanup_store_files("test-topic", 0).unwrap();

        // Directory should be deleted
        assert!(
            !partition_dir.exists(),
            "Partition directory should be deleted after cleanup_store_files"
        );
    }

    #[tokio::test]
    async fn test_per_partition_task_cancellation() {
        // Test that revoke cancels the per-partition setup task
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));

        let handler: ProcessorRebalanceHandler<String, TestProcessor> =
            ProcessorRebalanceHandler::new(
                store_manager,
                coordinator.clone(),
                offset_tracker,
                None,
                16, // rebalance_cleanup_parallelism
                Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
            );

        // First assignment
        let mut partitions = rdkafka::TopicPartitionList::new();
        partitions
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();

        handler.setup_assigned_partitions(&partitions);

        // Verify partition is owned
        let p0 = Partition::new("test-topic".to_string(), 0);
        assert!(
            coordinator.is_partition_owned(&p0),
            "Partition should be owned after assign"
        );

        // Revoke should remove ownership and cancel any setup tasks
        handler.setup_revoked_partitions(&partitions);

        // Verify partition is no longer owned
        assert!(
            !coordinator.is_partition_owned(&p0),
            "Partition should not be owned after revoke"
        );
    }

    #[tokio::test]
    async fn test_async_setup_sends_resume_command() {
        // Test that async_setup_assigned_partitions sends a Resume command
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));

        let handler: ProcessorRebalanceHandler<String, TestProcessor> =
            ProcessorRebalanceHandler::new(
                store_manager.clone(),
                coordinator,
                offset_tracker,
                None,
                16, // rebalance_cleanup_parallelism
                Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
            );

        // Create command channel
        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();

        // First do sync setup (required before async setup)
        let mut partitions = rdkafka::TopicPartitionList::new();
        partitions
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();
        partitions
            .add_partition_offset("test-topic", 1, Offset::Beginning)
            .unwrap();

        handler.setup_assigned_partitions(&partitions);

        // Now do async setup - should send Resume command
        handler.async_setup_assigned_partitions(&tx).await.unwrap();

        // Check that Resume command was sent (no importer, so only Resume is sent)
        let command = rx.try_recv().expect("Should have received a command");
        match command {
            ConsumerCommand::Resume(resume_partitions) => {
                assert_eq!(
                    resume_partitions.count(),
                    2,
                    "Resume command should contain all assigned partitions"
                );
            }
            ConsumerCommand::SeekPartitions(_) => {
                panic!("Handler has no importer; should not send SeekPartitions");
            }
        }

        // Verify stores were created
        assert_eq!(
            store_manager.get_active_store_count(),
            2,
            "Two stores should have been created"
        );
    }

    #[tokio::test]
    async fn test_overlapping_rebalances_only_last_sends_resume() {
        // Test that when multiple rebalances overlap, only the last one sends Resume.
        // This tests the counter-based coordination.
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));

        let handler: ProcessorRebalanceHandler<String, TestProcessor> =
            ProcessorRebalanceHandler::new(
                store_manager,
                coordinator.clone(),
                offset_tracker,
                None,
                16, // rebalance_cleanup_parallelism
                Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
            );

        // Create command channel
        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();

        // First rebalance - assign partition 0
        let mut partitions_a = rdkafka::TopicPartitionList::new();
        partitions_a
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();
        handler.setup_assigned_partitions(&partitions_a);

        // Second rebalance - assign partition 1 (overlapping, before A's async completes)
        let mut partitions_b = rdkafka::TopicPartitionList::new();
        partitions_b
            .add_partition_offset("test-topic", 1, Offset::Beginning)
            .unwrap();
        handler.setup_assigned_partitions(&partitions_b);

        // Counter should be 2 (two rebalances in progress)
        assert!(
            coordinator.is_rebalancing(),
            "Should be rebalancing with counter > 0"
        );

        // First async setup completes - should NOT send Resume (counter still > 0)
        handler.async_setup_assigned_partitions(&tx).await.unwrap();

        // No Resume yet (counter is still 1)
        // Give a moment for any potential Resume to be sent
        tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;

        // Counter should be 1 now
        assert!(
            coordinator.is_rebalancing(),
            "Should still be rebalancing (one more to go)"
        );

        // Second async setup completes - should send Resume (counter == 0)
        handler.async_setup_assigned_partitions(&tx).await.unwrap();

        // Should have received exactly one Resume command (from the last rebalance)
        let cmd = rx.try_recv().expect("Should have received Resume command");
        let tpl = match &cmd {
            ConsumerCommand::Resume(t) => t.clone(),
            ConsumerCommand::SeekPartitions(_) => {
                let cmd2 = rx
                    .try_recv()
                    .expect("Should have received Resume after SeekPartitions");
                match cmd2 {
                    ConsumerCommand::Resume(t) => t,
                    ConsumerCommand::SeekPartitions(_) => {
                        panic!("Expected Resume, got SeekPartitions")
                    }
                }
            }
        };
        // Should resume both partitions
        assert_eq!(tpl.count(), 2, "Should resume all owned partitions");

        // No more commands
        assert!(
            rx.try_recv().is_err(),
            "Should only receive one Resume command"
        );
    }

    #[tokio::test]
    async fn test_resume_only_owned_partitions() {
        // Test that async_setup_assigned_partitions resumes only owned partitions.
        // Simulates a race condition where revoke callback runs AFTER sync setup
        // but BEFORE async setup. We pass the ORIGINAL assignment list to async setup
        // (containing a now-revoked partition) to verify ownership filtering works.
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));

        let handler: ProcessorRebalanceHandler<String, TestProcessor> =
            ProcessorRebalanceHandler::new(
                store_manager.clone(),
                coordinator.clone(),
                offset_tracker,
                None,
                16, // rebalance_cleanup_parallelism
                Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
            );

        // Create command channel
        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();

        // Assign partitions 0, 1, 2
        let mut partitions = rdkafka::TopicPartitionList::new();
        partitions
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();
        partitions
            .add_partition_offset("test-topic", 1, Offset::Beginning)
            .unwrap();
        partitions
            .add_partition_offset("test-topic", 2, Offset::Beginning)
            .unwrap();

        handler.setup_assigned_partitions(&partitions);

        // Verify ownership via coordinator
        assert!(coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 0)));
        assert!(coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 1)));
        assert!(coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 2)));

        // Revoke partition 1 (removes from ownership)
        let mut revoked = rdkafka::TopicPartitionList::new();
        revoked
            .add_partition_offset("test-topic", 1, Offset::Beginning)
            .unwrap();
        handler.setup_revoked_partitions(&revoked);

        // Verify partition 1 is no longer owned
        assert!(!coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 1)));

        // Now do async setup - should resume only owned partitions (0 and 2)
        handler.async_setup_assigned_partitions(&tx).await.unwrap();

        // Check that Resume command contains only owned partitions
        let command = rx.try_recv().expect("Should have received a command");
        match command {
            ConsumerCommand::Resume(resume_partitions) => {
                assert_eq!(
                    resume_partitions.count(),
                    2,
                    "Resume command should only contain owned partitions (0 and 2)"
                );
                let elements = resume_partitions.elements();
                let partition_nums: Vec<i32> = elements.iter().map(|e| e.partition()).collect();
                assert!(
                    partition_nums.contains(&0),
                    "Partition 0 should be in Resume"
                );
                assert!(
                    partition_nums.contains(&2),
                    "Partition 2 should be in Resume"
                );
                assert!(
                    !partition_nums.contains(&1),
                    "Partition 1 should NOT be in Resume (not owned)"
                );
            }
            ConsumerCommand::SeekPartitions(_) => {
                let cmd2 = rx
                    .try_recv()
                    .expect("Should have received Resume after SeekPartitions");
                let ConsumerCommand::Resume(resume_partitions) = cmd2 else {
                    panic!("Expected Resume command, got {:?}", cmd2);
                };
                assert_eq!(resume_partitions.count(), 2);
                let partition_nums: Vec<i32> = resume_partitions
                    .elements()
                    .iter()
                    .map(|e| e.partition())
                    .collect();
                assert!(partition_nums.contains(&0));
                assert!(partition_nums.contains(&2));
                assert!(!partition_nums.contains(&1));
            }
        }

        // Verify stores
        assert!(
            store_manager.get("test-topic", 0).is_some(),
            "Store for partition 0 should exist"
        );
        assert!(
            store_manager.get("test-topic", 1).is_none(),
            "Store for partition 1 should NOT exist"
        );
        assert!(
            store_manager.get("test-topic", 2).is_some(),
            "Store for partition 2 should exist"
        );
    }

    #[tokio::test]
    async fn test_resume_skipped_when_no_owned_partitions() {
        // Test that Resume is skipped entirely when no partitions are owned
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));

        let handler: ProcessorRebalanceHandler<String, TestProcessor> =
            ProcessorRebalanceHandler::new(
                store_manager,
                coordinator.clone(),
                offset_tracker,
                None,
                16, // rebalance_cleanup_parallelism
                Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
            );

        // Create command channel
        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();

        // Assign partition 0
        let mut partitions = rdkafka::TopicPartitionList::new();
        partitions
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();

        handler.setup_assigned_partitions(&partitions);
        assert!(coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 0)));

        // Revoke partition 0 (no longer owned)
        handler.setup_revoked_partitions(&partitions);
        assert!(!coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 0)));

        // Now do async setup - should skip Resume entirely (no owned partitions)
        handler.async_setup_assigned_partitions(&tx).await.unwrap();

        // Should NOT have received a Resume command (no owned partitions)
        assert!(
            rx.try_recv().is_err(),
            "Should NOT have received a Resume command when no partitions are owned"
        );
    }

    #[tokio::test]
    async fn test_counter_not_leaked_on_finalize_error() {
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));

        let handler: ProcessorRebalanceHandler<String, TestProcessor> =
            ProcessorRebalanceHandler::new(
                store_manager,
                coordinator.clone(),
                offset_tracker,
                None,
                16,
                Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
            );

        let mut partitions = rdkafka::TopicPartitionList::new();
        partitions
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();
        handler.setup_assigned_partitions(&partitions);
        assert_eq!(coordinator.rebalancing_count(), 1);

        let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
        drop(rx);

        let result = handler.async_setup_assigned_partitions(&tx).await;
        assert!(result.is_err());

        assert_eq!(
            coordinator.rebalancing_count(),
            0,
            "Counter must be decremented even when finalize fails (channel broken)"
        );
    }

    #[tokio::test]
    async fn test_async_setup_skips_store_creation_for_unowned_partition() {
        // Test that async_setup_single_partition skips store creation
        // when the partition is no longer owned (was revoked during async setup).
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));

        let handler: ProcessorRebalanceHandler<String, TestProcessor> =
            ProcessorRebalanceHandler::new(
                store_manager.clone(),
                coordinator.clone(),
                offset_tracker,
                None,
                16, // rebalance_cleanup_parallelism
                Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
            );

        // Create command channel
        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();

        // Assign partitions 0 and 1
        let mut partitions = rdkafka::TopicPartitionList::new();
        partitions
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();
        partitions
            .add_partition_offset("test-topic", 1, Offset::Beginning)
            .unwrap();

        handler.setup_assigned_partitions(&partitions);

        // Verify ownership via coordinator
        assert!(coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 0)));
        assert!(coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 1)));

        // Before async setup, revoke partition 1 (simulating overlapping rebalance)
        let mut revoked = rdkafka::TopicPartitionList::new();
        revoked
            .add_partition_offset("test-topic", 1, Offset::Beginning)
            .unwrap();
        handler.setup_revoked_partitions(&revoked);

        // Verify partition 1 is no longer owned
        assert!(coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 0)));
        assert!(!coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 1)));

        // Now run async setup - should skip store creation for partition 1
        handler.async_setup_assigned_partitions(&tx).await.unwrap();

        // Verify: store for partition 0 should exist (still owned)
        assert!(
            store_manager.get("test-topic", 0).is_some(),
            "Store for partition 0 should exist (owned)"
        );

        // Verify: store for partition 1 should NOT exist (not owned, creation skipped)
        assert!(
            store_manager.get("test-topic", 1).is_none(),
            "Store for partition 1 should NOT exist (not owned, creation skipped)"
        );

        // Verify: the partition directory should not exist either
        let partition_1_dir = temp_dir.path().join("test-topic").join("1");
        assert!(
            !partition_1_dir.exists(),
            "Partition 1 directory should not exist (store creation was skipped)"
        );
    }

    #[tokio::test]
    async fn test_retained_partition_across_rebalance() {
        // KEY TEST: Verifies that partitions retained across rebalances are resumed.
        // Scenario: Rebalance A assigns [0, 1], then Rebalance B revokes [1] and assigns []
        // (partition 0 is retained). Partition 0 should be resumed by Rebalance B.
        //
        // This is a COMPLETE end-to-end test: A completes fully, then B interrupts.
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));

        let handler: ProcessorRebalanceHandler<String, TestProcessor> =
            ProcessorRebalanceHandler::new(
                store_manager.clone(),
                coordinator.clone(),
                offset_tracker,
                None,
                16, // rebalance_cleanup_parallelism
                Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
            );

        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();

        // ==================== REBALANCE A ====================
        // Rebalance A: Assign [0, 1] - SYNC
        let mut partitions_a = rdkafka::TopicPartitionList::new();
        partitions_a
            .add_partition_offset("test-topic", 0, Offset::Beginning)
            .unwrap();
        partitions_a
            .add_partition_offset("test-topic", 1, Offset::Beginning)
            .unwrap();
        handler.setup_assigned_partitions(&partitions_a);

        // Verify both owned
        assert!(coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 0)));
        assert!(coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 1)));

        // Rebalance A: Complete async setup - creates stores and sends Resume
        handler.async_setup_assigned_partitions(&tx).await.unwrap();

        // Verify A's stores were created
        assert!(
            store_manager.get("test-topic", 0).is_some(),
            "A should have created store for partition 0"
        );
        assert!(
            store_manager.get("test-topic", 1).is_some(),
            "A should have created store for partition 1"
        );

        // Drain A's Resume command
        let _ = rx.try_recv().expect("A should have sent Resume");

        // ==================== REBALANCE B ====================
        // Rebalance B starts - revoke partition 1
        let mut revoked = rdkafka::TopicPartitionList::new();
        revoked
            .add_partition_offset("test-topic", 1, Offset::Beginning)
            .unwrap();
        handler.setup_revoked_partitions(&revoked);

        // Verify store 1 was unregistered (but partition 0's store still exists)
        assert!(
            store_manager.get("test-topic", 0).is_some(),
            "Store for partition 0 should still exist"
        );
        assert!(
            store_manager.get("test-topic", 1).is_none(),
            "Store for partition 1 should be unregistered"
        );

        // Rebalance B: Assign empty (no new partitions, partition 0 is retained)
        let partitions_b = rdkafka::TopicPartitionList::new();
        handler.setup_assigned_partitions(&partitions_b); // This cancels A's token

        // Verify: partition 0 still owned, partition 1 not owned
        assert!(coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 0)));
        assert!(!coordinator.is_partition_owned(&Partition::new("test-topic".to_string(), 1)));

        // Rebalance B's async setup with empty list - should still resume partition 0!
        handler.async_setup_assigned_partitions(&tx).await.unwrap();

        // Should resume partition 0 (still owned) but not partition 1 (revoked)
        let command = rx
            .try_recv()
            .expect("Should have received Resume for retained partition");
        match command {
            ConsumerCommand::Resume(tpl) => {
                assert_eq!(tpl.count(), 1, "Should resume exactly 1 partition");
                let elements = tpl.elements();
                assert_eq!(elements[0].partition(), 0, "Should resume partition 0");
            }
            ConsumerCommand::SeekPartitions(_) => {
                let cmd2 = rx.try_recv().expect("Should have received Resume");
                let ConsumerCommand::Resume(tpl) = cmd2 else {
                    panic!("Expected Resume command, got {:?}", cmd2);
                };
                assert_eq!(tpl.count(), 1);
                assert_eq!(tpl.elements()[0].partition(), 0);
            }
        }

        // Verify partition 0's store still exists after B completes
        assert!(
            store_manager.get("test-topic", 0).is_some(),
            "Store for retained partition 0 should still exist"
        );
    }

    #[tokio::test]
    async fn test_ownership_across_multiple_topics() {
        // Verify ownership tracking works correctly across multiple topics
        let temp_dir = TempDir::new().unwrap();
        let store_config = DeduplicationStoreConfig {
            path: temp_dir.path().to_path_buf(),
            max_capacity: 1000,
            rocksdb: RocksDbConfig::default(),
        };
        let coordinator = create_test_tracker();
        let store_manager = Arc::new(StoreManager::new(store_config, coordinator.clone()));
        let offset_tracker = Arc::new(OffsetTracker::new(coordinator.clone()));

        let handler: ProcessorRebalanceHandler<String, TestProcessor> =
            ProcessorRebalanceHandler::new(
                store_manager.clone(),
                coordinator.clone(),
                offset_tracker,
                None,
                16, // rebalance_cleanup_parallelism
                Duration::from_secs(DEFAULT_LOCAL_CHECKPOINT_MAX_STALENESS_SECS),
            );

        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();

        // Assign partitions from two different topics
        let mut partitions = rdkafka::TopicPartitionList::new();
        partitions
            .add_partition_offset("topic-a", 0, Offset::Beginning)
            .unwrap();
        partitions
            .add_partition_offset("topic-a", 1, Offset::Beginning)
            .unwrap();
        partitions
            .add_partition_offset("topic-b", 0, Offset::Beginning)
            .unwrap();

        handler.setup_assigned_partitions(&partitions);

        // Verify all owned
        assert!(coordinator.is_partition_owned(&Partition::new("topic-a".to_string(), 0)));
        assert!(coordinator.is_partition_owned(&Partition::new("topic-a".to_string(), 1)));
        assert!(coordinator.is_partition_owned(&Partition::new("topic-b".to_string(), 0)));
        assert_eq!(coordinator.owned_partition_count(), 3);

        // Revoke topic-a partition 1 only
        let mut revoked = rdkafka::TopicPartitionList::new();
        revoked
            .add_partition_offset("topic-a", 1, Offset::Beginning)
            .unwrap();
        handler.setup_revoked_partitions(&revoked);

        // Verify correct ownership
        assert!(coordinator.is_partition_owned(&Partition::new("topic-a".to_string(), 0)));
        assert!(!coordinator.is_partition_owned(&Partition::new("topic-a".to_string(), 1)));
        assert!(coordinator.is_partition_owned(&Partition::new("topic-b".to_string(), 0)));
        assert_eq!(coordinator.owned_partition_count(), 2);

        // Complete async setup - should resume only owned partitions
        handler.async_setup_assigned_partitions(&tx).await.unwrap();

        let command = rx.try_recv().expect("Should have received Resume");
        match command {
            ConsumerCommand::Resume(tpl) => {
                assert_eq!(tpl.count(), 2, "Should resume 2 partitions");
                let elements = tpl.elements();
                let topics: Vec<&str> = elements.iter().map(|e| e.topic()).collect();
                assert!(topics.contains(&"topic-a"), "topic-a:0 should be resumed");
                assert!(topics.contains(&"topic-b"), "topic-b:0 should be resumed");
            }
            ConsumerCommand::SeekPartitions(_) => {
                let cmd2 = rx.try_recv().expect("Should have received Resume");
                let ConsumerCommand::Resume(tpl) = cmd2 else {
                    panic!("Expected Resume command, got {:?}", cmd2);
                };
                assert_eq!(tpl.count(), 2);
                let topics: Vec<String> = tpl
                    .elements()
                    .iter()
                    .map(|e| e.topic().to_string())
                    .collect();
                assert!(topics.contains(&"topic-a".to_string()));
                assert!(topics.contains(&"topic-b".to_string()));
            }
        }

        // Verify stores created for owned partitions only
        assert!(store_manager.get("topic-a", 0).is_some());
        assert!(store_manager.get("topic-a", 1).is_none());
        assert!(store_manager.get("topic-b", 0).is_some());
    }
}