Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

37.1 KB
Raw Blame
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;

use async_trait::async_trait;
use chrono::Utc;
use tokio_util::sync::CancellationToken;

use kafka_deduplicator::checkpoint::{
    hash_prefix_for_partition, CheckpointConfig, CheckpointExporter, CheckpointMetadata,
    CheckpointPlan, CheckpointUploader, CheckpointWorker, UploadCancelledError,
};
use kafka_deduplicator::kafka::types::Partition;
use kafka_deduplicator::store::TimestampMetadata;
use kafka_deduplicator::test_utils::test_helpers::{create_test_dedup_store, TestRawEventBuilder};

use anyhow::{Context, Result};
use tempfile::TempDir;
use tracing::info;

/// Mock uploader for testing that stores files in local filesystem.
/// Holds ownership of TempDir via Arc to ensure the directory persists for the uploader's lifetime
/// and survives cloning (multiple references share the same temp directory).
#[derive(Debug, Clone)]
pub struct MockUploader {
    /// TempDir that owns the upload directory (shared across clones, dropped when all refs gone)
    _temp_dir: Arc<TempDir>,
    /// Local directory where uploaded files are stored
    upload_dir: PathBuf,
    /// Whether the uploader should simulate being available
    available: bool,
}

impl MockUploader {
    pub fn new() -> Result<Self> {
        let temp_dir = TempDir::new()?;
        let upload_dir = temp_dir.path().to_path_buf();
        Ok(Self {
            _temp_dir: Arc::new(temp_dir),
            upload_dir,
            available: true,
        })
    }

    pub fn new_unavailable() -> Result<Self> {
        let temp_dir = TempDir::new()?;
        let upload_dir = temp_dir.path().to_path_buf();
        Ok(Self {
            _temp_dir: Arc::new(temp_dir),
            upload_dir,
            available: false,
        })
    }

    /// Get the stored files for testing
    pub async fn get_stored_files(&self) -> Result<HashMap<String, Vec<u8>>> {
        let mut files = HashMap::new();

        if !self.upload_dir.exists() {
            return Ok(files);
        }

        let mut stack = vec![self.upload_dir.clone()];

        while let Some(current_path) = stack.pop() {
            let entries = std::fs::read_dir(&current_path)?;

            for entry in entries {
                let entry = entry?;
                let path = entry.path();

                if path.is_dir() {
                    stack.push(path);
                } else {
                    let relative_path = path.strip_prefix(&self.upload_dir)?;
                    let key = relative_path.to_string_lossy().replace('\\', "/");
                    let data = tokio::fs::read(&path).await?;
                    files.insert(key, data);
                }
            }
        }

        Ok(files)
    }

    /// Clear all stored files
    pub async fn clear(&self) -> Result<()> {
        if self.upload_dir.exists() {
            tokio::fs::remove_dir_all(&self.upload_dir).await?;
            tokio::fs::create_dir_all(&self.upload_dir).await?;
        }
        Ok(())
    }

    /// Get count of stored files
    pub async fn file_count(&self) -> Result<usize> {
        if !self.upload_dir.exists() {
            return Ok(0);
        }
        let files = self.get_stored_files().await?;
        Ok(files.len())
    }
}

impl Default for MockUploader {
    fn default() -> Self {
        Self::new().unwrap()
    }
}

#[async_trait]
impl CheckpointUploader for MockUploader {
    async fn upload_checkpoint_with_plan_cancellable(
        &self,
        plan: &CheckpointPlan,
        cancel_token: Option<&CancellationToken>,
    ) -> Result<Vec<String>> {
        if let Some(token) = cancel_token {
            if token.is_cancelled() {
                info!("Mock uploader: cancelled before starting");
                return Err(UploadCancelledError {
                    reason: "before starting".to_string(),
                }
                .into());
            }
        }

        info!(
            "Mock uploading checkpoint with plan: {} files to upload to remote path: {}",
            plan.files_to_upload.len(),
            plan.info.get_remote_attempt_path(),
        );

        let remote_parent_path = self.upload_dir.join(plan.info.get_remote_attempt_path());
        tokio::fs::create_dir_all(remote_parent_path).await?;

        let mut files_to_upload = Vec::new();
        for local_file in &plan.files_to_upload {
            files_to_upload.push((
                local_file.local_path.clone(),
                plan.info.get_file_key(&local_file.filename),
            ));
        }

        let mut uploaded_keys = Vec::new();
        for (local_file_path, remote_file_path_str) in files_to_upload {
            if let Some(token) = cancel_token {
                if token.is_cancelled() {
                    info!("Mock uploader: cancelled during file upload");
                    return Err(UploadCancelledError {
                        reason: "during file upload".to_string(),
                    }
                    .into());
                }
            }
            let remote_file_path = self.upload_dir.join(&remote_file_path_str);
            if let Some(parent) = remote_file_path.parent() {
                tokio::fs::create_dir_all(parent).await?;
            }
            tokio::fs::copy(&local_file_path, &remote_file_path).await?;
            uploaded_keys.push(remote_file_path_str);
        }

        if let Some(token) = cancel_token {
            if token.is_cancelled() {
                info!("Mock uploader: cancelled before metadata upload");
                return Err(UploadCancelledError {
                    reason: "before metadata upload".to_string(),
                }
                .into());
            }
        }

        let metadata_key = self.upload_dir.join(plan.info.get_metadata_key());
        let metadata_json = &plan
            .info
            .metadata
            .to_json()
            .with_context(|| format!("Failed to serialize checkpoint metadata: {plan:?}"))?;
        tokio::fs::write(&metadata_key, metadata_json).await?;
        uploaded_keys.push(metadata_key.to_string_lossy().to_string());

        info!("Mock uploaded {} files with plan", uploaded_keys.len());
        Ok(uploaded_keys)
    }

    async fn is_available(&self) -> bool {
        self.available
    }
}

#[tokio::test]
async fn test_checkpoint_exporter_creation() {
    let uploader = MockUploader::new().unwrap();
    let exporter = CheckpointExporter::new(Box::new(uploader));
    assert!(exporter.is_available().await);
}

#[tokio::test]
async fn test_unavailable_uploader() {
    let tmp_store_dir = TempDir::new().unwrap();
    let test_topic = "test_unavailable_uploader";
    let test_partition = 0;
    let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);

    // Add some test data
    let events = vec![
        TestRawEventBuilder::new()
            .distinct_id("user1")
            .token("token1")
            .event("event1")
            .current_timestamp()
            .build(),
        TestRawEventBuilder::new()
            .distinct_id("user2")
            .token("token1")
            .event("event2")
            .current_timestamp()
            .build(),
    ];
    for event in &events {
        let key = event.into();
        let metadata = TimestampMetadata::new(event);
        store.put_timestamp_record(&key, &metadata).unwrap();
    }

    let tmp_checkpoint_dir = TempDir::new().unwrap();
    let config = CheckpointConfig {
        checkpoint_interval: Duration::from_secs(60),
        local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
        s3_bucket: "test-bucket".to_string(),
        s3_key_prefix: "test-prefix".to_string(),
        aws_region: Some("us-east-1".to_string()),
        ..Default::default()
    };

    let uploader = Box::new(MockUploader::new_unavailable().unwrap());
    let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));

    let local_base_dir = Path::new(&config.local_checkpoint_dir);
    let remote_namespace = config.s3_key_prefix.clone();
    let attempt_timestamp = Utc::now();
    let partition = Partition::new(test_topic.to_string(), test_partition);

    let worker = CheckpointWorker::new(
        1,
        local_base_dir,
        remote_namespace,
        partition.clone(),
        attempt_timestamp,
        exporter.clone(),
        None,
    );

    // The wrapper thread closure that is spawned to run this in
    // production will catch and log/stat these errors
    let result = worker.checkpoint_partition(&store, None).await;
    assert!(result.is_err());

    // No files should be uploaded when the remote storage is unavailable
    let file_count = uploader.file_count().await.unwrap();
    assert_eq!(
        file_count, 0,
        "No files should be uploaded when uploader is unavailable"
    );
}

#[tokio::test]
async fn test_unpopulated_exporter() {
    let tmp_store_dir = TempDir::new().unwrap();
    let test_topic = "test_unpopulated_exporter";
    let test_partition = 0;
    let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);

    // Add some test data
    let events = vec![
        TestRawEventBuilder::new()
            .distinct_id("user1")
            .token("token1")
            .event("event1")
            .current_timestamp()
            .build(),
        TestRawEventBuilder::new()
            .distinct_id("user2")
            .token("token1")
            .event("event2")
            .current_timestamp()
            .build(),
    ];
    for event in &events {
        let key = event.into();
        let metadata = TimestampMetadata::new(event);
        store.put_timestamp_record(&key, &metadata).unwrap();
    }

    let tmp_checkpoint_dir = TempDir::new().unwrap();
    let config = CheckpointConfig {
        checkpoint_interval: Duration::from_secs(60),
        local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
        s3_bucket: "test-bucket".to_string(),
        s3_key_prefix: "test-prefix".to_string(),
        aws_region: Some("us-east-1".to_string()),
        ..Default::default()
    };

    let local_base_dir = Path::new(&config.local_checkpoint_dir);
    let remote_namespace = config.s3_key_prefix.clone();
    let attempt_timestamp = Utc::now();
    let partition = Partition::new(test_topic.to_string(), test_partition);

    // without an exporter supplied to the worker, the checkpoint will
    // succeed and be created locally but never uploaded to remote storage
    let worker = CheckpointWorker::new(
        1,
        local_base_dir,
        remote_namespace,
        partition.clone(),
        attempt_timestamp,
        None,
        None,
    );

    // Checkpoint should still succeed even if uploader is unavailable
    let result = worker.checkpoint_partition(&store, None).await;
    assert!(result.is_ok());
    assert!(result.unwrap().is_none());
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_from_plan_with_no_previous_metadata() {
    let test_topic = "manual_cp_incremental";
    let test_partition = 0;
    let tmp_store_dir = TempDir::new().unwrap();
    let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);

    // Add some test data
    let events = vec![
        TestRawEventBuilder::new()
            .distinct_id("user1")
            .token("token1")
            .event("event1")
            .current_timestamp()
            .build(),
        TestRawEventBuilder::new()
            .distinct_id("user2")
            .token("token1")
            .event("event2")
            .current_timestamp()
            .build(),
    ];
    for event in &events {
        let key = event.into();
        let metadata = TimestampMetadata::new(event);
        store.put_timestamp_record(&key, &metadata).unwrap();
    }

    let tmp_checkpoint_dir = TempDir::new().unwrap();
    let config = CheckpointConfig {
        checkpoint_interval: Duration::from_secs(60),
        local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
        s3_bucket: "test-bucket".to_string(),
        s3_key_prefix: "test-prefix".to_string(),
        aws_region: Some("us-east-1".to_string()),
        ..Default::default()
    };

    let uploader = Box::new(MockUploader::new().unwrap());
    let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));

    let local_base_dir = Path::new(&config.local_checkpoint_dir);
    let remote_namespace = config.s3_key_prefix.clone();
    let partition = Partition::new(test_topic.to_string(), test_partition);
    let attempt_timestamp = Utc::now();

    let worker = CheckpointWorker::new(
        1,
        local_base_dir,
        remote_namespace,
        partition.clone(),
        attempt_timestamp,
        exporter.clone(),
        None,
    );

    // Perform checkpoint without previous metadata
    let result = worker.checkpoint_partition(&store, None).await;
    assert!(result.is_ok());

    let result = result.unwrap();
    assert!(result.is_some());
    let info = result.unwrap();

    // All remote paths (metadata and objects) now include hash prefix
    let hash = hash_prefix_for_partition(partition.topic(), partition.partition_number());
    let expected_remote_path = format!(
        "{}/{}/{}/{}/{}",
        hash,
        config.s3_key_prefix,
        partition.topic(),
        partition.partition_number(),
        CheckpointMetadata::generate_id(attempt_timestamp),
    );
    assert_eq!(info.get_remote_attempt_path(), expected_remote_path);

    let remote_checkpoint_files = uploader.get_stored_files().await.unwrap();
    assert!(!remote_checkpoint_files.is_empty());
    for k in remote_checkpoint_files.keys() {
        assert!(
            k.starts_with(&expected_remote_path),
            "all keys should be under hashed path, got: {k}"
        );
    }

    // Verify exported files contain expected RocksDB checkpoint files
    assert!(remote_checkpoint_files
        .keys()
        .any(|k| k.ends_with("CURRENT")));
    assert!(remote_checkpoint_files
        .keys()
        .any(|k| k.contains("MANIFEST-")));
    assert!(remote_checkpoint_files
        .keys()
        .any(|k| k.contains("OPTIONS-")));
    assert!(remote_checkpoint_files.keys().any(|k| k.ends_with(".sst")));
    assert!(remote_checkpoint_files.keys().any(|k| k.ends_with(".log")));
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_from_plan_with_previous_metadata() {
    // Note: detailed planner diffs are exercised in the planner test suite
    let test_topic = "test_cp_from_plan_with_prev_metadata";
    let test_partition = 0;
    let tmp_store_dir = TempDir::new().unwrap();
    let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);

    // Add some test data
    let events = vec![TestRawEventBuilder::new()
        .distinct_id("user1")
        .token("token1")
        .event("event1")
        .current_timestamp()
        .build()];
    for event in &events {
        let key = event.into();
        let metadata = TimestampMetadata::new(event);
        store.put_timestamp_record(&key, &metadata).unwrap();
    }

    let tmp_checkpoint_dir = TempDir::new().unwrap();
    let config = CheckpointConfig {
        checkpoint_interval: Duration::from_secs(60),
        local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
        s3_bucket: "test-bucket".to_string(),
        s3_key_prefix: "test-prefix".to_string(),
        aws_region: Some("us-east-1".to_string()),
        ..Default::default()
    };

    let uploader = Box::new(MockUploader::new().unwrap());
    let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));

    let local_base_dir = Path::new(&config.local_checkpoint_dir);
    let remote_namespace = config.s3_key_prefix.clone();
    let partition = Partition::new(test_topic.to_string(), test_partition);
    let attempt_timestamp = Utc::now();

    let worker = CheckpointWorker::new(
        1,
        local_base_dir,
        remote_namespace.clone(),
        partition.clone(),
        attempt_timestamp,
        exporter.clone(),
        None,
    );

    // Perform checkpoint without previous metadata
    let result = worker.checkpoint_partition(&store, None).await;
    assert!(result.is_ok());
    let result = result.unwrap();
    assert!(result.is_some());
    let orig_info = result.unwrap();

    // All remote paths (metadata and objects) now include hash prefix
    let hash = hash_prefix_for_partition(partition.topic(), partition.partition_number());
    let orig_expected_remote_path = format!(
        "{}/{}/{}/{}/{}",
        hash,
        config.s3_key_prefix,
        partition.topic(),
        partition.partition_number(),
        CheckpointMetadata::generate_id(attempt_timestamp),
    );
    assert_eq!(
        orig_info.get_remote_attempt_path(),
        orig_expected_remote_path
    );

    let orig_remote_checkpoint_files = uploader.get_stored_files().await.unwrap();
    assert!(!orig_remote_checkpoint_files.is_empty());
    for k in orig_remote_checkpoint_files.keys() {
        assert!(
            k.starts_with(&orig_expected_remote_path),
            "all keys should be under hashed path, got: {k}"
        );
    }

    // Verify exported files contain expected RocksDB checkpoint files, including SSTs
    assert!(orig_remote_checkpoint_files
        .keys()
        .any(|k| k.ends_with(".sst")));
    assert!(orig_remote_checkpoint_files
        .keys()
        .any(|k| k.ends_with("CURRENT")));
    assert!(orig_remote_checkpoint_files
        .keys()
        .any(|k| k.contains("MANIFEST-")));
    assert!(orig_remote_checkpoint_files
        .keys()
        .any(|k| k.contains("OPTIONS-")));
    assert!(orig_remote_checkpoint_files
        .keys()
        .any(|k| k.ends_with(".log")));

    // Do not add more test data so SST files don't change across checkpoints.
    // Await > 1 second so timestamp attempt directory changes between checkpoints
    tokio::time::sleep(Duration::from_millis(1100)).await;

    let next_attempt_timestamp = Utc::now();
    let next_checkpoint_id = CheckpointMetadata::generate_id(next_attempt_timestamp);
    let next_expected_remote_path = format!(
        "{}/{}/{}/{}/{}",
        hash,
        config.s3_key_prefix,
        partition.topic(),
        partition.partition_number(),
        next_checkpoint_id,
    );

    assert!(uploader.clear().await.is_ok());

    let worker_next = CheckpointWorker::new(
        2,
        local_base_dir,
        remote_namespace,
        partition.clone(),
        next_attempt_timestamp,
        exporter.clone(),
        None,
    );
    let result = worker_next
        .checkpoint_partition(&store, Some(&orig_info.metadata))
        .await;
    assert!(result.is_ok());
    let result = result.unwrap();
    assert!(result.is_some());

    // The new checkpoint metadata should contain a mix of new and reused files
    let info = result.unwrap();
    assert_eq!(info.get_remote_attempt_path(), next_expected_remote_path);
    let next_remote_checkpoint_files = uploader.get_stored_files().await.unwrap();

    assert!(!next_remote_checkpoint_files.is_empty());
    for k in next_remote_checkpoint_files.keys() {
        assert!(
            k.starts_with(&next_expected_remote_path),
            "all keys should be under hashed path, got: {k}"
        );
    }

    // there should be no new SST files uploaded in this checkpoint
    // because the original checkpoint uploaded them already
    assert!(next_remote_checkpoint_files
        .keys()
        .all(|k| !k.ends_with(".sst")));

    // Verify exported files contain expected RocksDB non-SST files
    // For now, we always upload the latest CURRENT, but
    // but this attempt, we don't upload new MANIFEST, OPTIONS,
    // or .log (WAL) files because the checksums will not be different
    assert!(next_remote_checkpoint_files
        .keys()
        .any(|k| k.ends_with("CURRENT")));
    assert!(next_remote_checkpoint_files
        .keys()
        .all(|k| !k.contains("MANIFEST-")));
    assert!(next_remote_checkpoint_files
        .keys()
        .all(|k| !k.contains("OPTIONS-")));
    assert!(next_remote_checkpoint_files
        .keys()
        .all(|k| !k.ends_with(".log")));
}

// ============================================================
// Cancellation Tests
// ============================================================

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_with_pre_cancelled_token_skips() {
    let test_topic = "test_pre_cancelled";
    let test_partition = 0;
    let tmp_store_dir = TempDir::new().unwrap();
    let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);

    // Add test data
    let event = TestRawEventBuilder::new()
        .distinct_id("user1")
        .token("token1")
        .event("event1")
        .current_timestamp()
        .build();
    let key = (&event).into();
    let metadata = TimestampMetadata::new(&event);
    store.put_timestamp_record(&key, &metadata).unwrap();

    let tmp_checkpoint_dir = TempDir::new().unwrap();
    let config = CheckpointConfig {
        checkpoint_interval: Duration::from_secs(60),
        local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
        s3_bucket: "test-bucket".to_string(),
        s3_key_prefix: "test-prefix".to_string(),
        aws_region: Some("us-east-1".to_string()),
        ..Default::default()
    };

    let uploader = Box::new(MockUploader::new().unwrap());
    let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));

    let partition = Partition::new(test_topic.to_string(), test_partition);
    let attempt_timestamp = Utc::now();

    let worker = CheckpointWorker::new(
        1,
        Path::new(&config.local_checkpoint_dir),
        config.s3_key_prefix.clone(),
        partition.clone(),
        attempt_timestamp,
        exporter.clone(),
        None,
    );

    // Create pre-cancelled token
    let cancel_token = CancellationToken::new();
    cancel_token.cancel();

    // Pre-cancelled checkpoint should be cooperatively skipped
    let result = worker
        .checkpoint_partition_cancellable(&store, None, Some(&cancel_token), Some("test"))
        .await;

    assert!(
        matches!(result, Ok(None)),
        "Checkpoint should be skipped when pre-cancelled, got: {result:?}"
    );

    // Verify no files were uploaded
    let file_count = uploader.file_count().await.unwrap();
    assert_eq!(
        file_count, 0,
        "No files should be uploaded when pre-cancelled"
    );
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_with_active_token_succeeds() {
    let test_topic = "test_active_token";
    let test_partition = 0;
    let tmp_store_dir = TempDir::new().unwrap();
    let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);

    // Add test data
    let event = TestRawEventBuilder::new()
        .distinct_id("user1")
        .token("token1")
        .event("event1")
        .current_timestamp()
        .build();
    let key = (&event).into();
    let metadata = TimestampMetadata::new(&event);
    store.put_timestamp_record(&key, &metadata).unwrap();

    let tmp_checkpoint_dir = TempDir::new().unwrap();
    let config = CheckpointConfig {
        checkpoint_interval: Duration::from_secs(60),
        local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
        s3_bucket: "test-bucket".to_string(),
        s3_key_prefix: "test-prefix".to_string(),
        aws_region: Some("us-east-1".to_string()),
        ..Default::default()
    };

    let uploader = Box::new(MockUploader::new().unwrap());
    let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));

    let partition = Partition::new(test_topic.to_string(), test_partition);
    let attempt_timestamp = Utc::now();

    let worker = CheckpointWorker::new(
        1,
        Path::new(&config.local_checkpoint_dir),
        config.s3_key_prefix.clone(),
        partition.clone(),
        attempt_timestamp,
        exporter.clone(),
        None,
    );

    // Create active (non-cancelled) token
    let cancel_token = CancellationToken::new();

    // Checkpoint should succeed with active token
    let result = worker
        .checkpoint_partition_cancellable(&store, None, Some(&cancel_token), None)
        .await;

    assert!(
        result.is_ok(),
        "Checkpoint should succeed with active token"
    );
    let info = result.unwrap();
    assert!(info.is_some(), "Should return CheckpointInfo");

    // Verify files were uploaded
    let file_count = uploader.file_count().await.unwrap();
    assert!(
        file_count > 0,
        "Files should be uploaded when token is active"
    );
}

#[tokio::test]
async fn test_checkpoint_without_token_succeeds() {
    // Verify the legacy non-cancellable path still works
    let test_topic = "test_no_token";
    let test_partition = 0;
    let tmp_store_dir = TempDir::new().unwrap();
    let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);

    // Add test data
    let event = TestRawEventBuilder::new()
        .distinct_id("user1")
        .token("token1")
        .event("event1")
        .current_timestamp()
        .build();
    let key = (&event).into();
    let metadata = TimestampMetadata::new(&event);
    store.put_timestamp_record(&key, &metadata).unwrap();

    let tmp_checkpoint_dir = TempDir::new().unwrap();
    let config = CheckpointConfig {
        checkpoint_interval: Duration::from_secs(60),
        local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
        s3_bucket: "test-bucket".to_string(),
        s3_key_prefix: "test-prefix".to_string(),
        aws_region: Some("us-east-1".to_string()),
        ..Default::default()
    };

    let uploader = Box::new(MockUploader::new().unwrap());
    let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));

    let partition = Partition::new(test_topic.to_string(), test_partition);
    let attempt_timestamp = Utc::now();

    let worker = CheckpointWorker::new(
        1,
        Path::new(&config.local_checkpoint_dir),
        config.s3_key_prefix.clone(),
        partition.clone(),
        attempt_timestamp,
        exporter.clone(),
        None,
    );

    // Use legacy non-cancellable method (no token)
    let result = worker.checkpoint_partition(&store, None).await;

    assert!(result.is_ok(), "Legacy checkpoint should succeed");
    let info = result.unwrap();
    assert!(info.is_some(), "Should return CheckpointInfo");

    // Verify files were uploaded
    let file_count = uploader.file_count().await.unwrap();
    assert!(file_count > 0, "Files should be uploaded");
}

// ============================================================
// spawn_blocking Tests
// ============================================================
// These tests verify that spawn_blocking correctly moves blocking RocksDB
// operations off the tokio worker threads to prevent runtime starvation.

/// Test that spawn_blocking doesn't starve the tokio runtime.
/// This is the core issue that spawn_blocking was introduced to solve.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_doesnt_starve_runtime() {
    use std::sync::atomic::{AtomicUsize, Ordering};

    let heartbeat_count = Arc::new(AtomicUsize::new(0));
    let count_clone = heartbeat_count.clone();

    // Simulate liveness check running every 50ms
    let heartbeat_task = tokio::spawn(async move {
        for _ in 0..10 {
            tokio::time::sleep(Duration::from_millis(50)).await;
            count_clone.fetch_add(1, Ordering::SeqCst);
        }
    });

    // Create a store with some data that will require actual RocksDB work
    let test_topic = "test_starve";
    let test_partition = 0;
    let tmp_store_dir = TempDir::new().unwrap();
    let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);

    // Write some test data to make checkpoint creation do real work
    for i in 0..100 {
        let event = TestRawEventBuilder::new()
            .distinct_id(&format!("user-{}", i))
            .token("test-token")
            .event(&format!("event-{}", i))
            .current_timestamp()
            .build();
        let key = (&event).into();
        let metadata = TimestampMetadata::new(&event);
        store.put_timestamp_record(&key, &metadata).unwrap();
    }

    // Create checkpoint worker
    let tmp_checkpoint_dir = TempDir::new().unwrap();
    let config = CheckpointConfig {
        checkpoint_interval: Duration::from_secs(60),
        local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
        s3_bucket: "test-bucket".to_string(),
        s3_key_prefix: "test-prefix".to_string(),
        aws_region: Some("us-east-1".to_string()),
        ..Default::default()
    };

    let worker = CheckpointWorker::new(
        1,
        Path::new(&config.local_checkpoint_dir),
        config.s3_key_prefix.clone(),
        Partition::new(test_topic.to_string(), test_partition),
        Utc::now(),
        None,
        None,
    );

    // Run checkpoint (which uses spawn_blocking for RocksDB operations)
    let checkpoint_task = tokio::spawn(async move {
        worker
            .checkpoint_partition_cancellable(
                &store,
                None,
                Some(&CancellationToken::new()),
                Some("test"),
            )
            .await
    });

    let (heartbeat_result, checkpoint_result) = tokio::join!(heartbeat_task, checkpoint_task);

    // With spawn_blocking: heartbeats continue during checkpoint
    // Without spawn_blocking: heartbeats would stall until checkpoint completes
    assert!(heartbeat_result.is_ok());
    let final_count = heartbeat_count.load(Ordering::SeqCst);
    assert!(
        final_count >= 8,
        "Heartbeats should continue during checkpoint (got {})",
        final_count
    );

    assert!(checkpoint_result.unwrap().is_ok());
}

/// Test that concurrent checkpoint operations complete successfully
/// when running on the blocking thread pool.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_concurrent_checkpoint_operations_complete() {
    let tmp_base_dir = TempDir::new().unwrap();

    let mut handles = vec![];

    // Create multiple workers for different partitions
    for partition_num in 0..5 {
        let tmp_checkpoint_dir = tmp_base_dir
            .path()
            .join(format!("checkpoints-{}", partition_num));
        let tmp_store_dir = tmp_base_dir.path().join(format!("store-{}", partition_num));

        let handle = tokio::spawn(async move {
            let test_topic = "test_concurrent";
            let store = create_test_dedup_store(&tmp_store_dir, test_topic, partition_num);

            // Write some test data
            for i in 0..20 {
                let event = TestRawEventBuilder::new()
                    .distinct_id(&format!("user-p{}-{}", partition_num, i))
                    .token("test-token")
                    .event(&format!("event-{}", i))
                    .current_timestamp()
                    .build();
                let key = (&event).into();
                let metadata = TimestampMetadata::new(&event);
                store.put_timestamp_record(&key, &metadata).unwrap();
            }

            let worker = CheckpointWorker::new(
                partition_num as u32,
                &tmp_checkpoint_dir,
                "test-namespace".to_string(),
                Partition::new(test_topic.to_string(), partition_num),
                Utc::now(),
                None,
                None,
            );

            worker
                .checkpoint_partition_cancellable(
                    &store,
                    None,
                    Some(&CancellationToken::new()),
                    Some("test"),
                )
                .await
        });

        handles.push(handle);
    }

    // All checkpoints should complete successfully despite limited tokio workers
    for handle in handles {
        let result = handle.await.unwrap();
        assert!(
            result.is_ok(),
            "All concurrent checkpoints should complete successfully"
        );
    }
}

/// Test that checkpoint operations produce correct results after thread migration.
/// Verifies that Arc cloning and spawn_blocking don't corrupt data.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_metadata_consistency_across_threads() {
    let test_topic = "test_consistency";
    let test_partition = 0;
    let tmp_store_dir = TempDir::new().unwrap();
    let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);

    // Use a fixed timestamp to ensure consistent keys across writes and reads
    let fixed_timestamp = "1234567890";

    // Write known data to store
    let test_events = vec![
        ("user-1", "token-1", "event-1"),
        ("user-2", "token-2", "event-2"),
        ("user-3", "token-3", "event-3"),
    ];

    for (user, token, event_name) in &test_events {
        let event = TestRawEventBuilder::new()
            .distinct_id(user)
            .token(token)
            .event(event_name)
            .timestamp(fixed_timestamp)
            .build();
        let key = (&event).into();
        let metadata = TimestampMetadata::new(&event);
        store.put_timestamp_record(&key, &metadata).unwrap();
    }

    // Create checkpoint (which clones store Arc into spawn_blocking)
    let tmp_checkpoint_dir = TempDir::new().unwrap();
    let config = CheckpointConfig {
        checkpoint_interval: Duration::from_secs(60),
        local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
        s3_bucket: "test-bucket".to_string(),
        s3_key_prefix: "test-prefix".to_string(),
        aws_region: Some("us-east-1".to_string()),
        ..Default::default()
    };

    let worker = CheckpointWorker::new(
        1,
        Path::new(&config.local_checkpoint_dir),
        config.s3_key_prefix.clone(),
        Partition::new(test_topic.to_string(), test_partition),
        Utc::now(),
        None,
        None,
    );

    let result = worker
        .checkpoint_partition_cancellable(
            &store,
            None,
            Some(&CancellationToken::new()),
            Some("test"),
        )
        .await;
    assert!(result.is_ok(), "Checkpoint should succeed");

    // Verify original store still has correct data (Arc wasn't corrupted)
    for (user, token, event_name) in &test_events {
        let event = TestRawEventBuilder::new()
            .distinct_id(user)
            .token(token)
            .event(event_name)
            .timestamp(fixed_timestamp)
            .build();
        let key = (&event).into();
        let metadata = store.get_timestamp_record(&key).unwrap();
        assert!(metadata.is_some(), "Original store should still have data");
        assert_eq!(
            metadata.unwrap().original_event.token.as_deref(),
            Some(*token),
            "Metadata should survive spawn_blocking clone"
        );
    }
}

/// Test that liveness handlers aren't starved during long checkpoint operations.
/// This simulates the production scenario that spawn_blocking was added to fix.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_liveness_check_during_checkpoint() {
    use std::sync::atomic::{AtomicBool, Ordering};

    let liveness_flag = Arc::new(AtomicBool::new(false));
    let flag_clone = liveness_flag.clone();

    let test_topic = "test_liveness";
    let test_partition = 0;
    let tmp_store_dir = TempDir::new().unwrap();
    let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);

    // Write data to make checkpoint take some time
    for i in 0..200 {
        let event = TestRawEventBuilder::new()
            .distinct_id(&format!("user-{}", i))
            .token("test-token")
            .event(&format!("event-{}", i))
            .current_timestamp()
            .build();
        let key = (&event).into();
        let metadata = TimestampMetadata::new(&event);
        store.put_timestamp_record(&key, &metadata).unwrap();
    }

    // Spawn liveness check on tokio runtime
    tokio::spawn(async move {
        tokio::time::sleep(Duration::from_millis(20)).await;
        flag_clone.store(true, Ordering::SeqCst);
    });

    // Start checkpoint operation
    let tmp_checkpoint_dir = TempDir::new().unwrap();
    let config = CheckpointConfig {
        checkpoint_interval: Duration::from_secs(60),
        local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
        s3_bucket: "test-bucket".to_string(),
        s3_key_prefix: "test-prefix".to_string(),
        aws_region: Some("us-east-1".to_string()),
        ..Default::default()
    };

    let worker = CheckpointWorker::new(
        1,
        Path::new(&config.local_checkpoint_dir),
        config.s3_key_prefix.clone(),
        Partition::new(test_topic.to_string(), test_partition),
        Utc::now(),
        None,
        None,
    );

    let _result = worker
        .checkpoint_partition_cancellable(
            &store,
            None,
            Some(&CancellationToken::new()),
            Some("test"),
        )
        .await;

    // Verify liveness check completed despite checkpoint running
    tokio::time::sleep(Duration::from_millis(50)).await;
    assert!(
        liveness_flag.load(Ordering::SeqCst),
        "Liveness check should complete while checkpoint blocks"
    );
}